From ecda41ee531101c9848b470c660d803b1ee01216 Mon Sep 17 00:00:00 2001
From: Michael Nguyen <michael.nguyen@parivedasolutions.com>
Date: Tue, 29 Oct 2019 15:55:12 -0500
Subject: [PATCH] updating record repo.
---
indexer-service-aws/pom.xml | 19 ++++
.../indexer/aws/IndexerAwsApplication.java | 4 +-
.../indexer/aws/model/RecordMetadata.java | 82 +++++++++++++++++
.../indexer/aws/model/RecordMetadataDoc.java | 37 ++++++++
.../osdu/indexer/aws/model/RecordState.java | 18 ++++
.../osdu/indexer/aws/model/SchemaDoc.java | 37 ++++++++
.../osdu/indexer/aws/model/SchemaItem.java | 21 +++++
.../indexer/aws/publish/PublisherImpl.java | 11 ++-
.../aws/service/AwsStorageServiceImpl.java | 89 +++++++++++++++++++
.../aws/util/AWSElasticSearchConfig.java | 33 -------
.../indexer/aws/util/AmazonSNSConfig.java | 39 --------
.../osdu/indexer/aws/util/IAMConfig.java | 24 -----
.../aws/util/dynamodb/DynamoDBConfig.java | 37 ++++++++
.../util/dynamodb/record/DynamoDBRecord.java | 20 +++++
.../record/LegalTagsTypeConverter.java | 49 ++++++++++
.../record/RecordMetadataTypeConverter.java | 52 +++++++++++
.../util/dynamodb/schema/DynamoDBSchema.java | 24 +++++
.../schema/SchemaExtTypeConverter.java | 46 ++++++++++
.../schema/SchemaItemTypeConverter.java | 48 ++++++++++
.../indexer/aws/util/s3/S3RecordClient.java | 43 +++++++++
.../src/main/resources/application.properties | 16 +++-
21 files changed, 648 insertions(+), 101 deletions(-)
create mode 100644 indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/RecordMetadata.java
create mode 100644 indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/RecordMetadataDoc.java
create mode 100644 indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/RecordState.java
create mode 100644 indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/SchemaDoc.java
create mode 100644 indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/SchemaItem.java
create mode 100644 indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/service/AwsStorageServiceImpl.java
delete mode 100644 indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/AWSElasticSearchConfig.java
delete mode 100644 indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/AmazonSNSConfig.java
delete mode 100644 indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IAMConfig.java
create mode 100644 indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/DynamoDBConfig.java
create mode 100644 indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/record/DynamoDBRecord.java
create mode 100644 indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/record/LegalTagsTypeConverter.java
create mode 100644 indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/record/RecordMetadataTypeConverter.java
create mode 100644 indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/schema/DynamoDBSchema.java
create mode 100644 indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/schema/SchemaExtTypeConverter.java
create mode 100644 indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/schema/SchemaItemTypeConverter.java
create mode 100644 indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/s3/S3RecordClient.java
diff --git a/indexer-service-aws/pom.xml b/indexer-service-aws/pom.xml
index be2416219..5c95e90e1 100644
--- a/indexer-service-aws/pom.xml
+++ b/indexer-service-aws/pom.xml
@@ -18,6 +18,25 @@
<artifactId>indexer-service-root</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-core</artifactId>
+ <version>1.11.651</version>
+ </dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-osdu-util</artifactId>
+ <version>0.0.2</version>
+ </dependency>
+ <dependency>
+ <groupId>com.github.derjust</groupId>
+ <artifactId>spring-data-dynamodb</artifactId>
+ <version>5.1.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ </dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
diff --git a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/IndexerAwsApplication.java b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/IndexerAwsApplication.java
index a3adb4f2e..f39452c46 100644
--- a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/IndexerAwsApplication.java
+++ b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/IndexerAwsApplication.java
@@ -2,10 +2,12 @@ package org.opengroup.osdu.indexer.aws;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.actuate.autoconfigure.elasticsearch.ElasticSearchRestHealthIndicatorAutoConfiguration;
+import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
import org.springframework.context.annotation.ComponentScan;
-@SpringBootApplication(exclude = {ElasticSearchRestHealthIndicatorAutoConfiguration.class})
+@SpringBootApplication(exclude = {ElasticSearchRestHealthIndicatorAutoConfiguration.class, SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
@ComponentScan({"org.opengroup.osdu.is.core", "org.opengroup.osdu.indexer"})
public class IndexerAwsApplication {
public static void main( String[] args )
diff --git a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/RecordMetadata.java b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/RecordMetadata.java
new file mode 100644
index 000000000..aa52ae69a
--- /dev/null
+++ b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/RecordMetadata.java
@@ -0,0 +1,82 @@
+package org.opengroup.osdu.indexer.aws.model;
+
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.indexer.model.*;
+import org.opengroup.osdu.is.core.util.AppException;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@Data
+@NoArgsConstructor
+public class RecordMetadata {
+ private String id;
+
+ private String kind;
+
+ private StorageAcl acl;
+
+ private Legal legal;
+
+ private RecordAncestry ancestry;
+
+ private List<String> gcsVersionPaths = new ArrayList<>();
+
+ private RecordState status;
+
+ private String user;
+
+ // epoch time
+ private long createTime;
+
+ private String modifyUser;
+
+ // epoch time
+ private long modifyTime;
+
+ public RecordMetadata(RecordIndexerPayload.Record record) {
+ this.id = record.getId();
+ this.kind = record.getKind();
+ this.acl = record.getAcl();
+ this.legal = record.getLegal();
+ this.ancestry = record.getAncestry();
+ }
+
+ public Long getLatestVersion() {
+ String latestVersionPath = this.gcsVersionPaths.get(gcsVersionPaths.size() - 1);
+ String[] versionTokens = latestVersionPath.split("/");
+ return Long.parseLong(versionTokens[versionTokens.length - 1]);
+ }
+
+ public boolean hasVersion() {
+ if (gcsVersionPaths.isEmpty()) {
+ return false;
+ } else {
+ return true;
+ }
+ }
+
+ public void addGcsPath(long version) {
+ this.gcsVersionPaths.add(String.format("%s/%s/%s", this.kind, this.id, version));
+ }
+
+ public String getVersionPath(Long version) {
+ for (String path : this.gcsVersionPaths) {
+ if (path.contains(Long.toString(version))) {
+ return path;
+ }
+ }
+
+ throw new AppException(HttpStatus.SC_NOT_FOUND, "Record version not found",
+ "The requested record version was not found");
+ }
+
+ public void resetGcsPath(List<String> gcsVersionPathList) {
+ this.gcsVersionPaths.clear();
+ for (String path: gcsVersionPathList) {
+ this.gcsVersionPaths.add(path);
+ }
+ }
+}
diff --git a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/RecordMetadataDoc.java b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/RecordMetadataDoc.java
new file mode 100644
index 000000000..caa128dcd
--- /dev/null
+++ b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/RecordMetadataDoc.java
@@ -0,0 +1,37 @@
+package org.opengroup.osdu.indexer.aws.model;
+
+import com.amazonaws.services.dynamodbv2.datamodeling.*;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.opengroup.osdu.indexer.aws.util.dynamodb.record.LegalTagsTypeConverter;
+import org.opengroup.osdu.indexer.aws.util.dynamodb.record.RecordMetadataTypeConverter;
+
+import java.util.Set;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@DynamoDBTable(tableName = "RecordMetadataRepository") // DynamoDB table name (without environment prefix)
+public class RecordMetadataDoc {
+
+ @DynamoDBHashKey(attributeName = "Id")
+ private String id;
+
+ @DynamoDBIndexHashKey(attributeName = "Kind", globalSecondaryIndexName = "KindStatusIndex")
+ private String kind;
+
+ @DynamoDBIndexRangeKey(attributeName = "Status", globalSecondaryIndexName = "KindStatusIndex")
+ private String status;
+
+ @DynamoDBIndexHashKey(attributeName = "User", globalSecondaryIndexName = "UserIndex")
+ private String user;
+
+ @DynamoDBTypeConverted(converter = RecordMetadataTypeConverter.class)
+ @DynamoDBAttribute(attributeName = "metadata")
+ private RecordMetadata metadata;
+
+ @DynamoDBTypeConverted(converter = LegalTagsTypeConverter.class)
+ @DynamoDBAttribute(attributeName = "LegalTags")
+ private Set<String> legaltags;
+}
diff --git a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/RecordState.java b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/RecordState.java
new file mode 100644
index 000000000..21c495140
--- /dev/null
+++ b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/RecordState.java
@@ -0,0 +1,18 @@
+package org.opengroup.osdu.indexer.aws.model;
+
+public enum RecordState {
+ /**
+ * An active record
+ */
+ active,
+
+ /**
+ * A deleted record
+ */
+ deleted,
+
+ /**
+ * A purged record
+ */
+ purged
+}
diff --git a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/SchemaDoc.java b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/SchemaDoc.java
new file mode 100644
index 000000000..def063bf8
--- /dev/null
+++ b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/SchemaDoc.java
@@ -0,0 +1,37 @@
+package org.opengroup.osdu.indexer.aws.model;
+
+import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBAttribute;
+import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBHashKey;
+import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBTable;
+import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBTypeConverted;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.opengroup.osdu.indexer.aws.util.dynamodb.schema.SchemaExtTypeConverter;
+import org.opengroup.osdu.indexer.aws.util.dynamodb.schema.SchemaItemTypeConverter;
+import org.opengroup.osdu.indexer.model.Schema;
+
+import java.util.List;
+import java.util.Map;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@DynamoDBTable(tableName = "SchemaRepository") // DynamoDB table name (without environment prefix)
+public class SchemaDoc {
+
+ @DynamoDBHashKey(attributeName = "Kind")
+ private String kind;
+
+ @DynamoDBTypeConverted(converter = SchemaExtTypeConverter.class)
+ @DynamoDBAttribute(attributeName = "ext")
+ private Map<String,Object> extension;
+
+ @DynamoDBAttribute(attributeName = "User")
+ private String user;
+
+ @DynamoDBTypeConverted(converter = SchemaItemTypeConverter.class)
+ @DynamoDBAttribute(attributeName = "schema")
+ private List<SchemaItem> schemaItems;
+}
+
diff --git a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/SchemaItem.java b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/SchemaItem.java
new file mode 100644
index 000000000..b00593b50
--- /dev/null
+++ b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/SchemaItem.java
@@ -0,0 +1,21 @@
+package org.opengroup.osdu.indexer.aws.model;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.Map;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class SchemaItem {
+
+ private String path;
+
+ private String kind;
+
+ @JsonInclude(value = JsonInclude.Include.NON_NULL)
+ private Map<String, Object> ext;
+}
\ No newline at end of file
diff --git a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/publish/PublisherImpl.java b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/publish/PublisherImpl.java
index 9a748fbb7..9e9fec608 100644
--- a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/publish/PublisherImpl.java
+++ b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/publish/PublisherImpl.java
@@ -1,5 +1,6 @@
package org.opengroup.osdu.indexer.aws.publish;
+import com.amazonaws.osdu.util.sns.SNSBuilder;
import com.amazonaws.services.sns.model.MessageAttributeValue;
import com.amazonaws.services.sns.model.PublishRequest;
import com.google.gson.Gson;
@@ -16,12 +17,20 @@ import java.util.Map;
@Component
public class PublisherImpl implements IPublisher {
- @Inject
+
private AmazonSNS snsClient;
@Value("${aws.sns.arn}")
private String amazonSNSTopic;
+ @Value("${aws.sns.region}")
+ private String amazonSNSRegion;
+
+ @Inject
+ public void init(){
+ snsClient = SNSBuilder.generateAmazonSNSClient(amazonSNSRegion);
+ }
+
public void publishStatusChangedTagsToTopic(DpsHeaders headers, JobStatus indexerBatchStatus) throws Exception
{
String json = new Gson().toJson(indexerBatchStatus.getStatusesList());
diff --git a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/service/AwsStorageServiceImpl.java b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/service/AwsStorageServiceImpl.java
new file mode 100644
index 000000000..f8b8f3072
--- /dev/null
+++ b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/service/AwsStorageServiceImpl.java
@@ -0,0 +1,89 @@
+package org.opengroup.osdu.indexer.aws.service;
+
+import com.google.gson.Gson;
+import org.checkerframework.checker.units.qual.A;
+import org.opengroup.osdu.indexer.aws.model.RecordMetadataDoc;
+import org.opengroup.osdu.indexer.aws.model.SchemaDoc;
+import org.opengroup.osdu.indexer.aws.util.dynamodb.record.DynamoDBRecord;
+import org.opengroup.osdu.indexer.aws.util.dynamodb.schema.DynamoDBSchema;
+import org.opengroup.osdu.indexer.aws.util.s3.S3RecordClient;
+import org.opengroup.osdu.indexer.model.*;
+import org.opengroup.osdu.indexer.service.StorageService;
+import org.opengroup.osdu.is.core.util.AppException;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Primary;
+import org.springframework.stereotype.Component;
+
+
+import javax.inject.Inject;
+import java.io.Console;
+import java.io.UnsupportedEncodingException;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+@Primary
+@Component
+public class AwsStorageServiceImpl implements StorageService {
+
+ @Inject
+ DynamoDBSchema dynamoDBSchema;
+
+ @Inject
+ DynamoDBRecord dynamoDBRecord;
+
+ @Inject
+ S3RecordClient s3Client;
+
+// private String getKeyNameForLatestVersion( recordMetadata){
+// return recordMetadata.getKind() + "/" + recordMetadata.getId() + "/" + recordMetadata.getLatestVersion();
+// }
+ @Override
+ public org.opengroup.osdu.indexer.model.Records getStorageRecords(List<String> ids) throws AppException, URISyntaxException {
+ Records records = new Records();
+ List<String> notFound = new ArrayList<>();
+ List<Records.Entity> entities = new ArrayList<>();
+ List<ConversionStatus> statuses = new ArrayList<>();
+ Gson gson = new Gson();
+ for (String id: ids) {
+ Optional<RecordMetadataDoc> doc = dynamoDBRecord.findById(id);
+ if (!doc.isPresent()){
+ notFound.add(id);
+ }
+
+ String record = s3Client.getRecord(doc.get().getMetadata());
+ Records.Entity entity = gson.fromJson(record, Records.Entity.class);
+ statuses.add(ConversionStatus.builder().status(doc.get().getStatus()).build());
+ entities.add(entity);
+ }
+ records.setNotFound(notFound);
+ records.setRecords(entities);
+ records.setConversionStatuses(statuses);
+ return records;
+ }
+
+ @Override
+ public RecordQueryResponse getRecordsByKind(RecordReindexRequest request) throws URISyntaxException {
+
+ return null;
+ }
+
+ @Override
+ public String getStorageSchema(String kind) throws URISyntaxException, UnsupportedEncodingException {
+ Optional<SchemaDoc> sd = dynamoDBSchema.findById(kind);
+ if (!sd.isPresent()) {
+ return null;
+ }
+ // Create a Schema object and assign the values retrieved from DynamoDB
+ Schema newSchema = new Schema();
+ newSchema.setKind(kind);
+ List<Schema.Mapping> mappings;
+ mappings = sd.get().getSchemaItems().stream().map(schemaItem -> Schema.Mapping.builder().kind(schemaItem.getKind()).path(schemaItem.getPath()).build()).collect(Collectors.toList());
+ newSchema.setSchema(mappings);
+ return new Gson().toJson(newSchema);
+ }
+}
diff --git a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/AWSElasticSearchConfig.java b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/AWSElasticSearchConfig.java
deleted file mode 100644
index b4ed0958e..000000000
--- a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/AWSElasticSearchConfig.java
+++ /dev/null
@@ -1,33 +0,0 @@
-package org.opengroup.osdu.indexer.aws.util;
-
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.AWSCredentialsProviderChain;
-import com.amazonaws.auth.AWSStaticCredentialsProvider;
-import com.amazonaws.services.elasticsearch.AWSElasticsearch;
-import com.amazonaws.services.elasticsearch.AWSElasticsearchClientBuilder;
-import com.amazonaws.services.elasticsearch.model.ElasticsearchClusterConfig;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.Import;
-
-import javax.inject.Inject;
-
-@Configuration
-@Import({IAMConfig.class})
-public class AWSElasticSearchConfig {
-
- @Value("${aws.sns.region}")
- private String amazonSNSRegion;
-
- @Inject
- private AWSCredentials amazonAWSCredentials;
-
- public AWSElasticsearch AWSElasticsearch() {
- return AWSElasticsearchClientBuilder.standard()
- .withRegion(amazonSNSRegion)
- .withCredentials(new AWSStaticCredentialsProvider(amazonAWSCredentials)).build();
-
- }
-
-}
diff --git a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/AmazonSNSConfig.java b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/AmazonSNSConfig.java
deleted file mode 100644
index 5ecbb027a..000000000
--- a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/AmazonSNSConfig.java
+++ /dev/null
@@ -1,39 +0,0 @@
-package org.opengroup.osdu.indexer.aws.util;
-
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.AWSStaticCredentialsProvider;
-import com.amazonaws.services.sns.AmazonSNS;
-import com.amazonaws.services.sns.AmazonSNSClientBuilder;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.Import;
-
-import javax.inject.Inject;
-
-@Configuration
-@Import({IAMConfig.class})
-public class AmazonSNSConfig {
-
- @Value("${aws.sns.region}")
- private String amazonSNSRegion;
-
- @Value("${aws.sns.topic-name}")
- private String amazonSNSTopicName;
-
- @Value("${aws.sns.arn}")
- private String amazonSNSARN;
-
-
- @Inject
- private AWSCredentials amazonAWSCredentials;
-
- @Bean
- public AmazonSNS AmazonSNS() {
- return AmazonSNSClientBuilder.standard()
- .withCredentials(new AWSStaticCredentialsProvider(amazonAWSCredentials))
- .withRegion(amazonSNSRegion)
- .build();
- }
-
-}
\ No newline at end of file
diff --git a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IAMConfig.java b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IAMConfig.java
deleted file mode 100644
index 12fe7b7c8..000000000
--- a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IAMConfig.java
+++ /dev/null
@@ -1,24 +0,0 @@
-package org.opengroup.osdu.indexer.aws.util;
-
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.BasicAWSCredentials;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-
-@Configuration
-public class IAMConfig {
-
- @Value("${aws.accessKeyId}")
- private String amazonAWSAccessKey;
-
- @Value("${aws.secretKey}")
- private String amazonAWSSecretKey;
-
- @Bean
- public AWSCredentials amazonAWSCredentials() {
- // Generate AWS basic credentials from the access keys provided in application.properties
- return new BasicAWSCredentials(
- amazonAWSAccessKey, amazonAWSSecretKey);
- }
-}
\ No newline at end of file
diff --git a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/DynamoDBConfig.java b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/DynamoDBConfig.java
new file mode 100644
index 000000000..7a7f00611
--- /dev/null
+++ b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/DynamoDBConfig.java
@@ -0,0 +1,37 @@
+package org.opengroup.osdu.indexer.aws.util.dynamodb;
+
+import com.amazonaws.osdu.util.dynamodb.DynamoDBBuilder;
+import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
+import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBMapper;
+import org.socialsignin.spring.data.dynamodb.repository.config.EnableDynamoDBRepositories;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Primary;
+
+@Configuration
+@EnableDynamoDBRepositories
+ (basePackages = "org.opengroup.osdu.indexer.aws")
+public class DynamoDBConfig {
+
+ @Value("${aws.dynamodb.endpoint}")
+ private String amazonDynamoDBEndpoint;
+
+ @Value("${aws.dynamodb.region}")
+ private String amazonDynamoDBRegion;
+
+ @Value("${aws.dynamodb.table.prefix}")
+ private String tablePrefix;
+
+ @Bean
+ public AmazonDynamoDB amazonDynamoDB() {
+ // Generate the DynamoDB client
+ return DynamoDBBuilder.generateDynamoDBClient(amazonDynamoDBEndpoint, amazonDynamoDBRegion);
+ }
+
+ @Bean
+ @Primary
+ public DynamoDBMapper DynamoDBMapper() {
+ return DynamoDBBuilder.generateDynamoDBMapper(amazonDynamoDB(), tablePrefix);
+ }
+}
diff --git a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/record/DynamoDBRecord.java b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/record/DynamoDBRecord.java
new file mode 100644
index 000000000..676c904d2
--- /dev/null
+++ b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/record/DynamoDBRecord.java
@@ -0,0 +1,20 @@
+package org.opengroup.osdu.indexer.aws.util.dynamodb.record;
+
+import com.amazonaws.osdu.util.dynamodb.DynamoDBRepository;
+import org.opengroup.osdu.indexer.aws.model.RecordMetadataDoc;
+import org.socialsignin.spring.data.dynamodb.repository.EnableScan;
+import org.socialsignin.spring.data.dynamodb.repository.EnableScanCount;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.Pageable;
+
+import java.util.List;
+
+@EnableScan
+@EnableScanCount
+public interface DynamoDBRecord extends DynamoDBRepository<RecordMetadataDoc, String> {
+ // Currently unsupported by the spring-data-dynamodb library, see: https://github.com/derjust/spring-data-dynamodb/issues/114
+ // No release timeline at this time
+ List<RecordMetadataDoc> findByKindAndStatus(String kind, String status);
+ Page<RecordMetadataDoc> findByKindAndStatus(String kind, String status, Pageable pageable);
+ List<RecordMetadataDoc> findByUser(String user);
+}
diff --git a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/record/LegalTagsTypeConverter.java b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/record/LegalTagsTypeConverter.java
new file mode 100644
index 000000000..b2aad3dd2
--- /dev/null
+++ b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/record/LegalTagsTypeConverter.java
@@ -0,0 +1,49 @@
+package org.opengroup.osdu.indexer.aws.util.dynamodb.record;
+
+import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBTypeConverter;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.opengroup.osdu.is.core.logging.JaxRsDpsLog;
+
+import javax.inject.Inject;
+import java.io.IOException;
+import java.util.Set;
+
+public class LegalTagsTypeConverter implements DynamoDBTypeConverter<String, Set<String>> {
+
+@Inject
+private JaxRsDpsLog logger;
+
+@Override
+// Converts an array of legaltag strings to a JSON string for DynamoDB
+public String convert(Set<String> legaltags) {
+ ObjectMapper objectMapper = new ObjectMapper();
+ try {
+ return objectMapper.writeValueAsString(legaltags);
+ } catch (JsonProcessingException e) {
+ logger.error(String.format("There was an error converting the schema to a JSON string. %s", e.getMessage()));
+ }
+ return null;
+ }
+
+@Override
+// Converts a JSON string of an array of legaltag strings to a list of legaltag strings
+public Set<String> unconvert(String legaltagsString) {
+ ObjectMapper objectMapper = new ObjectMapper();
+ try {
+ return objectMapper.readValue(legaltagsString, new TypeReference<Set<String>>(){});
+ } catch (JsonParseException e) {
+ logger.error(String.format("There was an error parsing the legaltags JSON string. %s", e.getMessage()));
+ } catch (JsonMappingException e) {
+ logger.error(String.format("There was an error mapping the legaltags JSON string. %s", e.getMessage()));
+ } catch (IOException e) {
+ logger.error(String.format("There was an IO exception while mapping the legaltags objects. %s", e.getMessage()));
+ } catch (Exception e) {
+ logger.error(String.format("There was an unknown exception legaltags the schema. %s", e.getMessage()));
+ }
+ return null;
+ }
+}
diff --git a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/record/RecordMetadataTypeConverter.java b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/record/RecordMetadataTypeConverter.java
new file mode 100644
index 000000000..832e8633c
--- /dev/null
+++ b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/record/RecordMetadataTypeConverter.java
@@ -0,0 +1,52 @@
+package org.opengroup.osdu.indexer.aws.util.dynamodb.record;
+
+import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBTypeConverter;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.opengroup.osdu.indexer.aws.model.RecordMetadata;
+import org.opengroup.osdu.indexer.aws.model.RecordMetadataDoc;
+import org.opengroup.osdu.is.core.logging.JaxRsDpsLog;
+
+import javax.inject.Inject;
+import java.io.IOException;
+
+public class RecordMetadataTypeConverter implements DynamoDBTypeConverter<String, RecordMetadata> {
+
+ @Inject
+ private JaxRsDpsLog logger;
+
+ @Override
+ // Converts RecordMetadata to a JSON string for DynamoDB
+ public String convert(RecordMetadata recordMetadata) {
+ ObjectMapper objectMapper = new ObjectMapper();
+ try {
+ return objectMapper.writeValueAsString(recordMetadata);
+ } catch (JsonProcessingException e) {
+ logger.error(String.format("There was an error converting the record metadata to a JSON string. %s", e.getMessage()));
+ }
+ return null;
+ }
+
+ @Override
+ // Converts a JSON string of an array of RecordMetadata to a RecordMetadata object
+ public RecordMetadata unconvert(String recordMetadataString) {
+ ObjectMapper objectMapper = new ObjectMapper();
+ objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ try {
+ return objectMapper.readValue(recordMetadataString, new TypeReference<RecordMetadata>(){});
+ } catch (JsonParseException e) {
+ logger.error(String.format("There was an error parsing the record metadata JSON string. %s", e.getMessage()));
+ } catch (JsonMappingException e) {
+ logger.error(String.format("There was an error mapping the record metadata JSON string. %s", e.getMessage()));
+ } catch (IOException e) {
+ logger.error(String.format("There was an IO exception while mapping the record metadata objects. %s", e.getMessage()));
+ } catch (Exception e) {
+ logger.error(String.format("There was an unknown exception converting the record metadata. %s", e.getMessage()));
+ }
+ return null;
+ }
+}
\ No newline at end of file
diff --git a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/schema/DynamoDBSchema.java b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/schema/DynamoDBSchema.java
new file mode 100644
index 000000000..628475112
--- /dev/null
+++ b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/schema/DynamoDBSchema.java
@@ -0,0 +1,24 @@
+// Copyright © Amazon Web Services
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.aws.util.dynamodb.schema;
+
+import com.amazonaws.osdu.util.dynamodb.DynamoDBRepository;
+import org.opengroup.osdu.indexer.aws.model.SchemaDoc;
+import org.socialsignin.spring.data.dynamodb.repository.EnableScan;
+import org.socialsignin.spring.data.dynamodb.repository.EnableScanCount;
+
+@EnableScan
+@EnableScanCount
+public interface DynamoDBSchema extends DynamoDBRepository<SchemaDoc, String> {}
diff --git a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/schema/SchemaExtTypeConverter.java b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/schema/SchemaExtTypeConverter.java
new file mode 100644
index 000000000..2b76a0e36
--- /dev/null
+++ b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/schema/SchemaExtTypeConverter.java
@@ -0,0 +1,46 @@
+package org.opengroup.osdu.indexer.aws.util.dynamodb.schema;
+
+import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBTypeConverter;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.opengroup.osdu.is.core.logging.JaxRsDpsLog;
+
+import javax.inject.Inject;
+import java.io.IOException;
+import java.util.Map;
+
+public class SchemaExtTypeConverter implements DynamoDBTypeConverter<String, Map<String, Object>> {
+ @Inject
+ private JaxRsDpsLog logger;
+
+ @Override
+ public String convert(Map<String, Object> stringObjectMap) {
+ ObjectMapper objectMapper = new ObjectMapper();
+ try {
+ return objectMapper.writeValueAsString(stringObjectMap);
+ } catch (JsonProcessingException e) {
+ logger.error(String.format("There was an error converting the schema to a JSON string. %s", e.getMessage()));
+ }
+ return null;
+ }
+
+ @Override
+ public Map<String, Object> unconvert(String s) {
+ ObjectMapper objectMapper = new ObjectMapper();
+ try {
+ return objectMapper.readValue(s, new TypeReference<Map<String, Object>>(){});
+ } catch (JsonParseException e) {
+ logger.error(String.format("There was an error parsing the schema JSON string. %s", e.getMessage()));
+ } catch (JsonMappingException e) {
+ logger.error(String.format("There was an error mapping the schema JSON string. %s", e.getMessage()));
+ } catch (IOException e) {
+ logger.error(String.format("There was an IO exception while mapping the schema objects. %s", e.getMessage()));
+ } catch (Exception e) {
+ logger.error(String.format("There was an unknown exception converting the schema. %s", e.getMessage()));
+ }
+ return null;
+ }
+}
diff --git a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/schema/SchemaItemTypeConverter.java b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/schema/SchemaItemTypeConverter.java
new file mode 100644
index 000000000..b7e8b6a82
--- /dev/null
+++ b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/schema/SchemaItemTypeConverter.java
@@ -0,0 +1,48 @@
+package org.opengroup.osdu.indexer.aws.util.dynamodb.schema;
+
+import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBTypeConverter;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.opengroup.osdu.core.api.storage.models.SchemaNode;
+import org.opengroup.osdu.indexer.aws.model.SchemaItem;
+import org.opengroup.osdu.is.core.logging.JaxRsDpsLog;
+
+import javax.inject.Inject;
+import java.io.IOException;
+import java.util.List;
+
+public class SchemaItemTypeConverter implements DynamoDBTypeConverter<String, List<SchemaItem>> {
+ @Inject
+ private JaxRsDpsLog logger;
+
+ @Override
+ public String convert(List<SchemaItem> schemaNodes) {
+ ObjectMapper objectMapper = new ObjectMapper();
+ try {
+ return objectMapper.writeValueAsString(schemaNodes);
+ } catch (JsonProcessingException e) {
+ logger.error(String.format("There was an error converting the schema to a JSON string. %s", e.getMessage()));
+ }
+ return null;
+ }
+
+ @Override
+ public List<SchemaItem> unconvert(String s) {
+ ObjectMapper objectMapper = new ObjectMapper();
+ try {
+ return objectMapper.readValue(s, new TypeReference<List<SchemaItem>>(){});
+ } catch (JsonParseException e) {
+ logger.error(String.format("There was an error parsing the schema JSON string. %s", e.getMessage()));
+ } catch (JsonMappingException e) {
+ logger.error(String.format("There was an error mapping the schema JSON string. %s", e.getMessage()));
+ } catch (IOException e) {
+ logger.error(String.format("There was an IO exception while mapping the schema objects. %s", e.getMessage()));
+ } catch (Exception e) {
+ logger.error(String.format("There was an unknown exception converting the schema. %s", e.getMessage()));
+ }
+ return null;
+ }
+}
diff --git a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/s3/S3RecordClient.java b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/s3/S3RecordClient.java
new file mode 100644
index 000000000..ca98fcafd
--- /dev/null
+++ b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/s3/S3RecordClient.java
@@ -0,0 +1,43 @@
+package org.opengroup.osdu.indexer.aws.util.s3;
+
+import com.amazonaws.AmazonServiceException;
+import com.amazonaws.SdkClientException;
+import com.amazonaws.osdu.util.s3.S3Builder;
+import com.amazonaws.services.s3.AmazonS3;
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.indexer.aws.model.RecordMetadata;
+import org.opengroup.osdu.is.core.util.AppException;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.stereotype.Component;
+
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+
+@Configuration
+public class S3RecordClient {
+
+ private AmazonS3 s3;
+
+ @Value("${aws.s3.records.bucket-name}")
+ private String recordsBucketName;
+
+
+ // Constructor Injection
+ public S3RecordClient(@Value("${aws.s3.region}") String awsS3Region, @Value("${aws.s3.endpoint}") String awsS3Endpoint){
+ s3 = S3Builder.generateS3Client(awsS3Endpoint, awsS3Region);
+ }
+
+
+ public String getRecord(RecordMetadata recordMetadata) {
+ String record = "";
+ String keyName = getKeyNameForLatestVersion(recordMetadata);
+ record = s3.getObjectAsString(recordsBucketName, keyName);
+ return record;
+ }
+
+ private String getKeyNameForLatestVersion(RecordMetadata recordMetadata){
+ return recordMetadata.getKind() + "/" + recordMetadata.getId() + "/" + recordMetadata.getLatestVersion();
+ }
+}
diff --git a/indexer-service-aws/src/main/resources/application.properties b/indexer-service-aws/src/main/resources/application.properties
index 603398728..698d068e6 100644
--- a/indexer-service-aws/src/main/resources/application.properties
+++ b/indexer-service-aws/src/main/resources/application.properties
@@ -8,19 +8,29 @@ DEFAULT_DATA_COUNTRY=US
CRON_INDEX_CLEANUP_THRESHOLD_DAYS=3
CRON_EMPTY_INDEX_CLEANUP_THRESHOLD_DAYS=7
-## AWS IAM configuration
-aws.accessKeyId=AKIA453FG2R346APY3VT
-aws.secretKey=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
+## AWS S3 configuration
+aws.s3.records.bucket-name=dev-osdu-data
+aws.s3.max-record-threads=2000
+aws.s3.enable-https=true
+aws.s3.region=us-east-1
+aws.s3.endpoint=s3.us-east-1.amazonaws.com
## AWS SNS configuration
aws.sns.region=us-east-1
aws.sns.arn=arn:aws:sns:us-east-1:888733619319:dev-osdu-messages
aws.sns.topic-name=dev-osdu-messages
+#Spring Configuration
spring.security.user.name=opendes@byoc.local
spring.security.user.password=123
spring.security.user.roles=service.storage.admin
+## AWS DynamoDB configuration
+aws.dynamodb.key=kind
+aws.dynamodb.table.prefix=dev-
+aws.dynamodb.region=us-east-1
+aws.dynamodb.endpoint=dynamodb.us-east-1.amazonaws.com
+
## AWS ES configuration
#aws.es.host=https://search-osdu-indexer-nj62ktooaiug2mjqzguzx5utr4.us-east-1.es.amazonaws.com
#aws.es.port=443
--
GitLab