updating record repo.

indexer-service-aws/pom.xml

@@ -18,6 +18,25 @@
           <artifactId>indexer-service-root</artifactId>
           <version>1.0-SNAPSHOT</version>
       </dependency>
+      <dependency>
+          <groupId>com.amazonaws</groupId>
+          <artifactId>aws-java-sdk-core</artifactId>
+          <version>1.11.651</version>
+      </dependency>
+      <dependency>
+          <groupId>com.amazonaws</groupId>
+          <artifactId>aws-osdu-util</artifactId>
+          <version>0.0.2</version>
+      </dependency>
+      <dependency>
+          <groupId>com.github.derjust</groupId>
+          <artifactId>spring-data-dynamodb</artifactId>
+          <version>5.1.0</version>
+      </dependency>
+      <dependency>
+          <groupId>org.projectlombok</groupId>
+          <artifactId>lombok</artifactId>
+      </dependency>
    <dependency>
       <groupId>com.amazonaws</groupId>
       <artifactId>aws-java-sdk</artifactId>

indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/IndexerAwsApplication.java

@@ -2,10 +2,12 @@ package org.opengroup.osdu.indexer.aws;
 
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.actuate.autoconfigure.elasticsearch.ElasticSearchRestHealthIndicatorAutoConfiguration;
+import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
 import org.springframework.context.annotation.ComponentScan;
 
-@SpringBootApplication(exclude = {ElasticSearchRestHealthIndicatorAutoConfiguration.class})
+@SpringBootApplication(exclude = {ElasticSearchRestHealthIndicatorAutoConfiguration.class, SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
 @ComponentScan({"org.opengroup.osdu.is.core", "org.opengroup.osdu.indexer"})
 public class IndexerAwsApplication {
     public static void main( String[] args )

indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/RecordMetadata.java

+package org.opengroup.osdu.indexer.aws.model;
+
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.indexer.model.*;
+import org.opengroup.osdu.is.core.util.AppException;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@Data
+@NoArgsConstructor
+public class RecordMetadata {
+    private String id;
+
+    private String kind;
+
+    private StorageAcl acl;
+
+    private Legal legal;
+
+    private RecordAncestry ancestry;
+
+    private List<String> gcsVersionPaths = new ArrayList<>();
+
+    private RecordState status;
+
+    private String user;
+
+    // epoch time
+    private long createTime;
+
+    private String modifyUser;
+
+    // epoch time
+    private long modifyTime;
+
+    public RecordMetadata(RecordIndexerPayload.Record record) {
+        this.id = record.getId();
+        this.kind = record.getKind();
+        this.acl = record.getAcl();
+        this.legal = record.getLegal();
+        this.ancestry = record.getAncestry();
+    }
+
+    public Long getLatestVersion() {
+        String latestVersionPath = this.gcsVersionPaths.get(gcsVersionPaths.size() - 1);
+        String[] versionTokens = latestVersionPath.split("/");
+        return Long.parseLong(versionTokens[versionTokens.length - 1]);
+    }
+
+    public boolean hasVersion() {
+        if (gcsVersionPaths.isEmpty()) {
+            return false;
+        } else {
+            return true;
+        }
+    }
+
+    public void addGcsPath(long version) {
+        this.gcsVersionPaths.add(String.format("%s/%s/%s", this.kind, this.id, version));
+    }
+
+    public String getVersionPath(Long version) {
+        for (String path : this.gcsVersionPaths) {
+            if (path.contains(Long.toString(version))) {
+                return path;
+            }
+        }
+
+        throw new AppException(HttpStatus.SC_NOT_FOUND, "Record version not found",
+                "The requested record version was not found");
+    }
+
+    public void resetGcsPath(List<String> gcsVersionPathList) {
+        this.gcsVersionPaths.clear();
+        for (String path: gcsVersionPathList) {
+            this.gcsVersionPaths.add(path);
+        }
+    }
+}

indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/RecordMetadataDoc.java

+package org.opengroup.osdu.indexer.aws.model;
+
+import com.amazonaws.services.dynamodbv2.datamodeling.*;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.opengroup.osdu.indexer.aws.util.dynamodb.record.LegalTagsTypeConverter;
+import org.opengroup.osdu.indexer.aws.util.dynamodb.record.RecordMetadataTypeConverter;
+
+import java.util.Set;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@DynamoDBTable(tableName = "RecordMetadataRepository") // DynamoDB table name (without environment prefix)
+public class RecordMetadataDoc {
+
+    @DynamoDBHashKey(attributeName = "Id")
+    private String id;
+
+    @DynamoDBIndexHashKey(attributeName = "Kind", globalSecondaryIndexName = "KindStatusIndex")
+    private String kind;
+
+    @DynamoDBIndexRangeKey(attributeName = "Status", globalSecondaryIndexName = "KindStatusIndex")
+    private String status;
+
+    @DynamoDBIndexHashKey(attributeName = "User", globalSecondaryIndexName = "UserIndex")
+    private String user;
+
+    @DynamoDBTypeConverted(converter = RecordMetadataTypeConverter.class)
+    @DynamoDBAttribute(attributeName = "metadata")
+    private RecordMetadata metadata;
+
+    @DynamoDBTypeConverted(converter = LegalTagsTypeConverter.class)
+    @DynamoDBAttribute(attributeName = "LegalTags")
+    private Set<String> legaltags;
+}

indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/RecordState.java

+package org.opengroup.osdu.indexer.aws.model;
+
+public enum RecordState {
+    /**
+     * An active record
+     */
+    active,
+
+    /**
+     * A deleted record
+     */
+    deleted,
+
+    /**
+     * A purged record
+     */
+    purged
+}

indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/SchemaDoc.java

+package org.opengroup.osdu.indexer.aws.model;
+
+import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBAttribute;
+import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBHashKey;
+import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBTable;
+import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBTypeConverted;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.opengroup.osdu.indexer.aws.util.dynamodb.schema.SchemaExtTypeConverter;
+import org.opengroup.osdu.indexer.aws.util.dynamodb.schema.SchemaItemTypeConverter;
+import org.opengroup.osdu.indexer.model.Schema;
+
+import java.util.List;
+import java.util.Map;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@DynamoDBTable(tableName = "SchemaRepository") // DynamoDB table name (without environment prefix)
+public class SchemaDoc {
+
+    @DynamoDBHashKey(attributeName = "Kind")
+    private String kind;
+
+    @DynamoDBTypeConverted(converter = SchemaExtTypeConverter.class)
+    @DynamoDBAttribute(attributeName = "ext")
+    private Map<String,Object> extension;
+
+    @DynamoDBAttribute(attributeName = "User")
+    private String user;
+
+    @DynamoDBTypeConverted(converter = SchemaItemTypeConverter.class)
+    @DynamoDBAttribute(attributeName = "schema")
+    private List<SchemaItem> schemaItems;
+}
+

indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/SchemaItem.java

+package org.opengroup.osdu.indexer.aws.model;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.Map;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class SchemaItem {
+
+    private String path;
+
+    private String kind;
+
+    @JsonInclude(value = JsonInclude.Include.NON_NULL)
+    private Map<String, Object> ext;
+}
\ No newline at end of file

indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/publish/PublisherImpl.java

 package org.opengroup.osdu.indexer.aws.publish;
 
+import com.amazonaws.osdu.util.sns.SNSBuilder;
 import com.amazonaws.services.sns.model.MessageAttributeValue;
 import com.amazonaws.services.sns.model.PublishRequest;
 import com.google.gson.Gson;
@@ -16,12 +17,20 @@ import java.util.Map;
 
 @Component
 public class PublisherImpl implements IPublisher {
-    @Inject
+
     private AmazonSNS snsClient;
 
     @Value("${aws.sns.arn}")
     private String amazonSNSTopic;
 
+    @Value("${aws.sns.region}")
+    private String amazonSNSRegion;
+
+    @Inject
+    public void init(){
+        snsClient = SNSBuilder.generateAmazonSNSClient(amazonSNSRegion);
+    }
+
     public void publishStatusChangedTagsToTopic(DpsHeaders headers, JobStatus indexerBatchStatus) throws Exception
     {
         String json = new Gson().toJson(indexerBatchStatus.getStatusesList());

indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/service/AwsStorageServiceImpl.java

+package org.opengroup.osdu.indexer.aws.service;
+
+import com.google.gson.Gson;
+import org.checkerframework.checker.units.qual.A;
+import org.opengroup.osdu.indexer.aws.model.RecordMetadataDoc;
+import org.opengroup.osdu.indexer.aws.model.SchemaDoc;
+import org.opengroup.osdu.indexer.aws.util.dynamodb.record.DynamoDBRecord;
+import org.opengroup.osdu.indexer.aws.util.dynamodb.schema.DynamoDBSchema;
+import org.opengroup.osdu.indexer.aws.util.s3.S3RecordClient;
+import org.opengroup.osdu.indexer.model.*;
+import org.opengroup.osdu.indexer.service.StorageService;
+import org.opengroup.osdu.is.core.util.AppException;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Primary;
+import org.springframework.stereotype.Component;
+
+
+import javax.inject.Inject;
+import java.io.Console;
+import java.io.UnsupportedEncodingException;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+@Primary
+@Component
+public class AwsStorageServiceImpl implements StorageService {
+
+    @Inject
+    DynamoDBSchema dynamoDBSchema;
+
+    @Inject
+    DynamoDBRecord dynamoDBRecord;
+
+    @Inject
+    S3RecordClient s3Client;
+
+//    private String getKeyNameForLatestVersion( recordMetadata){
+//        return recordMetadata.getKind() + "/" + recordMetadata.getId() + "/" + recordMetadata.getLatestVersion();
+//    }
+    @Override
+    public org.opengroup.osdu.indexer.model.Records getStorageRecords(List<String> ids) throws AppException, URISyntaxException {
+        Records records = new Records();
+        List<String> notFound = new ArrayList<>();
+        List<Records.Entity> entities = new ArrayList<>();
+        List<ConversionStatus> statuses = new ArrayList<>();
+        Gson gson = new Gson();
+        for (String id: ids) {
+            Optional<RecordMetadataDoc> doc = dynamoDBRecord.findById(id);
+            if (!doc.isPresent()){
+                notFound.add(id);
+            }
+
+            String record = s3Client.getRecord(doc.get().getMetadata());
+            Records.Entity entity = gson.fromJson(record, Records.Entity.class);
+            statuses.add(ConversionStatus.builder().status(doc.get().getStatus()).build());
+            entities.add(entity);
+        }
+        records.setNotFound(notFound);
+        records.setRecords(entities);
+        records.setConversionStatuses(statuses);
+        return records;
+    }
+
+    @Override
+    public RecordQueryResponse getRecordsByKind(RecordReindexRequest request) throws URISyntaxException {
+
+        return null;
+    }
+
+    @Override
+    public String getStorageSchema(String kind) throws URISyntaxException, UnsupportedEncodingException {
+        Optional<SchemaDoc> sd = dynamoDBSchema.findById(kind);
+        if (!sd.isPresent()) {
+            return null;
+        }
+        // Create a Schema object and assign the values retrieved from DynamoDB
+        Schema newSchema = new Schema();
+        newSchema.setKind(kind);
+        List<Schema.Mapping> mappings;
+        mappings = sd.get().getSchemaItems().stream().map(schemaItem -> Schema.Mapping.builder().kind(schemaItem.getKind()).path(schemaItem.getPath()).build()).collect(Collectors.toList());
+        newSchema.setSchema(mappings);
+        return new Gson().toJson(newSchema);
+    }
+}

indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/AWSElasticSearchConfig.java

-package org.opengroup.osdu.indexer.aws.util;
-
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.AWSCredentialsProviderChain;
-import com.amazonaws.auth.AWSStaticCredentialsProvider;
-import com.amazonaws.services.elasticsearch.AWSElasticsearch;
-import com.amazonaws.services.elasticsearch.AWSElasticsearchClientBuilder;
-import com.amazonaws.services.elasticsearch.model.ElasticsearchClusterConfig;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.Import;
-
-import javax.inject.Inject;
-
-@Configuration
-@Import({IAMConfig.class})
-public class AWSElasticSearchConfig {
-
-    @Value("${aws.sns.region}")
-    private String amazonSNSRegion;
-
-    @Inject
-    private AWSCredentials amazonAWSCredentials;
-
-    public AWSElasticsearch AWSElasticsearch() {
-        return AWSElasticsearchClientBuilder.standard()
-                .withRegion(amazonSNSRegion)
-                .withCredentials(new AWSStaticCredentialsProvider(amazonAWSCredentials)).build();
-
-    }
-
-}

indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/AmazonSNSConfig.java

-package org.opengroup.osdu.indexer.aws.util;
-
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.AWSStaticCredentialsProvider;
-import com.amazonaws.services.sns.AmazonSNS;
-import com.amazonaws.services.sns.AmazonSNSClientBuilder;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.Import;
-
-import javax.inject.Inject;
-
-@Configuration
-@Import({IAMConfig.class})
-public class AmazonSNSConfig {
-
-    @Value("${aws.sns.region}")
-    private String amazonSNSRegion;
-
-    @Value("${aws.sns.topic-name}")
-    private String amazonSNSTopicName;
-
-    @Value("${aws.sns.arn}")
-    private String amazonSNSARN;
-
-
-    @Inject
-    private AWSCredentials amazonAWSCredentials;
-
-    @Bean
-    public AmazonSNS AmazonSNS() {
-        return AmazonSNSClientBuilder.standard()
-                .withCredentials(new AWSStaticCredentialsProvider(amazonAWSCredentials))
-                .withRegion(amazonSNSRegion)
-                .build();
-    }
-
-}
\ No newline at end of file

indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IAMConfig.java

-package org.opengroup.osdu.indexer.aws.util;
-
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.BasicAWSCredentials;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-
-@Configuration
-public class IAMConfig {
-
-    @Value("${aws.accessKeyId}")
-    private String amazonAWSAccessKey;
-
-    @Value("${aws.secretKey}")
-    private String amazonAWSSecretKey;
-
-    @Bean
-    public AWSCredentials amazonAWSCredentials() {
-        // Generate AWS basic credentials from the access keys provided in application.properties
-        return new BasicAWSCredentials(
-                amazonAWSAccessKey, amazonAWSSecretKey);
-    }
-}
\ No newline at end of file

indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/DynamoDBConfig.java

+package org.opengroup.osdu.indexer.aws.util.dynamodb;
+
+import com.amazonaws.osdu.util.dynamodb.DynamoDBBuilder;
+import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
+import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBMapper;
+import org.socialsignin.spring.data.dynamodb.repository.config.EnableDynamoDBRepositories;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Primary;
+
+@Configuration
+@EnableDynamoDBRepositories
+        (basePackages = "org.opengroup.osdu.indexer.aws")
+public class DynamoDBConfig {
+
+    @Value("${aws.dynamodb.endpoint}")
+    private String amazonDynamoDBEndpoint;
+
+    @Value("${aws.dynamodb.region}")
+    private String amazonDynamoDBRegion;
+
+    @Value("${aws.dynamodb.table.prefix}")
+    private String tablePrefix;
+
+    @Bean
+    public AmazonDynamoDB amazonDynamoDB() {
+        // Generate the DynamoDB client
+        return DynamoDBBuilder.generateDynamoDBClient(amazonDynamoDBEndpoint, amazonDynamoDBRegion);
+    }
+
+    @Bean
+    @Primary
+    public DynamoDBMapper DynamoDBMapper() {
+        return DynamoDBBuilder.generateDynamoDBMapper(amazonDynamoDB(), tablePrefix);
+    }
+}

indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/record/DynamoDBRecord.java

+package org.opengroup.osdu.indexer.aws.util.dynamodb.record;
+
+import com.amazonaws.osdu.util.dynamodb.DynamoDBRepository;
+import org.opengroup.osdu.indexer.aws.model.RecordMetadataDoc;
+import org.socialsignin.spring.data.dynamodb.repository.EnableScan;
+import org.socialsignin.spring.data.dynamodb.repository.EnableScanCount;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.Pageable;
+
+import java.util.List;
+
+@EnableScan
+@EnableScanCount
+public interface DynamoDBRecord extends DynamoDBRepository<RecordMetadataDoc, String> {
+    // Currently unsupported by the spring-data-dynamodb library, see: https://github.com/derjust/spring-data-dynamodb/issues/114
+    // No release timeline at this time
+    List<RecordMetadataDoc> findByKindAndStatus(String kind, String status);
+    Page<RecordMetadataDoc> findByKindAndStatus(String kind, String status, Pageable pageable);
+    List<RecordMetadataDoc> findByUser(String user);
+}

indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/record/LegalTagsTypeConverter.java

+package org.opengroup.osdu.indexer.aws.util.dynamodb.record;
+
+import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBTypeConverter;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.opengroup.osdu.is.core.logging.JaxRsDpsLog;
+
+import javax.inject.Inject;
+import java.io.IOException;
+import java.util.Set;
+
+public class LegalTagsTypeConverter implements DynamoDBTypeConverter<String, Set<String>> {
+
+@Inject
+private JaxRsDpsLog logger;
+
+@Override
+// Converts an array of legaltag strings to a JSON string for DynamoDB
+public String convert(Set<String> legaltags) {
+    ObjectMapper objectMapper = new ObjectMapper();
+    try {
+        return objectMapper.writeValueAsString(legaltags);
+    } catch (JsonProcessingException e) {
+        logger.error(String.format("There was an error converting the schema to a JSON string. %s", e.getMessage()));
+    }
+    return null;
+    }
+
+@Override
+// Converts a JSON string of an array of legaltag strings to a list of legaltag strings
+public Set<String> unconvert(String legaltagsString) {
+    ObjectMapper objectMapper = new ObjectMapper();
+        try {
+            return objectMapper.readValue(legaltagsString, new TypeReference<Set<String>>(){});
+        } catch (JsonParseException e) {
+            logger.error(String.format("There was an error parsing the legaltags JSON string. %s", e.getMessage()));
+        } catch (JsonMappingException e) {
+            logger.error(String.format("There was an error mapping the legaltags JSON string. %s", e.getMessage()));
+        } catch (IOException e) {
+            logger.error(String.format("There was an IO exception while mapping the legaltags objects. %s", e.getMessage()));
+        } catch (Exception e) {
+            logger.error(String.format("There was an unknown exception legaltags the schema. %s", e.getMessage()));
+        }
+    return null;
+    }
+}

indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/record/RecordMetadataTypeConverter.java

+package org.opengroup.osdu.indexer.aws.util.dynamodb.record;
+
+import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBTypeConverter;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.opengroup.osdu.indexer.aws.model.RecordMetadata;
+import org.opengroup.osdu.indexer.aws.model.RecordMetadataDoc;
+import org.opengroup.osdu.is.core.logging.JaxRsDpsLog;
+
+import javax.inject.Inject;
+import java.io.IOException;
+
+public class RecordMetadataTypeConverter implements DynamoDBTypeConverter<String, RecordMetadata> {
+
+    @Inject
+    private JaxRsDpsLog logger;
+
+    @Override
+    // Converts RecordMetadata to a JSON string for DynamoDB
+    public String convert(RecordMetadata recordMetadata) {
+        ObjectMapper objectMapper = new ObjectMapper();
+        try {
+            return objectMapper.writeValueAsString(recordMetadata);
+        } catch (JsonProcessingException e) {
+            logger.error(String.format("There was an error converting the record metadata to a JSON string. %s", e.getMessage()));
+        }
+        return null;
+    }
+
+    @Override
+    // Converts a JSON string of an array of RecordMetadata to a RecordMetadata object
+    public RecordMetadata unconvert(String recordMetadataString) {
+        ObjectMapper objectMapper = new ObjectMapper();
+        objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+        try {
+            return objectMapper.readValue(recordMetadataString, new TypeReference<RecordMetadata>(){});
+        } catch (JsonParseException e) {
+            logger.error(String.format("There was an error parsing the record metadata JSON string. %s", e.getMessage()));
+        } catch (JsonMappingException e) {
+            logger.error(String.format("There was an error mapping the record metadata JSON string. %s", e.getMessage()));
+        } catch (IOException e) {
+            logger.error(String.format("There was an IO exception while mapping the record metadata objects. %s", e.getMessage()));
+        } catch (Exception e) {
+            logger.error(String.format("There was an unknown exception converting the record metadata. %s", e.getMessage()));
+        }
+        return null;
+    }
+}
\ No newline at end of file

indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/schema/DynamoDBSchema.java

+// Copyright © Amazon Web Services
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.aws.util.dynamodb.schema;
+
+import com.amazonaws.osdu.util.dynamodb.DynamoDBRepository;
+import org.opengroup.osdu.indexer.aws.model.SchemaDoc;
+import org.socialsignin.spring.data.dynamodb.repository.EnableScan;
+import org.socialsignin.spring.data.dynamodb.repository.EnableScanCount;
+
+@EnableScan
+@EnableScanCount
+public interface DynamoDBSchema extends DynamoDBRepository<SchemaDoc, String> {}

indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/schema/SchemaExtTypeConverter.java

+package org.opengroup.osdu.indexer.aws.util.dynamodb.schema;
+
+import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBTypeConverter;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.opengroup.osdu.is.core.logging.JaxRsDpsLog;
+
+import javax.inject.Inject;
+import java.io.IOException;
+import java.util.Map;
+
+public class SchemaExtTypeConverter implements DynamoDBTypeConverter<String, Map<String, Object>> {
+    @Inject
+    private JaxRsDpsLog logger;
+
+    @Override
+    public String convert(Map<String, Object> stringObjectMap) {
+        ObjectMapper objectMapper = new ObjectMapper();
+        try {
+            return objectMapper.writeValueAsString(stringObjectMap);
+        } catch (JsonProcessingException e) {
+            logger.error(String.format("There was an error converting the schema to a JSON string. %s", e.getMessage()));
+        }
+        return null;
+    }
+
+    @Override
+    public Map<String, Object> unconvert(String s) {
+        ObjectMapper objectMapper = new ObjectMapper();
+        try {
+            return objectMapper.readValue(s, new TypeReference<Map<String, Object>>(){});
+        } catch (JsonParseException e) {
+            logger.error(String.format("There was an error parsing the schema JSON string. %s", e.getMessage()));
+        } catch (JsonMappingException e) {
+            logger.error(String.format("There was an error mapping the schema JSON string. %s", e.getMessage()));
+        } catch (IOException e) {
+            logger.error(String.format("There was an IO exception while mapping the schema objects. %s", e.getMessage()));
+        } catch (Exception e) {
+            logger.error(String.format("There was an unknown exception converting the schema. %s", e.getMessage()));
+        }
+        return null;
+    }
+}

indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/schema/SchemaItemTypeConverter.java

+package org.opengroup.osdu.indexer.aws.util.dynamodb.schema;
+
+import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBTypeConverter;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.opengroup.osdu.core.api.storage.models.SchemaNode;
+import org.opengroup.osdu.indexer.aws.model.SchemaItem;
+import org.opengroup.osdu.is.core.logging.JaxRsDpsLog;
+
+import javax.inject.Inject;
+import java.io.IOException;
+import java.util.List;
+
+public class SchemaItemTypeConverter implements DynamoDBTypeConverter<String, List<SchemaItem>> {
+    @Inject
+    private JaxRsDpsLog logger;
+
+    @Override
+    public String convert(List<SchemaItem> schemaNodes) {
+        ObjectMapper objectMapper = new ObjectMapper();
+        try {
+            return objectMapper.writeValueAsString(schemaNodes);
+        } catch (JsonProcessingException e) {
+            logger.error(String.format("There was an error converting the schema to a JSON string. %s", e.getMessage()));
+        }
+        return null;
+    }
+
+    @Override
+    public List<SchemaItem> unconvert(String s) {
+        ObjectMapper objectMapper = new ObjectMapper();
+        try {
+            return objectMapper.readValue(s, new TypeReference<List<SchemaItem>>(){});
+        } catch (JsonParseException e) {
+            logger.error(String.format("There was an error parsing the schema JSON string. %s", e.getMessage()));
+        } catch (JsonMappingException e) {
+            logger.error(String.format("There was an error mapping the schema JSON string. %s", e.getMessage()));
+        } catch (IOException e) {
+            logger.error(String.format("There was an IO exception while mapping the schema objects. %s", e.getMessage()));
+        } catch (Exception e) {
+            logger.error(String.format("There was an unknown exception converting the schema. %s", e.getMessage()));
+        }
+        return null;
+    }
+}

indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/s3/S3RecordClient.java

+package org.opengroup.osdu.indexer.aws.util.s3;
+
+import com.amazonaws.AmazonServiceException;
+import com.amazonaws.SdkClientException;
+import com.amazonaws.osdu.util.s3.S3Builder;
+import com.amazonaws.services.s3.AmazonS3;
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.indexer.aws.model.RecordMetadata;
+import org.opengroup.osdu.is.core.util.AppException;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.stereotype.Component;
+
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+
+@Configuration
+public class S3RecordClient {
+
+    private AmazonS3 s3;
+
+    @Value("${aws.s3.records.bucket-name}")
+    private String recordsBucketName;
+
+
+    // Constructor Injection
+    public S3RecordClient(@Value("${aws.s3.region}") String awsS3Region,  @Value("${aws.s3.endpoint}") String awsS3Endpoint){
+        s3 = S3Builder.generateS3Client(awsS3Endpoint, awsS3Region);
+    }
+
+
+    public String getRecord(RecordMetadata recordMetadata) {
+        String record = "";
+        String keyName = getKeyNameForLatestVersion(recordMetadata);
+        record = s3.getObjectAsString(recordsBucketName, keyName);
+        return record;
+    }
+
+    private String getKeyNameForLatestVersion(RecordMetadata recordMetadata){
+        return recordMetadata.getKind() + "/" + recordMetadata.getId() + "/" + recordMetadata.getLatestVersion();
+    }
+}