diff --git a/indexer-service-aws/pom.xml b/indexer-service-aws/pom.xml index 588a2d34477bc534829ddec8a72ff23dd065f902..845b565ceb900294c24e8b417173a9ce3c564ba3 100644 --- a/indexer-service-aws/pom.xml +++ b/indexer-service-aws/pom.xml @@ -33,11 +33,6 @@ <artifactId>aws-osdu-util</artifactId> <version>0.0.5</version> </dependency> - <dependency> - <groupId>com.github.derjust</groupId> - <artifactId>spring-data-dynamodb</artifactId> - <version>5.1.0</version> - </dependency> <dependency> <groupId>org.projectlombok</groupId> <artifactId>lombok</artifactId> diff --git a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/RecordMetadata.java b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/RecordMetadata.java deleted file mode 100644 index aa52ae69a083c3e61fc0e9c973afaf5b96a8d378..0000000000000000000000000000000000000000 --- a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/RecordMetadata.java +++ /dev/null @@ -1,82 +0,0 @@ -package org.opengroup.osdu.indexer.aws.model; - -import lombok.Data; -import lombok.NoArgsConstructor; -import org.apache.http.HttpStatus; -import org.opengroup.osdu.indexer.model.*; -import org.opengroup.osdu.is.core.util.AppException; - -import java.util.ArrayList; -import java.util.List; - -@Data -@NoArgsConstructor -public class RecordMetadata { - private String id; - - private String kind; - - private StorageAcl acl; - - private Legal legal; - - private RecordAncestry ancestry; - - private List<String> gcsVersionPaths = new ArrayList<>(); - - private RecordState status; - - private String user; - - // epoch time - private long createTime; - - private String modifyUser; - - // epoch time - private long modifyTime; - - public RecordMetadata(RecordIndexerPayload.Record record) { - this.id = record.getId(); - this.kind = record.getKind(); - this.acl = record.getAcl(); - this.legal = record.getLegal(); - this.ancestry = record.getAncestry(); - } - - public Long getLatestVersion() { - String latestVersionPath = this.gcsVersionPaths.get(gcsVersionPaths.size() - 1); - String[] versionTokens = latestVersionPath.split("/"); - return Long.parseLong(versionTokens[versionTokens.length - 1]); - } - - public boolean hasVersion() { - if (gcsVersionPaths.isEmpty()) { - return false; - } else { - return true; - } - } - - public void addGcsPath(long version) { - this.gcsVersionPaths.add(String.format("%s/%s/%s", this.kind, this.id, version)); - } - - public String getVersionPath(Long version) { - for (String path : this.gcsVersionPaths) { - if (path.contains(Long.toString(version))) { - return path; - } - } - - throw new AppException(HttpStatus.SC_NOT_FOUND, "Record version not found", - "The requested record version was not found"); - } - - public void resetGcsPath(List<String> gcsVersionPathList) { - this.gcsVersionPaths.clear(); - for (String path: gcsVersionPathList) { - this.gcsVersionPaths.add(path); - } - } -} diff --git a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/RecordMetadataDoc.java b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/RecordMetadataDoc.java deleted file mode 100644 index caa128dcd38e62eafee1884bae315931e1cdc707..0000000000000000000000000000000000000000 --- a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/RecordMetadataDoc.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.opengroup.osdu.indexer.aws.model; - -import com.amazonaws.services.dynamodbv2.datamodeling.*; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; -import org.opengroup.osdu.indexer.aws.util.dynamodb.record.LegalTagsTypeConverter; -import org.opengroup.osdu.indexer.aws.util.dynamodb.record.RecordMetadataTypeConverter; - -import java.util.Set; - -@Data -@AllArgsConstructor -@NoArgsConstructor -@DynamoDBTable(tableName = "RecordMetadataRepository") // DynamoDB table name (without environment prefix) -public class RecordMetadataDoc { - - @DynamoDBHashKey(attributeName = "Id") - private String id; - - @DynamoDBIndexHashKey(attributeName = "Kind", globalSecondaryIndexName = "KindStatusIndex") - private String kind; - - @DynamoDBIndexRangeKey(attributeName = "Status", globalSecondaryIndexName = "KindStatusIndex") - private String status; - - @DynamoDBIndexHashKey(attributeName = "User", globalSecondaryIndexName = "UserIndex") - private String user; - - @DynamoDBTypeConverted(converter = RecordMetadataTypeConverter.class) - @DynamoDBAttribute(attributeName = "metadata") - private RecordMetadata metadata; - - @DynamoDBTypeConverted(converter = LegalTagsTypeConverter.class) - @DynamoDBAttribute(attributeName = "LegalTags") - private Set<String> legaltags; -} diff --git a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/RecordState.java b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/RecordState.java deleted file mode 100644 index 21c495140a1a8281c3c18b817b1040e39e1fd0e1..0000000000000000000000000000000000000000 --- a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/RecordState.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.opengroup.osdu.indexer.aws.model; - -public enum RecordState { - /** - * An active record - */ - active, - - /** - * A deleted record - */ - deleted, - - /** - * A purged record - */ - purged -} diff --git a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/SchemaDoc.java b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/SchemaDoc.java deleted file mode 100644 index def063bf85c9d4b8ec06309ed7d37be24e99d000..0000000000000000000000000000000000000000 --- a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/SchemaDoc.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.opengroup.osdu.indexer.aws.model; - -import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBAttribute; -import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBHashKey; -import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBTable; -import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBTypeConverted; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; -import org.opengroup.osdu.indexer.aws.util.dynamodb.schema.SchemaExtTypeConverter; -import org.opengroup.osdu.indexer.aws.util.dynamodb.schema.SchemaItemTypeConverter; -import org.opengroup.osdu.indexer.model.Schema; - -import java.util.List; -import java.util.Map; - -@Data -@AllArgsConstructor -@NoArgsConstructor -@DynamoDBTable(tableName = "SchemaRepository") // DynamoDB table name (without environment prefix) -public class SchemaDoc { - - @DynamoDBHashKey(attributeName = "Kind") - private String kind; - - @DynamoDBTypeConverted(converter = SchemaExtTypeConverter.class) - @DynamoDBAttribute(attributeName = "ext") - private Map<String,Object> extension; - - @DynamoDBAttribute(attributeName = "User") - private String user; - - @DynamoDBTypeConverted(converter = SchemaItemTypeConverter.class) - @DynamoDBAttribute(attributeName = "schema") - private List<SchemaItem> schemaItems; -} - diff --git a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/SchemaItem.java b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/SchemaItem.java deleted file mode 100644 index b00593b50e3a581a88f33a3960b7f5652bb97036..0000000000000000000000000000000000000000 --- a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/model/SchemaItem.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.opengroup.osdu.indexer.aws.model; - -import com.fasterxml.jackson.annotation.JsonInclude; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.util.Map; - -@Data -@AllArgsConstructor -@NoArgsConstructor -public class SchemaItem { - - private String path; - - private String kind; - - @JsonInclude(value = JsonInclude.Include.NON_NULL) - private Map<String, Object> ext; -} \ No newline at end of file diff --git a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/service/AwsStorageServiceImpl.java b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/service/AwsStorageServiceImpl.java deleted file mode 100644 index cf73caf3d301e33c3d70c0ba5d914b0995af95bc..0000000000000000000000000000000000000000 --- a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/service/AwsStorageServiceImpl.java +++ /dev/null @@ -1,85 +0,0 @@ -package org.opengroup.osdu.indexer.aws.service; -import com.amazonaws.osdu.util.dynamodb.DynamoDBQueryHelper; -import com.google.gson.Gson; -import org.opengroup.osdu.indexer.aws.model.RecordMetadataDoc; -import org.opengroup.osdu.indexer.aws.model.SchemaDoc; -import org.opengroup.osdu.indexer.aws.util.s3.S3RecordClient; -import org.opengroup.osdu.indexer.model.*; -import org.opengroup.osdu.indexer.service.StorageService; -import org.opengroup.osdu.is.core.util.AppException; -import org.springframework.context.annotation.Primary; -import org.springframework.stereotype.Component; - - -import javax.inject.Inject; -import java.io.UnsupportedEncodingException; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -@Primary -@Component -public class AwsStorageServiceImpl implements StorageService { - - @Inject - private DynamoDBQueryHelper queryHelper; - - @Inject - S3RecordClient s3Client; - - @Override - public org.opengroup.osdu.indexer.model.Records getStorageRecords(List<String> ids) throws AppException, URISyntaxException { - Records records = new Records(); - List<String> notFound = new ArrayList<>(); - List<Records.Entity> entities = new ArrayList<>(); - List<ConversionStatus> statuses = new ArrayList<>(); - Gson gson = new Gson(); - for (String id: ids) { - RecordMetadataDoc doc = queryHelper.loadByPrimaryKey(RecordMetadataDoc.class, id); - if (doc == null) { - notFound.add(id); - } - Records.Entity entity = createRecord(doc, gson); - statuses.add(ConversionStatus.builder().status(doc.getStatus()).build()); - entities.add(entity); - } - records.setNotFound(notFound); - records.setRecords(entities); - records.setConversionStatuses(statuses); - return records; - } - - private Records.Entity createRecord(RecordMetadataDoc recordMetadataDoc, Gson gson) { - String record = s3Client.getRecord(recordMetadataDoc.getMetadata()); - Records.Entity entity = gson.fromJson(record, Records.Entity.class); - entity.setId(recordMetadataDoc.getId()); - entity.setAcl(recordMetadataDoc.getMetadata().getAcl()); - entity.setAncestry(recordMetadataDoc.getMetadata().getAncestry()); - entity.setKind(recordMetadataDoc.getKind()); - Legal legal = new Legal(); - legal.setLegaltags(recordMetadataDoc.getLegaltags().toArray(new String[recordMetadataDoc.getLegaltags().size()])); - entity.setLegal(legal); - entity.setVersion(recordMetadataDoc.getMetadata().getLatestVersion()); - return entity; - } - - @Override - public RecordQueryResponse getRecordsByKind(RecordReindexRequest request) throws URISyntaxException { - - return null; - } - - @Override - public String getStorageSchema(String kind) throws URISyntaxException, UnsupportedEncodingException { - SchemaDoc sd = queryHelper.loadByPrimaryKey(SchemaDoc.class, kind); - if (sd == null) return null; - // Create a Schema object and assign the values retrieved from DynamoDB - Schema newSchema = new Schema(); - newSchema.setKind(kind); - List<Schema.Mapping> mappings; - mappings = sd.getSchemaItems().stream().map(schemaItem -> Schema.Mapping.builder().kind(schemaItem.getKind()).path(schemaItem.getPath()).build()).collect(Collectors.toList()); - newSchema.setSchema(mappings); - return new Gson().toJson(newSchema); - } -} diff --git a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/service/ElasticClientHandlerAws.java b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/service/ElasticClientHandlerAws.java index 5aaca8f731770c1467e4b98e312873858a691826..68c08a3512213ecf708276d3278f38a90394cad5 100644 --- a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/service/ElasticClientHandlerAws.java +++ b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/service/ElasticClientHandlerAws.java @@ -44,6 +44,7 @@ public class ElasticClientHandlerAws extends ElasticClientHandler { AWS4Signer signer = new AWS4Signer(); signer.setServiceName(serviceName); signer.setRegionName(region); + // TODO: Legal please review the use of this code coming from https://github.com/awslabs/aws-request-signing-apache-interceptor HttpRequestInterceptor interceptor = new AWSRequestSigningApacheInterceptor(serviceName, signer, credentials); return new RestHighLevelClient(RestClient.builder(HttpHost.create(host)).setHttpClientConfigCallback(configCallBack -> configCallBack.addInterceptorLast(interceptor))); diff --git a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/HeadersInfoAwsImpl.java b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/HeadersInfoAwsImpl.java index 9928d5ad9cbbbf803d1c37ede01022e6005301e3..8aed28fd00898cf05bf136317d121514218f3789 100644 --- a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/HeadersInfoAwsImpl.java +++ b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/HeadersInfoAwsImpl.java @@ -33,7 +33,6 @@ import javax.inject.Inject; @Primary @Log @Component -@RequestScope public class HeadersInfoAwsImpl implements IHeadersInfo { @Inject diff --git a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java new file mode 100644 index 0000000000000000000000000000000000000000..6c2886c6ddcffd9052dff979fdef8139cda65a53 --- /dev/null +++ b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java @@ -0,0 +1,46 @@ +package org.opengroup.osdu.indexer.aws.util; + +import com.amazonaws.osdu.util.sqs.SQSBuilder; +import com.amazonaws.services.sqs.AmazonSQS; +import com.amazonaws.services.sqs.model.SendMessageRequest; +import org.opengroup.osdu.core.api.DpsHeaders; +import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Primary; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; + +@Primary +@Component +public class IndexerQueueTaskBuilderAws extends IndexerQueueTaskBuilder { + + private AmazonSQS sqsClient; + + @Value("${aws.region}") + private String region; + + @Value("${aws.sqs.queue}") + private String queueName; + + @Inject + public void init() { + sqsClient = SQSBuilder.generateSqsClient(region); + } + + @Override + public void createWorkerTask(String payload, DpsHeaders headers) { + createTask(payload, headers); + } + + @Override + public void createReIndexTask(String payload,DpsHeaders headers) { + createTask(payload, headers); + } + + private void createTask(String payload, DpsHeaders headers) { + String queueUrl = sqsClient.getQueueUrl(queueName).getQueueUrl(); + SendMessageRequest messageRequest = new SendMessageRequest().withQueueUrl(queueUrl).withMessageBody(payload); + sqsClient.sendMessage(messageRequest); + } +} diff --git a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/record/LegalTagsTypeConverter.java b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/record/LegalTagsTypeConverter.java deleted file mode 100644 index f6486cdf8334e490a8e1a291303f66ef0070f9ad..0000000000000000000000000000000000000000 --- a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/record/LegalTagsTypeConverter.java +++ /dev/null @@ -1,53 +0,0 @@ -package org.opengroup.osdu.indexer.aws.util.dynamodb.record; - -import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBTypeConverter; -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.opengroup.osdu.is.core.logging.JaxRsDpsLog; -import org.opengroup.osdu.is.core.util.AppException; - -import javax.inject.Inject; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Set; - -public class LegalTagsTypeConverter implements DynamoDBTypeConverter<String, Set<String>> { - -@Inject -private JaxRsDpsLog logger; - -@Override -// Converts an array of legaltag strings to a JSON string for DynamoDB -public String convert(Set<String> legaltags) { - ObjectMapper objectMapper = new ObjectMapper(); - String converted; - try { - converted = objectMapper.writeValueAsString(legaltags); - } catch (JsonProcessingException e) { - throw new AppException(e.hashCode(), e.getMessage(), e.getOriginalMessage(), e); - } - return converted; - } - -@Override -// Converts a JSON string of an array of legaltag strings to a list of legaltag strings -public Set<String> unconvert(String legaltagsString) { - ObjectMapper objectMapper = new ObjectMapper(); - Set<String> unconvertedStrings; - try { - unconvertedStrings = objectMapper.readValue(legaltagsString, new TypeReference<Set<String>>(){}); - } catch (JsonParseException e) { - throw new AppException(e.hashCode(), e.getMessage(), e.getOriginalMessage(), e); - } catch (JsonMappingException e) { - throw new AppException(e.hashCode(), e.getMessage(), e.getOriginalMessage(), e); - } catch (IOException e) { - throw new AppException(e.hashCode(), e.getMessage(), e.getLocalizedMessage(), e); - } catch (Exception e) { - throw new AppException(e.hashCode(), e.getMessage(), e.getLocalizedMessage(), e); - } - return unconvertedStrings; - } -} diff --git a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/record/RecordMetadataTypeConverter.java b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/record/RecordMetadataTypeConverter.java deleted file mode 100644 index 40a7af75127a5fa96659ba1717edfe16479b55db..0000000000000000000000000000000000000000 --- a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/record/RecordMetadataTypeConverter.java +++ /dev/null @@ -1,55 +0,0 @@ -package org.opengroup.osdu.indexer.aws.util.dynamodb.record; - -import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBTypeConverter; -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.opengroup.osdu.indexer.aws.model.RecordMetadata; -import org.opengroup.osdu.indexer.aws.model.RecordMetadataDoc; -import org.opengroup.osdu.is.core.logging.JaxRsDpsLog; -import org.opengroup.osdu.is.core.util.AppException; - -import javax.inject.Inject; -import java.io.IOException; - -public class RecordMetadataTypeConverter implements DynamoDBTypeConverter<String, RecordMetadata> { - - @Inject - private JaxRsDpsLog logger; - - @Override - // Converts RecordMetadata to a JSON string for DynamoDB - public String convert(RecordMetadata recordMetadata) { - ObjectMapper objectMapper = new ObjectMapper(); - String convertedString; - try { - convertedString = objectMapper.writeValueAsString(recordMetadata); - } catch (JsonProcessingException e) { - throw new AppException(e.hashCode(), e.getMessage(), e.getOriginalMessage(), e); - } - return convertedString; - } - - @Override - // Converts a JSON string of an array of RecordMetadata to a RecordMetadata object - public RecordMetadata unconvert(String recordMetadataString) { - ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - RecordMetadata metadata; - try { - metadata = objectMapper.readValue(recordMetadataString, new TypeReference<RecordMetadata>(){}); - } catch (JsonParseException e) { - throw new AppException(e.hashCode(), e.getMessage(), e.getOriginalMessage(), e); - } catch (JsonMappingException e) { - throw new AppException(e.hashCode(), e.getMessage(), e.getOriginalMessage(), e); - } catch (IOException e) { - throw new AppException(e.hashCode(), e.getMessage(), e.getLocalizedMessage(), e); - } catch (Exception e) { - throw new AppException(e.hashCode(), e.getMessage(), e.getLocalizedMessage(), e); - } - return metadata; - } -} \ No newline at end of file diff --git a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/schema/SchemaExtTypeConverter.java b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/schema/SchemaExtTypeConverter.java deleted file mode 100644 index f62829f28f1b9d8f2b1305df15952008e3593907..0000000000000000000000000000000000000000 --- a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/schema/SchemaExtTypeConverter.java +++ /dev/null @@ -1,49 +0,0 @@ -package org.opengroup.osdu.indexer.aws.util.dynamodb.schema; - -import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBTypeConverter; -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.opengroup.osdu.is.core.logging.JaxRsDpsLog; -import org.opengroup.osdu.is.core.util.AppException; - -import javax.inject.Inject; -import java.io.IOException; -import java.util.Map; - -public class SchemaExtTypeConverter implements DynamoDBTypeConverter<String, Map<String, Object>> { - @Inject - private JaxRsDpsLog logger; - - @Override - public String convert(Map<String, Object> stringObjectMap) { - ObjectMapper objectMapper = new ObjectMapper(); - String convertedString = null; - try { - convertedString = objectMapper.writeValueAsString(stringObjectMap); - } catch (JsonProcessingException e) { - throw new AppException(e.hashCode(), e.getMessage(), e.getOriginalMessage(), e); - } - return convertedString; - } - - @Override - public Map<String, Object> unconvert(String s) { - ObjectMapper objectMapper = new ObjectMapper(); - Map<String, Object> unconvertedMap; - try { - unconvertedMap = objectMapper.readValue(s, new TypeReference<Map<String, Object>>(){}); - } catch (JsonParseException e) { - throw new AppException(e.hashCode(), e.getMessage(), e.getOriginalMessage(), e); - } catch (JsonMappingException e) { - throw new AppException(e.hashCode(), e.getMessage(), e.getOriginalMessage(), e); - } catch (IOException e) { - throw new AppException(e.hashCode(), e.getMessage(), e.getLocalizedMessage(), e); - } catch (Exception e) { - throw new AppException(e.hashCode(), e.getMessage(), e.getLocalizedMessage(), e); - } - return unconvertedMap; - } -} diff --git a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/schema/SchemaItemTypeConverter.java b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/schema/SchemaItemTypeConverter.java deleted file mode 100644 index c4ec5827548506b67c11aacb19962048ecceecb8..0000000000000000000000000000000000000000 --- a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/dynamodb/schema/SchemaItemTypeConverter.java +++ /dev/null @@ -1,51 +0,0 @@ -package org.opengroup.osdu.indexer.aws.util.dynamodb.schema; - -import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBTypeConverter; -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.opengroup.osdu.core.api.storage.models.SchemaNode; -import org.opengroup.osdu.indexer.aws.model.SchemaItem; -import org.opengroup.osdu.is.core.logging.JaxRsDpsLog; -import org.opengroup.osdu.is.core.util.AppException; - -import javax.inject.Inject; -import java.io.IOException; -import java.util.List; - -public class SchemaItemTypeConverter implements DynamoDBTypeConverter<String, List<SchemaItem>> { - @Inject - private JaxRsDpsLog logger; - - @Override - public String convert(List<SchemaItem> schemaNodes) { - ObjectMapper objectMapper = new ObjectMapper(); - String convertedString; - try { - convertedString = objectMapper.writeValueAsString(schemaNodes); - } catch (JsonProcessingException e) { - throw new AppException(e.hashCode(), e.getMessage(), e.getOriginalMessage(), e); - } - return convertedString; - } - - @Override - public List<SchemaItem> unconvert(String s) { - ObjectMapper objectMapper = new ObjectMapper(); - List<SchemaItem> schemaItems; - try { - schemaItems = objectMapper.readValue(s, new TypeReference<List<SchemaItem>>(){}); - } catch (JsonParseException e) { - throw new AppException(e.hashCode(), e.getMessage(), e.getOriginalMessage(), e); - } catch (JsonMappingException e) { - throw new AppException(e.hashCode(), e.getMessage(), e.getOriginalMessage(), e); - } catch (IOException e) { - throw new AppException(e.hashCode(), e.getMessage(), e.getLocalizedMessage(), e); - } catch (Exception e) { - throw new AppException(e.hashCode(), e.getMessage(), e.getLocalizedMessage(), e); - } - return schemaItems; - } -} diff --git a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/s3/S3RecordClient.java b/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/s3/S3RecordClient.java deleted file mode 100644 index ca98fcafdb6cb2aa3f727f1f62dd54ee685572ca..0000000000000000000000000000000000000000 --- a/indexer-service-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/s3/S3RecordClient.java +++ /dev/null @@ -1,43 +0,0 @@ -package org.opengroup.osdu.indexer.aws.util.s3; - -import com.amazonaws.AmazonServiceException; -import com.amazonaws.SdkClientException; -import com.amazonaws.osdu.util.s3.S3Builder; -import com.amazonaws.services.s3.AmazonS3; -import org.apache.http.HttpStatus; -import org.opengroup.osdu.indexer.aws.model.RecordMetadata; -import org.opengroup.osdu.is.core.util.AppException; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.stereotype.Component; - -import java.util.Map; -import java.util.concurrent.atomic.AtomicReference; - -@Configuration -public class S3RecordClient { - - private AmazonS3 s3; - - @Value("${aws.s3.records.bucket-name}") - private String recordsBucketName; - - - // Constructor Injection - public S3RecordClient(@Value("${aws.s3.region}") String awsS3Region, @Value("${aws.s3.endpoint}") String awsS3Endpoint){ - s3 = S3Builder.generateS3Client(awsS3Endpoint, awsS3Region); - } - - - public String getRecord(RecordMetadata recordMetadata) { - String record = ""; - String keyName = getKeyNameForLatestVersion(recordMetadata); - record = s3.getObjectAsString(recordsBucketName, keyName); - return record; - } - - private String getKeyNameForLatestVersion(RecordMetadata recordMetadata){ - return recordMetadata.getKind() + "/" + recordMetadata.getId() + "/" + recordMetadata.getLatestVersion(); - } -} diff --git a/indexer-service-aws/src/main/resources/application.properties b/indexer-service-aws/src/main/resources/application.properties index 514c4a10acf2e035c3c6bbb25d4023ac73a1a317..3607512c7bf23c52fb36ecd826c501f00526238d 100644 --- a/indexer-service-aws/src/main/resources/application.properties +++ b/indexer-service-aws/src/main/resources/application.properties @@ -8,6 +8,12 @@ DEFAULT_DATA_COUNTRY=US CRON_INDEX_CLEANUP_THRESHOLD_DAYS=3 CRON_EMPTY_INDEX_CLEANUP_THRESHOLD_DAYS=7 +## AWS DynamoDB configuration +aws.dynamodb.key=kind +aws.dynamodb.table.prefix=dev- +aws.dynamodb.region=us-east-1 +aws.dynamodb.endpoint=dynamodb.us-east-1.amazonaws.com + ## AWS S3 configuration aws.s3.records.bucket-name=dev-osdu-data aws.s3.max-record-threads=2000 @@ -20,16 +26,14 @@ aws.sns.region=us-east-1 aws.sns.arn=arn:aws:sns:us-east-1:888733619319:dev-osdu-indexer-messages aws.sns.topic-name=dev-osdu-indexer-messages +## AWS SQS Configuration +aws.sqs.queue=dev-osdu-storage-queue + #Spring Configuration spring.security.user.name=opendes@byoc.local spring.security.user.password=123 spring.security.user.roles=service.storage.admin -## AWS DynamoDB configuration -aws.dynamodb.key=kind -aws.dynamodb.table.prefix=dev- -aws.dynamodb.region=us-east-1 -aws.dynamodb.endpoint=dynamodb.us-east-1.amazonaws.com # AWS ES configuration aws.es.host=https://search-dev-osdu-indexer-i5bpf2gv4iv6ha2xi7rook2rga.us-east-1.es.amazonaws.com @@ -44,9 +48,10 @@ aws.es.serviceName=es GAE_SERVICE=indexer -STORAGE_SCHEMA_HOST=NotImplemented -STORAGE_QUERY_RECORD_HOST=NotImplemented -STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=NotImplemented +STORAGE_SCHEMA_HOST=http://localhost:8081/api/storage/v2/schemas +STORAGE_QUERY_RECORD_HOST=http://localhost:8081/api/storage/v2/query/records +STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=http://localhost:8081/api/storage/v2/query/records:batch STORAGE_RECORDS_BATCH_SIZE=20 +INDEXER_QUEUE_HOST=http://sqs.us-east-1.amazonaws.com/888733619319/dev-osdu-indexer-queue