Merge branch 'aws-integration' into 'master'

AWS Updates

See merge request !91

provider/indexer-aws/build-aws/Dockerfile

@@ -16,7 +16,14 @@
 FROM amazoncorretto:8
 
 ARG JAR_FILE=provider/indexer-aws/target/*spring-boot.jar
+
+#Default to using self signed generated TLS cert
+ENV USE_SELF_SIGNED_SSL_CERT true
+
 WORKDIR /
 COPY ${JAR_FILE} app.jar
+COPY /provider/indexer-aws/build-aws/ssl.sh /ssl.sh
+COPY /provider/indexer-aws/build-aws/entrypoint.sh /entrypoint.sh
 EXPOSE 8080
-ENTRYPOINT java $JAVA_OPTS -jar /app.jar
\ No newline at end of file
+
+ENTRYPOINT ["/bin/sh", "-c", ". /entrypoint.sh"]
\ No newline at end of file

provider/indexer-aws/build-aws/buildspec.yaml

@@ -27,6 +27,8 @@ phases:
     runtime-versions:
       java: corretto8
     commands:
+      # fix error noted here: https://github.com/yarnpkg/yarn/issues/7866
+      - curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
       - if [ $(echo $CODEBUILD_SOURCE_VERSION | grep -c  ^refs/heads.*) -eq 1 ]; then echo "Branch name found"; else echo "This build only supports branch builds" && exit 1; fi
       - apt-get update -y
       - apt-get install -y maven

provider/indexer-aws/build-aws/entrypoint.sh

+
+
+if [ -n $USE_SELF_SIGNED_SSL_CERT ];
+then    
+    export SSL_KEY_PASSWORD=$RANDOM$RANDOM$RANDOM;
+    export SSL_KEY_STORE_PASSWORD=$SSL_KEY_PASSWORD;
+    export SSL_KEY_STORE_DIR=/tmp/certs;
+    export SSL_KEY_STORE_NAME=osduonaws.p12;
+    export SSL_KEY_STORE_PATH=$SSL_KEY_STORE_DIR/$SSL_KEY_STORE_NAME;
+    export SSL_KEY_ALIAS=osduonaws;
+    
+    ./ssl.sh;
+fi
+
+java $JAVA_OPTS -jar /app.jar
\ No newline at end of file

provider/indexer-aws/build-aws/ssl.sh

+# Copyright © 2021 Amazon Web Services
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/usr/bin/env bash
+
+#Future: Support for using Amazon Cert Manager
+# if [ "$1" == "webserver" ] && [ -n $ACM_CERTIFICATE_ARN ];
+# then
+
+#   aws acm export-certificate --certificate-arn $ACM_CERTIFICATE_ARN --passphrase $(echo -n 'aws123' | openssl base64 -e) | jq -r '"\(.PrivateKey)"' > ${SSL_KEY_PATH}.enc
+#   openssl rsa -in ${SSL_KEY_PATH}.enc -out $SSL_KEY_PATH -passin pass:aws123
+#   aws acm get-certificate --certificate-arn $ACM_CERTIFICATE_ARN | jq -r '"\(.CertificateChain)"' > $SSL_CERT_PATH
+#   aws acm get-certificate --certificate-arn $ACM_CERTIFICATE_ARN | jq -r '"\(.Certificate)"' >> $SSL_CERT_PATH
+
+# fi
+
+if [ -n $USE_SELF_SIGNED_SSL_CERT ];
+then
+    mkdir -p $SSL_KEY_STORE_DIR
+    pushd $SSL_KEY_STORE_DIR
+    keytool -genkeypair -alias $SSL_KEY_ALIAS -keyalg RSA -keysize 2048 -storetype PKCS12 -keystore $SSL_KEY_STORE_NAME -validity 3650 -keypass $SSL_KEY_PASSWORD -storepass $SSL_KEY_PASSWORD -dname "CN=localhost, OU=AWS, O=Energy, L=Houston, ST=TX, C=US"
+    popd
+fi

provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/IndexerAwsApplication.java

@@ -14,15 +14,22 @@
 
 package org.opengroup.osdu.indexer.aws;
 
+import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
 import org.springframework.context.annotation.ComponentScan;
-import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.FilterType;
 
 @SpringBootApplication(exclude = { SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class })
-@ComponentScan({"org.opengroup.osdu"})
+@ComponentScan(
+        basePackages = {"org.opengroup.osdu"},
+        excludeFilters={
+                @ComponentScan.Filter(
+                        type=FilterType.ASSIGNABLE_TYPE,
+                        value=IndexerQueueTaskBuilder.class)
+        })
 public class IndexerAwsApplication {
 
     public static void main(String[] args) {

provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java

@@ -31,7 +31,7 @@ import org.springframework.stereotype.Component;
 import javax.inject.Inject;
 import java.util.HashMap;
 import java.util.Map;
-
+//
 @Primary
 @Component
 public class IndexerQueueTaskBuilderAws extends IndexerQueueTaskBuilder {
@@ -43,8 +43,8 @@ public class IndexerQueueTaskBuilderAws extends IndexerQueueTaskBuilder {
 
     private ParameterStorePropertySource ssm;
 
-    private String amazonSQSQueueUrl;
-
+    private String storageQueue;
+    private String dlq;
     private final String retryString = "retry";
 
     private Gson gson;
@@ -54,7 +54,8 @@ public class IndexerQueueTaskBuilderAws extends IndexerQueueTaskBuilder {
 
     @Value("${aws.storage.sqs.queue.url}")
     String sqsStorageQueueParameter;
-
+    @Value("${aws.indexer.sqs.dlq.url}")
+    String deadLetterQueueParameter;
 
     @Inject
     public void init() {
@@ -63,17 +64,26 @@ public class IndexerQueueTaskBuilderAws extends IndexerQueueTaskBuilder {
         gson =new Gson();
         SSMConfig ssmConfig = new SSMConfig();
         ssm = ssmConfig.amazonSSM();
-        amazonSQSQueueUrl = ssm.getProperty(sqsStorageQueueParameter).toString();
+        storageQueue = ssm.getProperty(sqsStorageQueueParameter).toString();
+        dlq = ssm.getProperty(deadLetterQueueParameter).toString();
     }
 
     @Override
     public void createWorkerTask(String payload, DpsHeaders headers) {
-        createTask(payload, headers);
+        this.createTask(payload, headers);
     }
 
+    @Override
+    public  void createWorkerTask(String payload, Long countDownMillis, DpsHeaders headers){
+        this.createTask(payload, headers);
+    }
     @Override
     public void createReIndexTask(String payload,DpsHeaders headers) {
-        createTask(payload, headers);
+        this.createTask(payload, headers);
+    }
+    @Override
+    public void createReIndexTask(String payload, Long countDownMillis, DpsHeaders headers){
+        this.createTask(payload, headers);
     }
 
     private void createTask(String payload, DpsHeaders headers) {
@@ -119,11 +129,19 @@ public class IndexerQueueTaskBuilderAws extends IndexerQueueTaskBuilder {
         );
 
         // Send a message with an attribute and a delay
-        final SendMessageRequest sendMessageRequest = new SendMessageRequest()
-                .withQueueUrl(amazonSQSQueueUrl)
-                .withMessageBody(message.getData())
-                .withDelaySeconds(new Integer(retryDelay))
-                .withMessageAttributes(messageAttributes);
+        final SendMessageRequest sendMessageRequest ;
+        if (retryCount< 10) {
+
+            sendMessageRequest = new SendMessageRequest()
+                    .withQueueUrl(storageQueue)
+                    .withMessageBody(message.getData())
+                    .withDelaySeconds(new Integer(retryDelay))
+                    .withMessageAttributes(messageAttributes);
+        }else{
+            sendMessageRequest = new SendMessageRequest()
+                    .withQueueUrl(dlq)
+                    .withMessageBody(message.getData());
+        }
         sqsClient.sendMessage(sendMessageRequest);
     }
 

provider/indexer-aws/src/main/resources/application.properties

@@ -59,3 +59,11 @@ aws.indexer.sns.topic.arn=${aws.ssm.prefix}/indexer/indexer-sns-topic-arn
 aws.storage.sns.topic.arn=${aws.ssm.prefix}/storage/storage-sns-topic-arn
 
 aws.storage.sqs.queue.url=${aws.ssm.prefix}/storage/storage-sqs-url
+aws.indexer.sqs.dlq.url=${aws.ssm.prefix}/indexer-queue/indexer-deadletter-queue-sqs-url
+
+server.ssl.enabled=${SSL_ENABLED:true}
+server.ssl.key-store-type=PKCS12
+server.ssl.key-store=${SSL_KEY_STORE_PATH:/certs/osduonaws.p12}
+server.ssl.key-alias=${SSL_KEY_ALIAS:osduonaws}
+server.ssl.key-password=${SSL_KEY_PASSWORD:}
+server.ssl.key-store-password=${SSL_KEY_STORE_PASSWORD:}
\ No newline at end of file