Merge branch 'feat/aws-schema-service-int' into 'master'

Feat/aws schema service int

See merge request !70

NOTICE

@@ -28,6 +28,20 @@ The following software have components provided under the terms of this license:
 - ASM Tree (from )
 - ASM Util (from )
 - ASM based accessors helper used by json-smart (from )
+- AWS Event Stream (from https://github.com/awslabs/aws-eventstream-java)
+- AWS Java SDK :: AWS Core (from https://aws.amazon.com/sdkforjava)
+- AWS Java SDK :: Annotations (from )
+- AWS Java SDK :: Auth (from https://aws.amazon.com/sdkforjava)
+- AWS Java SDK :: Core :: Protocols :: AWS Json Protocol (from https://aws.amazon.com/sdkforjava)
+- AWS Java SDK :: Core :: Protocols :: Protocol Core (from https://aws.amazon.com/sdkforjava)
+- AWS Java SDK :: HTTP Client Interface (from )
+- AWS Java SDK :: HTTP Clients :: Apache (from )
+- AWS Java SDK :: HTTP Clients :: Netty Non-Blocking I/O (from )
+- AWS Java SDK :: Profiles (from https://aws.amazon.com/sdkforjava)
+- AWS Java SDK :: Regions (from )
+- AWS Java SDK :: SDK Core (from https://aws.amazon.com/sdkforjava)
+- AWS Java SDK :: Services :: AWS Simple Systems Management (SSM) (from https://aws.amazon.com/sdkforjava)
+- AWS Java SDK :: Utilities (from )
 - AWS Java SDK for AWS AppSync (from https://aws.amazon.com/sdkforjava)
 - AWS Java SDK for AWS Application Auto Scaling (from https://aws.amazon.com/sdkforjava)
 - AWS Java SDK for AWS Application Discovery Service (from https://aws.amazon.com/sdkforjava)
@@ -353,10 +367,11 @@ The following software have components provided under the terms of this license:
 - Microsoft Azure Netty HTTP Client Library (from https://github.com/Azure/azure-sdk-for-java)
 - Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java)
 - Mockito (from http://mockito.org)
+- Mockito (from http://www.mockito.org)
 - Mockito (from http://mockito.org)
 - Mockito (from http://mockito.org)
-- Mockito (from http://www.mockito.org)
 - Mojo's Maven plugin for Cobertura (from http://mojo.codehaus.org/cobertura-maven-plugin/)
+- Netty Reactive Streams HTTP support (from )
 - Netty Reactive Streams Implementation (from )
 - Netty/Buffer (from http://netty.io/)
 - Netty/Codec (from )
@@ -448,6 +463,7 @@ The following software have components provided under the terms of this license:
 - aalto-xml (from )
 - aggs-matrix-stats (from https://github.com/elastic/elasticsearch)
 - aggs-matrix-stats (from https://github.com/elastic/elasticsearch)
+- aws-ssm-java-caching-client (from https://github.com/awslabs/aws-ssm-java-caching-client)
 - cli (from https://github.com/elastic/elasticsearch)
 - cli (from https://github.com/elastic/elasticsearch)
 - com.google.api.grpc:grpc-google-cloud-pubsub-v1 (from https://github.com/googleapis/googleapis)
@@ -974,6 +990,7 @@ public-domain
 ========================================================================
 The following software have components provided under the terms of this license:
 
+- AWS Java SDK :: SDK Core (from https://aws.amazon.com/sdkforjava)
 - AWS SDK for Java - Models (from https://aws.amazon.com/sdkforjava)
 - Asynchronous Http Client (from )
 - Bouncy Castle PKIX, CMS, EAC, TSP, PKCS, OCSP, CMP, and CRMF APIs (from http://www.bouncycastle.org/java.html)

provider/indexer-aws/azure-build.yml

-# Maven
-# Build your Java project and run tests with Apache Maven.
-# Add steps that analyze code, save build artifacts, deploy, and more:
-# https://docs.microsoft.com/azure/devops/pipelines/languages/java
-trigger:
-  branches:
-    include:
-      - master
-  paths:
-    exclude:
-      - README.md
-      - .gitignore
-pool:
-  name: dps-build
-  demands: maven
-steps:
-  - task: Maven@3
-    displayName: 'build, test, code coverage'
-    inputs:
-      mavenPomFile: 'pom.xml'
-      options: '--settings ./indexer-core/maven/settings.xml -DVSTS_FEED_TOKEN=$(VSTS_FEED_TOKEN) -U'
-      testResultsFiles: '**/*/TEST-*.xml'
-      codeCoverageToolOption: JaCoCo
-      goals: 'install'
-  - task: Maven@3
-    displayName: 'build, test, code coverage'
-    inputs:
-      mavenPomFile: 'pom.xml'
-      options: '--settings ./provider/indexer-aws/maven/settings.xml -DVSTS_FEED_TOKEN=$(VSTS_FEED_TOKEN) -U -P indexer-aws'
-      testResultsFiles: '**/*/TEST-*.xml'
-      codeCoverageToolOption: JaCoCo
-      goals: 'install'
-  - task: CopyFiles@2
-    displayName: 'Copy AWS artifacts for maven deploy to: $(build.artifactstagingdirectory)'
-    inputs:
-      SourceFolder:
-      Contents: |
-        provider/indexer-aws/CloudFormation
-        provider/indexer-aws/buildspec-post-deploy.yml
-        provider/indexer-aws/buildspec-pre-deploy.yml
-        provider/indexer-aws/target/*-spring-boot.jar
-      TargetFolder: '$(build.artifactstagingdirectory)'
-
-  - task: PublishBuildArtifacts@1
-    displayName: 'Publish Artifact: drop'
-    inputs:
-      PathtoPublish: '$(build.artifactstagingdirectory)'
-      ArtifactName: 'drop'
-      publishLocation: 'Container'
-    condition: succeededOrFailed()

provider/indexer-aws/build-aws/buildspec.yaml

@@ -57,7 +57,7 @@ phases:
       - printenv
 
       - echo "Building primary service assemblies..."
-      - mvn -B test install -pl indexer-core,provider/indexer-aws -Ddeployment.environment=prod
+      - mvn -ntp -B test install -pl indexer-core,provider/indexer-aws -Ddeployment.environment=prod
 
 
 #      - echo "Copying assemblies to dist..."
@@ -95,4 +95,4 @@ artifacts:
   name: ${REPO_NAME}_${BRANCH_NAME}_$(date +%F)_${CODEBUILD_BUILD_NUMBER}.zip
 cache:
   paths:
-    - "/root/.m2/**/*"
\ No newline at end of file
+    - "/root/.m2/**/*"

provider/indexer-aws/pom.xml

@@ -49,7 +49,7 @@
     <dependency>
         <groupId>org.opengroup.osdu.core.aws</groupId>
         <artifactId>os-core-lib-aws</artifactId>
-        <version>0.3.7</version>
+        <version>0.3.16</version>
     </dependency>
 
     <!-- AWS managed packages -->

provider/indexer-aws/src/main/resources/application.properties

@@ -22,7 +22,8 @@ aws.es.serviceName=es
 
 GAE_SERVICE=indexer
 
-SCHEMA_HOST=${HOST}/api/schema-service/v1/schema
+#reusing STORAGE_HOST variable here as the base url to point to schema service
+SCHEMA_HOST=${STORAGE_HOST}/api/schema-service/v1/schema
 
 STORAGE_SCHEMA_HOST=${STORAGE_HOST}/api/storage/v2/schemas
 STORAGE_QUERY_RECORD_HOST=${STORAGE_HOST}/api/storage/v2/query/records
@@ -41,7 +42,7 @@ aws.elasticache.cluster.schema.expiration=60
 MAX_CACHE_VALUE_SIZE=1000
 
 ## AWS Lambda configuration
-aws.lambda.get-groups-function-name=${ENVIRONMENT}-os-entitlements-GroupsFunction
+#aws.lambda.get-groups-function-name=${ENVIRONMENT}-os-entitlements-GroupsFunction
 
 
 ## Default DynamoDB Settings

testing/indexer-test-aws/build-aws/prepare-dist.sh

@@ -21,13 +21,13 @@ echo $INTEGRATION_TEST_OUTPUT_BIN_DIR
 rm -rf "$INTEGRATION_TEST_OUTPUT_DIR"
 mkdir -p "$INTEGRATION_TEST_OUTPUT_DIR" && mkdir -p "$INTEGRATION_TEST_OUTPUT_BIN_DIR"
 echo "Building integration testing assemblies and gathering artifacts..."
-mvn install -f "$INTEGRATION_TEST_SOURCE_DIR_CORE"/pom.xml
-mvn install dependency:copy-dependencies -DskipTests -f "$INTEGRATION_TEST_SOURCE_DIR_AWS"/pom.xml -DincludeGroupIds=org.opengroup.osdu -Dmdep.copyPom
+mvn -ntp install -f "$INTEGRATION_TEST_SOURCE_DIR_CORE"/pom.xml
+mvn -ntp install dependency:copy-dependencies -DskipTests -f "$INTEGRATION_TEST_SOURCE_DIR_AWS"/pom.xml -DincludeGroupIds=org.opengroup.osdu -Dmdep.copyPom
 cp "$INTEGRATION_TEST_SOURCE_DIR_AWS"/target/dependency/* "${INTEGRATION_TEST_OUTPUT_BIN_DIR}"
-(cd "${INTEGRATION_TEST_OUTPUT_BIN_DIR}" && ls *.jar | sed -e 's/\.jar$//' | xargs -I {} echo mvn install:install-file -Dfile={}.jar -DpomFile={}.pom >> install-deps.sh)
+(cd "${INTEGRATION_TEST_OUTPUT_BIN_DIR}" && ls *.jar | sed -e 's/\.jar$//' | xargs -I {} echo mvn -ntp install:install-file -Dfile={}.jar -DpomFile={}.pom >> install-deps.sh)
 chmod +x "${INTEGRATION_TEST_OUTPUT_BIN_DIR}"/install-deps.sh
-mvn clean -f "$INTEGRATION_TEST_SOURCE_DIR_AWS"/pom.xml
+mvn -ntp clean -f "$INTEGRATION_TEST_SOURCE_DIR_AWS"/pom.xml
 cp -R "$INTEGRATION_TEST_SOURCE_DIR_AWS"/* "${INTEGRATION_TEST_OUTPUT_DIR}"/
 
 #copy testing parent pom to output
-cp "$INTEGRATION_TEST_SOURCE_DIR/pom.xml" "${OUTPUT_DIR}/testing"
\ No newline at end of file
+cp "$INTEGRATION_TEST_SOURCE_DIR/pom.xml" "${OUTPUT_DIR}/testing"

testing/indexer-test-aws/build-aws/run-tests.sh

@@ -27,10 +27,11 @@ export DEFAULT_DATA_PARTITION_ID_TENANT2=common
 export ENTITLEMENTS_DOMAIN=testing.com
 export OTHER_RELEVANT_DATA_COUNTRIES=US
 export STORAGE_HOST=$STORAGE_URL
+export HOST=$SCHEMA_URL
 
 #### RUN INTEGRATION TEST #########################################################################
 
-mvn test -f "$SCRIPT_SOURCE_DIR"/../pom.xml -Dcucumber.options="--plugin junit:target/junit-report.xml"
+mvn -ntp test -f "$SCRIPT_SOURCE_DIR"/../pom.xml -Dcucumber.options="--plugin junit:target/junit-report.xml"
 TEST_EXIT_CODE=$?
 
 #### COPY TEST REPORTS #########################################################################
@@ -39,6 +40,7 @@ if [ -n "$1" ]
   then
     mkdir -p "$1"
     cp "$SCRIPT_SOURCE_DIR"/../target/junit-report.xml "$1"/os-indexer-junit-report.xml
+    cp  -R "$SCRIPT_SOURCE_DIR"/../target/surefire-reports "$1"
 fi
 
 exit $TEST_EXIT_CODE

testing/indexer-test-aws/pom.xml

@@ -51,7 +51,7 @@
         <dependency>
             <groupId>org.opengroup.osdu.core.aws</groupId>
             <artifactId>os-core-lib-aws</artifactId>
-            <version>0.3.7</version>
+            <version>0.3.16</version>
         </dependency>
 
         <!-- Testing -->
@@ -138,4 +138,4 @@
         </dependency>
 
     </dependencies>
-</project>
\ No newline at end of file
+</project>

testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java

@@ -20,7 +20,7 @@ import org.junit.runner.RunWith;
 
 @RunWith(Cucumber.class)
 @CucumberOptions(
-        features = "classpath:features/indexrecord/IndexRecord.feature",
+        features = "classpath:features/indexrecord/indexRecord-schema-service.feature",
         glue = {"classpath:org.opengroup.osdu.step_definitions/index/record"},
         plugin = {"pretty", "junit:target/cucumber-reports/TEST-indexrecord.xml"})
 public class RunTest {

testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java

@@ -15,7 +15,6 @@
 package org.opengroup.osdu.step_definitions.index.record;
 
 import lombok.extern.java.Log;
-import org.opengroup.osdu.common.RecordSteps;
 import org.opengroup.osdu.core.common.model.legal.Legal;
 import org.opengroup.osdu.util.AWSHTTPClient;
 
@@ -27,6 +26,7 @@ import cucumber.api.java.en.Then;
 import cucumber.api.java.en.When;
 import org.opengroup.osdu.util.ElasticUtilsAws;
 import org.opengroup.osdu.util.LegalTagUtilsAws;
+import org.opengroup.osdu.common.SchemaServiceRecordSteps;
 
 import java.util.HashSet;
 import java.util.Set;
@@ -34,7 +34,7 @@ import java.util.Set;
 import static org.opengroup.osdu.util.Config.getOtherRelevantDataCountries;
 
 @Log
-public class Steps extends RecordSteps {
+public class Steps extends SchemaServiceRecordSteps {
 
     protected LegalTagUtilsAws legalTagUtils;
     private String legalTagName;