Merge branch 'aws-sync-dev-to-master-Aug18' into 'master'

AWS sync dev to master

See merge request !417

.gitlab-ci.yml

@@ -39,12 +39,10 @@ include:
     file: "cloud-providers/gc-global.yml"
 
   - project: "osdu/platform/ci-cd-pipelines"
-    file: "cloud-providers/aws-global.yml"
+    file: "cloud-providers/ibm.yml"
 
-  - project: "osdu/platform/ci-cd-pipelines"
-    file: "cloud-providers/aws-maven.yml"
+  # aws
+  - local: "devops/aws/pipeline/override-stages.yml"
 
-  - project: "osdu/platform/ci-cd-pipelines"
-    file: "cloud-providers/ibm.yml"
+  - local: "devops/gc/pipeline/override-stages.yml"
 
-  - local: "devops/gc/pipeline/override-stages.yml"
\ No newline at end of file

devops/aws/pipeline/override-stages.yml

+include:
+  - project: "osdu/platform/ci-cd-pipelines"
+    file: "cloud-providers/aws-global.yml"
+  - project: "osdu/platform/ci-cd-pipelines"
+    file: "cloud-providers/aws-global-variables.yml"
+  - project: "osdu/platform/ci-cd-pipelines"
+    file: "cloud-providers/aws-global.yml"
+  - project: "osdu/platform/ci-cd-pipelines"
+    file: "cloud-providers/aws-maven.yml"
+
+aws-test-java:
+  script:
+    - export KUBECONFIG=/tmp/kubeconfig-${RANDOM}.yaml
+    - echo "ROLE $EKS_CLUSTER_MGMT_ROLE"
+    - echo "CLUSTER $EKS_CLUSTER_NAME"
+    - aws eks update-kubeconfig --region $AWS_REGION --name $EKS_CLUSTER_NAME --role-arn $EKS_CLUSTER_MGMT_ROLE
+    - chmod 600 $KUBECONFIG
+
+    - $MAVEN_BUILD $INTEGRATION_TEST_DIR maven-aws-integration-test-output.log ${AWS_MAVEN_TEST_COMMAND_OVERRIDE:-test} --update-snapshots -DdisableXmlReport=true
+

provider/notification-aws/pom.xml

@@ -76,7 +76,7 @@
         <dependency>
             <groupId>org.opengroup.osdu.core.aws</groupId>
             <artifactId>os-core-lib-aws</artifactId>
-            <version>0.21.0</version>
+            <version>0.23.0-rc2</version>
         </dependency>
         <dependency>
             <groupId>commons-io</groupId>
@@ -194,6 +194,29 @@
                     </execution>
                 </executions>
             </plugin>
+            <plugin>
+                <groupId>org.jacoco</groupId>
+                <artifactId>jacoco-maven-plugin</artifactId>
+                <version>0.8.10</version>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>prepare-agent</goal>
+                        </goals>
+                        <configuration>
+                            <!-- Sets the VM argument line used when unit tests are run. -->
+                            <propertyName>jacocoArgLine</propertyName>
+	                    </configuration>
+                    </execution>
+                    <execution>
+                        <id>report</id>
+                        <phase>test</phase>
+                        <goals>
+                            <goal>report</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
         </plugins>
     </build>
 </project>
\ No newline at end of file

testing/notification-test-aws/pom.xml

@@ -44,7 +44,7 @@
         <dependency>
             <groupId>org.opengroup.osdu.core.aws</groupId>
             <artifactId>os-core-lib-aws</artifactId>
-            <version>0.21.0</version>
+            <version>0.23.0-rc2</version>
         </dependency>
         <dependency>
             <groupId>com.amazonaws</groupId>
@@ -59,7 +59,7 @@
         <dependency>
             <groupId>org.opengroup.osdu</groupId>
             <artifactId>os-core-common</artifactId>
-            <version>0.21.0</version>
+            <version>0.22.0-rc4</version>
         </dependency>
 
         <dependency>

testing/notification-test-aws/src/test/java/org/opengroup/osdu/notification/subscriptions/LegalTagUtils.java

@@ -34,9 +34,6 @@ public class LegalTagUtils {
 
     protected static String getLegalUrl() {
         String legalUrl = System.getProperty("LEGAL_URL", System.getenv("LEGAL_URL"));
-        if (legalUrl == null || legalUrl.contains("-null")) {
-            legalUrl = "https://os-legal-dot-opendes.appspot.com/api/legal/v1/";
-        }
         return legalUrl;
     }
 

testing/notification-test-aws/src/test/java/org/opengroup/osdu/notification/subscriptions/RecordUtils.java

@@ -76,18 +76,8 @@ public class RecordUtils {
         return record;
     }
 
-    public static final String getAclSuffix(boolean isTestPartition) {
-        String environment = getEnvironment();
-        //build.gradle currently throws exception if a variable is set to empty or not set at all
-        //workaround by setting it to an "empty" string to construct the url
-        if (environment.equalsIgnoreCase("empty")) environment = "";
-        if (!environment.isEmpty())
-            environment = "." + environment;
-
-        return String.format("%s%s.%s", TenantUtils.getTenantName(isTestPartition), environment, domain);
+    public static String getAclSuffix(boolean isTestPartition) {
+        return String.format("%s.%s", TenantUtils.getTenantName(isTestPartition), domain);
     }
 
-    public static String getEnvironment() {
-        return System.getProperty("DEPLOY_ENV", System.getenv("DEPLOY_ENV"));
-    }
 }

testing/notification-test-aws/src/test/java/org/opengroup/osdu/notification/subscriptions/StorageTestUtils.java

@@ -54,9 +54,6 @@ public class StorageTestUtils {
 
     public static String getApiPath(String api) throws Exception {
         String baseUrl = System.getProperty("STORAGE_URL", System.getenv("STORAGE_URL"));
-        if (baseUrl == null || baseUrl.contains("-null")) {
-            baseUrl = "https://localhost:8443/api/storage/v2/";
-        }
         URL mergedURL = new URL(baseUrl + api);
         System.out.println(mergedURL.toString());
         return mergedURL.toString();

testing/notification-test-aws/src/test/java/org/opengroup/osdu/notification/subscriptions/TestNotificationsEndpoint.java

@@ -14,6 +14,7 @@ import org.opengroup.osdu.core.common.notification.SubscriptionException;
 import org.opengroup.osdu.core.common.notification.SubscriptionFactory;
 import org.opengroup.osdu.notification.util.*;
 
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.UUID;
@@ -22,20 +23,18 @@ import static org.junit.Assert.assertEquals;
 
 public class TestNotificationsEndpoint extends TestBase {
 
-    private String subscriptionId = null;
+    private String subscriptionId_TestPartition;
     private ISubscriptionService awssubscriptionService;
     private TestUtils testUtils;
     private static SubscriptionFactory awsfactory;
 
     private static final long NOW = System.currentTimeMillis();
-    private static final String RECORD_ID_PREFIX = TenantUtils.getTenantName(false) + ":query:";
     private static final String RECORD_ID_PREFIX_TEST = TenantUtils.getTenantName(true) + ":query:";
-    private static final String KIND = TenantUtils.getTenantName(false) + ":ds:query:1.0." + NOW;
     private static final String KIND_TEST = TenantUtils.getTenantName(true) + ":ds:query:1.0." + NOW;
     private static final String LEGAL_TAG = TenantUtils.getTenantName(false) + "-storage-" + System.currentTimeMillis();
     private static final String LEGAL_TAG_TEST = TenantUtils.getTenantName(true) + "-storage-" + System.currentTimeMillis();
 
-    protected static final String PARTITION_TEST = "performance-test";
+    protected static final String PARTITION_TEST = "opendes";
 
     protected static final String PERSISTABLE_REFERENCE = "%7B%22LB_CRS%22%3A%22%257B%2522WKT%2522%253A%2522PROJCS%255B%255C%2522British_National_Grid%255C%2522%252CGEOGCS%255B%255C%2522GCS_OSGB_1936%255C%2522%252CDATUM%255B%255C%2522D_OSGB_1936%255C%2522%252CSPHEROID%255B%255C%2522Airy_1830%255C%2522%252C6377563.396%252C299.3249646%255D%255D%252CPRIMEM%255B%255C%2522Greenwich%255C%2522%252C0.0%255D%252CUNIT%255B%255C%2522Degree%255C%2522%252C0.0174532925199433%255D%255D%252CPROJECTION%255B%255C%2522Transverse_Mercator%255C%2522%255D%252CPARAMETER%255B%255C%2522False_Easting%255C%2522%252C400000.0%255D%252CPARAMETER%255B%255C%2522False_Northing%255C%2522%252C-100000.0%255D%252CPARAMETER%255B%255C%2522Central_Meridian%255C%2522%252C-2.0%255D%252CPARAMETER%255B%255C%2522Scale_Factor%255C%2522%252C0.9996012717%255D%252CPARAMETER%255B%255C%2522Latitude_Of_Origin%255C%2522%252C49.0%255D%252CUNIT%255B%255C%2522Meter%255C%2522%252C1.0%255D%252CAUTHORITY%255B%255C%2522EPSG%255C%2522%252C27700%255D%255D%2522%252C%2522Type%2522%253A%2522LBCRS%2522%252C%2522EngineVersion%2522%253A%2522PE_10_3_1%2522%252C%2522AuthorityCode%2522%253A%257B%2522Authority%2522%253A%2522EPSG%2522%252C%2522Code%2522%253A%252227700%2522%257D%252C%2522Name%2522%253A%2522British_National_Grid%2522%257D%22%2C%22TRF%22%3A%22%257B%2522WKT%2522%253A%2522GEOGTRAN%255B%255C%2522OSGB_1936_To_WGS_1984_Petroleum%255C%2522%252CGEOGCS%255B%255C%2522GCS_OSGB_1936%255C%2522%252CDATUM%255B%255C%2522D_OSGB_1936%255C%2522%252CSPHEROID%255B%255C%2522Airy_1830%255C%2522%252C6377563.396%252C299.3249646%255D%255D%252CPRIMEM%255B%255C%2522Greenwich%255C%2522%252C0.0%255D%252CUNIT%255B%255C%2522Degree%255C%2522%252C0.0174532925199433%255D%255D%252CGEOGCS%255B%255C%2522GCS_WGS_1984%255C%2522%252CDATUM%255B%255C%2522D_WGS_1984%255C%2522%252CSPHEROID%255B%255C%2522WGS_1984%255C%2522%252C6378137.0%252C298.257223563%255D%255D%252CPRIMEM%255B%255C%2522Greenwich%255C%2522%252C0.0%255D%252CUNIT%255B%255C%2522Degree%255C%2522%252C0.0174532925199433%255D%255D%252CMETHOD%255B%255C%2522Position_Vector%255C%2522%255D%252CPARAMETER%255B%255C%2522X_Axis_Translation%255C%2522%252C446.448%255D%252CPARAMETER%255B%255C%2522Y_Axis_Translation%255C%2522%252C-125.157%255D%252CPARAMETER%255B%255C%2522Z_Axis_Translation%255C%2522%252C542.06%255D%252CPARAMETER%255B%255C%2522X_Axis_Rotation%255C%2522%252C0.15%255D%252CPARAMETER%255B%255C%2522Y_Axis_Rotation%255C%2522%252C0.247%255D%252CPARAMETER%255B%255C%2522Z_Axis_Rotation%255C%2522%252C0.842%255D%252CPARAMETER%255B%255C%2522Scale_Difference%255C%2522%252C-20.489%255D%252CAUTHORITY%255B%255C%2522EPSG%255C%2522%252C1314%255D%255D%2522%252C%2522Type%2522%253A%2522STRF%2522%252C%2522EngineVersion%2522%253A%2522PE_10_3_1%2522%252C%2522AuthorityCode%2522%253A%257B%2522Authority%2522%253A%2522EPSG%2522%252C%2522Code%2522%253A%25221314%2522%257D%252C%2522Name%2522%253A%2522OSGB_1936_To_WGS_1984_Petroleum%2522%257D%22%2C%22Type%22%3A%22EBCRS%22%2C%22EngineVersion%22%3A%22PE_10_3_1%22%2C%22Name%22%3A%22OSGB+1936+*+UKOOA-Pet+%2F+British+National+Grid+%5B27700%2C1314%5D%22%2C%22AuthorityCode%22%3A%7B%22Authority%22%3A%22MyCompany%22%2C%22Code%22%3A%2227700006%22%7D%7D";
     @BeforeClass
@@ -45,18 +44,19 @@ public class TestNotificationsEndpoint extends TestBase {
     }
 
     @Before
-    public void setup() throws Exception {
+    public void setup() {
         this.testUtils = new AwsTestUtils();
     }
 
     @After
     @Override
     public void tearDown() throws Exception {
-        //LegalTagUtils.delete(LEGAL_TAG, testUtils.getOpsToken());
+        LegalTagUtils.delete(LEGAL_TAG_TEST, testUtils.getOpsToken(), true);
+        LegalTagUtils.delete(LEGAL_TAG, testUtils.getOpsToken(), false);
         this.testUtils = null;
     }
 
-    private void createResource() throws Exception {
+    private void createResourceForTestParition() throws Exception {
         Map<String, String> headers = new HashMap<>();
         headers.put(DpsHeaders.DATA_PARTITION_ID, TestUtils.getOsduTenant());
         headers.put(DpsHeaders.AUTHORIZATION, testUtils.getOpsToken());
@@ -72,7 +72,7 @@ public class TestNotificationsEndpoint extends TestBase {
         //Create a new subscription to pub/sub
         Subscription subscription = new Subscription();
         subscription.setName("Subscription-test-for-notification");
-        subscription.setDescription("Subscription test for fetching notifications");
+        subscription.setDescription("Subscription with test Partition for fetching notifications");
         subscription.setTopic(Config.Instance().Topic);
         //This seems to be a bug. Don't need to add the string
         //subscription.setPushEndpoint(Config.Instance().HMACPushUrl + "hmac-integration-test");
@@ -83,9 +83,9 @@ public class TestNotificationsEndpoint extends TestBase {
         subscription.setSecret(secret);
         try {
             Subscription subscriptionCreated = awssubscriptionService.create(subscription);
-
+            System.out.println("Subscription created successfully");
             String notificationId = subscriptionCreated.getNotificationId();
-            subscriptionId = subscriptionCreated.getId();
+            subscriptionId_TestPartition = subscriptionCreated.getId();
             Config.Instance().NotificationId = notificationId;
         }catch (SubscriptionException e){
             System.out.println("Subscription exception inner response : " + e.getHttpResponse());
@@ -93,12 +93,6 @@ public class TestNotificationsEndpoint extends TestBase {
         }
     }
 
-    private ClientResponse createStorageRecord(final String correlationId) throws Exception {
-        String recordId = RECORD_ID_PREFIX + UUID.randomUUID().toString();
-        String jsonInput = RecordUtils.createJsonRecordWithReference(1, recordId, KIND, LEGAL_TAG, PERSISTABLE_REFERENCE, "CRS", false);
-        return StorageTestUtils.send("records", "PUT", StorageTestUtils.getHeaders(TenantUtils.getTenantName(false), testUtils.getAdminToken(), correlationId, false), jsonInput, "");
-    }
-
     private ClientResponse createStorageRecordForTestPartition(final String correlationId) throws Exception {
         String recordId = RECORD_ID_PREFIX_TEST + UUID.randomUUID().toString();
         String jsonInput = RecordUtils.createJsonRecordWithReference(1, recordId, KIND_TEST, LEGAL_TAG_TEST, PERSISTABLE_REFERENCE, "CRS", true);
@@ -106,10 +100,10 @@ public class TestNotificationsEndpoint extends TestBase {
     }
 
     @Test
-    public void testVerifyNotificationReceivedWhenDataPartitionIdIsDifferent() throws Exception {
+    public void testVerifyNotificationReceived() throws Exception {
         try {
             LegalTagUtils.create(LEGAL_TAG_TEST, testUtils.getOpsToken(), true);
-            createResource();
+            createResourceForTestParition();
             final String correlationId = UUID.randomUUID().toString();
             ClientResponse response = createStorageRecordForTestPartition(correlationId);
             assertEquals(201, response.getStatus());
@@ -118,40 +112,16 @@ public class TestNotificationsEndpoint extends TestBase {
 
             //Run Bash File to fetch logs from register endpoint and verify that notification was received
             String bashFileToExecute = "src/test/java/org/opengroup/osdu/notification/subscriptions/verify_register-logs.sh " + correlationId;
-            Process process = Runtime.getRuntime().exec(bashFileToExecute);
-            process.waitFor();
-            int exitValue = process.exitValue();
-            assertEquals(exitValue, 1);
-        } catch (Exception e) {
-            System.out.println("Error while verifying notification service" + e);
-        } finally {
-            awssubscriptionService.delete(subscriptionId);
-            LegalTagUtils.delete(LEGAL_TAG_TEST, testUtils.getOpsToken(), true);
-        }
-    }
 
-    @Test
-    public void testVerifyNotificationReceivedWhenDataPartitionIdIsSame() throws Exception {
-        try {
-            LegalTagUtils.create(LEGAL_TAG, testUtils.getOpsToken(), false);
-            createResource();
-            final String correlationId = UUID.randomUUID().toString();
-            ClientResponse response = createStorageRecord(correlationId);
-            assertEquals(201, response.getStatus());
-            //Executing notifications response to endpoints takes an upper bound of 120s.
-            Thread.sleep(120000);
-
-            //Run Bash File to fetch logs from register endpoint and verify that notification was received
-            String bashFileToExecute = "src/test/java/org/opengroup/osdu/notification/subscriptions/verify_register-logs.sh " + correlationId;
             Process process = Runtime.getRuntime().exec(bashFileToExecute);
             process.waitFor();
+
             int exitValue = process.exitValue();
             assertEquals(exitValue, 0);
         } catch (Exception e) {
-            System.out.println("Error while verifying notification service" + e);
+            throw e;
         } finally {
-            awssubscriptionService.delete(subscriptionId);
-            LegalTagUtils.delete(LEGAL_TAG_TEST, testUtils.getOpsToken(), false);
+            awssubscriptionService.delete(subscriptionId_TestPartition);
         }
     }
 }

testing/notification-test-aws/src/test/java/org/opengroup/osdu/notification/subscriptions/verify_register-logs.sh

-register_pod=$(kubectl get pods --all-namespaces -o=jsonpath='{range .items..metadata}{.name}{"\n"}{end}' | fgrep os-register-)
-register_logs=$(kubectl logs $register_pod -n osdu-instance-$OSDU_INSTANCE_NAME-core --since=2m)
+CORE_K8S_NAMESPACE="$AWS_SERVICE_NAMESPACE-core"
+register_pod=$(kubectl get pods -n $CORE_K8S_NAMESPACE | grep "Running" | grep "os-register" | awk '{ print $1 }')
+register_logs=$(kubectl logs $register_pod -n $CORE_K8S_NAMESPACE --since=2m)
 
 if echo $register_logs | grep $1; then
-  return 0
+  exit 0
 else
-  return 1
-fi
+  exit 1
+fi
\ No newline at end of file