Adding integration job for CI/CD

.fossa.yml

@@ -20,3 +20,8 @@ analyze:
     type: mvn
     target: provider/wks-gcp/pom.xml
     path: .
+  - name: os-wks-azure
+    type: mvn
+    target: provider/wks-azure/pom.xml
+    path: .
+    

.gitlab-ci.yml

@@ -2,7 +2,6 @@ variables:
   AZURE_SERVICE: wks
   AZURE_BUILD_SUBDIR: provider/wks-azure
   AZURE_TEST_SUBDIR: testing/wks-test-core
-  AZURE_SKIP_TEST: 'true'
 
 include:
   - project: 'osdu/platform/ci-cd-pipelines'
@@ -22,5 +21,5 @@ include:
     file: 'scanners/gitlab-ultimate.yml'
 
   - project: "osdu/platform/ci-cd-pipelines"
+    ref: master
     file: "cloud-providers/azure.yml"
-

azure-pipelines.yml

@@ -115,7 +115,7 @@ stages:
         INTEGRATION_TEST_AUDIENCE: $(INTEGRATION_TEST_AUDIENCE)
         INTEGRATION_TESTER : $(INTEGRATION_TESTER)
         WKS_BUCKET_NAME : $(WKS_BUCKET_NAME)
-        STORAGE_SERVICE_URL : $(STORAGE_SERVICE_URL)
+        STORAGE_URL : $(STORAGE_URL)
         
       
     - task: PublishBuildArtifacts@1

deployments/scripts/azure/deploy_mappings.py

@@ -14,8 +14,8 @@ def upload_files():
         local_file = join(local_folder, file)
         blob_client = blob_service_client.get_blob_client(container=container_name, blob=file)
         with open(local_file, "rb") as data:
-            blob_client.upload_blob(data, blob_type="BlockBlob")
+            blob_client.upload_blob(data, blob_type="BlockBlob", overwrite=True)
 
 
 if __name__=="__main__":
-    upload_files()
\ No newline at end of file
+    upload_files()

deployments/scripts/azure/requirements.txt

-os
 azure.storage.blob
\ No newline at end of file

devops/azure/chart/templates/deployment.yaml

@@ -15,22 +15,33 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: {{ .Release.Name }}-{{ .Chart.Name }}
+  name: {{ .Release.Name }}
   namespace: osdu
 spec:
   selector:
     matchLabels:
-      app: {{ .Release.Name }}-{{ .Chart.Name }}
+      app: {{ .Chart.Name }}
   template:
     metadata:
       labels:
-        app: {{ .Release.Name }}-{{ .Chart.Name }}
+        app: {{ .Chart.Name }}
         aadpodidbinding: osdu-identity
     spec:
+      volumes:
+        - name: azure-keyvault
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: azure-keyvault
       containers:
-        - name: {{ .Release.Name }}-{{ .Chart.Name }}
+        - name: {{ .Chart.Name }}
           image: {{ .Values.image.repository }}/{{ .Chart.Name }}-{{ .Values.image.branch }}:{{ .Values.image.tag | default .Chart.AppVersion }}
           imagePullPolicy: Always
+          volumeMounts:
+            - name: azure-keyvault
+              mountPath: "/mnt/azure-keyvault"
+              readOnly: true
           env:
           - name: spring_application_name
             value: wks
@@ -82,14 +93,16 @@ spec:
                 name: servicebus
                 key: servicebus
           - name: servicebus_topic_name
-            value: recordstopic
+            value: {{ .Values.azure.servicebusTopic }}
           - name: storage_account
             valueFrom:
               configMapKeyRef:
                 name: osdu-svc-properties
                 key: ENV_STORAGE_ACCOUNT
           - name: servicebus_topic_subscription
-            value: "recordstopicsubscription"
+            value: {{ .Values.azure.servicebusSubscription }}
+          - name: storage_container
+            value: {{ .Values.azure.storageContainer }}
           - name: server_port
             value: "80"
           - name: storage_service_endpoint

devops/azure/chart/templates/scaled_object.yaml

@@ -15,19 +15,19 @@
 apiVersion: keda.k8s.io/v1alpha1
 kind: ScaledObject
 metadata:
-  name: {{ .Release.Name }}-{{ .Chart.Name }}
+  name: {{ .Chart.Name }}
   namespace: osdu
   labels:
-    deploymentName: {{ .Release.Name }}-{{ .Chart.Name }}
+    deploymentName: {{ .Chart.Name }}
 spec:
   scaleTargetRef:
-    deploymentName: {{ .Release.Name }}-{{ .Chart.Name }}
+    deploymentName: {{ .Chart.Name }}
   triggers:
   - type: azure-servicebus
     metadata:
       type: serviceBusTrigger
       direction: in
       name: message
-      subscriptionName: {{ .Values.config.azure.servicebusSubscription }}
-      topicName: {{ .Values.config.azure.servicebusTopic }}
+      subscriptionName: {{ .Values.azure.servicebusSubscription }}
+      topicName: {{ .Values.azure.servicebusTopic }}
       connection: SERVICE_BUS

devops/azure/chart/values.yaml

@@ -12,10 +12,11 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-config:
-  name: wks
-  azure:
-    servicebusSubscription: recordstopicsubscription
-    servicebusTopic: recordstopic
+azure:
+  servicebusSubscription: wkssubscription
+  servicebusTopic: recordstopic
+  storageContainer: osdu-wks-mappings
 image:
+  branch: master
+  tag: latest
   repository: community.opengroup.org:5555/osdu/platform/data-flow/enrichment/wks
\ No newline at end of file

devops/azure/release.yaml

+---
+# Source: wks/templates/deployment.yaml
+#  Copyright © Microsoft Corporation
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: osdu-gitlab-wks
+  namespace: osdu
+spec:
+  selector:
+    matchLabels:
+      app: wks
+  template:
+    metadata:
+      labels:
+        app: wks
+        aadpodidbinding: osdu-identity
+    spec:
+      volumes:
+        - name: azure-keyvault
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: azure-keyvault
+      containers:
+        - name: wks
+          image: community.opengroup.org:5555/osdu/platform/data-flow/enrichment/wks/wks-master:latest
+          imagePullPolicy: Always
+          volumeMounts:
+            - name: azure-keyvault
+              mountPath: "/mnt/azure-keyvault"
+              readOnly: true
+          env:
+          - name: spring_application_name
+            value: wks
+          - name: KEYVAULT_URI
+            valueFrom:
+              configMapKeyRef:
+                name: osdu-svc-properties
+                key: ENV_KEYVAULT
+          - name: AZURE_TENANT_ID
+            valueFrom:
+              configMapKeyRef:
+                name: osdu-svc-properties
+                key: ENV_TENANT_ID
+          - name: AZURE_CLIENT_ID
+            valueFrom:
+              secretKeyRef:
+                name: clientid
+                key: clientid
+          - name: AZURE_CLIENT_SECRET
+            valueFrom:
+              secretKeyRef:
+                name: clientpassword
+                key: clientpassword
+          - name: aad_client_id
+            valueFrom:
+              secretKeyRef:
+                name: appid
+                key: appid
+          - name: AZURE_APP_RESOURCE_ID
+            valueFrom:
+              secretKeyRef:
+                name: appid
+                key: appid
+          - name: appinsights_key
+            valueFrom:
+              secretKeyRef:
+                name: appinsights
+                key: appinsights
+          - name: cosmosdb_database
+            value: osdu-db
+          - name: servicebus_namespace_name
+            valueFrom:
+              configMapKeyRef:
+                name: osdu-svc-properties
+                key: ENV_SERVICEBUS_NAMESPACE
+          - name: servicebus_connection_string
+            valueFrom:
+              secretKeyRef:
+                name: servicebus
+                key: servicebus
+          - name: servicebus_topic_name
+            value: recordstopic
+          - name: storage_account
+            valueFrom:
+              configMapKeyRef:
+                name: osdu-svc-properties
+                key: ENV_STORAGE_ACCOUNT
+          - name: servicebus_topic_subscription
+            value: wkssubscription
+          - name: storage_container
+            value: osdu-wks-mappings
+          - name: server_port
+            value: "80"
+          - name: storage_service_endpoint
+            value: "http://osdu-flux-storage/api/storage/v2"
+          - name: max_concurrent_calls
+            value: "1"
+          - name: executor_n_threads
+            value: "2"
+          - name: max_lock_renew_duration_seconds
+            value: "120"
+---
+# Source: wks/templates/scaled_object.yaml
+#  Copyright © Microsoft Corporation
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+apiVersion: keda.k8s.io/v1alpha1
+kind: ScaledObject
+metadata:
+  name: wks
+  namespace: osdu
+  labels:
+    deploymentName: wks
+spec:
+  scaleTargetRef:
+    deploymentName: wks
+  triggers:
+  - type: azure-servicebus
+    metadata:
+      type: serviceBusTrigger
+      direction: in
+      name: message
+      subscriptionName: wkssubscription
+      topicName: recordstopic
+      connection: SERVICE_BUS

provider/wks-azure/Dockerfile

-FROM openjdk:8-jdk-alpine
-ARG JAR_FILE=target/*spring-boot.jar
-COPY ${JAR_FILE} app.jar
-ENTRYPOINT ["java","-jar","/app.jar"]
\ No newline at end of file

provider/wks-azure/Readme.md

@@ -19,34 +19,23 @@ In order to run the service locally, you will need to have the following environ
 **System Environment required to run service**
 Refer to [application.properties](./src/main/resources/application.properties)
 
-Definitions for some variables used
+Definitions for Azure Environment variables used
 
 | name | value | description | sensitive? | source |
 | ---  | ---   | ---         | ---        | ---    |
-| `LOG_PREFIX` | `storage` | Logging prefix | no | - |
-| `server.servlet.contextPath` | `/api/storage/v2/` | Servlet context path | no | - |
-| `STORAGE_API` | ex `https://foo-storage.azurewebsites.net` | Storage API endpoint | no | output of infrastructure deployment |
-| `AUTHORIZE_API` | ex `https://foo-entitlements.azurewebsites.net` | Entitlements API endpoint | no | output of infrastructure deployment |
-| `AUTHORIZE_API_KEY` | `********` | The API key clients will need to use when calling the entitlements | yes | -- |
-| `azure.application-insights.instrumentation-key` | `********` | API Key for App Insights | yes | output of infrastructure deployment |
-| `azure.activedirectory.client-id` | `********` | AAD client application ID | yes | output of infrastructure deployment |
-| `azure.activedirectory.AppIdUri` | `api://${azure.activedirectory.client-id}` | URI for AAD Application | no | -- |
-| `azure.activedirectory.session-stateless` | `true` | Flag run in stateless mode (needed by AAD dependency) | no | -- |
-| `cosmosdb_account` | ex `devintosdur2cosmosacct` | Cosmos account name | no | output of infrastructure deployment |
-| `cosmosdb_key` | `********` | Key for CosmosDB | yes | output of infrastructure deployments |
+| `storage_service_endpoint` | ex `https://foo-storage.azurewebsites.net` | Storage API endpoint | no | output of infrastructure deployment |
+| `appinsights_key` | `********` | API Key for App Insights | yes | output of infrastructure deployment |
 | `cosmosdb_database` | ex `dev-osdu-r2-db` | Cosmos database for storage documents | no | output of infrastructure deployment |
-| `azure.storage.account-name` | ex `foo-storage-account` | Storage account for storing documents | no | output of infrastructure deployment |
-| `azure.storage.enable-https` | `true` | Used by spring boot starter library | no | - |
+| `storage_account` | ex `foo-storage-account` | Storage account for storing documents | no | output of infrastructure deployment |
+| `storage_container` | `opendes` | Storage Container name | no | - |
 | `servicebus_topic_name` | `recordstopic` | Topic for async messaging | no | output of infrastructure deployment |
-| `servicebus_namespace_name` | ex `foo-sb-namespace` | Namespace for async messaging | no | output of infrastructure deployment |
+| `servicebus_connection_string` | `` | Connection string for service bus | no | |
 | `KEYVAULT_URI` | ex `https://foo-keyvault.vault.azure.net/` | URI of KeyVault that holds application secrets | no | output of infrastructure deployment |
 | `AZURE_CLIENT_ID` | `********` | Identity to run the service locally. This enables access to Azure resources. You only need this if running locally | yes | keyvault secret: `$KEYVAULT_URI/secrets/app-dev-sp-username` |
 | `AZURE_TENANT_ID` | `********` | AD tenant to authenticate users from | yes | keyvault secret: `$KEYVAULT_URI/secrets/app-dev-sp-tenant-id` |
 | `AZURE_CLIENT_SECRET` | `********` | Secret for `$AZURE_CLIENT_ID` | yes | keyvault secret: `$KEYVAULT_URI/secrets/app-dev-sp-password` |
 
 
-
-
 ### Build and run the application
 After configuring your environment as specified above, you can follow these steps to build and run the application. These steps should be invoked from the *repository root*.
 ```bash

provider/wks-azure/src/main/resources/application.properties

@@ -30,7 +30,7 @@ azure.keyvault.url=${KEYVAULT_URI}
 
 # Azure Storage configuration
 azure.storage.account-name=${storage_account}
-azure.storage.container-name=opendes
+azure.storage.container-name=${storage_container}
 
 # Azure Service Bus configuration
 azure.servicebus.topic-name=${servicebus_topic_name}

testing/wks-test-core/Readme.md

@@ -22,11 +22,11 @@ wks-core-test contains the step definitions and features for the Cucumber Integr
   
   ```
    INTEGRATION_TESTER (System identity to assume for API calls. Note: this user must have entitlements configured already)
-   TESTER_SERVICEPRINCIPAL_SECRET (service account key which has admins api access)
+   AZURE_TESTER_SERVICEPRINCIPAL_SECRET (service account key which has admins api access)
    AZURE_AD_TENANT_ID (AD tenant to authenticate users from)
    AZURE_AD_APP_RESOURCE_ID (AAD client application ID)
    VENDOR (cloud provider name eg. azure, gcp)
-   STORAGE_SERVICE_URL (storage service endpoint)
+   STORAGE_URL (storage service endpoint)
    
    ```
 

testing/wks-test-core/src/test/java/org/opengroup/osdu/wks/stepdefs/IntegrationTestStepDefs.java

@@ -350,7 +350,8 @@ public class IntegrationTestStepDefs implements En {
 		JsonElement jsonBody = new Gson().fromJson(payload, JsonElement.class);
 		payload = new Gson().toJson(jsonBody);
 		this.context.setInputPayload(payload);
-		String storageURL = System.getProperty("STORAGE_SERVICE_URL", System.getenv("STORAGE_SERVICE_URL"));
+		// Changing environment variable name to STORAGE_URL to maintain consistancy in naming
+		String storageURL = System.getProperty("STORAGE_URL", System.getenv("STORAGE_URL"));
 		HttpRequest httpRequest = HttpRequest.builder().url(storageURL + AutomationConstants.POST_ENDPOINT)
 				.body(jsonBody.toString()).httpMethod(HttpRequest.PUT).requestHeaders(this.context.getAuthHeaders())
 				.build();
@@ -360,7 +361,7 @@ public class IntegrationTestStepDefs implements En {
 	}
 
 	private void deleteFromStorage(String recordId) {
-		String storageURL = System.getProperty("STORAGE_SERVICE_URL", System.getenv("STORAGE_SERVICE_URL"))
+		String storageURL = System.getProperty("STORAGE_URL", System.getenv("STORAGE_URL"))
 				+ AutomationConstants.DELETE_ENDPOINT + recordId;
 		HttpRequest httpRequest = HttpRequest.builder().url(storageURL)
 				.httpMethod(HttpRequest.DELETE).requestHeaders(this.context.getAuthHeaders())
@@ -376,7 +377,7 @@ public class IntegrationTestStepDefs implements En {
 	}
 
 	private String storageGetResponse(String recordId) {
-		String reqURI = System.getProperty("STORAGE_SERVICE_URL", System.getenv("STORAGE_SERVICE_URL"))
+		String reqURI = System.getProperty("STORAGE_URL", System.getenv("STORAGE_URL"))
 				+ AutomationConstants.GET_ENDPOINT + recordId;
 		HttpRequest httpRequest = HttpRequest.builder().url(reqURI).httpMethod(HttpRequest.GET)
 				.requestHeaders(this.context.getAuthHeaders()).build();

testing/wks-test-core/src/test/java/org/opengroup/osdu/wks/util/AuthUtil.java

@@ -17,7 +17,7 @@ public class AuthUtil {
             case "azure":
 
                 String clientId = System.getProperty("INTEGRATION_TESTER", System.getenv("INTEGRATION_TESTER"));
-                String clientSecret = System.getProperty("TESTER_SERVICEPRINCIPAL_SECRET", System.getenv("TESTER_SERVICEPRINCIPAL_SECRET"));
+                String clientSecret = System.getProperty("AZURE_TESTER_SERVICEPRINCIPAL_SECRET", System.getenv("AZURE_TESTER_SERVICEPRINCIPAL_SECRET"));
                 String tenantId = System.getProperty("AZURE_AD_TENANT_ID", System.getenv("AZURE_AD_TENANT_ID"));
                 String appResourceId = System.getProperty("AZURE_AD_APP_RESOURCE_ID", System.getenv("AZURE_AD_APP_RESOURCE_ID"));
                 token = new AzureServicePrincipal().getIdToken(clientId, clientSecret, tenantId, appResourceId);

testing/wks-test-core/src/test/resources/logback.xml

+<configuration>
+    <root level="info">
+    </root>
+</configuration>
\ No newline at end of file

wks-core/src/test/resources/logback.xml

+<configuration>
+    <root level="info">
+    </root>
+</configuration>
\ No newline at end of file