Merge remote-tracking branch 'origin/master' into release/0.4

cloud-providers/aws.yml

@@ -57,6 +57,7 @@
     DELIVERY_INT_TEST_BUCKET_NAME: $AWS_DELIVERY_INT_TEST_BUCKET_NAME
     DEFAULT_DATA_PARTITION_ID_TENANT1: $AWS_DEFAULT_DATA_PARTITION_ID_TENANT1
     DEFAULT_DATA_PARTITION_ID_TENANT2: $AWS_DEFAULT_DATA_PARTITION_ID_TENANT2
+    DATA_PARTITION_ID: int-test-file #used by file->delivery int tests
     SEARCH_HOST: $AWS_SEARCH_HOST
     STORAGE_HOST: $AWS_STORAGE_HOST
     LEGAL_HOST: $AWS_LEGAL_HOST
@@ -83,6 +84,7 @@
     REGISTER_CUSTOM_PUSH_URL_HMAC: $AWS_REGISTER_CUSTOM_PUSH_URL_HMAC
     NOTIFICATION_BASE_URL: $AWS_NOTIFICATION_BASE_URL
     NOTIFICATION_REGISTER_BASE_URL: $AWS_NOTIFICATION_REGISTER_BASE_URL
+    FILE_SERVICE_HOST: $AWS_FILE_SERVICE_HOST
 
 .aws:
   tags: ['osdu-medium']

cloud-providers/azure.yml

@@ -107,12 +107,7 @@
     #WKS
     OS_WKS_SCHEMA_KIND: opendes:wks:wellbore:1.0.0
     #Unit
-    client_id: $AZURE_PRINCIPAL_ID
     VIRTUAL_SERVICE_HOST_NAME: $AZURE_DNS_NAME
-    #Crs_Catalog
-    AZURE_DEPLOY_TENANT: $AZURE_TENANT_ID
-    AZURE_DEPLOY_CLIENT_ID: $AZURE_PRINCIPAL_ID
-    AZURE_DEPLOY_CLIENT_SECRET: $AZURE_PRINCIPAL_SECRET
     #Register
     REGISTER_BASE_URL: https://${AZURE_DNS_NAME}/
     SUBSCRIPTION_ID: $AZURE_EVENT_SUBSCRIPTION_ID
@@ -189,6 +184,14 @@ azure_deploy:
     - cd devops/azure
     - echo "--set image.branch=$BRANCH --set image.tag=$TAG"
 
+    #Update Catalog
+    - |
+      if [ -f ../../data/*_catalog_*.json ]; then
+        accountKey=$(kubectl get secret airflow -n osdu -o jsonpath='{.data.azurestorageaccountkey}' | base64 -d)
+        accountName=$(kubectl get secret airflow -n osdu -o jsonpath='{.data.azurestorageaccountname}' | base64 -d)
+        az storage file upload --account-name $accountName --account-key $accountKey --share-name $SHARE_NAME --source ../../data/*_catalog_*.json
+      fi
+
     # Install Service
     - helm upgrade -i osdu-gitlab-$CI_PROJECT_NAME chart --set image.repository=${AZURE_REGISTRY}.azurecr.io --set image.branch=$BRANCH --set image.tag=$TAG
     # Increasing to 900s as rolling updates are happening and each service is expected to have minimum 2 containers.

cloud-providers/ibm.yml

@@ -133,6 +133,11 @@ ibm-deploy:
 
     #Conversion
     IBM_VIRTUAL_HOST_CRS_CONVERSION: $IBM_CONVERSION_SERVICE_HOST
+    
+    #Partition
+    PARTITION_BASE_URL: $IBM_PARTITION_BASE_URL
+    #CLIENT_TENANT: common
+    MY_TENANT: $TENANT_NAME
 
 ibm-test:
   stage: integration

cloud-providers/osdu-gcp-cloudrun.yml

@@ -137,7 +137,7 @@ osdu-gcp-containerize:
     - gcloud auth activate-service-account --key-file $OSDU_GCP_DEPLOY_FILE
     - gcloud config set project $OSDU_GCP_PROJECT
     - touch .gcloudignore
-    - gcloud builds submit --config provider/$OSDU_GCP_SERVICE-$OSDU_GCP_VENDOR/cloudbuild/cloudbuild.yaml --substitutions=_PROVIDER_NAME=$OSDU_GCP_VENDOR,_SHORT_SHA=$CI_COMMIT_SHORT_SHA,_PORT=$OSDU_GCP_PORT
+    - gcloud builds submit --config provider/$OSDU_GCP_SERVICE-$OSDU_GCP_VENDOR/cloudbuild/cloudbuild.yaml --substitutions=_GCP_SERVICE=$OSDU_GCP_SERVICE,_APPLICATION_NAME=$OSDU_GCP_APPLICATION_NAME,_PROVIDER_NAME=$OSDU_GCP_VENDOR,_SHORT_SHA=$CI_COMMIT_SHORT_SHA,_PORT=$OSDU_GCP_PORT
   only:
     variables:
       - $OSDU_GCP == 'true'
@@ -173,7 +173,7 @@ osdu-gcp-test:
     - .maven
   stage: integration
   needs: ["osdu-gcp-deploy"]
-  allow_failure: true
+  allow_failure: false
   variables:
     WORKFLOW_HOST: $OSDU_GCP_WORKFLOW_SERVICE_URL
     INGEST_HOST: $OSDU_GCP_INGEST_HOST
@@ -245,4 +245,4 @@ osdu-gcp-tagging:
     - gcloud config set project $OSDU_GCP_PROJECT
     - gcloud container images add-tag gcr.io/$OSDU_GCP_PROJECT/$OSDU_GCP_APPLICATION_NAME/$OSDU_GCP_SERVICE-$OSDU_GCP_VENDOR:$CI_COMMIT_SHORT_SHA gcr.io/$OSDU_GCP_PROJECT/$OSDU_GCP_APPLICATION_NAME/$OSDU_GCP_SERVICE-$OSDU_GCP_VENDOR:$CI_COMMIT_TAG
   only:
-    - tags
\ No newline at end of file
+    - tags