merged from master, resolved conflicts.

charts/helm-config.yaml

@@ -8,13 +8,13 @@ global:
   # Specify the azure environment specific values
   #
   azure:
-    tenant: ${AZURE_TENANT_ID}
-    subscription: ${ENV_SUBSCRIPTION_ID}
-    resourcegroup: ${ENV_BASE_NAME_21}-rg
-    identity: ${ENV_BASE_NAME_21}-osdu-identity
-    identity_id: ${ENV_IDENTITY_ID}
-    keyvault: ${ENV_BASE_NAME_21}-kv
-    appid: ${ENV_APP_ID}
+    tenant: #{tenant-id}#
+    subscription: #{subscription-id}#
+    resourcegroup: #{base-name-cr}#-rg
+    identity: #{base-name-cr}#-osdu-identity
+    identity_id: #{osdu-identity-id}#
+    keyvault: #{base-name-cr}#-kv
+    appid: #{aad-client-id}#
 
   ################################################################################
   # Specify the Ingress Settings
@@ -25,10 +25,11 @@ global:
   #     https://acme-v02.api.letsencrypt.org/directory --> Production Server
   #
   ingress:
-    hostname: ${ENV_HOST}
-    admin: ${ADMIN_EMAIL}
+    hostname: #{DNS_HOST}#
+    admin: #{ADMIN_EMAIL}#
     sslServer: https://acme-v02.api.letsencrypt.org/directory  # Production
 
+<<<<<<< HEAD
   ################################################################################
   # Specify the Gitlab branch being used for image creation
   # ie: community.opengroup.org:5555/osdu/platform/system/storage/{{ .Values.global.branch }}/storage:latest
@@ -45,3 +46,12 @@ global:
   istio:
     username: ${ISTIO_USERNAME}
     password: ${ISTIO_PASSWORD}
+=======
+################################################################################
+# Specify the istio specific values
+# based64 encoded username and password
+#
+istio:
+  username: #{istio-username}#
+  password: #{istio-password}#
+>>>>>>> master

charts/osdu-common/pipeline.yml

@@ -33,8 +33,6 @@ trigger:
   paths:
     include:
       - /charts/osdu-common/*
-      - /devops/tasks/*
-      - /osdu-common-pipeline.yml
     exclude:
       - /**/*.md
 
@@ -48,8 +46,6 @@ pr:
   paths:
     include:
       - /charts/osdu-common/*
-      - /devops/tasks/*
-      - /osdu-common-pipeline.yml
     exclude:
       - /**/*.md
 
@@ -61,6 +57,7 @@ resources:
 
 variables:
   - group: 'Azure - OSDU'
+  - group: 'Azure - OSDU secrets'
 
   - name: serviceName
     value: "osdu-common"
@@ -77,6 +74,7 @@ stages:
       serviceName: ${{ variables.serviceName }}
       chartPath: ${{ variables.chartPath }}
       valuesFile: ${{ variables.valuesFile }}
+      skipDeploy: ${{ variables.SKIP_DEPLOY }}
       providers:
         - name: Azure
           environments: ["dev"]

charts/osdu-common/templates/appgw-ingress.yaml

@@ -17,7 +17,7 @@ spec:
       http:
         paths:
           - backend:
-              serviceName: default-service
+              serviceName: osdu-common
               servicePort: 80
             path: /*
           - backend:
@@ -67,4 +67,8 @@ spec:
           - backend:
               serviceName: unit-service
               servicePort: 80
-            path: /api/unit/*            
+            path: /api/unit/*
+          - backend:
+              serviceName: file
+              servicePort: 80
+            path: /api/file/v2/*

charts/osdu-common/templates/default.service.yaml

 apiVersion: v1
 kind: Service
 metadata:
-  name: default-service
+  name: osdu-common
   namespace: osdu
   labels:
-    app: default-service
+    app: osdu-common
 spec:
   type: ClusterIP
   ports:
     - port: 80
   selector:
-    app: default-service
+    app: osdu-common
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: default-service
+  name: osdu-common
   namespace: osdu
 spec:
   selector:
     matchLabels:
-      app: default-service
+      app: osdu-common
   replicas: 1
   template:
     metadata:
       labels:
-        app: default-service
+        app: osdu-common
     spec:
       containers:
-        - name: default-service
+        - name: osdu-common
           image: neilpeterson/aks-helloworld:v1
           ports:
             - containerPort: 80
           env:
             - name: TITLE
-              value: "Azure OSDU Platform - (AKS)"
\ No newline at end of file
+              value: "Azure OSDU Platform - (AKS)"

charts/osdu-common/templates/kv-secrets.yaml

@@ -63,21 +63,27 @@ spec:
       key: servicebus-namespace
     - objectName: "opendes-sb-connection"
       key: servicebus-connection
-  - secretName: airflow-storage
+  - secretName: airflow
     type: Opaque
     data:
-    - objectName: airflow-storage-connection
+    - objectName: airflow-storage
       key: storage-account
     - objectName: airflow-storage-key
       key: storage-key
+    - objectName: airflow-remote-log-connection
+      key: remote-log-connection
+    - objectName: airflow-admin-password
+      key: admin-password
+    - objectName: airflow-fernet-key
+      key: fernet-key
   - secretName: elastic
     type: Opaque
     data:
-    - objectName: "elastic-endpoint"
+    - objectName: "opendes-elastic-endpoint"
       key: elastic-endpoint
-    - objectName: "elastic-username"
+    - objectName: "opendes-elastic-username"
       key: elastic-username
-    - objectName: "elastic-password"
+    - objectName: "opendes-elastic-password"
       key: elastic-password
   - secretName: postgres
     type: Opaque
@@ -141,17 +147,26 @@ spec:
         - |
           objectName: airflow-storage-connection
           objectType: secret
+        - |
+          objectName: airflow-remote-log-connection
+          objectType: secret
         - |
           objectName: airflow-storage-key
           objectType: secret
         - |
-          objectName: elastic-endpoint
+          objectName: airflow-admin-password
+          objectType: secret
+        - |
+          objectName: airflow-fernet-key
+          objectType: secret
+        - |
+          objectName: opendes-elastic-endpoint
           objectType: secret
         - |
-          objectName: elastic-username
+          objectName: opendes-elastic-username
           objectType: secret
         - |
-          objectName: elastic-password
+          objectName: opendes-elastic-password
           objectType: secret
         - |
           objectName: log-workspace-id

devops/build-stage.yml

@@ -71,7 +71,6 @@ stages:
         archiveFile: $(Build.ArtifactStagingDirectory)/${{ parameters.serviceBase }}-integration-tests.zip
         replaceExistingArchive: true
 
-
     - task: CopyFiles@2
       condition: ne('${{ parameters.copyFileContentsToFlatten }}', '')
       displayName: 'Copy Files to: $(build.artifactstagingdirectory)'

devops/chart-stages.yml

@@ -16,7 +16,7 @@
 parameters:
   providers: []
   serviceName: ""
-  skipDeploy: "false"
+  skipDeploy: false
   hldRegPath: "providers/azure/hld-registry"
 
 stages:
@@ -25,10 +25,10 @@ stages:
       - stage: 'Deploy_${{ provider.name }}_${{ environment }}'
 
         variables:
-          - group: 'Azure - Common'
-          - group: 'Azure Common Secrets'
+          - group: '${{ provider.name }} Target Env Secrets - ${{ environment }}'
           - group: '${{ provider.name }} Target Env - ${{ environment }}'
 
+
         jobs:
         - deployment: Deploy
           pool:
@@ -49,3 +49,9 @@ stages:
                     valuesFile: ${{ parameters.valuesFile }}
                     skipDeploy: ${{ parameters.skipDeploy }}
                     hldRegPath: ${{ parameters.hldRegPath }}
+
+                - template: tasks/flux-chart-wait.yml
+                  parameters:
+                    serviceName: ${{parameters.serviceName}}
+                    environment: ${{ environment }}
+                    imageRepoName: '${{ parameters.serviceName }}'

devops/deploy-stages.yml

@@ -33,8 +33,6 @@ stages:
     - stage: 'Deploy_${{ provider.name }}_${{ environment }}'
 
       variables:
-      - group: 'Azure - Common'
-      - group: 'Azure Common Secrets'
       - group: '${{ provider.name }} Target Env Secrets - ${{ environment }}'
       - group: '${{ provider.name }} Target Env - ${{ environment }}'
       - group: '${{ provider.name }} Service Release - ${{ parameters.serviceName }}'
@@ -69,7 +67,7 @@ stages:
                   testCoreMavenGoal: ${{ parameters.testCoreMavenGoal }}
                   testCoreMavenOptions: ${{ parameters.testCoreMavenOptions }}
                   artifactName: ${{ variables.artifactName }}
-                  elasticEndpoint: $(elastic-endpoint)
+                  elasticEndpoint: $(ELASTIC_ENDPOINT)
                   environment: ${{ environment }}
                   chartPath: ${{ parameters.chartPath }}
                   valuesFile: ${{ parameters.valuesFile }}

devops/tasks/acr-publish.yml

@@ -25,7 +25,7 @@ steps:
     condition: and(succeeded(), eq('${{ parameters.providerName }}', 'Azure'))
     env:
       JAR_FILE_PATH: ${{ parameters.jarFilePath }}
-      IMAGE: $(CONTAINER_REGISTRY_NAME).azurecr.io/${{ parameters.imageRepoName }}:$(Build.BuildId)-${{ parameters.environment }}
+      IMAGE: $(CONTAINER_REGISTRY_NAME).azurecr.io/${{ parameters.imageRepoName }}-${{ parameters.environment }}:$(Build.SourceVersion)
     inputs:
       azureSubscription: '$(SERVICE_CONNECTION_NAME)'
       addSpnToEnvironment: true
@@ -57,4 +57,4 @@ steps:
 
         docker push $IMAGE
         echo "Show recent tags of repository ${{ parameters.imageRepoName }}"
-        az acr repository show-tags -n $(CONTAINER_REGISTRY_NAME) --repository ${{ parameters.imageRepoName }} --top 10 --orderby time_desc
+        az acr repository show-tags -n $(CONTAINER_REGISTRY_NAME) --repository ${{ parameters.imageRepoName }}-${{ parameters.environment }} --top 10 --orderby time_desc

devops/tasks/aks-deployment-steps.yml

@@ -29,8 +29,14 @@ steps:
     - checkout: FluxRepo
       persistCredentials: true
     - task: "HelmInstaller@1"
+      displayName: 'Helm Install'
       inputs:
         helmVersionToInstall: 3.3.0
+    - task: qetza.replacetokens.replacetokens-task.replacetokens@3
+      displayName: 'Helm Values'
+      inputs:
+        targetFiles: |
+          $(Build.SourcesDirectory)/$(Build.Repository.Name)/${{parameters.valuesFile}} => $(Build.SourcesDirectory)/$(Build.Repository.Name)/${{parameters.chartPath}}/values.yaml
     - template: helm-template.yml
       parameters:
         serviceName: ${{parameters.serviceName}}

devops/tasks/debug.yml

+#  Copyright © Microsoft Corporation
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+#####################
+# README: GitOPS operations manifest repo tasks.
+#####################
+
+steps:
+  - task: Bash@3
+    displayName: "Debug"
+    condition: ne('${{ parameters.skipTest }}', false)
+    inputs:
+      targetType: "inline"
+      script: |
+        #!/usr/bin/env bash
+        set -euo pipefail
+
+        echo "Partition Service Test"
+        echo "--------------------------------------------------------------"
+        echo "-DINTEGRATION_TESTER=$INTEGRATION_TESTER"
+        echo "-DPARTITION_BASE_URL=$HOST_URL"
+        echo "-DMY_TENANT=$MY_TENANT"
+        echo "-DAZURE_TESTER_SERVICEPRINCIPAL_SECRET=$AZURE_TESTER_SERVICEPRINCIPAL_SECRET"
+        echo "-DAZURE_AD_TENANT_ID=$AZURE_TENANT_ID"
+        echo "-DAZURE_AD_APP_RESOURCE_ID=$AZURE_AD_APP_RESOURCE_ID"
+        echo "-DNO_DATA_ACCESS_TESTER=$NO_DATA_ACCESS_TESTER"
+        echo "-DNO_DATA_ACCESS_TESTER_SERVICEPRINCIPAL_SECRET=$NO_DATA_ACCESS_TESTER_SERVICEPRINCIPAL_SECRET"
+        echo "-DAZURE_AD_OTHER_APP_RESOURCE_ID=$AZURE_AD_OTHER_APP_RESOURCE_ID"
+        echo "-DENVIRONMENT=HOSTED"
+

devops/tasks/deployment-steps.yml

@@ -42,7 +42,7 @@ steps:
       environment: ${{ parameters.environment }}
       jarFilePath: $(DetectJarScript.JAR_FILE_PATH)
       providerName: ${{ parameters.providerName }}
-      imageRepoName: 'svc-${{ parameters.serviceName }}'
+      imageRepoName: '${{ parameters.serviceName }}'
 
   - template: aks-deployment-steps.yml
     parameters:
@@ -75,9 +75,15 @@ steps:
       options: ${{ parameters.testCoreMavenOptions }}
       publishJUnitResults: ${{ parameters.mavenPublishJUnitResults }}
 
+  - template: flux-service-wait.yml
+    parameters:
+      serviceName: ${{parameters.serviceName}}
+      environment: ${{ parameters.environment }}
+      imageRepoName: '${{ parameters.serviceName }}'
+
   - task: Maven@3
     displayName: 'Maven run integration test'
-    condition: ${{ parameters.skipTest }}
+    condition: and(succeeded(), ne('${{ parameters.skipTest }}', false))
     inputs:
       mavenPomFile: '$(System.DefaultWorkingDirectory)/$(MAVEN_INTEGRATION_TEST_POM_FILE_PATH)'
       goals: ${{ parameters.integrationTestMavenGoal }}

devops/tasks/flux-chart-wait.yml

+#  Copyright � Microsoft Corporation
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+# This template is responsible for detecting a service being alive.
+steps:
+  - task: AzureCLI@1
+    displayName: 'Flux Deployment Wait'
+    env:
+      SERVICE_NAME: ${{parameters.serviceName}}
+      BASE_NAME_SR: $(base-name-sr)
+    inputs:
+      azureSubscription: '$(SERVICE_CONNECTION_NAME)'
+      addSpnToEnvironment: true
+      scriptLocation: inlineScript
+      inlineScript: |
+        #!/usr/bin/env bash
+        set -euo pipefail
+
+        echo "Logging in to AKS"
+        echo "------------------------------------"
+        sudo az aks install-cli
+        az aks get-credentials -g $BASE_NAME_SR-rg -n $BASE_NAME_SR-aks
+
+        echo "Checking Deployment $SERVICE_NAME Available"
+        echo "------------------------------------"
+        attempt_counter=0
+        max_attempts=30
+        until kubectl get deployment $SERVICE_NAME -n osdu
+        do
+          if [ ${attempt_counter} -eq ${max_attempts} ];then
+            echo "Deployment Image not updated, integration tests are skipped"
+            exit 1
+          fi
+          attempt_counter=$(($attempt_counter+1))
+          sleep 30
+        done
+
+        kubectl wait --for=condition=available --timeout=600s deployment/$SERVICE_NAME -n osdu

devops/tasks/flux-service-wait.yml

+#  Copyright � Microsoft Corporation
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+# This template is responsible for detecting a service being alive.
+steps:
+  - task: AzureCLI@1
+    displayName: 'Flux Deployment Wait'
+    env:
+      IMAGE: $(CONTAINER_REGISTRY_NAME).azurecr.io/${{ parameters.imageRepoName }}-${{ parameters.environment }}:$(Build.SourceVersion)
+      SERVICE_NAME: ${{parameters.serviceName}}
+      BASE_NAME_SR: $(base-name-sr)
+    inputs:
+      azureSubscription: '$(SERVICE_CONNECTION_NAME)'
+      addSpnToEnvironment: true
+      scriptLocation: inlineScript
+      inlineScript: |
+        #!/usr/bin/env bash
+        set -euo pipefail
+
+        echo "Logging in to AKS"
+        echo "------------------------------------"
+        sudo az aks install-cli
+        az aks get-credentials -g $BASE_NAME_SR-rg -n $BASE_NAME_SR-aks
+
+        echo "Checking Deployment $SERVICE_NAME Exists"
+        echo "----------------------------------------"
+        attempt_counter=0
+        max_attempts=30
+        until kubectl get deployment $SERVICE_NAME -n osdu
+        do
+          if [ ${attempt_counter} -eq ${max_attempts} ];then
+            echo "Deployment Image not updated, integration tests are skipped"
+            exit 1
+          fi
+          attempt_counter=$(($attempt_counter+1))
+          sleep 30
+        done
+
+
+
+        echo "Checking Deployment $IMAGE updated"
+        echo "------------------------------------"
+        attempt_counter=0
+        max_attempts=30
+        while [ $(kubectl get deployment $SERVICE_NAME -n osdu -o=jsonpath='{$.spec.template.spec.containers[:1].image}') != $IMAGE ]
+        do
+          if [ ${attempt_counter} -eq ${max_attempts} ];then
+            echo "Deployment Image not updated, integration tests are skipped"
+            exit 1
+          fi
+          attempt_counter=$(($attempt_counter+1))
+          sleep 30
+        done
+
+
+        echo "Checking Deployment $SERVICE_NAME Available"
+        echo "----------------------------------------"
+        kubectl wait --for=condition=available --timeout=600s deployment/$SERVICE_NAME -n osdu

devops/tasks/gitops.yml

@@ -18,7 +18,7 @@
 
 steps:
   - task: Bash@3
-    displayName: "Commit to Branch"
+    displayName: "Flux Commit"
     env:
       CHART_PATH: ${{parameters.chartPath}}
       BRANCH_NAME: ${{ parameters.branchName }}
@@ -53,24 +53,20 @@ steps:
         }
 
         function git_commit() {
-          # echo "GIT REMOVE"
-          # Cannot delete as all the manifests from multiple repos would be coming over to here.
-          # rm -rf ./$(MANIFEST_GENERATION_PATH)/*/
-
           echo "COPY YAML FILES FROM $GITOPS_MANIFEST_DIRECTORY..."
           cp -rf $GITOPS_MANIFEST_DIRECTORY .
 
           echo "SKIP DEPLOY: $SKIP_DEPLOY, HLD-REG: $HLD_REG_PATH"
-          if [[ $SKIP_DEPLOY == false ]]
+          if [[ $SKIP_DEPLOY == true ]]
           then
+              echo "***********************"
+              echo "FLUX DEPLOYMENT SKIPPED"
+              echo "***********************"
+          else
               echo "COPYING YAML FILES TO $HLD_REG_PATH/$SERVICE_NAME"
               mkdir -p ./$HLD_REG_PATH/$SERVICE_NAME
               rm -rf ./$HLD_REG_PATH/$SERVICE_NAME/*
               cp -rf $GITOPS_MANIFEST_DIRECTORY/$SERVICE_NAME/templates/* ./$HLD_REG_PATH/$SERVICE_NAME
-          else
-              echo "***********************"
-              echo "FLUX DEPLOYMENT SKIPPED"
-              echo "***********************"
           fi
 
           git add -A

devops/tasks/helm-template.yml

@@ -15,7 +15,7 @@
 steps:
   - task: Bash@3
     name: GenerateHelmTemplate
-    displayName: Generate Template
+    displayName: Helm Template
     env:
       CHART_PATH: ${{parameters.chartPath}}
       VALUES_FILE: ${{parameters.valuesFile}}
@@ -30,9 +30,10 @@ steps:
 
           cd $(Build.SourcesDirectory)/$(Build.Repository.Name)
           mkdir $CHART_PATH/$GENERATION_PATH
+          cat $(Build.SourcesDirectory)/$(Build.Repository.Name)/${{parameters.chartPath}}/values.yaml
 
-          echo "Creating Helm Values File"
-          envsubst < $VALUES_FILE > $CHART_PATH/values.yml
+          # echo "Creating Helm Values File"
+          # envsubst < $VALUES_FILE > $CHART_PATH/values.yml
 
           echo "Extracting Manifest"
-          helm template $SERVICE_NAME $CHART_PATH -f $CHART_PATH/values.yml --output-dir $CHART_PATH/$GENERATION_PATH
+          helm template $SERVICE_NAME $CHART_PATH -f $(Build.SourcesDirectory)/$(Build.Repository.Name)/${{parameters.chartPath}}/values.yaml --output-dir $CHART_PATH/$GENERATION_PATH