Merge branch 'haaggarw/AddingAzurePipelines' into 'master'

Adding Azure pipelines

See merge request !64

.gitlab-ci.yml

@@ -16,9 +16,11 @@ variables:
   AZURE_BUILD_SUBDIR: provider/schema-azure
   AZURE_TEST_SUBDIR: testing/schema-test-core
   AZURE_DEPLOYMENTS_SUBDIR: deployments/scripts/azure
-  INTEGRATION_TESTER: $AZURE_PRINCIPAL_ID
-  AZURE_TESTER_SERVICEPRINCIPAL_SECRET: $AZURE_PRINCIPAL_SECRET
+  AZURE_DEPLOYMENTS_SCRIPTS_SUBDIR: deployments/scripts
+  AZURE_CLIENT_ID: $AZURE_PRINCIPAL_ID
+  AZURE_CLIENT_SECRET: $AZURE_PRINCIPAL_SECRET
   DATA_PARTITION: opendes
+  AZURE_AD_APP_RESOURCE_ID: $AZURE_APP_ID
   AZURE_STORAGE_CONNECTION_STRING: DefaultEndpointsProtocol=https;AccountName=${AZURE_BASE}data;AccountKey=${AZURE_STORAGE_KEY};EndpointSuffix=core.windows.net
 
 
@@ -62,4 +64,4 @@ include:
     ref: 'master'
     file: 'cloud-providers/osdu-gcp-cloudrun.yml'
 
-  - local: "/devops/azure/bootstrap.yaml"
+  - local: "/devops/azure/gitlab-bootstrap.yml"

deployments/scripts/azure/Token.py

@@ -5,9 +5,9 @@ class AzureToken(object):
 
     def get_azure_id_token(self):
         tenant_id = os.getenv('AZURE_TENANT_ID')
-        resource_id = os.getenv('AZURE_APP_ID')
-        client_id = os.getenv('INTEGRATION_TESTER')
-        client_secret = os.getenv('AZURE_TESTER_SERVICEPRINCIPAL_SECRET')
+        resource_id = os.getenv('AZURE_AD_APP_RESOURCE_ID')
+        client_id = os.getenv('AZURE_CLIENT_ID')
+        client_secret = os.getenv('AZURE_CLIENT_SECRET')
         
         if tenant_id is None:
             print('Please pass tenant Id to generate token')

deployments/scripts/azure/bootstrap.sh

 export AZURE_SCHEMA_URL=https://$AZURE_DNS_NAME/api/schema-service/v1/schema
 BEARER_TOKEN=`python $AZURE_DEPLOYMENTS_SUBDIR/Token.py`
 export BEARER_TOKEN=$BEARER_TOKEN
-python deployments/scripts/DeploySharedSchemas.py -u $AZURE_SCHEMA_URL
+python $AZURE_DEPLOYMENTS_SCRIPTS_SUBDIR/DeploySharedSchemas.py -u $AZURE_SCHEMA_URL
\ No newline at end of file

devops/azure/bootstrap-stage.yml

+parameters:
+  artifactName: 'drop'
+  serviceName: ''
+  providers: []
+
+stages:
+
+  - ${{ each provider in parameters.providers }}:
+      - ${{ each environment in provider.environments }}:
+          - stage: 'Bootstrap_${{ provider.name }}_${{ environment }}'
+
+            variables:
+              - group: '${{ provider.name }} Target Env Secrets - ${{ environment }}'
+              - group: '${{ provider.name }} Target Env - ${{ environment }}'
+              - group: '${{ provider.name }} Service Release - ${{ parameters.serviceName }}'
+
+              - name: artifactName
+                value: 'drop'
+
+            jobs:
+              - deployment: Bootstrap
+                pool:
+                  name: $(AGENT_POOL)
+                  demands:
+                    - maven
+                environment: ${{ environment }}
+                strategy:
+                  runOnce:
+                    deploy:
+                      steps:
+                        - download: none
+                        - task: DownloadBuildArtifacts@0
+                          displayName: 'Pull Build Artifacts'
+                          inputs:
+                            artifactName: '${{ variables.artifactName }}'
+                            downloadPath: '$(System.DefaultWorkingDirectory)'
+                        - task: UsePythonVersion@0
+                          inputs:
+                            versionSpec: '3.8'
+                            addToPath: true
+                            architecture: 'x64'
+                        - task: Bash@3
+                          displayName: 'Bootstrap Schemas'
+                          inputs:
+                            targetType: 'inline'
+                            script: |
+                              export AZURE_DEPLOYMENTS_SUBDIR=$(System.DefaultWorkingDirectory)/${{ parameters.artifactName }}/deployments/scripts/azure
+                              # These variables are required for generating the token
+                              export AZURE_TENANT_ID=$(AZURE_TENANT_ID)
+                              export AZURE_AD_APP_RESOURCE_ID=$(AZURE_AD_APP_RESOURCE_ID)
+                              export AZURE_CLIENT_ID=$(INTEGRATION_TESTER)
+                              export AZURE_CLIENT_SECRET=$(AZURE_TESTER_SERVICEPRINCIPAL_SECRET)
+                              export DATA_PARTITION=$(DATA_PARTITION_ID)
+
+                              pip install -r $AZURE_DEPLOYMENTS_SUBDIR/requirements.txt
+                              chmod +x $AZURE_DEPLOYMENTS_SUBDIR/bootstrap.sh
+                              $AZURE_DEPLOYMENTS_SUBDIR/bootstrap.sh

devops/azure/chart/helm-config.yml

+#  Copyright © Microsoft Corporation
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+# This file contains the essential configs for the osdu on azure helm chart
+global:
+
+  # Service(s) Replica Count
+  replicaCount: 1
+
+################################################################################
+# Specify the Gitlab branch being used for image creation
+# ie: community.opengroup.org:5555/osdu/platform/system/storage/{{ .Values.global.branch }}/storage:latest
+#
+image:
+  repository: #{container-registry}#.azurecr.io
+  branch: #{ENVIRONMENT_NAME}#
+  tag: #{Build.SourceVersion}#

devops/azure/chart/templates/deployment.yaml

@@ -29,70 +29,73 @@ spec:
         aadpodidbinding: osdu-identity
     spec:
       volumes:
-      - name: azure-keyvault
-        csi:
-          driver: secrets-store.csi.k8s.io
-          readOnly: true
-          volumeAttributes:
-            secretProviderClass: azure-keyvault
-      containers:
-      - name: {{ .Chart.Name }}
-        image: {{ .Values.image.repository }}/{{ .Chart.Name }}-{{ .Values.image.branch }}:{{ .Values.image.tag | default .Chart.AppVersion }}
-        imagePullPolicy: Always
-        ports:
-        - containerPort: 80
-        readinessProbe:
-          httpGet:
-            path: /api/schema-service/v1/swagger-ui.html
-            port: 80
-        volumeMounts:
         - name: azure-keyvault
-          mountPath: "/mnt/azure-keyvault"
-          readOnly: true
-        env:
-        - name: spring_application_name
-          value: schema
-        - name: LOG_PREFIX
-          value: "schema"
-        - name: server.servlet.contextPath
-          value: /api/schema-service/v1/
-        - name: server.port
-          value: "80"
-        - name: KEYVAULT_URI
-          valueFrom:
-            configMapKeyRef:
-              name: osdu-svc-properties
-              key: ENV_KEYVAULT
-        - name: AZURE_TENANT_ID
-          valueFrom:
-            secretKeyRef:
-              name: active-directory
-              key: tenantid
-        - name: AZURE_CLIENT_ID
-          valueFrom:
-            secretKeyRef:
-              name: active-directory
-              key: principal-clientid
-        - name: AZURE_CLIENT_SECRET
-          valueFrom:
-            secretKeyRef:
-              name: active-directory
-              key: principal-clientpassword
-        - name: aad_client_id
-          valueFrom:
-            secretKeyRef:
-              name: active-directory
-              key: application-appid
-        - name: appinsights_key
-          valueFrom:
-            secretKeyRef:
-              name: central-logging
-              key: appinsights
-        - name: cosmosdb_database
-          value: osdu-db
-        - name: storage_account
-          value: osdumvpdp1glabky7vdata
-        - name: entitlements_service_endpoint
-          value: http://entitlements-azure/entitlements/v1
-        - name: entitlements_service_api_key
-          value: "OBSOLETE"
\ No newline at end of file
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: azure-keyvault
+      containers:
+        - name: {{ .Chart.Name }}
+          image: {{ .Values.image.repository }}/{{ .Chart.Name }}-{{ .Values.image.branch }}:{{ .Values.image.tag | default .Chart.AppVersion }}
+          imagePullPolicy: Always
+          ports:
+            - containerPort: 80
+          readinessProbe:
+            httpGet:
+              path: /api/schema-service/v1/swagger-ui.html
+              port: 80
+          volumeMounts:
+            - name: azure-keyvault
+              mountPath: "/mnt/azure-keyvault"
+              readOnly: true
+          env:
+            - name: spring_application_name
+              value: schema
+            - name: LOG_PREFIX
+              value: "schema"
+            - name: server.servlet.contextPath
+              value: /api/schema-service/v1/
+            - name: server.port
+              value: "80"
+            - name: KEYVAULT_URI
+              valueFrom:
+                configMapKeyRef:
+                  name: osdu-svc-properties
+                  key: ENV_KEYVAULT
+            - name: AZURE_TENANT_ID
+              valueFrom:
+                secretKeyRef:
+                  name: active-directory
+                  key: tenantid
+            - name: AZURE_CLIENT_ID
+              valueFrom:
+                secretKeyRef:
+                  name: active-directory
+                  key: principal-clientid
+            - name: AZURE_CLIENT_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: active-directory
+                  key: principal-clientpassword
+            - name: aad_client_id
+              valueFrom:
+                secretKeyRef:
+                  name: active-directory
+                  key: application-appid
+            - name: appinsights_key
+              valueFrom:
+                secretKeyRef:
+                  name: central-logging
+                  key: appinsights
+            - name: storage_account
+              valueFrom:
+                secretKeyRef:
+                  name: storage
+                  key: storage-account
+            - name: cosmosdb_database
+              value: osdu-db
+            - name: entitlements_service_endpoint
+              value: http://entitlements-azure/entitlements/v1
+            - name: entitlements_service_api_key
+              value: "OBSOLETE"
\ No newline at end of file

devops/azure/deploy-stage.yml

+#  Copyright � Microsoft Corporation
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+#####################
+# README: Defines an common set of supportive deploy stages that any external app service pipeline can call into.
+#####################
+
+parameters:
+  providers: []
+  serviceName: ''
+  hldRegPath: "providers/azure/hld-registry"
+  testCoreMavenPomFile: ''
+  testCoreMavenOptions: ''
+  integrationTestMavenGoal: 'package'
+  testCoreMavenGoal: 'install'
+  skipDeploy: false
+  skipTest: 'false'
+  dockerFile: ''
+  checkoutRepo: self
+
+stages:
+
+  - ${{ each provider in parameters.providers }}:
+      - ${{ each environment in provider.environments }}:
+          - stage: 'Deploy_${{ provider.name }}_${{ environment }}'
+
+            variables:
+              - group: '${{ provider.name }} Target Env Secrets - ${{ environment }}'
+              - group: '${{ provider.name }} Target Env - ${{ environment }}'
+              - group: '${{ provider.name }} Service Release - ${{ parameters.serviceName }}'
+
+              - name: artifactName
+                value: 'drop'
+
+            jobs:
+              - deployment: Deploy
+                pool:
+                  name: $(AGENT_POOL)
+                  demands:
+                    - maven
+                environment: ${{ environment }}
+                strategy:
+                  runOnce:
+                    deploy:
+                      steps:
+                        - download: none
+
+                        - task: DownloadBuildArtifacts@0
+                          displayName: 'Pull Build Artifacts'
+                          inputs:
+                            artifactName: '${{ variables.artifactName }}'
+                            downloadPath: '$(System.DefaultWorkingDirectory)'
+
+                        - template: deployment-steps.yml
+                          parameters:
+                            serviceName: ${{ parameters.serviceName }}
+                            providerName: ${{ provider.name }}
+                            testCoreMavenPomFile: ${{ parameters.testCoreMavenPomFile }}
+                            testCoreMavenGoal: ${{ parameters.testCoreMavenGoal }}
+                            integrationTestMavenGoal: ${{ parameters.integrationTestMavenGoal }}
+                            testCoreMavenOptions: ${{ parameters.testCoreMavenOptions }}
+                            artifactName: ${{ variables.artifactName }}
+                            elasticEndpoint: $(ELASTIC_ENDPOINT)
+                            environment: ${{ environment }}
+                            chartPath: ${{ parameters.chartPath }}
+                            valuesFile: ${{ parameters.valuesFile }}
+                            dockerFile: ${{ parameters.dockerFile }}
+                            hldRegPath: ${{ parameters.hldRegPath }}
+                            skipDeploy: ${{ parameters.skipDeploy }}
+                            skipTest: ${{ parameters.skipTest }}
+                            runPythonTest: ${{ parameters.runPythonTest }}
+                            testPythonFilePath: ${{ parameters.testPythonFilePath }}
+                            testPythonFile: ${{ parameters.testPythonFile }}
+                            checkoutRepo: ${{ parameters.checkoutRepo }}

devops/azure/deployment-steps.yml

+#  Copyright � Microsoft Corporation
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+#####################
+# README: Defines a common set of deployment steps that the deploy stage can call into.
+#####################
+
+parameters:
+  artifactName: 'drop'
+  serviceName: ''
+  providerName: ''
+  jarFilePattern: '*-SNAPSHOT*.jar'
+  integrationTestMavenGoal: 'package'
+  mavenPublishJUnitResults: false
+  testCoreMavenPomFile: ''
+  testCoreMavenGoal: 'install'
+  testCoreMavenOptions: ''
+  testArtifactLocation: 'drop/deploy'
+  elasticEndpoint: ''
+  skipTest: ''
+  dockerFile: ''
+  testPythonFilePath: ''
+  testPythonFile: ''
+  runPythonTest: ''
+  checkoutRepo: self
+
+steps:
+  - template: /devops/tasks/detect-jar.yml@TemplateRepo
+    parameters:
+      stepName: DetectJarScript
+      providerName: ${{ parameters.providerName }}
+      pomFilePath: $(MAVEN_DEPLOY_POM_FILE_PATH)
+      jarFilePattern: ${{ parameters.jarFilePattern }}
+
+  - template: /devops/tasks/acr-publish.yml@TemplateRepo
+    parameters:
+      environment: ${{ parameters.environment }}
+      jarFilePath: $(DetectJarScript.JAR_FILE_PATH)
+      providerName: ${{ parameters.providerName }}
+      imageRepoName: ${{ parameters.serviceName }}
+      artifactName: ${{ parameters.artifactName }}
+      dockerFile: ${{ parameters.dockerFile }}
+
+  - template: /devops/tasks/aks-deployment-steps.yml@TemplateRepo
+    parameters:
+      serviceName: ${{ parameters.serviceName }}
+      providerName: ${{ parameters.providerName }}
+      environment: ${{ parameters.environment }}
+      chartPath: ${{ parameters.chartPath }}
+      valuesFile: ${{ parameters.valuesFile }}
+      hldRegPath: ${{ parameters.hldRegPath }}
+      skipDeploy: ${{ parameters.skipDeploy }}
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+
+  - task: ExtractFiles@1
+    displayName: 'Extract ${{parameters.serviceName}} integration test suite '
+    inputs:
+      archiveFilePatterns: '$(System.DefaultWorkingDirectory)/${{ parameters.artifactName }}/${{parameters.serviceName}}-integration-tests.zip'
+      destinationFolder: '$(System.DefaultWorkingDirectory)/${{ parameters.testArtifactLocation }}'
+
+  - template: /devops/tasks/elastic-setup.yml@TemplateRepo
+    parameters:
+      stepName: ExtractElasticHostAndPort
+      providerName: ${{ parameters.providerName }}
+      elasticEndpoint: ${{ parameters.elasticEndpoint }}
+
+  - task: Maven@3
+    displayName: 'Maven build, test, and install ${{parameters.providerName}}-test-core'
+    condition: ne('${{ parameters.testCoreMavenPomFile }}', '')
+    inputs:
+      mavenPomFile: '$(System.DefaultWorkingDirectory)/${{ parameters.testArtifactLocation }}/${{ parameters.testCoreMavenPomFile }}'
+      goals: ${{ parameters.testCoreMavenGoal }}
+      options: ${{ parameters.testCoreMavenOptions }}
+      publishJUnitResults: ${{ parameters.mavenPublishJUnitResults }}
+
+  - template: /devops/tasks/flux-service-wait.yml@TemplateRepo
+    parameters:
+      serviceName: ${{parameters.serviceName}}
+      environment: ${{ parameters.environment }}
+      imageRepoName: '${{ parameters.serviceName }}'
+
+  - task: Maven@3
+    displayName: 'Maven run integration test'
+    condition: and(succeeded(), eq('${{ parameters.skipTest }}', 'false'))
+    inputs:
+      mavenPomFile: '$(System.DefaultWorkingDirectory)/$(MAVEN_INTEGRATION_TEST_POM_FILE_PATH)'
+      goals: ${{ parameters.integrationTestMavenGoal }}
+      options: $(MAVEN_INTEGRATION_TEST_OPTIONS)
+      publishJUnitResults: ${{ parameters.mavenPublishJUnitResults }}
+
+  - template: /devops/tasks/python-integration-test.yml@TemplateRepo
+    parameters:
+      stepName: PythonRunIntegrationTests
+      testPythonFilePath: ${{ parameters.testPythonFilePath }}
+      testPythonFile: ${{ parameters.testPythonFile }}
+      runPythonTest: ${{ parameters.runPythonTest }}

devops/azure/development-pipeline.yml

+#  Copyright © Microsoft Corporation
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+trigger:
+  batch: true
+  branches:
+    include:
+      - master
+  paths:
+    exclude:
+      - /**/*.md
+      - .gitignore
+      - /docs
+      - /provider/schema-aws
+      - /provider/schema-gcp
+      - /provider/schema-ibm
+
+resources:
+  repositories:
+    - repository: FluxRepo
+      type: git
+      name: k8-gitops-manifests
+    - repository: TemplateRepo
+      type: git
+      name: infra-azure-provisioning
+
+variables:
+  - group: 'Azure - OSDU'
+  - group: 'Azure - OSDU Secrets'
+
+  - name: serviceName
+    value: "schema-service"
+  - name: chartPath
+    value: "devops/azure/chart"
+  - name: valuesFile
+    value: "devops/azure/chart/helm-config.yml"
+  - name: 'MANIFEST_REPO'
+    value: $[ resources.repositories['FluxRepo'].name ]
+  - name: 'MAVEN_CACHE_FOLDER'
+    value: $(Pipeline.Workspace)/.m2/repository
+  - name: SKIP_TESTS
+    value: 'false'
+
+stages:
+  - template: /devops/build-stage.yml@TemplateRepo
+    parameters:
+      mavenGoal: 'package'
+      mavenPublishJUnitResults: true
+      serviceCoreMavenOptions: '-P schema-core --settings .mvn/community-maven.settings.xml'
+      mavenOptions: '-P schema-azure --settings .mvn/community-maven.settings.xml -Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
+      copyFileContents: |
+        pom.xml
+        provider/schema-azure/maven/settings.xml
+        provider/schema-azure/pom.xml
+        provider/schema-azure/target/*-spring-boot.jar
+        .mvn/community-maven.settings.xml
+        deployments/**
+      copyFileContentsToFlatten: ''
+      mavenSettingsFile: '.mvn/community-maven.settings.xml'
+      serviceBase: ${{ variables.serviceName }}
+      testingRootFolder: 'testing'
+      chartPath: ${{ variables.chartPath }}
+  - template: deploy-stage.yml
+    parameters:
+      serviceName: ${{ variables.serviceName }}
+      chartPath: ${{ variables.chartPath }}
+      valuesFile: ${{ variables.valuesFile }}
+      testCoreMavenPomFile: 'testing/schema-test-core/pom.xml'
+      testCoreMavenOptions: '--settings $(System.DefaultWorkingDirectory)/drop/.mvn/community-maven.settings.xml -DskipTests -DskipITs'
+      integrationTestMavenGoal: 'verify'
+      skipDeploy: ${{ variables.SKIP_DEPLOY }}
+      skipTest: ${{ variables.SKIP_TESTS }}
+      providers:
+        -  name: Azure
+           environments: ['dev']
+  - template: bootstrap-stage.yml
+    parameters:
+      serviceName: ${{ variables.serviceName }}
+      providers:
+        -  name: Azure
+           environments: ['dev']
\ No newline at end of file

devops/azure/bootstrap.yaml → devops/azure/gitlab-bootstrap.yml

@@ -11,7 +11,6 @@ azure_bootstrap:
       - $AZURE == 'true'
 
 azure_test:
-  needs: ["azure_bootstrap"]
   only:
     variables:
       - $AZURE == 'true'

devops/azure/pipeline.yml

+#  Copyright © Microsoft Corporation
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+trigger:
+  batch: true
+  branches:
+    include:
+      - master
+  paths:
+    exclude:
+      - /**/*.md
+      - .gitignore
+      - /docs
+      - /provider/schema-aws
+      - /provider/schema-gcp
+      - /provider/schema-ibm
+
+resources:
+  repositories:
+    - repository: FluxRepo
+      type: git
+      name: k8-gitops-manifests
+    - repository: TemplateRepo
+      type: git
+      name: infra-azure-provisioning
+
+variables:
+  - group: 'Azure - OSDU'