Added CI/CD pipeline and updated base e2e test infra for OSDU

.fossa.yml

+# Generated by FOSSA CLI (https://github.com/fossas/fossa-cli)
+# Visit https://fossa.com to learn more
+
+version: 2
+cli:
+  server: https://app.fossa.com
+  fetcher: custom
+  project: git@community.opengroup.org:osdu/platform/domain-data-mgmt-services/seismic/seismic-dms-suite/seismic-store-service.git
+analyze:
+  modules:
+  - name: .
+    type: npm
+    target: .
+    path: .
+  - name: .
+    type: npm
+    target: .
+    path: .

.gitlab-ci.yml

+variables:
+  #seismic store service variables
+  ENVIRONMENT: cloud
+  ISGITLAB: "true"
+  PORT: 80
+  REPLICA: 1
+  UTEST_RUNTIME_IMAGE: seistore-svc-runtime
+  #aws variables
+  #azure variables  
+  AZURE: 'true'
+  AZURE_SERVICE: seismic-store-service
+  #gcp variables
+  GCP: 'true'
+  GCP_APPLICATION_NAME: os-seismic-store-service
+  GCP_DEPLOY_FILE: $OSDU_GCP_DEPLOY_FILE
+  GCP_PROJECT: opendes-evt
+  GCP_SERVICE: seismic-store-service
+  GCP_VENDOR: gcp
+  #ibm variables
+  IBM: 'true'
+  IBM_OPENSHIFT_PROJECT: og-cicd
+  IBM_VENDOR: ibm
+
+include: 
+  # pipeline logic
+  - project: "osdu/platform/ci-cd-pipelines"
+    file: "standard-setup.yml"
+  
+  # build
+  - project: "osdu/platform/ci-cd-pipelines"
+    file: "build/seismic-store-service.yml"
+    
+  # containerize
+  - project: "osdu/platform/ci-cd-pipelines"
+    file: "containerize/seismic-store-service.yml"
+    
+  # deploy
+    #azure
+  - project: "osdu/platform/ci-cd-pipelines"
+    file: "cloud-providers/azure-seismic-store-service.yml"
+    
+    #ibm  
+  - project: "osdu/platform/ci-cd-pipelines"
+    file: "cloud-providers/ibm-seismic-store-service.yml"
+    
+    #gcp
+  - project: "osdu/platform/ci-cd-pipelines"
+    file: "cloud-providers/gcp-seismic-store-service.yml"
\ No newline at end of file

NOTICE

+# 3rd-Party Software License Notice
+Generated by fossa-cli (https://github.com/fossas/fossa-cli).
+This software includes the following software and licenses:
+
+========================================================================
+Apache-2.0
+========================================================================
+The following software have components provided under the terms of this license:
+
+- @google-cloud/datastore (from https://www.npmjs.com/package/@google-cloud/datastore)
+- @google-cloud/logging (from https://www.npmjs.com/package/@google-cloud/logging)
+- @google-cloud/logging-winston (from https://www.npmjs.com/package/@google-cloud/logging-winston)
+- @google-cloud/storage (from https://www.npmjs.com/package/@google-cloud/storage)
+- @google-cloud/trace-agent (from https://www.npmjs.com/package/@google-cloud/trace-agent)
+- @opencensus/core (from https://www.npmjs.com/package/@opencensus/core)
+- @opencensus/exporter-stackdriver (from https://www.npmjs.com/package/@opencensus/exporter-stackdriver)
+- jwtproxy (from https://www.npmjs.com/package/jwtproxy)
+- request (from https://github.com/request/request#readme)
+- typescript (from https://www.npmjs.com/package/typescript)
+
+========================================================================
+BSD-2-Clause
+========================================================================
+The following software have components provided under the terms of this license:
+
+- dotenv (from https://www.npmjs.com/package/dotenv)
+- node-pre-gyp (from https://github.com/mapbox/node-pre-gyp)
+
+========================================================================
+BSD-3-Clause
+========================================================================
+The following software have components provided under the terms of this license:
+
+- node-pre-gyp (from https://github.com/mapbox/node-pre-gyp)
+- xss-filters (from https://www.npmjs.com/package/xss-filters)
+- yamljs (from https://www.npmjs.com/package/yamljs)
+
+========================================================================
+CC-BY-4.0
+========================================================================
+The following software have components provided under the terms of this license:
+
+- typescript (from https://www.npmjs.com/package/typescript)
+
+========================================================================
+ISC
+========================================================================
+The following software have components provided under the terms of this license:
+
+- request-promise (from https://github.com/request/request-promise)
+- yargs-parser (from https://github.com/yargs/yargs-parser#readme)
+
+========================================================================
+MIT
+========================================================================
+The following software have components provided under the terms of this license:
+
+- @azure/cosmos (from https://www.npmjs.com/package/@azure/cosmos)
+- @azure/identity (from https://www.npmjs.com/package/@azure/identity)
+- @azure/keyvault-secrets (from https://www.npmjs.com/package/@azure/keyvault-secrets)
+- @azure/storage-blob (from https://www.npmjs.com/package/@azure/storage-blob)
+- applicationinsights (from https://www.npmjs.com/package/applicationinsights)
+- applicationinsights-native-metrics (from https://www.npmjs.com/package/applicationinsights-native-metrics)
+- body-parser (from https://github.com/expressjs/body-parser)
+- cors (from https://github.com/expressjs/cors/)
+- express (from https://github.com/expressjs/express)
+- extend (from https://github.com/justmoon/node-extend#readme)
+- jsonwebtoken (from https://github.com/auth0/node-jsonwebtoken#readme)
+- lodash (from https://lodash.com/)
+- minimist (from https://github.com/substack/minimist)
+- mkdirp (from https://github.com/substack/node-mkdirp#readme)
+- redis (from https://github.com/NodeRedis/node_redis)
+- redlock (from https://www.npmjs.com/package/redlock)
+- redlock-async (from https://www.npmjs.com/package/redlock-async)
+- replace-in-file (from https://www.npmjs.com/package/replace-in-file)
+- request-promise (from https://github.com/request/request-promise)
+- typescript (from https://www.npmjs.com/package/typescript)
+- winston (from https://github.com/winstonjs/winston#readme)
+- yamljs (from https://www.npmjs.com/package/yamljs)
+- yargs (from https://github.com/bcoe/yargs#readme)
+- yargs-parser (from https://github.com/yargs/yargs-parser#readme)
+
+========================================================================
+unknown
+========================================================================
+The following software have components provided under the terms of this license:
+
+- xss-filters (from https://www.npmjs.com/package/xss-filters)
+
+

devops/azure/chart/Chart.yaml

 apiVersion: v2
-name: sdms
+name: seismic-store-service
 appVersion: "latest"
 description: Helm Chart for installing sdms service.
 version: 0.1.0

devops/azure/chart/helm-config.yaml

@@ -13,6 +13,6 @@ configEnv:
   port: #{PORT}#
 
 image:
-  repository: #{CONTAINER_REGISTRY_NAME}#.azurecr.io/#{utest.runtime.image}#
+  repository: #{CONTAINER_REGISTRY_NAME}#
   branch: master
-  tag: #{Build.SourceBranchName}#-#{Build.BuildNumber}#
\ No newline at end of file
+  tag: #{IMAGE_TAG}#
\ No newline at end of file

devops/azure/chart/templates/azure-istio-auth-policy.yaml

+apiVersion: security.istio.io/v1beta1
+kind: AuthorizationPolicy
+metadata:
+  name: {{ .Release.Name }}-jwt-authz
+  namespace: {{ .Values.global.namespace }}
+spec:
+  selector:
+    matchLabels:
+      app: {{ .Release.Name }}
+  action: DENY
+  rules:
+    - from:
+        - source:
+            notRequestPrincipals: ["*"]
+      to:
+        - operation:
+            notPaths: ["/","*/index.html",
+                       "*/v2/api-docs",
+                       "*/swagger","*/swagger-resources","*/swagger-ui.html",
+                       "*/actuator/health", "*/health",
+                       "*/configuration/ui","*/configuration/security",
+                       "/seistore-svc/api/v3/swagger-resources/*",
+                       "/seistore-svc/api/v3/*",
+                       "/seistore-svc/api/v3/webjars/*"]

devops/azure/chart/templates/configmap.yaml

@@ -5,7 +5,6 @@ metadata:
   namespace: {{ .Values.global.namespace }}
 data:
   CLOUDPROVIDER: {{ .Values.configEnv.cloudProvider }}
-  KEYVAULT_URL: {{ .Values.configEnv.keyvaultUrl }}
   DES_SERVICE_HOST: https://{{ .Values.configEnv.desServiceHost }}
   REDIS_INSTANCE_PORT: "{{ .Values.configEnv.redisInstancePort }}"
   APP_ENVIRONMENT_IDENTIFIER: {{ .Values.configEnv.appEnvironmentIdentifier }}

devops/azure/chart/templates/deployment.yaml

@@ -4,7 +4,7 @@ metadata:
   name: {{ .Release.Name }}
   namespace: {{ .Values.global.namespace }}
   labels:
-    app: {{ .Chart.Name }}
+    app: {{ .Release.Name }}
 spec:
   replicas: {{ .Values.global.replicaCount }}
   selector:
@@ -13,11 +13,11 @@ spec:
   template:
     metadata:
       labels:
-        app: {{ .Chart.Name }}
+        app: {{ .Release.Name }}
         aadpodidbinding: {{ .Values.global.podidentity }}
     spec:
       containers:
-        - name: {{ .Chart.Name }}
+        - name: {{ .Release.Name }}
           image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
           livenessProbe:
             httpGet:
@@ -42,6 +42,12 @@ spec:
           ports:
             - protocol: TCP
               containerPort: 80
+          env:
+          - name: KEYVAULT_URL
+            valueFrom:
+              configMapKeyRef:
+                name: {{ .Values.configEnv.keyvaultUrl }}
+                key: ENV_KEYVAULT
           envFrom:
             - configMapRef:
                 name: seistore-svc-properties

devops/azure/chart/templates/service.yaml

@@ -4,13 +4,13 @@ metadata:
   name: {{ .Release.Name }}
   namespace: {{ .Values.global.namespace }}
   labels:
-    app: {{ .Chart.Name }}
+    app: {{ .Release.Name }}
     chart: {{ .Values.image.tag }}
-    release: {{ .Chart.Name }}
+    release: {{ .Release.Name }}
 spec:
   ports:
   - protocol: TCP
     port: 80
     targetPort: 80
   selector:
-    app: {{ .Chart.Name }}
\ No newline at end of file
+    app: {{ .Release.Name }}
\ No newline at end of file

devops/azure/development-pipeline.yml

+trigger:
+  batch: true
+  branches:
+    include:
+      - master
+  paths:
+    exclude:
+      - /**/*.md
+      - .gitignore
+      - /docs
+
+resources:
+  repositories:
+  - repository: FluxRepo
+    type: git
+    name: k8-gitops-manifests
+
+variables:
+  - group: 'Azure - OSDU'
+  - name: serviceName
+    value: "seismic-store-service"
+  - name: environmentPrefix
+    value: ""
+  - name: chartPath
+    value: "devops/azure/chart"
+  - name: valuesFile
+    value: "devops/azure/chart/helm-config.yaml"
+  - name: 'MANIFEST_REPO'
+    value: $[ resources.repositories['FluxRepo'].name ]
+  - name: isGitlab
+    value: true
+    
+stages:
+
+  - template: template/build-stage.yml
+    parameters:
+      serviceName: ${{ variables.serviceName }}
+      providers:
+        -  name: Azure
+  - template: template/deploy-stage.yml
+    parameters:
+      serviceName: ${{ variables.serviceName }}
+      environmentPrefix: ${{ variables.environmentPrefix }}
+      installRequirements: false
+      providers:
+        -  name: Azure
+           environments:
+             - name: 'dev'
+               performE2Etests: true
+               continueIfE2Efails: true
+               fluxEnabled: true
\ No newline at end of file

devops/azure/pipeline.yml

@@ -13,23 +13,24 @@ resources:
   repositories:
   - repository: FluxRepo
     type: git
-    endpoint: slbswtserviceconnection
-    name: osdu-delfi/r3-gitops-manifests
-    ref: trunk
+    name: k8-gitops-manifests
+
 
 variables:
-  - group: 'R3MVP - OSDU'
-  
+  - group: 'Azure - OSDU'
   - name: serviceName
-    value: "sdms"
+    value: "seismic-store-service"
+  - name: environmentPrefix
+    value: ""
   - name: chartPath
     value: "devops/azure/chart"
-  - name: hldRegPath
-    value: "providers/azure/hld-registry"
+  - name: valuesFile
+    value: "devops/azure/chart/helm-config.yaml"
   - name: 'MANIFEST_REPO'
-    value: r3-gitops-manifests
-  - name: environmentPrefix
-    value: "r3mvp"
+    value: $[ resources.repositories['FluxRepo'].name ]
+  - name: isGitlab
+    value: true
+
     
 stages:
 
@@ -41,27 +42,12 @@ stages:
   - template: template/deploy-stage.yml
     parameters:
       serviceName: ${{ variables.serviceName }}
-      chartPath: ${{ variables.chartPath }}
-      hldRegPath: ${{ variables.hldRegPath }}
-      manifestRepo: ${{ variables.MANIFEST_REPO }}
       environmentPrefix: ${{ variables.environmentPrefix }}
       installRequirements: false
       providers:
         -  name: Azure
            environments:
              - name: 'dev'
-               fluxEnabled: true
-               performE2Etests: true
-               continueIfE2Efails: true
-             - name: 'qa'
-               fluxEnabled: true
-               performE2Etests: true
-               continueIfE2Efails: false
-             - name: 'cvx'
-               fluxEnabled: true
                performE2Etests: false
                continueIfE2Efails: false
-             - name: 'prd'
-               fluxEnabled: true
-               performE2Etests: false
-               continueIfE2Efails: false
\ No newline at end of file
+               fluxEnabled: true
\ No newline at end of file

devops/azure/template/deploy-stage.yml

@@ -3,10 +3,9 @@ parameters:
   providers: []
   manifestRepo: ''
   environmentPrefix: ""
-  fluxEnabled: false
-  chartPath: ""
-  hldRegPath: ""
+  fluxEnabled: true
   installRequirements: false
+  isGitlab: false
 
 stages:
 
@@ -14,14 +13,14 @@ stages:
   - ${{ each environment in provider.environments }}:
     - stage: 'Deploy_${{ provider.name }}_${{ environment.name }}'
       variables:
-      - group: 'R3MVP - ${{ provider.name }} Target Env - ${{ environment.name }}'
-      - group: 'R3MVP - ${{ provider.name }} Service Release - ${{ parameters.serviceName }}'
+      - group: '${{ provider.name }} Target Env - ${{ environment.name }}'
+      - group: '${{ provider.name }} Service Release - ${{ parameters.serviceName }}'
       jobs:
       - deployment: Deploy_Image
         pool:
           name: $(AGENT_POOL)
           vmImage: $(AGENT_IMAGE)
-        environment: ${{ environment.name }}
+        environment: ${{ parameters.environmentPrefix }}${{ environment.name }}
         strategy:
           runOnce:
             deploy:
@@ -51,9 +50,8 @@ stages:
                 parameters:
                   serviceName: ${{ parameters.serviceName }}
                   providerName: ${{ provider.name }}
-                  chartPath: ${{ parameters.chartPath }}
-                  manifestRepo: ${{ parameters.manifestRepo }}
-                  hldRegPath: "${{ parameters.hldRegPath }}"
+                  chartPath: $(chartPath)
+                  hldRegPath: "$(hldRegPath)"
                   environment: ${{ environment.name }}
                   fluxEnabled: ${{ environment.fluxEnabled }}
                   installRequirements: ${{ parameters.installRequirements }}
@@ -69,4 +67,5 @@ stages:
         - template: task/e2e-steps.yml
           parameters:
             installRequirements: ${{ parameters.installRequirements }}
-            continueIfE2Efails: ${{ parameters.continueIfE2Efails }}
\ No newline at end of file
+            continueIfE2Efails: ${{ parameters.continueIfE2Efails }}
+            isGitlab: ${{ parameters.isGitlab }}
\ No newline at end of file

devops/azure/template/task/aks-deployment-steps.yml

@@ -28,7 +28,6 @@ steps:
     # self to trigger clone of the current repo.
     - checkout: self
       persistCredentials: true
-    # Uncomment if flux enabled
     - checkout: FluxRepo
       persistCredentials: true
     - task: Bash@3
@@ -92,7 +91,7 @@ steps:
         branchName: ${{parameters.environment}}
         hldRegPath: ${{parameters.hldRegPath}}
         skipDeploy: ${{parameters.skipDeploy}}
-        manifestRepo: ${{parameters.manifestRepo}}
+        manifestRepo: ${{ parameters.manifestRepo }}
         fluxEnabled: ${{ parameters.fluxEnabled }}
     - template: kube.yml
       parameters:

devops/azure/template/task/deployment-helm-steps.yml

 parameters:
   serviceName: ''
   providerName: ''
-  chartPath: ''
+  chartPath: 'devops/azure/chart'
   manifestRepo: ''
-  hldRegPath: ''
+  hldRegPath: 'providers/azure/hld-registry'
   environment: ''
   fluxEnabled: false
   installRequirements: false

devops/azure/template/task/e2e-steps.yml

 parameters:
   installRequirements: false
   continueIfE2Efails: false
+  isGitlab: false
 steps:
   - template: e2e.yml
     parameters:
       installRequirements: ${{ parameters.installRequirements }}
-      continueIfE2Efails: ${{ parameters.continueIfE2Efails }}
\ No newline at end of file
+      continueIfE2Efails: ${{ parameters.continueIfE2Efails }}
+      isGitlab: ${{ parameters.isGitlab }}
\ No newline at end of file

devops/azure/template/task/e2e.yml

 parameters:
   installRequirements: false
   continueIfE2Efails: false
+  isGitlab: false
 steps:
     # self to trigger clone of the current repo.
     - checkout: self
@@ -9,9 +10,12 @@ steps:
       parameters:
         installRequirements: ${{ parameters.installRequirements }}
     - template: generate-token.yml
+      parameters:
+        isGitlab: ${{ parameters.isGitlab }}
     - template: run-e2e.yml
       parameters:
         continueIfE2Efails: ${{ parameters.continueIfE2Efails }}
+        isGitlab: ${{ parameters.isGitlab }}
     - task: PublishTestResults@1
       inputs:
         testResultsFormat: 'JUnit'

devops/azure/template/task/generate-token.yml

+parameters:
+  isGitlab: false
 steps:
+    - task: Bash@3
+      displayName: "Generate token"
+      condition: |
+        eq(${{ parameters.isGitlab }},true)
+      env:
+        AZURE_AD_APP_RESOURCE_ID: $(aad-client-id)
+        AZURE_AD_TENANT_ID: $(tenant-id)
+        INTEGRATION_TESTER: $(app-dev-sp-username)
+        AZURE_TESTER_SERVICEPRINCIPAL_SECRET: $(app-dev-sp-password)
+      inputs:
+        targetType: "inline"
+        script: |
+          #!/usr/bin/env bash
+
+          svctoken=$(python devops/scripts/azure_jwt_client.py)
+          echo "##vso[task.setvariable variable=e2eIdToken]$svctoken"
     - task: Bash@3
       displayName: "Generate tokens"
+      condition: |
+        eq(${{ parameters.isGitlab }},false)
       env:
         e2esauthurl: $(e2esauthurl)
         sauthSvcApiKey: $(sauthSvcApiKey)
@@ -21,7 +41,6 @@ steps:
         e2esauthTesterSvcAccountSecretSa: $(e2esauthTesterSvcAccountSecretSa)
         e2esauthTesterSvcAccountSecretSe: $(e2esauthTesterSvcAccountSecretSe)
         e2esauthTesterSvcAccountSecretSv: $(e2esauthTesterSvcAccountSecretSv)
-        
       inputs:
         targetType: "inline"
         script: |

devops/azure/template/task/gitops.yml

@@ -9,6 +9,22 @@ parameters:
   fluxEnabled: false
 
 steps:
+  - task: AzureCLI@1
+    name: LoadSP
+    displayName: 'Load SP'
+    condition: |
+      eq(${{ parameters.fluxEnabled }},true)
+    inputs:
+      azureSubscription: '$(SERVICE_CONNECTION_NAME)'
+      addSpnToEnvironment: true
+      scriptLocation: inlineScript
+      inlineScript: |
+       !/usr/bin/env bash
+        set -euo pipefail
+        
+        echo "##vso[task.setvariable variable=ARM_CLIENT_ID]${servicePrincipalId}"
+        echo "##vso[task.setvariable variable=ARM_CLIENT_SECRET]${servicePrincipalKey}"
+        echo "##vso[task.setvariable variable=ARM_TENANT_ID]${tenantId}"
   - task: Bash@3
     displayName: "Flux Commit"
     condition: |
@@ -20,13 +36,21 @@ steps:
       SERVICE_NAME: ${{parameters.serviceName}}
       HLD_REG_PATH: ${{parameters.hldRegPath}}
       GENERATION_PATH: ${{parameters.generationPath}}
-      MANIFEST_REPO: ${{parameters.manifestRepo}}
+      BASE_NAME_SR: $(base-name-sr)
+      AZURE_CLIENT_ID: $(ARM_CLIENT_ID)
+      AZURE_SECRET: $(ARM_CLIENT_SECRET)
+      AZURE_TENANT: $(ARM_TENANT_ID)
+      runtimeTag: "$(Build.SourceBranchName)-$(Build.BuildNumber)"
+      MANIFEST_REPO: ${{ parameters.manifestRepo }}
     inputs:
       targetType: "inline"
       script: |
         #!/usr/bin/env bash
         set -euo pipefail
 
+        az login --service-principal --username $AZURE_CLIENT_ID --password $AZURE_SECRET --tenant $AZURE_TENANT
+        az aks get-credentials -g $BASE_NAME_SR-rg -n $BASE_NAME_SR-aks
+
         GITOPS_MANIFEST_DIRECTORY=$(Build.SourcesDirectory)/$(Build.Repository.Name)/$CHART_PATH/$GENERATION_PATH
         cd $(Build.SourcesDirectory)/$MANIFEST_REPO
 
@@ -89,4 +113,5 @@ steps:
         git_commit
         git_pull
         git_push
-        sleep 10m
+        sleep 9m
+        kubectl rollout status deployment.v1.apps/$SERVICE_NAME -n osdu --timeout=900s

devops/azure/template/task/helm-template.yml

@@ -14,15 +14,19 @@ steps:
       CHART_PATH: ${{parameters.chartPath}}
       SERVICE_NAME: ${{parameters.serviceName}}
       GENERATION_PATH: ${{parameters.generationPath}}
+      runtimeTag: "$(Build.SourceBranchName)-$(Build.BuildNumber)"
+      CONTAINER_REGISTRY_NAME: $(CONTAINER_REGISTRY_NAME)
+      utest_runtime_image: $(utest.runtime.image)
     inputs:
       targetType: 'inline'
       script: |
         #!/usr/bin/env bash
-
+        
           cd $(Build.SourcesDirectory)/$(Build.Repository.Name)
-
+        
           mkdir $CHART_PATH/$GENERATION_PATH
           cat $(Build.SourcesDirectory)/$(Build.Repository.Name)/$CHART_PATH/values.yaml
 
           echo "Extracting Manifest"
-          helm template $SERVICE_NAME $CHART_PATH -f $(Build.SourcesDirectory)/$(Build.Repository.Name)/$CHART_PATH/values.yaml --output-dir $CHART_PATH/$GENERATION_PATH
+          helm template $SERVICE_NAME $CHART_PATH -f $(Build.SourcesDirectory)/$(Build.Repository.Name)/$CHART_PATH/values.yaml --output-dir $CHART_PATH/$GENERATION_PATH --set image.tag=$runtimeTag --set image.repository=$CONTAINER_REGISTRY_NAME.azurecr.io/$utest_runtime_image
+          cat $CHART_PATH/$GENERATION_PATH/sdms/templates/deployment.yaml

devops/azure/template/task/push-runtime-image.yml

@@ -46,9 +46,6 @@ steps:
         echo "------------------------------------"
         az acr login -n $(CONTAINER_REGISTRY_NAME)
 
-        previousRuntimeTag=$(az acr repository show-tags -n $(CONTAINER_REGISTRY_NAME) --repository seistore-svc-runtime --top 1 --orderby time_desc | jq .[0] -r)