Commit 05ec184d authored by Daniel Perez's avatar Daniel Perez Committed by Konstantin Khottchenkov
Browse files

Added CI/CD pipeline and updated base e2e test infra for OSDU

parent af5e97cd
# Generated by FOSSA CLI (https://github.com/fossas/fossa-cli)
# Visit https://fossa.com to learn more
version: 2
cli:
server: https://app.fossa.com
fetcher: custom
project: git@community.opengroup.org:osdu/platform/domain-data-mgmt-services/seismic/seismic-dms-suite/seismic-store-service.git
analyze:
modules:
- name: .
type: npm
target: .
path: .
- name: .
type: npm
target: .
path: .
variables:
#seismic store service variables
ENVIRONMENT: cloud
ISGITLAB: "true"
PORT: 80
REPLICA: 1
UTEST_RUNTIME_IMAGE: seistore-svc-runtime
#aws variables
#azure variables
AZURE: 'true'
AZURE_SERVICE: seismic-store-service
#gcp variables
GCP: 'true'
GCP_APPLICATION_NAME: os-seismic-store-service
GCP_DEPLOY_FILE: $OSDU_GCP_DEPLOY_FILE
GCP_PROJECT: opendes-evt
GCP_SERVICE: seismic-store-service
GCP_VENDOR: gcp
#ibm variables
IBM: 'true'
IBM_OPENSHIFT_PROJECT: og-cicd
IBM_VENDOR: ibm
include:
# pipeline logic
- project: "osdu/platform/ci-cd-pipelines"
file: "standard-setup.yml"
# build
- project: "osdu/platform/ci-cd-pipelines"
file: "build/seismic-store-service.yml"
# containerize
- project: "osdu/platform/ci-cd-pipelines"
file: "containerize/seismic-store-service.yml"
# deploy
#azure
- project: "osdu/platform/ci-cd-pipelines"
file: "cloud-providers/azure-seismic-store-service.yml"
#ibm
- project: "osdu/platform/ci-cd-pipelines"
file: "cloud-providers/ibm-seismic-store-service.yml"
#gcp
- project: "osdu/platform/ci-cd-pipelines"
file: "cloud-providers/gcp-seismic-store-service.yml"
\ No newline at end of file
# 3rd-Party Software License Notice
Generated by fossa-cli (https://github.com/fossas/fossa-cli).
This software includes the following software and licenses:
========================================================================
Apache-2.0
========================================================================
The following software have components provided under the terms of this license:
- @google-cloud/datastore (from https://www.npmjs.com/package/@google-cloud/datastore)
- @google-cloud/logging (from https://www.npmjs.com/package/@google-cloud/logging)
- @google-cloud/logging-winston (from https://www.npmjs.com/package/@google-cloud/logging-winston)
- @google-cloud/storage (from https://www.npmjs.com/package/@google-cloud/storage)
- @google-cloud/trace-agent (from https://www.npmjs.com/package/@google-cloud/trace-agent)
- @opencensus/core (from https://www.npmjs.com/package/@opencensus/core)
- @opencensus/exporter-stackdriver (from https://www.npmjs.com/package/@opencensus/exporter-stackdriver)
- jwtproxy (from https://www.npmjs.com/package/jwtproxy)
- request (from https://github.com/request/request#readme)
- typescript (from https://www.npmjs.com/package/typescript)
========================================================================
BSD-2-Clause
========================================================================
The following software have components provided under the terms of this license:
- dotenv (from https://www.npmjs.com/package/dotenv)
- node-pre-gyp (from https://github.com/mapbox/node-pre-gyp)
========================================================================
BSD-3-Clause
========================================================================
The following software have components provided under the terms of this license:
- node-pre-gyp (from https://github.com/mapbox/node-pre-gyp)
- xss-filters (from https://www.npmjs.com/package/xss-filters)
- yamljs (from https://www.npmjs.com/package/yamljs)
========================================================================
CC-BY-4.0
========================================================================
The following software have components provided under the terms of this license:
- typescript (from https://www.npmjs.com/package/typescript)
========================================================================
ISC
========================================================================
The following software have components provided under the terms of this license:
- request-promise (from https://github.com/request/request-promise)
- yargs-parser (from https://github.com/yargs/yargs-parser#readme)
========================================================================
MIT
========================================================================
The following software have components provided under the terms of this license:
- @azure/cosmos (from https://www.npmjs.com/package/@azure/cosmos)
- @azure/identity (from https://www.npmjs.com/package/@azure/identity)
- @azure/keyvault-secrets (from https://www.npmjs.com/package/@azure/keyvault-secrets)
- @azure/storage-blob (from https://www.npmjs.com/package/@azure/storage-blob)
- applicationinsights (from https://www.npmjs.com/package/applicationinsights)
- applicationinsights-native-metrics (from https://www.npmjs.com/package/applicationinsights-native-metrics)
- body-parser (from https://github.com/expressjs/body-parser)
- cors (from https://github.com/expressjs/cors/)
- express (from https://github.com/expressjs/express)
- extend (from https://github.com/justmoon/node-extend#readme)
- jsonwebtoken (from https://github.com/auth0/node-jsonwebtoken#readme)
- lodash (from https://lodash.com/)
- minimist (from https://github.com/substack/minimist)
- mkdirp (from https://github.com/substack/node-mkdirp#readme)
- redis (from https://github.com/NodeRedis/node_redis)
- redlock (from https://www.npmjs.com/package/redlock)
- redlock-async (from https://www.npmjs.com/package/redlock-async)
- replace-in-file (from https://www.npmjs.com/package/replace-in-file)
- request-promise (from https://github.com/request/request-promise)
- typescript (from https://www.npmjs.com/package/typescript)
- winston (from https://github.com/winstonjs/winston#readme)
- yamljs (from https://www.npmjs.com/package/yamljs)
- yargs (from https://github.com/bcoe/yargs#readme)
- yargs-parser (from https://github.com/yargs/yargs-parser#readme)
========================================================================
unknown
========================================================================
The following software have components provided under the terms of this license:
- xss-filters (from https://www.npmjs.com/package/xss-filters)
apiVersion: v2
name: sdms
name: seismic-store-service
appVersion: "latest"
description: Helm Chart for installing sdms service.
version: 0.1.0
......
......@@ -13,6 +13,6 @@ configEnv:
port: #{PORT}#
image:
repository: #{CONTAINER_REGISTRY_NAME}#.azurecr.io/#{utest.runtime.image}#
repository: #{CONTAINER_REGISTRY_NAME}#
branch: master
tag: #{Build.SourceBranchName}#-#{Build.BuildNumber}#
\ No newline at end of file
tag: #{IMAGE_TAG}#
\ No newline at end of file
apiVersion: security.istio.io/v1beta1
kind: AuthorizationPolicy
metadata:
name: {{ .Release.Name }}-jwt-authz
namespace: {{ .Values.global.namespace }}
spec:
selector:
matchLabels:
app: {{ .Release.Name }}
action: DENY
rules:
- from:
- source:
notRequestPrincipals: ["*"]
to:
- operation:
notPaths: ["/","*/index.html",
"*/v2/api-docs",
"*/swagger","*/swagger-resources","*/swagger-ui.html",
"*/actuator/health", "*/health",
"*/configuration/ui","*/configuration/security",
"/seistore-svc/api/v3/swagger-resources/*",
"/seistore-svc/api/v3/*",
"/seistore-svc/api/v3/webjars/*"]
......@@ -5,7 +5,6 @@ metadata:
namespace: {{ .Values.global.namespace }}
data:
CLOUDPROVIDER: {{ .Values.configEnv.cloudProvider }}
KEYVAULT_URL: {{ .Values.configEnv.keyvaultUrl }}
DES_SERVICE_HOST: https://{{ .Values.configEnv.desServiceHost }}
REDIS_INSTANCE_PORT: "{{ .Values.configEnv.redisInstancePort }}"
APP_ENVIRONMENT_IDENTIFIER: {{ .Values.configEnv.appEnvironmentIdentifier }}
......
......@@ -4,7 +4,7 @@ metadata:
name: {{ .Release.Name }}
namespace: {{ .Values.global.namespace }}
labels:
app: {{ .Chart.Name }}
app: {{ .Release.Name }}
spec:
replicas: {{ .Values.global.replicaCount }}
selector:
......@@ -13,11 +13,11 @@ spec:
template:
metadata:
labels:
app: {{ .Chart.Name }}
app: {{ .Release.Name }}
aadpodidbinding: {{ .Values.global.podidentity }}
spec:
containers:
- name: {{ .Chart.Name }}
- name: {{ .Release.Name }}
image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
livenessProbe:
httpGet:
......@@ -42,6 +42,12 @@ spec:
ports:
- protocol: TCP
containerPort: 80
env:
- name: KEYVAULT_URL
valueFrom:
configMapKeyRef:
name: {{ .Values.configEnv.keyvaultUrl }}
key: ENV_KEYVAULT
envFrom:
- configMapRef:
name: seistore-svc-properties
......@@ -4,13 +4,13 @@ metadata:
name: {{ .Release.Name }}
namespace: {{ .Values.global.namespace }}
labels:
app: {{ .Chart.Name }}
app: {{ .Release.Name }}
chart: {{ .Values.image.tag }}
release: {{ .Chart.Name }}
release: {{ .Release.Name }}
spec:
ports:
- protocol: TCP
port: 80
targetPort: 80
selector:
app: {{ .Chart.Name }}
\ No newline at end of file
app: {{ .Release.Name }}
\ No newline at end of file
trigger:
batch: true
branches:
include:
- master
paths:
exclude:
- /**/*.md
- .gitignore
- /docs
resources:
repositories:
- repository: FluxRepo
type: git
name: k8-gitops-manifests
variables:
- group: 'Azure - OSDU'
- name: serviceName
value: "seismic-store-service"
- name: environmentPrefix
value: ""
- name: chartPath
value: "devops/azure/chart"
- name: valuesFile
value: "devops/azure/chart/helm-config.yaml"
- name: 'MANIFEST_REPO'
value: $[ resources.repositories['FluxRepo'].name ]
- name: isGitlab
value: true
stages:
- template: template/build-stage.yml
parameters:
serviceName: ${{ variables.serviceName }}
providers:
- name: Azure
- template: template/deploy-stage.yml
parameters:
serviceName: ${{ variables.serviceName }}
environmentPrefix: ${{ variables.environmentPrefix }}
installRequirements: false
providers:
- name: Azure
environments:
- name: 'dev'
performE2Etests: true
continueIfE2Efails: true
fluxEnabled: true
\ No newline at end of file
......@@ -13,23 +13,24 @@ resources:
repositories:
- repository: FluxRepo
type: git
endpoint: slbswtserviceconnection
name: osdu-delfi/r3-gitops-manifests
ref: trunk
name: k8-gitops-manifests
variables:
- group: 'R3MVP - OSDU'
- group: 'Azure - OSDU'
- name: serviceName
value: "sdms"
value: "seismic-store-service"
- name: environmentPrefix
value: ""
- name: chartPath
value: "devops/azure/chart"
- name: hldRegPath
value: "providers/azure/hld-registry"
- name: valuesFile
value: "devops/azure/chart/helm-config.yaml"
- name: 'MANIFEST_REPO'
value: r3-gitops-manifests
- name: environmentPrefix
value: "r3mvp"
value: $[ resources.repositories['FluxRepo'].name ]
- name: isGitlab
value: true
stages:
......@@ -41,27 +42,12 @@ stages:
- template: template/deploy-stage.yml
parameters:
serviceName: ${{ variables.serviceName }}
chartPath: ${{ variables.chartPath }}
hldRegPath: ${{ variables.hldRegPath }}
manifestRepo: ${{ variables.MANIFEST_REPO }}
environmentPrefix: ${{ variables.environmentPrefix }}
installRequirements: false
providers:
- name: Azure
environments:
- name: 'dev'
fluxEnabled: true
performE2Etests: true
continueIfE2Efails: true
- name: 'qa'
fluxEnabled: true
performE2Etests: true
continueIfE2Efails: false
- name: 'cvx'
fluxEnabled: true
performE2Etests: false
continueIfE2Efails: false
- name: 'prd'
fluxEnabled: true
performE2Etests: false
continueIfE2Efails: false
\ No newline at end of file
fluxEnabled: true
\ No newline at end of file
......@@ -3,10 +3,9 @@ parameters:
providers: []
manifestRepo: ''
environmentPrefix: ""
fluxEnabled: false
chartPath: ""
hldRegPath: ""
fluxEnabled: true
installRequirements: false
isGitlab: false
stages:
......@@ -14,14 +13,14 @@ stages:
- ${{ each environment in provider.environments }}:
- stage: 'Deploy_${{ provider.name }}_${{ environment.name }}'
variables:
- group: 'R3MVP - ${{ provider.name }} Target Env - ${{ environment.name }}'
- group: 'R3MVP - ${{ provider.name }} Service Release - ${{ parameters.serviceName }}'
- group: '${{ provider.name }} Target Env - ${{ environment.name }}'
- group: '${{ provider.name }} Service Release - ${{ parameters.serviceName }}'
jobs:
- deployment: Deploy_Image
pool:
name: $(AGENT_POOL)
vmImage: $(AGENT_IMAGE)
environment: ${{ environment.name }}
environment: ${{ parameters.environmentPrefix }}${{ environment.name }}
strategy:
runOnce:
deploy:
......@@ -51,9 +50,8 @@ stages:
parameters:
serviceName: ${{ parameters.serviceName }}
providerName: ${{ provider.name }}
chartPath: ${{ parameters.chartPath }}
manifestRepo: ${{ parameters.manifestRepo }}
hldRegPath: "${{ parameters.hldRegPath }}"
chartPath: $(chartPath)
hldRegPath: "$(hldRegPath)"
environment: ${{ environment.name }}
fluxEnabled: ${{ environment.fluxEnabled }}
installRequirements: ${{ parameters.installRequirements }}
......@@ -69,4 +67,5 @@ stages:
- template: task/e2e-steps.yml
parameters:
installRequirements: ${{ parameters.installRequirements }}
continueIfE2Efails: ${{ parameters.continueIfE2Efails }}
\ No newline at end of file
continueIfE2Efails: ${{ parameters.continueIfE2Efails }}
isGitlab: ${{ parameters.isGitlab }}
\ No newline at end of file
......@@ -28,7 +28,6 @@ steps:
# self to trigger clone of the current repo.
- checkout: self
persistCredentials: true
# Uncomment if flux enabled
- checkout: FluxRepo
persistCredentials: true
- task: Bash@3
......@@ -92,7 +91,7 @@ steps:
branchName: ${{parameters.environment}}
hldRegPath: ${{parameters.hldRegPath}}
skipDeploy: ${{parameters.skipDeploy}}
manifestRepo: ${{parameters.manifestRepo}}
manifestRepo: ${{ parameters.manifestRepo }}
fluxEnabled: ${{ parameters.fluxEnabled }}
- template: kube.yml
parameters:
......
parameters:
serviceName: ''
providerName: ''
chartPath: ''
chartPath: 'devops/azure/chart'
manifestRepo: ''
hldRegPath: ''
hldRegPath: 'providers/azure/hld-registry'
environment: ''
fluxEnabled: false
installRequirements: false
......
parameters:
installRequirements: false
continueIfE2Efails: false
isGitlab: false
steps:
- template: e2e.yml
parameters:
installRequirements: ${{ parameters.installRequirements }}
continueIfE2Efails: ${{ parameters.continueIfE2Efails }}
\ No newline at end of file
continueIfE2Efails: ${{ parameters.continueIfE2Efails }}
isGitlab: ${{ parameters.isGitlab }}
\ No newline at end of file
parameters:
installRequirements: false
continueIfE2Efails: false
isGitlab: false
steps:
# self to trigger clone of the current repo.
- checkout: self
......@@ -9,9 +10,12 @@ steps:
parameters:
installRequirements: ${{ parameters.installRequirements }}
- template: generate-token.yml
parameters:
isGitlab: ${{ parameters.isGitlab }}
- template: run-e2e.yml
parameters:
continueIfE2Efails: ${{ parameters.continueIfE2Efails }}
isGitlab: ${{ parameters.isGitlab }}
- task: PublishTestResults@1
inputs:
testResultsFormat: 'JUnit'
......
parameters:
isGitlab: false
steps:
- task: Bash@3
displayName: "Generate token"
condition: |
eq(${{ parameters.isGitlab }},true)
env:
AZURE_AD_APP_RESOURCE_ID: $(aad-client-id)
AZURE_AD_TENANT_ID: $(tenant-id)
INTEGRATION_TESTER: $(app-dev-sp-username)
AZURE_TESTER_SERVICEPRINCIPAL_SECRET: $(app-dev-sp-password)
inputs:
targetType: "inline"
script: |
#!/usr/bin/env bash
svctoken=$(python devops/scripts/azure_jwt_client.py)
echo "##vso[task.setvariable variable=e2eIdToken]$svctoken"
- task: Bash@3
displayName: "Generate tokens"
condition: |
eq(${{ parameters.isGitlab }},false)
env:
e2esauthurl: $(e2esauthurl)
sauthSvcApiKey: $(sauthSvcApiKey)
......@@ -21,7 +41,6 @@ steps:
e2esauthTesterSvcAccountSecretSa: $(e2esauthTesterSvcAccountSecretSa)
e2esauthTesterSvcAccountSecretSe: $(e2esauthTesterSvcAccountSecretSe)
e2esauthTesterSvcAccountSecretSv: $(e2esauthTesterSvcAccountSecretSv)
inputs:
targetType: "inline"
script: |
......
......@@ -9,6 +9,22 @@ parameters:
fluxEnabled: false
steps:
- task: AzureCLI@1
name: LoadSP
displayName: 'Load SP'
condition: |
eq(${{ parameters.fluxEnabled }},true)
inputs:
azureSubscription: '$(SERVICE_CONNECTION_NAME)'
addSpnToEnvironment: true
scriptLocation: inlineScript
inlineScript: |
!/usr/bin/env bash
set -euo pipefail
echo "##vso[task.setvariable variable=ARM_CLIENT_ID]${servicePrincipalId}"
echo "##vso[task.setvariable variable=ARM_CLIENT_SECRET]${servicePrincipalKey}"
echo "##vso[task.setvariable variable=ARM_TENANT_ID]${tenantId}"
- task: Bash@3
displayName: "Flux Commit"
condition: |
......@@ -20,13 +36,21 @@ steps:
SERVICE_NAME: ${{parameters.serviceName}}
HLD_REG_PATH: ${{parameters.hldRegPath}}
GENERATION_PATH: ${{parameters.generationPath}}
MANIFEST_REPO: ${{parameters.manifestRepo}}
BASE_NAME_SR: $(base-name-sr)
AZURE_CLIENT_ID: $(ARM_CLIENT_ID)
AZURE_SECRET: $(ARM_CLIENT_SECRET)
AZURE_TENANT: $(ARM_TENANT_ID)
runtimeTag: "$(Build.SourceBranchName)-$(Build.BuildNumber)"
MANIFEST_REPO: ${{ parameters.manifestRepo }}
inputs:
targetType: "inline"
script: |
#!/usr/bin/env bash
set -euo pipefail
az login --service-principal --username $AZURE_CLIENT_ID --password $AZURE_SECRET --tenant $AZURE_TENANT
az aks get-credentials -g $BASE_NAME_SR-rg -n $BASE_NAME_SR-aks
GITOPS_MANIFEST_DIRECTORY=$(Build.SourcesDirectory)/$(Build.Repository.Name)/$CHART_PATH/$GENERATION_PATH
cd $(Build.SourcesDirectory)/$MANIFEST_REPO
......@@ -89,4 +113,5 @@ steps:
git_commit
git_pull
git_push
sleep 10m
sleep 9m
kubectl rollout status deployment.v1.apps/$SERVICE_NAME -n osdu --timeout=900s
......@@ -14,15 +14,19 @@ steps:
CHART_PATH: ${{parameters.chartPath}}
SERVICE_NAME: ${{parameters.serviceName}}
GENERATION_PATH: ${{parameters.generationPath}}
runtimeTag: "$(Build.SourceBranchName)-$(Build.BuildNumber)"
CONTAINER_REGISTRY_NAME: $(CONTAINER_REGISTRY_NAME)
utest_runtime_image: $(utest.runtime.image)
inputs:
targetType: 'inline'
script: |
#!/usr/bin/env bash
cd $(Build.SourcesDirectory)/$(Build.Repository.Name)
mkdir $CHART_PATH/$GENERATION_PATH
cat $(Build.SourcesDirectory)/$(Build.Repository.Name)/$CHART_PATH/values.yaml
echo "Extracting Manifest"
helm template $SERVICE_NAME $CHART_PATH -f $(Build.SourcesDirectory)/$(Build.Repository.Name)/$CHART_PATH/values.yaml --output-dir $CHART_PATH/$GENERATION_PATH
helm template $SERVICE_NAME $CHART_PATH -f $(Build.SourcesDirectory)/$(Build.Repository.Name)/$CHART_PATH/values.yaml --output-dir $CHART_PATH/$GENERATION_PATH --set image.tag=$runtimeTag --set image.repository=$CONTAINER_REGISTRY_NAME.azurecr.io/$utest_runtime_image
cat $CHART_PATH/$GENERATION_PATH/sdms/templates/deployment.yaml
......@@ -46,9 +46,6 @@ steps:
echo "------------------------------------"
az acr login -n $(CONTAINER_REGISTRY_NAME)
previousRuntimeTag=$(az acr repository show-tags -n $(CONTAINER_REGISTRY_NAME) --repository seistore-svc-runtime --top 1 --orderby time_desc | jq .[0] -r)