Migrate Wellbore to common gke pipeline [GONRG-2850]

.gitlab-ci.yml

@@ -27,14 +27,14 @@ variables:
   AWS_TEST_SUBDIR: tests/aws-test/build-aws
   AWS_INT_TEST_TYPE: python
 
-  OSDU_GCP_PROJECT_NAME: nice-etching-277309
-  OSDU_GCP_CLUSTER: asm-primary
-  OSDU_GCP_ZONE: us-central1-c
   OSDU_GCP_SERVICE: wellbore
-  OSDU_GCP_HELM_DEPLOYMENT_DIR: devops/gcp/osdu-helm
-  OSDU_GCP_SERVICE_PATH: /api/os-wellbore-ddms
-  OSDU_GCP_HEALTH_PATH: /api/os-wellbore-ddms/healthz
-
+  OSDU_GCP_VENDOR: gcp
+  OSDU_GCP_HELM_PACKAGE_CHARTS: "devops/gcp/deploy devops/gcp/configmap"
+  OSDU_GCP_HELM_CONFIG_SERVICE_VARS: "--set data.os_wellbore_ddms_data_project_id=$OSDU_GCP_PROJECT --set data.service_host_search=$OSDU_GCP_SERVICE_HOST_SEARCH --set data.sa_key=$OSDU_GCP_INTEGRATION_TESTER"
+  OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS: "--set data.image=$CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA"
+  OSDU_GCP_HELM_CONFIG_SERVICE: wellbore-config
+  OSDU_GCP_HELM_DEPLOYMENT_SERVICE: wellbore-deploy
+  OSDU_GCP_INT_TEST_TYPE: python
 
 include:
   - project: "osdu/platform/ci-cd-pipelines"
@@ -50,7 +50,7 @@ include:
     file: "scanners/gitlab-ultimate.yml"
 
   - project: "osdu/platform/ci-cd-pipelines"
-    file: "cloud-providers/osdu-gcp-stateful-k8s.yml"
+    file: "cloud-providers/osdu-gcp-gke.yml"
 
   - project: "osdu/platform/ci-cd-pipelines"
     file: "cloud-providers/ibm-wellbore.yml"
@@ -92,7 +92,7 @@ containerize:
     - docker push $IMAGE_TAG
     - docker push $CI_REGISTRY_IMAGE:latest
 
-osdu-gcp-containerize:
+osdu-gcp-containerize-gitlab:
   stage: containerize
   image: docker:19.03
   cache: {}
@@ -101,8 +101,8 @@ osdu-gcp-containerize:
     variables:
       - $OSDU_GCP == 'true'
   variables:
-    IMAGE_TAG: $CI_REGISTRY_IMAGE/osdu-gcp-$OSDU_GCP_SERVICE:$CI_COMMIT_SHA
-    IMAGE_TAG_COMMUNITY: $CI_REGISTRY_IMAGE/osdu-gcp-$OSDU_GCP_SERVICE:gcp-community
+    IMAGE_TAG: $CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA
+    IMAGE_TAG_LATEST: $CI_REGISTRY_IMAGE/osdu-gcp:latest
   script:
     - echo ---- ---- ---- SYSTEM DEPENDENCIES ---- ---- ----
     - apk update
@@ -119,11 +119,11 @@ osdu-gcp-containerize:
     - commit_branch=$commit_id
     - echo $commit_branch
     - echo ---- ---- BUILD IMAGE
-    - docker build -t $IMAGE_TAG -t $IMAGE_TAG_COMMUNITY -t=$CI_REGISTRY_IMAGE:latest --rm . -f ./build/Dockerfile --build-arg PIP_WHEEL_DIR=python-packages --build-arg build_date="$current_utc_date" --build-arg build_number=$commit_id --build-arg commit_id=$commit_id --build-arg build_origin="Gitlab" --build-arg commit_branch=$commit_branch
+    - docker build -t $IMAGE_TAG -t $IMAGE_TAG_LATEST --rm . -f ./build/Dockerfile --build-arg PIP_WHEEL_DIR=python-packages --build-arg build_date="$current_utc_date" --build-arg build_number=$commit_id --build-arg commit_id=$commit_id --build-arg build_origin="Gitlab" --build-arg commit_branch=$commit_branch
     - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
     - echo ---- ---- PUSH IMAGE
     - docker push $IMAGE_TAG
-    - docker push $IMAGE_TAG_COMMUNITY
+    - docker push $IMAGE_TAG_LATEST
 
 osdu-gcp-containerize-for-release:
   stage: containerize
@@ -132,10 +132,9 @@ osdu-gcp-containerize-for-release:
   tags: ["osdu-medium"]
   only:
     variables:
-      - $OSDU_GCP == 'true' && $RELEASE =~ /(v)(\d+\.)(\d+\.)\w+/i
+      - $OSDU_GCP == 'true' && $CI_COMMIT_TAG != null
   variables:
-    IMAGE_TAG: $CI_REGISTRY_IMAGE/osdu-gcp-$OSDU_GCP_SERVICE:gcp-$RELEASE
-    IMAGE_TAG_RELEASE: $CI_REGISTRY_IMAGE/osdu-gcp-$OSDU_GCP_SERVICE:gcp-release
+    IMAGE_TAG_RELEASE: $CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_TAG
   script:
     - echo ---- ---- ---- SYSTEM DEPENDENCIES ---- ---- ----
     - apk update
@@ -152,16 +151,16 @@ osdu-gcp-containerize-for-release:
     - commit_branch=$commit_id
     - echo $commit_branch
     - echo ---- ---- BUILD IMAGE
-    - docker build -t $IMAGE_TAG -t $IMAGE_TAG_RELEASE -t=$CI_REGISTRY_IMAGE:latest --rm . -f ./build/Dockerfile --build-arg PIP_WHEEL_DIR=python-packages --build-arg build_date="$current_utc_date" --build-arg build_number=$commit_id --build-arg commit_id=$commit_id --build-arg build_origin="Gitlab" --build-arg commit_branch=$commit_branch
+    - docker build -t $IMAGE_TAG_RELEASE --rm . -f ./build/Dockerfile --build-arg PIP_WHEEL_DIR=python-packages --build-arg build_date="$current_utc_date" --build-arg build_number=$commit_id --build-arg commit_id=$commit_id --build-arg build_origin="Gitlab" --build-arg commit_branch=$commit_branch
     - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
     - echo ---- ---- PUSH IMAGE
-    - docker push $IMAGE_TAG
     - docker push $IMAGE_TAG_RELEASE
 
 osdu-gcp-test-python:
+  extends: .osdu-gcp-variables
   stage: integration
   image: gcr.io/google.com/cloudsdktool/cloud-sdk
-  needs: ['osdu-gcp-deploy']
+  needs: ["osdu-gcp-deploy-deployment"]
   only:
     variables:
       - $OSDU_GCP == 'true' && $OSDU_GCP_INT_TEST_TYPE == 'python'
@@ -178,17 +177,21 @@ osdu-gcp-test-python:
     - cd tests/integration
     - echo $OSDU_GCP_INTEGRATION_TESTER | base64 -d > file.json
     - gcloud auth activate-service-account --key-file file.json
-    - gcloud config set project $OSDU_GCP_PROJECT_NAME
+    - gcloud config set project $OSDU_GCP_PROJECT
     - >
       python gen_postman_env.py
       --token $(gcloud auth print-access-token)
-      --base_url ${OSDU_GCP_URL}${OSDU_GCP_SERVICE_PATH}
+      --base_url $OSDU_GCP_WELLBORE_BASE_URL
       --cloud_provider $OSDU_GCP_VENDOR
       --data_partition $OSDU_GCP_TENANT
       --acl_domain $DOMAIN
-      --legal_tag $OSDU_GCP_LEGAL_TAG
+      --legal_tag $LEGAL_TAG
     - pytest ./functional --environment="./generated/postman_environment.json" --filter-tag=basic
 
+# Disable maven job in gcp common gke pipeline
+osdu-gcp-test:
+  extends:
+    - .osdu-gcp-variables
 
 # Allow failure on deployments
 

devops/gcp/.helmignore

-# Patterns to ignore when building packages.
-# This supports shell glob matching, relative path matching, and
-# negation (prefixed with !). Only one pattern per line.
-.DS_Store
-# Common VCS dirs
-.git/
-.gitignore
-.bzr/
-.bzrignore
-.hg/
-.hgignore
-.svn/
-# Common backup files
-*.swp
-*.bak
-*.tmp
-*.orig
-*~
-# Various IDEs
-.project
-.idea/
-*.tmproj
-.vscode/

devops/gcp/Chart.yaml

-apiVersion: v2
-name: gcp
-description: A Helm chart for Kubernetes
-
-# A chart can be either an 'application' or a 'library' chart.
-#
-# Application charts are a collection of templates that can be packaged into versioned archives
-# to be deployed.
-#
-# Library charts provide useful utilities or functions for the chart developer. They're included as
-# a dependency of application charts to inject those utilities and functions into the rendering
-# pipeline. Library charts do not define any templates and therefore cannot be deployed.
-type: application
-
-# This is the chart version. This version number should be incremented each time you make changes
-# to the chart and its templates, including the app version.
-# Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.0
-
-# This is the version number of the application being deployed. This version number should be
-# incremented each time you make changes to the application. Versions are not expected to
-# follow Semantic Versioning. They should reflect the version the application is using.
-# It is recommended to use it with quotes.
-appVersion: "1.16.0"

devops/osdu-gcp/configmap/values.yaml → devops/gcp/configmap/values.yaml

@@ -3,10 +3,10 @@
 # Declare variables to be passed into your templates.
 
 data:
-  os_wellbore_ddms_data_project_id: "nice-etching-277309"
+  os_wellbore_ddms_data_project_id: ""
   os_wellbore_ddms_data_project_credentials: "/tmp/key/key.json"
-  service_host_search: ""
-  service_host_storage: ""
+  service_host_search: "http://search/api/search"
+  service_host_storage: "http://storage/api/storage"
   sa_key: ""
 
 conf:

devops/osdu-gcp/deploy/templates/deployment.yaml → devops/gcp/deploy/templates/deployment.yaml

@@ -51,4 +51,3 @@ spec:
             limits:
               cpu: "{{ .Values.data.limits_cpu }}"
               memory: "{{ .Values.data.limits_memory }}"
-

devops/osdu-gcp/deploy/values.yaml → devops/gcp/deploy/values.yaml

-# Default values for wellbore-configmap.
+# Default values for wellbore-deploy.
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 

devops/gcp/infra/os-infra-configmap-community.yaml

-kind: ConfigMap
-apiVersion: v1
-data:
-  # System sevice variables
-  CLOUD_PROVIDER: gcp
-  OS_WELLBORE_DDMS_DATA_PROJECT_ID: nice-etching-277309
-  SERVICE_HOST_SEARCH: https://os-search-attcrcktoa-uc.a.run.app/api/search
-  SERVICE_HOST_STORAGE: https://os-storage-attcrcktoa-uc.a.run.app/api/storage
-  # Custom variables
-  IMAGE_NAME: community.opengroup.org:5555/osdu/platform/domain-data-mgmt-services/wellbore/wellbore-domain-services/osdu-gcp-wellbore-ddms
-  IMAGE_TAG: gcp-community
-  HEALTH_URL: /api/os-wellbore-ddms/healthz
-  API_URL: /api/os-wellbore-ddms
-  SERVICE_NAME: wellbore
-  TENANT: osdu
-  LEGAL_TAG: osdu-demo-legaltag
-  DEPLOYMENT_DIR: devops/gcp/osdu-helm
-
-metadata:
-  name: wellbore
-  namespace: osdu
----
-kind: Secret
-apiVersion: v1
-metadata:
-  name: wellbore
-  namespace: osdu
-type: Opaque
-data:
-  "key.json": ""
\ No newline at end of file

devops/gcp/osdu-helm/.helmignore

-# Patterns to ignore when building packages.
-# This supports shell glob matching, relative path matching, and
-# negation (prefixed with !). Only one pattern per line.
-.DS_Store
-# Common VCS dirs
-.git/
-.gitignore
-.bzr/
-.bzrignore
-.hg/
-.hgignore
-.svn/
-# Common backup files
-*.swp
-*.bak
-*.tmp
-*.orig
-*~
-# Various IDEs
-.project
-.idea/
-*.tmproj
-.vscode/

devops/gcp/osdu-helm/Chart.yaml

-apiVersion: v2
-name: gcp
-description: A Helm chart for Kubernetes
-
-# A chart can be either an 'application' or a 'library' chart.
-#
-# Application charts are a collection of templates that can be packaged into versioned archives
-# to be deployed.
-#
-# Library charts provide useful utilities or functions for the chart developer. They're included as
-# a dependency of application charts to inject those utilities and functions into the rendering
-# pipeline. Library charts do not define any templates and therefore cannot be deployed.
-type: application
-
-# This is the chart version. This version number should be incremented each time you make changes
-# to the chart and its templates, including the app version.
-# Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.0
-
-# This is the version number of the application being deployed. This version number should be
-# incremented each time you make changes to the application. Versions are not expected to
-# follow Semantic Versioning. They should reflect the version the application is using.
-# It is recommended to use it with quotes.
-appVersion: "1.16.0"

devops/gcp/osdu-helm/templates/_helpers.tpl

-{{/*
-Expand the name of the chart.
-*/}}
-{{- define "gcp.name" -}}
-{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
-{{- end }}
-
-{{/*
-Create a default fully qualified app name.
-We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
-If release name contains chart name it will be used as a full name.
-*/}}
-{{- define "gcp.fullname" -}}
-{{- if .Values.fullnameOverride }}
-{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
-{{- else }}
-{{- $name := default .Chart.Name .Values.nameOverride }}
-{{- if contains $name .Release.Name }}
-{{- .Release.Name | trunc 63 | trimSuffix "-" }}
-{{- else }}
-{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
-{{- end }}
-{{- end }}
-{{- end }}
-
-{{/*
-Create chart name and version as used by the chart label.
-*/}}
-{{- define "gcp.chart" -}}
-{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
-{{- end }}
-
-{{/*
-Common labels
-*/}}
-{{- define "gcp.labels" -}}
-helm.sh/chart: {{ include "gcp.chart" . }}
-{{ include "gcp.selectorLabels" . }}
-{{- if .Chart.AppVersion }}
-app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
-{{- end }}
-app.kubernetes.io/managed-by: {{ .Release.Service }}
-{{- end }}
-
-{{/*
-Selector labels
-*/}}
-{{- define "gcp.selectorLabels" -}}
-app.kubernetes.io/name: {{ include "gcp.name" . }}
-app.kubernetes.io/instance: {{ .Release.Name }}
-{{- end }}
-
-{{/*
-Create the name of the service account to use
-*/}}
-{{- define "gcp.serviceAccountName" -}}
-{{- if .Values.serviceAccount.create }}
-{{- default (include "gcp.fullname" .) .Values.serviceAccount.name }}
-{{- else }}
-{{- default "default" .Values.serviceAccount.name }}
-{{- end }}
-{{- end }}

devops/gcp/osdu-helm/templates/deployment.yaml

-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  namespace: {{ .Values.namespace }}
-  name: {{ include "gcp.fullname" . }}
-  labels:
-    {{- include "gcp.labels" . | nindent 4 }}
-spec:
-  replicas: {{ .Values.replicaCount }}
-  selector:
-    matchLabels:
-      {{- include "gcp.selectorLabels" . | nindent 6 }}
-  template:
-    metadata:
-      labels:
-        {{- include "gcp.selectorLabels" . | nindent 8 }}
-      annotations:
-        sidecar.istio.io/rewriteAppHTTPProbers: "true"
-    spec:
-      volumes:
-        - name: service-account-key
-          secret:
-            secretName:  {{ .Values.secret.name }}
-      containers:
-        - name: {{ .Chart.Name }}
-          volumeMounts:
-            - mountPath: /tmp/key
-              name: service-account-key
-          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
-          imagePullPolicy: {{ .Values.image.pullPolicy }}
-          envFrom:
-            - configMapRef:
-                name: {{ .Values.configMap.name }}
-          env:
-            - name: OS_WELLBORE_DDMS_DATA_PROJECT_CREDENTIALS
-              value: /tmp/key/{{ .Values.secret.key }}
-          ports:
-            - name: http
-              containerPort: {{ .Values.service.port }}
-              protocol: TCP
-          livenessProbe:
-            httpGet:
-              path: /api/os-wellbore-ddms/healthz
-              port: http
-          readinessProbe:
-            httpGet:
-              path: /api/os-wellbore-ddms/healthz
-              port: http
-          resources:
-            {{- toYaml .Values.resources | nindent 12 }}

devops/gcp/osdu-helm/templates/service.yaml

-apiVersion: v1
-kind: Service
-metadata:
-  name: {{ include "gcp.fullname" . }}
-  namespace: {{ .Values.namespace }}
-  labels:
-    {{- include "gcp.labels" . | nindent 4 }}
-spec:
-  ports:
-    - port: 80
-      targetPort: {{ .Values.service.port }}
-      protocol: TCP
-      name: http
-  selector:
-    {{- include "gcp.selectorLabels" . | nindent 4 }}

devops/gcp/osdu-helm/templates/virtual-service.yaml

-apiVersion: networking.istio.io/v1alpha3
-kind: VirtualService
-metadata:
-  name: {{ include "gcp.fullname" . }}
-  namespace: {{ .Values.namespace }}
-spec:
-  hosts:
-    - "*"
-  gateways:
-    - service-gateway
-  http:
-    - match:
-        - uri:
-            prefix: /api/os-wellbore-ddms
-      route:
-        - destination:
-            host: {{ include "gcp.fullname" . }}
-            port:
-              number: 80

devops/gcp/osdu-helm/values.yaml

-# Default values for gcp.
-# This is a YAML-formatted file.
-# Declare variables to be passed into your templates.
-namespace: default
-
-configMap:
-  data:
-    cloudProvider: gcp
-    projectId: nice-etching-277309
-    searchEndpoint: https://os-search-attcrcktoa-uc.a.run.app/api/search
-    storageEndpoint: https://os-storage-attcrcktoa-uc.a.run.app/api/storage
-  name: wellbore
-
-secret:
-  name: wellbore
-  key: "key.json"
-  gcpSA: "" # ENTER in base64 service-account value
-
-replicaCount: 1
-
-image:
-  repository: "community.opengroup.org:5555/osdu/platform/domain-data-mgmt-services/wellbore/wellbore-domain-services/osdu-gcp-wellbore-ddms"
-  pullPolicy: Always
-  tag: "gcp-community"
-
-service:
-  port: 8080
-
-resources:
-  {}
-  # We usually recommend not to specify default resources and to leave this as a conscious
-  # choice for the user. This also increases chances charts run on environments with little
-  # resources, such as Minikube. If you do want to specify resources, uncomment the following
-  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
-  # limits:
-  #   cpu: 800m
-  #   memory: 512Mi
-  # requests:
-  #   cpu: 700m
-  #   memory: 512Mi