Merge branch 'gcp-release-pipeline' into 'master'

Refactor pipeline for release [GONRG-4856]

See merge request !194

.gitlab-ci.yml

@@ -39,7 +39,7 @@ include:
     file: "cloud-providers/ibm.yml"
 
   - project: "osdu/platform/ci-cd-pipelines"
-    file: "cloud-providers/osdu-gcp-gke.yml"
+    file: "cloud-providers/osdu-gcp-global.yml"
 
   - local: "devops/gcp/pipeline/override-stages.yml"
 

devops/gcp/configmap/templates/configmap-bootstrap.yml

@@ -7,11 +7,16 @@ metadata:
   namespace: "{{ .Release.Namespace }}"
 data:
   PARTITION_NAME: "{{ .Values.data.partitionName }}.{{ .Release.Namespace }}.svc.cluster.local"
-  PROJECT_ID: "{{ .Values.data.projectId }}"
+  SERVICE_ACCOUNT: {{ printf "%s@%s.iam.gserviceaccount.com" .Values.data.datafierSa .Values.data.projectId | quote }}
   DATA_PARTITION_ID: "{{ .Values.data.dataPartitionId }}"
-  DATAFIER_SA: "{{ .Values.data.datafierSa }}"
   ENVIRONMENT: "{{ .Values.data.springProfilesActive }}"
+  {{- if .Values.data.dataProjectId }}
+  PROJECT_ID: "{{ .Values.data.dataProjectId }}"
+  {{- else }}
+  PROJECT_ID: "{{ .Values.data.projectId }}"
+  {{- end }}
   {{- if .Values.conf.onPremEnabled }}
+  SERVICE_ACCOUNT: {{ print "%s@service.local" .Values.data.datafierSa | quote }}
   DOMAIN: "{{ .Values.data.domain }}"
   {{- else }}
   AUDIENCES: "{{ .Values.data.googleAudiences }}"

devops/gcp/configmap/values.yaml

@@ -4,6 +4,7 @@ data:
   springProfilesActive: "gcp"
   # gcp
   projectId: ""
+  dataProjectId: ""
   partitionAdminAccounts: ""
   googleAudiences: ""
   serviceAccountTail: ".iam.gserviceaccount.com"

devops/gcp/pipeline/override-stages.yml

-variables:
-  OSDU_GCP_ENABLE_BOOTSTRAP: "true"
-  OSDU_GCP_SERVICE: partition
-  OSDU_GCP_VENDOR: gcp
-  OSDU_GCP_HELM_CONFIG_SERVICE_VARS: >
-    --set data.partitionAdminAccounts=$OSDU_GCP_PARTITION_ADMIN_ACCOUNTS
-    --set data.projectId=$OSDU_GCP_PROJECT
-    --set data.googleAudiences=$GOOGLE_AUDIENCE
-    --set data.logLevel=INFO
-    --set data.dataPartitionId=$OSDU_GCP_TENANT
-  OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS: >
-    --set data.image=$CI_REGISTRY_IMAGE/osdu-gcp-$OSDU_GCP_SERVICE:$CI_COMMIT_SHORT_SHA
-    --set data.bootstrapImage=$CI_REGISTRY_IMAGE/osdu-gcp-bootstrap-$OSDU_GCP_SERVICE:$CI_COMMIT_SHORT_SHA
-    --set data.serviceAccountName=$OSDU_GCP_SERVICE-k8s
-    --set data.bootstrapServiceAccountName=$OSDU_GCP_SERVICE-k8s
-  OSDU_GCP_HELM_CONFIG_SERVICE_VARS_DEV2: >
-    --set data.partitionAdminAccounts=$OSDU_GCP_DEV2_PARTITION_ADMIN_ACCOUNTS
-    --set data.projectId=$OSDU_GCP_PROJECT
-    --set data.googleAudiences=$GOOGLE_AUDIENCE
-    --set data.logLevel=INFO
-    --set data.dataPartitionId=$OSDU_GCP_TENANT
-  OSDU_GCP_HELM_CONFIG_SERVICE: partition-config
-  OSDU_GCP_HELM_DEPLOYMENT_SERVICE: partition-deploy
-
-osdu-gcp-deploy-deployment:
-  needs:
-    - osdu-gcp-containerize-gitlab
-    - osdu-gcp-containerize-bootstrap-gitlab
-    - osdu-gcp-deploy-configmap
-  after_script:
-    - echo ----- Verify Bootstrap -----
-    - kubectl rollout status deployment.v1.apps/$OSDU_GCP_SERVICE-bootstrap -n $OSDU_GCP_HELM_NAMESPACE --timeout=900s
-    - POD=$(kubectl get pod --sort-by=.metadata.creationTimestamp -n $OSDU_GCP_HELM_NAMESPACE | grep $OSDU_GCP_SERVICE-bootstrap | tail -1 | awk '{print $1}')
-    - STATUS=$(kubectl wait -n $OSDU_GCP_HELM_NAMESPACE --for=condition=Ready pod/$POD --timeout=300s)
-    - echo $STATUS
-    - if [[ "$STATUS" != *"met"* ]]; then echo "POD didn't start correctly" ; exit 1 ; fi
-
-osdu-gcp-dev2-deploy-deployment:
-  needs:
-    - osdu-gcp-containerize-gitlab
-    - osdu-gcp-containerize-bootstrap-gitlab
-    - osdu-gcp-dev2-deploy-configmap
-
-osdu-gcp-anthos-deploy-deployment:
-  needs:
-    - osdu-gcp-containerize-gitlab
-    - osdu-gcp-containerize-bootstrap-gitlab
-    - osdu-gcp-anthos-deploy-configmap
-
-osdu-gcp-test:
-  variables:
-    CLIENT_TENANT: osdu
-
-osdu-gcp-dev2-test:
-  variables:
-    CLIENT_TENANT: devtwo
+variables:
+  OSDU_GCP_ENABLE_BOOTSTRAP: "true"
+  OSDU_GCP_SERVICE: partition
+  OSDU_GCP_VENDOR: gcp
+  # FIXME move variables below to common pipeline
+  OSDU_GCP_HELM_CONFIG_SERVICE_VARS: >
+    --set data.partitionAdminAccounts=$OSDU_GCP_PARTITION_ADMIN_ACCOUNTS
+    --set data.projectId=$OSDU_GCP_PROJECT
+    --set data.googleAudiences=$GOOGLE_AUDIENCE
+    --set data.logLevel=INFO
+    --set data.dataPartitionId=$OSDU_GCP_TENANT
+  OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS: >
+    --set data.image=$CI_REGISTRY_IMAGE/osdu-gcp-$OSDU_GCP_SERVICE:$CI_COMMIT_SHORT_SHA
+    --set data.bootstrapImage=$CI_REGISTRY_IMAGE/osdu-gcp-bootstrap-$OSDU_GCP_SERVICE:$CI_COMMIT_SHORT_SHA
+    --set data.serviceAccountName=$OSDU_GCP_SERVICE-k8s
+    --set data.bootstrapServiceAccountName=$OSDU_GCP_SERVICE-k8s
+  OSDU_GCP_HELM_CONFIG_SERVICE: partition-config
+  OSDU_GCP_HELM_DEPLOYMENT_SERVICE: partition-deploy
+
+# FIXME refactor using reference
+osdu-gcp-deploy-deployment:
+  after_script:
+    - echo ----- Verify Bootstrap -----
+    - kubectl rollout status deployment.v1.apps/$OSDU_GCP_SERVICE-bootstrap -n $OSDU_GCP_HELM_NAMESPACE --timeout=900s
+    - POD=$(kubectl get pod --sort-by=.metadata.creationTimestamp -n $OSDU_GCP_HELM_NAMESPACE | grep $OSDU_GCP_SERVICE-bootstrap | tail -1 | awk '{print $1}')
+    - STATUS=$(kubectl wait -n $OSDU_GCP_HELM_NAMESPACE --for=condition=Ready pod/$POD --timeout=300s)
+    - echo $STATUS
+    - if [[ "$STATUS" != *"met"* ]]; then echo "POD didn't start correctly" ; exit 1 ; fi
+
+osdu-gcp-test:
+  variables:
+    CLIENT_TENANT: osdu
+
+osdu-gcp-dev2-test:
+  variables:
+    CLIENT_TENANT: devtwo

provider/partition-gcp/bootstrap/bootstrap_partition.sh

@@ -14,7 +14,7 @@ generate_post_data() {
     },
     "serviceAccount": {
       "sensitive": false,
-      "value": "${SERVICEACCOUNT}"
+      "value": "${SERVICE_ACCOUNT}"
     },
     "complianceRuleSet": {
       "sensitive": false,
@@ -144,8 +144,6 @@ EOF
 if [ "$ENVIRONMENT" == "anthos" ]
 then
 
-  SERVICEACCOUNT=$DATAFIER_SA@service.local
-
   status_code=$(curl -X POST \
     --url "http://${PARTITION_NAME}/api/partition/v1/partitions/${DATA_PARTITION_ID}" --write-out "%{http_code}" --silent --output "/dev/null" \
     -H "Content-Type: application/json" \
@@ -173,8 +171,6 @@ then
 
   IDENTITY_TOKEN=$(gcloud auth print-identity-token --audiences="${AUDIENCES}")
 
-  SERVICEACCOUNT=${DATAFIER_SA}@${PROJECT_ID}.iam.gserviceaccount.com
-
   status_code=$(curl -X POST \
      --url "http://${PARTITION_NAME}/api/partition/v1/partitions/${DATA_PARTITION_ID}" --write-out "%{http_code}" --silent --output "/dev/null" \
      -H "Authorization: Bearer ${IDENTITY_TOKEN}" \