From 10b08f3b74d0080245bd99743871e141a5e26e3b Mon Sep 17 00:00:00 2001
From: "Mikhail Piatliou (EPAM)" <mikhail_piatliou@epam.com>
Date: Mon, 13 Jun 2022 09:35:25 +0000
Subject: [PATCH] Refactor pipeline for release [GONRG-4856]
---
.gitlab-ci.yml | 2 +-
.../templates/configmap-bootstrap.yml | 9 +-
devops/gcp/configmap/values.yaml | 1 +
devops/gcp/pipeline/override-stages.yml | 92 ++++++++-----------
.../bootstrap/bootstrap_partition.sh | 6 +-
5 files changed, 46 insertions(+), 64 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index accbeca4b..510974681 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -39,7 +39,7 @@ include:
file: "cloud-providers/ibm.yml"
- project: "osdu/platform/ci-cd-pipelines"
- file: "cloud-providers/osdu-gcp-gke.yml"
+ file: "cloud-providers/osdu-gcp-global.yml"
- local: "devops/gcp/pipeline/override-stages.yml"
diff --git a/devops/gcp/configmap/templates/configmap-bootstrap.yml b/devops/gcp/configmap/templates/configmap-bootstrap.yml
index 1bbe1ba05..e4160375e 100644
--- a/devops/gcp/configmap/templates/configmap-bootstrap.yml
+++ b/devops/gcp/configmap/templates/configmap-bootstrap.yml
@@ -7,11 +7,16 @@ metadata:
namespace: "{{ .Release.Namespace }}"
data:
PARTITION_NAME: "{{ .Values.data.partitionName }}.{{ .Release.Namespace }}.svc.cluster.local"
- PROJECT_ID: "{{ .Values.data.projectId }}"
+ SERVICE_ACCOUNT: {{ printf "%s@%s.iam.gserviceaccount.com" .Values.data.datafierSa .Values.data.projectId | quote }}
DATA_PARTITION_ID: "{{ .Values.data.dataPartitionId }}"
- DATAFIER_SA: "{{ .Values.data.datafierSa }}"
ENVIRONMENT: "{{ .Values.data.springProfilesActive }}"
+ {{- if .Values.data.dataProjectId }}
+ PROJECT_ID: "{{ .Values.data.dataProjectId }}"
+ {{- else }}
+ PROJECT_ID: "{{ .Values.data.projectId }}"
+ {{- end }}
{{- if .Values.conf.onPremEnabled }}
+ SERVICE_ACCOUNT: {{ print "%s@service.local" .Values.data.datafierSa | quote }}
DOMAIN: "{{ .Values.data.domain }}"
{{- else }}
AUDIENCES: "{{ .Values.data.googleAudiences }}"
diff --git a/devops/gcp/configmap/values.yaml b/devops/gcp/configmap/values.yaml
index f7661110a..d89345f88 100644
--- a/devops/gcp/configmap/values.yaml
+++ b/devops/gcp/configmap/values.yaml
@@ -4,6 +4,7 @@ data:
springProfilesActive: "gcp"
# gcp
projectId: ""
+ dataProjectId: ""
partitionAdminAccounts: ""
googleAudiences: ""
serviceAccountTail: ".iam.gserviceaccount.com"
diff --git a/devops/gcp/pipeline/override-stages.yml b/devops/gcp/pipeline/override-stages.yml
index a44b3f4b3..f59dece9c 100644
--- a/devops/gcp/pipeline/override-stages.yml
+++ b/devops/gcp/pipeline/override-stages.yml
@@ -1,56 +1,36 @@
-variables:
- OSDU_GCP_ENABLE_BOOTSTRAP: "true"
- OSDU_GCP_SERVICE: partition
- OSDU_GCP_VENDOR: gcp
- OSDU_GCP_HELM_CONFIG_SERVICE_VARS: >
- --set data.partitionAdminAccounts=$OSDU_GCP_PARTITION_ADMIN_ACCOUNTS
- --set data.projectId=$OSDU_GCP_PROJECT
- --set data.googleAudiences=$GOOGLE_AUDIENCE
- --set data.logLevel=INFO
- --set data.dataPartitionId=$OSDU_GCP_TENANT
- OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS: >
- --set data.image=$CI_REGISTRY_IMAGE/osdu-gcp-$OSDU_GCP_SERVICE:$CI_COMMIT_SHORT_SHA
- --set data.bootstrapImage=$CI_REGISTRY_IMAGE/osdu-gcp-bootstrap-$OSDU_GCP_SERVICE:$CI_COMMIT_SHORT_SHA
- --set data.serviceAccountName=$OSDU_GCP_SERVICE-k8s
- --set data.bootstrapServiceAccountName=$OSDU_GCP_SERVICE-k8s
- OSDU_GCP_HELM_CONFIG_SERVICE_VARS_DEV2: >
- --set data.partitionAdminAccounts=$OSDU_GCP_DEV2_PARTITION_ADMIN_ACCOUNTS
- --set data.projectId=$OSDU_GCP_PROJECT
- --set data.googleAudiences=$GOOGLE_AUDIENCE
- --set data.logLevel=INFO
- --set data.dataPartitionId=$OSDU_GCP_TENANT
- OSDU_GCP_HELM_CONFIG_SERVICE: partition-config
- OSDU_GCP_HELM_DEPLOYMENT_SERVICE: partition-deploy
-
-osdu-gcp-deploy-deployment:
- needs:
- - osdu-gcp-containerize-gitlab
- - osdu-gcp-containerize-bootstrap-gitlab
- - osdu-gcp-deploy-configmap
- after_script:
- - echo ----- Verify Bootstrap -----
- - kubectl rollout status deployment.v1.apps/$OSDU_GCP_SERVICE-bootstrap -n $OSDU_GCP_HELM_NAMESPACE --timeout=900s
- - POD=$(kubectl get pod --sort-by=.metadata.creationTimestamp -n $OSDU_GCP_HELM_NAMESPACE | grep $OSDU_GCP_SERVICE-bootstrap | tail -1 | awk '{print $1}')
- - STATUS=$(kubectl wait -n $OSDU_GCP_HELM_NAMESPACE --for=condition=Ready pod/$POD --timeout=300s)
- - echo $STATUS
- - if [[ "$STATUS" != *"met"* ]]; then echo "POD didn't start correctly" ; exit 1 ; fi
-
-osdu-gcp-dev2-deploy-deployment:
- needs:
- - osdu-gcp-containerize-gitlab
- - osdu-gcp-containerize-bootstrap-gitlab
- - osdu-gcp-dev2-deploy-configmap
-
-osdu-gcp-anthos-deploy-deployment:
- needs:
- - osdu-gcp-containerize-gitlab
- - osdu-gcp-containerize-bootstrap-gitlab
- - osdu-gcp-anthos-deploy-configmap
-
-osdu-gcp-test:
- variables:
- CLIENT_TENANT: osdu
-
-osdu-gcp-dev2-test:
- variables:
- CLIENT_TENANT: devtwo
+variables:
+ OSDU_GCP_ENABLE_BOOTSTRAP: "true"
+ OSDU_GCP_SERVICE: partition
+ OSDU_GCP_VENDOR: gcp
+ # FIXME move variables below to common pipeline
+ OSDU_GCP_HELM_CONFIG_SERVICE_VARS: >
+ --set data.partitionAdminAccounts=$OSDU_GCP_PARTITION_ADMIN_ACCOUNTS
+ --set data.projectId=$OSDU_GCP_PROJECT
+ --set data.googleAudiences=$GOOGLE_AUDIENCE
+ --set data.logLevel=INFO
+ --set data.dataPartitionId=$OSDU_GCP_TENANT
+ OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS: >
+ --set data.image=$CI_REGISTRY_IMAGE/osdu-gcp-$OSDU_GCP_SERVICE:$CI_COMMIT_SHORT_SHA
+ --set data.bootstrapImage=$CI_REGISTRY_IMAGE/osdu-gcp-bootstrap-$OSDU_GCP_SERVICE:$CI_COMMIT_SHORT_SHA
+ --set data.serviceAccountName=$OSDU_GCP_SERVICE-k8s
+ --set data.bootstrapServiceAccountName=$OSDU_GCP_SERVICE-k8s
+ OSDU_GCP_HELM_CONFIG_SERVICE: partition-config
+ OSDU_GCP_HELM_DEPLOYMENT_SERVICE: partition-deploy
+
+# FIXME refactor using reference
+osdu-gcp-deploy-deployment:
+ after_script:
+ - echo ----- Verify Bootstrap -----
+ - kubectl rollout status deployment.v1.apps/$OSDU_GCP_SERVICE-bootstrap -n $OSDU_GCP_HELM_NAMESPACE --timeout=900s
+ - POD=$(kubectl get pod --sort-by=.metadata.creationTimestamp -n $OSDU_GCP_HELM_NAMESPACE | grep $OSDU_GCP_SERVICE-bootstrap | tail -1 | awk '{print $1}')
+ - STATUS=$(kubectl wait -n $OSDU_GCP_HELM_NAMESPACE --for=condition=Ready pod/$POD --timeout=300s)
+ - echo $STATUS
+ - if [[ "$STATUS" != *"met"* ]]; then echo "POD didn't start correctly" ; exit 1 ; fi
+
+osdu-gcp-test:
+ variables:
+ CLIENT_TENANT: osdu
+
+osdu-gcp-dev2-test:
+ variables:
+ CLIENT_TENANT: devtwo
diff --git a/provider/partition-gcp/bootstrap/bootstrap_partition.sh b/provider/partition-gcp/bootstrap/bootstrap_partition.sh
index 8f2a15208..a20ff5954 100644
--- a/provider/partition-gcp/bootstrap/bootstrap_partition.sh
+++ b/provider/partition-gcp/bootstrap/bootstrap_partition.sh
@@ -14,7 +14,7 @@ generate_post_data() {
},
"serviceAccount": {
"sensitive": false,
- "value": "${SERVICEACCOUNT}"
+ "value": "${SERVICE_ACCOUNT}"
},
"complianceRuleSet": {
"sensitive": false,
@@ -144,8 +144,6 @@ EOF
if [ "$ENVIRONMENT" == "anthos" ]
then
- SERVICEACCOUNT=$DATAFIER_SA@service.local
-
status_code=$(curl -X POST \
--url "http://${PARTITION_NAME}/api/partition/v1/partitions/${DATA_PARTITION_ID}" --write-out "%{http_code}" --silent --output "/dev/null" \
-H "Content-Type: application/json" \
@@ -173,8 +171,6 @@ then
IDENTITY_TOKEN=$(gcloud auth print-identity-token --audiences="${AUDIENCES}")
- SERVICEACCOUNT=${DATAFIER_SA}@${PROJECT_ID}.iam.gserviceaccount.com
-
status_code=$(curl -X POST \
--url "http://${PARTITION_NAME}/api/partition/v1/partitions/${DATA_PARTITION_ID}" --write-out "%{http_code}" --silent --output "/dev/null" \
-H "Authorization: Bearer ${IDENTITY_TOKEN}" \
--
GitLab