Gonrg 5231 refactor gcp pipeline

devops/gcp/bootstrap-osdu-module/Dockerfile

-FROM google/cloud-sdk:slim
+FROM gcr.io/google.com/cloudsdktool/cloud-sdk:alpine
 
-COPY ./devops/gcp/bootstrap-osdu-module/bootstrap_policy.sh /opt 
-COPY . /opt/
+COPY ./requirements_bootstrap.txt ./devops/gcp/bootstrap-osdu-module/bootstrap_policy.sh /opt/
+COPY ./deployment/ /opt/deployment
+COPY ./devops/gcp/bootstrap-osdu-module /opt/devops/gcp/bootstrap-osdu-module
 
 RUN chmod 775 /opt/bootstrap_policy.sh
 
+RUN apk add py3-pip
+
 RUN pip3 install -r /opt/requirements_bootstrap.txt
 
 RUN pip3 install -r /opt/devops/gcp/bootstrap-osdu-module/requirements.txt
 
 CMD ["/bin/bash", "-c", "/opt/bootstrap_policy.sh"]
-

devops/gcp/bootstrap-osdu-module/bootstrap_policy.sh

@@ -6,12 +6,9 @@ BEARER_TOKEN=$(gcloud auth print-identity-token --audiences="${AUDIENCES}")
 export BEARER_TOKEN
 
 echo "Achive bundle of policies and push to bucket"
-tar -czf bundle.tar.gz --directory='./opt/deployment/default-policies' --exclude='./bootstrap_sequence.json' . --verbose
+tar -czf bundle.tar.gz --directory='/opt/deployment/default-policies' --exclude='./bootstrap_sequence.json' . --verbose
 mkdir --parents ./opt/policies ; mv bundle.tar.gz "$_"
-python3 /opt/devops/gcp/bootstrap-osdu-module/DataPartitionBundles.py
-mv /opt/devops/gcp/bootstrap-osdu-module/bundle-"${DATA_PARTITION}".tar.gz ./opt/policies
+python3 ./devops/gcp/bootstrap-osdu-module/DataPartitionBundles.py
+mv ./bundle-"${DATA_PARTITION}".tar.gz ./opt/policies
 gsutil rsync ./opt/policies gs://"${POLICY_BUCKET}"/
-echo "Achive bundle of policies and push to bucket - DONE!"
-
-echo "Bootstrap Policy Service"
-python3 /opt/deployment/scripts/BootstrapDefaultPolicies.py -u "${POLICY_URL}"
+echo "Archive bundle of policies and push to bucket - DONE!"

devops/gcp/configmap_opa/templates/opa-env-configmap.yaml

+apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    app: "{{ .Values.conf.appName }}"
+  name: "{{ .Values.conf.envConfig }}"
+  namespace: "{{ .Release.Namespace }}"
+data:
+  LEGAL_BASE_URL: "{{ .Values.data.legalHost }}"
+  ENTITLEMENTS_BASE_URL: "{{ .Values.data.entitlementsHost }}"

devops/gcp/configmap_opa/values.yaml

 data:
   bucketName: ""
   scopes: "https://www.googleapis.com/auth/devstorage.read_only"
+  legalHost: "http://legal"
+  entitlementsHost: "http://entitlements"
 
 conf:
   configmap: "opa-config"
+  envConfig: "opa-env-config"
   appName: "opa"
   dataPartitionId: ""
   min_delay_seconds: 6

devops/gcp/opa/templates/deployment.yaml

@@ -20,7 +20,7 @@ spec:
     spec:
       containers:
       - name: "{{ .Values.conf.appName }}"
-        image: openpolicyagent/opa:latest
+        image: "{{ .Values.data.image }}"
         imagePullPolicy: "{{ .Values.data.imagePullPolicy }}"
         ports:
           - containerPort: 8181
@@ -39,11 +39,9 @@ spec:
         volumeMounts:
           - mountPath: /config
             name: "{{ .Values.conf.configmap }}"
-        env:
-        - name: ENTITLEMENTS_BASE_URL
-          value: "http://entitlements"
-        - name: LEGAL_BASE_URL
-          value: "http://legal"
+        envFrom:
+        - configMapRef:
+            name: "{{ .Values.conf.envConfig }}"
       volumes:
         - name: "{{ .Values.conf.configmap }}"
           configMap:

devops/gcp/opa/values.yaml

@@ -7,11 +7,12 @@ data:
   requestsMemory: "128M"
   limitsCpu: "1"
   limitsMemory: "1G"
-  image: ""
+  image: "openpolicyagent/opa:latest"
   imagePullPolicy: "IfNotPresent"
   serviceAccountName: "opa-k8s"
 
 conf:
   appName: "opa"
   configmap: "opa-config"
+  envConfig: "opa-env-config"
   domain: ""

devops/gcp/pipeline/override-stages.yml

@@ -7,10 +7,6 @@ variables:
   OSDU_GCP_OPA_SERVICE: opa
   OSDU_GCP_HELM_OPA_DIR: "devops/gcp/opa"
   OSDU_GCP_HELM_OPA_CONFIG_DIR: "devops/gcp/configmap_opa"
-  # FIXME
-  OSDU_GCP_HELM_OPA_CONFIG_SERVICE_VARS: >-
-    --set data.bucketName=$OSDU_GCP_POLICY_BUCKET
-    --set conf.dataPartitionId=osdu
   OSDU_GCP_INT_TEST_TYPE: python
 
 osdu-gcp-helm-charts-master:
@@ -22,11 +18,11 @@ osdu-gcp-helm-charts-master:
 
 osdu-gcp-helm-charts-release:
   script:
-    - VER=$(echo $CI_COMMIT_TAG | sed "s/^v//")
-    - helm cm-push $OSDU_GCP_HELM_CONFIG_DIR ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/helm/stable --version $VER --username gitlab-ci-token --password $CI_JOB_TOKEN
-    - helm cm-push $OSDU_GCP_HELM_DEPLOYMENT_DIR ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/helm/stable --version $VER --username gitlab-ci-token --password $CI_JOB_TOKEN
-    - helm cm-push $OSDU_GCP_HELM_OPA_DIR ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/helm/stable --version $VER --username gitlab-ci-token --password $CI_JOB_TOKEN
-    - helm cm-push $OSDU_GCP_HELM_OPA_CONFIG_DIR ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/helm/stable --version $VER --username gitlab-ci-token --password $CI_JOB_TOKEN
+    - !reference [.define_version, script]
+    - helm cm-push $OSDU_GCP_HELM_CONFIG_DIR ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/helm/stable --version $VERSION --username gitlab-ci-token --password $CI_JOB_TOKEN
+    - helm cm-push $OSDU_GCP_HELM_DEPLOYMENT_DIR ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/helm/stable --version $VERSION --username gitlab-ci-token --password $CI_JOB_TOKEN
+    - helm cm-push $OSDU_GCP_HELM_OPA_DIR ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/helm/stable --version $VERSION --username gitlab-ci-token --password $CI_JOB_TOKEN
+    - helm cm-push $OSDU_GCP_HELM_OPA_CONFIG_DIR ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/helm/stable --version $VERSION --username gitlab-ci-token --password $CI_JOB_TOKEN
 
 .common_test_config:
   script:
@@ -34,42 +30,23 @@ osdu-gcp-helm-charts-release:
   - gcloud auth activate-service-account --key-file OSDU_GCP_INTEGRATION_TESTER.json
   - gcloud config set project $OSDU_GCP_PROJECT
 
-.common_deploy_config:
-  script:
-    - gcloud auth activate-service-account --key-file $OSDU_GCP_DEPLOY_FILE
-    - gcloud config set project $OSDU_GCP_PROJECT
-    - gcloud container clusters get-credentials $OSDU_GCP_CLUSTER --zone $OSDU_GCP_ZONE --project $OSDU_GCP_PROJECT
-    - curl https://get.helm.sh/helm-v3.5.2-linux-amd64.tar.gz -s -o helm.tgz; tar -zxf helm.tgz; mv linux-amd64/helm /usr/local/bin/
-    - apt-get update && apt-get install -y jq
-    - HELM_REVISION=$(helm history $OSDU_GCP_SERVICE-deploy -o json | jq '.[] | select(.status == "pending-upgrade").revision')
-    - if [[ ! -z "$HELM_REVISION" ]]; then kubectl delete secret sh.helm.release.v1.$OSDU_GCP_SERVICE-deploy.v$HELM_REVISION; fi
-
-.verify_deploy:
-  script:
-    - echo ----- Verify Deployment -----
-    - kubectl rollout status deployment.v1.apps/$OSDU_GCP_SERVICE -n $OSDU_GCP_HELM_NAMESPACE --timeout=900s
-    - POD=$(kubectl get pod --sort-by=.metadata.creationTimestamp -n $OSDU_GCP_HELM_NAMESPACE | grep $OSDU_GCP_SERVICE | tail -1 | awk '{print $1}')
-    - STATUS=$(kubectl wait -n $OSDU_GCP_HELM_NAMESPACE --for=condition=Ready pod/$POD --timeout=300s)
-    - echo $STATUS
-    - >
-      if [[ "$STATUS" != *"met"* ]];
-      then echo "POD didn't start correctly"; exit 1; fi
-
 osdu-gcp-deploy-configmap-opa:
   tags: ["osdu-small"]
   extends: .osdu-gcp-variables
-  image: gcr.io/google.com/cloudsdktool/cloud-sdk
+  image: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine
   stage: deploy
   needs: ["osdu-gcp-deploy-deployment"]
   script:
-    - !reference [.common_deploy_config, script]
+    - gcloud auth activate-service-account --key-file $OSDU_GCP_DEPLOY_FILE
+    - !reference [.common_config, script]
     - helm upgrade $OSDU_GCP_OPA_CONFIG_SERVICE $OSDU_GCP_HELM_OPA_CONFIG_DIR
       --install
       --create-namespace
       --namespace=$OSDU_GCP_HELM_NAMESPACE
       --wait
       --history-max=3
-      $OSDU_GCP_HELM_OPA_CONFIG_SERVICE_VARS
+      --set data.bucketName=$OSDU_GCP_POLICY_BUCKET
+      --set conf.dataPartitionId=$DATA_PARTITION_ID
   rules:
     - if: '$OSDU_GCP == "1" && $CI_COMMIT_BRANCH =~ /^release/'
       when: never
@@ -81,11 +58,12 @@ osdu-gcp-deploy-configmap-opa:
 osdu-gcp-dev2-deploy-configmap-opa:
   tags: ["osdu-small"]
   extends: .osdu-gcp-dev2-variables
-  image: gcr.io/google.com/cloudsdktool/cloud-sdk
+  image: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine
   stage: deploy
   needs: ["osdu-gcp-dev2-deploy-deployment"]
   script:
-    - !reference [.common_deploy_config, script]
+    - gcloud auth activate-service-account --key-file $OSDU_GCP_DEPLOY_FILE
+    - !reference [.common_config, script]
     - helm upgrade $OSDU_GCP_OPA_CONFIG_SERVICE $OSDU_GCP_HELM_OPA_CONFIG_DIR
       --install
       --create-namespace
@@ -93,7 +71,7 @@ osdu-gcp-dev2-deploy-configmap-opa:
       --wait
       --history-max=3
       --set data.bucket_name=$OSDU_GCP_POLICY_BUCKET
-      --set conf.data_partition_id=devtwo
+      --set conf.data_partition_id=$DATA_PARTITION_ID
   rules:
     - if: '$OSDU_GCP == "1" && $CI_COMMIT_BRANCH =~ /^release/'
       when: on_success
@@ -102,13 +80,14 @@ osdu-gcp-dev2-deploy-configmap-opa:
 
 osdu-gcp-deploy-opa:
   tags: ["osdu-small"]
-  image: gcr.io/google.com/cloudsdktool/cloud-sdk
+  image: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine
   stage: deploy
   cache: {}
   extends: .osdu-gcp-variables
   needs: ["osdu-gcp-deploy-configmap-opa"]
   script:
-    - !reference [.common_deploy_config, script]
+    - gcloud auth activate-service-account --key-file $OSDU_GCP_DEPLOY_FILE
+    - !reference [.common_config, script]
     - helm upgrade $OSDU_GCP_OPA_SERVICE $OSDU_GCP_HELM_OPA_DIR
       --install
       --create-namespace
@@ -126,13 +105,14 @@ osdu-gcp-deploy-opa:
 
 osdu-gcp-dev2-deploy-opa:
   tags: ["osdu-small"]
-  image: gcr.io/google.com/cloudsdktool/cloud-sdk
+  image: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine
   stage: deploy
   cache: {}
   extends: .osdu-gcp-dev2-variables
   needs: ["osdu-gcp-dev2-deploy-configmap-opa"]
   script:
-    - !reference [.common_deploy_config, script]
+    - gcloud auth activate-service-account --key-file $OSDU_GCP_DEPLOY_FILE
+    - !reference [.common_config, script]
     - helm upgrade $OSDU_GCP_OPA_SERVICE $OSDU_GCP_HELM_OPA_DIR
       --install
       --create-namespace
@@ -149,12 +129,13 @@ osdu-gcp-dev2-deploy-opa:
 osdu-gcp-bootstrap:
   tags: ["osdu-small"]
   stage: bootstrap
-  image: gcr.io/google.com/cloudsdktool/cloud-sdk
+  image: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine
   extends: .osdu-gcp-variables
   needs: ["osdu-gcp-deploy-opa"]
   script:
     - !reference [.common_test_config, script]
     - >
+    - apk add py3-pip
     - pip install -r requirements_bootstrap.txt
     - tar -czf bundle.tar.gz --directory='./deployment/default-policies/' --exclude='./bootstrap_sequence.json' . --verbose
     - mkdir --parents ./policies ; mv bundle.tar.gz $_
@@ -172,12 +153,13 @@ osdu-gcp-bootstrap:
 osdu-gcp-dev2-bootstrap:
   tags: ["osdu-small"]
   stage: bootstrap
-  image: gcr.io/google.com/cloudsdktool/cloud-sdk
+  image: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine
   extends: .osdu-gcp-dev2-variables
   needs: ["osdu-gcp-dev2-deploy-opa"]
   script:
     - !reference [.common_test_config, script]
     - >
+    - apk add py3-pip
     - pip install -r requirements_bootstrap.txt
     - tar -czf bundle.tar.gz --directory='./deployment/default-policies/' --exclude='./bootstrap_sequence.json' . --verbose
     - mkdir --parents ./policies ; mv bundle.tar.gz $_
@@ -190,71 +172,55 @@ osdu-gcp-dev2-bootstrap:
     - if: '$OSDU_GCP == "1" && $CI_COMMIT_TAG'
       when: on_success
 
-#FIXME there is no python 3.9 image with gcloud
 osdu-gcp-test-python:
   tags: ["osdu-small"]
-  image: ubuntu:20.04
+  image: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine
   needs: ["osdu-gcp-bootstrap"]
+  extends: .osdu-gcp-variables
+  variables:
+    POLICY_BUCKET: $OSDU_GCP_POLICY_BUCKET
+    CLOUD_PROVIDER: $OSDU_GCP_VENDOR
+    OPA_URL: $OSDU_GCP_OPA_URL
+    ENTITLEMENTS_BASE_URL: $HOST
+    LEGAL_BASE_URL: $HOST
+    BUNDLE_PAUSE: 36
+    PARTITION_BASE_URL: $HOST
   script:
+    - !reference [.common_test_config, script]
     - >
-    - apt-get update && apt-get install -y apt-transport-https ca-certificates gnupg curl
-    - echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
-    - curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
-    - apt-get update && apt-get -y install google-cloud-cli
-    - export DEBIAN_FRONTEND=noninteractive
-    - apt -y install software-properties-common
-    - add-apt-repository -y ppa:deadsnakes/ppa 
-    - apt -y install python3.9 python3.9-venv
-    - python3.9 -m venv env
-    - source env/bin/activate
-    - echo $OSDU_GCP_INTEGRATION_TESTER | base64 -d > OSDU_GCP_INTEGRATION_TESTER.json
-    - gcloud auth activate-service-account --key-file OSDU_GCP_INTEGRATION_TESTER.json
-    - gcloud config set project $OSDU_GCP_PROJECT
-    - export BEARER_TOKEN=`gcloud auth print-access-token`
+    - apk add py3-pip
     - pip install -r requirements.txt
     - pip install -r requirements_dev.txt
     - cd app
     - pip install -r requirements.txt
-    - export POLICY_BUCKET=$OSDU_GCP_POLICY_BUCKET
-    - export CLOUD_PROVIDER=gcp
-    - export OPA_URL=$OSDU_GCP_OPA_URL
-    - export ENTITLEMENTS_BASE_URL=$HOST
-    - export LEGAL_BASE_URL=$HOST
-    - export BUNDLE_PAUSE=36
-    - export PARTITION_BASE_URL=$HOST
     - export GOOGLE_APPLICATION_CREDENTIALS="../OSDU_GCP_INTEGRATION_TESTER.json"
+    - export BEARER_TOKEN=`gcloud auth print-access-token`
     - python3 -m pytest --token=$BEARER_TOKEN --service_url=$HOST --data_partition=$DATA_PARTITION
 
 osdu-gcp-dev2-test-python:
   tags: ["osdu-small"]
-  image: gcr.io/google.com/cloudsdktool/cloud-sdk
+  image: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine
   needs: ["osdu-gcp-dev2-bootstrap"]
+  extends: .osdu-gcp-dev2-variables
+  variables:
+    POLICY_BUCKET: $OSDU_GCP_POLICY_BUCKET
+    CLOUD_PROVIDER: $OSDU_GCP_VENDOR
+    OPA_URL: $OSDU_GCP_OPA_URL
+    ENTITLEMENTS_BASE_URL: $HOST
+    LEGAL_BASE_URL: $HOST
+    BUNDLE_PAUSE: 36
+    PARTITION_BASE_URL: $HOST
   script:
     - !reference [.common_test_config, script]
     - >
-    - export BEARER_TOKEN=`gcloud auth print-access-token`
+    - apk add py3-pip
     - pip install -r requirements.txt
     - pip install -r requirements_dev.txt
     - cd app
     - pip install -r requirements.txt
-    - export CLOUD_PROVIDER=gcp
-    - export BUNDLE_PAUSE=30
-    - echo ENTITLEMENTS_BASE_URL $ENTITLEMENTS_BASE_URL
-    - echo LEGAL_BASE_URL $LEGAL_BASE_URL
-    - export ENTITLEMENTS_BASE_URL=$HOST
-    - export LEGAL_BASE_URL=$HOST
-    - export OPA_URL=$OSDU_GCP_OPA_URL
-    - echo BUNDLE_PAUSE $BUNDLE_PAUSE
-    - echo DOMAIN was $DOMAIN
-    - export DOMAIN=group
-    - echo DOMAIN $DOMAIN
-    - echo CLOUD_PROVIDER $CLOUD_PROVIDER
-    - echo ENTITLEMENTS_BASE_URL $ENTITLEMENTS_BASE_URL
-    - echo LEGAL_BASE_URL $LEGAL_BASE_URL
-    - echo DATA_PARTITION $DATA_PARTITION
-    - echo OPA_URL $OPA_URL
-    - echo service_url $OSDU_GCP_POLICY_URL
-    - python3 -m pytest --token=$BEARER_TOKEN --service_url=$OSDU_GCP_POLICY_URL --data_partition=$DATA_PARTITION
+    - export GOOGLE_APPLICATION_CREDENTIALS="../OSDU_GCP_INTEGRATION_TESTER.json"
+    - export BEARER_TOKEN=`gcloud auth print-access-token`
+    - python3 -m pytest --token=$BEARER_TOKEN --service_url=$HOST --data_partition=$DATA_PARTITION
 
 osdu-gcp-containerize-bootstrap-gitlab:
   variables: