Merge branch 'gcp-int-tests-anthos' into 'master'

Update partition policy, refactored pipeline [GONRG-4958]

See merge request !191

.gitlab-ci.yml

@@ -13,33 +13,6 @@ variables:
   IBM_BUILD_SUBDIR: provider/partition-ibm
   IBM_INT_TEST_SUBDIR: testing/partition-test-ibm
 
-  OSDU_GCP_ENABLE_BOOTSTRAP: "true"
-  OSDU_GCP_SERVICE: partition
-  OSDU_GCP_VENDOR: gcp
-  OSDU_GCP_HELM_CONFIG_SERVICE_VARS: >
-    --set data.partitionAdminAccounts=$OSDU_GCP_PARTITION_ADMIN_ACCOUNTS
-    --set data.projectId=$OSDU_GCP_PROJECT
-    --set data.googleAudiences=$GOOGLE_AUDIENCE
-    --set data.logLevel=INFO
-    --set data.dataPartitionId=$OSDU_GCP_TENANT
-  OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS: >
-    --set data.image=$CI_REGISTRY_IMAGE/osdu-gcp-$OSDU_GCP_SERVICE:$CI_COMMIT_SHORT_SHA
-    --set data.bootstrapImage=$CI_REGISTRY_IMAGE/osdu-gcp-bootstrap-$OSDU_GCP_SERVICE:$CI_COMMIT_SHORT_SHA
-    --set data.serviceAccountName=$OSDU_GCP_SERVICE-k8s
-    --set data.bootstrapServiceAccountName=$OSDU_GCP_SERVICE-k8s
-  OSDU_GCP_HELM_CONFIG_SERVICE_VARS_DEV2: >
-    --set data.partitionAdminAccounts=$OSDU_GCP_PARTITION_ADMIN_ACCOUNTS_DEV2
-    --set data.projectId=$OSDU_GCP_PROJECT
-    --set data.googleAudiences=$GOOGLE_AUDIENCE
-    --set data.logLevel=INFO
-    --set data.dataPartitionId=$OSDU_GCP_TENANT
-  # FIXME add value below to DEV2 pipeline
-  OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS_DEV2: >
-    --set data.bootstrapImage=$CI_REGISTRY_IMAGE/osdu-gcp-bootstrap-$OSDU_GCP_SERVICE:$CI_COMMIT_SHORT_SHA
-    --set data.bootstrapServiceAccountName=$OSDU_GCP_BOOTSTRAP_SERVICE_ACCOUNT
-  OSDU_GCP_HELM_CONFIG_SERVICE: partition-config
-  OSDU_GCP_HELM_DEPLOYMENT_SERVICE: partition-deploy
-
 include:
   - project: "osdu/platform/ci-cd-pipelines"
     file: "standard-setup.yml"
@@ -68,47 +41,7 @@ include:
   - project: "osdu/platform/ci-cd-pipelines"
     file: "cloud-providers/osdu-gcp-gke.yml"
 
+  - local: "devops/gcp/pipeline/override-stages.yml"
+
   - project: "osdu/platform/ci-cd-pipelines"
     file: "publishing/pages.yml"
-
-osdu-gcp-deploy-deployment:
-  needs:
-    - osdu-gcp-containerize-gitlab
-    - osdu-gcp-containerize-bootstrap-gitlab
-    - osdu-gcp-deploy-configmap
-  after_script:
-    - echo ----- Verify Bootstrap -----
-    - kubectl rollout status deployment.v1.apps/$OSDU_GCP_SERVICE-bootstrap -n $OSDU_GCP_HELM_NAMESPACE --timeout=900s
-    - POD=$(kubectl get pod --sort-by=.metadata.creationTimestamp -n $OSDU_GCP_HELM_NAMESPACE | grep $OSDU_GCP_SERVICE-bootstrap | tail -1 | awk '{print $1}')
-    - STATUS=$(kubectl wait -n $OSDU_GCP_HELM_NAMESPACE --for=condition=Ready pod/$POD --timeout=300s)
-    - echo $STATUS
-    - if [[ "$STATUS" != *"met"* ]]; then echo "POD didn't start correctly" ; exit 1 ; fi
-
-osdu-gcp-dev2-deploy-deployment:
-  variables:
-    OSDU_GCP_BOOTSTRAP_SERVICE_ACCOUNT: workload-gke-bootstrap-sa
-  needs:
-    - osdu-gcp-containerize-gitlab
-    - osdu-gcp-containerize-bootstrap-gitlab
-    - osdu-gcp-dev2-deploy-configmap
-  after_script:
-    - echo ----- Verify Bootstrap -----
-    - kubectl rollout status deployment.v1.apps/$OSDU_GCP_SERVICE-bootstrap -n $OSDU_GCP_HELM_NAMESPACE --timeout=900s
-    - POD=$(kubectl get pod --sort-by=.metadata.creationTimestamp -n $OSDU_GCP_HELM_NAMESPACE | grep $OSDU_GCP_SERVICE-bootstrap | tail -1 | awk '{print $1}')
-    - STATUS=$(kubectl wait -n $OSDU_GCP_HELM_NAMESPACE --for=condition=Ready pod/$POD --timeout=300s)
-    - echo $STATUS
-    - if [[ "$STATUS" != *"met"* ]]; then echo "POD didn't start correctly" ; exit 1 ; fi
-
-osdu-gcp-anthos-deploy-deployment:
-  needs:
-    - osdu-gcp-containerize-gitlab
-    - osdu-gcp-containerize-bootstrap-gitlab
-    - osdu-gcp-anthos-deploy-configmap
-
-osdu-gcp-test:
-  variables:
-    CLIENT_TENANT: osdu
-
-osdu-gcp-dev2-test:
-  variables:
-    CLIENT_TENANT: devtwo

devops/gcp/deploy/templates/partition-authorization-policy.yml → devops/gcp/deploy/templates/authorization-policy.yml

 {{- if .Values.conf.onPremEnabled }}
-{{- range $key, $spec := .Values.authorizations }}
 apiVersion: security.istio.io/v1beta1
 kind: AuthorizationPolicy
 metadata:
-  name: {{ (print $key "-default") | lower | quote }}
-  labels:
-    app.kubernetes.io/name: {{ $key | quote }}
-    app.kubernetes.io/managed-by: {{ $.Release.Service | quote }}
-  namespace: {{ $.Release.Namespace | quote }}
+  name: "{{ .Values.conf.appName }}-jwt-policy"
+  namespace: "{{ .Release.Namespace }}"
 spec:
   selector:
     matchLabels:
-{{- toYaml $spec.matchLabels | nindent 6 }}
+      app: "{{ .Values.conf.appName }}"
   action: ALLOW
   rules:
   - from:
@@ -55,5 +51,28 @@ spec:
         - GET
         paths:
         - /api/partition/v1/*
-{{- end }}
+  {{- if .Values.conf.cicdEnabled }}
+  - from:
+    - source:
+       requestPrincipals: ["*"]
+    to:
+    - operation:
+        methods:
+        - POST
+        - PATCH
+        - GET
+        - DELETE
+        - OPTIONS
+        paths:
+        - /api/partition/v1/*
+    when:
+    - key: request.auth.claims[iss]
+      values:
+      - "https://keycloak.{{ .Values.conf.domain }}/auth/realms/{{ .Values.auth.realm }}"
+      - "http://keycloak.{{ .Values.conf.domain }}/auth/realms/{{ .Values.auth.realm }}"
+      - "http://keycloak.{{ .Release.Namespace }}.svc.cluster.local/auth/realms/{{ .Values.auth.realm }}"
+    - key: request.auth.claims[email]
+      values:
+      - "integration-tester@service.local"
+  {{- end }}
 {{- end }}

devops/gcp/deploy/templates/request-authentication.yml

+{{- if .Values.conf.onPremEnabled }}
+apiVersion: security.istio.io/v1beta1
+kind: RequestAuthentication
+metadata:
+  name: "{{ .Values.conf.appName }}-jwt-policy"
+  namespace: "{{ .Release.Namespace }}"
+spec:
+  selector:
+    matchLabels:
+      app: "{{ .Values.conf.appName }}"
+  jwtRules:
+  - issuer: "https://keycloak.{{ .Values.conf.domain }}/auth/realms/{{ .Values.auth.realm }}"
+    jwksUri: "http://keycloak.{{ .Release.Namespace }}.svc.cluster.local/auth/realms/{{ .Values.auth.realm }}/protocol/openid-connect/certs"
+    forwardOriginalToken: true
+  - issuer: "http://keycloak.{{ .Values.conf.domain }}/auth/realms/{{ .Values.auth.realm }}"
+    jwksUri: "http://keycloak.{{ .Release.Namespace }}.svc.cluster.local/auth/realms/{{ .Values.auth.realm }}/protocol/openid-connect/certs"
+    forwardOriginalToken: true
+  - issuer: "http://keycloak.{{ .Release.Namespace }}.svc.cluster.local/auth/realms/{{ .Values.auth.realm }}"
+    jwksUri: "http://keycloak.{{ .Release.Namespace }}.svc.cluster.local/auth/realms/{{ .Values.auth.realm }}/protocol/openid-connect/certs"
+    forwardOriginalToken: true
+{{- end }}

devops/gcp/deploy/values.yaml

@@ -12,14 +12,13 @@ data:
   bootstrapImage: ""
   bootstrapServiceAccountName: ""
 conf:
-  configmap: "partition-config"
-  secret: "partition-postgres-secret"
   appName: "partition"
-  onPremEnabled: false
+  cicdEnabled: false
+  configmap: "partition-config"
   domain: ""
+  onPremEnabled: false
+  secret: "partition-postgres-secret"
 namespacePolicy:
-  mtlsMode: STRICT
-authorizations:
-  partitionPolicy:
-    matchLabels:
-      app: partition
+    mtlsMode: STRICT
+auth:
+    realm: "osdu"

devops/gcp/pipeline/override-stages.yml

+variables:
+  OSDU_GCP_ENABLE_BOOTSTRAP: "true"
+  OSDU_GCP_SERVICE: partition
+  OSDU_GCP_VENDOR: gcp
+  OSDU_GCP_HELM_CONFIG_SERVICE_VARS: >
+    --set data.partitionAdminAccounts=$OSDU_GCP_PARTITION_ADMIN_ACCOUNTS
+    --set data.projectId=$OSDU_GCP_PROJECT
+    --set data.googleAudiences=$GOOGLE_AUDIENCE
+    --set data.logLevel=INFO
+    --set data.dataPartitionId=$OSDU_GCP_TENANT
+  OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS: >
+    --set data.image=$CI_REGISTRY_IMAGE/osdu-gcp-$OSDU_GCP_SERVICE:$CI_COMMIT_SHORT_SHA
+    --set data.bootstrapImage=$CI_REGISTRY_IMAGE/osdu-gcp-bootstrap-$OSDU_GCP_SERVICE:$CI_COMMIT_SHORT_SHA
+    --set data.serviceAccountName=$OSDU_GCP_SERVICE-k8s
+    --set data.bootstrapServiceAccountName=$OSDU_GCP_SERVICE-k8s
+  OSDU_GCP_HELM_CONFIG_SERVICE_VARS_DEV2: >
+    --set data.partitionAdminAccounts=$OSDU_GCP_PARTITION_ADMIN_ACCOUNTS_DEV2
+    --set data.projectId=$OSDU_GCP_PROJECT
+    --set data.googleAudiences=$GOOGLE_AUDIENCE
+    --set data.logLevel=INFO
+    --set data.dataPartitionId=$OSDU_GCP_TENANT
+  # FIXME add value below to DEV2 pipeline
+  OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS_DEV2: >
+    --set data.bootstrapImage=$CI_REGISTRY_IMAGE/osdu-gcp-bootstrap-$OSDU_GCP_SERVICE:$CI_COMMIT_SHORT_SHA
+    --set data.bootstrapServiceAccountName=$OSDU_GCP_BOOTSTRAP_SERVICE_ACCOUNT
+  OSDU_GCP_HELM_CONFIG_SERVICE: partition-config
+  OSDU_GCP_HELM_DEPLOYMENT_SERVICE: partition-deploy
+
+osdu-gcp-deploy-deployment:
+  needs:
+    - osdu-gcp-containerize-gitlab
+    - osdu-gcp-containerize-bootstrap-gitlab
+    - osdu-gcp-deploy-configmap
+  after_script:
+    - echo ----- Verify Bootstrap -----
+    - kubectl rollout status deployment.v1.apps/$OSDU_GCP_SERVICE-bootstrap -n $OSDU_GCP_HELM_NAMESPACE --timeout=900s
+    - POD=$(kubectl get pod --sort-by=.metadata.creationTimestamp -n $OSDU_GCP_HELM_NAMESPACE | grep $OSDU_GCP_SERVICE-bootstrap | tail -1 | awk '{print $1}')
+    - STATUS=$(kubectl wait -n $OSDU_GCP_HELM_NAMESPACE --for=condition=Ready pod/$POD --timeout=300s)
+    - echo $STATUS
+    - if [[ "$STATUS" != *"met"* ]]; then echo "POD didn't start correctly" ; exit 1 ; fi
+
+osdu-gcp-dev2-deploy-deployment:
+  variables:
+    OSDU_GCP_BOOTSTRAP_SERVICE_ACCOUNT: workload-gke-bootstrap-sa
+  needs:
+    - osdu-gcp-containerize-gitlab
+    - osdu-gcp-containerize-bootstrap-gitlab
+    - osdu-gcp-dev2-deploy-configmap
+  after_script:
+    - echo ----- Verify Bootstrap -----
+    - kubectl rollout status deployment.v1.apps/$OSDU_GCP_SERVICE-bootstrap -n $OSDU_GCP_HELM_NAMESPACE --timeout=900s
+    - POD=$(kubectl get pod --sort-by=.metadata.creationTimestamp -n $OSDU_GCP_HELM_NAMESPACE | grep $OSDU_GCP_SERVICE-bootstrap | tail -1 | awk '{print $1}')
+    - STATUS=$(kubectl wait -n $OSDU_GCP_HELM_NAMESPACE --for=condition=Ready pod/$POD --timeout=300s)
+    - echo $STATUS
+    - if [[ "$STATUS" != *"met"* ]]; then echo "POD didn't start correctly" ; exit 1 ; fi
+
+osdu-gcp-anthos-deploy-deployment:
+  needs:
+    - osdu-gcp-containerize-gitlab
+    - osdu-gcp-containerize-bootstrap-gitlab
+    - osdu-gcp-anthos-deploy-configmap
+
+osdu-gcp-test:
+  variables:
+    CLIENT_TENANT: osdu
+
+osdu-gcp-dev2-test:
+  variables:
+    CLIENT_TENANT: devtwo