[GONRG-7838] Extend legal bootstrap to all partitions

NOTICE

@@ -31,7 +31,7 @@ The following software have components provided under the terms of this license:
 - Doxia :: FML Module (from https://repo1.maven.org/maven2/org/apache/maven/doxia/doxia-module-fml)
 - Doxia :: Sink API (from https://repo1.maven.org/maven2/org/apache/maven/doxia/doxia-sink-api)
 - Doxia :: XDoc Module (from https://repo1.maven.org/maven2/org/apache/maven/doxia/doxia-module-xdoc)
-- Maven Core (from https://repo1.maven.org/maven2/org/apache/maven/maven-core)
+- Maven Core (from https://maven.apache.org/ref/4.0.0-alpha-8/maven-core/, https://repo1.maven.org/maven2/org/apache/maven/maven-core)
 - Maven Error Diagnostics (from https://repo1.maven.org/maven2/org/apache/maven/maven-error-diagnostics)
 - Maven Monitor (from https://repo1.maven.org/maven2/org/apache/maven/maven-monitor)
 - Maven Plugin Descriptor Model (from https://repo1.maven.org/maven2/org/apache/maven/maven-plugin-descriptor)
@@ -367,7 +367,7 @@ The following software have components provided under the terms of this license:
 - Byte Buddy Java agent (from https://repo1.maven.org/maven2/net/bytebuddy/byte-buddy-agent)
 - ClassMate (from http://github.com/cowtowncoder/java-classmate)
 - Cloud Key Management Service (KMS) API (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-cloudkms)
-- Cloud Storage JSON API v1-rev20231012-2.0.0 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-storage)
+- Cloud Storage JSON API v1-rev20231028-2.0.0 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-storage)
 - CloudWatch Metrics for AWS Java SDK (from https://aws.amazon.com/sdkforjava)
 - Cobertura (from http://cobertura.sourceforge.net)
 - Cobertura Limited Runtime (from http://cobertura.sourceforge.net)
@@ -464,20 +464,20 @@ The following software have components provided under the terms of this license:
 - Kotlin Stdlib Common (from https://kotlinlang.org/)
 - Kotlin Stdlib Jdk7 (from <https://kotlinlang.org/>, https://kotlinlang.org/)
 - Kotlin Stdlib Jdk8 (from <https://kotlinlang.org/>, https://kotlinlang.org/)
-- Maven Artifact (from https://repo1.maven.org/maven2/org/apache/maven/maven-artifact)
+- Maven Artifact (from https://maven.apache.org/ref/4.0.0-alpha-8/maven-artifact/, https://repo1.maven.org/maven2/org/apache/maven/maven-artifact)
 - Maven Artifact Manager (from https://repo1.maven.org/maven2/org/apache/maven/maven-artifact-manager)
-- Maven Core (from https://repo1.maven.org/maven2/org/apache/maven/maven-core)
+- Maven Core (from https://maven.apache.org/ref/4.0.0-alpha-8/maven-core/, https://repo1.maven.org/maven2/org/apache/maven/maven-core)
 - Maven Error Diagnostics (from https://repo1.maven.org/maven2/org/apache/maven/maven-error-diagnostics)
-- Maven Model (from https://repo1.maven.org/maven2/org/apache/maven/maven-model)
+- Maven Model (from https://maven.apache.org/ref/4.0.0-alpha-8/maven-model/, https://repo1.maven.org/maven2/org/apache/maven/maven-model)
 - Maven Monitor (from https://repo1.maven.org/maven2/org/apache/maven/maven-monitor)
-- Maven Plugin API (from https://repo1.maven.org/maven2/org/apache/maven/maven-plugin-api)
+- Maven Plugin API (from https://maven.apache.org/ref/4.0.0-alpha-8/maven-plugin-api/, https://repo1.maven.org/maven2/org/apache/maven/maven-plugin-api)
 - Maven Plugin Descriptor Model (from https://repo1.maven.org/maven2/org/apache/maven/maven-plugin-descriptor)
 - Maven Plugin Parameter Documenter API (from https://repo1.maven.org/maven2/org/apache/maven/maven-plugin-parameter-documenter)
 - Maven Plugin Registry Model (from https://repo1.maven.org/maven2/org/apache/maven/maven-plugin-registry)
 - Maven Profile Model (from https://repo1.maven.org/maven2/org/apache/maven/maven-profile)
 - Maven Project (from https://repo1.maven.org/maven2/org/apache/maven/maven-project)
-- Maven Repository Metadata Model (from https://repo1.maven.org/maven2/org/apache/maven/maven-repository-metadata)
-- Maven Settings (from https://repo1.maven.org/maven2/org/apache/maven/maven-settings)
+- Maven Repository Metadata Model (from https://maven.apache.org/ref/4.0.0-alpha-8/maven-repository-metadata/, https://repo1.maven.org/maven2/org/apache/maven/maven-repository-metadata)
+- Maven Settings (from https://maven.apache.org/ref/4.0.0-alpha-8/maven-settings/, https://repo1.maven.org/maven2/org/apache/maven/maven-settings)
 - Metrics Core (from https://repo1.maven.org/maven2/io/dropwizard/metrics/metrics-core)
 - Microsoft Application Insights Java Agent (from https://github.com/Microsoft/ApplicationInsights-Java)
 - Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java)
@@ -969,7 +969,6 @@ The following software have components provided under the terms of this license:
 - Apache HttpClient (from http://hc.apache.org/httpcomponents-client, http://hc.apache.org/httpcomponents-client-ga, https://repo1.maven.org/maven2/org/apache/httpcomponents/client5/httpclient5)
 - Apache HttpClient Cache (from http://hc.apache.org/httpcomponents-client, http://hc.apache.org/httpcomponents-client-ga)
 - Apache Log4j API (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-api)
-- Apache Log4j Core (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core)
 - Apache Log4j SLF4J Binding (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-slf4j-impl)
 - Apache Log4j to SLF4J Adapter (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-to-slf4j)
 - Azure Java Client Authentication Library for AutoRest (from https://github.com/Azure/autorest-clientruntime-for-java)

devops/gc/bootstrap_legalstatus_update/update_legal_status.sh

@@ -27,45 +27,31 @@
 
 set -ex
 
-update_legal_status_baremetal() {
-
-  DATA_PARTITION_ID=$1
-
-  ID_TOKEN="$(curl --location --silent --globoff --request POST "${OPENID_PROVIDER_URL}/protocol/openid-connect/token" \
-    --header "data-partition-id: ${DATA_PARTITION_ID}" \
-    --header "Content-Type: application/x-www-form-urlencoded" \
-    --data-urlencode "grant_type=client_credentials" \
-    --data-urlencode "scope=openid" \
-    --data-urlencode "client_id=${OPENID_PROVIDER_CLIENT_ID}" \
-    --data-urlencode "client_secret=${OPENID_PROVIDER_CLIENT_SECRET}" | jq -r ".id_token")"
-  export ID_TOKEN
-
-  status_code=$(curl --location --globoff --request GET "${LEGAL_HOST}/api/legal/v1/jobs/updateLegalTagStatus" \
-    --write-out "%{http_code}" --silent --output "output.txt" \
-    --header "data-partition-id: ${DATA_PARTITION_ID}" \
-    --header "Authorization: Bearer ${ID_TOKEN}")
-
-  if [ "$status_code" == 204 ]; then
-    echo "Legal status update completed successfully!"
+get_token() {
+  if [ "${ONPREM_ENABLED}" == "true" ]; then
+    # id token
+    TOKEN="$(curl --location --silent --globoff --request POST "${OPENID_PROVIDER_URL}/protocol/openid-connect/token" \
+      --header "Content-Type: application/x-www-form-urlencoded" \
+      --data-urlencode "grant_type=client_credentials" \
+      --data-urlencode "scope=openid" \
+      --data-urlencode "client_id=${OPENID_PROVIDER_CLIENT_ID}" \
+      --data-urlencode "client_secret=${OPENID_PROVIDER_CLIENT_SECRET}" | jq -r ".id_token")"
+    export TOKEN
   else
-    echo "Legal status update failed!"
-    cat /opt/output.txt | jq
-    exit 1
+    # access token
+    TOKEN="$(gcloud auth print-access-token)"
+    export TOKEN
   fi
-
 }
 
-update_legal_status_gc() {
+update_legal_status() {
 
   DATA_PARTITION_ID=$1
 
-  ACCESS_TOKEN="$(gcloud auth print-access-token)"
-  export ACCESS_TOKEN
-
   status_code=$(curl --location --globoff --request GET "${LEGAL_HOST}/api/legal/v1/jobs/updateLegalTagStatus" \
     --write-out "%{http_code}" --silent --output "output.txt" \
     --header "data-partition-id: ${DATA_PARTITION_ID}" \
-    --header "Authorization: Bearer ${ACCESS_TOKEN}")
+    --header "Authorization: Bearer ${TOKEN}")
 
   if [ "$status_code" == 204 ]; then
     echo "Legal status update completed successfully!"
@@ -78,7 +64,7 @@ update_legal_status_gc() {
 }
 
 # Check variables
-source ./validate-env.sh "DATA_PARTITION_ID"
+source ./validate-env.sh "PARTITION_HOST"
 source ./validate-env.sh "LEGAL_HOST"
 if [[ "${ONPREM_ENABLED}" == "true" ]]; then
   source ./validate-env.sh "OPENID_PROVIDER_URL"
@@ -86,13 +72,33 @@ if [[ "${ONPREM_ENABLED}" == "true" ]]; then
   source ./validate-env.sh "OPENID_PROVIDER_CLIENT_SECRET"
 fi
 
-# Update legal status for all partitions
+# Get list of partitions 
+status_code=$(curl --location --request GET \
+    --url "${PARTITION_HOST}/api/partition/v1/partitions" \
+    --write-out "%{http_code}" --silent --output "output.txt")
 
-if [[ "${ONPREM_ENABLED}" == "true" ]]; then
-  update_legal_status_baremetal "${DATA_PARTITION_ID}"
+if [ "$status_code" == 200 ]; then
+    partitions=$(cat /opt/output.txt | xargs)           # unquote
+    partitions=${partitions:1:-1}                       # remove []
+    IFS=',' read -ra PARTITIONS <<<"${partitions},"     # append ',' for single partition case
 else
-  update_legal_status_gc "${DATA_PARTITION_ID}"
+    echo "$status_code: Partition service is not available"
+    cat /opt/output.txt
+    exit 1
 fi
 
+# Update legal status for all partitions
+for PARTITION in "${PARTITIONS[@]}"; do
+  if [[ "$PARTITION" == "system" ]]; then
+      continue
+  fi
+  get_token
+  update_legal_status "${PARTITION}"
+done
+
+# cleanly exit envoy if present
+set +e
 curl -X POST http://localhost:15000/quitquitquit
+set -e
+
 exit 0

devops/gc/deploy/templates/bootstrap-deployment.yaml

-{{- if .Values.global.dataBootstrapEnabled }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -42,4 +41,3 @@ spec:
             allowPrivilegeEscalation: false
             runAsNonRoot: true
       serviceAccountName: {{ .Values.data.bootstrapServiceAccountName | quote }}
-{{- end }}

devops/gc/deploy/templates/configmap-bootstrap.yaml

-{{- if .Values.global.dataBootstrapEnabled }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -7,9 +6,8 @@ metadata:
   name: {{ printf "%s-bootstrap" .Values.conf.configmap | quote }}
   namespace: {{ .Release.Namespace | quote }}
 data:
-  DATA_PARTITION_ID: {{ .Values.data.dataPartitionId | quote }}
   ENTITLEMENTS_HOST: {{ .Values.data.entitlementsHost | quote }}
   LEGAL_HOST: {{ .Values.data.legalHost | quote }}
+  PARTITION_HOST: {{ .Values.data.partitionHost | quote }}
   DEFAULT_LEGAL_TAG: {{ .Values.data.defaultLegalTag | quote }}
   ONPREM_ENABLED: {{ .Values.global.onPremEnabled | quote }}
-{{- end }}

devops/gc/deploy/templates/legal-status-update-configmap.yaml

@@ -4,7 +4,6 @@ metadata:
   name: {{ printf "%s-status-update" .Values.conf.configmap | quote }}
   namespace: {{ .Release.Namespace | quote }}
 data:
-  DATA_PARTITION_ID: {{ .Values.data.dataPartitionId | quote }}
   LEGAL_HOST: {{ .Values.data.legalHost | quote }}
+  PARTITION_HOST: {{ .Values.data.partitionHost | quote }}
   ONPREM_ENABLED: {{ .Values.global.onPremEnabled | quote }}
-

devops/gc/deploy/values.yaml

@@ -13,10 +13,10 @@ data:
   logLevel: "ERROR"
   springProfilesActive: "gcp"
   acceptHttp: "true"
-  dataPartitionId: ""
   entitlementsHost: "http://entitlements"
   defaultLegalTag: "default-data-tag"
   legalHost: "http://legal"
+  partitionHost: "http://partition"
   # deployments
   requestsCpu: "5m"
   requestsMemory: "325Mi"

provider/legal-gc/bootstrap/bootstrap.sh

@@ -7,28 +7,30 @@
 
 set -ex
 
-source ./validate-env.sh "DATA_PARTITION_ID"
+source ./validate-env.sh "PARTITION_HOST"
 source ./validate-env.sh "LEGAL_HOST"
 source ./validate-env.sh "ENTITLEMENTS_HOST"
 source ./validate-env.sh "DEFAULT_LEGAL_TAG"
 
-get_token_onprem() {
-    ID_TOKEN="$(curl --location --request POST "${OPENID_PROVIDER_URL}/protocol/openid-connect/token" \
-        --header "Content-Type: application/x-www-form-urlencoded" \
-        --data-urlencode "grant_type=client_credentials" \
-        --data-urlencode "scope=openid" \
-        --data-urlencode "client_id=${OPENID_PROVIDER_CLIENT_ID}" \
-        --data-urlencode "client_secret=${OPENID_PROVIDER_CLIENT_SECRET}" | jq -r ".id_token")"
-    export ID_TOKEN
-}
-
-get_token_gc() {
-    ID_TOKEN=$(gcloud auth print-identity-token)
+get_token() {
+    if [ "${ONPREM_ENABLED}" == "true" ]; then
+        ID_TOKEN="$(curl --location --request POST "${OPENID_PROVIDER_URL}/protocol/openid-connect/token" \
+            --header "Content-Type: application/x-www-form-urlencoded" \
+            --data-urlencode "grant_type=client_credentials" \
+            --data-urlencode "scope=openid" \
+            --data-urlencode "client_id=${OPENID_PROVIDER_CLIENT_ID}" \
+            --data-urlencode "client_secret=${OPENID_PROVIDER_CLIENT_SECRET}" | jq -r ".id_token")"
+    else
+        ID_TOKEN=$(gcloud auth print-identity-token)
+    fi
     export ID_TOKEN
 }
 
 check_entitlements_readiness() {
-    status_code=$(curl --retry 1 --location -globoff --request GET \
+    
+    DATA_PARTITION_ID=$1
+
+    status_code=$(curl --retry 1 --location --globoff --request GET \
         "${ENTITLEMENTS_HOST}/api/entitlements/v2/groups" \
         --write-out "%{http_code}" --silent --output "/dev/null" \
         --header 'Content-Type: application/json' \
@@ -64,6 +66,8 @@ create_legaltag() {
 }
 EOF
 
+    DATA_PARTITION_ID=$1
+
     # FIXME update after default tag logic is defined
     status_code=$(curl --location -g --request POST \
         --url "${LEGAL_HOST}/api/legal/v1/legaltags" \
@@ -84,21 +88,35 @@ EOF
     rm /opt/output.txt
 }
 
+# Get list of partitions 
+status_code=$(curl --location --request GET \
+    --url "${PARTITION_HOST}/api/partition/v1/partitions" \
+    --write-out "%{http_code}" --silent --output "output.txt")
+
+if [ "$status_code" == 200 ]; then
+    partitions=$(cat /opt/output.txt | xargs)           # unquote
+    partitions=${partitions:1:-1}                       # remove []
+    IFS=',' read -ra PARTITIONS <<<"${partitions},"     # append ',' for single partition case
+else
+    echo "$status_code: Partition service is not available"
+    cat /opt/output.txt
+    exit 1
+fi
+
 if [ "${ONPREM_ENABLED}" == "true" ]; then
     source ./validate-env.sh "OPENID_PROVIDER_URL"
     source ./validate-env.sh "OPENID_PROVIDER_CLIENT_ID"
     source ./validate-env.sh "OPENID_PROVIDER_CLIENT_SECRET"
-
-    get_token_onprem
-
-else
-
-    get_token_gc
-
 fi
 
-check_entitlements_readiness
-
-create_legaltag
+# Bootstrapping legal tag for each partition
+for PARTITION in "${PARTITIONS[@]}"; do
+    if [[ "${PARTITION}" == "system" ]]; then
+        continue
+    fi
+    get_token
+    check_entitlements_readiness ${PARTITION}
+    create_legaltag ${PARTITION}
+done
 
 touch /tmp/bootstrap_ready

provider/legal-gc/bootstrap/validate-env.sh

-#!/bin/bash
-#  Copyright 2020 Google LLC
-#  Copyright 2017-2019, Schlumberger
-#  Copyright 2022 EPAM
+#!/usr/bin/env bash
+#  Copyright 2023 Google LLC
+#  Copyright 2023 EPAM
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -13,12 +12,18 @@
 #  distributed under the License is distributed on an "AS IS" BASIS,
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
-#  limitations under the License.
+#  limitations under the License. 
+
+{ set +x ;} 2> /dev/null # disable output to prevent secret logging
+set -e
 
 ENV_VAR_NAME=$1
 
 if [ "${!ENV_VAR_NAME}" = "" ]
 then
     echo "Missing environment variable '$ENV_VAR_NAME'. Please provide all variables and try again"
+    { set -x ;} 2> /dev/null # enable output back
     exit 1
 fi
+
+{ set -x ;} 2> /dev/null # enable output back