diff --git a/NOTICE b/NOTICE index 856a94e8130c0fd44aa123d718b03cb711c76ad8..bf272e6ee3849b5bf1452c7fa04b48325a38650e 100644 --- a/NOTICE +++ b/NOTICE @@ -456,7 +456,7 @@ The following software have components provided under the terms of this license: - Jackson extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-jackson) - Jackson module: Afterburner (from http://wiki.fasterxml.com/JacksonHome, https://github.com/FasterXML/jackson-modules-base) - Jackson module: JAXB Annotations (from http://github.com/FasterXML/jackson-module-jaxb-annotations, http://wiki.fasterxml.com/JacksonJAXBAnnotations, https://github.com/FasterXML/jackson-modules-base) -- Jackson-annotations (from http://github.com/FasterXML/jackson, http://wiki.fasterxml.com/JacksonHome) +- Jackson-annotations (from http://github.com/FasterXML/jackson, http://wiki.fasterxml.com/JacksonHome, https://github.com/FasterXML/jackson) - Jackson-core (from http://wiki.fasterxml.com/JacksonHome, https://github.com/FasterXML/jackson-core) - Jackson-dataformat-XML (from http://wiki.fasterxml.com/JacksonExtensionXmlDataBinding, https://github.com/FasterXML/jackson-dataformat-xml) - Jackson-dataformat-YAML (from https://github.com/FasterXML/jackson, https://github.com/FasterXML/jackson-dataformats-text) @@ -635,7 +635,7 @@ The following software have components provided under the terms of this license: - io.grpc:grpc-protobuf-lite (from https://github.com/grpc/grpc-java) - io.grpc:grpc-stub (from https://github.com/grpc/grpc-java) - ion-java (from https://github.com/amzn/ion-java/, https://github.com/amznlabs/ion-java/) -- jackson-databind (from http://github.com/FasterXML/jackson, http://wiki.fasterxml.com/JacksonHome) +- jackson-databind (from http://github.com/FasterXML/jackson, http://wiki.fasterxml.com/JacksonHome, https://github.com/FasterXML/jackson) - java-cloudant (from https://cloudant.com) - javatuples (from http://www.javatuples.org) - javax.annotation-api (from http://jcp.org/en/jsr/detail?id=250) diff --git a/devops/gcp/configmap/README.md b/devops/gcp/configmap/README.md index 57dc3db04b3520c4e7d72a079385535a0aaebfbb..fd2b4ff27bfc5007f6891e39fe4255f753fb4a4e 100644 --- a/devops/gcp/configmap/README.md +++ b/devops/gcp/configmap/README.md @@ -120,6 +120,7 @@ cat ~/.config/gcloud/application_default_credentials.json | grep client_id |------|-------------|------|---------|---------| **partitionName** | name of the partition | string | partition | yes **dataPartitionId** | ID of data partition | string | - | yes +**dataPartitionIdList** | List of data partitions | array of strings | [] | no **datafierSa** | datafier service account | string | datafier | yes ### Bootstrap on-prem variables @@ -136,6 +137,7 @@ cat ~/.config/gcloud/application_default_credentials.json | grep client_id **configmap** | configmap to be used | string | partition-config | yes **appName** | name of the app | string | partition | yes **onPremEnabled** | whether on-prem is enabled | boolean | false | yes +**multiPartitionEnabled** | whether multipartition is enabled | boolean | false | yes ### Install the helm chart diff --git a/devops/gcp/configmap/templates/configmap-bootstrap.yml b/devops/gcp/configmap/templates/configmap-bootstrap.yml index 9917f21158cea47808f390e04d2bc8e438cbcf64..7da45e0682cfd4239884ac1078e090acd97dc4fd 100644 --- a/devops/gcp/configmap/templates/configmap-bootstrap.yml +++ b/devops/gcp/configmap/templates/configmap-bootstrap.yml @@ -18,9 +18,15 @@ data: {{- if .Values.conf.onPremEnabled }} SERVICE_ACCOUNT: {{ printf "%s@service.local" .Values.data.datafierSa | quote }} DOMAIN: "{{ .Values.data.domain }}" - MINIO_ENDPOINT: "{{ default "http://minio:9000" .Values.data.minioExternalEndpoint }}" - FILE_MINIO_ENDPOINT: "{{ default (printf "https://s3.%s" .Values.data.domain) .Values.data.minioExternalEndpoint }}" + MINIO_ENDPOINT: {{ default "http://minio:9000" .Values.data.minioExternalEndpoint | quote }} + FILE_MINIO_ENDPOINT: {{ default (printf "https://s3.%s" .Values.data.domain) .Values.data.minioExternalEndpoint | quote }} {{- else }} SERVICE_ACCOUNT: {{ printf "%s@%s.iam.gserviceaccount.com" .Values.data.datafierSa .Values.data.projectId | quote }} AUDIENCES: "{{ .Values.data.googleAudiences }}" {{- end }} + {{- if .Values.conf.multiPartitionEnabled }} + DATA_PARTITION_ID_LIST: {{ join "," .Values.data.dataPartitionIdList | quote }} + MULTI_PARTITON_ENABLED: "{{ .Values.conf.multiPartitionEnabled }}" + {{- else }} + MULTI_PARTITON_ENABLED: "{{ .Values.conf.multiPartitionEnabled }}" + {{- end }} diff --git a/devops/gcp/configmap/values.yaml b/devops/gcp/configmap/values.yaml index b0b7f10c887595945e886a09fe811b46a14bf238..83cb378b54ba0695ae1c7c3e0408cde8cb356ec9 100644 --- a/devops/gcp/configmap/values.yaml +++ b/devops/gcp/configmap/values.yaml @@ -11,12 +11,14 @@ data: # bootstrap common partitionName: "partition" dataPartitionId: "" + dataPartitionIdList: [] datafierSa: "datafier" partitionCleanUpEnabled: "false" - # bootstrap variables onprem + # bootstrap onprem domain: "" minioExternalEndpoint: "" # use only if external minio is configured conf: configmap: "partition-config" appName: "partition" onPremEnabled: false + multiPartitionEnabled: false diff --git a/provider/partition-gcp/bootstrap/bootstrap_partition.sh b/provider/partition-gcp/bootstrap/bootstrap_partition.sh index 3423ae084cb47521486ec7e6e363b42509bd373c..8a471136da8ca5844f5de970b8650c5606faaf2e 100644 --- a/provider/partition-gcp/bootstrap/bootstrap_partition.sh +++ b/provider/partition-gcp/bootstrap/bootstrap_partition.sh @@ -5,23 +5,27 @@ set -ex source ./data_anthos.sh source ./data_gcp.sh +# Bootstrap Partition service on Anthos (on-prem) bootstrap_anthos() { - if [ "$PARTITION_CLEAN_UP_ENABLED" == "true" ] - then + + DATA_PARTITION_ID=$1 + DATA_PARTITION_ID_UPPER=$2 + + if [[ "${PARTITION_CLEAN_UP_ENABLED}" == "true" ]]; then echo "Partition cleanup enabled, will delete partition ${DATA_PARTITION_ID}" + delete_status_code=$(curl -X DELETE \ - --url "http://${PARTITION_NAME}/api/partition/v1/partitions/${DATA_PARTITION_ID}" --write-out "%{http_code}" --silent --output "/dev/null" \ - -H "Content-Type: application/json") - - if [ "$delete_status_code" == 204 ] || [ "$delete_status_code" == 404 ] - then - echo "Partition deletion was successful, with status code: ${delete_status_code}" - else - echo "Not able to delete partition, status code is: ${delete_status_code}" - exit 1 - fi + --url "http://${PARTITION_NAME}/api/partition/v1/partitions/${DATA_PARTITION_ID}" --write-out "%{http_code}" --silent --output "/dev/null" \ + -H "Content-Type: application/json") + + if [[ "${delete_status_code}" == 204 ]] || [[ "${delete_status_code}" == 404 ]]; then + echo "Partition deletion was successful with status code: ${delete_status_code}" + else + echo "Not able to delete partition with status code: ${delete_status_code}" + exit 1 + fi else - echo "Partition cleanup not enabled, skipping deletion" + echo "Partition cleanup is not enabled, skipping deletion" fi status_code=$(curl -X POST \ @@ -30,42 +34,45 @@ bootstrap_anthos() { --data-raw "$(generate_post_data_anthos)") # shellcheck disable=SC2002 - if [ "$status_code" == 201 ] - then + if [[ "${status_code}" == 201 ]]; then echo "Partition bootstrap finished successfully!" - elif [ "$status_code" == 409 ] - then + elif [[ "${status_code}" == 409 ]]; then + patch_status_code=$(curl -X PATCH \ --url "http://${PARTITION_NAME}/api/partition/v1/partitions/${DATA_PARTITION_ID}" --write-out "%{http_code}" --silent --output "/dev/null" \ -H "Content-Type: application/json" \ --data-raw "$(generate_post_data_anthos)") - echo "Partition was patched because Postgres Database had already had entities! Status code of patching: $patch_status_code" + + echo "Partition was patched because Postgres Database had already had entities! Status code: ${patch_status_code}" else - echo "Exiting with status code: $status_code" + echo "Exiting with status code: ${status_code}" exit 1 fi } +# Bootstrap Partition service on Google Cloud bootstrap_gcp() { + echo "sleep to prevent 500 response from the Partition service, due to timeout of creation for Workload Identity" sleep 20 + DATA_PARTITION_ID=$1 + DATA_PARTITION_ID_UPPER=$2 IDENTITY_TOKEN=$(gcloud auth print-identity-token --audiences="${AUDIENCES}") - if [ "$PARTITION_CLEAN_UP_ENABLED" == "true" ] - then - echo "Partition cleanup enabled, will delete partition ${DATA_PARTITION_ID}" - delete_status_code=$(curl -X DELETE \ - --url "http://${PARTITION_NAME}/api/partition/v1/partitions/${DATA_PARTITION_ID}" --write-out "%{http_code}" --silent --output "/dev/null" \ - -H "Authorization: Bearer ${IDENTITY_TOKEN}") - - if [ "$delete_status_code" == 204 ] || [ "$delete_status_code" == 404 ] - then - echo "Partition deletion was successful, with status code: ${delete_status_code}" - else - echo "Not able to delete partition, status code is: ${delete_status_code}" - exit 1 - fi + if [[ "${PARTITION_CLEAN_UP_ENABLED}" == "true" ]]; then + echo "Partition cleanup enabled, will delete partition ${DATA_PARTITION_ID}" + + delete_status_code=$(curl -X DELETE \ + --url "http://${PARTITION_NAME}/api/partition/v1/partitions/${DATA_PARTITION_ID}" --write-out "%{http_code}" --silent --output "/dev/null" \ + -H "Authorization: Bearer ${IDENTITY_TOKEN}") + + if [[ "${delete_status_code}" == 204 ]] || [[ "${delete_status_code}" == 404 ]]; then + echo "Partition deletion was successful with status code: ${delete_status_code}" + else + echo "Not able to delete partition with status code: ${delete_status_code}" + exit 1 + fi else echo "Partition cleanup not enabled, skipping deletion" fi @@ -77,29 +84,35 @@ bootstrap_gcp() { --data-raw "$(generate_post_data_gcp)") # shellcheck disable=SC2002 - if [ "$status_code" == 201 ] - then + if [[ "${status_code}" == 201 ]]; then echo "Partition bootstrap finished successfully!" - elif [ "$status_code" == 409 ] - then + elif [[ "${status_code}" == 409 ]]; then + patch_status_code=$(curl -X PATCH \ --url "http://${PARTITION_NAME}/api/partition/v1/partitions/${DATA_PARTITION_ID}" --write-out "%{http_code}" --silent --output "/dev/null" \ -H "Authorization: Bearer ${IDENTITY_TOKEN}" \ -H "Content-Type: application/json" \ --data-raw "$(generate_post_data_gcp)") - echo "Partition was patched because Datastore had already had entities! Status code of patching: $patch_status_code" + + echo "Partition was patched because Datastore had already had entities! Status code: ${patch_status_code}" else - echo "Exiting with status code: $status_code" + echo "Exiting with status code: ${status_code}" exit 1 fi } -if [ "$ENVIRONMENT" == "anthos" ] -then - bootstrap_anthos -elif [ "$ENVIRONMENT" == "gcp" ] -then - bootstrap_gcp +if [[ "${ENVIRONMENT}" == "anthos" && "${MULTI_PARTITON_ENABLED}" == "false" ]]; then + bootstrap_anthos "${DATA_PARTITION_ID}" "${DATA_PARTITION_ID^^}" +elif [[ "${ENVIRONMENT}" == "gcp" && "${MULTI_PARTITON_ENABLED}" == "false" ]]; then + bootstrap_gcp "${DATA_PARTITION_ID}" "${DATA_PARTITION_ID^^}" +elif [[ "${ENVIRONMENT}" == "gcp" && "${MULTI_PARTITON_ENABLED}" == "true" ]]; then + + IFS=',' read -ra PARTITIONS <<< "${DATA_PARTITION_ID_LIST}" + PARTITIONS=("${DATA_PARTITION_ID}" "${PARTITIONS[@]}") + + for PARTITION in "${PARTITIONS[@]}"; do + bootstrap_gcp "${PARTITION}" "${PARTITION^^}" + done fi touch /tmp/bootstrap_ready diff --git a/provider/partition-gcp/bootstrap/data_anthos.sh b/provider/partition-gcp/bootstrap/data_anthos.sh index 6de13608e238d978488a8a8a2b4d7b5c79e7e620..df38c2bce4cdaa5af5361c5a00149b5229f20b27 100644 --- a/provider/partition-gcp/bootstrap/data_anthos.sh +++ b/provider/partition-gcp/bootstrap/data_anthos.sh @@ -1,7 +1,5 @@ #!/usr/bin/env bash -DATA_PARTITION_ID_UPPER=${DATA_PARTITION_ID^^} - generate_post_data_anthos() { cat <<EOF { diff --git a/provider/partition-gcp/bootstrap/data_gcp.sh b/provider/partition-gcp/bootstrap/data_gcp.sh index 7f78a10cfcc9b394f369cf1674de9cce9646f626..7e4f1815cba8b928aeabc056e9d6474050077266 100644 --- a/provider/partition-gcp/bootstrap/data_gcp.sh +++ b/provider/partition-gcp/bootstrap/data_gcp.sh @@ -1,7 +1,5 @@ #!/usr/bin/env bash -DATA_PARTITION_ID_UPPER=${DATA_PARTITION_ID^^} - generate_post_data_gcp() { cat <<EOF {