diff --git a/devops/gc/deploy/README.md b/devops/gc/deploy/README.md index 3b01ba31a9b11f2ef4a32eabede6d34e30e2c29d..dacc1754235180f52bad3f4931e35084179803b6 100644 --- a/devops/gc/deploy/README.md +++ b/devops/gc/deploy/README.md @@ -109,7 +109,7 @@ First you need to set variables in **values.yaml** file using any code editor. S **data.springProfilesActive** | active spring profile | string | gcp | yes **data.projectId** | your Google Cloud project id | string | - | only in case of Google Cloud installation **data.dataProjectId** | in case of multiproject cloud installation (services and data stored in different project) the name of data project | string | - | only in case of multiproject installation -**data.partitionName** | partition host | string | partition | yes +**data.partitionHost** | partition host | string | partition | yes **data.partitionNamespace** | datastore namespace where partition will store the data | string | partition | yes **data.dataPartitionId** | data partition id | string | - | yes **data.datafierSa** | datafier service account | string | datafier | yes diff --git a/devops/gc/deploy/templates/configmap-bootstrap.yaml b/devops/gc/deploy/templates/configmap-bootstrap.yaml index afbb66d287fb734e4e158e3005e75edc44eb2615..6cda8ffe7f6f17735a9d7c4e7a95177b05187359 100644 --- a/devops/gc/deploy/templates/configmap-bootstrap.yaml +++ b/devops/gc/deploy/templates/configmap-bootstrap.yaml @@ -6,7 +6,7 @@ metadata: name: "{{ .Values.conf.configmap }}-bootstrap" namespace: "{{ .Release.Namespace }}" data: - PARTITION_NAME: {{ .Values.data.partitionName | quote }} + PARTITION_HOST: {{ .Values.data.partitionHost | quote }} DATA_PARTITION_ID: {{ .Values.data.dataPartitionId | quote }} ENVIRONMENT: {{ .Values.data.springProfilesActive | quote }} INDEXER_AUGMENTER_ENABLED: {{ .Values.data.indexerAugmenterEnabled | quote }} diff --git a/devops/gc/deploy/values.yaml b/devops/gc/deploy/values.yaml index cb8915632355f80b9532ca3851ea82612bf8d512..7ed6262b397074d6c5467eb878bdbc8d6e951400 100644 --- a/devops/gc/deploy/values.yaml +++ b/devops/gc/deploy/values.yaml @@ -12,7 +12,7 @@ data: dataProjectId: "" dataPartitionId: "" dataPartitionIdList: [] - partitionName: "partition" + partitionHost: "partition" partitionNamespace: "partition" datafierSa: "datafier" bucketPrefix: "refi" diff --git a/provider/partition-gc/bootstrap/Dockerfile b/provider/partition-gc/bootstrap/Dockerfile index db30642df6c2ec6d2a7088d4cda9b0246ac4e97c..4df7fd64c9dc8674f41e07af0fc38bc927d4b13e 100644 --- a/provider/partition-gc/bootstrap/Dockerfile +++ b/provider/partition-gc/bootstrap/Dockerfile @@ -5,6 +5,7 @@ ENV PREFIX=$(pwd) RUN apk update && apk add \ bash \ curl \ + jq \ && chmod 775 bootstrap_partition.sh RUN addgroup -g 10001 -S nonroot \ && adduser -h /opt -G nonroot -S -u 10001 nonroot diff --git a/provider/partition-gc/bootstrap/bootstrap_partition.sh b/provider/partition-gc/bootstrap/bootstrap_partition.sh index 0df32dd89a8c5bb5a145dc1215955fc192ce94d2..b0935b22ebe2a98c21d83b90b472e4bd6455a1b4 100644 --- a/provider/partition-gc/bootstrap/bootstrap_partition.sh +++ b/provider/partition-gc/bootstrap/bootstrap_partition.sh @@ -2,50 +2,26 @@ set -ex +source ./helpers.sh source ./data_baremetal.sh source ./data_gc.sh -# Bootstrap Partition service on Baremetal (on-prem) -bootstrap_baremetal() { - - DATA_PARTITION_ID=$1 - DATA_PARTITION_ID_UPPER=$2 - - status_code=$(curl -X POST \ - --url "http://${PARTITION_NAME}/api/partition/v1/partitions/${DATA_PARTITION_ID}" --write-out "%{http_code}" --silent --output "/dev/null" \ - -H "Content-Type: application/json" \ - --data-raw "$(generate_post_data_baremetal)") - - # shellcheck disable=SC2002 - if [[ "${status_code}" == 201 ]]; then - echo "Partition bootstrap finished successfully!" - elif [[ "${status_code}" == 409 ]]; then - - patch_status_code=$(curl -X PATCH \ - --url "http://${PARTITION_NAME}/api/partition/v1/partitions/${DATA_PARTITION_ID}" --write-out "%{http_code}" --silent --output "/dev/null" \ - -H "Content-Type: application/json" \ - --data-raw "$(generate_post_data_baremetal)") - - echo "Partition was patched because Postgres Database had already had entities! Status code: ${patch_status_code}" - else - echo "Exiting with status code: ${status_code}" - exit 1 - fi -} - # Bootstrap Partition service on Google Cloud -bootstrap_gc() { +bootstrap_partition() { echo "sleep to prevent 500 response from the Partition service, due to timeout of creation for Workload Identity" sleep 20 DATA_PARTITION_ID=$1 - DATA_PARTITION_ID_UPPER=$2 + BOOTSTRAP_DATA=$2 + + echo "Bootstrapping partition: $DATA_PARTITION_ID" + echo "$BOOTSTRAP_DATA" | jq status_code=$(curl -X POST \ - --url "http://${PARTITION_NAME}/api/partition/v1/partitions/${DATA_PARTITION_ID}" --write-out "%{http_code}" --silent --output "/dev/null" \ + --url "http://${PARTITION_HOST}/api/partition/v1/partitions/${DATA_PARTITION_ID}" --write-out "%{http_code}" --silent --output "/dev/null" \ -H "Content-Type: application/json" \ - --data-raw "$(generate_post_data_gc)") + --data-raw "$BOOTSTRAP_DATA") # shellcheck disable=SC2002 if [[ "${status_code}" == 201 ]]; then @@ -53,28 +29,38 @@ bootstrap_gc() { elif [[ "${status_code}" == 409 ]]; then patch_status_code=$(curl -X PATCH \ - --url "http://${PARTITION_NAME}/api/partition/v1/partitions/${DATA_PARTITION_ID}" --write-out "%{http_code}" --silent --output "/dev/null" \ + --url "http://${PARTITION_HOST}/api/partition/v1/partitions/${DATA_PARTITION_ID}" --write-out "%{http_code}" --silent --output "/dev/null" \ -H "Content-Type: application/json" \ - --data-raw "$(generate_post_data_gc)") + --data-raw "$BOOTSTRAP_DATA") - echo "Partition was patched because Datastore had already had entities! Status code: ${patch_status_code}" + echo "Partition was patched because Datastore already has entities! Status code: ${patch_status_code}" else echo "Exiting with status code: ${status_code}" exit 1 fi } -if [[ "${ENVIRONMENT}" == "anthos" && "${DATA_PARTITION_ID_LIST}" == "" ]]; then - bootstrap_baremetal "${DATA_PARTITION_ID}" "${DATA_PARTITION_ID^^}" -elif [[ "${ENVIRONMENT}" == "gcp" && "${DATA_PARTITION_ID_LIST}" == "" ]]; then - bootstrap_gc "${DATA_PARTITION_ID}" "${DATA_PARTITION_ID^^}" -elif [[ "${ENVIRONMENT}" == "gcp" && "${DATA_PARTITION_ID_LIST}" != "" ]]; then +# Bootstrap system partition +if [[ "${ENVIRONMENT}" == "gcp" ]]; then + bootstrap_partition "${DATA_PARTITION_ID}" "$(gc_system_partition_data)" +elif [[ "${ENVIRONMENT}" == "anthos" ]]; then + bootstrap_partition "${DATA_PARTITION_ID}" "$(baremetal_system_partition_data)" +fi + +# Bootstrap additional partitions +if [[ "${ENVIRONMENT}" == "gcp" && "${DATA_PARTITION_ID_LIST}" != "" ]]; then + IFS=',' read -ra PARTITIONS <<< "${DATA_PARTITION_ID_LIST}" + for PARTITION in "${PARTITIONS[@]}"; do + additional_partition_data=$(merge "gc_system_partition_data" "gc_additional_partition_data") + bootstrap_partition "${PARTITION}" "$additional_partition_data" + done +elif [[ "${ENVIRONMENT}" == "anthos" && "${DATA_PARTITION_ID_LIST}" != "" ]]; then IFS=',' read -ra PARTITIONS <<< "${DATA_PARTITION_ID_LIST}" - PARTITIONS=("${DATA_PARTITION_ID}" "${PARTITIONS[@]}") for PARTITION in "${PARTITIONS[@]}"; do - bootstrap_gc "${PARTITION}" "${PARTITION^^}" + additional_partition_data=$(merge "baremetal_system_partition_data" "baremetal_additional_partition_data") + bootstrap_partition "${PARTITION}" "$additional_partition_data" done fi diff --git a/provider/partition-gc/bootstrap/data_baremetal.sh b/provider/partition-gc/bootstrap/data_baremetal.sh index d9c1dc032963df7bf6d73686a6cf1bbb07c65a43..4b453df1d559a84b7d1abd721b4777aee866b73a 100644 --- a/provider/partition-gc/bootstrap/data_baremetal.sh +++ b/provider/partition-gc/bootstrap/data_baremetal.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash -generate_post_data_baremetal() { +# FIXME (GONRG-7695): Move elastic properties to additional partition when resolved +# FIXME (GONRG-7696): Move rabbitmq properties to additional partition when resolved +baremetal_system_partition_data() { + DATA_PARTITION_ID_UPPER="${DATA_PARTITION_ID^^}" cat <<EOF { "properties": { @@ -24,10 +27,6 @@ generate_post_data_baremetal() { "sensitive": false, "value": "${DATA_PARTITION_ID}" }, - "policy-service-enabled": { - "sensitive": false, - "value": "false" - }, "bucket": { "sensitive": false, "value": "${BUCKET_PREFIX}-${DATA_PARTITION_ID}-records" @@ -64,9 +63,9 @@ generate_post_data_baremetal() { "sensitive": false, "value": "${MINIO_IGNORE_CERT_CHECK}" }, - "obm.minio.external.endpoint": { + "kubernetes-secret-name": { "sensitive": false, - "value": "${MINIO_EXTERNAL_ENDPOINT}" + "value": "eds-${DATA_PARTITION_ID}" }, "oqm.rabbitmq.amqp.host": { "sensitive": false, @@ -127,16 +126,32 @@ generate_post_data_baremetal() { "elasticsearch.password": { "sensitive": true, "value": "ELASTIC_PASS" + } + } +} +EOF +} + +baremetal_additional_partition_data() { + DATA_PARTITION_ID_UPPER="${DATA_PARTITION_ID^^}" + cat <<EOF +{ + "properties": { + "index-augmenter-enabled": { + "sensitive": false, + "value": "${INDEXER_AUGMENTER_ENABLED}" }, - "kubernetes-secret-name": { + "policy-service-enabled": { "sensitive": false, - "value": "eds-${DATA_PARTITION_ID}" + "value": "false" }, - "index-augmenter-enabled": { + "obm.minio.external.endpoint": { "sensitive": false, - "value": "${INDEXER_AUGMENTER_ENABLED}" + "value": "${MINIO_EXTERNAL_ENDPOINT}" } } } EOF } + + diff --git a/provider/partition-gc/bootstrap/data_gc.sh b/provider/partition-gc/bootstrap/data_gc.sh index c78dc1c53895c590c543c274db117c2314a608d1..20f625f9c99efcf85e815f0716cf77dc6c22c2ad 100644 --- a/provider/partition-gc/bootstrap/data_gc.sh +++ b/provider/partition-gc/bootstrap/data_gc.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash -generate_post_data_gc() { +# FIXME (GONRG-7695): Move elastic properties to additional partition when resolved +gc_system_partition_data() { + DATA_PARTITION_ID_UPPER="${DATA_PARTITION_ID^^}" cat <<EOF { "properties": { @@ -24,10 +26,6 @@ generate_post_data_gc() { "sensitive": false, "value": "${DATA_PARTITION_ID}" }, - "policy-service-enabled": { - "sensitive": false, - "value": "false" - }, "bucket": { "sensitive": false, "value": "${PROJECT_ID}-${DATA_PARTITION_ID}-records" @@ -40,10 +38,6 @@ generate_post_data_gc() { "sensitive": false, "value": "[${DATA_PARTITION_ID},${DATA_PARTITION_ID}]" }, - "reservoir-connection": { - "sensitive": true, - "value": "RESERVOIR_POSTGRES_CONN_STRING_OSDU" - }, "elasticsearch.host": { "sensitive": true, "value": "ELASTIC_HOST" @@ -59,6 +53,19 @@ generate_post_data_gc() { "elasticsearch.password": { "sensitive": true, "value": "ELASTIC_PASS" + } + } +} +EOF +} + +gc_additional_partition_data() { + cat <<EOF +{ + "properties": { + "policy-service-enabled": { + "sensitive": false, + "value": "false" }, "kubernetes-secret-name": { "sensitive": false, @@ -67,6 +74,10 @@ generate_post_data_gc() { "index-augmenter-enabled": { "sensitive": false, "value": "${INDEXER_AUGMENTER_ENABLED}" + }, + "reservoir-connection": { + "sensitive": true, + "value": "RESERVOIR_POSTGRES_CONN_STRING_OSDU" } } } diff --git a/provider/partition-gc/bootstrap/helpers.sh b/provider/partition-gc/bootstrap/helpers.sh new file mode 100644 index 0000000000000000000000000000000000000000..8aa2a0fa870a3a1b3c8f826437d904523c7cb6b4 --- /dev/null +++ b/provider/partition-gc/bootstrap/helpers.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +merge() { + local system_data_function_name="$1" + local additional_data_function_name="$2" + local json1 + local json2 + + json1="$($system_data_function_name)" + json2="$($additional_data_function_name)" + + jq -n --argjson json1 "$json1" --argjson json2 "$json2" \ + '$json1.properties + $json2.properties | { properties: . }' +} +