Merge branch 'GONRG-7655-Separate-Bootstrap-for-System-and-General-Partition' into 'master'

[GONRG-7655] Added separate bootstrap for system partition

See merge request !427

devops/gc/deploy/README.md

@@ -109,7 +109,7 @@ First you need to set variables in **values.yaml** file using any code editor. S
 **data.springProfilesActive** | active spring profile | string | gcp | yes
 **data.projectId** | your Google Cloud project id | string | - | only in case of Google Cloud installation
 **data.dataProjectId** | in case of multiproject cloud installation (services and data stored in different project) the name of data project | string | - | only in case of multiproject installation
-**data.partitionName** | partition host | string | partition | yes
+**data.partitionHost** | partition host | string | partition | yes
 **data.partitionNamespace** | datastore namespace where partition will store the data | string | partition | yes
 **data.dataPartitionId** | data partition id | string | - | yes
 **data.datafierSa** | datafier service account | string | datafier | yes

devops/gc/deploy/templates/configmap-bootstrap.yaml

@@ -6,7 +6,7 @@ metadata:
   name: "{{ .Values.conf.configmap }}-bootstrap"
   namespace: "{{ .Release.Namespace }}"
 data:
-  PARTITION_NAME: {{ .Values.data.partitionName | quote }}
+  PARTITION_HOST: {{ .Values.data.partitionHost | quote }}
   DATA_PARTITION_ID: {{ .Values.data.dataPartitionId | quote }}
   ENVIRONMENT: {{ .Values.data.springProfilesActive | quote }}
   INDEXER_AUGMENTER_ENABLED: {{ .Values.data.indexerAugmenterEnabled | quote }}

devops/gc/deploy/values.yaml

@@ -12,7 +12,7 @@ data:
   dataProjectId: ""
   dataPartitionId: ""
   dataPartitionIdList: []
-  partitionName: "partition"
+  partitionHost: "partition"
   partitionNamespace: "partition"
   datafierSa: "datafier"
   bucketPrefix: "refi"

provider/partition-gc/bootstrap/Dockerfile

@@ -5,6 +5,7 @@ ENV PREFIX=$(pwd)
 RUN apk update && apk add \
     bash \
     curl \
+    jq \
     && chmod 775 bootstrap_partition.sh
 RUN addgroup -g 10001 -S nonroot \
   && adduser -h /opt -G nonroot -S -u 10001 nonroot

provider/partition-gc/bootstrap/bootstrap_partition.sh

@@ -2,50 +2,26 @@
 
 set -ex
 
+source ./helpers.sh
 source ./data_baremetal.sh
 source ./data_gc.sh
 
-# Bootstrap Partition service on Baremetal (on-prem)
-bootstrap_baremetal() {
-
-  DATA_PARTITION_ID=$1
-  DATA_PARTITION_ID_UPPER=$2
-
-  status_code=$(curl -X POST \
-    --url "http://${PARTITION_NAME}/api/partition/v1/partitions/${DATA_PARTITION_ID}" --write-out "%{http_code}" --silent --output "/dev/null" \
-    -H "Content-Type: application/json" \
-    --data-raw "$(generate_post_data_baremetal)")
-
-  # shellcheck disable=SC2002
-  if [[ "${status_code}" == 201 ]]; then
-    echo "Partition bootstrap finished successfully!"
-  elif [[ "${status_code}" == 409 ]]; then
-
-    patch_status_code=$(curl -X PATCH \
-    --url "http://${PARTITION_NAME}/api/partition/v1/partitions/${DATA_PARTITION_ID}" --write-out "%{http_code}" --silent --output "/dev/null" \
-    -H "Content-Type: application/json" \
-    --data-raw "$(generate_post_data_baremetal)")
-
-    echo "Partition was patched because Postgres Database had already had entities! Status code: ${patch_status_code}"
-  else
-    echo "Exiting with status code: ${status_code}"
-    exit 1
-  fi
-}
-
 # Bootstrap Partition service on Google Cloud
-bootstrap_gc() {
+bootstrap_partition() {
 
   echo "sleep to prevent 500 response from the Partition service, due to timeout of creation for Workload Identity"
   sleep 20
 
   DATA_PARTITION_ID=$1
-  DATA_PARTITION_ID_UPPER=$2
+  BOOTSTRAP_DATA=$2
+  
+  echo "Bootstrapping partition: $DATA_PARTITION_ID"
+  echo "$BOOTSTRAP_DATA" | jq
 
   status_code=$(curl -X POST \
-     --url "http://${PARTITION_NAME}/api/partition/v1/partitions/${DATA_PARTITION_ID}" --write-out "%{http_code}" --silent --output "/dev/null" \
+     --url "http://${PARTITION_HOST}/api/partition/v1/partitions/${DATA_PARTITION_ID}" --write-out "%{http_code}" --silent --output "/dev/null" \
      -H "Content-Type: application/json" \
-     --data-raw "$(generate_post_data_gc)")
+     --data-raw "$BOOTSTRAP_DATA")
 
   # shellcheck disable=SC2002
   if [[ "${status_code}" == 201 ]]; then
@@ -53,28 +29,38 @@ bootstrap_gc() {
   elif [[ "${status_code}" == 409 ]]; then
 
     patch_status_code=$(curl -X PATCH \
-    --url "http://${PARTITION_NAME}/api/partition/v1/partitions/${DATA_PARTITION_ID}" --write-out "%{http_code}" --silent --output "/dev/null" \
+    --url "http://${PARTITION_HOST}/api/partition/v1/partitions/${DATA_PARTITION_ID}" --write-out "%{http_code}" --silent --output "/dev/null" \
     -H "Content-Type: application/json" \
-    --data-raw "$(generate_post_data_gc)")
+    --data-raw "$BOOTSTRAP_DATA")
 
-    echo "Partition was patched because Datastore had already had entities! Status code: ${patch_status_code}"
+    echo "Partition was patched because Datastore already has entities! Status code: ${patch_status_code}"
   else
     echo "Exiting with status code: ${status_code}"
     exit 1
   fi
 }
 
-if [[ "${ENVIRONMENT}" == "anthos" && "${DATA_PARTITION_ID_LIST}" == "" ]]; then
-  bootstrap_baremetal "${DATA_PARTITION_ID}" "${DATA_PARTITION_ID^^}"
-elif [[ "${ENVIRONMENT}" == "gcp" && "${DATA_PARTITION_ID_LIST}" == "" ]]; then
-  bootstrap_gc "${DATA_PARTITION_ID}" "${DATA_PARTITION_ID^^}"
-elif [[ "${ENVIRONMENT}" == "gcp" && "${DATA_PARTITION_ID_LIST}" != "" ]]; then
+# Bootstrap system partition
+if [[ "${ENVIRONMENT}" == "gcp" ]]; then
+  bootstrap_partition "${DATA_PARTITION_ID}" "$(gc_system_partition_data)"
+elif [[ "${ENVIRONMENT}" == "anthos" ]]; then
+  bootstrap_partition "${DATA_PARTITION_ID}" "$(baremetal_system_partition_data)"
+fi
+
+# Bootstrap additional partitions
+if [[ "${ENVIRONMENT}" == "gcp" && "${DATA_PARTITION_ID_LIST}" != "" ]]; then
+  IFS=',' read -ra PARTITIONS <<< "${DATA_PARTITION_ID_LIST}"
 
+  for PARTITION in "${PARTITIONS[@]}"; do
+    additional_partition_data=$(merge "gc_system_partition_data" "gc_additional_partition_data")
+    bootstrap_partition "${PARTITION}" "$additional_partition_data"
+  done
+elif [[ "${ENVIRONMENT}" == "anthos" && "${DATA_PARTITION_ID_LIST}" != "" ]]; then
   IFS=',' read -ra PARTITIONS <<< "${DATA_PARTITION_ID_LIST}"
-  PARTITIONS=("${DATA_PARTITION_ID}" "${PARTITIONS[@]}")
 
   for PARTITION in "${PARTITIONS[@]}"; do
-    bootstrap_gc "${PARTITION}" "${PARTITION^^}"
+    additional_partition_data=$(merge "baremetal_system_partition_data" "baremetal_additional_partition_data")
+    bootstrap_partition "${PARTITION}" "$additional_partition_data"
   done
 fi
 

provider/partition-gc/bootstrap/data_baremetal.sh

 #!/usr/bin/env bash
 
-generate_post_data_baremetal() {
+# FIXME (GONRG-7695): Move elastic properties to additional partition when resolved
+# FIXME (GONRG-7696): Move rabbitmq properties to additional partition when resolved
+baremetal_system_partition_data() {
+  DATA_PARTITION_ID_UPPER="${DATA_PARTITION_ID^^}"
   cat <<EOF
 {
   "properties": {
@@ -24,10 +27,6 @@ generate_post_data_baremetal() {
       "sensitive": false,
       "value": "${DATA_PARTITION_ID}"
     },
-    "policy-service-enabled": {
-      "sensitive": false,
-      "value": "false"
-    },
     "bucket": {
       "sensitive": false,
       "value": "${BUCKET_PREFIX}-${DATA_PARTITION_ID}-records"
@@ -64,9 +63,9 @@ generate_post_data_baremetal() {
       "sensitive": false,
       "value": "${MINIO_IGNORE_CERT_CHECK}"
     },
-    "obm.minio.external.endpoint": {
+    "kubernetes-secret-name": {
       "sensitive": false,
-      "value": "${MINIO_EXTERNAL_ENDPOINT}"
+      "value": "eds-${DATA_PARTITION_ID}"
     },
     "oqm.rabbitmq.amqp.host": {
       "sensitive": false,
@@ -127,16 +126,32 @@ generate_post_data_baremetal() {
     "elasticsearch.password": {
       "sensitive": true,
       "value": "ELASTIC_PASS"
+    }
+  }
+}
+EOF
+}
+
+baremetal_additional_partition_data() {
+  DATA_PARTITION_ID_UPPER="${DATA_PARTITION_ID^^}"
+  cat <<EOF
+{
+  "properties": {
+    "index-augmenter-enabled": {
+      "sensitive": false,
+      "value": "${INDEXER_AUGMENTER_ENABLED}"
     },
-    "kubernetes-secret-name": {
+    "policy-service-enabled": {
       "sensitive": false,
-      "value": "eds-${DATA_PARTITION_ID}"
+      "value": "false"
     },
-    "index-augmenter-enabled": {
+    "obm.minio.external.endpoint": {
       "sensitive": false,
-      "value": "${INDEXER_AUGMENTER_ENABLED}"
+      "value": "${MINIO_EXTERNAL_ENDPOINT}"
     }
   }
 }
 EOF
 }
+
+

provider/partition-gc/bootstrap/data_gc.sh

 #!/usr/bin/env bash
 
-generate_post_data_gc() {
+# FIXME (GONRG-7695): Move elastic properties to additional partition when resolved
+gc_system_partition_data() {
+  DATA_PARTITION_ID_UPPER="${DATA_PARTITION_ID^^}"
   cat <<EOF
 {
   "properties": {
@@ -24,10 +26,6 @@ generate_post_data_gc() {
       "sensitive": false,
       "value": "${DATA_PARTITION_ID}"
     },
-    "policy-service-enabled": {
-      "sensitive": false,
-      "value": "false"
-    },
     "bucket": {
       "sensitive": false,
       "value": "${PROJECT_ID}-${DATA_PARTITION_ID}-records"
@@ -40,10 +38,6 @@ generate_post_data_gc() {
       "sensitive": false,
       "value": "[${DATA_PARTITION_ID},${DATA_PARTITION_ID}]"
     },
-    "reservoir-connection": {
-      "sensitive": true,
-      "value": "RESERVOIR_POSTGRES_CONN_STRING_OSDU"
-    },
     "elasticsearch.host": {
       "sensitive": true,
       "value": "ELASTIC_HOST"
@@ -59,6 +53,19 @@ generate_post_data_gc() {
     "elasticsearch.password": {
       "sensitive": true,
       "value": "ELASTIC_PASS"
+    }
+  }
+}
+EOF
+}
+
+gc_additional_partition_data() {
+  cat <<EOF
+{
+  "properties": {
+    "policy-service-enabled": {
+      "sensitive": false,
+      "value": "false"
     },
     "kubernetes-secret-name": {
       "sensitive": false,
@@ -67,6 +74,10 @@ generate_post_data_gc() {
     "index-augmenter-enabled": {
       "sensitive": false,
       "value": "${INDEXER_AUGMENTER_ENABLED}"
+    },
+    "reservoir-connection": {
+      "sensitive": true,
+      "value": "RESERVOIR_POSTGRES_CONN_STRING_OSDU"
     }
   }
 }

provider/partition-gc/bootstrap/helpers.sh

+#!/usr/bin/env bash
+
+merge() {
+  local system_data_function_name="$1"
+  local additional_data_function_name="$2"
+  local json1
+  local json2
+  
+  json1="$($system_data_function_name)"
+  json2="$($additional_data_function_name)"
+  
+  jq -n --argjson json1 "$json1" --argjson json2 "$json2" \
+    '$json1.properties + $json2.properties | { properties: . }'
+}
+