diff --git a/devops/core-plus/bootstrap/bootstrap_partition.sh b/devops/core-plus/bootstrap/bootstrap_partition.sh index 2df85929a1dfe19bb1eb5edf618caca3ab6e68e0..60b5e3853520167e1a554e80cfe5f700a7ffc869 100644 --- a/devops/core-plus/bootstrap/bootstrap_partition.sh +++ b/devops/core-plus/bootstrap/bootstrap_partition.sh @@ -2,7 +2,6 @@ set -ex -source ./helpers.sh source ./data_core.sh DATA_PARTITION_URL="http://${PARTITION_HOST}/api/partition/v1/partitions/${DATA_PARTITION_ID}" @@ -40,7 +39,6 @@ bootstrap_partition() { # Bootstrap additional partition export DATA_PARTITION_ID_VALUE="${DATA_PARTITION_ID}" -additional_partition_data=$(merge "core_system_partition_data" "core_additional_partition_data") -bootstrap_partition "${DATA_PARTITION_ID}" "$additional_partition_data" "${DATA_PARTITION_URL}" +bootstrap_partition "${DATA_PARTITION_ID}" "$(core_partition_data)" "${DATA_PARTITION_URL}" touch /tmp/bootstrap_ready diff --git a/devops/core-plus/bootstrap/data_core.sh b/devops/core-plus/bootstrap/data_core.sh index 003b15253654d66901711323bcd9cbd166b036d8..1f5a96a28e5f8c770aea526c89939b7380539034 100644 --- a/devops/core-plus/bootstrap/data_core.sh +++ b/devops/core-plus/bootstrap/data_core.sh @@ -2,7 +2,7 @@ # FIXME (GONRG-7695): Move elastic properties to additional partition when resolved # FIXME (GONRG-7696): Move rabbitmq properties to additional partition when resolved -core_system_partition_data() { +core_partition_data() { DATA_PARTITION_ID_UPPER="${DATA_PARTITION_ID_VALUE^^}" cat <<EOF { @@ -115,19 +115,19 @@ core_system_partition_data() { "sensitive": true, "value": "RABBITMQ_ADMIN_PASSWORD" }, - "elasticsearch.host": { + "elasticsearch.8.host": { "sensitive": true, "value": "ELASTIC_HOST${PARTITION_SUFFIX}" }, - "elasticsearch.port": { + "elasticsearch.8.port": { "sensitive": true, "value": "ELASTIC_PORT${PARTITION_SUFFIX}" }, - "elasticsearch.user": { + "elasticsearch.8.user": { "sensitive": true, "value": "ELASTIC_USER${PARTITION_SUFFIX}" }, - "elasticsearch.password": { + "elasticsearch.8.password": { "sensitive": true, "value": "ELASTIC_PASS${PARTITION_SUFFIX}" }, @@ -135,14 +135,6 @@ core_system_partition_data() { "sensitive": false, "value": "${INDEXER_AUGMENTER_ENABLED}" }, - "policy-service-enabled": { - "sensitive": false, - "value": "false" - }, - "obm.minio.external.endpoint": { - "sensitive": false, - "value": "${MINIO_EXTERNAL_ENDPOINT}" - }, "entitlements.datasource.url": { "sensitive": true, "value": "ENT_PG_URL${PARTITION_SUFFIX}" @@ -166,24 +158,6 @@ core_system_partition_data() { "schema.bucket.name": { "sensitive": false, "value": "${BUCKET_PREFIX}-${DATA_PARTITION_ID_VALUE}-schema" - } - } -} -EOF -} - -core_additional_partition_data() { - DATA_PARTITION_ID_UPPER="${DATA_PARTITION_ID_VALUE^^}" - cat <<EOF -{ - "properties": { - "index-augmenter-enabled": { - "sensitive": false, - "value": "${INDEXER_AUGMENTER_ENABLED}" - }, - "policy-service-enabled": { - "sensitive": false, - "value": "false" }, "obm.minio.external.endpoint": { "sensitive": false, @@ -191,11 +165,27 @@ core_additional_partition_data() { }, "wellbore-dms-bucket": { "sensitive": false, - "value": "${BUCKET_PREFIX}-logstore-osdu" + "value": "${BUCKET_PREFIX}-${DATA_PARTITION_ID_VALUE}-wellbore" }, "sd.ksd.k8s.namespace": { "sensitive": false, "value": "secret-admin" + }, + "featureFlag.eds.enabled": { + "sensitive": false, + "value": "${EDS_ENABLED}" + }, + "featureFlag.opa.enabled": { + "sensitive": false, + "value": "${POLICY_SERVICE_ENABLED}" + }, + "featureFlag.policy.enabled": { + "sensitive": false, + "value": "${POLICY_SERVICE_ENABLED}" + }, + "featureFlag.autocomplete.enabled": { + "sensitive": false, + "value": "${AUTOCOMPLETE_ENABLED}" } } } diff --git a/devops/core-plus/bootstrap/helpers.sh b/devops/core-plus/bootstrap/helpers.sh deleted file mode 100644 index 8aa2a0fa870a3a1b3c8f826437d904523c7cb6b4..0000000000000000000000000000000000000000 --- a/devops/core-plus/bootstrap/helpers.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -merge() { - local system_data_function_name="$1" - local additional_data_function_name="$2" - local json1 - local json2 - - json1="$($system_data_function_name)" - json2="$($additional_data_function_name)" - - jq -n --argjson json1 "$json1" --argjson json2 "$json2" \ - '$json1.properties + $json2.properties | { properties: . }' -} - diff --git a/devops/gc/deploy/templates/configmap-bootstrap.yaml b/devops/gc/deploy/templates/configmap-bootstrap.yaml index 956eb8683a5ae18988e9429d77429bd600387dea..67bcf04cd2155c1aa501ad22a1a19e09cd7c0577 100644 --- a/devops/gc/deploy/templates/configmap-bootstrap.yaml +++ b/devops/gc/deploy/templates/configmap-bootstrap.yaml @@ -13,7 +13,6 @@ data: AUTOCOMPLETE_ENABLED: {{ .Values.data.autocompleteEnabled | quote }} PARTITION_SUFFIX: {{ .Values.data.partitionSuffix | quote }} DATA_PARTITION_ID: {{ .Values.global.dataPartitionId | quote }} - ENVIRONMENT: "gcp" {{- if .Values.data.dataProjectId }} PROJECT_ID: "{{ .Values.data.dataProjectId }}" {{- else }} diff --git a/provider/partition-gc/bootstrap/bootstrap_partition.sh b/provider/partition-gc/bootstrap/bootstrap_partition.sh index f43a53317184a4c3ba333dccd9cbc6171c8de842..fc944f22eb78eb2cec75af5ba6ba606c071fd3d1 100644 --- a/provider/partition-gc/bootstrap/bootstrap_partition.sh +++ b/provider/partition-gc/bootstrap/bootstrap_partition.sh @@ -1,9 +1,22 @@ #!/usr/bin/env bash +# Copyright 2024 Google LLC +# Copyright 2024 EPAM +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. set -ex -source ./helpers.sh -source ./data_baremetal.sh +source ./system_gc.sh source ./data_gc.sh DATA_PARTITION_URL="http://${PARTITION_HOST}/api/partition/v1/partitions/${DATA_PARTITION_ID}" @@ -43,21 +56,15 @@ bootstrap_partition() { fi } -# Bootstrap system partition -if [[ "${ENVIRONMENT}" == "gcp" ]]; then - # Specifying "system" partition for GC installation - export SYSTEM_PARTITION_ID="system" - export SYSTEM_PARTITION_URL="http://${PARTITION_HOST}/api/partition/v1/partition/${SYSTEM_PARTITION_ID}" - export DATA_PARTITION_ID_VALUE="${SYSTEM_PARTITION_ID}" - bootstrap_partition "${SYSTEM_PARTITION_ID}" "$(gc_system_partition_data)" "${SYSTEM_PARTITION_URL}" - - # Bootstrap additional partition - export DATA_PARTITION_ID_VALUE="${DATA_PARTITION_ID}" - additional_partition_data=$(merge "gc_system_partition_data" "gc_additional_partition_data") - bootstrap_partition "${DATA_PARTITION_ID}" "$additional_partition_data" "${DATA_PARTITION_URL}" -elif [[ "${ENVIRONMENT}" == "anthos" ]]; then - export DATA_PARTITION_ID_VALUE="${DATA_PARTITION_ID}" - bootstrap_partition "${DATA_PARTITION_ID}" "$(baremetal_system_partition_data)" "${DATA_PARTITION_URL}" -fi +# Bootstrap system and data partitions +# Specifying "system" partition for GC installation +export SYSTEM_PARTITION_ID="system" +export SYSTEM_PARTITION_URL="http://${PARTITION_HOST}/api/partition/v1/partition/${SYSTEM_PARTITION_ID}" +export DATA_PARTITION_ID_VALUE="${SYSTEM_PARTITION_ID}" +bootstrap_partition "${SYSTEM_PARTITION_ID}" "$(gc_system_partition_data)" "${SYSTEM_PARTITION_URL}" + +# Bootstrap additional partition +export DATA_PARTITION_ID_VALUE="${DATA_PARTITION_ID}" +bootstrap_partition "${DATA_PARTITION_ID}" "$(gc_partition_data)" "${DATA_PARTITION_URL}" touch /tmp/bootstrap_ready diff --git a/provider/partition-gc/bootstrap/data_baremetal.sh b/provider/partition-gc/bootstrap/data_baremetal.sh deleted file mode 100644 index 40fd9d065e4359fc1f0854644cdf84551d4fe5c3..0000000000000000000000000000000000000000 --- a/provider/partition-gc/bootstrap/data_baremetal.sh +++ /dev/null @@ -1,193 +0,0 @@ -#!/usr/bin/env bash - -# FIXME (GONRG-7695): Move elastic properties to additional partition when resolved -# FIXME (GONRG-7696): Move rabbitmq properties to additional partition when resolved -baremetal_system_partition_data() { - DATA_PARTITION_ID_UPPER=$(echo "${DATA_PARTITION_ID_VALUE//-/_}" | tr '[:lower:]' '[:upper:]') - cat <<EOF -{ - "properties": { - "projectId": { - "sensitive": false, - "value": "${BUCKET_PREFIX}" - }, - "eds.enabled": { - "sensitive": false, - "value": "${EDS_ENABLED}" - }, - "serviceAccount": { - "sensitive": false, - "value": "${SERVICE_ACCOUNT}" - }, - "complianceRuleSet": { - "sensitive": false, - "value": "shared" - }, - "dataPartitionId": { - "sensitive": false, - "value": "${DATA_PARTITION_ID_VALUE}" - }, - "name": { - "sensitive": false, - "value": "${DATA_PARTITION_ID_VALUE}" - }, - "bucket": { - "sensitive": false, - "value": "${BUCKET_PREFIX}-${DATA_PARTITION_ID_VALUE}-records" - }, - "crmAccountID": { - "sensitive": false, - "value": "[${DATA_PARTITION_ID_VALUE},${DATA_PARTITION_ID_VALUE}]" - }, - "osm.postgres.datasource.url": { - "sensitive": true, - "value": "POSTGRES_DATASOURCE_URL${PARTITION_SUFFIX}" - }, - "osm.postgres.datasource.username": { - "sensitive": true, - "value": "POSTGRES_DB_USERNAME${PARTITION_SUFFIX}" - }, - "osm.postgres.datasource.password": { - "sensitive": true, - "value": "POSTGRES_DB_PASSWORD${PARTITION_SUFFIX}" - }, - "obm.minio.endpoint": { - "sensitive": false, - "value": "${MINIO_ENDPOINT}" - }, - "obm.minio.accessKey": { - "sensitive": true, - "value": "MINIO_ACCESS_KEY" - }, - "obm.minio.secretKey": { - "sensitive": true, - "value": "MINIO_SECRET_KEY" - }, - "obm.minio.ignoreCertCheck": { - "sensitive": false, - "value": "${MINIO_IGNORE_CERT_CHECK}" - }, - "obm.minio.ui.endpoint": { - "sensitive": false, - "value": "${MINIO_UI_ENDPOINT}" - }, - "kubernetes-secret-name": { - "sensitive": false, - "value": "eds-${DATA_PARTITION_ID_VALUE}" - }, - "oqm.rabbitmq.amqp.host": { - "sensitive": false, - "value": "rabbitmq" - }, - "oqm.rabbitmq.amqp.port": { - "sensitive": false, - "value": "5672" - }, - "oqm.rabbitmq.amqp.path": { - "sensitive": false, - "value": "" - }, - "oqm.rabbitmq.amqp.username": { - "sensitive": true, - "value": "RABBITMQ_ADMIN_USERNAME" - }, - "oqm.rabbitmq.amqp.password": { - "sensitive": true, - "value": "RABBITMQ_ADMIN_PASSWORD" - }, - "oqm.rabbitmq.admin.schema": { - "sensitive": false, - "value": "http" - }, - "oqm.rabbitmq.admin.host": { - "sensitive": false, - "value": "rabbitmq" - }, - "oqm.rabbitmq.admin.port": { - "sensitive": false, - "value": "15672" - }, - "oqm.rabbitmq.admin.path": { - "sensitive": false, - "value": "/api" - }, - "oqm.rabbitmq.admin.username": { - "sensitive": true, - "value": "RABBITMQ_ADMIN_USERNAME" - }, - "oqm.rabbitmq.admin.password": { - "sensitive": true, - "value": "RABBITMQ_ADMIN_PASSWORD" - }, - "elasticsearch.host": { - "sensitive": true, - "value": "ELASTIC_HOST${PARTITION_SUFFIX}" - }, - "elasticsearch.port": { - "sensitive": true, - "value": "ELASTIC_PORT${PARTITION_SUFFIX}" - }, - "elasticsearch.user": { - "sensitive": true, - "value": "ELASTIC_USER${PARTITION_SUFFIX}" - }, - "elasticsearch.password": { - "sensitive": true, - "value": "ELASTIC_PASS${PARTITION_SUFFIX}" - }, - "index-augmenter-enabled": { - "sensitive": false, - "value": "${INDEXER_AUGMENTER_ENABLED}" - }, - "featureFlag.policy.enabled": { - "sensitive": false, - "value": "${POLICY_SERVICE_ENABLED}" - }, - "obm.minio.external.endpoint": { - "sensitive": false, - "value": "${MINIO_EXTERNAL_ENDPOINT}" - }, - "entitlements.datasource.url": { - "sensitive": true, - "value": "ENT_PG_URL${PARTITION_SUFFIX}" - }, - "entitlements.datasource.username": { - "sensitive": true, - "value": "ENT_PG_USER${PARTITION_SUFFIX}" - }, - "entitlements.datasource.password": { - "sensitive": true, - "value": "ENT_PG_PASS${PARTITION_SUFFIX}" - }, - "entitlements.datasource.schema": { - "sensitive": true, - "value": "ENT_PG_SCHEMA_${DATA_PARTITION_ID_UPPER}" - }, - "system.schema.bucket.name": { - "sensitive": false, - "value": "${BUCKET_PREFIX}-system-schema" - }, - "schema.bucket.name": { - "sensitive": false, - "value": "${BUCKET_PREFIX}-${DATA_PARTITION_ID_VALUE}-schema" - }, - "featureFlag.opa.enabled": { - "sensitive": false, - "value": "${POLICY_SERVICE_ENABLED}" - }, - "featureFlag.autocomplete.enabled": { - "sensitive": false, - "value": "${AUTOCOMPLETE_ENABLED}" - }, - "wellbore-dms-bucket": { - "sensitive": false, - "value": "${BUCKET_PREFIX}-${DATA_PARTITION_ID_VALUE}-wellbore" - }, - "sd.ksd.k8s.namespace": { - "sensitive": false, - "value": "secret-admin" - } - } -} -EOF -} diff --git a/provider/partition-gc/bootstrap/data_gc.sh b/provider/partition-gc/bootstrap/data_gc.sh index e5a7c8d096462e47110ab21ab779b457474f68ea..9864641dcfde2cf0e978c0d4215340f7291dd92e 100644 --- a/provider/partition-gc/bootstrap/data_gc.sh +++ b/provider/partition-gc/bootstrap/data_gc.sh @@ -1,7 +1,6 @@ #!/usr/bin/env bash -# FIXME (GONRG-7695): Move elastic properties to additional partition when resolved -gc_system_partition_data() { +gc_partition_data() { DATA_PARTITION_ID_UPPER=$(echo "${DATA_PARTITION_ID_VALUE//-/_}" | tr '[:lower:]' '[:upper:]') cat <<EOF { @@ -26,34 +25,10 @@ gc_system_partition_data() { "sensitive": false, "value": "${DATA_PARTITION_ID_VALUE}" }, - "bucket": { - "sensitive": false, - "value": "${PROJECT_ID}-${DATA_PARTITION_ID_VALUE}-records" - }, - "seismicBucket": { - "sensitive": false, - "value": "${PROJECT_ID}-${DATA_PARTITION_ID_VALUE}-ss-seismic" - }, "crmAccountID": { "sensitive": false, "value": "[${DATA_PARTITION_ID_VALUE},${DATA_PARTITION_ID_VALUE}]" }, - "elasticsearch.host": { - "sensitive": true, - "value": "ELASTIC_HOST${PARTITION_SUFFIX}" - }, - "elasticsearch.port": { - "sensitive": true, - "value": "ELASTIC_PORT${PARTITION_SUFFIX}" - }, - "elasticsearch.user": { - "sensitive": true, - "value": "ELASTIC_USER${PARTITION_SUFFIX}" - }, - "elasticsearch.password": { - "sensitive": true, - "value": "ELASTIC_PASS${PARTITION_SUFFIX}" - }, "entitlements.datasource.url": { "sensitive": true, "value": "ENT_PG_URL${PARTITION_SUFFIX}" @@ -70,39 +45,14 @@ gc_system_partition_data() { "sensitive": true, "value": "ENT_PG_SCHEMA_${DATA_PARTITION_ID_UPPER}" }, + "bucket": { + "sensitive": false, + "value": "${PROJECT_ID}-${DATA_PARTITION_ID_VALUE}-records" + }, "reservoir-connection": { "sensitive": true, "value": "RESERVOIR_POSTGRES_CONN_STRING${PARTITION_SUFFIX}" }, - "system.schema.bucket.name": { - "sensitive": false, - "value": "${PROJECT_ID}-system-schema" - }, - "system.featureFlag.eds.enabled": { - "sensitive": false, - "value": "${EDS_ENABLED}" - }, - "system.featureFlag.opa.enabled": { - "sensitive": false, - "value": "${POLICY_SERVICE_ENABLED}" - }, - "system.featureFlag.policy.enabled": { - "sensitive": false, - "value": "${POLICY_SERVICE_ENABLED}" - }, - "system.featureFlag.autocomplete.enabled": { - "sensitive": false, - "value": "${AUTOCOMPLETE_ENABLED}" - } - } -} -EOF -} - -gc_additional_partition_data() { - cat <<EOF -{ - "properties": { "kubernetes-secret-name": { "sensitive": false, "value": "eds-${DATA_PARTITION_ID_VALUE}" @@ -138,6 +88,26 @@ gc_additional_partition_data() { "sd.ksd.k8s.namespace": { "sensitive": false, "value": "secret-admin" + }, + "seismicBucket": { + "sensitive": false, + "value": "${PROJECT_ID}-${DATA_PARTITION_ID_VALUE}-ss-seismic" + }, + "elasticsearch.8.host": { + "sensitive": true, + "value": "ELASTIC_HOST${PARTITION_SUFFIX}" + }, + "elasticsearch.8.port": { + "sensitive": true, + "value": "ELASTIC_PORT${PARTITION_SUFFIX}" + }, + "elasticsearch.8.user": { + "sensitive": true, + "value": "ELASTIC_USER${PARTITION_SUFFIX}" + }, + "elasticsearch.8.password": { + "sensitive": true, + "value": "ELASTIC_PASS${PARTITION_SUFFIX}" } } } diff --git a/provider/partition-gc/bootstrap/helpers.sh b/provider/partition-gc/bootstrap/helpers.sh deleted file mode 100644 index 8aa2a0fa870a3a1b3c8f826437d904523c7cb6b4..0000000000000000000000000000000000000000 --- a/provider/partition-gc/bootstrap/helpers.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -merge() { - local system_data_function_name="$1" - local additional_data_function_name="$2" - local json1 - local json2 - - json1="$($system_data_function_name)" - json2="$($additional_data_function_name)" - - jq -n --argjson json1 "$json1" --argjson json2 "$json2" \ - '$json1.properties + $json2.properties | { properties: . }' -} - diff --git a/provider/partition-gc/bootstrap/system_gc.sh b/provider/partition-gc/bootstrap/system_gc.sh new file mode 100644 index 0000000000000000000000000000000000000000..e506a0df2d1b1952e34876132ba15703df50ae6e --- /dev/null +++ b/provider/partition-gc/bootstrap/system_gc.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +gc_system_partition_data() { + DATA_PARTITION_ID_UPPER=$(echo "${DATA_PARTITION_ID_VALUE//-/_}" | tr '[:lower:]' '[:upper:]') + cat <<EOF +{ + "properties": { + "projectId": { + "sensitive": false, + "value": "${PROJECT_ID}" + }, + "serviceAccount": { + "sensitive": false, + "value": "${SERVICE_ACCOUNT}" + }, + "complianceRuleSet": { + "sensitive": false, + "value": "shared" + }, + "dataPartitionId": { + "sensitive": false, + "value": "${DATA_PARTITION_ID_VALUE}" + }, + "name": { + "sensitive": false, + "value": "${DATA_PARTITION_ID_VALUE}" + }, + "crmAccountID": { + "sensitive": false, + "value": "[${DATA_PARTITION_ID_VALUE},${DATA_PARTITION_ID_VALUE}]" + }, + "entitlements.datasource.url": { + "sensitive": true, + "value": "ENT_PG_URL${PARTITION_SUFFIX}" + }, + "entitlements.datasource.username": { + "sensitive": true, + "value": "ENT_PG_USER${PARTITION_SUFFIX}" + }, + "entitlements.datasource.password": { + "sensitive": true, + "value": "ENT_PG_PASS${PARTITION_SUFFIX}" + }, + "entitlements.datasource.schema": { + "sensitive": true, + "value": "ENT_PG_SCHEMA_${DATA_PARTITION_ID_UPPER}" + }, + "system.schema.bucket.name": { + "sensitive": false, + "value": "${PROJECT_ID}-system-schema" + }, + "system.featureFlag.eds.enabled": { + "sensitive": false, + "value": "${EDS_ENABLED}" + }, + "system.featureFlag.opa.enabled": { + "sensitive": false, + "value": "${POLICY_SERVICE_ENABLED}" + }, + "system.featureFlag.policy.enabled": { + "sensitive": false, + "value": "${POLICY_SERVICE_ENABLED}" + }, + "system.featureFlag.autocomplete.enabled": { + "sensitive": false, + "value": "${AUTOCOMPLETE_ENABLED}" + } + } +} +EOF +}