Skip to content
Snippets Groups Projects
Commit 7f916a87 authored by Mikhail Piatliou (EPAM)'s avatar Mikhail Piatliou (EPAM)
Browse files

Merge branch 'gcp-multipartition' into 'master' 

Add multipartition bootstrap [GONRG-5697]

See merge request !271
parents 8222a6bd ef7cd339
No related branches found
No related tags found
1 merge request!271Add multipartition bootstrap [GONRG-5697]
Pipeline #142637 failed
Stage: build
Stage: coverage
Stage: containerize
Stage: scan
Stage: deploy
Stage: integration
Stage: publish
Hide whitespace changes
Inline Side-by-side
devops/gcp/configmap/README.md
+ 2
− 0
View file @ 7f916a87
......@@ -120,6 +120,7 @@ cat ~/.config/gcloud/application_default_credentials.json | grep client_id
|------|-------------|------|---------|---------|
**partitionName** | name of the partition | string | partition | yes
**dataPartitionId** | ID of data partition | string | - | yes
**dataPartitionIdList** | List of data partitions | array of strings | [] | no
**datafierSa** | datafier service account | string | datafier | yes
### Bootstrap on-prem variables
......@@ -136,6 +137,7 @@ cat ~/.config/gcloud/application_default_credentials.json | grep client_id
**configmap** | configmap to be used | string | partition-config | yes
**appName** | name of the app | string | partition | yes
**onPremEnabled** | whether on-prem is enabled | boolean | false | yes
**multiPartitionEnabled** | whether multipartition is enabled | boolean | false | yes
### Install the helm chart
......
devops/gcp/configmap/templates/configmap-bootstrap.yml
+ 8
− 2
View file @ 7f916a87
......@@ -18,9 +18,15 @@ data:
{{- if .Values.conf.onPremEnabled }}
SERVICE_ACCOUNT: {{ printf "%s@service.local" .Values.data.datafierSa | quote }}
DOMAIN: "{{ .Values.data.domain }}"
MINIO_ENDPOINT: "{{ default "http://minio:9000" .Values.data.minioExternalEndpoint }}"
FILE_MINIO_ENDPOINT: "{{ default (printf "https://s3.%s" .Values.data.domain) .Values.data.minioExternalEndpoint }}"
MINIO_ENDPOINT: {{ default "http://minio:9000" .Values.data.minioExternalEndpoint | quote }}
FILE_MINIO_ENDPOINT: {{ default (printf "https://s3.%s" .Values.data.domain) .Values.data.minioExternalEndpoint | quote }}
{{- else }}
SERVICE_ACCOUNT: {{ printf "%s@%s.iam.gserviceaccount.com" .Values.data.datafierSa .Values.data.projectId | quote }}
AUDIENCES: "{{ .Values.data.googleAudiences }}"
{{- end }}
{{- if .Values.conf.multiPartitionEnabled }}
DATA_PARTITION_ID_LIST: {{ join "," .Values.data.dataPartitionIdList | quote }}
MULTI_PARTITON_ENABLED: "{{ .Values.conf.multiPartitionEnabled }}"
{{- else }}
MULTI_PARTITON_ENABLED: "{{ .Values.conf.multiPartitionEnabled }}"
{{- end }}
devops/gcp/configmap/values.yaml
+ 3
− 1
View file @ 7f916a87
......@@ -11,12 +11,14 @@ data:
# bootstrap common
partitionName: "partition"
dataPartitionId: ""
dataPartitionIdList: []
datafierSa: "datafier"
partitionCleanUpEnabled: "false"
# bootstrap variables onprem
# bootstrap onprem
domain: ""
minioExternalEndpoint: "" # use only if external minio is configured
conf:
configmap: "partition-config"
appName: "partition"
onPremEnabled: false
multiPartitionEnabled: false
provider/partition-gcp/bootstrap/bootstrap_partition.sh
+ 58
− 45
View file @ 7f916a87
......@@ -5,23 +5,27 @@ set -ex
source ./data_anthos.sh
source ./data_gcp.sh
# Bootstrap Partition service on Anthos (on-prem)
bootstrap_anthos() {
if [ "$PARTITION_CLEAN_UP_ENABLED" == "true" ]
then
DATA_PARTITION_ID=$1
DATA_PARTITION_ID_UPPER=$2
if [[ "${PARTITION_CLEAN_UP_ENABLED}" == "true" ]]; then
echo "Partition cleanup enabled, will delete partition ${DATA_PARTITION_ID}"
delete_status_code=$(curl -X DELETE \
--url "http://${PARTITION_NAME}/api/partition/v1/partitions/${DATA_PARTITION_ID}" --write-out "%{http_code}" --silent --output "/dev/null" \
-H "Content-Type: application/json")
if [ "$delete_status_code" == 204 ] || [ "$delete_status_code" == 404 ]
then
echo "Partition deletion was successful, with status code: ${delete_status_code}"
else
echo "Not able to delete partition, status code is: ${delete_status_code}"
exit 1
fi
--url "http://${PARTITION_NAME}/api/partition/v1/partitions/${DATA_PARTITION_ID}" --write-out "%{http_code}" --silent --output "/dev/null" \
-H "Content-Type: application/json")
if [[ "${delete_status_code}" == 204 ]] || [[ "${delete_status_code}" == 404 ]]; then
echo "Partition deletion was successful with status code: ${delete_status_code}"
else
echo "Not able to delete partition with status code: ${delete_status_code}"
exit 1
fi
else
echo "Partition cleanup not enabled, skipping deletion"
echo "Partition cleanup is not enabled, skipping deletion"
fi
status_code=$(curl -X POST \
......@@ -30,42 +34,45 @@ bootstrap_anthos() {
--data-raw "$(generate_post_data_anthos)")
# shellcheck disable=SC2002
if [ "$status_code" == 201 ]
then
if [[ "${status_code}" == 201 ]]; then
echo "Partition bootstrap finished successfully!"
elif [ "$status_code" == 409 ]
then
elif [[ "${status_code}" == 409 ]]; then
patch_status_code=$(curl -X PATCH \
--url "http://${PARTITION_NAME}/api/partition/v1/partitions/${DATA_PARTITION_ID}" --write-out "%{http_code}" --silent --output "/dev/null" \
-H "Content-Type: application/json" \
--data-raw "$(generate_post_data_anthos)")
echo "Partition was patched because Postgres Database had already had entities! Status code of patching: $patch_status_code"
echo "Partition was patched because Postgres Database had already had entities! Status code: ${patch_status_code}"
else
echo "Exiting with status code: $status_code"
echo "Exiting with status code: ${status_code}"
exit 1
fi
}
# Bootstrap Partition service on Google Cloud
bootstrap_gcp() {
echo "sleep to prevent 500 response from the Partition service, due to timeout of creation for Workload Identity"
sleep 20
DATA_PARTITION_ID=$1
DATA_PARTITION_ID_UPPER=$2
IDENTITY_TOKEN=$(gcloud auth print-identity-token --audiences="${AUDIENCES}")
if [ "$PARTITION_CLEAN_UP_ENABLED" == "true" ]
then
echo "Partition cleanup enabled, will delete partition ${DATA_PARTITION_ID}"
delete_status_code=$(curl -X DELETE \
--url "http://${PARTITION_NAME}/api/partition/v1/partitions/${DATA_PARTITION_ID}" --write-out "%{http_code}" --silent --output "/dev/null" \
-H "Authorization: Bearer ${IDENTITY_TOKEN}")
if [ "$delete_status_code" == 204 ] || [ "$delete_status_code" == 404 ]
then
echo "Partition deletion was successful, with status code: ${delete_status_code}"
else
echo "Not able to delete partition, status code is: ${delete_status_code}"
exit 1
fi
if [[ "${PARTITION_CLEAN_UP_ENABLED}" == "true" ]]; then
echo "Partition cleanup enabled, will delete partition ${DATA_PARTITION_ID}"
delete_status_code=$(curl -X DELETE \
--url "http://${PARTITION_NAME}/api/partition/v1/partitions/${DATA_PARTITION_ID}" --write-out "%{http_code}" --silent --output "/dev/null" \
-H "Authorization: Bearer ${IDENTITY_TOKEN}")
if [[ "${delete_status_code}" == 204 ]] || [[ "${delete_status_code}" == 404 ]]; then
echo "Partition deletion was successful with status code: ${delete_status_code}"
else
echo "Not able to delete partition with status code: ${delete_status_code}"
exit 1
fi
else
echo "Partition cleanup not enabled, skipping deletion"
fi
......@@ -77,29 +84,35 @@ bootstrap_gcp() {
--data-raw "$(generate_post_data_gcp)")
# shellcheck disable=SC2002
if [ "$status_code" == 201 ]
then
if [[ "${status_code}" == 201 ]]; then
echo "Partition bootstrap finished successfully!"
elif [ "$status_code" == 409 ]
then
elif [[ "${status_code}" == 409 ]]; then
patch_status_code=$(curl -X PATCH \
--url "http://${PARTITION_NAME}/api/partition/v1/partitions/${DATA_PARTITION_ID}" --write-out "%{http_code}" --silent --output "/dev/null" \
-H "Authorization: Bearer ${IDENTITY_TOKEN}" \
-H "Content-Type: application/json" \
--data-raw "$(generate_post_data_gcp)")
echo "Partition was patched because Datastore had already had entities! Status code of patching: $patch_status_code"
echo "Partition was patched because Datastore had already had entities! Status code: ${patch_status_code}"
else
echo "Exiting with status code: $status_code"
echo "Exiting with status code: ${status_code}"
exit 1
fi
}
if [ "$ENVIRONMENT" == "anthos" ]
then
bootstrap_anthos
elif [ "$ENVIRONMENT" == "gcp" ]
then
bootstrap_gcp
if [[ "${ENVIRONMENT}" == "anthos" && "${MULTI_PARTITON_ENABLED}" == "false" ]]; then
bootstrap_anthos "${DATA_PARTITION_ID}" "${DATA_PARTITION_ID^^}"
elif [[ "${ENVIRONMENT}" == "gcp" && "${MULTI_PARTITON_ENABLED}" == "false" ]]; then
bootstrap_gcp "${DATA_PARTITION_ID}" "${DATA_PARTITION_ID^^}"
elif [[ "${ENVIRONMENT}" == "gcp" && "${MULTI_PARTITON_ENABLED}" == "true" ]]; then
IFS=',' read -ra PARTITIONS <<< "${DATA_PARTITION_ID_LIST}"
PARTITIONS=("${DATA_PARTITION_ID}" "${PARTITIONS[@]}")
for PARTITION in "${PARTITIONS[@]}"; do
bootstrap_gcp "${PARTITION}" "${PARTITION^^}"
done
fi
touch /tmp/bootstrap_ready
provider/partition-gcp/bootstrap/data_anthos.sh
+ 0
− 2
View file @ 7f916a87
#!/usr/bin/env bash
DATA_PARTITION_ID_UPPER=${DATA_PARTITION_ID^^}
generate_post_data_anthos() {
cat <<EOF
{
......
provider/partition-gcp/bootstrap/data_gcp.sh
+ 0
− 2
View file @ 7f916a87
#!/usr/bin/env bash
DATA_PARTITION_ID_UPPER=${DATA_PARTITION_ID^^}
generate_post_data_gcp() {
cat <<EOF
{
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment