diff --git a/devops/gcp/bootstrap-osdu-module/DataPartitionBundles.py b/devops/gcp/bootstrap-osdu-module/DataPartitionBundles.py index 019252f74df478ec73154420cab6576b3132e790..2d224cf7a01c9c8ed39d2e33775b0e4cbbc32f63 100644 --- a/devops/gcp/bootstrap-osdu-module/DataPartitionBundles.py +++ b/devops/gcp/bootstrap-osdu-module/DataPartitionBundles.py @@ -1,6 +1,7 @@ import os import tarfile from jinja2 import Environment, FileSystemLoader, select_autoescape +import argparse class BootstrapDataPartitionBundles: @@ -35,5 +36,8 @@ class BootstrapDataPartitionBundles: # Initialize class and upload bundles if __name__ == '__main__': - BootstrapDataPartitionBundles.create_and_upload_dp_bundles(os.environ.get('DATA_PARTITION')) + parser = argparse.ArgumentParser() + parser.add_argument('--partition', required=True, type=str) + args = parser.parse_args() + BootstrapDataPartitionBundles.create_and_upload_dp_bundles(args.partition) diff --git a/devops/gcp/bootstrap-osdu-module/Dockerfile b/devops/gcp/bootstrap-osdu-module/Dockerfile index bc94a140035dbc8dd13cf9adb3ecad08b21f58da..78cec86cba57d2b0676c7777d83c649fc5651a8f 100644 --- a/devops/gcp/bootstrap-osdu-module/Dockerfile +++ b/devops/gcp/bootstrap-osdu-module/Dockerfile @@ -6,11 +6,7 @@ COPY ./deployment/ /opt/deployment COPY ./devops/gcp/bootstrap-osdu-module /opt/devops/gcp/bootstrap-osdu-module RUN chmod 775 /opt/bootstrap_policy.sh - RUN apk add py3-pip - -RUN pip3 install -r /opt/requirements_bootstrap.txt - -RUN pip3 install -r /opt/devops/gcp/bootstrap-osdu-module/requirements.txt +RUN pip3 install -r /opt/requirements_bootstrap.txt -r /opt/devops/gcp/bootstrap-osdu-module/requirements.txt CMD ["/bin/bash", "-c", "/opt/bootstrap_policy.sh && sleep 365d"] diff --git a/devops/gcp/bootstrap-osdu-module/bootstrap_policy.sh b/devops/gcp/bootstrap-osdu-module/bootstrap_policy.sh index 710062279e68be04a99cf0800dbf80429c7424be..7efcb5999b35694a2d6f625a549f50c098715d12 100644 --- a/devops/gcp/bootstrap-osdu-module/bootstrap_policy.sh +++ b/devops/gcp/bootstrap-osdu-module/bootstrap_policy.sh @@ -1,54 +1,79 @@ #!/usr/bin/env bash +# +# The following script renders and archives bundles of policies for instance and partition level +# After that archives are uploaded to GCS bucket or MinIO bucket set -ex -create_bundles() { +create_instance_bundles() { + # Renders and archives intance level policies + echo "Archiving bundle of instance policies..." + tar -czf bundle.tar.gz --directory='/opt/deployment/default-policies' --exclude='./bootstrap_sequence.json' . --verbose + mkdir --parents /opt/policies ; mv bundle.tar.gz "$_" + echo "Instance policies archive is ready" +} - echo "Archive bundle of policies" - tar -czf bundle.tar.gz --directory='/opt/deployment/default-policies' --exclude='./bootstrap_sequence.json' . --verbose - mkdir --parents /opt/policies ; mv bundle.tar.gz "$_" - python3 /opt/devops/gcp/bootstrap-osdu-module/DataPartitionBundles.py - mv /opt/bundle-"${DATA_PARTITION}".tar.gz /opt/policies - echo "Archive is ready" +create_partition_bundle() { + # Renders and archives policies for data_partition + # Creates archive named bundle-<data_partition>.tar.gz in /opt/policies + # Args: $1 - data_partition_id + DATA_PARTITION=$1 + echo "Archiving bundle of policies for parition: ${DATA_PARTITION}..." + python3 /opt/devops/gcp/bootstrap-osdu-module/DataPartitionBundles.py --partition "${DATA_PARTITION}" + mv /opt/bundle-"${DATA_PARTITION}".tar.gz /opt/policies + echo "${DATA_PARTITION} partition archive is ready" } bootstrap_gcs() { - - echo "Push archive to GCS bucket" + echo "Push archives to GCS bucket" gsutil rsync /opt/policies gs://"${POLICY_BUCKET}"/ - echo "Bootsrap finished successfully" - + echo "Bootstrap finished successfully" } bootstrap_minio() { - echo "Installing Minio Client (mc) tool" apk add wget wget https://dl.min.io/client/mc/release/linux-amd64/mc chmod +x mc && mv mc /usr/bin/mc echo "mc tool installed successfully, configuring" mc alias set minio "${MINIO_HOST}":"${MINIO_PORT}" "${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}" - echo "Push archive to Minio bucket" + echo "Push archives to Minio bucket" mc mirror --overwrite /opt/policies minio/"${POLICY_BUCKET}" - echo "Bootsrap finished successfully" - + echo "Bootstrap finished successfully" } +# Main part + +source ./validate-env.sh "DATA_PARTITION" +source ./validate-env.sh "POLICY_BUCKET" + +## Creating instance bundles +create_instance_bundles + +## Creating partition bundles +if [[ "${DATA_PARTITION_ID_LIST}" == "" ]]; then + # Single partition case + create_partition_bundle "$DATA_PARTITION" +else + # Multipartition case + IFS=',' read -ra PARTITIONS <<< "${DATA_PARTITION_ID_LIST}" + PARTITIONS=("${DATA_PARTITION}" "${PARTITIONS[@]}") + + for PARTITION in "${PARTITIONS[@]}"; do + create_partition_bundle "${PARTITION}" + done +fi + +## Uploading bundles to gcs/minio bucket if [ "${ONPREM_ENABLED}" == "true" ] then - source ./validate-env.sh "DATA_PARTITION" - source ./validate-env.sh "POLICY_BUCKET" source ./validate-env.sh "MINIO_HOST" source ./validate-env.sh "MINIO_ACCESS_KEY" source ./validate-env.sh "MINIO_SECRET_KEY" source ./validate-env.sh "MINIO_PORT" - create_bundles bootstrap_minio else - source ./validate-env.sh "DATA_PARTITION" - source ./validate-env.sh "POLICY_BUCKET" - create_bundles bootstrap_gcs fi diff --git a/devops/gcp/bootstrap-osdu-module/requirements.txt b/devops/gcp/bootstrap-osdu-module/requirements.txt index 1c579e7d0639eeeb2ebd3335779789856451969d..a8f7292e0382579f9d1b4c8bbddd28698e46ddc2 100644 --- a/devops/gcp/bootstrap-osdu-module/requirements.txt +++ b/devops/gcp/bootstrap-osdu-module/requirements.txt @@ -1 +1 @@ -jinja2 \ No newline at end of file +jinja2==3.1.2 diff --git a/devops/gcp/configmap/README.md b/devops/gcp/configmap/README.md index 57198dfe02aa88996492197d1064b27070f4c1e3..ee7988b31de7d47d3bfbcafa6dfa44cbbac9f827 100644 --- a/devops/gcp/configmap/README.md +++ b/devops/gcp/configmap/README.md @@ -56,6 +56,7 @@ First you need to set variables in **values.yaml** file using any code editor. S | Name | Description | Type | Default |Required | |------|-------------|------|---------|---------| **dataPartitionId** | ID of data partition | string | - | yes +**dataPartitionIdList** | list of secondary data partition ids in case of multipartition | string | - | no **onPremEnabled** | whether on-prem is enabled | boolean | false | yes ### Install the helm chart diff --git a/devops/gcp/configmap/templates/policy-configmap-bootstrap.yaml b/devops/gcp/configmap/templates/policy-configmap-bootstrap.yaml index 65c2dba7d874a9c02982e93ebdd585567c4d208c..b95a6c140113e4310f5a18a1474ddaa4b5d2d60a 100644 --- a/devops/gcp/configmap/templates/policy-configmap-bootstrap.yaml +++ b/devops/gcp/configmap/templates/policy-configmap-bootstrap.yaml @@ -8,4 +8,5 @@ metadata: data: POLICY_BUCKET: "{{ .Values.data.bucketName }}" DATA_PARTITION: "{{ .Values.data.dataPartitionId }}" + DATA_PARTITION_ID_LIST: {{ join "," .Values.data.dataPartitionIdList | quote }} ONPREM_ENABLED: "{{ .Values.conf.onPremEnabled }}" diff --git a/devops/gcp/configmap/values.yaml b/devops/gcp/configmap/values.yaml index 98f59b93eb50f8a8f70a8949c6c5301d7a01c9ae..da6951c0cf702276c1f8f8f361bb7bb9d1e886d9 100644 --- a/devops/gcp/configmap/values.yaml +++ b/devops/gcp/configmap/values.yaml @@ -7,6 +7,7 @@ data: bucketName: "" useBundles: "yes" dataPartitionId: "" + dataPartitionIdList: [] #on-prem only minioHost: "http://minio:9000" diff --git a/devops/gcp/configmap_opa/README.md b/devops/gcp/configmap_opa/README.md index cdf15904bc0273f35a597e18aa7db8eebeb861e0..d933e8309511aa70cdeedaaf6b48a7d2b61be192 100644 --- a/devops/gcp/configmap_opa/README.md +++ b/devops/gcp/configmap_opa/README.md @@ -43,6 +43,7 @@ First you need to set variables in **values.yaml** file using any code editor. S **envConfig** | configmap with env vars | string | opa-env-config | yes **appName** | name of the app | string | opa | yes **dataPartitionId** | data partition id | string | - | yes +**dataPartitionIdList** | list of secondary data partition ids in case of multipartition | string | - | no **onPremEnabled** | whether on-prem is enabled | boolean | false | yes **minDelaySeconds** | min delay for bundle download | num | 6 | yes **maxDelaySeconds** | max delay for bundle download | num | 12 | yes diff --git a/devops/gcp/configmap_opa/templates/opa-configmap.yaml b/devops/gcp/configmap_opa/templates/opa-configmap.yaml index 7da14e263f7a2da46a2cf23f2e0359a7cc090705..872d772160dca4c3562dbdea70bc857e2594d159 100644 --- a/devops/gcp/configmap_opa/templates/opa-configmap.yaml +++ b/devops/gcp/configmap_opa/templates/opa-configmap.yaml @@ -21,12 +21,20 @@ data: service: gcs # NOTE ?alt=media is required resource: 'bundle.tar.gz?alt=media' - osdu/partition/{{ .Values.conf.dataPartitionId }}: + osdu/partition/{{ .Values.data.dataPartitionId }}: service: gcs - resource: 'bundle-{{ .Values.conf.dataPartitionId }}.tar.gz?alt=media' + resource: 'bundle-{{ .Values.data.dataPartitionId }}.tar.gz?alt=media' polling: min_delay_seconds: {{ .Values.conf.minDelaySeconds }} max_delay_seconds: {{ .Values.conf.maxDelaySeconds }} + {{- range (compact .Values.data.dataPartitionIdList) }} + osdu/partition/{{ . }}: + service: gcs + resource: 'bundle-{{ . }}.tar.gz?alt=media' + polling: + min_delay_seconds: {{ $.Values.conf.minDelaySeconds }} + max_delay_seconds: {{ $.Values.conf.maxDelaySeconds }} + {{- end }} {{- else }} config.yaml: | services: @@ -41,6 +49,11 @@ data: resource: bundle.tar.gz osdu/partition/{{ .Values.data.dataPartitionId }}: service: s3 - resource: bundle-{{ .Values.data.dataPartitionId }}.tar.gz + resource: 'bundle-{{ .Values.data.dataPartitionId }}.tar.gz' + {{- range (compact .Values.data.dataPartitionIdList) }} + osdu/partition/{{ . }}: + service: s3 + resource: 'bundle-{{ . }}.tar.gz' + {{- end }} {{- end }} diff --git a/devops/gcp/configmap_opa/values.yaml b/devops/gcp/configmap_opa/values.yaml index 4c03f32d9647b9c6fbffdbe988984b447faf52b6..8b36523d4b452fbaad271f1a21200a703407b771 100644 --- a/devops/gcp/configmap_opa/values.yaml +++ b/devops/gcp/configmap_opa/values.yaml @@ -4,6 +4,7 @@ data: legalHost: "http://legal" entitlementsHost: "http://entitlements" dataPartitionId: "" + dataPartitionIdList: [] conf: configmap: "opa-config"