[GONRG-5913] Added multipartition support

devops/gcp/bootstrap-osdu-module/DataPartitionBundles.py

 import os
 import tarfile
 from jinja2 import Environment, FileSystemLoader, select_autoescape
+import argparse
 
 
 class BootstrapDataPartitionBundles:
@@ -35,5 +36,8 @@ class BootstrapDataPartitionBundles:
 
 # Initialize class and upload bundles
 if __name__ == '__main__':
-    BootstrapDataPartitionBundles.create_and_upload_dp_bundles(os.environ.get('DATA_PARTITION'))
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--partition', required=True, type=str)
+    args = parser.parse_args()
+    BootstrapDataPartitionBundles.create_and_upload_dp_bundles(args.partition)
 

devops/gcp/bootstrap-osdu-module/Dockerfile

@@ -6,11 +6,7 @@ COPY ./deployment/ /opt/deployment
 COPY ./devops/gcp/bootstrap-osdu-module /opt/devops/gcp/bootstrap-osdu-module
 
 RUN chmod 775 /opt/bootstrap_policy.sh
-
 RUN apk add py3-pip
-
-RUN pip3 install -r /opt/requirements_bootstrap.txt
-
-RUN pip3 install -r /opt/devops/gcp/bootstrap-osdu-module/requirements.txt
+RUN pip3 install -r /opt/requirements_bootstrap.txt -r /opt/devops/gcp/bootstrap-osdu-module/requirements.txt
 
 CMD ["/bin/bash", "-c", "/opt/bootstrap_policy.sh && sleep 365d"]

devops/gcp/bootstrap-osdu-module/bootstrap_policy.sh

 #!/usr/bin/env bash
+#
+# The following script renders and archives bundles of policies for instance and partition level
+# After that archives are uploaded to GCS bucket or MinIO bucket
 
 set -ex
 
-create_bundles() {
+create_instance_bundles() {
+  # Renders and archives intance level policies
+  echo "Archiving bundle of instance policies..."
+  tar -czf bundle.tar.gz --directory='/opt/deployment/default-policies' --exclude='./bootstrap_sequence.json' . --verbose
+  mkdir --parents /opt/policies ; mv bundle.tar.gz "$_"
+  echo "Instance policies archive is ready"
+}
 
-    echo "Archive bundle of policies"
-    tar -czf bundle.tar.gz --directory='/opt/deployment/default-policies' --exclude='./bootstrap_sequence.json' . --verbose
-    mkdir --parents /opt/policies ; mv bundle.tar.gz "$_"
-    python3 /opt/devops/gcp/bootstrap-osdu-module/DataPartitionBundles.py
-    mv /opt/bundle-"${DATA_PARTITION}".tar.gz /opt/policies
-    echo "Archive is ready"
+create_partition_bundle() {
+  # Renders and archives policies for data_partition
+  # Creates archive named bundle-<data_partition>.tar.gz in /opt/policies
+  # Args: $1 - data_partition_id
 
+  DATA_PARTITION=$1
+  echo "Archiving bundle of policies for parition: ${DATA_PARTITION}..."
+  python3 /opt/devops/gcp/bootstrap-osdu-module/DataPartitionBundles.py --partition "${DATA_PARTITION}"
+  mv /opt/bundle-"${DATA_PARTITION}".tar.gz /opt/policies
+  echo "${DATA_PARTITION} partition archive is ready"
 }
 
 bootstrap_gcs() {
-
-    echo "Push archive to GCS bucket"
+    echo "Push archives to GCS bucket"
     gsutil rsync /opt/policies gs://"${POLICY_BUCKET}"/
-    echo "Bootsrap finished successfully"
-
+    echo "Bootstrap finished successfully"
 }
 
 bootstrap_minio() {
-
     echo "Installing Minio Client (mc) tool"
     apk add wget
     wget https://dl.min.io/client/mc/release/linux-amd64/mc
     chmod +x mc && mv mc /usr/bin/mc
     echo "mc tool installed successfully, configuring"
     mc alias set minio "${MINIO_HOST}":"${MINIO_PORT}" "${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}"
-    echo "Push archive to Minio bucket"
+    echo "Push archives to Minio bucket"
     mc mirror --overwrite /opt/policies minio/"${POLICY_BUCKET}"
-    echo "Bootsrap finished successfully"
-
+    echo "Bootstrap finished successfully"
 }
 
+# Main part
+
+source ./validate-env.sh "DATA_PARTITION"
+source ./validate-env.sh "POLICY_BUCKET"
+
+## Creating instance bundles
+create_instance_bundles
+
+## Creating partition bundles
+if [[ "${DATA_PARTITION_ID_LIST}" == "" ]]; then
+  # Single partition case
+  create_partition_bundle "$DATA_PARTITION"
+else
+  # Multipartition case
+  IFS=',' read -ra PARTITIONS <<< "${DATA_PARTITION_ID_LIST}"
+  PARTITIONS=("${DATA_PARTITION}" "${PARTITIONS[@]}")
+
+  for PARTITION in "${PARTITIONS[@]}"; do
+    create_partition_bundle "${PARTITION}"
+  done
+fi
+
+## Uploading bundles to gcs/minio bucket
 if [ "${ONPREM_ENABLED}" == "true" ]
 then
-  source ./validate-env.sh "DATA_PARTITION"
-  source ./validate-env.sh "POLICY_BUCKET"
   source ./validate-env.sh "MINIO_HOST"
   source ./validate-env.sh "MINIO_ACCESS_KEY"
   source ./validate-env.sh "MINIO_SECRET_KEY"
   source ./validate-env.sh "MINIO_PORT"
-  create_bundles
   bootstrap_minio
 else
-  source ./validate-env.sh "DATA_PARTITION"
-  source ./validate-env.sh "POLICY_BUCKET"
-  create_bundles
   bootstrap_gcs
 fi
 

devops/gcp/bootstrap-osdu-module/requirements.txt

-jinja2
\ No newline at end of file
+jinja2==3.1.2

devops/gcp/configmap/README.md

@@ -56,6 +56,7 @@ First you need to set variables in **values.yaml** file using any code editor. S
 | Name | Description | Type | Default |Required |
 |------|-------------|------|---------|---------|
 **dataPartitionId** | ID of data partition | string | - | yes
+**dataPartitionIdList** | list of secondary data partition ids in case of multipartition | string | - | no
 **onPremEnabled** | whether on-prem is enabled | boolean | false | yes
 
 ### Install the helm chart

devops/gcp/configmap/templates/policy-configmap-bootstrap.yaml

@@ -8,4 +8,5 @@ metadata:
 data:
   POLICY_BUCKET: "{{ .Values.data.bucketName }}"
   DATA_PARTITION: "{{ .Values.data.dataPartitionId }}"
+  DATA_PARTITION_ID_LIST: {{ join "," .Values.data.dataPartitionIdList | quote }}
   ONPREM_ENABLED: "{{ .Values.conf.onPremEnabled }}"

devops/gcp/configmap/values.yaml

@@ -7,6 +7,7 @@ data:
   bucketName: ""
   useBundles: "yes"
   dataPartitionId: ""
+  dataPartitionIdList: []
   #on-prem only
   minioHost: "http://minio:9000"
 

devops/gcp/configmap_opa/README.md

@@ -43,6 +43,7 @@ First you need to set variables in **values.yaml** file using any code editor. S
 **envConfig** | configmap with env vars | string | opa-env-config | yes
 **appName** | name of the app | string | opa | yes
 **dataPartitionId** | data partition id | string | - | yes
+**dataPartitionIdList** | list of secondary data partition ids in case of multipartition | string | - | no
 **onPremEnabled** | whether on-prem is enabled | boolean | false | yes
 **minDelaySeconds** | min delay for bundle download | num | 6 | yes
 **maxDelaySeconds** | max delay for bundle download | num | 12 | yes

devops/gcp/configmap_opa/templates/opa-configmap.yaml

@@ -21,12 +21,20 @@ data:
         service: gcs
         # NOTE ?alt=media is required
         resource: 'bundle.tar.gz?alt=media'
-      osdu/partition/{{ .Values.conf.dataPartitionId }}:
+      osdu/partition/{{ .Values.data.dataPartitionId }}:
         service: gcs
-        resource: 'bundle-{{ .Values.conf.dataPartitionId }}.tar.gz?alt=media'
+        resource: 'bundle-{{ .Values.data.dataPartitionId }}.tar.gz?alt=media'
         polling:
           min_delay_seconds: {{ .Values.conf.minDelaySeconds }}
           max_delay_seconds: {{ .Values.conf.maxDelaySeconds }}
+      {{- range (compact .Values.data.dataPartitionIdList) }}
+      osdu/partition/{{ . }}:
+        service: gcs
+        resource: 'bundle-{{ . }}.tar.gz?alt=media'
+        polling:
+          min_delay_seconds: {{ $.Values.conf.minDelaySeconds }}
+          max_delay_seconds: {{ $.Values.conf.maxDelaySeconds }}
+      {{- end }}
 {{- else }}
   config.yaml: |
     services:
@@ -41,6 +49,11 @@ data:
         resource: bundle.tar.gz
       osdu/partition/{{ .Values.data.dataPartitionId }}:
         service: s3
-        resource: bundle-{{ .Values.data.dataPartitionId }}.tar.gz
+        resource: 'bundle-{{ .Values.data.dataPartitionId }}.tar.gz'
+      {{- range (compact .Values.data.dataPartitionIdList) }}
+      osdu/partition/{{ . }}:
+        service: s3
+        resource: 'bundle-{{ . }}.tar.gz'
+      {{- end }}
 {{- end }}
 

devops/gcp/configmap_opa/values.yaml

@@ -4,6 +4,7 @@ data:
   legalHost: "http://legal"
   entitlementsHost: "http://entitlements"
   dataPartitionId: ""
+  dataPartitionIdList: []
 
 conf:
   configmap: "opa-config"