Add Data Init Docker Containers

tools/data_initialization_agent/access_token/get_access_token.sh

+#!/bin/bash
+
+i=0
+
+while [[ $i -lt 3 ]]; do
+    i=$(expr $i + 1)
+
+    response=$(curl 'http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https%3A%2F%2Fmanagement.azure.com%2F' -H Metadata:true -s)
+
+    if [ -z "$response" -a "$response"==" " ]; then
+        continue
+    fi
+
+    # Get Access Token
+    access_token=$(echo $response | python -c 'import sys, json; print (json.load(sys.stdin)["access_token"])')
+    if [ ! -z "$access_token" -a "$access_token" != " " ]; then
+        echo $access_token
+        exit 0
+    fi
+done
+
+echo "TOKEN_FETCH_FAILURE"
+exit 1
\ No newline at end of file

tools/data_initialization_agent/entitlements_init/Dockerfile

+FROM python:3.9.6-alpine3.14
+
+RUN apk add --no-cache --upgrade bash
+RUN apk add --no-cache --upgrade curl
+
+WORKDIR /usr/src/app
+
+COPY access_token/get_access_token.sh ./
+COPY entitlements_init/data_init.sh ./
+
+RUN chmod +x ./get_access_token.sh
+RUN chmod +x ./data_init.sh
+ENTRYPOINT [ "./data_init.sh" ]
\ No newline at end of file

tools/data_initialization_agent/entitlements_init/data_init.sh

+#!/bin/bash
+
+OSDU_URI=${OSDU_HOST}
+
+if [[ ${OSDU_HOST} != "https://"* ]] || [[ ${OSDU_HOST} != "http://"* ]]; then
+  OSDU_URI="https://${OSDU_HOST}"
+fi
+
+echo "Trying to Fetch Access Token"
+ACCESS_TOKEN=$(sh ./get_access_token.sh)
+if [[ "$ACCESS_TOKEN" == "TOKEN_FETCH_FAILURE" ]]; then
+  echo "Failure fetching Access Token"
+  exit 1
+fi
+echo "Access Token fetched successfully."
+
+IFS=',' read -r -a partitions_array <<< ${PARTITIONS}
+
+partition_count=0
+partition_initialized_count=0
+partition_admin_initialized_count=0
+
+for index in "${!partitions_array[@]}"
+do
+  partition_count=$(expr $partition_count + 1)
+  echo "Intitializing Entitlements for Partition: ${partitions_array[index]}"
+
+  OSDU_ENTITLEMENTS_INIT_URI=${OSDU_URI}/api/entitlements/v2/tenant-provisioning
+  echo "Entitlements Partition Initialization Endpoint: ${OSDU_ENTITLEMENTS_INIT_URI}"
+
+  i=0
+  partition_initialized=false
+  while [[ $i -lt 3 ]]; do
+    i=$(expr $i + 1)
+
+    init_response=$(curl -s -w " Http_Status_Code:%{http_code} " \
+      -X POST \
+      -H 'Content-Type: application/json' \
+      -H "Authorization: Bearer $ACCESS_TOKEN" \
+      -H "data-partition-id: ${partitions_array[index]}" \
+      $OSDU_ENTITLEMENTS_INIT_URI)
+  
+    echo "Init Reponse: $init_response"
+
+    if [ -z "$init_response" -a "$init_response"==" " ]; then
+      echo "Initialization Failed, Empty Reponse. Iteration $i."
+      continue
+    fi
+
+    # Status code check. succeed only if 2xx
+    # quit for partition if 404 or 400.
+    # 401 or 403, then retry after getting access token.
+    # else sleep for 1min and retry
+    if [[ ${init_response} != *"Http_Status_Code:2"* ]];then
+      if [[ ${init_response} == *"Http_Status_Code:400"* ]] || [[ ${init_response} == *"Http_Status_Code:404"* ]];then
+        echo "Entitlements Init for partition ${partitions_array[index]} failed with response $init_response"
+        break
+      fi
+
+      echo "Sleeping for 1min."
+      sleep 1m
+
+      if [[ ${init_response} == *"Http_Status_Code:401"* ]] || [[ ${init_response} == *"Http_Status_Code:403"* ]];then
+        echo "Trying to Re-Fetch Access Token"
+        ACCESS_TOKEN=$(sh ./get_access_token.sh)
+        if [[ "$ACCESS_TOKEN" == "TOKEN_FETCH_FAILURE" ]]; then
+          echo "Failure re-fetching Access Token"
+          exit 1
+        fi
+        echo "Access Token re-fetched successfully."
+      fi
+
+      continue
+    else
+      echo "Entitlements for Partition ${partitions_array[index]} Initialized successfully."
+      partition_initialized_count=$(expr $partition_initialized_count + 1)
+      partition_initialized=true
+
+      break
+    fi
+  done
+
+  if [ "$partition_initialized" != true ] ; then
+    continue
+  fi
+
+  OSDU_ENTITLEMENTS_ADD_OPS_URI=${OSDU_URI}/api/entitlements/v2/groups/users.datalake.ops@${partitions_array[index]}.$SERVICE_DOMAIN/members
+  echo "Entitlements Partition Add Ops Endpoint: ${OSDU_ENTITLEMENTS_ADD_OPS_URI}"
+
+  i=0
+  while [[ $i -lt 3 ]]; do
+    i=$(expr $i + 1)
+
+    init_response=$(curl -s -w " Http_Status_Code:%{http_code} " \
+      -X POST \
+      -H 'Content-Type: application/json' \
+      -H "Authorization: Bearer $ACCESS_TOKEN" \
+      -H "data-partition-id: ${partitions_array[index]}" \
+      -d "{"email":"$ADMIN_ID", "role": "MEMBER"}" \
+      $OSDU_ENTITLEMENTS_ADD_OPS_URI)
+    
+    echo "Init Reponse: $init_response"
+
+    if [ -z "$init_response" -a "$init_response"==" " ]; then
+      echo "Add Ops Member Failed, Empty Reponse. Iteration $i."
+      continue
+    fi
+
+    # Status code check. succeed only if 2xx
+    # quit for partition if 404 or 400.
+    # 401 or 403, then retry after getting access token.
+    # else sleep for 1min and retry
+    if [[ ${init_response} != *"Http_Status_Code:2"* ]];then
+      if [[ ${init_response} == *"Http_Status_Code:400"* ]] || [[ ${init_response} == *"Http_Status_Code:404"* ]];then
+        echo "Add Ops Member for partition ${partitions_array[index]} failed with response $init_response"
+        break
+      fi
+
+      echo "Sleeping for 1min."
+      sleep 1m
+
+      if [[ ${init_response} == *"Http_Status_Code:401"* ]] || [[ ${init_response} == *"Http_Status_Code:403"* ]];then
+        echo "Trying to Re-Fetch Access Token"
+        ACCESS_TOKEN=$(sh ./get_access_token.sh)
+        if [[ "$ACCESS_TOKEN" == "TOKEN_FETCH_FAILURE" ]]; then
+          echo "Failure re-fetching Access Token"
+          exit 1
+        fi
+        echo "Access Token re-fetched successfully."
+      fi
+
+      continue
+    else
+      echo "Ops Member for Partition ${partitions_array[index]} Initialized successfully."
+      partition_admin_initialized_count=$(expr $partition_admin_initialized_count + 1)
+
+      break
+    fi
+  done
+done
+
+if [ "$partition_count" -ne "$partition_initialized_count" ] || [ "$partition_admin_initialized_count" -ne "$partition_initialized_count" ]; then
+  echo "Entitlements for $partition_initialized_count partition(s) of total $partition_count partition(s) initialized successfully."
+  echo "$partition_admin_initialized_count partition(s) of total $partition_initialized_count initialized with Ops Member."
+  exit 1
+else
+  echo "Entitlements for All $partition_initialized_count partition(s) initialized successfully."
+  echo "Ops Members for all of $partition_admin_initialized_count partition(s) added successfully."
+  exit 0
+fi
\ No newline at end of file

tools/data_initialization_agent/entitlements_init/docker-compose.yaml

+version: '3'
+services:
+  entitlements-data-init:
+    build:
+      context: ../
+      dockerfile: ./entitlements_init/Dockerfile
+    image: msosdu.azurecr.io/entitlements-data-init:$VERSION
+    environment:
+      ADMIN_ID: $ADMIN_ID
+      SERVICE_DOMAIN: $SERVICE_DOMAIN
+      OSDU_HOST: $OSDU_HOST
+      PARTITIONS: $PARTITIONS # comma separated list of partitions
+      VERSION: $VERSION
\ No newline at end of file

tools/data_initialization_agent/partition_init/Dockerfile

+FROM python:3.9.6-alpine3.14
+
+RUN apk add --no-cache --upgrade bash
+RUN apk add --no-cache --upgrade curl
+
+WORKDIR /usr/src/app
+
+COPY access_token/get_access_token.sh ./
+COPY partition_init/data_init.sh ./
+COPY partition_init/partition_init_api_payload.json ./
+
+RUN chmod +x ./get_access_token.sh
+RUN chmod +x ./data_init.sh
+ENTRYPOINT [ "./data_init.sh" ]
\ No newline at end of file

tools/data_initialization_agent/partition_init/data_init.sh

+#!/bin/bash
+
+OSDU_URI=${OSDU_HOST}
+
+if [[ ${OSDU_HOST} != "https://"* ]] || [[ ${OSDU_HOST} != "http://"* ]]; then
+  OSDU_URI="https://${OSDU_HOST}"
+fi
+
+echo "Trying to Fetch Access Token"
+ACCESS_TOKEN=$(sh ./get_access_token.sh)
+if [[ "$ACCESS_TOKEN" == "TOKEN_FETCH_FAILURE" ]]; then
+  echo "Failure fetching Access Token"
+  exit 1
+fi
+echo "Access Token fetched successfully."
+
+IFS=',' read -r -a partitions_array <<< ${PARTITIONS}
+
+partition_count=0
+partition_initialized_count=0
+
+for index in "${!partitions_array[@]}"
+do
+  partition_count=$(expr $partition_count + 1)
+  echo "Intitializing Partition: ${partitions_array[index]}"
+
+  OSDU_PARTITION_INIT_URI=${OSDU_URI}/api/partition/v1/partitions/${partitions_array[index]}
+  echo "Partition Initialization Endpoint: ${OSDU_PARTITION_INIT_URI}"
+
+  i=0
+  while [[ $i -lt 3 ]]; do
+    i=$(expr $i + 1)
+
+    init_response=$(curl -s -w " Http_Status_Code:%{http_code} " \
+      -X POST \
+      -H 'Content-Type: application/json' \
+      -H "Authorization: Bearer $ACCESS_TOKEN" \
+      -H "data-partition-id: ${partitions_array[index]}" \
+      -d "@partition_init_api_payload.json" \
+      $OSDU_PARTITION_INIT_URI)
+  
+    echo "Init Reponse: $init_response"
+
+    if [ -z "$init_response" -a "$init_response"==" " ]; then
+      echo "Initialization Failed, Empty Reponse. Iteration $i."
+      continue
+    fi
+
+    # Status code check. succeed only if 2xx
+    # quit for partition if 404 or 400.
+    # 401 or 403, then retry after getting access token.
+    # else sleep for 1min and retry
+    if [[ ${init_response} != *"Http_Status_Code:2"* ]];then
+      if [[ ${init_response} == *"Http_Status_Code:400"* ]] || [[ ${init_response} == *"Http_Status_Code:404"* ]];then
+        echo "Partition Init for partition ${partitions_array[index]} failed with response $init_response"
+        break
+      fi
+
+      echo "Sleeping for 1min."
+      sleep 1m
+
+      if [[ ${init_response} == *"Http_Status_Code:401"* ]] || [[ ${init_response} == *"Http_Status_Code:403"* ]];then
+        echo "Trying to Re-Fetch Access Token"
+        ACCESS_TOKEN=$(sh ./get_access_token.sh)
+        if [[ "$ACCESS_TOKEN" == "TOKEN_FETCH_FAILURE" ]]; then
+          echo "Failure re-fetching Access Token"
+          exit 1
+        fi
+        echo "Access Token re-fetched successfully."
+      fi
+
+      continue
+    else
+      echo "Partition ${partitions_array[index]} Initialized successfully."
+      partition_initialized_count=$(expr $partition_initialized_count + 1)
+
+      break
+    fi
+  done
+done
+
+if [ "$partition_count" -ne "$partition_initialized_count" ]; then
+  echo "$partition_initialized_count partition(s) of total $partition_count partition(s) initialized successfully."
+  exit 1
+else
+ echo "All $partition_initialized_count partition(s) initialized successfully."
+ exit 0
+fi
\ No newline at end of file

tools/data_initialization_agent/partition_init/docker-compose.yaml

+version: '3'
+services:
+  partition-data-init:
+    build:
+      context: ../
+      dockerfile: ./partition_init/Dockerfile
+    image: msosdu.azurecr.io/partition-data-init:$VERSION
+    environment:
+      OSDU_HOST: $OSDU_HOST
+      PARTITIONS: $PARTITIONS # comma separated list of partitions
+      VERSION: $VERSION
\ No newline at end of file

tools/data_initialization_agent/partition_init/partition_init_api_payload.json

+{
+    "properties": {
+        "compliance-ruleset": {
+            "value": "shared"
+        },
+        "elastic-endpoint": {
+            "sensitive": true,
+            "value": "elastic-endpoint"
+        },
+        "elastic-username": {
+            "sensitive": true,
+            "value": "elastic-username"
+        },
+        "elastic-password": {
+            "sensitive": true,
+            "value": "elastic-password"
+        },
+        "cosmos-connection": {
+            "sensitive": true,
+            "value": "cosmos-connection"
+        },
+        "cosmos-endpoint": {
+            "sensitive": true,
+            "value": "cosmos-endpoint"
+        },
+        "cosmos-primary-key": {
+            "sensitive": true,
+            "value": "cosmos-primary-key"
+        },
+        "sb-connection": {
+            "sensitive": true,
+            "value": "sb-connection"
+        },
+        "sb-namespace": {
+            "sensitive": true,
+            "value": "sb-namespace"
+        },
+        "storage-account-key": {
+            "sensitive": true,
+            "value": "storage-key"
+        },
+        "storage-account-name": {
+            "sensitive": true,
+            "value": "storage"
+        },
+        "ingest-storage-account-name": {
+            "sensitive": true,
+            "value": "ingest-storage"
+        },
+        "ingest-storage-account-key": {
+            "sensitive": true,
+            "value": "ingest-storage-key"
+        },
+        "eventgrid-recordstopic": {
+            "sensitive": true,
+            "value": "eventgrid-recordstopic"
+        },
+        "eventgrid-recordstopic-accesskey": {
+            "sensitive": true,
+            "value": "eventgrid-recordstopic-accesskey"
+        },
+        "eventgrid-legaltagschangedtopic": {
+            "sensitive": true,
+            "value": "eventgrid-legaltagschangedtopic"
+        },
+        "eventgrid-legaltagschangedtopic-accesskey": {
+            "sensitive": true,
+            "value": "eventgrid-legaltagschangedtopic-accesskey"
+        },
+        "eventgrid-resourcegroup": {
+            "sensitive": true,
+            "value": "eventgrid-resourcegroup"
+        },
+        "encryption-key-identifier": {
+            "sensitive": true,
+            "value": "encryption-key-identifier"
+        },
+        "sdms-storage-account-name": {
+            "sensitive": true,
+            "value": "sdms-storage"
+        },
+        "sdms-storage-account-key": {
+            "sensitive": true,
+            "value": "sdms-storage-key"
+        },
+        "eventgrid-schemanotificationtopic": {
+            "sensitive": true,
+            "value": "eventgrid-schemachangedtopic"
+        },
+         "eventgrid-schemanotificationtopic-accesskey": {
+            "sensitive": true,
+            "value": "eventgrid-schemachangedtopic-accesskey"
+        },
+        "eventgrid-gsmtopic": {
+            "sensitive": true,
+            "value": "eventgrid-statuschangedtopic"
+        },
+         "eventgrid-gsmtopic-accesskey": {
+            "sensitive": true,
+            "value": "eventgrid-statuschangedtopic-accesskey"
+        },
+        "eventgrid-statuschangedtopic": {
+            "sensitive": true,
+            "value": "eventgrid-statuschangedtopic"
+        },
+         "eventgrid-statuschangedtopic-accesskey": {
+            "sensitive": true,
+            "value": "eventgrid-statuschangedtopic-accesskey"
+        },
+        "eventgrid-schemachangedtopic": {
+            "sensitive": true,
+            "value": "eventgrid-schemachangedtopic"
+        },
+         "eventgrid-schemachangedtopic-accesskey": {
+            "sensitive": true,
+            "value": "eventgrid-schemachangedtopic-accesskey"
+        }
+    }
+}

tools/data_seeding_agent/seed_data.sh

@@ -7,14 +7,14 @@ az login --service-principal -u $ARM_CLIENT_ID -p $ARM_CLIENT_SECRET --tenant $A
 # in the container  legal-service-azure-configuration.
 ENV_VAULT=$(az keyvault list --resource-group $RESOURCE_GROUP_NAME --query [].name -otsv)
 
-IFS=',' read -r -a partitions <<< ${PARTITIONS}
+IFS=',' read -r -a partitions_array <<< ${PARTITIONS}
 
-for index in "${!partitions[@]}"
+for index in "${!partitions_array[@]}"
 do
-    echo "Ingesting file for partition: $index. ${partitions[index]}"
+    echo "Ingesting file for partition: $index. ${partitions_array[index]}"
     
-    STORAGE_ACCOUNT_NAME=$(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/${partitions[index]}-storage --query value -otsv)
-    STORAGE_ACCOUNT_KEY=$(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/${partitions[index]}-storage-key --query value -otsv)
+    STORAGE_ACCOUNT_NAME=$(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/${partitions_array[index]}-storage --query value -otsv)
+    STORAGE_ACCOUNT_KEY=$(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/${partitions_array[index]}-storage-key --query value -otsv)
     FILE_NAME=Legal_COO.json
 
     if [ -z "$STORAGE_ACCOUNT_NAME" -a "$STORAGE_ACCOUNT_NAME"==" " ]; then
@@ -59,7 +59,7 @@ do
         fi
     fi
 
-    echo "File ingested for partition: $index. ${partitions[index]}"
+    echo "File ingested for partition: $index. ${partitions_array[index]}"
     
 done