Commit 7b922af4 authored by Aryaan Singh's avatar Aryaan Singh Committed by MANISH KUMAR
Browse files

Add Data Init Docker Containers

parent 7fdec0e3
#!/bin/bash
i=0
while [[ $i -lt 3 ]]; do
i=$(expr $i + 1)
response=$(curl 'http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https%3A%2F%2Fmanagement.azure.com%2F' -H Metadata:true -s)
if [ -z "$response" -a "$response"==" " ]; then
continue
fi
# Get Access Token
access_token=$(echo $response | python -c 'import sys, json; print (json.load(sys.stdin)["access_token"])')
if [ ! -z "$access_token" -a "$access_token" != " " ]; then
echo $access_token
exit 0
fi
done
echo "TOKEN_FETCH_FAILURE"
exit 1
\ No newline at end of file
FROM python:3.9.6-alpine3.14
RUN apk add --no-cache --upgrade bash
RUN apk add --no-cache --upgrade curl
WORKDIR /usr/src/app
COPY access_token/get_access_token.sh ./
COPY entitlements_init/data_init.sh ./
RUN chmod +x ./get_access_token.sh
RUN chmod +x ./data_init.sh
ENTRYPOINT [ "./data_init.sh" ]
\ No newline at end of file
#!/bin/bash
OSDU_URI=${OSDU_HOST}
if [[ ${OSDU_HOST} != "https://"* ]] || [[ ${OSDU_HOST} != "http://"* ]]; then
OSDU_URI="https://${OSDU_HOST}"
fi
echo "Trying to Fetch Access Token"
ACCESS_TOKEN=$(sh ./get_access_token.sh)
if [[ "$ACCESS_TOKEN" == "TOKEN_FETCH_FAILURE" ]]; then
echo "Failure fetching Access Token"
exit 1
fi
echo "Access Token fetched successfully."
IFS=',' read -r -a partitions_array <<< ${PARTITIONS}
partition_count=0
partition_initialized_count=0
partition_admin_initialized_count=0
for index in "${!partitions_array[@]}"
do
partition_count=$(expr $partition_count + 1)
echo "Intitializing Entitlements for Partition: ${partitions_array[index]}"
OSDU_ENTITLEMENTS_INIT_URI=${OSDU_URI}/api/entitlements/v2/tenant-provisioning
echo "Entitlements Partition Initialization Endpoint: ${OSDU_ENTITLEMENTS_INIT_URI}"
i=0
partition_initialized=false
while [[ $i -lt 3 ]]; do
i=$(expr $i + 1)
init_response=$(curl -s -w " Http_Status_Code:%{http_code} " \
-X POST \
-H 'Content-Type: application/json' \
-H "Authorization: Bearer $ACCESS_TOKEN" \
-H "data-partition-id: ${partitions_array[index]}" \
$OSDU_ENTITLEMENTS_INIT_URI)
echo "Init Reponse: $init_response"
if [ -z "$init_response" -a "$init_response"==" " ]; then
echo "Initialization Failed, Empty Reponse. Iteration $i."
continue
fi
# Status code check. succeed only if 2xx
# quit for partition if 404 or 400.
# 401 or 403, then retry after getting access token.
# else sleep for 1min and retry
if [[ ${init_response} != *"Http_Status_Code:2"* ]];then
if [[ ${init_response} == *"Http_Status_Code:400"* ]] || [[ ${init_response} == *"Http_Status_Code:404"* ]];then
echo "Entitlements Init for partition ${partitions_array[index]} failed with response $init_response"
break
fi
echo "Sleeping for 1min."
sleep 1m
if [[ ${init_response} == *"Http_Status_Code:401"* ]] || [[ ${init_response} == *"Http_Status_Code:403"* ]];then
echo "Trying to Re-Fetch Access Token"
ACCESS_TOKEN=$(sh ./get_access_token.sh)
if [[ "$ACCESS_TOKEN" == "TOKEN_FETCH_FAILURE" ]]; then
echo "Failure re-fetching Access Token"
exit 1
fi
echo "Access Token re-fetched successfully."
fi
continue
else
echo "Entitlements for Partition ${partitions_array[index]} Initialized successfully."
partition_initialized_count=$(expr $partition_initialized_count + 1)
partition_initialized=true
break
fi
done
if [ "$partition_initialized" != true ] ; then
continue
fi
OSDU_ENTITLEMENTS_ADD_OPS_URI=${OSDU_URI}/api/entitlements/v2/groups/users.datalake.ops@${partitions_array[index]}.$SERVICE_DOMAIN/members
echo "Entitlements Partition Add Ops Endpoint: ${OSDU_ENTITLEMENTS_ADD_OPS_URI}"
i=0
while [[ $i -lt 3 ]]; do
i=$(expr $i + 1)
init_response=$(curl -s -w " Http_Status_Code:%{http_code} " \
-X POST \
-H 'Content-Type: application/json' \
-H "Authorization: Bearer $ACCESS_TOKEN" \
-H "data-partition-id: ${partitions_array[index]}" \
-d "{"email":"$ADMIN_ID", "role": "MEMBER"}" \
$OSDU_ENTITLEMENTS_ADD_OPS_URI)
echo "Init Reponse: $init_response"
if [ -z "$init_response" -a "$init_response"==" " ]; then
echo "Add Ops Member Failed, Empty Reponse. Iteration $i."
continue
fi
# Status code check. succeed only if 2xx
# quit for partition if 404 or 400.
# 401 or 403, then retry after getting access token.
# else sleep for 1min and retry
if [[ ${init_response} != *"Http_Status_Code:2"* ]];then
if [[ ${init_response} == *"Http_Status_Code:400"* ]] || [[ ${init_response} == *"Http_Status_Code:404"* ]];then
echo "Add Ops Member for partition ${partitions_array[index]} failed with response $init_response"
break
fi
echo "Sleeping for 1min."
sleep 1m
if [[ ${init_response} == *"Http_Status_Code:401"* ]] || [[ ${init_response} == *"Http_Status_Code:403"* ]];then
echo "Trying to Re-Fetch Access Token"
ACCESS_TOKEN=$(sh ./get_access_token.sh)
if [[ "$ACCESS_TOKEN" == "TOKEN_FETCH_FAILURE" ]]; then
echo "Failure re-fetching Access Token"
exit 1
fi
echo "Access Token re-fetched successfully."
fi
continue
else
echo "Ops Member for Partition ${partitions_array[index]} Initialized successfully."
partition_admin_initialized_count=$(expr $partition_admin_initialized_count + 1)
break
fi
done
done
if [ "$partition_count" -ne "$partition_initialized_count" ] || [ "$partition_admin_initialized_count" -ne "$partition_initialized_count" ]; then
echo "Entitlements for $partition_initialized_count partition(s) of total $partition_count partition(s) initialized successfully."
echo "$partition_admin_initialized_count partition(s) of total $partition_initialized_count initialized with Ops Member."
exit 1
else
echo "Entitlements for All $partition_initialized_count partition(s) initialized successfully."
echo "Ops Members for all of $partition_admin_initialized_count partition(s) added successfully."
exit 0
fi
\ No newline at end of file
version: '3'
services:
entitlements-data-init:
build:
context: ../
dockerfile: ./entitlements_init/Dockerfile
image: msosdu.azurecr.io/entitlements-data-init:$VERSION
environment:
ADMIN_ID: $ADMIN_ID
SERVICE_DOMAIN: $SERVICE_DOMAIN
OSDU_HOST: $OSDU_HOST
PARTITIONS: $PARTITIONS # comma separated list of partitions
VERSION: $VERSION
\ No newline at end of file
FROM python:3.9.6-alpine3.14
RUN apk add --no-cache --upgrade bash
RUN apk add --no-cache --upgrade curl
WORKDIR /usr/src/app
COPY access_token/get_access_token.sh ./
COPY partition_init/data_init.sh ./
COPY partition_init/partition_init_api_payload.json ./
RUN chmod +x ./get_access_token.sh
RUN chmod +x ./data_init.sh
ENTRYPOINT [ "./data_init.sh" ]
\ No newline at end of file
#!/bin/bash
OSDU_URI=${OSDU_HOST}
if [[ ${OSDU_HOST} != "https://"* ]] || [[ ${OSDU_HOST} != "http://"* ]]; then
OSDU_URI="https://${OSDU_HOST}"
fi
echo "Trying to Fetch Access Token"
ACCESS_TOKEN=$(sh ./get_access_token.sh)
if [[ "$ACCESS_TOKEN" == "TOKEN_FETCH_FAILURE" ]]; then
echo "Failure fetching Access Token"
exit 1
fi
echo "Access Token fetched successfully."
IFS=',' read -r -a partitions_array <<< ${PARTITIONS}
partition_count=0
partition_initialized_count=0
for index in "${!partitions_array[@]}"
do
partition_count=$(expr $partition_count + 1)
echo "Intitializing Partition: ${partitions_array[index]}"
OSDU_PARTITION_INIT_URI=${OSDU_URI}/api/partition/v1/partitions/${partitions_array[index]}
echo "Partition Initialization Endpoint: ${OSDU_PARTITION_INIT_URI}"
i=0
while [[ $i -lt 3 ]]; do
i=$(expr $i + 1)
init_response=$(curl -s -w " Http_Status_Code:%{http_code} " \
-X POST \
-H 'Content-Type: application/json' \
-H "Authorization: Bearer $ACCESS_TOKEN" \
-H "data-partition-id: ${partitions_array[index]}" \
-d "@partition_init_api_payload.json" \
$OSDU_PARTITION_INIT_URI)
echo "Init Reponse: $init_response"
if [ -z "$init_response" -a "$init_response"==" " ]; then
echo "Initialization Failed, Empty Reponse. Iteration $i."
continue
fi
# Status code check. succeed only if 2xx
# quit for partition if 404 or 400.
# 401 or 403, then retry after getting access token.
# else sleep for 1min and retry
if [[ ${init_response} != *"Http_Status_Code:2"* ]];then
if [[ ${init_response} == *"Http_Status_Code:400"* ]] || [[ ${init_response} == *"Http_Status_Code:404"* ]];then
echo "Partition Init for partition ${partitions_array[index]} failed with response $init_response"
break
fi
echo "Sleeping for 1min."
sleep 1m
if [[ ${init_response} == *"Http_Status_Code:401"* ]] || [[ ${init_response} == *"Http_Status_Code:403"* ]];then
echo "Trying to Re-Fetch Access Token"
ACCESS_TOKEN=$(sh ./get_access_token.sh)
if [[ "$ACCESS_TOKEN" == "TOKEN_FETCH_FAILURE" ]]; then
echo "Failure re-fetching Access Token"
exit 1
fi
echo "Access Token re-fetched successfully."
fi
continue
else
echo "Partition ${partitions_array[index]} Initialized successfully."
partition_initialized_count=$(expr $partition_initialized_count + 1)
break
fi
done
done
if [ "$partition_count" -ne "$partition_initialized_count" ]; then
echo "$partition_initialized_count partition(s) of total $partition_count partition(s) initialized successfully."
exit 1
else
echo "All $partition_initialized_count partition(s) initialized successfully."
exit 0
fi
\ No newline at end of file
version: '3'
services:
partition-data-init:
build:
context: ../
dockerfile: ./partition_init/Dockerfile
image: msosdu.azurecr.io/partition-data-init:$VERSION
environment:
OSDU_HOST: $OSDU_HOST
PARTITIONS: $PARTITIONS # comma separated list of partitions
VERSION: $VERSION
\ No newline at end of file
{
"properties": {
"compliance-ruleset": {
"value": "shared"
},
"elastic-endpoint": {
"sensitive": true,
"value": "elastic-endpoint"
},
"elastic-username": {
"sensitive": true,
"value": "elastic-username"
},
"elastic-password": {
"sensitive": true,
"value": "elastic-password"
},
"cosmos-connection": {
"sensitive": true,
"value": "cosmos-connection"
},
"cosmos-endpoint": {
"sensitive": true,
"value": "cosmos-endpoint"
},
"cosmos-primary-key": {
"sensitive": true,
"value": "cosmos-primary-key"
},
"sb-connection": {
"sensitive": true,
"value": "sb-connection"
},
"sb-namespace": {
"sensitive": true,
"value": "sb-namespace"
},
"storage-account-key": {
"sensitive": true,
"value": "storage-key"
},
"storage-account-name": {
"sensitive": true,
"value": "storage"
},
"ingest-storage-account-name": {
"sensitive": true,
"value": "ingest-storage"
},
"ingest-storage-account-key": {
"sensitive": true,
"value": "ingest-storage-key"
},
"eventgrid-recordstopic": {
"sensitive": true,
"value": "eventgrid-recordstopic"
},
"eventgrid-recordstopic-accesskey": {
"sensitive": true,
"value": "eventgrid-recordstopic-accesskey"
},
"eventgrid-legaltagschangedtopic": {
"sensitive": true,
"value": "eventgrid-legaltagschangedtopic"
},
"eventgrid-legaltagschangedtopic-accesskey": {
"sensitive": true,
"value": "eventgrid-legaltagschangedtopic-accesskey"
},
"eventgrid-resourcegroup": {
"sensitive": true,
"value": "eventgrid-resourcegroup"
},
"encryption-key-identifier": {
"sensitive": true,
"value": "encryption-key-identifier"
},
"sdms-storage-account-name": {
"sensitive": true,
"value": "sdms-storage"
},
"sdms-storage-account-key": {
"sensitive": true,
"value": "sdms-storage-key"
},
"eventgrid-schemanotificationtopic": {
"sensitive": true,
"value": "eventgrid-schemachangedtopic"
},
"eventgrid-schemanotificationtopic-accesskey": {
"sensitive": true,
"value": "eventgrid-schemachangedtopic-accesskey"
},
"eventgrid-gsmtopic": {
"sensitive": true,
"value": "eventgrid-statuschangedtopic"
},
"eventgrid-gsmtopic-accesskey": {
"sensitive": true,
"value": "eventgrid-statuschangedtopic-accesskey"
},
"eventgrid-statuschangedtopic": {
"sensitive": true,
"value": "eventgrid-statuschangedtopic"
},
"eventgrid-statuschangedtopic-accesskey": {
"sensitive": true,
"value": "eventgrid-statuschangedtopic-accesskey"
},
"eventgrid-schemachangedtopic": {
"sensitive": true,
"value": "eventgrid-schemachangedtopic"
},
"eventgrid-schemachangedtopic-accesskey": {
"sensitive": true,
"value": "eventgrid-schemachangedtopic-accesskey"
}
}
}
......@@ -7,14 +7,14 @@ az login --service-principal -u $ARM_CLIENT_ID -p $ARM_CLIENT_SECRET --tenant $A
# in the container legal-service-azure-configuration.
ENV_VAULT=$(az keyvault list --resource-group $RESOURCE_GROUP_NAME --query [].name -otsv)
IFS=',' read -r -a partitions <<< ${PARTITIONS}
IFS=',' read -r -a partitions_array <<< ${PARTITIONS}
for index in "${!partitions[@]}"
for index in "${!partitions_array[@]}"
do
echo "Ingesting file for partition: $index. ${partitions[index]}"
echo "Ingesting file for partition: $index. ${partitions_array[index]}"
STORAGE_ACCOUNT_NAME=$(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/${partitions[index]}-storage --query value -otsv)
STORAGE_ACCOUNT_KEY=$(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/${partitions[index]}-storage-key --query value -otsv)
STORAGE_ACCOUNT_NAME=$(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/${partitions_array[index]}-storage --query value -otsv)
STORAGE_ACCOUNT_KEY=$(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/${partitions_array[index]}-storage-key --query value -otsv)
FILE_NAME=Legal_COO.json
if [ -z "$STORAGE_ACCOUNT_NAME" -a "$STORAGE_ACCOUNT_NAME"==" " ]; then
......@@ -59,7 +59,7 @@ do
fi
fi
echo "File ingested for partition: $index. ${partitions[index]}"
echo "File ingested for partition: $index. ${partitions_array[index]}"
done
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment