Commit ea2cfa06 authored by MANISH KUMAR's avatar MANISH KUMAR
Browse files

Merge branch 'users/arysingh/DataAgentsConfigMapFix' into 'master'

Add Config Map Modifications To Data Agents

See merge request !469
parents 58ecef52 e54b1192
Pipeline #62102 passed with stages
in 2 minutes and 16 seconds
FROM python:3.9.6-alpine3.14
RUN apk add --no-cache --upgrade bash
RUN apk add --no-cache --upgrade curl
RUN apk update
RUN apk add libc6-compat
RUN apk add --no-cache --upgrade ca-certificates bash curl wget gettext jq bind-tools \
&& wget -q https://storage.googleapis.com/kubernetes-release/release/v1.21.2/bin/linux/amd64/kubectl -O /usr/local/bin/kubectl \
&& chmod +x /usr/local/bin/kubectl \
&& chmod g+rwx /root \
&& mkdir /config \
&& chmod g+rwx /config
WORKDIR /usr/src/app
......
#!/bin/bash
currentStatus=""
currentMessage=""
az login --identity
ENV_AKS=$(az aks list --resource-group $RESOURCE_GROUP_NAME --query [].name -otsv)
az aks get-credentials --resource-group $RESOURCE_GROUP_NAME --name $ENV_AKS
kubectl config set-context $RESOURCE_GROUP_NAME --cluster $ENV_AKS
OSDU_URI=${OSDU_HOST}
if [[ ${OSDU_HOST} != "https://"* ]] || [[ ${OSDU_HOST} != "http://"* ]]; then
......@@ -11,140 +19,196 @@ ACCESS_TOKEN=$(sh ./get_access_token.sh)
if [[ "$ACCESS_TOKEN" == "TOKEN_FETCH_FAILURE" ]]; then
echo "Failure fetching Access Token"
exit 1
fi
echo "Access Token fetched successfully."
IFS=',' read -r -a partitions_array <<< ${PARTITIONS}
partition_count=0
partition_initialized_count=0
partition_admin_initialized_count=0
for index in "${!partitions_array[@]}"
do
partition_count=$(expr $partition_count + 1)
echo "Intitializing Entitlements for Partition: ${partitions_array[index]}"
else
echo "Access Token fetched successfully."
OSDU_ENTITLEMENTS_INIT_URI=${OSDU_URI}/api/entitlements/v2/tenant-provisioning
echo "Entitlements Partition Initialization Endpoint: ${OSDU_ENTITLEMENTS_INIT_URI}"
IFS=',' read -r -a partitions_array <<< ${PARTITIONS}
partition_count=0
partition_initialized_count=0
partition_admin_initialized_count=0
for index in "${!partitions_array[@]}"
do
partition_count=$(expr $partition_count + 1)
echo "Intitializing Entitlements for Partition: ${partitions_array[index]}"
OSDU_ENTITLEMENTS_INIT_URI=${OSDU_URI}/api/entitlements/v2/tenant-provisioning
echo "Entitlements Partition Initialization Endpoint: ${OSDU_ENTITLEMENTS_INIT_URI}"
i=0
partition_initialized=false
while [[ $i -lt 3 ]]; do
init_response=$(curl -s -w " Http_Status_Code:%{http_code} " \
-X POST \
-H 'Content-Type: application/json' \
-H "Authorization: Bearer $ACCESS_TOKEN" \
-H "data-partition-id: ${partitions_array[index]}" \
$OSDU_ENTITLEMENTS_INIT_URI)
echo "Init Reponse: $init_response"
if [ -z "$init_response" -a "$init_response"==" " ]; then
echo "Initialization Failed, Empty Reponse. Iteration $i."
continue
fi
# Status code check. succeed only if 2xx
# quit for partition if 404 or 400.
# 401 or 403, then retry after getting access token.
# else sleep for 1min and retry
if [[ ${init_response} != *"Http_Status_Code:2"* ]];then
if [[ ${init_response} == *"Http_Status_Code:400"* ]] || [[ ${init_response} == *"Http_Status_Code:404"* ]];then
currentStatus="failure"
currentMessage="${currentMessage}. Entitlements Init for partition ${partitions_array[index]} failed with response $init_response. "
echo "Entitlements Init for partition ${partitions_array[index]} failed with response $init_response"
break
fi
echo "Sleeping for 1min."
sleep 1m
if [[ ${init_response} == *"Http_Status_Code:401"* ]] || [[ ${init_response} == *"Http_Status_Code:403"* ]];then
echo "Trying to Re-Fetch Access Token"
ACCESS_TOKEN=$(sh ./get_access_token.sh)
if [[ "$ACCESS_TOKEN" == "TOKEN_FETCH_FAILURE" ]]; then
currentStatus="failure"
currentMessage="${currentMessage}. Failure re-fetching Access Token. "
echo "Failure re-fetching Access Token"
break
fi
echo "Access Token re-fetched successfully."
fi
continue
else
currentMessage="${currentMessage}. Entitlements for Partition ${partitions_array[index]} Initialized successfully. "
echo "Entitlements for Partition ${partitions_array[index]} Initialized successfully."
partition_initialized_count=$(expr $partition_initialized_count + 1)
partition_initialized=true
break
fi
i=0
partition_initialized=false
while [[ $i -lt 3 ]]; do
i=$(expr $i + 1)
i=$(expr $i + 1)
done
init_response=$(curl -s -w " Http_Status_Code:%{http_code} " \
-X POST \
-H 'Content-Type: application/json' \
-H "Authorization: Bearer $ACCESS_TOKEN" \
-H "data-partition-id: ${partitions_array[index]}" \
$OSDU_ENTITLEMENTS_INIT_URI)
echo "Init Reponse: $init_response"
if [[ $i -ge 3 ]]; then
currentStatus="failure"
currentMessage="${currentMessage}. Entitlements Init: Max Number of retries reached. "
fi
if [ -z "$init_response" -a "$init_response"==" " ]; then
echo "Initialization Failed, Empty Reponse. Iteration $i."
if [ "$partition_initialized" != true ] ; then
currentStatus="failure"
currentMessage="${currentMessage}. Skipping Adding Admin as Entitlements Init has failed. "
echo "Skipping Adding Admin as Entitlements Init has failed."
continue
fi
# Status code check. succeed only if 2xx
# quit for partition if 404 or 400.
# 401 or 403, then retry after getting access token.
# else sleep for 1min and retry
if [[ ${init_response} != *"Http_Status_Code:2"* ]];then
if [[ ${init_response} == *"Http_Status_Code:400"* ]] || [[ ${init_response} == *"Http_Status_Code:404"* ]];then
echo "Entitlements Init for partition ${partitions_array[index]} failed with response $init_response"
break
echo "Adding Admin User Entitlements for Partition: ${partitions_array[index]}"
OSDU_ENTITLEMENTS_ADD_OPS_URI=${OSDU_URI}/api/entitlements/v2/groups/users.datalake.ops@${partitions_array[index]}.$SERVICE_DOMAIN/members
echo "Entitlements Partition Add Ops Endpoint: ${OSDU_ENTITLEMENTS_ADD_OPS_URI}"
i=0
while [[ $i -lt 3 ]]; do
init_response=$(curl -s -w " Http_Status_Code:%{http_code} " \
-X POST \
-H 'Content-Type: application/json' \
-H "Authorization: Bearer $ACCESS_TOKEN" \
-H "data-partition-id: ${partitions_array[index]}" \
-d "{"email":"$ADMIN_ID", "role": "MEMBER"}" \
$OSDU_ENTITLEMENTS_ADD_OPS_URI)
echo "Init Reponse: $init_response"
if [ -z "$init_response" -a "$init_response"==" " ]; then
echo "Add Ops Member Failed, Empty Reponse. Iteration $i."
continue
fi
echo "Sleeping for 1min."
sleep 1m
if [[ ${init_response} == *"Http_Status_Code:401"* ]] || [[ ${init_response} == *"Http_Status_Code:403"* ]];then
echo "Trying to Re-Fetch Access Token"
ACCESS_TOKEN=$(sh ./get_access_token.sh)
if [[ "$ACCESS_TOKEN" == "TOKEN_FETCH_FAILURE" ]]; then
echo "Failure re-fetching Access Token"
exit 1
# Status code check. succeed only if 2xx
# quit for partition if 404 or 400.
# 401 or 403, then retry after getting access token.
# else sleep for 1min and retry
if [[ ${init_response} != *"Http_Status_Code:2"* ]];then
if [[ ${init_response} == *"Http_Status_Code:400"* ]] || [[ ${init_response} == *"Http_Status_Code:404"* ]];then
currentStatus="failure"
currentMessage="${currentMessage}. Add Ops Member for partition ${partitions_array[index]} failed with response $init_response. "
echo "Add Ops Member for partition ${partitions_array[index]} failed with response $init_response"
break
fi
echo "Sleeping for 1min."
sleep 1m
if [[ ${init_response} == *"Http_Status_Code:401"* ]] || [[ ${init_response} == *"Http_Status_Code:403"* ]];then
echo "Trying to Re-Fetch Access Token"
ACCESS_TOKEN=$(sh ./get_access_token.sh)
if [[ "$ACCESS_TOKEN" == "TOKEN_FETCH_FAILURE" ]]; then
currentStatus="failure"
currentMessage="${currentMessage}. Failure re-fetching Access Token. "
echo "Failure re-fetching Access Token"
break
fi
echo "Access Token re-fetched successfully."
fi
echo "Access Token re-fetched successfully."
continue
else
currentMessage="${currentMessage}. Ops Member for Partition ${partitions_array[index]} Initialized successfully. "
echo "Ops Member for Partition ${partitions_array[index]} Initialized successfully."
partition_admin_initialized_count=$(expr $partition_admin_initialized_count + 1)
break
fi
continue
else
echo "Entitlements for Partition ${partitions_array[index]} Initialized successfully."
partition_initialized_count=$(expr $partition_initialized_count + 1)
partition_initialized=true
i=$(expr $i + 1)
done
break
if [[ $i -ge 3 ]]; then
currentStatus="failure"
currentMessage="${currentMessage}. Adding Admin User: Max Number of retries reached. "
fi
done
if [ "$partition_initialized" != true ] ; then
continue
if [ "$partition_count" -ne "$partition_initialized_count" ] || [ "$partition_admin_initialized_count" -ne "$partition_initialized_count" ]; then
currentStatus="failure"
currentMessage="${currentMessage}. Entitlements for $partition_initialized_count partition(s) of total $partition_count partition(s) initialized successfully. "
currentMessage="${currentMessage}. $partition_admin_initialized_count partition(s) of total $partition_initialized_count initialized with Ops Member. "
echo "Entitlements for $partition_initialized_count partition(s) of total $partition_count partition(s) initialized successfully."
echo "$partition_admin_initialized_count partition(s) of total $partition_initialized_count initialized with Ops Member."
else
currentMessage="${currentMessage}. Entitlements for All $partition_initialized_count partition(s) initialized successfully. "
currentMessage="${currentMessage}. Ops Members for all of $partition_admin_initialized_count partition(s) added successfully. "
echo "Entitlements for All $partition_initialized_count partition(s) initialized successfully."
echo "Ops Members for all of $partition_admin_initialized_count partition(s) added successfully."
fi
fi
OSDU_ENTITLEMENTS_ADD_OPS_URI=${OSDU_URI}/api/entitlements/v2/groups/users.datalake.ops@${partitions_array[index]}.$SERVICE_DOMAIN/members
echo "Entitlements Partition Add Ops Endpoint: ${OSDU_ENTITLEMENTS_ADD_OPS_URI}"
i=0
while [[ $i -lt 3 ]]; do
i=$(expr $i + 1)
if [ -z "$currentStatus" -a "$currentStatus"==" " ]; then
currentStatus="success"
fi
echo "Current Status: ${currentStatus}"
echo "Current Message: ${currentMessage}"
init_response=$(curl -s -w " Http_Status_Code:%{http_code} " \
-X POST \
-H 'Content-Type: application/json' \
-H "Authorization: Bearer $ACCESS_TOKEN" \
-H "data-partition-id: ${partitions_array[index]}" \
-d "{"email":"$ADMIN_ID", "role": "MEMBER"}" \
$OSDU_ENTITLEMENTS_ADD_OPS_URI)
echo "Init Reponse: $init_response"
if [ ! -z "$CONFIG_MAP_NAME" -a "$CONFIG_MAP_NAME" != " " ]; then
Status=$(kubectl get configmap $CONFIG_MAP_NAME -o jsonpath='{.data.status}')
Message=$(kubectl get configmap $CONFIG_MAP_NAME -o jsonpath='{.data.message}')
if [ -z "$init_response" -a "$init_response"==" " ]; then
echo "Add Ops Member Failed, Empty Reponse. Iteration $i."
continue
if [[ ${Status} == *"success"* ]]; then # If status is already failed, do not over-write in any case.
Status="${currentStatus}"
fi
Message="${Message}. Entitlements Data Initialization: ${currentMessage}. "
# Status code check. succeed only if 2xx
# quit for partition if 404 or 400.
# 401 or 403, then retry after getting access token.
# else sleep for 1min and retry
if [[ ${init_response} != *"Http_Status_Code:2"* ]];then
if [[ ${init_response} == *"Http_Status_Code:400"* ]] || [[ ${init_response} == *"Http_Status_Code:404"* ]];then
echo "Add Ops Member for partition ${partitions_array[index]} failed with response $init_response"
break
fi
echo "Sleeping for 1min."
sleep 1m
if [[ ${init_response} == *"Http_Status_Code:401"* ]] || [[ ${init_response} == *"Http_Status_Code:403"* ]];then
echo "Trying to Re-Fetch Access Token"
ACCESS_TOKEN=$(sh ./get_access_token.sh)
if [[ "$ACCESS_TOKEN" == "TOKEN_FETCH_FAILURE" ]]; then
echo "Failure re-fetching Access Token"
exit 1
fi
echo "Access Token re-fetched successfully."
fi
continue
else
echo "Ops Member for Partition ${partitions_array[index]} Initialized successfully."
partition_admin_initialized_count=$(expr $partition_admin_initialized_count + 1)
break
fi
done
done
## Update ConfigMap
kubectl create configmap $CONFIG_MAP_NAME --from-literal=status="$Status" --from-literal=message="$Message" -o yaml --dry-run=client | kubectl replace -f -
fi
if [ "$partition_count" -ne "$partition_initialized_count" ] || [ "$partition_admin_initialized_count" -ne "$partition_initialized_count" ]; then
echo "Entitlements for $partition_initialized_count partition(s) of total $partition_count partition(s) initialized successfully."
echo "$partition_admin_initialized_count partition(s) of total $partition_initialized_count initialized with Ops Member."
exit 1
if [[ ${currentStatus} == "success" ]]; then
exit 0
else
echo "Entitlements for All $partition_initialized_count partition(s) initialized successfully."
echo "Ops Members for all of $partition_admin_initialized_count partition(s) added successfully."
exit 0
exit 1
fi
\ No newline at end of file
......@@ -9,5 +9,7 @@ services:
ADMIN_ID: $ADMIN_ID
SERVICE_DOMAIN: $SERVICE_DOMAIN
OSDU_HOST: $OSDU_HOST
RESOURCE_GROUP_NAME: $RESOURCE_GROUP_NAME
PARTITIONS: $PARTITIONS # comma separated list of partitions
CONFIG_MAP_NAME: $CONFIG_MAP_NAME
VERSION: $VERSION
\ No newline at end of file
FROM python:3.9.6-alpine3.14
RUN apk add --no-cache --upgrade bash
RUN apk add --no-cache --upgrade curl
RUN apk update
RUN apk add libc6-compat
RUN apk add --no-cache --upgrade ca-certificates bash curl wget gettext jq bind-tools \
&& wget -q https://storage.googleapis.com/kubernetes-release/release/v1.21.2/bin/linux/amd64/kubectl -O /usr/local/bin/kubectl \
&& chmod +x /usr/local/bin/kubectl \
&& chmod g+rwx /root \
&& mkdir /config \
&& chmod g+rwx /config
WORKDIR /usr/src/app
......
#!/bin/bash
currentStatus=""
currentMessage=""
az login --identity
ENV_AKS=$(az aks list --resource-group $RESOURCE_GROUP_NAME --query [].name -otsv)
az aks get-credentials --resource-group $RESOURCE_GROUP_NAME --name $ENV_AKS
kubectl config set-context $RESOURCE_GROUP_NAME --cluster $ENV_AKS
OSDU_URI=${OSDU_HOST}
if [[ ${OSDU_HOST} != "https://"* ]] || [[ ${OSDU_HOST} != "http://"* ]]; then
......@@ -9,80 +17,118 @@ fi
echo "Trying to Fetch Access Token"
ACCESS_TOKEN=$(sh ./get_access_token.sh)
if [[ "$ACCESS_TOKEN" == "TOKEN_FETCH_FAILURE" ]]; then
echo "Failure fetching Access Token"
exit 1
fi
echo "Access Token fetched successfully."
IFS=',' read -r -a partitions_array <<< ${PARTITIONS}
partition_count=0
partition_initialized_count=0
for index in "${!partitions_array[@]}"
do
partition_count=$(expr $partition_count + 1)
echo "Intitializing Partition: ${partitions_array[index]}"
OSDU_PARTITION_INIT_URI=${OSDU_URI}/api/partition/v1/partitions/${partitions_array[index]}
echo "Partition Initialization Endpoint: ${OSDU_PARTITION_INIT_URI}"
i=0
while [[ $i -lt 3 ]]; do
i=$(expr $i + 1)
currentStatus="failure"
currentMessage="${currentMessage}. Failure fetching Access Token. "
else
echo "Access Token fetched successfully."
init_response=$(curl -s -w " Http_Status_Code:%{http_code} " \
-X POST \
-H 'Content-Type: application/json' \
-H "Authorization: Bearer $ACCESS_TOKEN" \
-H "data-partition-id: ${partitions_array[index]}" \
-d "@partition_init_api_payload.json" \
$OSDU_PARTITION_INIT_URI)
IFS=',' read -r -a partitions_array <<< ${PARTITIONS}
partition_count=0
partition_initialized_count=0
for index in "${!partitions_array[@]}"
do
partition_count=$(expr $partition_count + 1)
echo "Intitializing Partition: ${partitions_array[index]}"
OSDU_PARTITION_INIT_URI=${OSDU_URI}/api/partition/v1/partitions/${partitions_array[index]}
echo "Partition Initialization Endpoint: ${OSDU_PARTITION_INIT_URI}"
i=0
while [[ $i -lt 3 ]]; do
init_response=$(curl -s -w " Http_Status_Code:%{http_code} " \
-X POST \
-H 'Content-Type: application/json' \
-H "Authorization: Bearer $ACCESS_TOKEN" \
-H "data-partition-id: ${partitions_array[index]}" \
-d "@partition_init_api_payload.json" \
$OSDU_PARTITION_INIT_URI)
echo "Init Reponse: $init_response"
if [ -z "$init_response" -a "$init_response"==" " ]; then
echo "Initialization Failed, Empty Reponse. Iteration $i."
continue
fi
# Status code check. succeed only if 2xx
# quit for partition if 404 or 400.
# 401 or 403, then retry after getting access token.
# else sleep for 1min and retry
if [[ ${init_response} != *"Http_Status_Code:2"* ]];then
if [[ ${init_response} == *"Http_Status_Code:400"* ]] || [[ ${init_response} == *"Http_Status_Code:404"* ]];then
currentStatus="failure"
currentMessage="${currentMessage}. Partition Init for partition ${partitions_array[index]} failed with response $init_response. "
echo "Partition Init for partition ${partitions_array[index]} failed with response $init_response"
break
fi
echo "Sleeping for 1min."
sleep 1m
if [[ ${init_response} == *"Http_Status_Code:401"* ]] || [[ ${init_response} == *"Http_Status_Code:403"* ]];then
echo "Trying to Re-Fetch Access Token"
ACCESS_TOKEN=$(sh ./get_access_token.sh)
if [[ "$ACCESS_TOKEN" == "TOKEN_FETCH_FAILURE" ]]; then
currentStatus="failure"
currentMessage="${currentMessage}. Failure re-fetching Access Token. "
echo "Failure re-fetching Access Token"
break
fi
echo "Access Token re-fetched successfully."
fi
continue
else
currentMessage="${currentMessage}. Partition ${partitions_array[index]} Initialized successfully. "
echo "Partition ${partitions_array[index]} Initialized successfully."
partition_initialized_count=$(expr $partition_initialized_count + 1)
echo "Init Reponse: $init_response"
if [ -z "$init_response" -a "$init_response"==" " ]; then
echo "Initialization Failed, Empty Reponse. Iteration $i."
continue
fi
# Status code check. succeed only if 2xx
# quit for partition if 404 or 400.
# 401 or 403, then retry after getting access token.
# else sleep for 1min and retry
if [[ ${init_response} != *"Http_Status_Code:2"* ]];then
if [[ ${init_response} == *"Http_Status_Code:400"* ]] || [[ ${init_response} == *"Http_Status_Code:404"* ]];then
echo "Partition Init for partition ${partitions_array[index]} failed with response $init_response"
break
fi
i=$(expr $i + 1)
done
echo "Sleeping for 1min."
sleep 1m
if [[ $i -ge 3 ]]; then
currentStatus="failure"
currentMessage="${currentMessage}. Max Number of retries reached. "
fi
done
if [ "$partition_count" -ne "$partition_initialized_count" ]; then
currentStatus="failure"
currentMessage="${currentMessage}. $partition_initialized_count partition(s) of total $partition_count partition(s) initialized successfully. "
echo "$partition_initialized_count partition(s) of total $partition_count partition(s) initialized successfully."
else
currentMessage="${currentMessage}. All $partition_initialized_count partition(s) initialized successfully. "
echo "All $partition_initialized_count partition(s) initialized successfully."
fi
fi
if [[ ${init_response} == *"Http_Status_Code:401"* ]] || [[ ${init_response} == *"Http_Status_Code:403"* ]];then
echo "Trying to Re-Fetch Access Token"
ACCESS_TOKEN=$(sh ./get_access_token.sh)
if [[ "$ACCESS_TOKEN" == "TOKEN_FETCH_FAILURE" ]]; then
echo "Failure re-fetching Access Token"
exit 1
fi
echo "Access Token re-fetched successfully."
fi
if [ -z "$currentStatus" -a "$currentStatus"==" " ]; then
currentStatus="success"
fi
echo "Current Status: ${currentStatus}"
echo "Current Message: ${currentMessage}"
continue
else
echo "Partition ${partitions_array[index]} Initialized successfully."
partition_initialized_count=$(expr $partition_initialized_count + 1)
if [ ! -z "$CONFIG_MAP_NAME" -a "$CONFIG_MAP_NAME" != " " ]; then
Status=$(kubectl get configmap $CONFIG_MAP_NAME -o jsonpath='{.data.status}')
Message=$(kubectl get configmap $CONFIG_MAP_NAME -o jsonpath='{.data.message}')
break
if [[ ${Status} == *"success"* ]]; then # If status is already failed, do not over-write in any case.
Status="${currentStatus}"
fi
done
done
Message="${Message}. Partition Data Initialization: ${currentMessage}. "
## Update ConfigMap
kubectl create configmap $CONFIG_MAP_NAME --from-literal=status="$Status" --from-literal=message="$Message" -o yaml --dry-run=client | kubectl replace -f -
fi
if [ "$partition_count" -ne "$partition_initialized_count" ]; then
echo "$partition_initialized_count partition(s) of total $partition_count partition(s) initialized successfully."
exit 1
if [[ ${currentStatus} == "success" ]]; then
exit 0
else
echo "All $partition_initialized_count partition(s) initialized successfully."
exit 0
exit 1
fi
\ No newline at end of file
......@@ -7,5 +7,7 @@ services:
image: msosdu.azurecr.io/partition-data-init:$VERSION
environment:
OSDU_HOST: $OSDU_HOST
RESOURCE_GROUP_NAME: $RESOURCE_GROUP_NAME
PARTITIONS: $PARTITIONS # comma separated list of partitions
CONFIG_MAP_NAME: $CONFIG_MAP_NAME
VERSION: $VERSION
\ No newline at end of file