Commit 9b22c55e authored by Vivek Ojha's avatar Vivek Ojha
Browse files

Merge branch 'users/arysingh/testinghotfix' into 'master'

Fixes for Data Seeding & Data Init Agents

See merge request !477
parents 0082f0c0 ce6c0247
Pipeline #63388 passed with stages
in 2 minutes and 5 seconds
FROM python:3.9.6-alpine3.14
FROM mcr.microsoft.com/azure-cli:2.15.1
RUN apk update
RUN apk add ca-certificates wget && update-ca-certificates
RUN apk add libc6-compat
RUN apk add --no-cache --upgrade ca-certificates bash curl wget gettext jq bind-tools \
RUN apk add ca-certificates bash curl wget gettext jq bind-tools \
&& wget -q https://storage.googleapis.com/kubernetes-release/release/v1.21.2/bin/linux/amd64/kubectl -O /usr/local/bin/kubectl \
&& chmod +x /usr/local/bin/kubectl \
&& chmod g+rwx /root \
&& mkdir /config \
&& chmod g+rwx /config
ENV PYTHONUNBUFFERED=1
RUN apk add --update --no-cache python3 && ln -sf python3 /usr/bin/python
RUN python3 -m ensurepip
RUN pip3 install --no-cache --upgrade pip setuptools
WORKDIR /usr/src/app
COPY access_token/get_access_token.sh ./
......
......@@ -3,11 +3,6 @@
currentStatus=""
currentMessage=""
az login --identity
ENV_AKS=$(az aks list --resource-group $RESOURCE_GROUP_NAME --query [].name -otsv)
az aks get-credentials --resource-group $RESOURCE_GROUP_NAME --name $ENV_AKS
kubectl config set-context $RESOURCE_GROUP_NAME --cluster $ENV_AKS
OSDU_URI=${OSDU_HOST}
if [[ ${OSDU_HOST} != "https://"* ]] || [[ ${OSDU_HOST} != "http://"* ]]; then
......@@ -26,6 +21,7 @@ else
partition_count=0
partition_initialized_count=0
partition_user_initialized_count=0
partition_admin_initialized_count=0
for index in "${!partitions_array[@]}"
......@@ -39,7 +35,8 @@ else
i=0
partition_initialized=false
while [[ $i -lt 3 ]]; do
i=$(expr $i + 1)
init_response=$(curl -s -w " Http_Status_Code:%{http_code} " \
-X POST \
-H 'Content-Type: application/json' \
......@@ -90,8 +87,6 @@ else
break
fi
i=$(expr $i + 1)
done
if [[ $i -ge 3 ]]; then
......@@ -101,11 +96,81 @@ else
if [ "$partition_initialized" != true ] ; then
currentStatus="failure"
currentMessage="${currentMessage}. Skipping Adding Admin as Entitlements Init has failed. "
echo "Skipping Adding Admin as Entitlements Init has failed."
currentMessage="${currentMessage}. Skipping Adding User as an Admin as Entitlements Init has failed. "
echo "Skipping Adding User as an Admin as Entitlements Init has failed."
continue
fi
echo "Creating User Entitlements for Partition: ${partitions_array[index]}"
OSDU_ENTITLEMENTS_CREATE_USER_URI=${OSDU_URI}/api/entitlements/v2/groups/users@${partitions_array[index]}.$SERVICE_DOMAIN/members
echo "Entitlements Partition Create User Endpoint: ${OSDU_ENTITLEMENTS_CREATE_USER_URI}"
i=0
while [[ $i -lt 3 ]]; do
i=$(expr $i + 1)
init_response=$(curl -s -w " Http_Status_Code:%{http_code} " \
-X POST \
-H 'Content-Type: application/json' \
-H "Authorization: Bearer $ACCESS_TOKEN" \
-H "data-partition-id: ${partitions_array[index]}" \
-d "{\"email\":\"$ADMIN_ID\", \"role\": \"MEMBER\"}" \
$OSDU_ENTITLEMENTS_CREATE_USER_URI)
echo "Init Reponse: $init_response"
if [ -z "$init_response" -a "$init_response"==" " ]; then
echo "Create User Failed, Empty Reponse. Iteration $i."
continue
fi
# Status code check. succeed only if 2xx
# quit for partition if 404 or 400.
# 401 or 403, then retry after getting access token.
# else sleep for 1min and retry
if [[ ${init_response} != *"Http_Status_Code:2"* ]] && [[ ${init_response} != *"Http_Status_Code:409"* ]];then
if [[ ${init_response} == *"Http_Status_Code:400"* ]] || [[ ${init_response} == *"Http_Status_Code:404"* ]];then
currentStatus="failure"
currentMessage="${currentMessage}. Create User for partition ${partitions_array[index]} failed with response $init_response. "
echo "Create User for partition ${partitions_array[index]} failed with response $init_response"
break
fi
echo "Sleeping for 1min."
sleep 1m
if [[ ${init_response} == *"Http_Status_Code:401"* ]] || [[ ${init_response} == *"Http_Status_Code:403"* ]];then
echo "Trying to Re-Fetch Access Token"
ACCESS_TOKEN=$(sh ./get_access_token.sh)
if [[ "$ACCESS_TOKEN" == "TOKEN_FETCH_FAILURE" ]]; then
currentStatus="failure"
currentMessage="${currentMessage}. Failure re-fetching Access Token. "
echo "Failure re-fetching Access Token"
break
fi
echo "Access Token re-fetched successfully."
fi
continue
else
if [[ ${init_response} == *"Http_Status_Code:409"* ]];then
currentMessage="${currentMessage}. HTTP Status Code: 409 -> Admin User ${partitions_array[index]} Already Exists. "
echo "HTTP Status Code: 409 -> User ${partitions_array[index]} Already Exists."
fi
currentMessage="${currentMessage}. User for Partition ${partitions_array[index]} Initialized successfully. "
echo "User for Partition ${partitions_array[index]} Initialized successfully."
partition_user_initialized_count=$(expr $partition_user_initialized_count + 1)
break
fi
done
if [[ $i -ge 3 ]]; then
currentStatus="failure"
currentMessage="${currentMessage}. Creating User: Max Number of retries reached. "
fi
echo "Adding Admin User Entitlements for Partition: ${partitions_array[index]}"
OSDU_ENTITLEMENTS_ADD_OPS_URI=${OSDU_URI}/api/entitlements/v2/groups/users.datalake.ops@${partitions_array[index]}.$SERVICE_DOMAIN/members
......@@ -113,13 +178,14 @@ else
i=0
while [[ $i -lt 3 ]]; do
i=$(expr $i + 1)
init_response=$(curl -s -w " Http_Status_Code:%{http_code} " \
-X POST \
-H 'Content-Type: application/json' \
-H "Authorization: Bearer $ACCESS_TOKEN" \
-H "data-partition-id: ${partitions_array[index]}" \
-d "{"email":"$ADMIN_ID", "role": "MEMBER"}" \
-d "{\"email\":\"$ADMIN_ID\", \"role\": \"MEMBER\"}" \
$OSDU_ENTITLEMENTS_ADD_OPS_URI)
echo "Init Reponse: $init_response"
......@@ -133,7 +199,7 @@ else
# quit for partition if 404 or 400.
# 401 or 403, then retry after getting access token.
# else sleep for 1min and retry
if [[ ${init_response} != *"Http_Status_Code:2"* ]];then
if [[ ${init_response} != *"Http_Status_Code:2"* ]] && [[ ${init_response} != *"Http_Status_Code:409"* ]];then
if [[ ${init_response} == *"Http_Status_Code:400"* ]] || [[ ${init_response} == *"Http_Status_Code:404"* ]];then
currentStatus="failure"
currentMessage="${currentMessage}. Add Ops Member for partition ${partitions_array[index]} failed with response $init_response. "
......@@ -158,14 +224,16 @@ else
continue
else
if [[ ${init_response} == *"Http_Status_Code:409"* ]];then
currentMessage="${currentMessage}. HTTP Status Code: 409 -> Admin User ${partitions_array[index]} Already Exists. "
echo "HTTP Status Code: 409 -> Partition ${partitions_array[index]} Already Exists."
fi
currentMessage="${currentMessage}. Ops Member for Partition ${partitions_array[index]} Initialized successfully. "
echo "Ops Member for Partition ${partitions_array[index]} Initialized successfully."
partition_admin_initialized_count=$(expr $partition_admin_initialized_count + 1)
break
fi
i=$(expr $i + 1)
done
if [[ $i -ge 3 ]]; then
......@@ -174,16 +242,20 @@ else
fi
done
if [ "$partition_count" -ne "$partition_initialized_count" ] || [ "$partition_admin_initialized_count" -ne "$partition_initialized_count" ]; then
if [ "$partition_count" -ne "$partition_initialized_count" ] || [ "$partition_user_initialized_count" -ne "$partition_initialized_count" ] || [ "$partition_admin_initialized_count" -ne "$partition_initialized_count" ]; then
currentStatus="failure"
currentMessage="${currentMessage}. Entitlements for $partition_initialized_count partition(s) of total $partition_count partition(s) initialized successfully. "
currentMessage="${currentMessage}. $partition_user_initialized_count partition(s) of total $partition_initialized_count initialized with User. "
currentMessage="${currentMessage}. $partition_admin_initialized_count partition(s) of total $partition_initialized_count initialized with Ops Member. "
echo "Entitlements for $partition_initialized_count partition(s) of total $partition_count partition(s) initialized successfully."
echo "$partition_user_initialized_count partition(s) of total $partition_initialized_count initialized with User."
echo "$partition_admin_initialized_count partition(s) of total $partition_initialized_count initialized with Ops Member."
else
currentMessage="${currentMessage}. Entitlements for All $partition_initialized_count partition(s) initialized successfully. "
currentMessage="${currentMessage}. User for all of $partition_user_initialized_count partition(s) added successfully. "
currentMessage="${currentMessage}. Ops Members for all of $partition_admin_initialized_count partition(s) added successfully. "
echo "Entitlements for All $partition_initialized_count partition(s) initialized successfully."
echo "User for all of $partition_user_initialized_count partition(s) added successfully."
echo "Ops Members for all of $partition_admin_initialized_count partition(s) added successfully."
fi
fi
......@@ -195,16 +267,21 @@ echo "Current Status: ${currentStatus}"
echo "Current Message: ${currentMessage}"
if [ ! -z "$CONFIG_MAP_NAME" -a "$CONFIG_MAP_NAME" != " " ]; then
Status=$(kubectl get configmap $CONFIG_MAP_NAME -o jsonpath='{.data.status}')
Message=$(kubectl get configmap $CONFIG_MAP_NAME -o jsonpath='{.data.message}')
az login --identity --username $OSDU_IDENTITY_ID
ENV_AKS=$(az aks list --resource-group $RESOURCE_GROUP_NAME --query [].name -otsv)
az aks get-credentials --resource-group $RESOURCE_GROUP_NAME --name $ENV_AKS
kubectl config set-context $RESOURCE_GROUP_NAME --cluster $ENV_AKS
if [[ ${Status} == *"success"* ]]; then # If status is already failed, do not over-write in any case.
Status="${currentStatus}"
fi
Message="${Message}. Entitlements Data Initialization: ${currentMessage}. "
Status=$(kubectl get configmap $CONFIG_MAP_NAME -o jsonpath='{.data.status}')
Message=$(kubectl get configmap $CONFIG_MAP_NAME -o jsonpath='{.data.message}')
Message="${Message}Entitlements Init Message: ${currentMessage}. "
## Update ConfigMap
kubectl create configmap $CONFIG_MAP_NAME --from-literal=status="$Status" --from-literal=message="$Message" -o yaml --dry-run=client | kubectl replace -f -
## Update ConfigMap
kubectl create configmap $CONFIG_MAP_NAME \
--from-literal=status="$currentStatus" \
--from-literal=message="$Message" \
-o yaml --dry-run=client | kubectl replace -f -
fi
if [[ ${currentStatus} == "success" ]]; then
......
......@@ -9,6 +9,7 @@ services:
ADMIN_ID: $ADMIN_ID
SERVICE_DOMAIN: $SERVICE_DOMAIN
OSDU_HOST: $OSDU_HOST
OSDU_IDENTITY_ID: $OSDU_IDENTITY_ID
RESOURCE_GROUP_NAME: $RESOURCE_GROUP_NAME
PARTITIONS: $PARTITIONS # comma separated list of partitions
CONFIG_MAP_NAME: $CONFIG_MAP_NAME
......
FROM python:3.9.6-alpine3.14
FROM mcr.microsoft.com/azure-cli:2.15.1
RUN apk update
RUN apk add ca-certificates wget && update-ca-certificates
RUN apk add libc6-compat
RUN apk add --no-cache --upgrade ca-certificates bash curl wget gettext jq bind-tools \
RUN apk add ca-certificates bash curl wget gettext jq bind-tools \
&& wget -q https://storage.googleapis.com/kubernetes-release/release/v1.21.2/bin/linux/amd64/kubectl -O /usr/local/bin/kubectl \
&& chmod +x /usr/local/bin/kubectl \
&& chmod g+rwx /root \
&& mkdir /config \
&& chmod g+rwx /config
ENV PYTHONUNBUFFERED=1
RUN apk add --update --no-cache python3 && ln -sf python3 /usr/bin/python
RUN python3 -m ensurepip
RUN pip3 install --no-cache --upgrade pip setuptools
WORKDIR /usr/src/app
COPY access_token/get_access_token.sh ./
......
......@@ -3,11 +3,6 @@
currentStatus=""
currentMessage=""
az login --identity
ENV_AKS=$(az aks list --resource-group $RESOURCE_GROUP_NAME --query [].name -otsv)
az aks get-credentials --resource-group $RESOURCE_GROUP_NAME --name $ENV_AKS
kubectl config set-context $RESOURCE_GROUP_NAME --cluster $ENV_AKS
OSDU_URI=${OSDU_HOST}
if [[ ${OSDU_HOST} != "https://"* ]] || [[ ${OSDU_HOST} != "http://"* ]]; then
......@@ -37,6 +32,7 @@ else
i=0
while [[ $i -lt 3 ]]; do
i=$(expr $i + 1)
init_response=$(curl -s -w " Http_Status_Code:%{http_code} " \
-X POST \
......@@ -53,11 +49,11 @@ else
continue
fi
# Status code check. succeed only if 2xx
# Status code check. succeed only if 2xx, or 409
# quit for partition if 404 or 400.
# 401 or 403, then retry after getting access token.
# else sleep for 1min and retry
if [[ ${init_response} != *"Http_Status_Code:2"* ]];then
if [[ ${init_response} != *"Http_Status_Code:2"* ]] && [[ ${init_response} != *"Http_Status_Code:409"* ]];then
if [[ ${init_response} == *"Http_Status_Code:400"* ]] || [[ ${init_response} == *"Http_Status_Code:404"* ]];then
currentStatus="failure"
currentMessage="${currentMessage}. Partition Init for partition ${partitions_array[index]} failed with response $init_response. "
......@@ -82,14 +78,16 @@ else
continue
else
if [[ ${init_response} == *"Http_Status_Code:409"* ]];then
currentMessage="${currentMessage}. HTTP Status Code: 409 -> Partition ${partitions_array[index]} Already Exists. "
echo "HTTP Status Code: 409 -> Partition ${partitions_array[index]} Already Exists."
fi
currentMessage="${currentMessage}. Partition ${partitions_array[index]} Initialized successfully. "
echo "Partition ${partitions_array[index]} Initialized successfully."
partition_initialized_count=$(expr $partition_initialized_count + 1)
break
fi
i=$(expr $i + 1)
done
if [[ $i -ge 3 ]]; then
......@@ -115,16 +113,21 @@ echo "Current Status: ${currentStatus}"
echo "Current Message: ${currentMessage}"
if [ ! -z "$CONFIG_MAP_NAME" -a "$CONFIG_MAP_NAME" != " " ]; then
Status=$(kubectl get configmap $CONFIG_MAP_NAME -o jsonpath='{.data.status}')
Message=$(kubectl get configmap $CONFIG_MAP_NAME -o jsonpath='{.data.message}')
az login --identity --username $OSDU_IDENTITY_ID
ENV_AKS=$(az aks list --resource-group $RESOURCE_GROUP_NAME --query [].name -otsv)
az aks get-credentials --resource-group $RESOURCE_GROUP_NAME --name $ENV_AKS
kubectl config set-context $RESOURCE_GROUP_NAME --cluster $ENV_AKS
if [[ ${Status} == *"success"* ]]; then # If status is already failed, do not over-write in any case.
Status="${currentStatus}"
fi
Message="${Message}. Partition Data Initialization: ${currentMessage}. "
Status=$(kubectl get configmap $CONFIG_MAP_NAME -o jsonpath='{.data.status}')
Message=$(kubectl get configmap $CONFIG_MAP_NAME -o jsonpath='{.data.message}')
Message="${Message}Partition Init Message: ${currentMessage}. "
## Update ConfigMap
kubectl create configmap $CONFIG_MAP_NAME --from-literal=status="$Status" --from-literal=message="$Message" -o yaml --dry-run=client | kubectl replace -f -
## Update ConfigMap
kubectl create configmap $CONFIG_MAP_NAME \
--from-literal=status="$currentStatus" \
--from-literal=message="$Message" \
-o yaml --dry-run=client | kubectl replace -f -
fi
if [[ ${currentStatus} == "success" ]]; then
......
......@@ -7,6 +7,7 @@ services:
image: msosdu.azurecr.io/partition-data-init:$VERSION
environment:
OSDU_HOST: $OSDU_HOST
OSDU_IDENTITY_ID: $OSDU_IDENTITY_ID
RESOURCE_GROUP_NAME: $RESOURCE_GROUP_NAME
PARTITIONS: $PARTITIONS # comma separated list of partitions
CONFIG_MAP_NAME: $CONFIG_MAP_NAME
......
......@@ -11,6 +11,7 @@ services:
ARM_CLIENT_ID: $ARM_CLIENT_ID
ADMIN_ID: $ADMIN_ID
ARM_CLIENT_SECRET: $ARM_CLIENT_SECRET
OSDU_IDENTITY_ID: $OSDU_IDENTITY_ID
PARTITIONS: $PARTITIONS # comma separated list of partitions
RESOURCE_GROUP_NAME: $RESOURCE_GROUP_NAME
CONFIG_MAP_NAME: $CONFIG_MAP_NAME
......
......@@ -4,13 +4,9 @@ currentStatus=""
currentMessage=""
# This logs the Azure CLI in using the configured service principal.
az login --service-principal -u $ARM_CLIENT_ID -p $ARM_CLIENT_SECRET --tenant $ARM_TENANT_ID
az account set -s $ARM_SUBSCRIPTION_ID
# Merge AKS context with current k8s cluster
ENV_AKS=$(az aks list --resource-group $RESOURCE_GROUP_NAME --query [].name -otsv)
az aks get-credentials --resource-group $RESOURCE_GROUP_NAME --name $ENV_AKS
kubectl config set-context $RESOURCE_GROUP_NAME --cluster $ENV_AKS
# az login --service-principal -u $ARM_CLIENT_ID -p $ARM_CLIENT_SECRET --tenant $ARM_TENANT_ID
# az account set -s $ARM_SUBSCRIPTION_ID
az login --identity --username $OSDU_IDENTITY_ID
# The Legal_COO.json file needs to be loaded into the Data Partition Storage Account,
# in the container legal-service-azure-configuration.
......@@ -103,16 +99,21 @@ echo "Current Status: ${currentStatus}"
echo "Current Message: ${currentMessage}"
if [ ! -z "$CONFIG_MAP_NAME" -a "$CONFIG_MAP_NAME" != " " ]; then
Status=$(kubectl get configmap $CONFIG_MAP_NAME -o jsonpath='{.data.status}')
Message=$(kubectl get configmap $CONFIG_MAP_NAME -o jsonpath='{.data.message}')
az login --identity --username $OSDU_IDENTITY_ID
ENV_AKS=$(az aks list --resource-group $RESOURCE_GROUP_NAME --query [].name -otsv)
az aks get-credentials --resource-group $RESOURCE_GROUP_NAME --name $ENV_AKS
kubectl config set-context $RESOURCE_GROUP_NAME --cluster $ENV_AKS
if [[ ${Status} == *"success"* ]]; then # If status is already failed, do not over-write in any case.
Status="${currentStatus}"
fi
Message="${Message}. Data Seeding Message: ${currentMessage}. "
Status=$(kubectl get configmap $CONFIG_MAP_NAME -o jsonpath='{.data.status}')
Message=$(kubectl get configmap $CONFIG_MAP_NAME -o jsonpath='{.data.message}')
Message="${Message}Static File Data Seeding Message: ${currentMessage}. "
## Update ConfigMap
kubectl create configmap $CONFIG_MAP_NAME --from-literal=status="$Status" --from-literal=message="$Message" -o yaml --dry-run=client | kubectl replace -f -
## Update ConfigMap
kubectl create configmap $CONFIG_MAP_NAME \
--from-literal=status="$currentStatus" \
--from-literal=message="$Message" \
-o yaml --dry-run=client | kubectl replace -f -
fi
if [[ ${currentStatus} == "success" ]]; then
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment