Commit cb3ec827 authored by Komal Makkar's avatar Komal Makkar
Browse files

Merge branch 'master' of...

Merge branch 'master' of https://community.opengroup.org/osdu/platform/deployment-and-operations/infra-azure-provisioning into users/komakkar/releaseNotes
parents 692fc8ca 322dd74b
Pipeline #65190 failed with stages
in 1 minute and 23 seconds
......@@ -340,7 +340,11 @@ airflow:
key: ENV_KEYVAULT
# Needed for installing python osdu python sdk. In future this will be changed
- name: CI_COMMIT_TAG
value: "v0.10.0"
value: "v0.11.0"
- name: AIRFLOW_VAR_AZURE_DNS_HOST
value: #{DNS_HOST}#
- name: AIRFLOW_VAR_AZURE_ENABLE_MSI
value: "false"
extraConfigmapMounts:
- name: remote-log-config
mountPath: /opt/airflow/config
......@@ -365,7 +369,8 @@ airflow:
"pyyaml==5.4.1",
"requests==2.25.1",
"tenacity==8.0.1",
"https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_api-0.10.0.tar.gz"
"https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_api-0.11.0.tar.gz",
"https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_airflow-0.0.1.tar.gz"
]
extraVolumeMounts:
- name: azure-keyvault
......
......@@ -290,11 +290,15 @@ airflow:
value: "#{AAD_CLIENT_ID}#"
- name: AIRFLOW_VAR_APPINSIGHTS_KEY
value: "#{appinsights-key}#"
- name: AIRFLOW_VAR_AZURE_DNS_HOST
value: #{DNS_HOST}#
- name: AIRFLOW_VAR_AZURE_ENABLE_MSI
value: "false"
- name: PYTHONPATH
value: "/opt/celery"
# Needed for installing python osdu python sdk. In future this will be changed
- name: CI_COMMIT_TAG
value: "v0.10.0"
value: "v0.11.0"
extraConfigmapMounts:
- name: remote-log-config
mountPath: /opt/airflow/config
......@@ -323,7 +327,8 @@ airflow:
"pyyaml==5.4.1",
"requests==2.25.1",
"tenacity==8.0.1",
"https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_api-0.10.0.tar.gz"
"https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_api-0.11.0.tar.gz",
"https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_airflow-0.0.1.tar.gz"
]
extraVolumeMounts:
- name: azure-keyvault
......
......@@ -305,11 +305,15 @@ airflow:
secretKeyRef:
name: central-logging
key: appinsights
- name: AIRFLOW_VAR_AZURE_DNS_HOST
value: #{DNS_HOST}#
- name: AIRFLOW_VAR_AZURE_ENABLE_MSI
value: "false"
- name: PYTHONPATH
value: "/opt/celery"
# Needed for installing python osdu python sdk. In future this will be changed
- name: CI_COMMIT_TAG
value: "v0.10.0"
value: "v0.11.0"
extraConfigmapMounts:
- name: remote-log-config
mountPath: /opt/airflow/config
......@@ -338,7 +342,8 @@ airflow:
"pyyaml==5.4.1",
"requests==2.25.1",
"tenacity==8.0.1",
"https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_api-0.10.0.tar.gz"
"https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_api-0.11.0.tar.gz",
"https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_airflow-0.0.1.tar.gz"
]
extraVolumeMounts:
- name: azure-keyvault
......
# Copyright © Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#####################
# README: Defines a Stage to execute End to End Tests for DAGs
#####################
parameters:
outputDagsFolder: 'output_dags'
dockerfilePath: ''
environmentVars: ''
stages:
- stage: 'Execute_End_to_End_Tests'
jobs:
- job: Execute_End_to_End_Tests
displayName: Execute End to End Test Cases
pool:
name: $(AGENT_POOL)
steps:
- task: DownloadPipelineArtifact@2
displayName: Download Dag files
inputs:
artifact: ${{ parameters.outputDagsFolder }}
path: $(System.DefaultWorkingDirectory)/output_dags
- task: AzureCLI@1
displayName: Execute End to End Tests
env:
AZURE_TENANT_ID: $(app-dev-sp-tenant-id)
inputs:
azureSubscription: $(SERVICE_CONNECTION_NAME)
addSpnToEnvironment: true
scriptLocation: inlineScript
inlineScript: |
// Generate the env file for docker run
NEWLINE=$'\n'
cat > .env << EOF
${{ parameters.environmentVars }}
EOF
docker_image_name=execute-end-to-end-tests:$(Build.SourceVersion)
docker build -t $docker_image_name --file ${{ parameters.dockerfilePath }} .
docker run --env-file .env $docker_image_name
# Copyright © Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#####################
# README: Defines a Stage to execute End to End Tests for DAGs
#####################
parameters:
outputDagsFolder: 'output_dags'
environmentVars: ''
postmanCollection: ''
stages:
- stage: 'Execute_End_to_End_Tests'
jobs:
- job: Execute_End_to_End_Tests
displayName: Execute End to End Test Cases
pool:
name: $(AGENT_POOL)
steps:
- task: DownloadPipelineArtifact@2
displayName: Download Dag files
inputs:
artifact: ${{ parameters.outputDagsFolder }}
path: $(System.DefaultWorkingDirectory)/output_dags
- task: UsePythonVersion@0
inputs:
versionSpec: '3.8'
- task: NodeTool@0
inputs:
versionSpec: '14.x'
- task: AzureCLI@1
displayName: Execute End to End Tests
env:
AZURE_TENANT_ID: $(app-dev-sp-tenant-id)
AZURE_POSTMAN_ENVIRONMENT_FILE_URL: $AZURE_POSTMAN_ENVIRONMENT_FILE_URL
AZURE_POSTMAN_COLLECTION_FILE_URL: ${{ parameters.postmanCollection }}
CLIENT_ID: $(aad-client-id)
inputs:
azureSubscription: $(SERVICE_CONNECTION_NAME)
addSpnToEnvironment: true
scriptLocation: inlineScript
inlineScript: |
NEWLINE=$'\n'
cat > python_env_vars.txt << EOF
${{ parameters.environmentVars }}
EOF
cat > run_postman_collection.py << EOF
import requests
import os
import json
def bootstrap():
f = open('./output_dags/workflow_request_body.json', 'r', encoding='utf-8')
data = json.load(f)
f.close()
dag_name = data[0]['workflowName']
prepare_environment(dag_name)
prepare_postman_collection()
def prepare_postman_collection():
# Downloading the postman collection json file
url_collection = os.environ.get('AZURE_POSTMAN_COLLECTION_FILE_URL')
r = requests.get(url_collection, allow_redirects=True)
open('collection_file.json', 'wb').write(r.content)
def prepare_environment(dag_name):
# Downloading the environment file to trigger the postman collections
url_env_file = os.environ.get('AZURE_POSTMAN_ENVIRONMENT_FILE_URL')
r = requests.get(url_env_file, allow_redirects=True)
open('env_file.json', 'wb').write(r.content)
env_file = open('env_file.json', "r")
data = json.load(env_file)
# Setting env variables
postman_var_to_env_dict = {}
python_vars = open("python_env_vars.txt")
for line in python_vars:
key, value = line.split()
postman_var_to_env_dict[key] = eval(value)
for value in data['values']:
if value['key'] in postman_var_to_env_dict.keys():
value['value'] = postman_var_to_env_dict[value['key']]
env_file = open('env_file.json', "w")
json.dump(data, env_file)
env_file.close()
if __name__ == "__main__":
bootstrap()
EOF
pip install requests
python ./run_postman_collection.py
npm install -g newman
newman run collection_file.json -e env_file.json
FROM python:3.9.6-alpine3.14
FROM mcr.microsoft.com/azure-cli:2.15.1
RUN apk update
RUN apk add ca-certificates wget && update-ca-certificates
RUN apk add libc6-compat
RUN apk add --no-cache --upgrade ca-certificates bash curl wget gettext jq bind-tools \
RUN apk add ca-certificates bash curl wget gettext jq bind-tools \
&& wget -q https://storage.googleapis.com/kubernetes-release/release/v1.21.2/bin/linux/amd64/kubectl -O /usr/local/bin/kubectl \
&& chmod +x /usr/local/bin/kubectl \
&& chmod g+rwx /root \
&& mkdir /config \
&& chmod g+rwx /config
ENV PYTHONUNBUFFERED=1
RUN apk add --update --no-cache python3 && ln -sf python3 /usr/bin/python
RUN python3 -m ensurepip
RUN pip3 install --no-cache --upgrade pip setuptools
WORKDIR /usr/src/app
COPY access_token/get_access_token.sh ./
......
......@@ -3,11 +3,6 @@
currentStatus=""
currentMessage=""
az login --identity
ENV_AKS=$(az aks list --resource-group $RESOURCE_GROUP_NAME --query [].name -otsv)
az aks get-credentials --resource-group $RESOURCE_GROUP_NAME --name $ENV_AKS
kubectl config set-context $RESOURCE_GROUP_NAME --cluster $ENV_AKS
OSDU_URI=${OSDU_HOST}
if [[ ${OSDU_HOST} != "https://"* ]] || [[ ${OSDU_HOST} != "http://"* ]]; then
......@@ -26,6 +21,7 @@ else
partition_count=0
partition_initialized_count=0
partition_user_initialized_count=0
partition_admin_initialized_count=0
for index in "${!partitions_array[@]}"
......@@ -39,7 +35,8 @@ else
i=0
partition_initialized=false
while [[ $i -lt 3 ]]; do
i=$(expr $i + 1)
init_response=$(curl -s -w " Http_Status_Code:%{http_code} " \
-X POST \
-H 'Content-Type: application/json' \
......@@ -90,8 +87,6 @@ else
break
fi
i=$(expr $i + 1)
done
if [[ $i -ge 3 ]]; then
......@@ -101,11 +96,81 @@ else
if [ "$partition_initialized" != true ] ; then
currentStatus="failure"
currentMessage="${currentMessage}. Skipping Adding Admin as Entitlements Init has failed. "
echo "Skipping Adding Admin as Entitlements Init has failed."
currentMessage="${currentMessage}. Skipping Adding User as an Admin as Entitlements Init has failed. "
echo "Skipping Adding User as an Admin as Entitlements Init has failed."
continue
fi
echo "Creating User Entitlements for Partition: ${partitions_array[index]}"
OSDU_ENTITLEMENTS_CREATE_USER_URI=${OSDU_URI}/api/entitlements/v2/groups/users@${partitions_array[index]}.$SERVICE_DOMAIN/members
echo "Entitlements Partition Create User Endpoint: ${OSDU_ENTITLEMENTS_CREATE_USER_URI}"
i=0
while [[ $i -lt 3 ]]; do
i=$(expr $i + 1)
init_response=$(curl -s -w " Http_Status_Code:%{http_code} " \
-X POST \
-H 'Content-Type: application/json' \
-H "Authorization: Bearer $ACCESS_TOKEN" \
-H "data-partition-id: ${partitions_array[index]}" \
-d "{\"email\":\"$ADMIN_ID\", \"role\": \"MEMBER\"}" \
$OSDU_ENTITLEMENTS_CREATE_USER_URI)
echo "Init Reponse: $init_response"
if [ -z "$init_response" -a "$init_response"==" " ]; then
echo "Create User Failed, Empty Reponse. Iteration $i."
continue
fi
# Status code check. succeed only if 2xx
# quit for partition if 404 or 400.
# 401 or 403, then retry after getting access token.
# else sleep for 1min and retry
if [[ ${init_response} != *"Http_Status_Code:2"* ]] && [[ ${init_response} != *"Http_Status_Code:409"* ]];then
if [[ ${init_response} == *"Http_Status_Code:400"* ]] || [[ ${init_response} == *"Http_Status_Code:404"* ]];then
currentStatus="failure"
currentMessage="${currentMessage}. Create User for partition ${partitions_array[index]} failed with response $init_response. "
echo "Create User for partition ${partitions_array[index]} failed with response $init_response"
break
fi
echo "Sleeping for 1min."
sleep 1m
if [[ ${init_response} == *"Http_Status_Code:401"* ]] || [[ ${init_response} == *"Http_Status_Code:403"* ]];then
echo "Trying to Re-Fetch Access Token"
ACCESS_TOKEN=$(sh ./get_access_token.sh)
if [[ "$ACCESS_TOKEN" == "TOKEN_FETCH_FAILURE" ]]; then
currentStatus="failure"
currentMessage="${currentMessage}. Failure re-fetching Access Token. "
echo "Failure re-fetching Access Token"
break
fi
echo "Access Token re-fetched successfully."
fi
continue
else
if [[ ${init_response} == *"Http_Status_Code:409"* ]];then
currentMessage="${currentMessage}. HTTP Status Code: 409 -> Admin User ${partitions_array[index]} Already Exists. "
echo "HTTP Status Code: 409 -> User ${partitions_array[index]} Already Exists."
fi
currentMessage="${currentMessage}. User for Partition ${partitions_array[index]} Initialized successfully. "
echo "User for Partition ${partitions_array[index]} Initialized successfully."
partition_user_initialized_count=$(expr $partition_user_initialized_count + 1)
break
fi
done
if [[ $i -ge 3 ]]; then
currentStatus="failure"
currentMessage="${currentMessage}. Creating User: Max Number of retries reached. "
fi
echo "Adding Admin User Entitlements for Partition: ${partitions_array[index]}"
OSDU_ENTITLEMENTS_ADD_OPS_URI=${OSDU_URI}/api/entitlements/v2/groups/users.datalake.ops@${partitions_array[index]}.$SERVICE_DOMAIN/members
......@@ -113,13 +178,14 @@ else
i=0
while [[ $i -lt 3 ]]; do
i=$(expr $i + 1)
init_response=$(curl -s -w " Http_Status_Code:%{http_code} " \
-X POST \
-H 'Content-Type: application/json' \
-H "Authorization: Bearer $ACCESS_TOKEN" \
-H "data-partition-id: ${partitions_array[index]}" \
-d "{"email":"$ADMIN_ID", "role": "MEMBER"}" \
-d "{\"email\":\"$ADMIN_ID\", \"role\": \"MEMBER\"}" \
$OSDU_ENTITLEMENTS_ADD_OPS_URI)
echo "Init Reponse: $init_response"
......@@ -133,7 +199,7 @@ else
# quit for partition if 404 or 400.
# 401 or 403, then retry after getting access token.
# else sleep for 1min and retry
if [[ ${init_response} != *"Http_Status_Code:2"* ]];then
if [[ ${init_response} != *"Http_Status_Code:2"* ]] && [[ ${init_response} != *"Http_Status_Code:409"* ]];then
if [[ ${init_response} == *"Http_Status_Code:400"* ]] || [[ ${init_response} == *"Http_Status_Code:404"* ]];then
currentStatus="failure"
currentMessage="${currentMessage}. Add Ops Member for partition ${partitions_array[index]} failed with response $init_response. "
......@@ -158,14 +224,16 @@ else
continue
else
if [[ ${init_response} == *"Http_Status_Code:409"* ]];then
currentMessage="${currentMessage}. HTTP Status Code: 409 -> Admin User ${partitions_array[index]} Already Exists. "
echo "HTTP Status Code: 409 -> Partition ${partitions_array[index]} Already Exists."
fi
currentMessage="${currentMessage}. Ops Member for Partition ${partitions_array[index]} Initialized successfully. "
echo "Ops Member for Partition ${partitions_array[index]} Initialized successfully."
partition_admin_initialized_count=$(expr $partition_admin_initialized_count + 1)
break
fi
i=$(expr $i + 1)
done
if [[ $i -ge 3 ]]; then
......@@ -174,16 +242,20 @@ else
fi
done
if [ "$partition_count" -ne "$partition_initialized_count" ] || [ "$partition_admin_initialized_count" -ne "$partition_initialized_count" ]; then
if [ "$partition_count" -ne "$partition_initialized_count" ] || [ "$partition_user_initialized_count" -ne "$partition_initialized_count" ] || [ "$partition_admin_initialized_count" -ne "$partition_initialized_count" ]; then
currentStatus="failure"
currentMessage="${currentMessage}. Entitlements for $partition_initialized_count partition(s) of total $partition_count partition(s) initialized successfully. "
currentMessage="${currentMessage}. $partition_user_initialized_count partition(s) of total $partition_initialized_count initialized with User. "
currentMessage="${currentMessage}. $partition_admin_initialized_count partition(s) of total $partition_initialized_count initialized with Ops Member. "
echo "Entitlements for $partition_initialized_count partition(s) of total $partition_count partition(s) initialized successfully."
echo "$partition_user_initialized_count partition(s) of total $partition_initialized_count initialized with User."
echo "$partition_admin_initialized_count partition(s) of total $partition_initialized_count initialized with Ops Member."
else
currentMessage="${currentMessage}. Entitlements for All $partition_initialized_count partition(s) initialized successfully. "
currentMessage="${currentMessage}. User for all of $partition_user_initialized_count partition(s) added successfully. "
currentMessage="${currentMessage}. Ops Members for all of $partition_admin_initialized_count partition(s) added successfully. "
echo "Entitlements for All $partition_initialized_count partition(s) initialized successfully."
echo "User for all of $partition_user_initialized_count partition(s) added successfully."
echo "Ops Members for all of $partition_admin_initialized_count partition(s) added successfully."
fi
fi
......@@ -195,16 +267,21 @@ echo "Current Status: ${currentStatus}"
echo "Current Message: ${currentMessage}"
if [ ! -z "$CONFIG_MAP_NAME" -a "$CONFIG_MAP_NAME" != " " ]; then
Status=$(kubectl get configmap $CONFIG_MAP_NAME -o jsonpath='{.data.status}')
Message=$(kubectl get configmap $CONFIG_MAP_NAME -o jsonpath='{.data.message}')
az login --identity --username $OSDU_IDENTITY_ID
ENV_AKS=$(az aks list --resource-group $RESOURCE_GROUP_NAME --query [].name -otsv)
az aks get-credentials --resource-group $RESOURCE_GROUP_NAME --name $ENV_AKS
kubectl config set-context $RESOURCE_GROUP_NAME --cluster $ENV_AKS
if [[ ${Status} == *"success"* ]]; then # If status is already failed, do not over-write in any case.
Status="${currentStatus}"
fi
Message="${Message}. Entitlements Data Initialization: ${currentMessage}. "
Status=$(kubectl get configmap $CONFIG_MAP_NAME -o jsonpath='{.data.status}')
Message=$(kubectl get configmap $CONFIG_MAP_NAME -o jsonpath='{.data.message}')
Message="${Message}Entitlements Init Message: ${currentMessage}. "
## Update ConfigMap
kubectl create configmap $CONFIG_MAP_NAME --from-literal=status="$Status" --from-literal=message="$Message" -o yaml --dry-run=client | kubectl replace -f -
## Update ConfigMap
kubectl create configmap $CONFIG_MAP_NAME \
--from-literal=status="$currentStatus" \
--from-literal=message="$Message" \
-o yaml --dry-run=client | kubectl replace -f -
fi
if [[ ${currentStatus} == "success" ]]; then
......
......@@ -9,6 +9,7 @@ services:
ADMIN_ID: $ADMIN_ID
SERVICE_DOMAIN: $SERVICE_DOMAIN
OSDU_HOST: $OSDU_HOST
OSDU_IDENTITY_ID: $OSDU_IDENTITY_ID
RESOURCE_GROUP_NAME: $RESOURCE_GROUP_NAME
PARTITIONS: $PARTITIONS # comma separated list of partitions
CONFIG_MAP_NAME: $CONFIG_MAP_NAME
......
FROM python:3.9.6-alpine3.14
FROM mcr.microsoft.com/azure-cli:2.15.1
RUN apk update
RUN apk add ca-certificates wget && update-ca-certificates
RUN apk add libc6-compat
RUN apk add --no-cache --upgrade ca-certificates bash curl wget gettext jq bind-tools \
RUN apk add ca-certificates bash curl wget gettext jq bind-tools \
&& wget -q https://storage.googleapis.com/kubernetes-release/release/v1.21.2/bin/linux/amd64/kubectl -O /usr/local/bin/kubectl \
&& chmod +x /usr/local/bin/kubectl \
&& chmod g+rwx /root \
&& mkdir /config \
&& chmod g+rwx /config
ENV PYTHONUNBUFFERED=1
RUN apk add --update --no-cache python3 && ln -sf python3 /usr/bin/python
RUN python3 -m ensurepip
RUN pip3 install --no-cache --upgrade pip setuptools
WORKDIR /usr/src/app
COPY access_token/get_access_token.sh ./
......
......@@ -3,11 +3,6 @@
currentStatus=""
currentMessage=""
az login --identity
ENV_AKS=$(az aks list --resource-group $RESOURCE_GROUP_NAME --query [].name -otsv)
az aks get-credentials --resource-group $RESOURCE_GROUP_NAME --name $ENV_AKS
kubectl config set-context $RESOURCE_GROUP_NAME --cluster $ENV_AKS
OSDU_URI=${OSDU_HOST}
if [[ ${OSDU_HOST} != "https://"* ]] || [[ ${OSDU_HOST} != "http://"* ]]; then
......@@ -37,6 +32,7 @@ else
i=0
while [[ $i -lt 3 ]]; do
i=$(expr $i + 1)
init_response=$(curl -s -w " Http_Status_Code:%{http_code} " \
-X POST \
......@@ -53,11 +49,11 @@ else
continue
fi
# Status code check. succeed only if 2xx
# Status code check. succeed only if 2xx, or 409
# quit for partition if 404 or 400.
# 401 or 403, then retry after getting access token.
# else sleep for 1min and retry
if [[ ${init_response} != *"Http_Status_Code:2"* ]];then
if [[ ${init_response} != *"Http_Status_Code:2"* ]] && [[ ${init_response} != *"Http_Status_Code:409"* ]];then
if [[ ${init_response} == *"Http_Status_Code:400"* ]] || [[ ${init_response} == *"Http_Status_Code:404"* ]];then
currentStatus="failure"
currentMessage="${currentMessage}. Partition Init for partition ${partitions_array[index]} failed with response $init_response. "
......@@ -82,14 +78,16 @@ else
continue
else
if [[ ${init_response} == *"Http_Status_Code:409"* ]];then
currentMessage="${currentMessage}. HTTP Status Code: 409 -> Partition ${partitions_array[index]} Already Exists. "
echo "HTTP Status Code: 409 -> Partition ${partitions_array[index]} Already Exists."
fi
currentMessage="${currentMessage}. Partition ${partitions_array[index]} Initialized successfully. "
echo "Partition ${partitions_array[index]} Initialized successfully."
partition_initialized_count=$(expr $partition_initialized_count + 1)
break
fi
i=$(expr $i + 1)
done
if [[ $i -ge 3 ]]; then
......@@ -115,16 +113,21 @@ echo "Current Status: ${currentStatus}"
echo "Current Message: ${currentMessage}"
if [ ! -z "$CONFIG_MAP_NAME" -a "$CONFIG_MAP_NAME" != " " ]; then
Status=$(kubectl get configmap $CONFIG_MAP_NAME -o jsonpath='{.data.status}')