Commit c336cc81 authored by Kishore Battula's avatar Kishore Battula
Browse files

Merge branch 'haaggarw/Azure_Dag' into 'master'

Adding pipeline for deploying Azure Dags

See merge request osdu/platform/ci-cd-pipelines!253
parents 4b185e9f b64f3eee
Pipeline #45326 passed with stages
in 20 seconds
.vscode
\ No newline at end of file
.vscode
.idea
.gradle
\ No newline at end of file
# EXPECTED PIPELINE INHERITED GROUP VARIABLES
# --------------------------------------------------------------------------------
# AZURE (Protected Branch)
# AZURE_APP_ID (Protected Branch)
# AZURE_APP_ID_OTHER (Protected Branch)
# AZURE_APP_OID_OTHER (Protected Branch)
# AZURE_BASE (Protected Branch)
# AZURE_BASENAME_21 (Protected Branch)
# AZURE_DNS_NAME (Protected Branch)
# AZURE_ELASTIC_HOST (Protected Branch)
# AZURE_ELASTIC_PASSWORD (Protected Branch/Masked Variable)
# AZURE_INVALID_JWT (Protected Branch)
# AZURE_NO_ACCESS_ID (Protected Branch)
# AZURE_NO_ACCESS_SECRET (Protected Branch/Masked Variable)
# AZURE_PRINCIPAL_ID (Protected Branch)
# AZURE_PRINCIPAL_SECRET (Protected Branch/Masked Variable)
# AZURE_REGISTRY (Protected Branch)
# AZURE_SERVICEBUS_KEY (Protected Branch/Masked Variable)
# AZURE_STORAGE_KEY (Protected Branch/Masked Variable)
# AZURE_SUBSCRIPTION_ID (Protected Branch)
# AZURE_SUBSCRIPTION_NAME (Protected Branch)
# AZURE_TENANT_ID (Protected Branch)
# AZURE_APPINSIGHTS_KEY (Protected Branch/Masked Variable)
# EXPECTED PIPELINE VARIABLES
# --------------------------------------------------------------------------------
# AZURE_TEST_SUBDIR
.azure_variables:
variables:
WORKFLOW_URL: https://${AZURE_DNS_NAME}/api/workflow/v1/
AZURE_AD_TENANT_ID: $AZURE_TENANT_ID
AZURE_CLIENT_ID: $AZURE_PRINCIPAL_ID
AZURE_CLIENT_SECRET: $AZURE_PRINCIPAL_SECRET
AZURE_AD_APP_RESOURCE_ID: $AZURE_APP_ID
SHARED_TENANT: opendes
DATA_PARTITION_ID: opendes
# JOBS
# --------------------------------------------------------------------------------
# This stage generates the output dag folder by running a docker container which will
# built via Dockerfile present in DAD repository root. This Dockerfile should contain the
# logic to generate all the prerequisite artifacts for the DAG and output all the
# Dag related files in folder
azure_create_dag:
tags: ["osdu-medium"]
image: danielscholl/azure-build-image
stage: containerize
needs: ["compile-and-unit-test"]
variables:
PARSER_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:${CI_COMMIT_SHA}
DAG_IMAGE: ${CI_PROJECT_NAME}-dag-${CI_COMMIT_REF_SLUG}:${CI_COMMIT_SHA}
extends:
- .azure_variables
before_script:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- az --version
- az login --service-principal -u $AZURE_PRINCIPAL_ID -p $AZURE_PRINCIPAL_SECRET --tenant $AZURE_TENANT_ID
script:
- |
# Generating environment file to be passed while running the docker container
cat > .env << EOF
CI_REGISTRY_USER=${CI_REGISTRY_USER}
CI_REGISTRY_PASSWORD=${CI_REGISTRY_PASSWORD}
CI_REGISTRY=${CI_REGISTRY}
AZURE_REGISTRY=${AZURE_REGISTRY}
AZURE_PRINCIPAL_ID=${AZURE_PRINCIPAL_ID}
AZURE_PRINCIPAL_SECRET=${AZURE_PRINCIPAL_SECRET}
AZURE_TENANT_ID=${AZURE_TENANT_ID}
CI_PROJECT_NAME=${CI_PROJECT_NAME}
CI_COMMIT_REF_SLUG=${CI_COMMIT_REF_SLUG}
CI_COMMIT_SHA=${CI_COMMIT_SHA}
CI_REGISTRY_IMAGE=${CI_REGISTRY_IMAGE}
AZURE_DEPLOYMENTS_SCRIPTS_SUBDIR=${AZURE_DEPLOYMENTS_SCRIPTS_SUBDIR}
PARSER_IMAGE=${PARSER_IMAGE}
DAG_IMAGE=${DAG_IMAGE}
SHARED_TENANT=$SHARED_TENANT
AZURE_DNS_NAME=$AZURE_DNS_NAME
EOF
- docker build -t $CI_REGISTRY_IMAGE/$PARSER_IMAGE-orchestrator --file $AZURE_DEPLOYMENTS_SCRIPTS_SUBDIR/dockerFolder/output_dags_dockerfile .
- |
# Checking if container with name docker_generate_dags exists then delete
# Container with a name is required to perform the copy operation below
if [ "$(docker ps -a | grep docker_generate_dags)" ]; then
docker stop docker_generate_dags
docker rm docker_generate_dags
fi
- docker run --name "docker_generate_dags" -v /var/run/docker.sock:/var/run/docker.sock --env-file .env $CI_REGISTRY_IMAGE/$PARSER_IMAGE-orchestrator
# Copying the dag folder from the docker container to he host
- docker cp "docker_generate_dags:$AZURE_OUTPUT_DAG_FOLDER" .
artifacts:
paths:
- "output_dags"
only:
variables:
- $AZURE == 'true'
azure_copy_dag:
image: danielscholl/azure-build-image
tags: ["osdu-medium"]
stage: deploy
needs: ["azure_create_dag"]
extends:
- .azure_variables
before_script:
- az login --service-principal -u $AZURE_PRINCIPAL_ID -p $AZURE_PRINCIPAL_SECRET --tenant $AZURE_TENANT_ID
- az aks get-credentials -g $AZURE_UNIQUE-rg -n $AZURE_UNIQUE-aks
script:
- |
# Installing the Azcopy utility
apk add --update coreutils && rm -rf /var/cache/apk/*
mkdir -p tmp
cd tmp
wget -O azcopy_v10.tar.gz https://aka.ms/downloadazcopy-v10-linux && tar -xf azcopy_v10.tar.gz --strip-components=1
cp ./azcopy /usr/bin/
cd ..
- EXPIRE=$(date -u -d "59 minutes" '+%Y-%m-%dT%H:%M:%SZ')
- START=$(date -u -d "-1 minute" '+%Y-%m-%dT%H:%M:%SZ')
- accountKey=$(kubectl get secret airflow -n osdu -o jsonpath='{.data.azurestorageaccountkey}' | base64 -d)
- accountName=$(kubectl get secret airflow -n osdu -o jsonpath='{.data.azurestorageaccountname}' | base64 -d)
# Generating the SAS Token required for Authorization
- AZURE_STORAGE_SAS_TOKEN=$(az storage account generate-sas --account-name $accountName --account-key $accountKey --start $START --expiry $EXPIRE --https-only --resource-types sco --services f --permissions cwdlur -o tsv)
- cd output_dags
- |
if [ -d "./dags/" ]; then
# Copying all the contents inside the dags folder
azcopy cp "./dags/*" "https://${accountName}.file.core.windows.net/airflowdags/dags?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true
cd dags
# Now syncing only the folders which are part of source to remove the deleted files
for directory in *; do
if [ -d "$directory" ]; then
azcopy sync "./$directory/" "https://${accountName}.file.core.windows.net/airflowdags/dags/$directory?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true --delete-destination=true
fi
done
cd ..
fi
- |
if [ -d "./plugins/" ]; then
# Copying all the contents inside the plugins folder
azcopy cp "./plugins/*" "https://${accountName}.file.core.windows.net/airflowdags/plugins?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true
cd plugins
if [ -d "operators" ]; then
cd operators
# Syncing only the sub folders which are part of source in operators folder
for directory in *; do
if [ -d "$directory" ]; then
azcopy sync "./$directory/" "https://${accountName}.file.core.windows.net/airflowdags/plugins/operators/$directory?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true --delete-destination=true
fi
done
cd ..
fi
if [ -d "hooks" ]; then
cd hooks
# Syncing only the sub folders which are part of source in hooks folder
for directory in *; do
if [ -d "$directory" ]; then
azcopy sync "./$directory/" "https://${accountName}.file.core.windows.net/airflowdags/plugins/hooks/$directory?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true --delete-destination=true
fi
done
cd ..
fi
cd ..
fi
artifacts:
paths:
- "output_dags"
only:
variables:
- $AZURE == 'true'
azure_register_dag:
tags: ["osdu-medium"]
image: python:3.8
stage: bootstrap
needs: ["azure_copy_dag"]
variables:
PARSER_IMAGE: ${AZURE_REGISTRY}.azurecr.io/${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:${CI_COMMIT_SHA}
extends:
- .azure_variables
script:
- |
cat > requirements.txt << EOF
msal
EOF
- |
# Python script for generating the Bearer Token
cat > Token.py << EOF
import os
import msal
class AzureToken(object):
def get_azure_id_token(self):
tenant_id = os.getenv('AZURE_TENANT_ID')
resource_id = os.getenv('AZURE_AD_APP_RESOURCE_ID')
client_id = os.getenv('AZURE_CLIENT_ID')
client_secret = os.getenv('AZURE_CLIENT_SECRET')
if tenant_id is None:
print('Please pass tenant Id to generate token')
exit(1)
if resource_id is None:
print('Please pass resource Id to generate token')
exit(1)
if client_id is None:
print('Please pass client Id to generate token')
exit(1)
if client_secret is None:
print('Please pass client secret to generate token')
exit(1)
try:
authority_host_uri = 'https://login.microsoftonline.com'
authority_uri = authority_host_uri + '/' + tenant_id
scope = [resource_id + '/.default']
app = msal.ConfidentialClientApplication(client_id=client_id, authority=authority_uri, client_credential=client_secret)
result = app.acquire_token_for_client(scopes=scope)
token = 'Bearer ' + result.get('access_token')
print(token)
return token
except Exception as e:
print(e)
if __name__ == '__main__':
AzureToken().get_azure_id_token()
EOF
- |
# Python script for registering the DAG by calling Workflow service API
cat > register_dag.py << EOF
import json
import requests
import os
class RegisterDag:
def __init__(self):
self.token = os.environ.get('BEARER_TOKEN')
self.data_partition_id = os.environ.get('SHARED_TENANT')
self.workflow_service_url = os.environ.get('WORKFLOW_URL') + "workflow"
def register(self):
self.register_dag()
def register_dag(self):
with open('./output_dags/workflow_request_body.json', 'r', encoding='utf-8') as f:
file_data = json.load(f)
for data in file_data:
data = json.dumps(data)
headers = {
'Content-Type': 'application/json',
'Authorization': self.token,
'data-partition-id': self.data_partition_id
}
response = requests.post(self.workflow_service_url, headers=headers, data=data)
if response.status_code == 200:
workflow_id = response.json().get('workflowId')
print("DAG registered with workflowId: {0}".format(workflow_id))
elif response.status_code == 409:
workflow_id = response.json().get('workflowId')
print("DAG is already registered with workflowId: {0}".format(workflow_id))
else:
print("Error while registering DAG {0}".format(response.raise_for_status()))
exit(1)
if __name__ == "__main__":
RegisterDag().register()
EOF
- pip install -r ./requirements.txt
- export BEARER_TOKEN=$(python ./Token.py)
- echo $BEARER_TOKEN
- python ./register_dag.py
only:
variables:
- $AZURE == 'true'
azure_test_dag:
tags: ["osdu-medium"]
image: $CI_REGISTRY/danielscholl/azure-maven/azure-maven:v1.0
stage: testDag
needs: ["azure_register_dag"]
variables:
AZURE_SERVICE: csv-parser
AZURE_TENANT_ID: $AZURE_TENANT_ID
KEYVAULT_URI: $AZURE_KEYVAULT_URI
partition_service_endpoint: https://${AZURE_DNS_NAME}/api/partition/v1
storage_service_endpoint: https://${AZURE_DNS_NAME}/api/storage/v2
schema_service_endpoint: https://${AZURE_DNS_NAME}/api/schema-service/v1
search_service_endpoint: https://${AZURE_DNS_NAME}/api/search/v2
unit_service_endpoint: https://${AZURE_DNS_NAME}/api/unit/v2/unit/symbol
legal_service_endpoint: https://${AZURE_DNS_NAME}/api/legal/v1
file_service_endpoint: https://${AZURE_DNS_NAME}/api/file/v2
DMS_KIND: opendes:osdudemo:wellbore:1.0.0
DMS_ACL: data.test1@opendes.contoso.com
DMS_LEGAL_TAG: opendes-public-usa-dataset-7643990
AZURE_STORAGE_CONTAINER: opendes
aad_client_id: $AZURE_APP_ID
extends:
- .azure_variables
script:
- mvn clean install -pl $AZURE_PARSER_BUILD_SUBDIR -am
- mvn clean verify -f $AZURE_TEST_SUBDIR/pom.xml
only:
variables:
- $AZURE == 'true'
except:
variables:
- $AZURE_SKIP_TEST == 'true'
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment