Skip to content
Snippets Groups Projects
Commit d618799f authored by preeti singh[Microsoft]'s avatar preeti singh[Microsoft] Committed by Kishore Battula
Browse files

Preeti/ManifestIngestionPipeline ado

parent 8cd3a5c2
No related branches found
No related tags found
1 merge request!55Preeti/ManifestIngestionPipeline ado
Hide whitespace changes
Inline Side-by-side
devops/azure/development-pipeline.yml 0 → 100644
+ 296
− 0
View file @ d618799f
# This pipeline is for dev env
# Docker
# Build a Docker image
# https://docs.microsoft.com/azure/devops/pipelines/languages/docker
trigger:
- master
resources:
- repo: self
#I have included dev environment specific libraries because it is intended to run on dev env at present.
variables:
- group: 'Azure - OSDU'
- group: 'Azure - Common'
- group: 'Azure Target Env - dev'
- group: 'Azure Target Env Secrets - dev'
steps:
- task: AzureCLI@1
displayName: 'Docker Build + ACR Push'
inputs:
azureSubscription: $(SERVICE_CONNECTION_NAME)
addSpnToEnvironment: true
scriptLocation: inlineScript
inlineScript: |
function writeDockerFile() {
echo 'FROM mcr.microsoft.com/azure-cli
VOLUME /tmp
USER root
ADD /src/dags ./dags
ADD /src/plugins ./plugins
ADD bootstrap.sh ./bootstrap.sh
RUN chmod +x bootstrap.sh \
&& apk add --update coreutils && rm -rf /var/cache/apk/* \
&& echo "Setup AzCopy.." \
&& mkdir -p tmp \
&& cd tmp \
&& wget -O azcopy_v10.tar.gz https://aka.ms/downloadazcopy-v10-linux && tar -xf azcopy_v10.tar.gz --strip-components=1 \
&& cp ./azcopy /usr/bin/ \
&& cd ..' > Dockerfile
}
function writeDockerRunCommands(){
echo '#!/bin/bash
EXPIRE=$(date -u -d "9 minutes" '+%Y-%m-%dT%H:%M:%SZ')
START=$(date -u -d "-1 minute" '+%Y-%m-%dT%H:%M:%SZ')
AZURE_STORAGE_SAS_TOKEN=$(az storage account generate-sas --account-name $env_storage --account-key $env_storage_key --start $START --expiry $EXPIRE --https-only --resource-types sco --services bfqt --permissions dlrcuaw -o tsv )
azcopy cp "/plugins/*" "https://${env_storage}.file.core.windows.net/airflowdags/plugins/?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true
azcopy cp "/dags/*" "https://${env_storage}.file.core.windows.net/airflowdags/dags/?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true
echo "copied files and folders for dags and plugins***"
if [ -d "./dags/" ]; then
cd dags
for directory in *; do
if [ -d "$directory" ]; then
echo "syncing $directory"
azcopy sync "./$directory" "https://${env_storage}.file.core.windows.net/airflowdags/dags/$directory?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true --delete-destination=true
fi
echo "$directory sync done"
done
cd ..
fi
echo "synced files and folders for dags"
pwd
cd plugins/operators
pwd
for directory in *; do
if [ -d "$directory" ]; then
azcopy sync "./$directory/" "https://${env_storage}.file.core.windows.net/airflowdags/plugins/operators/$directory?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true --delete-destination=true
fi
echo "$directory sync done"
done
echo "synced files and folders for plugins/operators"' > bootstrap.sh
}
echo "out from writing docker file block"
curl -L https://aka.ms/acr/installaad/bash | /bin/bash
echo "Logging in to the ACR Registry"
echo "------------------------------------"
az acr login -n $(CONTAINER_REGISTRY_NAME)
writeDockerRunCommands
writeDockerFile
echo "Tagging and Pushing $IMAGE to ACR for JAR $jar_file_name from jar file path"
docker build -t azcli-image .
echo "Build done"
docker run --entrypoint ./bootstrap.sh -e "env_storage_key=$(AIRFLOW_STORAGE_ACCOUNT_KEY)" -e "env_storage=$(AIRFLOW_STORAGE_ACCOUNT)" azcli-image
- task: AzureCLI@1
displayName: 'Docker Build + generate token and register dag'
inputs:
azureSubscription: $(SERVICE_CONNECTION_NAME)
addSpnToEnvironment: true
scriptLocation: inlineScript
inlineScript: |
function writeDockerFile() {
echo 'FROM python
VOLUME /tmp
USER root
ADD requirements.txt ./requirements.txt
ADD Token.py ./Token.py
ADD register_dag.py ./register_dag.py
ADD bootstrapPython.sh ./bootstrapPython.sh
RUN chmod +x register_dag.py \
&& chmod +x Token.py \
&& chmod +x requirements.txt \
&& chmod +x bootstrapPython.sh \
&& pip install -r requirements.txt' > Dockerfile
}
echo 'msal' > requirements.txt
cat > Token.py << EOF
import os
import msal
class AzureToken(object):
def get_azure_id_token(self):
tenant_id = os.getenv('AZURE_DEPLOY_TENANT')
resource_id = os.getenv('AZURE_AD_APP_RESOURCE_ID')
client_id = os.getenv('INTEGRATION_TESTER')
client_secret = os.getenv('AZURE_TESTER_SERVICEPRINCIPAL_SECRET')
if tenant_id is None:
print('Please pass tenant Id to generate token')
exit(1)
if resource_id is None:
print('Please pass resource Id to generate token')
exit(1)
if client_id is None:
print('Please pass client Id to generate token')
exit(1)
if client_secret is None:
print('Please pass client secret to generate token')
exit(1)
try:
authority_host_uri = 'https://login.microsoftonline.com'
authority_uri = authority_host_uri + '/' + tenant_id
scopes = [resource_id + '/.default']
app = msal.ConfidentialClientApplication(client_id=client_id, authority=authority_uri, client_credential=client_secret)
result = app.acquire_token_for_client(scopes=scopes)
token = 'Bearer ' + result.get('access_token')
print(token)
return token
except Exception as e:
print(e)
if __name__ == '__main__':
AzureToken().get_azure_id_token()
EOF
cat > register_dag.py << EOF
import json
import requests
import os
import argparse
class RegisterDag:
def __init__(self):
self.token = os.environ.get('BEARER_TOKEN')
self.data_partition_id = os.environ.get('SHARED_TENANT')
self.workflow_service_url = os.environ.get('WORKFLOW_URL')
self.dag_name = f"Osdu_ingest"
print(f"data_partition_id: {self.data_partition_id}")
print(f"workflow_service_url: {self.workflow_service_url}")
print(f"dag_name: {self.dag_name}")
def register(self):
self.register_dag()
def register_dag(self):
data = json.dumps({
"description": "This is for manifest ingestion",
"registrationInstructions": {
"dagName": self.dag_name,
},
"workflowName": self.dag_name
})
headers = {
'Content-Type': 'application/json',
'Authorization': self.token,
'data-partition-id': self.data_partition_id
#'correlation-id': self.service_name
}
response = requests.post(self.workflow_service_url, headers=headers, data=data)
if response.status_code == 200:
workflow_id = response.json().get('workflowId')
print("DAG registered with workflowId: {0}".format(workflow_id))
elif response.status_code == 409:
workflow_id = response.json().get('workflowId')
print("DAG was already registered with workflowId: {0}".format(workflow_id))
else:
print("Error while registering DAG {0}".format(response.raise_for_status()))
exit(1)
if __name__ == "__main__":
RegisterDag().register()
EOF
echo '#!/usr/bin/env sh
###############################
## ARGUMENT INPUT ##
###############################
usage() { echo "Usage: bootstrapPython.sh"; }
printf "\n"
echo "------------------------------------";
echo "BOOTSTRAPPYTHON -- START";
echo "------------------------------------";
if [ -z $AZURE_DEPLOY_TENANT ]; then
echo 'ERROR: AZURE_DEPLOY_TENANT not provided'
usage;
elif [ -z $DNS_HOST ]; then
echo 'ERROR: DNS_HOST not provided'
usage;
elif [ -z $AZURE_TESTER_SERVICEPRINCIPAL_SECRET ]; then
echo 'ERROR: AZURE_TESTER_SERVICEPRINCIPAL_SECRET not provided'
usage;
elif [ -z $INTEGRATION_TESTER ]; then
echo 'ERROR: INTEGRATION_TESTER not provided'
usage;
elif [ -z $AZURE_AD_APP_RESOURCE_ID ]; then
echo 'ERROR: AZURE_AD_APP_RESOURCE_ID not provided'
usage;
else
export AZURE_DNS_NAME=$DNS_HOST
export WORKFLOW_URL=https://${DNS_HOST}/api/workflow/v1/workflow
fi
if [ -z $DATA_PARTITION ]; then
SHARED_TENANT="opendes"
else
SHARED_TENANT=$DATA_PARTITION
fi
export SHARED_TENANT
echo "AZURE_DNS_NAME: $AZURE_DNS_NAME"
echo "WORKFLOW_URL: $WORKFLOW_URL"
echo "SHARED_TENANT: $SHARED_TENANT"
printf "\n"
echo "Execute Python Script Token.py";
echo "------------------------------------";
BEARER_TOKEN=$(python ./Token.py)
export BEARER_TOKEN
echo "$(echo $BEARER_TOKEN | cut -c -20)***********"
printf "\n"
echo "Execute Python Script register_dag.py";
echo "------------------------------------";
python ./register_dag.py
if test $? = '1'; then
printf "\n"
echo "------------------------------------";
echo "BOOTSTRAP -- ERROR";
echo "------------------------------------";
exit 1
else
printf "\n"
echo "------------------------------------";
echo "BOOTSTRAP -- FINISH";
echo "------------------------------------";
exit 0
fi' >bootstrapPython.sh
echo "out from writing docker file"
curl -L https://aka.ms/acr/installaad/bash | /bin/bash
echo "Logging in to the ACR Registry"
echo "------------------------------------"
az acr login -n $(CONTAINER_REGISTRY_NAME)
writeDockerFile
echo "Tagging and Pushing $IMAGE to ACR "
docker build -t python-image .
echo "Build done"
docker run --entrypoint ./bootstrapPython.sh -e "DNS_HOST=$(DNS_HOST)" -e "AZURE_DEPLOY_TENANT=$(AZURE_DEPLOY_TENANT)" -e "AZURE_TESTER_SERVICEPRINCIPAL_SECRET=$(AZURE_TESTER_SERVICEPRINCIPAL_SECRET)" -e "INTEGRATION_TESTER=$(INTEGRATION_TESTER)" -e "AZURE_AD_APP_RESOURCE_ID=$(AZURE_AD_APP_RESOURCE_ID)" python-image
devops/azure/pipeline.yml 0 → 100644
+ 296
− 0
View file @ d618799f
#This pipeline is for demo env
# Docker
# Build a Docker image
# https://docs.microsoft.com/azure/devops/pipelines/languages/docker
trigger:
- master
resources:
- repo: self
#I have included dev environment specific libraries because it is intended to run on demo env .
variables:
- group: 'Azure - OSDU'
- group: 'Azure - Common'
- group: 'Azure Target Env - demo'
- group: 'Azure Target Env Secrets - demo'
steps:
- task: AzureCLI@1
displayName: 'Docker Build + ACR Push'
inputs:
azureSubscription: $(SERVICE_CONNECTION_NAME)
addSpnToEnvironment: true
scriptLocation: inlineScript
inlineScript: |
function writeDockerFile() {
echo 'FROM mcr.microsoft.com/azure-cli
VOLUME /tmp
USER root
ADD /src/dags ./dags
ADD /src/plugins ./plugins
ADD bootstrap.sh ./bootstrap.sh
RUN chmod +x bootstrap.sh \
&& apk add --update coreutils && rm -rf /var/cache/apk/* \
&& echo "Setup AzCopy.." \
&& mkdir -p tmp \
&& cd tmp \
&& wget -O azcopy_v10.tar.gz https://aka.ms/downloadazcopy-v10-linux && tar -xf azcopy_v10.tar.gz --strip-components=1 \
&& cp ./azcopy /usr/bin/ \
&& cd ..' > Dockerfile
}
function writeDockerRunCommands(){
echo '#!/bin/bash
EXPIRE=$(date -u -d "9 minutes" '+%Y-%m-%dT%H:%M:%SZ')
START=$(date -u -d "-1 minute" '+%Y-%m-%dT%H:%M:%SZ')
AZURE_STORAGE_SAS_TOKEN=$(az storage account generate-sas --account-name $env_storage --account-key $env_storage_key --start $START --expiry $EXPIRE --https-only --resource-types sco --services bfqt --permissions dlrcuaw -o tsv )
azcopy cp "/plugins/*" "https://${env_storage}.file.core.windows.net/airflowdags/plugins/?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true
azcopy cp "/dags/*" "https://${env_storage}.file.core.windows.net/airflowdags/dags/?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true
echo "copied files and folders for dags and plugins***"
if [ -d "./dags/" ]; then
cd dags
for directory in *; do
if [ -d "$directory" ]; then
echo "syncing $directory"
azcopy sync "./$directory" "https://${env_storage}.file.core.windows.net/airflowdags/dags/$directory?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true --delete-destination=true
fi
echo "$directory sync done"
done
cd ..
fi
echo "synced files and folders for dags"
pwd
cd plugins/operators
pwd
for directory in *; do
if [ -d "$directory" ]; then
azcopy sync "./$directory/" "https://${env_storage}.file.core.windows.net/airflowdags/plugins/operators/$directory?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true --delete-destination=true
fi
echo "$directory sync done"
done
echo "synced files and folders for plugins/operators"' > bootstrap.sh
}
echo "out from writing docker file block"
curl -L https://aka.ms/acr/installaad/bash | /bin/bash
echo "Logging in to the ACR Registry"
echo "------------------------------------"
az acr login -n $(CONTAINER_REGISTRY_NAME)
writeDockerRunCommands
writeDockerFile
echo "Tagging and Pushing $IMAGE to ACR for JAR $jar_file_name from jar file path"
docker build -t azcli-image .
echo "Build done"
docker run --entrypoint ./bootstrap.sh -e "env_storage_key=$(AIRFLOW_STORAGE_ACCOUNT_KEY)" -e "env_storage=$(AIRFLOW_STORAGE_ACCOUNT)" azcli-image
- task: AzureCLI@1
displayName: 'Docker Build + generate token and register dag'
inputs:
azureSubscription: $(SERVICE_CONNECTION_NAME)
addSpnToEnvironment: true
scriptLocation: inlineScript
inlineScript: |
function writeDockerFile() {
echo 'FROM python
VOLUME /tmp
USER root
ADD requirements.txt ./requirements.txt
ADD Token.py ./Token.py
ADD register_dag.py ./register_dag.py
ADD bootstrapPython.sh ./bootstrapPython.sh
RUN chmod +x register_dag.py \
&& chmod +x Token.py \
&& chmod +x requirements.txt \
&& chmod +x bootstrapPython.sh \
&& pip install -r requirements.txt' > Dockerfile
}
echo 'msal' > requirements.txt
cat > Token.py << EOF
import os
import msal
class AzureToken(object):
def get_azure_id_token(self):
tenant_id = os.getenv('AZURE_DEPLOY_TENANT')
resource_id = os.getenv('AZURE_AD_APP_RESOURCE_ID')
client_id = os.getenv('INTEGRATION_TESTER')
client_secret = os.getenv('AZURE_TESTER_SERVICEPRINCIPAL_SECRET')
if tenant_id is None:
print('Please pass tenant Id to generate token')
exit(1)
if resource_id is None:
print('Please pass resource Id to generate token')
exit(1)
if client_id is None:
print('Please pass client Id to generate token')
exit(1)
if client_secret is None:
print('Please pass client secret to generate token')
exit(1)
try:
authority_host_uri = 'https://login.microsoftonline.com'
authority_uri = authority_host_uri + '/' + tenant_id
scopes = [resource_id + '/.default']
app = msal.ConfidentialClientApplication(client_id=client_id, authority=authority_uri, client_credential=client_secret)
result = app.acquire_token_for_client(scopes=scopes)
token = 'Bearer ' + result.get('access_token')
print(token)
return token
except Exception as e:
print(e)
if __name__ == '__main__':
AzureToken().get_azure_id_token()
EOF
cat > register_dag.py << EOF
import json
import requests
import os
import argparse
class RegisterDag:
def __init__(self):
self.token = os.environ.get('BEARER_TOKEN')
self.data_partition_id = os.environ.get('SHARED_TENANT')
self.workflow_service_url = os.environ.get('WORKFLOW_URL')
self.dag_name = f"Osdu_ingest"
print(f"data_partition_id: {self.data_partition_id}")
print(f"workflow_service_url: {self.workflow_service_url}")
print(f"dag_name: {self.dag_name}")
def register(self):
self.register_dag()
def register_dag(self):
data = json.dumps({
"description": "This is for manifest ingestion",
"registrationInstructions": {
"dagName": self.dag_name,
},
"workflowName": self.dag_name
})
headers = {
'Content-Type': 'application/json',
'Authorization': self.token,
'data-partition-id': self.data_partition_id
#'correlation-id': self.service_name
}
response = requests.post(self.workflow_service_url, headers=headers, data=data)
if response.status_code == 200:
workflow_id = response.json().get('workflowId')
print("DAG registered with workflowId: {0}".format(workflow_id))
elif response.status_code == 409:
workflow_id = response.json().get('workflowId')
print("DAG was already registered with workflowId: {0}".format(workflow_id))
else:
print("Error while registering DAG {0}".format(response.raise_for_status()))
exit(1)
if __name__ == "__main__":
RegisterDag().register()
EOF
echo '#!/usr/bin/env sh
###############################
## ARGUMENT INPUT ##
###############################
usage() { echo "Usage: bootstrapPython.sh"; }
printf "\n"
echo "------------------------------------";
echo "BOOTSTRAPPYTHON -- START";
echo "------------------------------------";
if [ -z $AZURE_DEPLOY_TENANT ]; then
echo 'ERROR: AZURE_DEPLOY_TENANT not provided'
usage;
elif [ -z $DNS_HOST ]; then
echo 'ERROR: DNS_HOST not provided'
usage;
elif [ -z $AZURE_TESTER_SERVICEPRINCIPAL_SECRET]; then
echo 'ERROR: AZURE_TESTER_SERVICEPRINCIPAL_SECRET not provided'
usage;
elif [ -z $INTEGRATION_TESTER ]; then
echo 'ERROR: INTEGRATION_TESTER not provided'
usage;
elif [ -z $AZURE_AD_APP_RESOURCE_ID ]; then
echo 'ERROR: AZURE_AD_APP_RESOURCE_ID not provided'
usage;
else
export AZURE_DNS_NAME=$DNS_HOST
export WORKFLOW_URL=https://${DNS_HOST}/api/workflow/v1/workflow
fi
if [ -z $DATA_PARTITION ]; then
SHARED_TENANT="opendes"
else
SHARED_TENANT=$DATA_PARTITION
fi
export SHARED_TENANT
echo "AZURE_DNS_NAME: $AZURE_DNS_NAME"
echo "WORKFLOW_URL: $WORKFLOW_URL"
echo "SHARED_TENANT: $SHARED_TENANT"
printf "\n"
echo "Execute Python Script Token.py";
echo "------------------------------------";
BEARER_TOKEN=$(python ./Token.py)
export BEARER_TOKEN
echo "$(echo $BEARER_TOKEN | cut -c -20)***********"
printf "\n"
echo "Execute Python Script register_dag.py";
echo "------------------------------------";
python ./register_dag.py
if test $? = '1'; then
printf "\n"
echo "------------------------------------";
echo "BOOTSTRAP -- ERROR";
echo "------------------------------------";
exit 1
else
printf "\n"
echo "------------------------------------";
echo "BOOTSTRAP -- FINISH";
echo "------------------------------------";
exit 0
fi' >bootstrapPython.sh
echo "out from writing docker file"
curl -L https://aka.ms/acr/installaad/bash | /bin/bash
echo "Logging in to the ACR Registry"
echo "------------------------------------"
az acr login -n $(CONTAINER_REGISTRY_NAME)
writeDockerFile
echo "Tagging and Pushing $IMAGE to ACR "
docker build -t python-image .
echo "Build done"
docker run --entrypoint ./bootstrapPython.sh -e "DNS_HOST=$(DNS_HOST)" -e "AZURE_DEPLOY_TENANT=$(AZURE_DEPLOY_TENANT)" -e "AZURE_TESTER_SERVICEPRINCIPAL_SECRET=$(AZURE_TESTER_SERVICEPRINCIPAL_SECRET)" -e "INTEGRATION_TESTER=$(INTEGRATION_TESTER)" -e "AZURE_AD_APP_RESOURCE_ID=$(AZURE_AD_APP_RESOURCE_ID)" python-image
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment