Commit 105cc7a5 authored by Kishore Battula's avatar Kishore Battula
Browse files

Merge branch 'haaggarw/packaged_dag_support' into 'master'

Support for Packaged DAGs in Azure DAG pipeline

See merge request !283
parents 6d84568a cab986b9
Pipeline #52056 passed with stages
in 24 seconds
......@@ -135,48 +135,56 @@ azure_copy_dag:
- cd output_dags
- |
if [ -d "./dags/" ]; then
# Copying all the contents inside the dags folder
azcopy cp "./dags/*" "https://${accountName}.file.core.windows.net/airflowdags/dags?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true
cd dags
# Now syncing only the folders which are part of source to remove the deleted files
for directory in *; do
if [ -d "$directory" ]; then
azcopy sync "./$directory/" "https://${accountName}.file.core.windows.net/airflowdags/dags/$directory?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true --delete-destination=true
if [ "$AZURE_DEPLOY_PACKAGED_DAG" == "true" ]; then
echo "Packaged Dags are enabled"
if [ -d "./dags/" ]; then
# Copying the zipped dag inside the dags folder
azcopy cp "./dags/*.zip" "https://${accountName}.file.core.windows.net/airflowdags/dags?${AZURE_STORAGE_SAS_TOKEN}"
fi
else
echo "Packaged Dags are disabled"
if [ -d "./dags/" ]; then
# Copying all the contents inside the dags folder
azcopy cp "./dags/*" "https://${accountName}.file.core.windows.net/airflowdags/dags?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true
cd dags
# Now syncing only the folders which are part of source to remove the deleted files
for directory in *; do
if [ -d "$directory" ]; then
azcopy sync "./$directory/" "https://${accountName}.file.core.windows.net/airflowdags/dags/$directory?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true --delete-destination=true
fi
done
cd ..
fi
done
cd ..
fi
- |
if [ -d "./plugins/" ]; then
# Copying all the contents inside the plugins folder
azcopy cp "./plugins/*" "https://${accountName}.file.core.windows.net/airflowdags/plugins?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true
cd plugins
if [ -d "operators" ]; then
cd operators
# Syncing only the sub folders which are part of source in operators folder
for directory in *; do
if [ -d "$directory" ]; then
azcopy sync "./$directory/" "https://${accountName}.file.core.windows.net/airflowdags/plugins/operators/$directory?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true --delete-destination=true
if [ -d "./plugins/" ]; then
# Copying all the contents inside the plugins folder
azcopy cp "./plugins/*" "https://${accountName}.file.core.windows.net/airflowdags/plugins?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true
cd plugins
if [ -d "operators" ]; then
cd operators
# Syncing only the sub folders which are part of source in operators folder
for directory in *; do
if [ -d "$directory" ]; then
azcopy sync "./$directory/" "https://${accountName}.file.core.windows.net/airflowdags/plugins/operators/$directory?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true --delete-destination=true
fi
done
cd ..
fi
done
cd ..
fi
if [ -d "hooks" ]; then
cd hooks
# Syncing only the sub folders which are part of source in hooks folder
for directory in *; do
if [ -d "$directory" ]; then
azcopy sync "./$directory/" "https://${accountName}.file.core.windows.net/airflowdags/plugins/hooks/$directory?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true --delete-destination=true
if [ -d "hooks" ]; then
cd hooks
# Syncing only the sub folders which are part of source in hooks folder
for directory in *; do
if [ -d "$directory" ]; then
azcopy sync "./$directory/" "https://${accountName}.file.core.windows.net/airflowdags/plugins/hooks/$directory?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true --delete-destination=true
fi
done
cd ..
fi
done
cd ..
fi
cd ..
cd ..
fi
fi
artifacts:
......@@ -316,11 +324,15 @@ azure_test_dag:
unit_service_endpoint: https://${AZURE_DNS_NAME}/api/unit/v2/unit/symbol
legal_service_endpoint: https://${AZURE_DNS_NAME}/api/legal/v1
file_service_endpoint: https://${AZURE_DNS_NAME}/api/file/v2
DMS_KIND: opendes:osdudemo:wellbore:1.0.0
DMS_ACL: data.test1@opendes.contoso.com
DMS_KIND: opendes:testcsv:wellbore:1.0.0
DMS_ACL: data.default.viewers@opendes.contoso.com
DMS_LEGAL_TAG: opendes-public-usa-dataset-7643990
AZURE_STORAGE_CONTAINER: opendes
aad_client_id: $AZURE_APP_ID
AZURE_AD_TENANT_ID: $AZURE_TENANT_ID
AZURE_CLIENT_ID: $AZURE_PRINCIPAL_ID
AZURE_CLIENT_SECRET: $AZURE_PRINCIPAL_SECRET
AZURE_AD_APP_RESOURCE_ID: $AZURE_APP_ID
DATA_PARTITION_ID: opendes
extends:
- .azure_variables
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment