Commit cab986b9 authored by harshit aggarwal's avatar harshit aggarwal Committed by Kishore Battula
Browse files

Support for Packaged DAGs in Azure DAG pipeline

parent 6d84568a
...@@ -135,6 +135,14 @@ azure_copy_dag: ...@@ -135,6 +135,14 @@ azure_copy_dag:
- cd output_dags - cd output_dags
- | - |
if [ "$AZURE_DEPLOY_PACKAGED_DAG" == "true" ]; then
echo "Packaged Dags are enabled"
if [ -d "./dags/" ]; then
# Copying the zipped dag inside the dags folder
azcopy cp "./dags/*.zip" "https://${accountName}.file.core.windows.net/airflowdags/dags?${AZURE_STORAGE_SAS_TOKEN}"
fi
else
echo "Packaged Dags are disabled"
if [ -d "./dags/" ]; then if [ -d "./dags/" ]; then
# Copying all the contents inside the dags folder # Copying all the contents inside the dags folder
azcopy cp "./dags/*" "https://${accountName}.file.core.windows.net/airflowdags/dags?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true azcopy cp "./dags/*" "https://${accountName}.file.core.windows.net/airflowdags/dags?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true
...@@ -149,7 +157,6 @@ azure_copy_dag: ...@@ -149,7 +157,6 @@ azure_copy_dag:
cd .. cd ..
fi fi
- |
if [ -d "./plugins/" ]; then if [ -d "./plugins/" ]; then
# Copying all the contents inside the plugins folder # Copying all the contents inside the plugins folder
azcopy cp "./plugins/*" "https://${accountName}.file.core.windows.net/airflowdags/plugins?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true azcopy cp "./plugins/*" "https://${accountName}.file.core.windows.net/airflowdags/plugins?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true
...@@ -178,6 +185,7 @@ azure_copy_dag: ...@@ -178,6 +185,7 @@ azure_copy_dag:
fi fi
cd .. cd ..
fi fi
fi
artifacts: artifacts:
paths: paths:
...@@ -316,11 +324,15 @@ azure_test_dag: ...@@ -316,11 +324,15 @@ azure_test_dag:
unit_service_endpoint: https://${AZURE_DNS_NAME}/api/unit/v2/unit/symbol unit_service_endpoint: https://${AZURE_DNS_NAME}/api/unit/v2/unit/symbol
legal_service_endpoint: https://${AZURE_DNS_NAME}/api/legal/v1 legal_service_endpoint: https://${AZURE_DNS_NAME}/api/legal/v1
file_service_endpoint: https://${AZURE_DNS_NAME}/api/file/v2 file_service_endpoint: https://${AZURE_DNS_NAME}/api/file/v2
DMS_KIND: opendes:osdudemo:wellbore:1.0.0 DMS_KIND: opendes:testcsv:wellbore:1.0.0
DMS_ACL: data.test1@opendes.contoso.com DMS_ACL: data.default.viewers@opendes.contoso.com
DMS_LEGAL_TAG: opendes-public-usa-dataset-7643990 DMS_LEGAL_TAG: opendes-public-usa-dataset-7643990
AZURE_STORAGE_CONTAINER: opendes
aad_client_id: $AZURE_APP_ID aad_client_id: $AZURE_APP_ID
AZURE_AD_TENANT_ID: $AZURE_TENANT_ID
AZURE_CLIENT_ID: $AZURE_PRINCIPAL_ID
AZURE_CLIENT_SECRET: $AZURE_PRINCIPAL_SECRET
AZURE_AD_APP_RESOURCE_ID: $AZURE_APP_ID
DATA_PARTITION_ID: opendes
extends: extends:
- .azure_variables - .azure_variables
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment