There is a security vulnerability in SSH key-generation using GitKraken < v8.0.1. If you used this tool to create SSH keys, please update GitKraken and regenerate. If you need help with this, contact forum-support@opengroup.org

Commit 105cc7a5 authored by Kishore Battula's avatar Kishore Battula
Browse files

Merge branch 'haaggarw/packaged_dag_support' into 'master'

Support for Packaged DAGs in Azure DAG pipeline

See merge request !283
parents 6d84568a cab986b9
Pipeline #52056 passed with stages
in 24 seconds
...@@ -135,48 +135,56 @@ azure_copy_dag: ...@@ -135,48 +135,56 @@ azure_copy_dag:
- cd output_dags - cd output_dags
- | - |
if [ -d "./dags/" ]; then if [ "$AZURE_DEPLOY_PACKAGED_DAG" == "true" ]; then
# Copying all the contents inside the dags folder echo "Packaged Dags are enabled"
azcopy cp "./dags/*" "https://${accountName}.file.core.windows.net/airflowdags/dags?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true if [ -d "./dags/" ]; then
cd dags # Copying the zipped dag inside the dags folder
azcopy cp "./dags/*.zip" "https://${accountName}.file.core.windows.net/airflowdags/dags?${AZURE_STORAGE_SAS_TOKEN}"
# Now syncing only the folders which are part of source to remove the deleted files fi
for directory in *; do else
if [ -d "$directory" ]; then echo "Packaged Dags are disabled"
azcopy sync "./$directory/" "https://${accountName}.file.core.windows.net/airflowdags/dags/$directory?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true --delete-destination=true if [ -d "./dags/" ]; then
# Copying all the contents inside the dags folder
azcopy cp "./dags/*" "https://${accountName}.file.core.windows.net/airflowdags/dags?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true
cd dags
# Now syncing only the folders which are part of source to remove the deleted files
for directory in *; do
if [ -d "$directory" ]; then
azcopy sync "./$directory/" "https://${accountName}.file.core.windows.net/airflowdags/dags/$directory?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true --delete-destination=true
fi
done
cd ..
fi fi
done
cd ..
fi
- | if [ -d "./plugins/" ]; then
if [ -d "./plugins/" ]; then # Copying all the contents inside the plugins folder
# Copying all the contents inside the plugins folder azcopy cp "./plugins/*" "https://${accountName}.file.core.windows.net/airflowdags/plugins?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true
azcopy cp "./plugins/*" "https://${accountName}.file.core.windows.net/airflowdags/plugins?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true cd plugins
cd plugins
if [ -d "operators" ]; then
if [ -d "operators" ]; then cd operators
cd operators # Syncing only the sub folders which are part of source in operators folder
# Syncing only the sub folders which are part of source in operators folder for directory in *; do
for directory in *; do if [ -d "$directory" ]; then
if [ -d "$directory" ]; then azcopy sync "./$directory/" "https://${accountName}.file.core.windows.net/airflowdags/plugins/operators/$directory?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true --delete-destination=true
azcopy sync "./$directory/" "https://${accountName}.file.core.windows.net/airflowdags/plugins/operators/$directory?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true --delete-destination=true fi
done
cd ..
fi fi
done
cd ..
fi
if [ -d "hooks" ]; then if [ -d "hooks" ]; then
cd hooks cd hooks
# Syncing only the sub folders which are part of source in hooks folder # Syncing only the sub folders which are part of source in hooks folder
for directory in *; do for directory in *; do
if [ -d "$directory" ]; then if [ -d "$directory" ]; then
azcopy sync "./$directory/" "https://${accountName}.file.core.windows.net/airflowdags/plugins/hooks/$directory?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true --delete-destination=true azcopy sync "./$directory/" "https://${accountName}.file.core.windows.net/airflowdags/plugins/hooks/$directory?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true --delete-destination=true
fi
done
cd ..
fi fi
done cd ..
cd .. fi
fi
cd ..
fi fi
artifacts: artifacts:
...@@ -316,11 +324,15 @@ azure_test_dag: ...@@ -316,11 +324,15 @@ azure_test_dag:
unit_service_endpoint: https://${AZURE_DNS_NAME}/api/unit/v2/unit/symbol unit_service_endpoint: https://${AZURE_DNS_NAME}/api/unit/v2/unit/symbol
legal_service_endpoint: https://${AZURE_DNS_NAME}/api/legal/v1 legal_service_endpoint: https://${AZURE_DNS_NAME}/api/legal/v1
file_service_endpoint: https://${AZURE_DNS_NAME}/api/file/v2 file_service_endpoint: https://${AZURE_DNS_NAME}/api/file/v2
DMS_KIND: opendes:osdudemo:wellbore:1.0.0 DMS_KIND: opendes:testcsv:wellbore:1.0.0
DMS_ACL: data.test1@opendes.contoso.com DMS_ACL: data.default.viewers@opendes.contoso.com
DMS_LEGAL_TAG: opendes-public-usa-dataset-7643990 DMS_LEGAL_TAG: opendes-public-usa-dataset-7643990
AZURE_STORAGE_CONTAINER: opendes
aad_client_id: $AZURE_APP_ID aad_client_id: $AZURE_APP_ID
AZURE_AD_TENANT_ID: $AZURE_TENANT_ID
AZURE_CLIENT_ID: $AZURE_PRINCIPAL_ID
AZURE_CLIENT_SECRET: $AZURE_PRINCIPAL_SECRET
AZURE_AD_APP_RESOURCE_ID: $AZURE_APP_ID
DATA_PARTITION_ID: opendes
extends: extends:
- .azure_variables - .azure_variables
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment