Commit 1d7713ab authored by preeti singh[Microsoft]'s avatar preeti singh[Microsoft]
Browse files

Delete azure-manifest-ingestion.yml

parent 609ac274
Pipeline #42738 passed with stage
in 15 seconds
# EXPECTED PIPELINE INHERITED GROUP VARIABLES
# --------------------------------------------------------------------------------
# AZURE (Protected Branch)
# AZURE_APP_ID (Protected Branch)
# AZURE_APP_ID_OTHER (Protected Branch)
# AZURE_APP_OID_OTHER (Protected Branch)
# AZURE_BASE (Protected Branch)
# AZURE_BASENAME_21 (Protected Branch)
# AZURE_DNS_NAME (Protected Branch)
# AZURE_ELASTIC_HOST (Protected Branch)
# AZURE_ELASTIC_PASSWORD (Protected Branch/Masked Variable)
# AZURE_INVALID_JWT (Protected Branch)
# AZURE_NO_ACCESS_ID (Protected Branch)
# AZURE_NO_ACCESS_SECRET (Protected Branch/Masked Variable)
# AZURE_PRINCIPAL_ID (Protected Branch)
# AZURE_PRINCIPAL_SECRET (Protected Branch/Masked Variable)
# AZURE_REGISTRY (Protected Branch)
# AZURE_SERVICEBUS_KEY (Protected Branch/Masked Variable)
# AZURE_STORAGE_KEY (Protected Branch/Masked Variable)
# AZURE_SUBSCRIPTION_ID (Protected Branch)
# AZURE_SUBSCRIPTION_NAME (Protected Branch)
# AZURE_TENANT_ID (Protected Branch)
# AZURE_APPINSIGHTS_KEY (Protected Branch/Masked Variable)
# EXPECTED PIPELINE VARIABLES
# --------------------------------------------------------------------------------
# AZURE_TEST_SUBDIR
.azure_variables:
variables:
WORKFLOW_URL: https://${AZURE_DNS_NAME}/api/workflow/v1/
AZURE_AD_TENANT_ID: $AZURE_TENANT_ID
AZURE_CLIENT_ID: $AZURE_PRINCIPAL_ID
AZURE_CLIENT_SECRET: $AZURE_PRINCIPAL_SECRET
AZURE_AD_APP_RESOURCE_ID: $AZURE_APP_ID
SHARED_TENANT: opendes
DATA_PARTITION_ID: opendes
# JOBS
# --------------------------------------------------------------------------------
azure_create_dag:
tags: ["osdu-medium"]
image: danielscholl/azure-build-image
stage: createDag
extends:
- .azure_variables
script:
- |
mkdir output_dags
cp ./src/dags output_dags
cp ./src/plugins output_dags
artifacts:
paths:
- "output_dags"
only:
variables:
- $AZURE == 'true'
azure_copy_dag:
image: danielscholl/azure-build-image
tags: ["osdu-medium"]
stage: copyDag
needs: ["azure_create_dag"]
extends:
- .azure_variables
before_script:
- az login --service-principal -u $AZURE_PRINCIPAL_ID -p $AZURE_PRINCIPAL_SECRET --tenant $AZURE_TENANT_ID
- az aks get-credentials -g $AZURE_UNIQUE-rg -n $AZURE_UNIQUE-aks
script:
- |
apk add --update coreutils && rm -rf /var/cache/apk/*
mkdir -p tmp
cd tmp
wget -O azcopy_v10.tar.gz https://aka.ms/downloadazcopy-v10-linux && tar -xf azcopy_v10.tar.gz --strip-components=1
cp ./azcopy /usr/bin/
cd ..
- EXPIRE=$(date -u -d "9 minutes" '+%Y-%m-%dT%H:%M:%SZ')
- START=$(date -u -d "-1 minute" '+%Y-%m-%dT%H:%M:%SZ')
- accountKey=$(kubectl get secret airflow -n osdu -o jsonpath='{.data.azurestorageaccountkey}' | base64 -d)
- accountName=$(kubectl get secret airflow -n osdu -o jsonpath='{.data.azurestorageaccountname}' | base64 -d)
- AZURE_STORAGE_SAS_TOKEN=$(az storage account generate-sas --account-name $accountName --account-key $accountKey --start $START --expiry $EXPIRE --https-only --resource-types sco --services f --permissions cwdlur -o tsv)
- cd output_dags
- |
if [ -d "./dags/" ]; then
azcopy cp "./dags/*.py" "https://${accountName}.file.core.windows.net/airflowdags/dags?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true
cd dags
for directory in *; do
if [ -d "$directory" ]; then
azcopy cp "./$directory/" "https://${accountName}.file.core.windows.net/airflowdags/dags?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true
azcopy sync "./$directory/" "https://${accountName}.file.core.windows.net/airflowdags/dags/$directory?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true --delete-destination=true
fi
done
cd ..
fi
- |
if [ -d "./plugins/" ]; then
cd plugins/operators
for directory in *; do
if [ -d "$directory" ]; then
azcopy cp "./$directory/" "https://${accountName}.file.core.windows.net/airflowdags/plugins?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true
azcopy sync "./$directory/" "https://${accountName}.file.core.windows.net/airflowdags/plugins/$directory?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true --delete-destination=true
fi
done
fi
only:
variables:
- $AZURE == 'true'
azure_register_dag:
tags: ["osdu-medium"]
image: python:3.8
stage: registerDag
needs: ["azure_copy_dag"]
extends:
- .azure_variables
script:
- |
cat > requirements.txt << EOF
msal
EOF
- |
cat > Token.py << EOF
import os
import msal
class AzureToken(object):
def get_azure_id_token(self):
tenant_id = os.getenv('AZURE_TENANT_ID')
resource_id = os.getenv('AZURE_AD_APP_RESOURCE_ID')
client_id = os.getenv('AZURE_CLIENT_ID')
client_secret = os.getenv('AZURE_CLIENT_SECRET')
if tenant_id is None:
print('Please pass tenant Id to generate token')
exit(1)
if resource_id is None:
print('Please pass resource Id to generate token')
exit(1)
if client_id is None:
print('Please pass client Id to generate token')
exit(1)
if client_secret is None:
print('Please pass client secret to generate token')
exit(1)
try:
authority_host_uri = 'https://login.microsoftonline.com'
authority_uri = authority_host_uri + '/' + tenant_id
scope = [resource_id + '/.default']
app = msal.ConfidentialClientApplication(client_id=client_id, authority=authority_uri, client_credential=client_secret)
result = app.acquire_token_for_client(scopes=scope)
token = 'Bearer ' + result.get('access_token')
print(token)
return token
except Exception as e:
print(e)
if __name__ == '__main__':
AzureToken().get_azure_id_token()
EOF
- |
cat > register_dag.py << EOF
import json
import requests
import os
class RegisterDag:
def __init__(self):
self.token = os.environ.get('BEARER_TOKEN')
self.data_partition_id = os.environ.get('SHARED_TENANT')
self.workflow_service_url = os.environ.get('WORKFLOW_URL') + "workflow"
self.dag_name = f"Osdu_ingest"
def register(self):
self.register_dag()
def register_dag(self):
data = json.dumps({
"description": self.service_name,
"registrationInstructions": {
"dagName": self.dag_name,
},
"workflowName": self.dag_name
})
headers = {
'Content-Type': 'application/json',
'Authorization': self.token,
'data-partition-id': self.data_partition_id,
}
response = requests.post(self.workflow_service_url, headers=headers, data=data)
if response.status_code == 200:
workflow_id = response.json().get('workflowId')
print("DAG registered with workflowId: {0}".format(workflow_id))
elif response.status_code == 409:
workflow_id = response.json().get('workflowId')
print("DAG is already registered with workflowId: {0}".format(self.dag_name))
else:
print("Error while registering DAG {0}".format(response.raise_for_status()))
exit(1)
if __name__ == "__main__":
RegisterDag().register()
EOF
- pip install -r ./requirements.txt
- export BEARER_TOKEN=$(python ./Token.py)
- echo $BEARER_TOKEN
- python ./register_dag.py
only:
variables:
- $AZURE == 'true'
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment