pipeline.yml

#This pipeline is for demo env
# Docker 
# Build a Docker image 
# https://docs.microsoft.com/azure/devops/pipelines/languages/docker 
trigger: 
  - master 
resources: 
  - repo: self 
#I have included dev environment specific libraries because it is intended to run on demo env .
variables:
  - group: 'Azure - OSDU'
  - group: 'Azure - Common'
  - group: 'Azure Target Env - demo'
  - group: 'Azure Target Env Secrets - demo'

steps: 
    
    - task: AzureCLI@1 
      displayName: 'Running Unit test' 
       
      inputs: 
        azureSubscription: $(SERVICE_CONNECTION_NAME)
        addSpnToEnvironment: true 
        scriptLocation: inlineScript 
        inlineScript: | 
          function writeDockerFile() {
            echo 'FROM apache/airflow
            VOLUME /tmp
            USER root
            ADD bootstrap_ut.sh ./bootstrap_ut.sh
            RUN chmod +x bootstrap_ut.sh
            ADD tests ./tests
            ADD src/ ./src
            ADD .pylintrc ./.pylintrc
            RUN chmod +x tests/set_airflow_env.sh
            RUN chmod +x tests/unit_tests.sh' > Dockerfile            
          }
          echo "out from writing docker file block" 
          echo '#!/bin/bash
          export PATH=/root/.local/bin:$PATH
          pip install --upgrade pip
          pip install apache-airflow
          pip install Flask-Bcrypt
          pip install apache-airflow[statsd]
          pip install apache-airflow[kubernetes]
          pip install apache-airflow-backport-providers-microsoft-azure
          pip install https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_api-0.0.4.tar.gz
          #ignoring this test for now as it belongs to r2.We need to removed all the tests related to r2 as its deprecated.
          rm  tests/plugin-unit-tests/test_process_manifest_r2.py
          tests/./unit_tests.sh  || EXIT_CODE=$?
          echo $EXIT_CODE
          exit ${EXIT_CODE}' > bootstrap_ut.sh
          writeDockerFile
          docker build -t python-ut .
          echo "Build done"
          docker run --entrypoint ./bootstrap_ut.sh python-ut

    
    - task: AzureCLI@1 
      displayName: 'Deploy DAGs' 
       
      inputs: 
        azureSubscription: $(SERVICE_CONNECTION_NAME)
        addSpnToEnvironment: true 
        scriptLocation: inlineScript 
        inlineScript: | 
          function writeDockerFile() { 
            echo 'FROM mcr.microsoft.com/azure-cli 
            VOLUME /tmp 
            USER root 
            ADD /src/dags ./dags
            ADD /src/plugins ./plugins
            ADD bootstrap.sh ./bootstrap.sh
            RUN chmod +x bootstrap.sh \
            && apk add --update coreutils && rm -rf /var/cache/apk/* \
            && echo "Setup AzCopy.." \
            && mkdir -p tmp \
            && cd tmp \
            && wget -O azcopy_v10.tar.gz https://aka.ms/downloadazcopy-v10-linux && tar -xf azcopy_v10.tar.gz --strip-components=1 \
            && cp ./azcopy /usr/bin/ \
            && cd ..' > Dockerfile
          } 
          function writeDockerRunCommands(){
            echo '#!/bin/bash
            EXPIRE=$(date -u -d "9 minutes" '+%Y-%m-%dT%H:%M:%SZ')
            START=$(date -u -d "-1 minute" '+%Y-%m-%dT%H:%M:%SZ')

            AZURE_STORAGE_SAS_TOKEN=$(az storage account generate-sas --account-name $env_storage  --account-key $env_storage_key --start $START --expiry $EXPIRE --https-only --resource-types sco --services bfqt --permissions dlrcuaw -o tsv )
            azcopy cp "/plugins/*" "https://${env_storage}.file.core.windows.net/airflowdags/plugins/?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true
            azcopy cp "/dags/*" "https://${env_storage}.file.core.windows.net/airflowdags/dags/?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true
            echo "copied files and folders for dags and plugins***"

            if [ -d "./dags/" ]; then
              cd dags
              for directory in *; do  
                if [ -d "$directory" ]; then
                  echo "syncing $directory"
                  azcopy sync "./$directory" "https://${env_storage}.file.core.windows.net/airflowdags/dags/$directory?${AZURE_STORAGE_SAS_TOKEN}"     --recursive=true --delete-destination=true
                fi
                echo "$directory sync done"
              done
              cd ..
            fi
            
            echo "synced files and folders for dags"
            pwd
     
            cd plugins/operators
            pwd
       
            for directory in *; do
              if [ -d "$directory" ]; then
                azcopy sync "./$directory/" "https://${env_storage}.file.core.windows.net/airflowdags/plugins/operators/$directory?${AZURE_STORAGE_SAS_TOKEN}" --recursive=true --delete-destination=true
              fi
              echo "$directory sync done"
            done
            echo "synced files and folders for plugins/operators"' > bootstrap.sh
          }
              echo "out from writing docker file block" 
              
              writeDockerRunCommands
              writeDockerFile 
              docker build -t azcli-image . 
              echo "Build done" 
              docker run --entrypoint ./bootstrap.sh -e "env_storage_key=$(AIRFLOW_STORAGE_ACCOUNT_KEY)" -e "env_storage=$(AIRFLOW_STORAGE_ACCOUNT)" azcli-image
    - task: AzureCLI@1 
      displayName: 'Docker Build + generate token and register dag' 
       
      inputs: 
        azureSubscription: $(SERVICE_CONNECTION_NAME)
        addSpnToEnvironment: true 
        scriptLocation: inlineScript 
        inlineScript: | 
          function writeDockerFile() { 
            echo 'FROM python 
            VOLUME /tmp 
            USER root 
            ADD requirements.txt ./requirements.txt
            ADD Token.py ./Token.py
            ADD register_dag.py ./register_dag.py
            ADD bootstrapPython.sh ./bootstrapPython.sh
            RUN chmod +x register_dag.py \
            && chmod +x Token.py \
            && chmod +x requirements.txt \
            && chmod +x bootstrapPython.sh \
            && pip install -r requirements.txt' > Dockerfile
          }
          echo 'msal' > requirements.txt
          cat > Token.py << EOF
          import os
          import msal

          class AzureToken(object):

            def get_azure_id_token(self):
              tenant_id = os.getenv('AZURE_DEPLOY_TENANT')
              resource_id = os.getenv('AZURE_AD_APP_RESOURCE_ID')
              client_id = os.getenv('INTEGRATION_TESTER')
              client_secret = os.getenv('AZURE_TESTER_SERVICEPRINCIPAL_SECRET')

              if tenant_id is None:
                  print('Please pass tenant Id to generate token')
                  exit(1)
              if resource_id is None:
                  print('Please pass resource Id to generate token')
                  exit(1)
              if client_id is None:
                  print('Please pass client Id to generate token')
                  exit(1)
              if client_secret is None:
                  print('Please pass client secret to generate token')
                  exit(1)

              try:
                  authority_host_uri = 'https://login.microsoftonline.com'
                  authority_uri = authority_host_uri + '/' + tenant_id
                  scopes = [resource_id + '/.default']
                  app = msal.ConfidentialClientApplication(client_id=client_id, authority=authority_uri, client_credential=client_secret)
                  result = app.acquire_token_for_client(scopes=scopes)
                  token = 'Bearer ' +  result.get('access_token')
                  print(token)
                  return token
              except Exception as e:
                  print(e)

          if __name__ == '__main__':
            AzureToken().get_azure_id_token()
          EOF

          cat > register_dag.py << EOF

          import json
          import requests
          import os
          import argparse


          class RegisterDag:

              def __init__(self):
                  self.token = os.environ.get('BEARER_TOKEN')
                  self.data_partition_id = os.environ.get('SHARED_TENANT')
                  self.workflow_service_url = os.environ.get('WORKFLOW_URL')
                  self.dag_name = f"Osdu_ingest"

                  print(f"data_partition_id: {self.data_partition_id}")
                  print(f"workflow_service_url: {self.workflow_service_url}")
                  print(f"dag_name: {self.dag_name}")
                  

              def register(self):
                  self.register_dag()


              def register_dag(self):
                  data = json.dumps({
                        "description": "This is for manifest ingestion",
                        "registrationInstructions": {
                          "dagName": self.dag_name,
                        },
                        "workflowName": self.dag_name
                      })

                  headers = {
                      'Content-Type': 'application/json',
                      'Authorization': self.token,
                      'data-partition-id': self.data_partition_id
                      #'correlation-id': self.service_name
                      
                  }

                  response = requests.post(self.workflow_service_url, headers=headers, data=data)
                  if response.status_code == 200:
                      workflow_id = response.json().get('workflowId')
                      print("DAG registered with workflowId: {0}".format(workflow_id))
                  
                  elif response.status_code == 409:
                    workflow_id = response.json().get('workflowId')
                    print("DAG was already registered with workflowId: {0}".format(workflow_id))
                  
                  else:
                      print("Error while registering DAG {0}".format(response.raise_for_status()))
                      exit(1)


          if __name__ == "__main__":
              RegisterDag().register()

          EOF

              echo '#!/usr/bin/env sh
              ###############################
              ## ARGUMENT INPUT            ##
              ###############################
              usage() { echo "Usage: bootstrapPython.sh"; }

              printf "\n"
              echo "------------------------------------";
              echo "BOOTSTRAPPYTHON -- START";
              echo "------------------------------------";

              if [ -z $AZURE_DEPLOY_TENANT ]; then
                echo 'ERROR: AZURE_DEPLOY_TENANT not provided'
                usage;
              elif [ -z $DNS_HOST ]; then
                echo 'ERROR: DNS_HOST not provided'
                usage;
              elif [ -z $AZURE_TESTER_SERVICEPRINCIPAL_SECRET]; then
                echo 'ERROR: AZURE_TESTER_SERVICEPRINCIPAL_SECRET not provided'
                usage;
              elif [ -z $INTEGRATION_TESTER ]; then
                echo 'ERROR: INTEGRATION_TESTER not provided'
                usage;
              elif [ -z $AZURE_AD_APP_RESOURCE_ID ]; then
                echo 'ERROR: AZURE_AD_APP_RESOURCE_ID not provided'
                usage;
              else
                export AZURE_DNS_NAME=$DNS_HOST
                export WORKFLOW_URL=https://${DNS_HOST}/api/workflow/v1/workflow
              fi

              if [ -z $DATA_PARTITION ]; then
                SHARED_TENANT="opendes"
              else
                SHARED_TENANT=$DATA_PARTITION
              fi
              export SHARED_TENANT

              echo "AZURE_DNS_NAME: $AZURE_DNS_NAME"
              echo "WORKFLOW_URL: $WORKFLOW_URL"
              echo "SHARED_TENANT: $SHARED_TENANT"

              printf "\n"
              echo "Execute Python Script Token.py";
              echo "------------------------------------";

              BEARER_TOKEN=$(python ./Token.py)
              export BEARER_TOKEN

              echo "$(echo $BEARER_TOKEN | cut -c -20)***********"

              printf "\n"
              echo "Execute Python Script register_dag.py";
              echo "------------------------------------";

              python ./register_dag.py 

              if test $? = '1'; then
              printf "\n"
                echo "------------------------------------";
                echo "BOOTSTRAP -- ERROR";
                echo "------------------------------------";
                exit 1
              else
                printf "\n"
                echo "------------------------------------";
                echo "BOOTSTRAP -- FINISH";
                echo "------------------------------------";
                exit 0
              fi' >bootstrapPython.sh

              echo "out from writing docker file" 
              writeDockerFile
 
              docker build -t python-image . 
              echo "Build done" 
              docker run --entrypoint ./bootstrapPython.sh -e "DNS_HOST=$(DNS_HOST)" -e "AZURE_DEPLOY_TENANT=$(AZURE_DEPLOY_TENANT)" -e "AZURE_TESTER_SERVICEPRINCIPAL_SECRET=$(AZURE_TESTER_SERVICEPRINCIPAL_SECRET)" -e "INTEGRATION_TESTER=$(INTEGRATION_TESTER)" -e "AZURE_AD_APP_RESOURCE_ID=$(AZURE_AD_APP_RESOURCE_ID)" python-image