ci: migrate python image to MCR

deployments/scripts/azure/DAG.Dockerfile

+FROM mcr.microsoft.com/mirror/docker/library/python:3.8-slim
+
+ARG DATA_PARTITION=opendes
+
+ENV BUILD_PACKAGES \
+  python3-cryptography
+
+ENV PYTHONUNBUFFERED=1 \
+  PATH="/home/osdu/.local/bin:${PATH}" \
+  DATA_PARTITION=${DATA_PARTITION} \
+  DAG_DIRECTORY=../airflow/workflow-svc-v2 \
+  DAG_TASK_IMAGE=${DAG_TASK_IMAGE} \
+  AZURE_DEPLOYMENTS_SUBDIR=. \
+  AZURE_DEPLOYMENTS_SCRIPTS_SUBDIR=.
+
+USER root
+
+RUN set -x \
+  echo "**** install Python ****" && \
+  apt-get update && \
+  apt-get install -y -o APT::Keep-Downloaded-Packages=false ${BUILD_PACKAGES} &&\
+  rm -rf /var/cache/apt/archives/*
+
+# Install Requirements
+ADD deployments/scripts/azure/requirements.txt ./requirements.txt
+RUN pip install -r requirements.txt
+
+# Add debian repo to install busybox package
+RUN \
+  echo '@debian http://deb.debian.org/debian' >> /etc/apt/repositories && \
+  echo '@debian http://deb.debian.org/debian-security' >> /etc/apt/repositories
+
+RUN \
+  apt-get update && \
+  apt-get install busybox -y
+
+# Add Scripts
+WORKDIR /home/osdu
+ADD deployments/scripts/azure/*.py scripts/
+ADD deployments/scripts/azure/bootstrap.sh scripts/
+RUN chmod +x scripts/bootstrap.sh
+
+# Add DAGs
+ADD airflow airflow
+
+# Execute as non root user.
+RUN addgroup --system osdu && adduser --system osdu --ingroup osdu
+USER osdu
+WORKDIR /home/osdu/scripts
+
+ENTRYPOINT ["/home/osdu/scripts/bootstrap.sh"]
\ No newline at end of file

deployments/scripts/azure/Dockerfile

-FROM community.opengroup.org:5555/osdu/platform/deployment-and-operations/base-containers-azure/alpine-python3:0.0.2
+FROM mcr.microsoft.com/mirror/docker/library/python:3.8-slim
 
 ARG DATA_PARTITION=opendes
 
 ENV BUILD_PACKAGES \
-  py-cryptography
+  python3-cryptography
 
 ENV PYTHONUNBUFFERED=1 \
   PATH="/home/osdu/.local/bin:${PATH}" \
@@ -17,15 +17,25 @@ USER root
 
 RUN set -x \
   echo "**** install Python ****" && \
-  apk update && \
-  apk add --no-cache ${BUILD_PACKAGES} &&\
-  rm -rf /var/cache/apk/*
+  apt-get update && \
+  apt-get install -y -o APT::Keep-Downloaded-Packages=false ${BUILD_PACKAGES} &&\
+  rm -rf /var/cache/apt/archives/*
 
 # Install Requirements
 ADD deployments/scripts/azure/requirements.txt ./requirements.txt
 RUN pip install -r requirements.txt
 
+# Add debian repo to install busybox package
+RUN \
+  echo '@debian http://deb.debian.org/debian' >> /etc/apt/repositories && \
+  echo '@debian http://deb.debian.org/debian-security' >> /etc/apt/repositories
+
+RUN \
+  apt-get update && \
+  apt-get install busybox -y
+
 # Add Scripts
+WORKDIR /home/osdu
 ADD deployments/scripts/azure/*.py scripts/
 ADD deployments/scripts/azure/bootstrap.sh scripts/
 RUN chmod +x scripts/bootstrap.sh
@@ -34,7 +44,8 @@ RUN chmod +x scripts/bootstrap.sh
 ADD airflow airflow
 
 # Execute as non root user.
+RUN addgroup --system osdu && adduser --system osdu --ingroup osdu
 USER osdu
 WORKDIR /home/osdu/scripts
 
-ENTRYPOINT ["/home/osdu/scripts/bootstrap.sh"]
+ENTRYPOINT ["/home/osdu/scripts/bootstrap.sh"]
\ No newline at end of file

devops/azure/override-stages.yml

@@ -24,7 +24,7 @@ azure_build_dag:
   image: danielscholl/azure-build-image
   only: 
     variables: 
-      - "$AZURE == 'true'"
+      - "$AZURE == '1'"
   script: 
     - |
         cat > .env << EOF
@@ -39,7 +39,7 @@ azure_build_dag:
         BUILD_VERSION=$CI_COMMIT_SHA
         EOF
           
-        docker build -t $CI_REGISTRY_IMAGE/$DAG_TASK_IMAGE --file DAG.Dockerfile .
+        docker build -t $CI_REGISTRY_IMAGE/$DAG_TASK_IMAGE --file deployments/scripts/azure/DAG.Dockerfile .
         docker build -t $CI_REGISTRY_IMAGE/$DAG_LOAD_IMAGE --file deployments/scripts/azure/Dockerfile .
         docker push $CI_REGISTRY_IMAGE/$DAG_TASK_IMAGE
         docker push $CI_REGISTRY_IMAGE/$DAG_LOAD_IMAGE
@@ -76,7 +76,7 @@ azure_register_dag:
     - azure_build_dag
   only: 
     variables: 
-      - "$AZURE == 'true'"
+      - "$AZURE == '1'"
   script: 
     - |
         cat > requirements.txt << EOF
@@ -149,13 +149,28 @@ azure_register_dag:
             self.register_dag()
         
           def register_dag(self):
-            data = '{"workflowName":"$DAG_NAME","description":"$DAG_NAME","registrationInstructions":{"dagName":"$DAG_NAME"}}'
+            # To register DAG workflow name must match pattern ^[a-zA-Z0-9._-]{1,64}$
+            dag_name = '$DAG_NAME'
+            formatted_dag_name = dag_name[:63]
+
+            data = json.dumps(
+              {
+                "workflowName": formatted_dag_name,
+                "description": formatted_dag_name,
+                "registrationInstructions":
+                  {
+                    "dagName": formatted_dag_name
+                  }
+              })
+
             headers = {
                   'Content-Type': 'application/json',
                   'Authorization': self.token,
                   'data-partition-id': self.data_partition_id
                 }
 
+            print(f"Attempting to register DAG with name {formatted_dag_name}")
+
             response = requests.post(self.workflow_service_url, headers=headers, data=data)
             if response.status_code == 200:
               workflow_id = response.json().get('workflowId')
@@ -164,7 +179,8 @@ azure_register_dag:
               workflow_id = response.json().get('workflowId')
               print("DAG is already registered with workflowId: {0}".format(workflow_id))
             else:
-              print("Error while registering DAG {0}".format(response.raise_for_status()))
+              print(f"Error while registering DAG due to: {response.json().get('message')}")
+              print("Error: {0}".format(response.raise_for_status()))
          
         if __name__ == "__main__":
           RegisterDag().register()
@@ -197,7 +213,7 @@ azure_copy_dag:
     - azure_register_dag
   only: 
     variables: 
-      - "$AZURE == 'true'"
+      - "$AZURE == '1'"
   script: 
     - |
         # Installing the Azcopy utility