Merge branch 'fix/upgrade-azure-pipeline' into 'master'

feat: upgrade base azure pipeline and added m13 support

See merge request !90

.gitlab-ci.yml

@@ -92,55 +92,51 @@ build_parser:
     - |
       export EXTRA_GCP_DOCKER_TAG="";
       export EXTRA_ANTHOS_DOCKER_TAG="";
-      export EXTRA_AZURE_DOCKER_TAG="";
 
       if [ "$CI_COMMIT_TAG" != "" ]; then
         EXTRA_GCP_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/osdu-gcp-$OSDU_GCP_SERVICE:$CI_COMMIT_TAG";
         EXTRA_ANTHOS_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/osdu-gcp-anthos-$OSDU_GCP_SERVICE:$CI_COMMIT_TAG";
-        EXTRA_AZURE_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/$OSDU_AZURE_POSTFIX:$CI_COMMIT_TAG";
       elif [ "$CI_COMMIT_REF_NAME" = "master" ]; then
         EXTRA_GCP_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/osdu-gcp-$OSDU_GCP_SERVICE:latest";
         EXTRA_ANTHOS_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/osdu-gcp-anthos-$OSDU_GCP_SERVICE:latest";
-        EXTRA_AZURE_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/$OSDU_AZURE_POSTFIX:latest";
       fi
     - docker build --target gcp -t $CI_REGISTRY_IMAGE/osdu-gcp-$OSDU_GCP_SERVICE:$CI_COMMIT_SHORT_SHA $EXTRA_GCP_DOCKER_TAG --rm -f ./build/Dockerfile .
     - docker build --target anthos -t $CI_REGISTRY_IMAGE/osdu-gcp-anthos-$OSDU_GCP_SERVICE:$CI_COMMIT_SHORT_SHA $EXTRA_ANTHOS_DOCKER_TAG --rm -f ./build/Dockerfile .
-    - docker build --target azure -t $OSDU_AZURE_BASE_IMAGE_TAG $EXTRA_AZURE_DOCKER_TAG --rm -f ./build/Dockerfile .
     - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
     - docker push $CI_REGISTRY_IMAGE/osdu-gcp-$OSDU_GCP_SERVICE
     - docker push $CI_REGISTRY_IMAGE/osdu-gcp-anthos-$OSDU_GCP_SERVICE
-    - docker push $CI_REGISTRY_IMAGE/$OSDU_AZURE_POSTFIX
 
 build_parser_azure:
   stage: build
-  tags: [ "osdu-small" ]
+  needs: []
+  tags: [ "osdu-medium" ]
   image: danielscholl/azure-build-image
   variables:
     SHA_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:${CI_COMMIT_SHA}
     LATEST_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:latest
     RELEASE_IMAGE: release-${CI_COMMIT_TAG}:${CI_PROJECT_NAME}-${CI_COMMIT_TAG}
-
   before_script:
     - az login --service-principal -u $AZURE_PRINCIPAL_ID -p $AZURE_PRINCIPAL_SECRET --tenant $AZURE_TENANT_ID
     - az acr login -n $AZURE_REGISTRY
   script:
     - |
       if [ "$CI_COMMIT_TAG" != "" ]; then
-        docker build --target azure -t ${AZURE_REGISTRY}.azurecr.io/$RELEASE_IMAGE --rm -f ./build/Dockerfile .
+        docker build --target azure -t ${AZURE_REGISTRY}.azurecr.io/$RELEASE_IMAGE --rm -f ./build/providers/azure/Dockerfile .
         docker push ${AZURE_REGISTRY}.azurecr.io/$RELEASE_IMAGE
       elif [ "$CI_COMMIT_REF_NAME" = "master" ]; then
-        docker build --target azure -t ${AZURE_REGISTRY}.azurecr.io/$LATEST_IMAGE --rm -f ./build/Dockerfile .
+        docker build --target azure -t ${AZURE_REGISTRY}.azurecr.io/$LATEST_IMAGE --rm -f ./build/providers/azure/Dockerfile .
         docker push ${AZURE_REGISTRY}.azurecr.io/$LATEST_IMAGE
       fi
-    - docker build --target azure -t ${AZURE_REGISTRY}.azurecr.io/$SHA_IMAGE --rm -f ./build/Dockerfile .
+
+    - docker build --target azure -t ${AZURE_REGISTRY}.azurecr.io/$SHA_IMAGE --rm -f ./build/providers/azure/Dockerfile .
     - docker push ${AZURE_REGISTRY}.azurecr.io/$SHA_IMAGE
   only:
     variables:
-      - $AZURE == 'true'
+      - $AZURE == '1'
 
 azure_standalone_tests:
   stage: test
-  needs: [ "test_parser" ]
+  needs: []
   variables:
     AZURE_SKIP_STANDALONE_TESTS: "true"
 
@@ -175,17 +171,3 @@ azure_build_dag:
       DATA_PARTITION=$DATA_PARTITION
       OSDU_AZURE_BASE_URL=$OSDU_AZURE_BASE_URL
       EOF
-
-azure_dag_postman_tests:
-  before_script:
-    - |
-      # Generating environment file to be passed while running the postman tests
-      cat > python_env_vars.txt << EOF
-      TENANT_ID os.environ.get('AZURE_TENANT_ID')
-      CLIENT_ID os.environ.get('CLIENT_ID')
-      CLIENT_SECRET os.environ.get('CLIENT_SECRET')
-      HOSTNAME os.environ.get('AZURE_DNS_NAME')
-      refresh_token os.environ.get('AZURE_REFRESH_TOKEN')
-      WORKFLOW_HOST os.environ.get('AZURE_DNS_NAME')
-      witsmlFilePath os.environ.get('AZURE_TEST_WITSML_FILE_PATH')
-      EOF

build/Dockerfile

@@ -57,15 +57,6 @@ COPY "build/providers/${PROVIDER_NAME}/requirements.txt" ./
 RUN pip install -r "requirements.txt"
 COPY build/providers/gcp/osdu_api.ini ./
 
-## azure
-FROM base as azure
-
-ENV CLOUD_PROVIDER azure
-
-COPY build/providers/azure/requirements.txt ./
-COPY build/providers/azure/osdu_api.ini ./
-RUN pip install -r requirements.txt
-
 ## anthos
 FROM base as anthos
 

build/providers/azure/Dockerfile

+FROM mcr.microsoft.com/mirror/docker/library/python:3.8-slim as base
+
+ENV HOME="/home/witsml_parser"
+
+WORKDIR $HOME
+
+ENV BUILD_DIR="build"
+ENV WITSML_PARSER_DIR="energistics/src/witsml_parser"
+ENV ENERGISTICS_LIBS="${WITSML_PARSER_DIR}/energistics"
+ENV OSDU_CORE_LIBS_DIR="${WITSML_PARSER_DIR}/osdu/ingestion-dags/src/dags/libs"
+ENV OSDU_CORE_PROVIDERS_DIR="${WITSML_PARSER_DIR}/osdu/ingestion-dags/src/dags/providers"
+
+COPY ./${BUILD_DIR}/requirements.txt ./
+COPY ./${WITSML_PARSER_DIR}/*.py ./
+COPY ./${ENERGISTICS_LIBS} ./energistics
+COPY ./${BUILD_DIR}/providers/gcp/osdu_api.ini ./
+
+RUN pip install --upgrade pip && \
+    pip install -r "./energistics/requirements.txt" && \
+    pip install -r "./requirements.txt"
+
+ENV PYTHONPATH="$PYTHONPATH:$HOME"
+
+FROM base as azure
+
+ENV CLOUD_PROVIDER azure
+
+COPY build/providers/azure/requirements.txt ./
+COPY build/providers/azure/osdu_api.ini ./
+RUN pip install -r requirements.txt

deployments/scripts/azure/dag_bootstrap/Dockerfile

 FROM msosdu.azurecr.io/data-seed-base:v1
 
-#Environment Variables while running the docker(These are not required to be set in build pipelines)
+# Environment Variables while running the docker(These are not required to be set in build pipelines)
 
 # AZURE_TENANT_ID
 # AZURE_CLIENT_ID

deployments/scripts/azure/dockerFolder/output_dags_dockerfile

-FROM danielscholl/azure-build-image
+FROM mcr.microsoft.com/mirror/docker/library/python:3.8-slim
 
 USER root
 
 WORKDIR /home
 
 COPY ./energistics/src/dags/energistics/witsml_parser_dag_bootstrap.py ./src/energistics/witsml_parser_dag_bootstrap.py
-COPY ./energistics/src/dags/requirements.txt ./src/requirements.txt
 
+COPY ./deployments/scripts/azure/requirements.txt ./scripts/requirements.txt
 COPY ./deployments/scripts/azure/prepare_dags.sh ./scripts/prepare_dags.sh
+
 COPY ./deployments/scripts/azure/setup/render_dag_file.py ./scripts/render_dag_file.py
 COPY ./deployments/scripts/azure/setup/output_dag_folder.py ./scripts/output_dag_folder.py
 

deployments/scripts/azure/prepare_dags.sh

@@ -21,9 +21,22 @@ echo "K8S_NAMESPACE: $K8S_NAMESPACE"
 echo "DATA_PARTITION: $DATA_PARTITION"
 echo "OSDU_AZURE_BASE_URL: $OSDU_AZURE_BASE_URL"
 
-pip install -r "/home/src/requirements.txt"
 pip install -r "/home/scripts/requirements.txt"
 
 python /home/scripts/render_dag_file.py -f $AZURE_DAG_DIRECTORY/witsml_parser_dag_bootstrap.py
 mv $AZURE_DAG_DIRECTORY/witsml_parser_dag_bootstrap.py $AZURE_DAG_DIRECTORY/witsml_parser_dag.py
 python /home/scripts/output_dag_folder.py
+
+if test $? = '1'; then
+  printf "\n"
+  echo "------------------------------------";
+  echo "Creating Dags -- ERROR";
+  echo "------------------------------------";
+  exit 1
+else
+  printf "\n"
+  echo "------------------------------------";
+  echo "Creating Dags -- FINISH";
+  echo "------------------------------------";
+  exit 0
+fi

deployments/scripts/azure/requirements.txt

+markupsafe==2.0.1
 Jinja2==2.10.1

deployments/scripts/azure/setup/render_dag_file.py

@@ -43,7 +43,8 @@ class DAGFileRenderer:
     def _render_template(self, file) -> str:
         env = Environment(
             loader=FileSystemLoader(
-                searchpath=os.getenv("AZURE_DAG_DIRECTORY", default="src/energistics")
+                searchpath=os.getenv("AZURE_DAG_DIRECTORY",
+                                     default="src/energistics")
             ),
             variable_start_string='{|', variable_end_string='|}',
         )