diff --git a/deployments/scripts/azure/build_dag.sh b/deployments/scripts/azure/build_dag.sh index 4b52f5312f512d3812f17a2d231b2b32a19a7ac8..c92ad45c357c7c7e06b79792c20d30002b9f1b41 100644 --- a/deployments/scripts/azure/build_dag.sh +++ b/deployments/scripts/azure/build_dag.sh @@ -17,6 +17,7 @@ echo "Building Dag -- START "; echo "------------------------------------"; export NAMESPACE=airflow +export BUILD_VERSION=$BUILD_VERSION if [ -z $AZURE_DNS_NAME ]; then echo 'ERROR: AZURE_DNS_NAME not provided' @@ -33,17 +34,10 @@ else fi export SHARED_TENANT -if [ -z $PARSER_IMAGE ]; then - CSV_DOCKER_IMAGE=${AZURE_REGISTRY}.azurecr.io/${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:${CI_COMMIT_SHA} -else - CSV_DOCKER_IMAGE=${PARSER_IMAGE} -fi -export CSV_DOCKER_IMAGE - echo "AZURE_DNS_NAME: $AZURE_DNS_NAME" echo "WORKFLOW_URL: $WORKFLOW_URL" echo "SHARED_TENANT: $SHARED_TENANT" -echo "CSV_DOCKER_IMAGE: $CSV_DOCKER_IMAGE" +echo "BUILD_VERSION: $BUILD_VERSION" printf "\n" echo "Execute Python Script create_dag.py"; diff --git a/deployments/scripts/azure/create_dag.py b/deployments/scripts/azure/create_dag.py index e9880058cf37a30e54083f0bd30476dc111f353a..61ca18a991140a9c08423502212e099e4f408f55 100644 --- a/deployments/scripts/azure/create_dag.py +++ b/deployments/scripts/azure/create_dag.py @@ -8,14 +8,11 @@ from jinja2 import Environment, FileSystemLoader class CreateDag: def __init__(self, file_path): - self.docker_image = os.environ.get('CSV_DOCKER_IMAGE') + self.build_version = os.environ.get("BUILD_VERSION") self.dns_host_name = "{{ var.value.azure_dns_host }}" self.namespace = os.environ.get('NAMESPACE') self.dag_folder_path = os.environ.get('CSV_PARSER_DAG_FOLDER_PATH', '../airflowdags/osdu_csv_parser') - self.file_path = file_path - self.service_name = self.docker_image[self.docker_image.index("/") + 1: self.docker_image.index(":")] - self.build_version = self.docker_image[self.docker_image.index(":") + 1:] self.dag_name = f"csv-parser-{self.build_version}"[:64] self.env_vars = { @@ -35,17 +32,14 @@ class CreateDag: } self.kubernetes_pod_operator_options = { - "labels": { - "aadpodidbinding": "osdu-identity" + "labels": { + "aadpodidbinding": "osdu-identity" + } } - } - print(f"docker_image: {self.docker_image}") print(f"file_path: {self.file_path}") - print(f"service_name: {self.service_name}") print(f"build_version: {self.build_version}") print(f"dag_name: {self.dag_name}") - print(f"docker_image: {self.docker_image}") def _render_template(self, file) -> str: env = Environment( @@ -56,7 +50,7 @@ class CreateDag: params = { "DAG_NAME": self.dag_name, - "DOCKER_IMAGE": self.docker_image, + "DOCKER_IMAGE": "{{ var.value.dag_image_acr }}/csv-parser:" + self.build_version, "DNS_HOST": self.dns_host_name, "NAMESPACE": self.namespace, "ENV_VARS": json.dumps(self.env_vars, indent=4), diff --git a/deployments/scripts/azure/dockerFolder/build_dag_dockerfile b/deployments/scripts/azure/dockerFolder/build_dag_dockerfile index 6ba50105dc1fea8006074644b52750de4f7f596b..3f7b0d298e88deda57117e3f659467be1e40a639 100644 --- a/deployments/scripts/azure/dockerFolder/build_dag_dockerfile +++ b/deployments/scripts/azure/dockerFolder/build_dag_dockerfile @@ -6,7 +6,6 @@ ARG AZURE_DEPLOYMENTS_SCRIPTS_SUBDIR=/ ENV PYTHONUNBUFFERED=1 \ PATH="/home/osdu/.local/bin:${PATH}" \ - CSV_PARSER_IMAGE=${PARSER_IMAGE} \ SHARED_TENANT=${SHARED_TENANT} \ AZURE_DNS_NAME=$AZURE_DNS_NAME diff --git a/deployments/scripts/azure/prepare_dags.sh b/deployments/scripts/azure/prepare_dags.sh index 378b16ab7428f9f6fb5b8db4f4281ba16dd9b617..fb367c0a812a682b7b11cf1d4d8c84dcc8fc8e7c 100644 --- a/deployments/scripts/azure/prepare_dags.sh +++ b/deployments/scripts/azure/prepare_dags.sh @@ -29,27 +29,27 @@ if [ "$AZURE_GITLAB_PIPELINE_RUN" == "true" ]; then docker push $CI_REGISTRY_IMAGE/$DAG_IMAGE echo "Pushing to Azure Container Registry" - docker tag $CI_REGISTRY_IMAGE/$PARSER_IMAGE ${AZURE_REGISTRY}.azurecr.io/$PARSER_IMAGE + docker tag $CI_REGISTRY_IMAGE/$PARSER_IMAGE ${AZURE_REGISTRY}.azurecr.io/csv-parser:$CI_COMMIT_SHA docker tag $CI_REGISTRY_IMAGE/$DAG_IMAGE ${AZURE_REGISTRY}.azurecr.io/$DAG_IMAGE - docker push ${AZURE_REGISTRY}.azurecr.io/$PARSER_IMAGE + docker push ${AZURE_REGISTRY}.azurecr.io/csv-parser:$CI_COMMIT_SHA docker push ${AZURE_REGISTRY}.azurecr.io/$DAG_IMAGE - export PARSER_IMAGE=${AZURE_REGISTRY}.azurecr.io/$PARSER_IMAGE + export BUILD_VERSION=$CI_COMMIT_SHA export DAG_IMAGE=$CI_REGISTRY_IMAGE/$DAG_IMAGE else echo "Build docker images" - docker build --file ./$AZURE_DEPLOYMENTS_SCRIPTS_SUBDIR/dockerFolder/csv_parser_dockerfile -t $PARSER_IMAGE . + docker build --file ./$AZURE_DEPLOYMENTS_SCRIPTS_SUBDIR/dockerFolder/csv_parser_dockerfile -t ${AZURE_REGISTRY}.azurecr.io/csv-parser:$BUILD_VERSION . docker build --build-arg AZURE_DEPLOYMENTS_SCRIPTS_SUBDIR=$AZURE_DEPLOYMENTS_SCRIPTS_SUBDIR --file ./$AZURE_DEPLOYMENTS_SCRIPTS_SUBDIR/dockerFolder/build_dag_dockerfile -t $DAG_IMAGE . echo "Pushing to Azure Container Registry" - docker push $PARSER_IMAGE + docker push ${AZURE_REGISTRY}.azurecr.io/csv-parser:$BUILD_VERSION docker push $DAG_IMAGE fi cat > .env << EOF -PARSER_IMAGE=$PARSER_IMAGE +BUILD_VERSION=$BUILD_VERSION SHARED_TENANT=$SHARED_TENANT AZURE_DNS_NAME=$AZURE_DNS_NAME EOF diff --git a/devops/azure/dev-pipeline.yml b/devops/azure/dev-pipeline.yml index e6c95679f35cd21ee59b5c750f502ea8bbc48cd9..2772dedfe75bf6cbd2cff74a5871571db346646d 100644 --- a/devops/azure/dev-pipeline.yml +++ b/devops/azure/dev-pipeline.yml @@ -60,7 +60,7 @@ stages: parameters: dockerfilePath: 'deployments/scripts/azure/dockerFolder/output_dags_dockerfile' outputDagFolder: '/home/output_dags' - environmentVars: 'AZURE_REGISTRY=$(CONTAINER_REGISTRY_NAME)${NEWLINE}AZURE_PRINCIPAL_ID=$(AZURE_DEPLOY_CLIENT_ID)${NEWLINE}AZURE_PRINCIPAL_SECRET=$(AZURE_DEPLOY_CLIENT_SECRET)${NEWLINE}AZURE_TENANT_ID=$(AZURE_DEPLOY_TENANT)${NEWLINE}PARSER_IMAGE=$(CONTAINER_REGISTRY_NAME).azurecr.io/csv-parser-dev:$(Build.SourceVersion)${NEWLINE}DAG_IMAGE=$(CONTAINER_REGISTRY_NAME).azurecr.io/csv-parser-dag-dev:$(Build.SourceVersion)${NEWLINE}SHARED_TENANT=$(OSDU_TENANT)${NEWLINE}AZURE_DNS_NAME=$(AZURE_DNS_NAME)${NEWLINE}AZURE_GITLAB_PIPELINE_RUN=false' + environmentVars: 'AZURE_REGISTRY=$(CONTAINER_REGISTRY_NAME)${NEWLINE}AZURE_PRINCIPAL_ID=$(AZURE_DEPLOY_CLIENT_ID)${NEWLINE}AZURE_PRINCIPAL_SECRET=$(AZURE_DEPLOY_CLIENT_SECRET)${NEWLINE}AZURE_TENANT_ID=$(AZURE_DEPLOY_TENANT)${NEWLINE}BUILD_VERSION=$(Build.SourceVersion)${NEWLINE}DAG_IMAGE=$(CONTAINER_REGISTRY_NAME).azurecr.io/csv-parser-dag-dev:$(Build.SourceVersion)${NEWLINE}SHARED_TENANT=$(OSDU_TENANT)${NEWLINE}AZURE_DNS_NAME=$(AZURE_DNS_NAME)${NEWLINE}AZURE_GITLAB_PIPELINE_RUN=false' - template: /devops/dag-pipeline-stages/copy-dag.yml@TemplateRepo parameters: