Skip to content
Snippets Groups Projects
Commit 103ba129 authored by Marija Dukic's avatar Marija Dukic
Browse files

ci: migrate python image to MCR

parent 1b6e28ba
No related branches found
No related tags found
1 merge request!76ci: migrate python image to MCR
FROM mcr.microsoft.com/mirror/docker/library/python:3.8-slim
ARG DATA_PARTITION=opendes
ENV BUILD_PACKAGES \
python3-cryptography
ENV PYTHONUNBUFFERED=1 \
PATH="/home/osdu/.local/bin:${PATH}" \
DATA_PARTITION=${DATA_PARTITION} \
DAG_DIRECTORY=../airflow/workflow-svc-v2 \
DAG_TASK_IMAGE=${DAG_TASK_IMAGE} \
AZURE_DEPLOYMENTS_SUBDIR=. \
AZURE_DEPLOYMENTS_SCRIPTS_SUBDIR=.
USER root
RUN set -x \
echo "**** install Python ****" && \
apt-get update && \
apt-get install -y -o APT::Keep-Downloaded-Packages=false ${BUILD_PACKAGES} &&\
rm -rf /var/cache/apt/archives/*
# Install Requirements
ADD deployments/scripts/azure/requirements.txt ./requirements.txt
RUN pip install -r requirements.txt
# Add debian repo to install busybox package
RUN \
echo '@debian http://deb.debian.org/debian' >> /etc/apt/repositories && \
echo '@debian http://deb.debian.org/debian-security' >> /etc/apt/repositories
RUN \
apt-get update && \
apt-get install busybox -y
# Add Scripts
WORKDIR /home/osdu
ADD deployments/scripts/azure/*.py scripts/
ADD deployments/scripts/azure/bootstrap.sh scripts/
RUN chmod +x scripts/bootstrap.sh
# Add DAGs
ADD airflow airflow
# Execute as non root user.
RUN addgroup --system osdu && adduser --system osdu --ingroup osdu
USER osdu
WORKDIR /home/osdu/scripts
ENTRYPOINT ["/home/osdu/scripts/bootstrap.sh"]
\ No newline at end of file
FROM community.opengroup.org:5555/osdu/platform/deployment-and-operations/base-containers-azure/alpine-python3:0.0.2
FROM mcr.microsoft.com/mirror/docker/library/python:3.8-slim
ARG DATA_PARTITION=opendes
ENV BUILD_PACKAGES \
py-cryptography
python3-cryptography
ENV PYTHONUNBUFFERED=1 \
PATH="/home/osdu/.local/bin:${PATH}" \
......@@ -17,15 +17,25 @@ USER root
RUN set -x \
echo "**** install Python ****" && \
apk update && \
apk add --no-cache ${BUILD_PACKAGES} &&\
rm -rf /var/cache/apk/*
apt-get update && \
apt-get install -y -o APT::Keep-Downloaded-Packages=false ${BUILD_PACKAGES} &&\
rm -rf /var/cache/apt/archives/*
# Install Requirements
ADD deployments/scripts/azure/requirements.txt ./requirements.txt
RUN pip install -r requirements.txt
# Add debian repo to install busybox package
RUN \
echo '@debian http://deb.debian.org/debian' >> /etc/apt/repositories && \
echo '@debian http://deb.debian.org/debian-security' >> /etc/apt/repositories
RUN \
apt-get update && \
apt-get install busybox -y
# Add Scripts
WORKDIR /home/osdu
ADD deployments/scripts/azure/*.py scripts/
ADD deployments/scripts/azure/bootstrap.sh scripts/
RUN chmod +x scripts/bootstrap.sh
......@@ -34,7 +44,8 @@ RUN chmod +x scripts/bootstrap.sh
ADD airflow airflow
# Execute as non root user.
RUN addgroup --system osdu && adduser --system osdu --ingroup osdu
USER osdu
WORKDIR /home/osdu/scripts
ENTRYPOINT ["/home/osdu/scripts/bootstrap.sh"]
ENTRYPOINT ["/home/osdu/scripts/bootstrap.sh"]
\ No newline at end of file
......@@ -24,7 +24,7 @@ azure_build_dag:
image: danielscholl/azure-build-image
only:
variables:
- "$AZURE == 'true'"
- "$AZURE == '1'"
script:
- |
cat > .env << EOF
......@@ -39,7 +39,7 @@ azure_build_dag:
BUILD_VERSION=$CI_COMMIT_SHA
EOF
docker build -t $CI_REGISTRY_IMAGE/$DAG_TASK_IMAGE --file DAG.Dockerfile .
docker build -t $CI_REGISTRY_IMAGE/$DAG_TASK_IMAGE --file deployments/scripts/azure/DAG.Dockerfile .
docker build -t $CI_REGISTRY_IMAGE/$DAG_LOAD_IMAGE --file deployments/scripts/azure/Dockerfile .
docker push $CI_REGISTRY_IMAGE/$DAG_TASK_IMAGE
docker push $CI_REGISTRY_IMAGE/$DAG_LOAD_IMAGE
......@@ -76,7 +76,7 @@ azure_register_dag:
- azure_build_dag
only:
variables:
- "$AZURE == 'true'"
- "$AZURE == '1'"
script:
- |
cat > requirements.txt << EOF
......@@ -149,13 +149,28 @@ azure_register_dag:
self.register_dag()
def register_dag(self):
data = '{"workflowName":"$DAG_NAME","description":"$DAG_NAME","registrationInstructions":{"dagName":"$DAG_NAME"}}'
# To register DAG workflow name must match pattern ^[a-zA-Z0-9._-]{1,64}$
dag_name = '$DAG_NAME'
formatted_dag_name = dag_name[:63]
data = json.dumps(
{
"workflowName": formatted_dag_name,
"description": formatted_dag_name,
"registrationInstructions":
{
"dagName": formatted_dag_name
}
})
headers = {
'Content-Type': 'application/json',
'Authorization': self.token,
'data-partition-id': self.data_partition_id
}
print(f"Attempting to register DAG with name {formatted_dag_name}")
response = requests.post(self.workflow_service_url, headers=headers, data=data)
if response.status_code == 200:
workflow_id = response.json().get('workflowId')
......@@ -164,7 +179,8 @@ azure_register_dag:
workflow_id = response.json().get('workflowId')
print("DAG is already registered with workflowId: {0}".format(workflow_id))
else:
print("Error while registering DAG {0}".format(response.raise_for_status()))
print(f"Error while registering DAG due to: {response.json().get('message')}")
print("Error: {0}".format(response.raise_for_status()))
if __name__ == "__main__":
RegisterDag().register()
......@@ -197,7 +213,7 @@ azure_copy_dag:
- azure_register_dag
only:
variables:
- "$AZURE == 'true'"
- "$AZURE == '1'"
script:
- |
# Installing the Azcopy utility
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment