Commit ae97dff5 authored by David Diederich's avatar David Diederich
Browse files

Merge branch 'cherry-pick-d3a3f618' into 'release/0.16'

cherry-pick-d3a3f618 into release/0.16

See merge request !94
parents e193b1c3 3e82f858
Pipeline #132669 failed with stages
in 41 minutes and 27 seconds
......@@ -92,55 +92,51 @@ build_parser:
- |
export EXTRA_GCP_DOCKER_TAG="";
export EXTRA_ANTHOS_DOCKER_TAG="";
export EXTRA_AZURE_DOCKER_TAG="";
if [ "$CI_COMMIT_TAG" != "" ]; then
EXTRA_GCP_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/osdu-gcp-$OSDU_GCP_SERVICE:$CI_COMMIT_TAG";
EXTRA_ANTHOS_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/osdu-gcp-anthos-$OSDU_GCP_SERVICE:$CI_COMMIT_TAG";
EXTRA_AZURE_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/$OSDU_AZURE_POSTFIX:$CI_COMMIT_TAG";
elif [ "$CI_COMMIT_REF_NAME" = "master" ]; then
EXTRA_GCP_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/osdu-gcp-$OSDU_GCP_SERVICE:latest";
EXTRA_ANTHOS_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/osdu-gcp-anthos-$OSDU_GCP_SERVICE:latest";
EXTRA_AZURE_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/$OSDU_AZURE_POSTFIX:latest";
fi
- docker build --target gcp -t $CI_REGISTRY_IMAGE/osdu-gcp-$OSDU_GCP_SERVICE:$CI_COMMIT_SHORT_SHA $EXTRA_GCP_DOCKER_TAG --rm -f ./build/Dockerfile .
- docker build --target anthos -t $CI_REGISTRY_IMAGE/osdu-gcp-anthos-$OSDU_GCP_SERVICE:$CI_COMMIT_SHORT_SHA $EXTRA_ANTHOS_DOCKER_TAG --rm -f ./build/Dockerfile .
- docker build --target azure -t $OSDU_AZURE_BASE_IMAGE_TAG $EXTRA_AZURE_DOCKER_TAG --rm -f ./build/Dockerfile .
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- docker push $CI_REGISTRY_IMAGE/osdu-gcp-$OSDU_GCP_SERVICE
- docker push $CI_REGISTRY_IMAGE/osdu-gcp-anthos-$OSDU_GCP_SERVICE
- docker push $CI_REGISTRY_IMAGE/$OSDU_AZURE_POSTFIX
build_parser_azure:
stage: build
tags: [ "osdu-small" ]
needs: []
tags: [ "osdu-medium" ]
image: danielscholl/azure-build-image
variables:
SHA_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:${CI_COMMIT_SHA}
LATEST_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:latest
RELEASE_IMAGE: release-${CI_COMMIT_TAG}:${CI_PROJECT_NAME}-${CI_COMMIT_TAG}
before_script:
- az login --service-principal -u $AZURE_PRINCIPAL_ID -p $AZURE_PRINCIPAL_SECRET --tenant $AZURE_TENANT_ID
- az acr login -n $AZURE_REGISTRY
script:
- |
if [ "$CI_COMMIT_TAG" != "" ]; then
docker build --target azure -t ${AZURE_REGISTRY}.azurecr.io/$RELEASE_IMAGE --rm -f ./build/Dockerfile .
docker build --target azure -t ${AZURE_REGISTRY}.azurecr.io/$RELEASE_IMAGE --rm -f ./build/providers/azure/Dockerfile .
docker push ${AZURE_REGISTRY}.azurecr.io/$RELEASE_IMAGE
elif [ "$CI_COMMIT_REF_NAME" = "master" ]; then
docker build --target azure -t ${AZURE_REGISTRY}.azurecr.io/$LATEST_IMAGE --rm -f ./build/Dockerfile .
docker build --target azure -t ${AZURE_REGISTRY}.azurecr.io/$LATEST_IMAGE --rm -f ./build/providers/azure/Dockerfile .
docker push ${AZURE_REGISTRY}.azurecr.io/$LATEST_IMAGE
fi
- docker build --target azure -t ${AZURE_REGISTRY}.azurecr.io/$SHA_IMAGE --rm -f ./build/Dockerfile .
- docker build --target azure -t ${AZURE_REGISTRY}.azurecr.io/$SHA_IMAGE --rm -f ./build/providers/azure/Dockerfile .
- docker push ${AZURE_REGISTRY}.azurecr.io/$SHA_IMAGE
only:
variables:
- $AZURE == 'true'
- $AZURE == '1'
azure_standalone_tests:
stage: test
needs: [ "test_parser" ]
needs: []
variables:
AZURE_SKIP_STANDALONE_TESTS: "true"
......@@ -175,17 +171,3 @@ azure_build_dag:
DATA_PARTITION=$DATA_PARTITION
OSDU_AZURE_BASE_URL=$OSDU_AZURE_BASE_URL
EOF
azure_dag_postman_tests:
before_script:
- |
# Generating environment file to be passed while running the postman tests
cat > python_env_vars.txt << EOF
TENANT_ID os.environ.get('AZURE_TENANT_ID')
CLIENT_ID os.environ.get('CLIENT_ID')
CLIENT_SECRET os.environ.get('CLIENT_SECRET')
HOSTNAME os.environ.get('AZURE_DNS_NAME')
refresh_token os.environ.get('AZURE_REFRESH_TOKEN')
WORKFLOW_HOST os.environ.get('AZURE_DNS_NAME')
witsmlFilePath os.environ.get('AZURE_TEST_WITSML_FILE_PATH')
EOF
......@@ -57,15 +57,6 @@ COPY "build/providers/${PROVIDER_NAME}/requirements.txt" ./
RUN pip install -r "requirements.txt"
COPY build/providers/gcp/osdu_api.ini ./
## azure
FROM base as azure
ENV CLOUD_PROVIDER azure
COPY build/providers/azure/requirements.txt ./
COPY build/providers/azure/osdu_api.ini ./
RUN pip install -r requirements.txt
## anthos
FROM base as anthos
......
FROM mcr.microsoft.com/mirror/docker/library/python:3.8-slim as base
ENV HOME="/home/witsml_parser"
WORKDIR $HOME
ENV BUILD_DIR="build"
ENV WITSML_PARSER_DIR="energistics/src/witsml_parser"
ENV ENERGISTICS_LIBS="${WITSML_PARSER_DIR}/energistics"
ENV OSDU_CORE_LIBS_DIR="${WITSML_PARSER_DIR}/osdu/ingestion-dags/src/dags/libs"
ENV OSDU_CORE_PROVIDERS_DIR="${WITSML_PARSER_DIR}/osdu/ingestion-dags/src/dags/providers"
COPY ./${BUILD_DIR}/requirements.txt ./
COPY ./${WITSML_PARSER_DIR}/*.py ./
COPY ./${ENERGISTICS_LIBS} ./energistics
COPY ./${BUILD_DIR}/providers/gcp/osdu_api.ini ./
RUN pip install --upgrade pip && \
pip install -r "./energistics/requirements.txt" && \
pip install -r "./requirements.txt"
ENV PYTHONPATH="$PYTHONPATH:$HOME"
FROM base as azure
ENV CLOUD_PROVIDER azure
COPY build/providers/azure/requirements.txt ./
COPY build/providers/azure/osdu_api.ini ./
RUN pip install -r requirements.txt
FROM msosdu.azurecr.io/data-seed-base:v1
#Environment Variables while running the docker(These are not required to be set in build pipelines)
# Environment Variables while running the docker(These are not required to be set in build pipelines)
# AZURE_TENANT_ID
# AZURE_CLIENT_ID
......
FROM danielscholl/azure-build-image
FROM mcr.microsoft.com/mirror/docker/library/python:3.8-slim
USER root
WORKDIR /home
COPY ./energistics/src/dags/energistics/witsml_parser_dag_bootstrap.py ./src/energistics/witsml_parser_dag_bootstrap.py
COPY ./energistics/src/dags/requirements.txt ./src/requirements.txt
COPY ./deployments/scripts/azure/requirements.txt ./scripts/requirements.txt
COPY ./deployments/scripts/azure/prepare_dags.sh ./scripts/prepare_dags.sh
COPY ./deployments/scripts/azure/setup/render_dag_file.py ./scripts/render_dag_file.py
COPY ./deployments/scripts/azure/setup/output_dag_folder.py ./scripts/output_dag_folder.py
......
......@@ -21,9 +21,22 @@ echo "K8S_NAMESPACE: $K8S_NAMESPACE"
echo "DATA_PARTITION: $DATA_PARTITION"
echo "OSDU_AZURE_BASE_URL: $OSDU_AZURE_BASE_URL"
pip install -r "/home/src/requirements.txt"
pip install -r "/home/scripts/requirements.txt"
python /home/scripts/render_dag_file.py -f $AZURE_DAG_DIRECTORY/witsml_parser_dag_bootstrap.py
mv $AZURE_DAG_DIRECTORY/witsml_parser_dag_bootstrap.py $AZURE_DAG_DIRECTORY/witsml_parser_dag.py
python /home/scripts/output_dag_folder.py
if test $? = '1'; then
printf "\n"
echo "------------------------------------";
echo "Creating Dags -- ERROR";
echo "------------------------------------";
exit 1
else
printf "\n"
echo "------------------------------------";
echo "Creating Dags -- FINISH";
echo "------------------------------------";
exit 0
fi
......@@ -43,7 +43,8 @@ class DAGFileRenderer:
def _render_template(self, file) -> str:
env = Environment(
loader=FileSystemLoader(
searchpath=os.getenv("AZURE_DAG_DIRECTORY", default="src/energistics")
searchpath=os.getenv("AZURE_DAG_DIRECTORY",
default="src/energistics")
),
variable_start_string='{|', variable_end_string='|}',
)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment