Merge branch 'feature/GONRG-3677-bootstrap' into 'master'

GONRG-3677-bootstrap: added Dockerfile and gitlab.ci file for building...

See merge request !29

.gitlab-ci.yml

-#  Copyright 2020 Google LLC
-#  Copyright 2020 EPAM Systems
+#  Copyright 2021 Google LLC
+#  Copyright 2021 EPAM Systems
 #  Copyright 2020 Microsoft
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,6 +13,7 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
+
 variables:
   AZURE_SERVICE: segy-to-vds-conversion
   AZURE_DEPLOYMENTS_SUBDIR: deployments/scripts/azure
@@ -25,6 +26,7 @@ variables:
   OSDU_GCP_VENDOR: gcp
   OSDU_GCP_K8S_NODE_POOLS: default-pool,pool-1,pool-2
   OSDU_GCP_SEISMIC_STORE_SERVICE_ENDPOINT: https://seismic-store-attcrcktoa-uc.a.run.app/api/v3
+  SEGY_VDS_DOCKER_IMAGE: "community.opengroup.org:5555/osdu/platform/domain-data-mgmt-services/seismic/open-vds/openvds-ingestion:latest"
 
 image: google/cloud-sdk:alpine
 
@@ -70,7 +72,6 @@ osdu-gcp-deploy:
     K8S_NAMESPACE: default
     K8S_NODE_POOLS: $OSDU_GCP_K8S_NODE_POOLS
     SD_SVC_URL: $OSDU_GCP_SEISMIC_STORE_SERVICE_ENDPOINT # i.e https://<ssdms_host>/seistore-svc/api/v3
-    SEGY_VDS_DOCKER_IMAGE: "community.opengroup.org:5555/osdu/platform/domain-data-mgmt-services/seismic/open-vds/openvds-ingestion:latest"
     GCP_DEPLOYMENTS_SUBDIR: deployments/scripts/gcp
     GCP_DAG_DIRECTORY: $CI_PROJECT_DIR/src/dags
   script:
@@ -88,3 +89,5 @@ include:
     file: "standard-setup.yml"
 
   - local: "/devops/azure/override-stages.yml"
+  # in case of any changes at project directory structure please make adjustments to Deployment Framework scripts
+  - local: "/deployments/scripts/gcp/bootstrap_segy_to_vds.yaml"

deployments/scripts/gcp/Dockerfile

+FROM google/cloud-sdk:slim
+
+COPY ./deployments/scripts/gcp /opt
+COPY ./src /opt
+
+ARG SEGY_VDS_DOCKER_IMAGE_ARG
+
+ENV SEGY_VDS_DOCKER_IMAGE=$SEGY_VDS_DOCKER_IMAGE_ARG
+ENV GCP_DEPLOYMENTS_SUBDIR=/opt
+ENV GCP_DAG_DIRECTORY=/opt/dags
+ENV PREFIX=$(pwd)
+
+RUN apt-get update && apt-get install -yq jq && pip3 install -r /opt/requirements.txt && chmod 775 /opt/bootstrap_segy_to_vds.sh
+
+CMD ["/bin/bash", "-c", "/opt/bootstrap_segy_to_vds.sh"]

deployments/scripts/gcp/bootstrap_segy_to_vds.sh

+#!/usr/bin/env bash
+
+set -ex
+
+###############################
+## ARGUMENT INPUT            ##
+###############################
+
+usage() { echo "Usage: bootstrap_segy_to_vds.sh"; }
+
+printf "\n"
+echo "------------------------------------";
+echo "BOOTSTRAP -- START";
+echo "------------------------------------";
+
+
+if [ -z $K8S_NAMESPACE ]; then
+  echo 'ERROR: K8S_NAMESPASE not provided'
+  usage;
+else
+  export NAMESPACE=$K8S_NAMESPACE
+fi
+
+if [ -z $K8S_NODE_POOLS ]; then
+  echo 'WARNING: K8S_NODE_POOLS not provided. Will be used default node pool'
+  export K8S_NODE_POOLS=default
+else
+  export K8S_NODE_POOLS=$K8S_NODE_POOLS
+fi
+
+if [ -z $SD_SVC_URL ]; then
+  echo 'ERROR: SD_SVC_URL not provided'
+  usage;
+else
+  export SD_SVC_URL=$SD_SVC_URL
+fi
+
+if [ -z $SEGY_VDS_DOCKER_IMAGE ]; then
+  echo 'ERROR: SEGY_VDS_DOCKER_IMAGE not provided'
+  usage;
+else
+  SEGY_VDS_DOCKER_IMAGE=${SEGY_VDS_DOCKER_IMAGE}
+fi
+export SEGY_VDS_DOCKER_IMAGE
+
+
+printf "\n"
+echo "SETTING POPULATION TO THE DAG FILE";
+echo "------------------------------------";
+
+if [ -z $OUTPUT_FILE ]; then
+  echo "THE ORIGINAL DAG FILE WILL BE OVERWRITTEN"
+  python3 $GCP_DEPLOYMENTS_SUBDIR/render_dag_file.py -f $GCP_DAG_DIRECTORY/segy_to_vds_ssdms_conversion_dag.py
+else
+  echo "RENDERED OUTPUT WILL BE SAVED INTO ${OUTPUT_FILE}"
+  python3 $GCP_DEPLOYMENTS_SUBDIR/render_dag_file.py -f $GCP_DAG_DIRECTORY/segy_to_vds_ssdms_conversion_dag.py -o ${OUTPUT_FILE}
+fi
+
+# get airflow bucket variable
+BUCKET=$(gcloud composer environments describe $COMP_NAME --location $LOCATION --format json | jq .config.dagGcsPrefix | tr -d '"' | sed -e 's|/dags||')
+
+gsutil -m rsync -R -x '(?!^segy_to_vds_ssdms_conversion_dag.py$)' /opt/dags  $BUCKET/dags/segy_to_vds

deployments/scripts/gcp/bootstrap_segy_to_vds.yaml

+containerize_osdu_gcp_segy_bootstrap:
+  stage: containerize
+  image: docker:19.03.15
+  tags: ["osdu-medium"]
+  services:
+    - docker:20.10.7-dind
+  variables:
+    IMAGE_NAME: osdu-bootstrap-segy
+  script:
+    # Gitlab Container Registry
+    - export EXTRA_DOCKER_TAG=""; if [ "$CI_COMMIT_TAG" != "" ] ; then EXTRA_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/$IMAGE_NAME:$CI_COMMIT_TAG" ; elif [ "$CI_COMMIT_REF_NAME" = "master" ] ; then EXTRA_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/$IMAGE_NAME:latest" ; fi
+    - docker build -t $CI_REGISTRY_IMAGE/$IMAGE_NAME:$CI_COMMIT_SHORT_SHA $EXTRA_DOCKER_TAG --build-arg SEGY_VDS_DOCKER_IMAGE_ARG=${SEGY_VDS_DOCKER_IMAGE} --file deployments/scripts/gcp/Dockerfile .
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - docker push $CI_REGISTRY_IMAGE/$IMAGE_NAME
+  only:
+    refs:
+      - master
+      - /.*bootstrap.*/
+      - tags