Merge branch 'GONRG-4004_Replace_bs_values_with_AF_vars' into 'master'

GONRG-4004: Replace bootstrap values with AF vars

See merge request !34

.gitlab-ci.yml

@@ -43,6 +43,7 @@ stages:
   - linters
   - containerize
   - bootstrap_dag
+  - publish_dag
   - deploy
   - build
   - scan

deployments/scripts/gcp/bootstrap.sh

@@ -16,36 +16,6 @@ echo "------------------------------------";
 echo "BOOTSTRAP -- START";
 echo "------------------------------------";
 
-
-if [ -z $K8S_NAMESPACE ]; then
-  echo 'ERROR: K8S_NAMESPASE not provided'
-  usage;
-else
-  export NAMESPACE=$K8S_NAMESPACE
-fi
-
-if [ -z $K8S_NODE_POOLS ]; then
-  echo 'WARNING: K8S_NODE_POOLS not provided. Will be used default node pool'
-  export K8S_NODE_POOLS=default
-else
-  export K8S_NODE_POOLS=$K8S_NODE_POOLS
-fi
-
-if [ -z $SD_SVC_URL ]; then
-  echo 'ERROR: SD_SVC_URL not provided'
-  usage;
-else
-  export SD_SVC_URL=$SD_SVC_URL
-fi
-
-if [ -z $SEGY_VDS_DOCKER_IMAGE ]; then
-  echo 'ERROR: SEGY_VDS_DOCKER_IMAGE not provided'
-  usage;
-else
-  export SEGY_VDS_DOCKER_IMAGE=$SEGY_VDS_DOCKER_IMAGE
-fi
-
-
 printf "\n"
 echo "SETTING POPULATION TO THE DAG FILE";
 echo "------------------------------------";

deployments/scripts/gcp/render_dag_file.py

@@ -8,41 +8,26 @@ class DAGFileRenderer:
 
     def __init__(self, file_path):
         self.file_path = file_path
-        self.docker_image = os.getenv("SEGY_VDS_DOCKER_IMAGE")
-        self.namespace = os.getenv("NAMESPACE")
-        self.sd_svc_url = os.getenv("SD_SVC_URL")
+        self.docker_image = "{{ var.value.gcp__image__segy_to_vds_converter }}"
+        # As for 25 FEB 2022, there is no namespace parameter in template_fields (https://github.com/apache/airflow/blob/2.2.0/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py#L165), 
+        # so we have to hardcode this value for now.
+        self.namespace = "default"
+        self.sd_svc_url = "{{ var.value.core__service__seismic__url }}"
         self.dag_name = "Segy_to_vds_conversion_sdms"
 
-    def _get_node_pools(self) -> List:
-        pools = os.getenv("K8S_NODE_POOLS")
-        return [p.strip() for p in pools.split(",")]
-
     def _get_resources(self):
-        return {
-            "request_memory": os.getenv("GCP_VDS_INGESTION_REQUEST_MEMORY", default="2Gi"),
-            "request_cpu": os.getenv("GCP_VDS_INGESTION_REQUEST_CPU", default="200m"),
-            "limit_memory": os.getenv("GCP_VDS_INGESTION_LIMIT_MEMORY", default="8Gi"),
-            "limit_cpu": os.getenv("GCP_VDS_INGESTION_LIMIT_CPU", default="1000m")
-        }
+        return """{
+                "request_memory": airflow.models.Variable.get("gcp__vds_ingestion_request_memory", default_var="2Gi"),
+                "request_cpu": airflow.models.Variable.get("gcp__vds_ingestion_request_cpu", default_var="200m"),
+                "limit_memory": airflow.models.Variable.get("gcp__vds_ingestion_limit_memory", default_var="8Gi"),
+                "limit_cpu": airflow.models.Variable.get("gcp__vds_ingestion_limit_cpu", default_var="1000m")
+            }"""
 
     def _prepare_operator_kwargs(self) -> Dict:
-        return {
-            "affinity": {
-                "nodeAffinity": {
-                    "requiredDuringSchedulingIgnoredDuringExecution": {
-                        "nodeSelectorTerms": [{
-                            "matchExpressions": [{
-                                "key": "cloud.google.com/gke-nodepool",
-                                "operator": "In",
-                                "values": self._get_node_pools()
-                            }]
-                        }]
-                    }
-                }
-            },
-            "resources": self._get_resources(),
-            "startup_timeout_seconds": os.getenv("GCP_VDS_INGESTION_STARTUP_TIMEOUT", default=300)
-        }
+        return f"""{{
+            "resources": {self._get_resources()},
+            "startup_timeout_seconds": 300
+        }}"""
 
     def _render_template(self, file) -> str:
         env = Environment(

deployments/scripts/gcp/requirements.txt

-Jinja2==2.10.1
\ No newline at end of file
+Jinja2==2.10.1
+markupsafe==2.0.1

devops/gcp/override-stages.yml

 .osdu_gcp_common_bootstrap_variables: &osdu_gcp_common_bootstrap_variables
   GCP_DAG_DIRECTORY: $CI_PROJECT_DIR/src/dags
   GCP_DEPLOYMENTS_SUBDIR: deployments/scripts/gcp
-  K8S_NAMESPACE: default
-  K8S_NODE_POOLS: $OSDU_GCP_K8S_NODE_POOLS
   OSDU_GCP_BOOTSTRAP_REQUIREMENTS: deployments/scripts/gcp/requirements.txt
   OSDU_GCP_BOOTSTRAP_SCRIPT: deployments/scripts/gcp/bootstrap.sh
 
@@ -12,15 +10,17 @@ osdu_gcp_bootstrap_dag_community:
   stage: bootstrap_dag
   variables:
     << : *osdu_gcp_common_bootstrap_variables
-    SD_SVC_URL: $OSDU_GCP_SEISMIC_STORE_SERVICE_COMMUNITY_ENDPOINT # i.e https://<ssdms_host>/seistore-svc/api/v3
-    SEGY_VDS_DOCKER_IMAGE: "community.opengroup.org:5555/osdu/platform/domain-data-mgmt-services/seismic/open-vds/openvds-ingestion:latest"
 
 osdu_gcp_bootstrap_dag_pre_ship:
   stage: bootstrap_dag
   variables:
     << : *osdu_gcp_common_bootstrap_variables
-    SD_SVC_URL: $OSDU_GCP_SEISMIC_STORE_SERVICE_PRE_SHIP_ENDPOINT # i.e https://<ssdms_host>/seistore-svc/api/v3
-    SEGY_VDS_DOCKER_IMAGE: $SEGY_VDS_DOCKER_IMAGE
+  
+osdu_gcp_publish_dag:
+  stage: publish_dag
+  variables:
+    BOOTSTRAPPED_DAG_PATH: ${OSDU_GCP_OUTPUT_DAG_DIR}/segy_to_vds_ssdms_conversion_dag.py
+    DAG_FILE: segy_to_vds_ssdms_conversion_dag.py
 
 osdu_gcp_deploy_composer_community:
   stage: deploy