Skip to content
Snippets Groups Projects
Commit 844700a4 authored by Yan Sushchynski (EPAM)'s avatar Yan Sushchynski (EPAM)
Browse files

Merge branch 'GONRG-4004_Replace_bs_values_with_AF_vars' into 'master'

GONRG-4004: Replace bootstrap values with AF vars

See merge request !34
parents e5bdc844 8b4472ee
No related branches found
No related tags found
3 merge requests!41Draft: Merge latest oVDS changes from 'master' to OSDU-SEGY-VDS-Azure-M8-Master,!40Draft: Merge Master latest changes to Azure,!34GONRG-4004: Replace bootstrap values with AF vars
Checking pipeline status
......@@ -43,6 +43,7 @@ stages:
- linters
- containerize
- bootstrap_dag
- publish_dag
- deploy
- build
- scan
......
......@@ -16,36 +16,6 @@ echo "------------------------------------";
echo "BOOTSTRAP -- START";
echo "------------------------------------";
if [ -z $K8S_NAMESPACE ]; then
echo 'ERROR: K8S_NAMESPASE not provided'
usage;
else
export NAMESPACE=$K8S_NAMESPACE
fi
if [ -z $K8S_NODE_POOLS ]; then
echo 'WARNING: K8S_NODE_POOLS not provided. Will be used default node pool'
export K8S_NODE_POOLS=default
else
export K8S_NODE_POOLS=$K8S_NODE_POOLS
fi
if [ -z $SD_SVC_URL ]; then
echo 'ERROR: SD_SVC_URL not provided'
usage;
else
export SD_SVC_URL=$SD_SVC_URL
fi
if [ -z $SEGY_VDS_DOCKER_IMAGE ]; then
echo 'ERROR: SEGY_VDS_DOCKER_IMAGE not provided'
usage;
else
export SEGY_VDS_DOCKER_IMAGE=$SEGY_VDS_DOCKER_IMAGE
fi
printf "\n"
echo "SETTING POPULATION TO THE DAG FILE";
echo "------------------------------------";
......
......@@ -8,41 +8,26 @@ class DAGFileRenderer:
def __init__(self, file_path):
self.file_path = file_path
self.docker_image = os.getenv("SEGY_VDS_DOCKER_IMAGE")
self.namespace = os.getenv("NAMESPACE")
self.sd_svc_url = os.getenv("SD_SVC_URL")
self.docker_image = "{{ var.value.gcp__image__segy_to_vds_converter }}"
# As for 25 FEB 2022, there is no namespace parameter in template_fields (https://github.com/apache/airflow/blob/2.2.0/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py#L165),
# so we have to hardcode this value for now.
self.namespace = "default"
self.sd_svc_url = "{{ var.value.core__service__seismic__url }}"
self.dag_name = "Segy_to_vds_conversion_sdms"
def _get_node_pools(self) -> List:
pools = os.getenv("K8S_NODE_POOLS")
return [p.strip() for p in pools.split(",")]
def _get_resources(self):
return {
"request_memory": os.getenv("GCP_VDS_INGESTION_REQUEST_MEMORY", default="2Gi"),
"request_cpu": os.getenv("GCP_VDS_INGESTION_REQUEST_CPU", default="200m"),
"limit_memory": os.getenv("GCP_VDS_INGESTION_LIMIT_MEMORY", default="8Gi"),
"limit_cpu": os.getenv("GCP_VDS_INGESTION_LIMIT_CPU", default="1000m")
}
return """{
"request_memory": airflow.models.Variable.get("gcp__vds_ingestion_request_memory", default_var="2Gi"),
"request_cpu": airflow.models.Variable.get("gcp__vds_ingestion_request_cpu", default_var="200m"),
"limit_memory": airflow.models.Variable.get("gcp__vds_ingestion_limit_memory", default_var="8Gi"),
"limit_cpu": airflow.models.Variable.get("gcp__vds_ingestion_limit_cpu", default_var="1000m")
}"""
def _prepare_operator_kwargs(self) -> Dict:
return {
"affinity": {
"nodeAffinity": {
"requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": [{
"key": "cloud.google.com/gke-nodepool",
"operator": "In",
"values": self._get_node_pools()
}]
}]
}
}
},
"resources": self._get_resources(),
"startup_timeout_seconds": os.getenv("GCP_VDS_INGESTION_STARTUP_TIMEOUT", default=300)
}
return f"""{{
"resources": {self._get_resources()},
"startup_timeout_seconds": 300
}}"""
def _render_template(self, file) -> str:
env = Environment(
......
Jinja2==2.10.1
\ No newline at end of file
Jinja2==2.10.1
markupsafe==2.0.1
.osdu_gcp_common_bootstrap_variables: &osdu_gcp_common_bootstrap_variables
GCP_DAG_DIRECTORY: $CI_PROJECT_DIR/src/dags
GCP_DEPLOYMENTS_SUBDIR: deployments/scripts/gcp
K8S_NAMESPACE: default
K8S_NODE_POOLS: $OSDU_GCP_K8S_NODE_POOLS
OSDU_GCP_BOOTSTRAP_REQUIREMENTS: deployments/scripts/gcp/requirements.txt
OSDU_GCP_BOOTSTRAP_SCRIPT: deployments/scripts/gcp/bootstrap.sh
......@@ -12,15 +10,17 @@ osdu_gcp_bootstrap_dag_community:
stage: bootstrap_dag
variables:
<< : *osdu_gcp_common_bootstrap_variables
SD_SVC_URL: $OSDU_GCP_SEISMIC_STORE_SERVICE_COMMUNITY_ENDPOINT # i.e https://<ssdms_host>/seistore-svc/api/v3
SEGY_VDS_DOCKER_IMAGE: "community.opengroup.org:5555/osdu/platform/domain-data-mgmt-services/seismic/open-vds/openvds-ingestion:latest"
osdu_gcp_bootstrap_dag_pre_ship:
stage: bootstrap_dag
variables:
<< : *osdu_gcp_common_bootstrap_variables
SD_SVC_URL: $OSDU_GCP_SEISMIC_STORE_SERVICE_PRE_SHIP_ENDPOINT # i.e https://<ssdms_host>/seistore-svc/api/v3
SEGY_VDS_DOCKER_IMAGE: $SEGY_VDS_DOCKER_IMAGE
osdu_gcp_publish_dag:
stage: publish_dag
variables:
BOOTSTRAPPED_DAG_PATH: ${OSDU_GCP_OUTPUT_DAG_DIR}/segy_to_vds_ssdms_conversion_dag.py
DAG_FILE: segy_to_vds_ssdms_conversion_dag.py
osdu_gcp_deploy_composer_community:
stage: deploy
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment