diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ac991c6a83456ca1a5eeb027771e5db90792386a..e49a44777e9708ea17fd07238f074a64924212f4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -59,13 +59,13 @@ include: file: "standard-setup.yml" - project: "osdu/platform/ci-cd-pipelines" - file: "cloud-providers/osdu-gcp-dag.yaml" + file: "cloud-providers/gc-dag.yaml" - project: "osdu/platform/ci-cd-pipelines" file: "cloud-providers/azure_dag.yml" - local: "/devops/azure/override-stages.yml" - - local: "/devops/gcp/pipeline/override-stages.yml" + - local: "/devops/gc/pipeline/override-stages.yml" - local: "/devops/ibm/ibm-stages.yml" - project: "osdu/platform/ci-cd-pipelines" @@ -73,4 +73,3 @@ include: - project: "osdu/platform/ci-cd-pipelines" file: "cloud-providers/aws-dag.yml" - diff --git a/deployments/scripts/anthos/README.md b/deployments/scripts/anthos/README.md index 4a371f5b6ddfd2d77dd742bb5d9ee8c5483fd9db..15837989bb234bbdfd348f870f6f3a26d9e05718 100644 --- a/deployments/scripts/anthos/README.md +++ b/deployments/scripts/anthos/README.md @@ -5,8 +5,8 @@ pip install Jinja2==2.10.1 # Setup Variables -OSDU_GCP_DEPLOYMENTS_SUBDIR="<path_to_folder>" # ie: path to GCP deployment scripts folder -OSDU_GCP_DAG_DIRECTORY="<path_to_folder>" # ie: segy-to-vds-conversion/airflow/workflow-svc-v2 +GC_DEPLOYMENTS_SUBDIR="<path_to_folder>" # ie: path to Google Cloud deployment scripts folder +GC_DAG_DIRECTORY="<path_to_folder>" # ie: segy-to-vds-conversion/airflow/workflow-svc-v2 OUTPUT_FILE="<path_to_output_file>" # ie. to avoid original file overwriting diff --git a/deployments/scripts/anthos/VERSION b/deployments/scripts/anthos/VERSION index 47d04a528837ea50434734bd7cca947d47c4e012..66333910a4beec729370ff97e5044b345383eba4 100644 --- a/deployments/scripts/anthos/VERSION +++ b/deployments/scripts/anthos/VERSION @@ -1 +1 @@ -0.18.0 \ No newline at end of file +0.18.0 diff --git a/deployments/scripts/anthos/bootstrap.sh b/deployments/scripts/anthos/bootstrap.sh index a90eaf4d1b206d141fbbefeb0882ea1ae43071ca..61100fd646b2e0814d9276cc8921462b7d43ed9f 100755 --- a/deployments/scripts/anthos/bootstrap.sh +++ b/deployments/scripts/anthos/bootstrap.sh @@ -22,10 +22,10 @@ echo "------------------------------------"; if [ -z $OUTPUT_FILE ]; then echo "THE ORIGINAL DAG FILE WILL BE OVERWRITTEN" - python3 $OSDU_GCP_DEPLOYMENTS_SUBDIR/render_dag_file.py -f $OSDU_GCP_DAG_DIRECTORY/segy_to_vds_ssdms_conversion_dag.py + python3 $GC_DEPLOYMENTS_SUBDIR/render_dag_file.py -f $GC_DAG_DIRECTORY/segy_to_vds_ssdms_conversion_dag.py else echo "RENDERED OUTPUT WILL BE SAVED INTO ${OUTPUT_FILE}" - python3 $OSDU_GCP_DEPLOYMENTS_SUBDIR/render_dag_file.py -f $OSDU_GCP_DAG_DIRECTORY/segy_to_vds_ssdms_conversion_dag.py -o ${OUTPUT_FILE} + python3 $GC_DEPLOYMENTS_SUBDIR/render_dag_file.py -f $GC_DAG_DIRECTORY/segy_to_vds_ssdms_conversion_dag.py -o ${OUTPUT_FILE} fi if test $? = '1'; then diff --git a/deployments/scripts/anthos/render_dag_file.py b/deployments/scripts/anthos/render_dag_file.py index fb0780caa504dcc5a26e7db96e1d5ff7fa13161f..ba53c41d07b6d2c1ee9f49ad73860ec6f13260fa 100644 --- a/deployments/scripts/anthos/render_dag_file.py +++ b/deployments/scripts/anthos/render_dag_file.py @@ -41,7 +41,7 @@ class DAGFileRenderer: def _render_template(self, file) -> str: env = Environment( - loader=FileSystemLoader(searchpath=os.getenv("OSDU_GCP_DAG_DIRECTORY", default="src/dags")), + loader=FileSystemLoader(searchpath=os.getenv("GC_DAG_DIRECTORY", default="src/dags")), variable_start_string='{|', variable_end_string='|}', ) template = env.get_template(os.path.basename(file.name)) diff --git a/deployments/scripts/gcp/README.md b/deployments/scripts/gc/README.md similarity index 82% rename from deployments/scripts/gcp/README.md rename to deployments/scripts/gc/README.md index 01f00b9a7eee42749901dcb5dd897ec4df080b18..db70a8cdd8b8a694f640656ee34d509e5cfabc82 100644 --- a/deployments/scripts/gcp/README.md +++ b/deployments/scripts/gc/README.md @@ -10,8 +10,8 @@ export SEGY_VDS_DOCKER_IMAGE="<image_tag>" export K8S_NODE_POOLS=<node_pools> # i.e. pool-1,pool-2 the value will be split by comma export SD_SVC_URL=<ssdms_host> # i.e https://<ssdms_host>/seistore-svc/api/v3 -OSDU_GCP_DEPLOYMENTS_SUBDIR="<path_to_folder>" # ie: path to Google Cloud deployment scripts folder -OSDU_GCP_DAG_DIRECTORY="<path_to_folder>" # ie: segy-to-vds-conversion/airflow/workflow-svc-v2 +GC_DEPLOYMENTS_SUBDIR="<path_to_folder>" # ie: path to Google Cloud deployment scripts folder +GC_DAG_DIRECTORY="<path_to_folder>" # ie: segy-to-vds-conversion/airflow/workflow-svc-v2 OUTPUT_FILE="<path_to_output_file>" # ie. to avoid original file overwriting diff --git a/deployments/scripts/gc/VERSION b/deployments/scripts/gc/VERSION new file mode 100644 index 0000000000000000000000000000000000000000..66333910a4beec729370ff97e5044b345383eba4 --- /dev/null +++ b/deployments/scripts/gc/VERSION @@ -0,0 +1 @@ +0.18.0 diff --git a/deployments/scripts/gcp/bootstrap.sh b/deployments/scripts/gc/bootstrap.sh old mode 100755 new mode 100644 similarity index 78% rename from deployments/scripts/gcp/bootstrap.sh rename to deployments/scripts/gc/bootstrap.sh index a90eaf4d1b206d141fbbefeb0882ea1ae43071ca..61100fd646b2e0814d9276cc8921462b7d43ed9f --- a/deployments/scripts/gcp/bootstrap.sh +++ b/deployments/scripts/gc/bootstrap.sh @@ -22,10 +22,10 @@ echo "------------------------------------"; if [ -z $OUTPUT_FILE ]; then echo "THE ORIGINAL DAG FILE WILL BE OVERWRITTEN" - python3 $OSDU_GCP_DEPLOYMENTS_SUBDIR/render_dag_file.py -f $OSDU_GCP_DAG_DIRECTORY/segy_to_vds_ssdms_conversion_dag.py + python3 $GC_DEPLOYMENTS_SUBDIR/render_dag_file.py -f $GC_DAG_DIRECTORY/segy_to_vds_ssdms_conversion_dag.py else echo "RENDERED OUTPUT WILL BE SAVED INTO ${OUTPUT_FILE}" - python3 $OSDU_GCP_DEPLOYMENTS_SUBDIR/render_dag_file.py -f $OSDU_GCP_DAG_DIRECTORY/segy_to_vds_ssdms_conversion_dag.py -o ${OUTPUT_FILE} + python3 $GC_DEPLOYMENTS_SUBDIR/render_dag_file.py -f $GC_DAG_DIRECTORY/segy_to_vds_ssdms_conversion_dag.py -o ${OUTPUT_FILE} fi if test $? = '1'; then diff --git a/deployments/scripts/gcp/render_dag_file.py b/deployments/scripts/gc/render_dag_file.py similarity index 96% rename from deployments/scripts/gcp/render_dag_file.py rename to deployments/scripts/gc/render_dag_file.py index 7ee8907d2f2cee0fef9e18f0c3951ef8f94d5d49..07368d35232c0a4dae0fbe4a0268fe29fa449ae3 100644 --- a/deployments/scripts/gcp/render_dag_file.py +++ b/deployments/scripts/gc/render_dag_file.py @@ -33,7 +33,7 @@ class DAGFileRenderer: def _render_template(self, file) -> str: env = Environment( - loader=FileSystemLoader(searchpath=os.getenv("OSDU_GCP_DAG_DIRECTORY", default="src/dags")), + loader=FileSystemLoader(searchpath=os.getenv("GC_DAG_DIRECTORY", default="src/dags")), variable_start_string='{|', variable_end_string='|}', ) template = env.get_template(os.path.basename(file.name)) diff --git a/deployments/scripts/gcp/requirements.txt b/deployments/scripts/gc/requirements.txt similarity index 100% rename from deployments/scripts/gcp/requirements.txt rename to deployments/scripts/gc/requirements.txt diff --git a/deployments/scripts/gcp/VERSION b/deployments/scripts/gcp/VERSION deleted file mode 100644 index 47d04a528837ea50434734bd7cca947d47c4e012..0000000000000000000000000000000000000000 --- a/deployments/scripts/gcp/VERSION +++ /dev/null @@ -1 +0,0 @@ -0.18.0 \ No newline at end of file diff --git a/devops/gcp/pipeline/override-stages.yml b/devops/gc/pipeline/override-stages.yml similarity index 54% rename from devops/gcp/pipeline/override-stages.yml rename to devops/gc/pipeline/override-stages.yml index a9a96996cfa1e89d82f9365ff021652e5e0deb7e..4ee22447b24bc4ea8f37a72e5084807306608fac 100644 --- a/devops/gcp/pipeline/override-stages.yml +++ b/devops/gc/pipeline/override-stages.yml @@ -1,39 +1,39 @@ variables: - OSDU_GCP_ENABLE_BOOTSTRAP: "false" - OSDU_GCP_SERVICE: segy-to-vds - OSDU_GCP_DAG_DIRECTORY: src/dags - OSDU_GCP_DAG_FILE: segy_to_vds_ssdms_conversion_dag.py - OUTPUT_FILE: $OSDU_GCP_RENDERED_DAG_FOLDER/$OSDU_GCP_DAG_FILE + GC_ENABLE_BOOTSTRAP: "false" + GC_SERVICE: segy-to-vds + GC_DAG_DIRECTORY: src/dags + GC_DAG_FILE: segy_to_vds_ssdms_conversion_dag.py + OUTPUT_FILE: $GC_RENDERED_DAG_FOLDER/$GC_DAG_FILE IMAGE_TAG_VARIABLE_NAME: gcp__image__segy_to_vds_converter - OSDU_GCP_GCR_REGISTRY: gcr.io/osdu-anthos + GC_GCR_REGISTRY: gcr.io/osdu-anthos # Postman tests COLLECTION_PATH: "Postman%20Collection/40_CICD_SEGY_OpenVDS_Conversion/SegyToOpenVDS%20conversion%20using%20Seisstore%20CI-CD%20v1.0.postman_collection.json" # Community, Dev2, Preship -osdu-gcp-render-dag: +gc-render-dag: needs: [] # Anthos -osdu-gcp-render-dag:anthos: +gc-render-dag:anthos: needs: [] -osdu-gcp-deploy-dag:community: +gc-deploy-dag:community: variables: # FIXME (GONRG-5847): an open-vds image is created in a different repo, need to implement a new logic on how to update the open-vds image in composer with sha/latest/release/tag versions - OSDU_GCP_COMPOSER_TAG: community.opengroup.org:5555/osdu/platform/domain-data-mgmt-services/seismic/open-vds/openvds-ingestion:latest + GC_COMPOSER_TAG: community.opengroup.org:5555/osdu/platform/domain-data-mgmt-services/seismic/open-vds/openvds-ingestion:latest -osdu-gcp-deploy-dag:anthos: +gc-deploy-dag:anthos: variables: IMAGE_TAG_VARIABLE_NAME: anthos__image__segy_to_vds_converter # FIXME (GONRG-5847): an open-vds image is created in a different repo, need to implement a new logic on how to update the open-vds image in composer with sha/latest/release/tag versions - OSDU_GCP_COMPOSER_TAG: community.opengroup.org:5555/osdu/platform/domain-data-mgmt-services/seismic/open-vds/openvds-ingestion:latest + GC_COMPOSER_TAG: community.opengroup.org:5555/osdu/platform/domain-data-mgmt-services/seismic/open-vds/openvds-ingestion:latest -osdu-gcp-deploy-dag:dev2: +gc-deploy-dag:dev2: variables: # FIXME (GONRG-5847): an open-vds image is created in a different repo, need to implement a new logic on how to update the open-vds image in composer with sha/latest/release/tag versions - OSDU_GCP_COMPOSER_TAG: community.opengroup.org:5555/osdu/platform/domain-data-mgmt-services/seismic/open-vds/openvds-ingestion:latest + GC_COMPOSER_TAG: community.opengroup.org:5555/osdu/platform/domain-data-mgmt-services/seismic/open-vds/openvds-ingestion:latest -osdu-gcp-deploy-dag:preship: +gc-deploy-dag:preship: variables: # FIXME (GONRG-5847): an open-vds image is created in a different repo, need to implement a new logic on how to update the open-vds image in composer with sha/latest/release/tag versions - OSDU_GCP_COMPOSER_TAG: community.opengroup.org:5555/osdu/platform/domain-data-mgmt-services/seismic/open-vds/openvds-ingestion:latest + GC_COMPOSER_TAG: community.opengroup.org:5555/osdu/platform/domain-data-mgmt-services/seismic/open-vds/openvds-ingestion:latest diff --git a/src/dags/segy_to_vds_conversion_dag.py b/src/dags/segy_to_vds_conversion_dag.py index 076379073213bfc6e1bcec3cc2917ce93ab3b557..df5a38e9ce893d898e6ec37a574b129dec010180 100644 --- a/src/dags/segy_to_vds_conversion_dag.py +++ b/src/dags/segy_to_vds_conversion_dag.py @@ -20,7 +20,7 @@ from datetime import timedelta import airflow from airflow import DAG from airflow.operators.dummy_operator import DummyOperator -from operators.gcp.kubernetes_pod_with_auth import \ +from operators.gc.kubernetes_pod_with_auth import \ KubernetesPodOperatorWithAuth default_args = { diff --git a/src/plugins/operators/gcp/__init__.py b/src/plugins/operators/gc/__init__.py similarity index 100% rename from src/plugins/operators/gcp/__init__.py rename to src/plugins/operators/gc/__init__.py diff --git a/src/plugins/operators/gcp/kubernetes_pod_with_auth.py b/src/plugins/operators/gc/kubernetes_pod_with_auth.py similarity index 100% rename from src/plugins/operators/gcp/kubernetes_pod_with_auth.py rename to src/plugins/operators/gc/kubernetes_pod_with_auth.py