Merge branch 'GONRG-3115_gcp_ini_file_generation' into 'master'

GONRG-3115: add ini file for GCP

See merge request !29

.gitlab-ci.yml

@@ -19,6 +19,8 @@ variables:
   OSDU_AIRFLOW_IMAGE: johnybear/osdu-airflow:python36-1
   OSDU_GCP_VENDOR: gcp
   OSDU_GCP_APPLICATION_NAME: energistics
+  OSDU_GCP_DATASET_BASE_URL: https://os-dataset-attcrcktoa-uc.a.run.app
+  OSDU_GCP_MOCK_BASE_URL: localhost
 
   WORKDIR: energistics
   SRCDIR:  energistics/src/
@@ -58,6 +60,11 @@ build_base_image:
   when: manual
   
 
+.gcp_update_ini_file: &gcp_update_ini_file
+  - echo ---- ---- ---- UPDATE ENV VARS ---- ---- ----
+  - sed -i "s|%(DATASET_BASE_URL)s|$OSDU_GCP_DATASET_BASE_URL|" build/providers/gcp/osdu_api.ini
+  - sed -i "s|%([_A-Z]*_BASE_URL)s|$OSDU_GCP_MOCK_BASE_URL|" build/providers/gcp/osdu_api.ini
+
 build_osdu_gcp_image:
   stage: csp_containerize
   tags: ["osdu-medium"]
@@ -66,6 +73,7 @@ build_osdu_gcp_image:
     - echo ---- ---- ---- SYSTEM DEPENDENCIES ---- ---- ----
     - apk update
     - apk add git
+    - *gcp_update_ini_file
     - echo ---- ---- ---- BUILD IMAGE ---- ---- ----
     - docker build -t $OSDU_GCP_IMAGE_TAG --rm -f ./build/providers/$OSDU_GCP_VENDOR/Dockerfile.energistics --build-arg PROVIDER_NAME=$OSDU_GCP_VENDOR --build-arg base_image=$BASE_IMAGE_TAG .
     - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
@@ -91,6 +99,7 @@ build_osdu_gcp_release_image:
     - echo ---- ---- ---- SYSTEM DEPENDENCIES ---- ---- ----
     - apk update
     - apk add git
+    - *gcp_update_ini_file
     - echo ---- ---- ---- BUILD IMAGE ---- ---- ----
     - docker build -t $OSDU_GCP_IMAGE_TAG:release-latest --rm -f ./build/providers/$OSDU_GCP_VENDOR/Dockerfile.energistics --build-arg PROVIDER_NAME=$OSDU_GCP_VENDOR --build-arg base_image=$BASE_IMAGE_TAG .
     - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY

build/providers/gcp/Dockerfile.energistics

@@ -23,3 +23,4 @@ ENV CLOUD_PROVIDER $PROVIDER_NAME
 
 COPY "build/providers/${PROVIDER_NAME}/requirements.txt" ./
 RUN pip install -r "requirements.txt"
+COPY build/providers/gcp/osdu_api.ini ./

build/providers/gcp/osdu_api.ini

+# Copyright 2021 Google LLC
+# Copyright 2021 EPAM
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[environment]
+data_partition_id=osdu
+storage_url=%(STORAGE_BASE_URL)s/api/storage/v2
+search_url=%(SEARCH_BASE_URL)s/api/search/v2
+legal_url=%(LEGAL_BASE_URL)s/api/legal/v1
+data_workflow_url=%(DATA_WORKFLOW_BASE_URL)s/api/data-workflow/v1
+file_dms_url=%(FILE_DMS_BASE_URL)s/api/filedms/v2
+dataset_url=%(DATASET_BASE_URL)s/api/dataset/v1
+entitlements_url=%(ENTITLEMENTS_BASE_URL)s/api/entitlements/v1
+schema_url=%(SCHEMA_BASE_URL)s/api/schema-service/v1
+ingestion_workflow_url=%(INGESTION_WORKFLOW_BASE_URL)s/api/workflow/v1
+use_service_principal=False
+
+[provider]
+name=gcp
\ No newline at end of file

build/requirements.txt

@@ -6,4 +6,4 @@ tenacity
 dataclasses==0.8;python_version<"3.7"
 
 --extra-index-url https://community.opengroup.org/api/v4/projects/148/packages/pypi/simple
-osdu_api==0.10.1.dev0+987cff77
+osdu_api==0.10.1.dev0+ad9ff99f

energistics/src/dags/energistics/witsml_parser_dag.py

@@ -81,7 +81,7 @@ process_energistics_op = KubernetesPodOperator(
         "python main.py"
         " --context '{{ dag_run.conf['execution_context'] | tojson }}'"
         " --file_service {{ var.value.core__service__file__host }}"
-        " --out /airflow/xcom/return.json",
+        " --out /airflow/xcom/return.json ",
     ],
     is_delete_operator_pod=True,
     image_pull_policy="Always",

energistics/src/docker_test/test_create_energistics_manifest.py

@@ -116,8 +116,9 @@ class TestCreateEnergisticsManifest:
 
         monkeypatch.setattr(create_energistics_manifest, "get_file_content",
                             lambda *args, **kwargs: get_file_content(xml_file))
+        token_refresher = BaseTokenRefresher(creds="test")
         context = Context("test", "test")
-        manifest_creator = create_energistics_manifest.EnergisticsManifestCreator(mock_file_handler, context)
+        manifest_creator = create_energistics_manifest.EnergisticsManifestCreator(mock_file_handler, token_refresher, context)
         manifest = manifest_creator.create_manifest(MOCK_EXECUTION_CONTEXT)
 
         jsonschema.validate(manifest, schema)

energistics/src/witsml_parser/create_energistics_manifest.py

@@ -23,6 +23,7 @@ import json
 import dataclasses
 import requests
 import tenacity
+from osdu_api.libs.auth.authorization import TokenRefresher
 from osdu_api.libs.context import Context
 from osdu_api.libs.handle_file import FileHandler
 from osdu_api.libs.utils import remove_trailing_colon
@@ -129,9 +130,11 @@ class EnergisticsManifestCreator:
     def __init__(
             self,
             file_handler: FileHandler,
+            token_refresher: TokenRefresher,
             payload_context: Context
     ) -> None:
         self.file_handler = file_handler
+        self.token_refresher = token_refresher
         self.payload_context = payload_context
 
     @staticmethod
@@ -229,7 +232,8 @@ class EnergisticsManifestCreator:
 
         if execution_context["Context"].get("dataset_id"):
             dataset_dms_client = DatasetDmsClient(DefaultConfigManager(), self.payload_context.data_partition_id)
-            retrieval_instructions_response = dataset_dms_client.get_retrieval_instructions(record_id=execution_context["Context"]["dataset_id"])
+            bearer_token = None if dataset_dms_client.use_service_principal else self.token_refresher.refresh_token()
+            retrieval_instructions_response = dataset_dms_client.get_retrieval_instructions(record_id=execution_context["Context"]["dataset_id"], bearer_token=bearer_token)
             
             # logger.info(f"Dataset retrieval response code: {retrieval_instructions_response.status_code}")
             print(f"Dataset retrieval response code: {retrieval_instructions_response.status_code}")

energistics/src/witsml_parser/main.py

@@ -31,7 +31,7 @@ def main(execution_context: dict, file_service_url: str):
     payload_context = Context.populate(execution_context)
     token_refresher = BaseTokenRefresher()
     file_handler = FileHandler(file_service_url, token_refresher, payload_context)
-    manifest_creator = EnergisticsManifestCreator(file_handler, payload_context)
+    manifest_creator = EnergisticsManifestCreator(file_handler, token_refresher, payload_context)
     manifest = manifest_creator.create_manifest(execution_context)
     return {"manifest": manifest}