Commit f5672ec1 authored by Siarhei Khaletski (EPAM)'s avatar Siarhei Khaletski (EPAM) 🚩
Browse files

Merge branch 'GONRG-3115_gcp_ini_file_generation' into 'master'

GONRG-3115: add ini file for GCP

See merge request osdu/platform/data-flow/ingestion/energistics-osdu-integration!29
parents c59bc9ef a3b9c18b
Pipeline #63569 skipped with stages
......@@ -19,6 +19,8 @@ variables:
OSDU_AIRFLOW_IMAGE: johnybear/osdu-airflow:python36-1
OSDU_GCP_VENDOR: gcp
OSDU_GCP_APPLICATION_NAME: energistics
OSDU_GCP_DATASET_BASE_URL: https://os-dataset-attcrcktoa-uc.a.run.app
OSDU_GCP_MOCK_BASE_URL: localhost
WORKDIR: energistics
SRCDIR: energistics/src/
......@@ -58,6 +60,11 @@ build_base_image:
when: manual
.gcp_update_ini_file: &gcp_update_ini_file
- echo ---- ---- ---- UPDATE ENV VARS ---- ---- ----
- sed -i "s|%(DATASET_BASE_URL)s|$OSDU_GCP_DATASET_BASE_URL|" build/providers/gcp/osdu_api.ini
- sed -i "s|%([_A-Z]*_BASE_URL)s|$OSDU_GCP_MOCK_BASE_URL|" build/providers/gcp/osdu_api.ini
build_osdu_gcp_image:
stage: csp_containerize
tags: ["osdu-medium"]
......@@ -66,6 +73,7 @@ build_osdu_gcp_image:
- echo ---- ---- ---- SYSTEM DEPENDENCIES ---- ---- ----
- apk update
- apk add git
- *gcp_update_ini_file
- echo ---- ---- ---- BUILD IMAGE ---- ---- ----
- docker build -t $OSDU_GCP_IMAGE_TAG --rm -f ./build/providers/$OSDU_GCP_VENDOR/Dockerfile.energistics --build-arg PROVIDER_NAME=$OSDU_GCP_VENDOR --build-arg base_image=$BASE_IMAGE_TAG .
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
......@@ -91,6 +99,7 @@ build_osdu_gcp_release_image:
- echo ---- ---- ---- SYSTEM DEPENDENCIES ---- ---- ----
- apk update
- apk add git
- *gcp_update_ini_file
- echo ---- ---- ---- BUILD IMAGE ---- ---- ----
- docker build -t $OSDU_GCP_IMAGE_TAG:release-latest --rm -f ./build/providers/$OSDU_GCP_VENDOR/Dockerfile.energistics --build-arg PROVIDER_NAME=$OSDU_GCP_VENDOR --build-arg base_image=$BASE_IMAGE_TAG .
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
......
......@@ -23,3 +23,4 @@ ENV CLOUD_PROVIDER $PROVIDER_NAME
COPY "build/providers/${PROVIDER_NAME}/requirements.txt" ./
RUN pip install -r "requirements.txt"
COPY build/providers/gcp/osdu_api.ini ./
# Copyright 2021 Google LLC
# Copyright 2021 EPAM
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[environment]
data_partition_id=osdu
storage_url=%(STORAGE_BASE_URL)s/api/storage/v2
search_url=%(SEARCH_BASE_URL)s/api/search/v2
legal_url=%(LEGAL_BASE_URL)s/api/legal/v1
data_workflow_url=%(DATA_WORKFLOW_BASE_URL)s/api/data-workflow/v1
file_dms_url=%(FILE_DMS_BASE_URL)s/api/filedms/v2
dataset_url=%(DATASET_BASE_URL)s/api/dataset/v1
entitlements_url=%(ENTITLEMENTS_BASE_URL)s/api/entitlements/v1
schema_url=%(SCHEMA_BASE_URL)s/api/schema-service/v1
ingestion_workflow_url=%(INGESTION_WORKFLOW_BASE_URL)s/api/workflow/v1
use_service_principal=False
[provider]
name=gcp
\ No newline at end of file
......@@ -6,4 +6,4 @@ tenacity
dataclasses==0.8;python_version<"3.7"
--extra-index-url https://community.opengroup.org/api/v4/projects/148/packages/pypi/simple
osdu_api==0.10.1.dev0+987cff77
osdu_api==0.10.1.dev0+ad9ff99f
......@@ -81,7 +81,7 @@ process_energistics_op = KubernetesPodOperator(
"python main.py"
" --context '{{ dag_run.conf['execution_context'] | tojson }}'"
" --file_service {{ var.value.core__service__file__host }}"
" --out /airflow/xcom/return.json",
" --out /airflow/xcom/return.json ",
],
is_delete_operator_pod=True,
image_pull_policy="Always",
......
......@@ -116,8 +116,9 @@ class TestCreateEnergisticsManifest:
monkeypatch.setattr(create_energistics_manifest, "get_file_content",
lambda *args, **kwargs: get_file_content(xml_file))
token_refresher = BaseTokenRefresher(creds="test")
context = Context("test", "test")
manifest_creator = create_energistics_manifest.EnergisticsManifestCreator(mock_file_handler, context)
manifest_creator = create_energistics_manifest.EnergisticsManifestCreator(mock_file_handler, token_refresher, context)
manifest = manifest_creator.create_manifest(MOCK_EXECUTION_CONTEXT)
jsonschema.validate(manifest, schema)
......@@ -23,6 +23,7 @@ import json
import dataclasses
import requests
import tenacity
from osdu_api.libs.auth.authorization import TokenRefresher
from osdu_api.libs.context import Context
from osdu_api.libs.handle_file import FileHandler
from osdu_api.libs.utils import remove_trailing_colon
......@@ -129,9 +130,11 @@ class EnergisticsManifestCreator:
def __init__(
self,
file_handler: FileHandler,
token_refresher: TokenRefresher,
payload_context: Context
) -> None:
self.file_handler = file_handler
self.token_refresher = token_refresher
self.payload_context = payload_context
@staticmethod
......@@ -229,7 +232,8 @@ class EnergisticsManifestCreator:
if execution_context["Context"].get("dataset_id"):
dataset_dms_client = DatasetDmsClient(DefaultConfigManager(), self.payload_context.data_partition_id)
retrieval_instructions_response = dataset_dms_client.get_retrieval_instructions(record_id=execution_context["Context"]["dataset_id"])
bearer_token = None if dataset_dms_client.use_service_principal else self.token_refresher.refresh_token()
retrieval_instructions_response = dataset_dms_client.get_retrieval_instructions(record_id=execution_context["Context"]["dataset_id"], bearer_token=bearer_token)
# logger.info(f"Dataset retrieval response code: {retrieval_instructions_response.status_code}")
print(f"Dataset retrieval response code: {retrieval_instructions_response.status_code}")
......
......@@ -31,7 +31,7 @@ def main(execution_context: dict, file_service_url: str):
payload_context = Context.populate(execution_context)
token_refresher = BaseTokenRefresher()
file_handler = FileHandler(file_service_url, token_refresher, payload_context)
manifest_creator = EnergisticsManifestCreator(file_handler, payload_context)
manifest_creator = EnergisticsManifestCreator(file_handler, token_refresher, payload_context)
manifest = manifest_creator.create_manifest(execution_context)
return {"manifest": manifest}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment