From efc0ff216371510e663745d1ef0910d3d4131127 Mon Sep 17 00:00:00 2001
From: Dmitrii <dmitrii_valuiskii@epam.com>
Date: Mon, 8 Feb 2021 12:41:28 +0300
Subject: [PATCH] GONRG-1709: update workflow service interaction
---
src/dags/default-ingest.py | 2 +-
src/dags/libs/update_status.py | 33 +-
src/dags/osdu-ingest-r2.py | 7 +-
src/dags/osdu-ingest-r3.py | 3 +-
src/plugins/operators/deprecated/__init__.py | 0
.../operators/deprecated/update_status.py | 96 +++
src/plugins/operators/process_manifest_r3.py | 8 +-
src/plugins/operators/update_status.py | 20 +-
.../data/invalid/EmptyManifest.json | 19 +-
.../data/invalid/NotOsduFormat.json | 23 +-
.../data/master/Wellbore.0.3.0.json | 101 +--
.../data/workProduct/SeismicTraceData.json | 768 +++++++++---------
.../test_manifest_processor_r3.py | 2 +-
.../test_manifest_traversal.py | 2 +-
tests/plugin-unit-tests/test_operators_r3.py | 29 +-
.../test_update_status_r3.py | 12 +-
16 files changed, 625 insertions(+), 500 deletions(-)
create mode 100644 src/plugins/operators/deprecated/__init__.py
create mode 100644 src/plugins/operators/deprecated/update_status.py
diff --git a/src/dags/default-ingest.py b/src/dags/default-ingest.py
index 7e0bfe2..5536623 100644
--- a/src/dags/default-ingest.py
+++ b/src/dags/default-ingest.py
@@ -21,7 +21,7 @@ from datetime import timedelta
import airflow.utils.dates
from airflow import DAG
from airflow.operators.python_operator import PythonOperator
-from operators.update_status import UpdateStatusOperator
+from operators.deprecated.update_status import UpdateStatusOperator
from libs.create_records import create_records # isort:skip
diff --git a/src/dags/libs/update_status.py b/src/dags/libs/update_status.py
index 3e44375..d6ac935 100644
--- a/src/dags/libs/update_status.py
+++ b/src/dags/libs/update_status.py
@@ -19,6 +19,7 @@ import json
import logging
import requests
+
from libs.context import Context
from libs.mixins import HeadersMixin
from libs.refresh_token import TokenRefresher, refresh_token
@@ -31,7 +32,9 @@ class UpdateStatus(HeadersMixin):
def __init__(
self,
+ workflow_name: str,
workflow_id: str,
+ run_id: str,
workflow_url: str,
status: str,
token_refresher: TokenRefresher,
@@ -39,8 +42,12 @@ class UpdateStatus(HeadersMixin):
) -> None:
"""Init the status update processor.
+ :param workflow_name: the name of the workflow
+ :type workflow_name: str
:param workflow_id: The id of the workflow
:type workflow_id: str
+ :param run_id: The id of workflow run instance
+ :type run_id: str
:param workflow_url: The base url of the Workflow service
:type workflow_url: str
:param status: The status
@@ -51,8 +58,10 @@ class UpdateStatus(HeadersMixin):
:type context: Context
"""
super().__init__(context)
+ self.workflow_name = workflow_name
self.workflow_url = workflow_url
self.workflow_id = workflow_id
+ self.run_id = run_id
self.context = context
self.status = status
self.token_refresher = token_refresher
@@ -61,6 +70,25 @@ class UpdateStatus(HeadersMixin):
def update_status_request(self, headers: dict) -> requests.Response:
"""Send request to update status.
+ :param headers: The request headers
+ :type headers: dict
+ :return: The Workflow server response
+ :rtype: requests.Response
+ """
+ request_body = {
+ "status": self.status
+ }
+ request_body = json.dumps(request_body)
+ logger.debug(f"Sending request '{request_body}'")
+ update_status_url = f"{self.workflow_url}/v1/workflow/{self.workflow_name}/workflowRun/{self.run_id}"
+ logger.debug(f"Workflow URL: {update_status_url}")
+ response = requests.put(update_status_url, request_body, headers=headers)
+ return response
+
+ @refresh_token()
+ def update_status_request_old(self, headers: dict) -> requests.Response:
+ """Send request to update status.
+
:param headers: The request headers
:type headers: dict
:return: The Workflow server response
@@ -78,4 +106,7 @@ class UpdateStatus(HeadersMixin):
def update_workflow_status(self):
"""Updates workflow status."""
headers = self.request_headers
- self.update_status_request(headers)
+ if self.workflow_name:
+ self.update_status_request(headers)
+ else:
+ self.update_status_request_old(headers)
diff --git a/src/dags/osdu-ingest-r2.py b/src/dags/osdu-ingest-r2.py
index f59aab8..90c7caa 100644
--- a/src/dags/osdu-ingest-r2.py
+++ b/src/dags/osdu-ingest-r2.py
@@ -19,9 +19,9 @@ from datetime import timedelta
import airflow
from airflow import DAG
+from operators.deprecated.update_status import UpdateStatusOperator
from operators.process_manifest_r2 import ProcessManifestOperatorR2
from operators.search_record_id import SearchRecordIdOperator
-from operators.update_status import UpdateStatusOperator
default_args = {
"start_date": airflow.utils.dates.days_ago(0),
@@ -30,8 +30,9 @@ default_args = {
"trigger_rule": "none_failed",
}
+workflow_name = "Osdu_ingest_r2"
dag = DAG(
- "Osdu_ingest_r2",
+ workflow_name,
default_args=default_args,
description="liveness monitoring dag",
schedule_interval=None,
@@ -46,7 +47,7 @@ update_status_running_op = UpdateStatusOperator(
update_status_finished_op = UpdateStatusOperator(
task_id="update_status_finished_task",
dag=dag,
- trigger_rule="all_done",
+ trigger_rule="all_done"
)
process_manifest_op = ProcessManifestOperatorR2(
diff --git a/src/dags/osdu-ingest-r3.py b/src/dags/osdu-ingest-r3.py
index 2354ed8..1177305 100644
--- a/src/dags/osdu-ingest-r3.py
+++ b/src/dags/osdu-ingest-r3.py
@@ -29,8 +29,9 @@ default_args = {
"trigger_rule": "none_failed",
}
+workflow_name = "Osdu_ingest"
dag = DAG(
- "Osdu_ingest",
+ workflow_name,
default_args=default_args,
description="liveness monitoring dag",
schedule_interval=None,
diff --git a/src/plugins/operators/deprecated/__init__.py b/src/plugins/operators/deprecated/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/plugins/operators/deprecated/update_status.py b/src/plugins/operators/deprecated/update_status.py
new file mode 100644
index 0000000..dfb2197
--- /dev/null
+++ b/src/plugins/operators/deprecated/update_status.py
@@ -0,0 +1,96 @@
+# Copyright 2020 Google LLC
+# Copyright 2020 EPAM Systems
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Update Status operator."""
+
+import copy
+import enum
+import logging
+
+from airflow.models import BaseOperator, Variable
+from libs.context import Context
+from libs.refresh_token import AirflowTokenRefresher
+from libs.exceptions import PipelineFailedError
+from libs.update_status import UpdateStatus
+
+
+logger = logging.getLogger()
+
+
+class UpdateStatusOperator(BaseOperator):
+ """Operator to update status."""
+ ui_color = '#10ECAA'
+ ui_fgcolor = '#000000'
+
+ class prev_ti_state(enum.Enum):
+ NONE = "running"
+ SUCCESS = "finished"
+ FAILED = "failed"
+
+ def get_previous_ti_statuses(self, context: dict) -> enum.Enum:
+ """Get status of previous tasks' executions.
+ Return corresponding enum value.
+
+ :param context: Airflow context
+ :type context: dict
+ :return: Previous status
+ :rtype: enum.Enum
+ """
+ dagrun = context['ti'].get_dagrun()
+ failed_ti = dagrun.get_task_instances(state='failed')
+ success_ti = dagrun.get_task_instances(state='success')
+ if not failed_ti and not success_ti: # There is no prev task so it can't have been failed
+ logger.info("There are no tasks before this one. So it has status RUNNING")
+ return self.prev_ti_state.NONE
+ if failed_ti:
+ logger.info("There are failed tasks before this one. So it has status FAILED")
+ return self.prev_ti_state.FAILED
+ logger.info("There are successed tasks before this one. So it has status SUCCESSED")
+ return self.prev_ti_state.SUCCESS
+
+ def pre_execute(self, context: dict):
+ self.status = self.get_previous_ti_statuses(context)
+
+ def execute(self, context: dict):
+ """Execute update workflow status.
+ If status assumed to be FINISHED then we check whether records
+ are searchable or not.
+ If they are then update status FINISHED else FAILED
+
+ :param context: Airflow context
+ :type context: dict
+ :raises PipelineFailedError: If any of the status is failed
+ """
+ conf = copy.deepcopy(context["dag_run"].conf)
+ logger.debug(f"Got conf {conf}.")
+ if "Payload" in conf:
+ payload_context = Context.populate(conf)
+ else:
+ payload_context = Context(data_partition_id=conf["data-partition-id"],
+ app_key=conf.get("AppKey", ""))
+ workflow_id = conf["WorkflowID"]
+ status = self.status.value
+ status_updater = UpdateStatus(
+ workflow_name="",
+ workflow_url=Variable.get("update_status_url"),
+ workflow_id=workflow_id,
+ run_id="",
+ status=status,
+ token_refresher=AirflowTokenRefresher(),
+ context=payload_context
+ )
+ status_updater.update_workflow_status()
+ if self.status is self.prev_ti_state.FAILED:
+ raise PipelineFailedError("Dag failed")
diff --git a/src/plugins/operators/process_manifest_r3.py b/src/plugins/operators/process_manifest_r3.py
index 26c29d7..37218db 100644
--- a/src/plugins/operators/process_manifest_r3.py
+++ b/src/plugins/operators/process_manifest_r3.py
@@ -56,7 +56,9 @@ class ProcessManifestOperatorR3(BaseOperator):
:param context: Airflow context
:type context: dict
"""
- payload_context = Context.populate(context["dag_run"].conf)
+
+ execution_context = context["dag_run"].conf["execution_context"]
+ payload_context = Context.populate(execution_context)
token_refresher = AirflowTokenRefresher()
file_handler = FileHandler(self.file_service_url, token_refresher, payload_context)
source_file_checker = SourceFileChecker()
@@ -66,8 +68,8 @@ class ProcessManifestOperatorR3(BaseOperator):
token_refresher,
payload_context
)
- manifest_schema = validator.validate_common_schema(context["dag_run"].conf)
- traversal = ManifestTraversal(context["dag_run"].conf, manifest_schema)
+ manifest_schema = validator.validate_common_schema(execution_context)
+ traversal = ManifestTraversal(execution_context, manifest_schema)
manifest_entities = traversal.traverse_manifest()
logger.debug(f"entities count: {len(manifest_entities)}")
valid_manifest_entities = validator.validate_manifest(manifest_entities)
diff --git a/src/plugins/operators/update_status.py b/src/plugins/operators/update_status.py
index 43c0126..3434c96 100644
--- a/src/plugins/operators/update_status.py
+++ b/src/plugins/operators/update_status.py
@@ -45,7 +45,7 @@ class UpdateStatusOperator(BaseOperator):
:param context: Airflow context
:type context: dict
- :return: Previous status
+ :return: Previous status
:rtype: enum.Enum
"""
dagrun = context['ti'].get_dagrun()
@@ -75,16 +75,20 @@ class UpdateStatusOperator(BaseOperator):
"""
conf = copy.deepcopy(context["dag_run"].conf)
logger.debug(f"Got conf {conf}.")
- if "Payload" in conf:
- payload_context = Context.populate(conf)
+ execution_context = conf["execution_context"]
+ if "Payload" in execution_context:
+ payload_context = Context.populate(execution_context)
else:
- payload_context = Context(data_partition_id=conf["data-partition-id"],
- app_key=conf.get("AppKey", ""))
- workflow_id = conf["WorkflowID"]
+ payload_context = Context(data_partition_id=execution_context["data-partition-id"],
+ app_key=execution_context.get("AppKey", ""))
+ workflow_name = conf["workflow_name"]
+ run_id = conf["run_id"]
status = self.status.value
status_updater = UpdateStatus(
- workflow_url=Variable.get("update_status_url"),
- workflow_id=workflow_id,
+ workflow_name=workflow_name,
+ workflow_url=Variable.get("workflow_url"),
+ workflow_id="",
+ run_id=run_id,
status=status,
token_refresher=AirflowTokenRefresher(),
context=payload_context
diff --git a/tests/plugin-unit-tests/data/invalid/EmptyManifest.json b/tests/plugin-unit-tests/data/invalid/EmptyManifest.json
index 0a1b070..433814d 100644
--- a/tests/plugin-unit-tests/data/invalid/EmptyManifest.json
+++ b/tests/plugin-unit-tests/data/invalid/EmptyManifest.json
@@ -1,11 +1,14 @@
{
- "Payload": {
- "authorization": "Bearer test",
- "data-partition-id": "opendes",
- "AppKey": "",
- "kind_version": "3.0.0"
+ "execution_context": {
+ "Payload": {
+ "authorization": "Bearer test",
+ "data-partition-id": "opendes",
+ "AppKey": "",
+ "kind_version": "3.0.0"
+ },
+ "$schema": "https://schema.osdu.opengroup.org/json/master-data/Wellbore.1.0.0.json",
+ "$filename": "load_Wellbore.1.0.0_350112350400.json"
},
- "$schema": "https://schema.osdu.opengroup.org/json/master-data/Wellbore.1.0.0.json",
- "$filename": "load_Wellbore.1.0.0_350112350400.json",
- "WorkflowID": "foo"
+ "workflow_name": "osdu_ingest",
+ "run_id": "foo"
}
diff --git a/tests/plugin-unit-tests/data/invalid/NotOsduFormat.json b/tests/plugin-unit-tests/data/invalid/NotOsduFormat.json
index 88b8815..cfe1cf7 100644
--- a/tests/plugin-unit-tests/data/invalid/NotOsduFormat.json
+++ b/tests/plugin-unit-tests/data/invalid/NotOsduFormat.json
@@ -1,12 +1,17 @@
{
- "Payload": {
- "authorization": "Bearer test",
- "data-partition-id": "opendes",
- "AppKey": "",
- "kind_version": "3.0.0"
+ "execution_context": {
+ "Payload": {
+ "authorization": "Bearer test",
+ "data-partition-id": "opendes",
+ "AppKey": "",
+ "kind_version": "3.0.0"
+ },
+ "$schema": "https://schema.osdu.opengroup.org/json/master-data/Wellbore.1.0.0.json",
+ "$filename": "load_Wellbore.1.0.0_350112350400.json",
+ "manifest": {
+ "test": "test"
+ }
},
- "$schema": "https://schema.osdu.opengroup.org/json/master-data/Wellbore.1.0.0.json",
- "$filename": "load_Wellbore.1.0.0_350112350400.json",
- "manifest": {"test": "test"},
- "WorkflowID": "foo"
+ "workflow_name": "osdu_ingest",
+ "run_id": "foo"
}
diff --git a/tests/plugin-unit-tests/data/master/Wellbore.0.3.0.json b/tests/plugin-unit-tests/data/master/Wellbore.0.3.0.json
index 9f1bbd6..f716b42 100644
--- a/tests/plugin-unit-tests/data/master/Wellbore.0.3.0.json
+++ b/tests/plugin-unit-tests/data/master/Wellbore.0.3.0.json
@@ -1,54 +1,57 @@
{
- "Payload": {
- "authorization": "Bearer test",
- "data-partition-id": "opendes",
- "AppKey": "",
- "kind_version": "3.0.0"
- },
- "$schema": "https://schema.osdu.opengroup.org/json/master-data/Wellbore.1.0.0.json",
- "$filename": "load_Wellbore.1.0.0_350112350400.json",
- "manifest": {
- "kind": "test:test:Manifest:1.0.0",
- "ReferenceData": [],
- "MasterData": [
- {
- "id": "opendes:master-data/Wellbore:350112350400",
- "kind": "opendes:osdu:TestMaster:0.3.0",
- "groupType": "master-data",
- "version": 1,
- "acl": {
- "owners": [
- "data.default.viewers@opendes.osdu-gcp.go3-nrg.projects.epam.com"
- ],
- "viewers": [
- "data.default.owners@opendes.osdu-gcp.go3-nrg.projects.epam.com"
- ]
- },
- "legal": {
- "legaltags": [
- "opendes-demo-legaltag"
- ],
- "otherRelevantDataCountries": [
- "srn:opendes:master-data/GeoPoliticalEntity:USA:"
- ],
- "status": "srn:opendes:reference-data/LegalStatus:public:1111"
- },
- "resourceHostRegionIDs": [
- "srn:opendes:reference-data/OSDURegion:US-EAST:"
+ "execution_context": {
+ "Payload": {
+ "authorization": "Bearer test",
+ "data-partition-id": "opendes",
+ "AppKey": "",
+ "kind_version": "3.0.0"
+ },
+ "$schema": "https://schema.osdu.opengroup.org/json/master-data/Wellbore.1.0.0.json",
+ "$filename": "load_Wellbore.1.0.0_350112350400.json",
+ "manifest": {
+ "kind": "test:test:Manifest:1.0.0",
+ "ReferenceData": [],
+ "MasterData": [
+ {
+ "id": "opendes:master-data/Wellbore:350112350400",
+ "kind": "opendes:osdu:TestMaster:0.3.0",
+ "groupType": "master-data",
+ "version": 1,
+ "acl": {
+ "owners": [
+ "data.default.viewers@opendes.osdu-gcp.go3-nrg.projects.epam.com"
+ ],
+ "viewers": [
+ "data.default.owners@opendes.osdu-gcp.go3-nrg.projects.epam.com"
+ ]
+ },
+ "legal": {
+ "legaltags": [
+ "opendes-demo-legaltag"
+ ],
+ "otherRelevantDataCountries": [
+ "srn:opendes:master-data/GeoPoliticalEntity:USA:"
+ ],
+ "status": "srn:opendes:reference-data/LegalStatus:public:1111"
+ },
+ "resourceHostRegionIDs": [
+ "srn:opendes:reference-data/OSDURegion:US-EAST:"
+ ],
+ "resourceObjectCreationDateTime": "2020-10-16T11:14:45-05:00",
+ "resourceVersionCreationDateTime": "2020-10-16T11:14:45-05:00",
+ "resourceSecurityClassification": "srn:opendes:reference-data/ResourceSecurityClassification:public:",
+ "source": "srn:opendes:master-data/Organisation:Oklahoma Corporation Commission:",
+ "existenceKind": "srn:opendes:reference-data/ExistenceKind:Active:",
+ "licenseState": "srn:opendes:reference-data/LicenseState:Unlicensed:",
+ "data": {
+ "SequenceNumber": 1
+ },
+ "schema": "test:test:GenericMasterData:1.0.0"
+ }
],
- "resourceObjectCreationDateTime": "2020-10-16T11:14:45-05:00",
- "resourceVersionCreationDateTime": "2020-10-16T11:14:45-05:00",
- "resourceSecurityClassification": "srn:opendes:reference-data/ResourceSecurityClassification:public:",
- "source": "srn:opendes:master-data/Organisation:Oklahoma Corporation Commission:",
- "existenceKind": "srn:opendes:reference-data/ExistenceKind:Active:",
- "licenseState": "srn:opendes:reference-data/LicenseState:Unlicensed:",
- "data": {
- "SequenceNumber": 1
- },
- "schema": "test:test:GenericMasterData:1.0.0"
+ "Data": {}
}
- ],
- "Data": {}
},
- "WorkflowID": "foo"
+ "workflow_name": "osdu_ingest",
+ "run_id": "foo"
}
diff --git a/tests/plugin-unit-tests/data/workProduct/SeismicTraceData.json b/tests/plugin-unit-tests/data/workProduct/SeismicTraceData.json
index 4a7da06..575fc30 100644
--- a/tests/plugin-unit-tests/data/workProduct/SeismicTraceData.json
+++ b/tests/plugin-unit-tests/data/workProduct/SeismicTraceData.json
@@ -1,50 +1,20 @@
{
- "Payload": {
- "authorization": "Bearer test",
- "data-partition-id": "opendes",
- "AppKey": "",
- "kind_version": "3.0.0"
- },
- "$schema": "https://schema.osdu.opengroup.org/json/master-data/Wellbore.1.0.0.json",
- "$filename": "load_Wellbore.1.0.0_350112350400.json",
- "manifest":
- {
- "ReferenceData": [],
- "MasterData": [],
- "Data": {
- "WorkProduct": {
- "kind": "opendes:osdu:WorkProduct:1.0.0",
- "groupType": "work-product",
- "acl": {
- "owners": [
- "ownergroup@testcompany.com"
- ],
- "viewers": [
- "viewgroup@testcompany.com"
- ]
- },
- "legal": {
- "legaltags": [
- "legaltag"
- ],
- "otherRelevantDataCountries": [
- "NO",
- "US"
- ]
- },
- "resourceSecurityClassification": "srn:opendes:reference-data/ResourceSecurityClassification:RESTRICTED:",
- "data": {
- "Name": "ST0202R08_PS_PSDM_RAW_PP_TIME.MIG_RAW",
- "Description": "Seismic Trace Data"
- },
- "ComponentsAssociativeIDs": [
- "wpc-1"
- ]
- },
- "WorkProductComponents": [
- {
- "kind": "opendes:osdu:SeismicTraceData:1.0.0",
- "groupType": "work-product-component",
+ "execution_context": {
+ "Payload": {
+ "authorization": "Bearer test",
+ "data-partition-id": "opendes",
+ "AppKey": "",
+ "kind_version": "3.0.0"
+ },
+ "$schema": "https://schema.osdu.opengroup.org/json/master-data/Wellbore.1.0.0.json",
+ "$filename": "load_Wellbore.1.0.0_350112350400.json",
+ "manifest": {
+ "ReferenceData": [],
+ "MasterData": [],
+ "Data": {
+ "WorkProduct": {
+ "kind": "opendes:osdu:WorkProduct:1.0.0",
+ "groupType": "work-product",
"acl": {
"owners": [
"ownergroup@testcompany.com"
@@ -63,369 +33,401 @@
]
},
"resourceSecurityClassification": "srn:opendes:reference-data/ResourceSecurityClassification:RESTRICTED:",
- "meta": [
- {
- "kind": "Unit",
- "name": "ms",
- "persistableReference": "{\"abcd\":{\"a\":0.0,\"b\":0.001,\"c\":1.0,\"d\":0.0},\"symbol\":\"ms\",\"baseMeasurement\":{\"ancestry\":\"T\",\"type\":\"UM\"},\"type\":\"UAD\"}",
- "unitOfMeasureID": "srn:opendes:reference-data/UnitOfMeasure:Energistics_UoM_ms:",
- "propertyNames": [
- "StartTime",
- "EndTime"
+ "data": {
+ "Name": "ST0202R08_PS_PSDM_RAW_PP_TIME.MIG_RAW",
+ "Description": "Seismic Trace Data"
+ },
+ "ComponentsAssociativeIDs": [
+ "wpc-1"
+ ]
+ },
+ "WorkProductComponents": [
+ {
+ "kind": "opendes:osdu:SeismicTraceData:1.0.0",
+ "groupType": "work-product-component",
+ "acl": {
+ "owners": [
+ "ownergroup@testcompany.com"
+ ],
+ "viewers": [
+ "viewgroup@testcompany.com"
]
},
- {
- "kind": "Unit",
- "name": "Amplitude",
- "persistableReference": "{\"abcd\":{\"a\":0.0,\"b\":1.0,\"c\":1.0,\"d\":0.0},\"symbol\":\"Euc\",\"baseMeasurement\":{\"ancestry\":\"1\",\"type\":\"UM\"},\"type\":\"UAD\"}",
- "unitOfMeasureID": "srn:opendes:reference-data/UnitOfMeasure:Energistics_UoM_Euc:",
- "propertyNames": [
- "RangeAmplitudeMax",
- "RangeAmplitudeMin"
+ "legal": {
+ "legaltags": [
+ "legaltag"
+ ],
+ "otherRelevantDataCountries": [
+ "NO",
+ "US"
]
- }
- ],
- "data": {
- "Name": "ST0202R08_PS_PSDM_RAW_PP_TIME.MIG_RAW",
- "Description": "Seismic Trace Data",
- "SpatialArea": {
- "AsIngestedCoordinates": {
- "type": "AnyCrsFeatureCollection",
- "CoordinateReferenceSystemID": "srn:opendes:reference-data/CoordinateReferenceSystem:BoundCRS.SLB.23031.1613:",
- "persistableReferenceCRS": "{\"lateBoundCRS\":{\"wkt\":\"PROJCS[\\\"ED_1950_UTM_Zone_31N\\\",GEOGCS[\\\"GCS_European_1950\\\",DATUM[\\\"D_European_1950\\\",SPHEROID[\\\"International_1924\\\",6378388.0,297.0]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],PROJECTION[\\\"Transverse_Mercator\\\"],PARAMETER[\\\"False_Easting\\\",500000.0],PARAMETER[\\\"False_Northing\\\",0.0],PARAMETER[\\\"Central_Meridian\\\",3.0],PARAMETER[\\\"Scale_Factor\\\",0.9996],PARAMETER[\\\"Latitude_Of_Origin\\\",0.0],UNIT[\\\"Meter\\\",1.0],AUTHORITY[\\\"EPSG\\\",23031]]\",\"ver\":\"PE_10_3_1\",\"name\":\"ED_1950_UTM_Zone_31N\",\"authCode\":{\"auth\":\"EPSG\",\"code\":\"23031\"},\"type\":\"LBC\"},\"singleCT\":{\"wkt\":\"GEOGTRAN[\\\"ED_1950_To_WGS_1984_24\\\",GEOGCS[\\\"GCS_European_1950\\\",DATUM[\\\"D_European_1950\\\",SPHEROID[\\\"International_1924\\\",6378388.0,297.0]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],GEOGCS[\\\"GCS_WGS_1984\\\",DATUM[\\\"D_WGS_1984\\\",SPHEROID[\\\"WGS_1984\\\",6378137.0,298.257223563]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],METHOD[\\\"Position_Vector\\\"],PARAMETER[\\\"X_Axis_Translation\\\",-90.365],PARAMETER[\\\"Y_Axis_Translation\\\",-101.13],PARAMETER[\\\"Z_Axis_Translation\\\",-123.384],PARAMETER[\\\"X_Axis_Rotation\\\",0.333],PARAMETER[\\\"Y_Axis_Rotation\\\",0.077],PARAMETER[\\\"Z_Axis_Rotation\\\",0.894],PARAMETER[\\\"Scale_Difference\\\",1.994],AUTHORITY[\\\"EPSG\\\",1613]]\",\"ver\":\"PE_10_3_1\",\"name\":\"ED_1950_To_WGS_1984_24\",\"authCode\":{\"auth\":\"EPSG\",\"code\":\"1613\"},\"type\":\"ST\"},\"ver\":\"PE_10_3_1\",\"name\":\"ED50 * EPSG-Nor S62 2001 / UTM zone 31N [23031,1613]\",\"authCode\":{\"auth\":\"SLB\",\"code\":\"23031024\"},\"type\":\"EBC\"}",
- "features": [
- {
- "type": "AnyCrsFeature",
- "properties": null,
- "geometry": {
- "type": "AnyCrsPolygon",
- "coordinates": [
- [
- [
- 438727.125,
- 6475514.5
- ],
- [
- 439888.34375,
- 6480172.0
- ],
- [
- 432562.59375,
- 6481998.5
- ],
- [
- 431401.375,
- 6477341.0
- ],
+ },
+ "resourceSecurityClassification": "srn:opendes:reference-data/ResourceSecurityClassification:RESTRICTED:",
+ "meta": [
+ {
+ "kind": "Unit",
+ "name": "ms",
+ "persistableReference": "{\"abcd\":{\"a\":0.0,\"b\":0.001,\"c\":1.0,\"d\":0.0},\"symbol\":\"ms\",\"baseMeasurement\":{\"ancestry\":\"T\",\"type\":\"UM\"},\"type\":\"UAD\"}",
+ "unitOfMeasureID": "srn:opendes:reference-data/UnitOfMeasure:Energistics_UoM_ms:",
+ "propertyNames": [
+ "StartTime",
+ "EndTime"
+ ]
+ },
+ {
+ "kind": "Unit",
+ "name": "Amplitude",
+ "persistableReference": "{\"abcd\":{\"a\":0.0,\"b\":1.0,\"c\":1.0,\"d\":0.0},\"symbol\":\"Euc\",\"baseMeasurement\":{\"ancestry\":\"1\",\"type\":\"UM\"},\"type\":\"UAD\"}",
+ "unitOfMeasureID": "srn:opendes:reference-data/UnitOfMeasure:Energistics_UoM_Euc:",
+ "propertyNames": [
+ "RangeAmplitudeMax",
+ "RangeAmplitudeMin"
+ ]
+ }
+ ],
+ "data": {
+ "Name": "ST0202R08_PS_PSDM_RAW_PP_TIME.MIG_RAW",
+ "Description": "Seismic Trace Data",
+ "SpatialArea": {
+ "AsIngestedCoordinates": {
+ "type": "AnyCrsFeatureCollection",
+ "CoordinateReferenceSystemID": "srn:opendes:reference-data/CoordinateReferenceSystem:BoundCRS.SLB.23031.1613:",
+ "persistableReferenceCRS": "{\"lateBoundCRS\":{\"wkt\":\"PROJCS[\\\"ED_1950_UTM_Zone_31N\\\",GEOGCS[\\\"GCS_European_1950\\\",DATUM[\\\"D_European_1950\\\",SPHEROID[\\\"International_1924\\\",6378388.0,297.0]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],PROJECTION[\\\"Transverse_Mercator\\\"],PARAMETER[\\\"False_Easting\\\",500000.0],PARAMETER[\\\"False_Northing\\\",0.0],PARAMETER[\\\"Central_Meridian\\\",3.0],PARAMETER[\\\"Scale_Factor\\\",0.9996],PARAMETER[\\\"Latitude_Of_Origin\\\",0.0],UNIT[\\\"Meter\\\",1.0],AUTHORITY[\\\"EPSG\\\",23031]]\",\"ver\":\"PE_10_3_1\",\"name\":\"ED_1950_UTM_Zone_31N\",\"authCode\":{\"auth\":\"EPSG\",\"code\":\"23031\"},\"type\":\"LBC\"},\"singleCT\":{\"wkt\":\"GEOGTRAN[\\\"ED_1950_To_WGS_1984_24\\\",GEOGCS[\\\"GCS_European_1950\\\",DATUM[\\\"D_European_1950\\\",SPHEROID[\\\"International_1924\\\",6378388.0,297.0]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],GEOGCS[\\\"GCS_WGS_1984\\\",DATUM[\\\"D_WGS_1984\\\",SPHEROID[\\\"WGS_1984\\\",6378137.0,298.257223563]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],METHOD[\\\"Position_Vector\\\"],PARAMETER[\\\"X_Axis_Translation\\\",-90.365],PARAMETER[\\\"Y_Axis_Translation\\\",-101.13],PARAMETER[\\\"Z_Axis_Translation\\\",-123.384],PARAMETER[\\\"X_Axis_Rotation\\\",0.333],PARAMETER[\\\"Y_Axis_Rotation\\\",0.077],PARAMETER[\\\"Z_Axis_Rotation\\\",0.894],PARAMETER[\\\"Scale_Difference\\\",1.994],AUTHORITY[\\\"EPSG\\\",1613]]\",\"ver\":\"PE_10_3_1\",\"name\":\"ED_1950_To_WGS_1984_24\",\"authCode\":{\"auth\":\"EPSG\",\"code\":\"1613\"},\"type\":\"ST\"},\"ver\":\"PE_10_3_1\",\"name\":\"ED50 * EPSG-Nor S62 2001 / UTM zone 31N [23031,1613]\",\"authCode\":{\"auth\":\"SLB\",\"code\":\"23031024\"},\"type\":\"EBC\"}",
+ "features": [
+ {
+ "type": "AnyCrsFeature",
+ "properties": null,
+ "geometry": {
+ "type": "AnyCrsPolygon",
+ "coordinates": [
[
- 438727.125,
- 6475514.5
+ [
+ 438727.125,
+ 6475514.5
+ ],
+ [
+ 439888.34375,
+ 6480172.0
+ ],
+ [
+ 432562.59375,
+ 6481998.5
+ ],
+ [
+ 431401.375,
+ 6477341.0
+ ],
+ [
+ 438727.125,
+ 6475514.5
+ ]
]
]
- ]
+ }
}
- }
- ]
- },
- "Wgs84Coordinates": {
- "type": "FeatureCollection",
- "features": [
- {
- "type": "Feature",
- "properties": null,
- "geometry": {
- "type": "Polygon",
- "coordinates": [
- [
- [
- 1.9496878,
- 58.4141503
- ],
- [
- 1.9683366,
- 58.4561357
- ],
- [
- 1.8422866,
- 58.4714655
- ],
+ ]
+ },
+ "Wgs84Coordinates": {
+ "type": "FeatureCollection",
+ "features": [
+ {
+ "type": "Feature",
+ "properties": null,
+ "geometry": {
+ "type": "Polygon",
+ "coordinates": [
[
- 1.8237804,
- 58.4294624
- ],
- [
- 1.9496878,
- 58.4141503
+ [
+ 1.9496878,
+ 58.4141503
+ ],
+ [
+ 1.9683366,
+ 58.4561357
+ ],
+ [
+ 1.8422866,
+ 58.4714655
+ ],
+ [
+ 1.8237804,
+ 58.4294624
+ ],
+ [
+ 1.9496878,
+ 58.4141503
+ ]
]
]
- ]
+ }
}
- }
- ]
+ ]
+ },
+ "OperationsApplied": [
+ "AsIngestedCoordinates converted to Wgs84Coordinates: Input CRS EPSG 23031 (ED50 / UTM zone 31N) to Target CRS EPSG 4326 (WGS84) using CT EPSG 1613 (ED50 to WGS 84 (24) - Norway - offshore south of 62°N - North Sea.)"
+ ],
+ "SpatialParameterTypeID": "srn:opendes:reference-data/SpatialParameterType:Outline:",
+ "SpatialGeometryTypeID": "srn:opendes:reference-data/SpatialGeometryType:Polygon:"
},
- "OperationsApplied": [
- "AsIngestedCoordinates converted to Wgs84Coordinates: Input CRS EPSG 23031 (ED50 / UTM zone 31N) to Target CRS EPSG 4326 (WGS84) using CT EPSG 1613 (ED50 to WGS 84 (24) - Norway - offshore south of 62°N - North Sea.)"
- ],
- "SpatialParameterTypeID": "srn:opendes:reference-data/SpatialParameterType:Outline:",
- "SpatialGeometryTypeID": "srn:opendes:reference-data/SpatialGeometryType:Polygon:"
- },
- "LiveTraceOutline": {
- "AsIngestedCoordinates": {
- "type": "AnyCrsFeatureCollection",
- "CoordinateReferenceSystemID": "srn:opendes:reference-data/CoordinateReferenceSystem:BoundCRS.SLB.23031.1613:",
- "persistableReferenceCRS": "{\"lateBoundCRS\":{\"wkt\":\"PROJCS[\\\"ED_1950_UTM_Zone_31N\\\",GEOGCS[\\\"GCS_European_1950\\\",DATUM[\\\"D_European_1950\\\",SPHEROID[\\\"International_1924\\\",6378388.0,297.0]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],PROJECTION[\\\"Transverse_Mercator\\\"],PARAMETER[\\\"False_Easting\\\",500000.0],PARAMETER[\\\"False_Northing\\\",0.0],PARAMETER[\\\"Central_Meridian\\\",3.0],PARAMETER[\\\"Scale_Factor\\\",0.9996],PARAMETER[\\\"Latitude_Of_Origin\\\",0.0],UNIT[\\\"Meter\\\",1.0],AUTHORITY[\\\"EPSG\\\",23031]]\",\"ver\":\"PE_10_3_1\",\"name\":\"ED_1950_UTM_Zone_31N\",\"authCode\":{\"auth\":\"EPSG\",\"code\":\"23031\"},\"type\":\"LBC\"},\"singleCT\":{\"wkt\":\"GEOGTRAN[\\\"ED_1950_To_WGS_1984_24\\\",GEOGCS[\\\"GCS_European_1950\\\",DATUM[\\\"D_European_1950\\\",SPHEROID[\\\"International_1924\\\",6378388.0,297.0]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],GEOGCS[\\\"GCS_WGS_1984\\\",DATUM[\\\"D_WGS_1984\\\",SPHEROID[\\\"WGS_1984\\\",6378137.0,298.257223563]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],METHOD[\\\"Position_Vector\\\"],PARAMETER[\\\"X_Axis_Translation\\\",-90.365],PARAMETER[\\\"Y_Axis_Translation\\\",-101.13],PARAMETER[\\\"Z_Axis_Translation\\\",-123.384],PARAMETER[\\\"X_Axis_Rotation\\\",0.333],PARAMETER[\\\"Y_Axis_Rotation\\\",0.077],PARAMETER[\\\"Z_Axis_Rotation\\\",0.894],PARAMETER[\\\"Scale_Difference\\\",1.994],AUTHORITY[\\\"EPSG\\\",1613]]\",\"ver\":\"PE_10_3_1\",\"name\":\"ED_1950_To_WGS_1984_24\",\"authCode\":{\"auth\":\"EPSG\",\"code\":\"1613\"},\"type\":\"ST\"},\"ver\":\"PE_10_3_1\",\"name\":\"ED50 * EPSG-Nor S62 2001 / UTM zone 31N [23031,1613]\",\"authCode\":{\"auth\":\"SLB\",\"code\":\"23031024\"},\"type\":\"EBC\"}",
- "features": [
- {
- "type": "AnyCrsFeature",
- "properties": null,
- "geometry": {
- "type": "AnyCrsPolygon",
- "coordinates": [
- [
- [
- 438727.125,
- 6475514.5
- ],
+ "LiveTraceOutline": {
+ "AsIngestedCoordinates": {
+ "type": "AnyCrsFeatureCollection",
+ "CoordinateReferenceSystemID": "srn:opendes:reference-data/CoordinateReferenceSystem:BoundCRS.SLB.23031.1613:",
+ "persistableReferenceCRS": "{\"lateBoundCRS\":{\"wkt\":\"PROJCS[\\\"ED_1950_UTM_Zone_31N\\\",GEOGCS[\\\"GCS_European_1950\\\",DATUM[\\\"D_European_1950\\\",SPHEROID[\\\"International_1924\\\",6378388.0,297.0]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],PROJECTION[\\\"Transverse_Mercator\\\"],PARAMETER[\\\"False_Easting\\\",500000.0],PARAMETER[\\\"False_Northing\\\",0.0],PARAMETER[\\\"Central_Meridian\\\",3.0],PARAMETER[\\\"Scale_Factor\\\",0.9996],PARAMETER[\\\"Latitude_Of_Origin\\\",0.0],UNIT[\\\"Meter\\\",1.0],AUTHORITY[\\\"EPSG\\\",23031]]\",\"ver\":\"PE_10_3_1\",\"name\":\"ED_1950_UTM_Zone_31N\",\"authCode\":{\"auth\":\"EPSG\",\"code\":\"23031\"},\"type\":\"LBC\"},\"singleCT\":{\"wkt\":\"GEOGTRAN[\\\"ED_1950_To_WGS_1984_24\\\",GEOGCS[\\\"GCS_European_1950\\\",DATUM[\\\"D_European_1950\\\",SPHEROID[\\\"International_1924\\\",6378388.0,297.0]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],GEOGCS[\\\"GCS_WGS_1984\\\",DATUM[\\\"D_WGS_1984\\\",SPHEROID[\\\"WGS_1984\\\",6378137.0,298.257223563]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],METHOD[\\\"Position_Vector\\\"],PARAMETER[\\\"X_Axis_Translation\\\",-90.365],PARAMETER[\\\"Y_Axis_Translation\\\",-101.13],PARAMETER[\\\"Z_Axis_Translation\\\",-123.384],PARAMETER[\\\"X_Axis_Rotation\\\",0.333],PARAMETER[\\\"Y_Axis_Rotation\\\",0.077],PARAMETER[\\\"Z_Axis_Rotation\\\",0.894],PARAMETER[\\\"Scale_Difference\\\",1.994],AUTHORITY[\\\"EPSG\\\",1613]]\",\"ver\":\"PE_10_3_1\",\"name\":\"ED_1950_To_WGS_1984_24\",\"authCode\":{\"auth\":\"EPSG\",\"code\":\"1613\"},\"type\":\"ST\"},\"ver\":\"PE_10_3_1\",\"name\":\"ED50 * EPSG-Nor S62 2001 / UTM zone 31N [23031,1613]\",\"authCode\":{\"auth\":\"SLB\",\"code\":\"23031024\"},\"type\":\"EBC\"}",
+ "features": [
+ {
+ "type": "AnyCrsFeature",
+ "properties": null,
+ "geometry": {
+ "type": "AnyCrsPolygon",
+ "coordinates": [
[
- 439888.34375,
- 6480172.0
- ],
- [
- 432562.59375,
- 6481998.5
- ],
- [
- 431401.375,
- 6477341.0
- ],
- [
- 438727.125,
- 6475514.5
+ [
+ 438727.125,
+ 6475514.5
+ ],
+ [
+ 439888.34375,
+ 6480172.0
+ ],
+ [
+ 432562.59375,
+ 6481998.5
+ ],
+ [
+ 431401.375,
+ 6477341.0
+ ],
+ [
+ 438727.125,
+ 6475514.5
+ ]
]
]
- ]
+ }
}
- }
- ]
- },
- "Wgs84Coordinates": {
- "type": "FeatureCollection",
- "features": [
- {
- "type": "Feature",
- "properties": null,
- "geometry": {
- "type": "Polygon",
- "coordinates": [
- [
- [
- 1.9496878,
- 58.4141503
- ],
- [
- 1.9683366,
- 58.4561357
- ],
- [
- 1.8422866,
- 58.4714655
- ],
+ ]
+ },
+ "Wgs84Coordinates": {
+ "type": "FeatureCollection",
+ "features": [
+ {
+ "type": "Feature",
+ "properties": null,
+ "geometry": {
+ "type": "Polygon",
+ "coordinates": [
[
- 1.8237804,
- 58.4294624
- ],
- [
- 1.9496878,
- 58.4141503
+ [
+ 1.9496878,
+ 58.4141503
+ ],
+ [
+ 1.9683366,
+ 58.4561357
+ ],
+ [
+ 1.8422866,
+ 58.4714655
+ ],
+ [
+ 1.8237804,
+ 58.4294624
+ ],
+ [
+ 1.9496878,
+ 58.4141503
+ ]
]
]
- ]
+ }
}
- }
- ]
+ ]
+ },
+ "OperationsApplied": [
+ "AsIngestedCoordinates converted to Wgs84Coordinates: Input CRS EPSG 23031 (ED50 / UTM zone 31N) to Target CRS EPSG 4326 (WGS84) using CT EPSG 1613 (ED50 to WGS 84 (24) - Norway - offshore south of 62°N - North Sea.)"
+ ],
+ "SpatialParameterTypeID": "srn:opendes:reference-data/SpatialParameterType:Outline:",
+ "SpatialGeometryTypeID": "srn:opendes:reference-data/SpatialGeometryType:Polygon:"
},
- "OperationsApplied": [
- "AsIngestedCoordinates converted to Wgs84Coordinates: Input CRS EPSG 23031 (ED50 / UTM zone 31N) to Target CRS EPSG 4326 (WGS84) using CT EPSG 1613 (ED50 to WGS 84 (24) - Norway - offshore south of 62°N - North Sea.)"
+ "PrincipalAcquisitionProjectID": "srn:opendes:master-data/SeismicAcquisitionProject:ST0202R08:",
+ "ProcessingProjectID": "srn:opendes:master-data/SeismicProcessingProject:ST0202R08:",
+ "SeismicTraceDataDimensionalityTypeID": "srn:opendes:reference-data/SeismicTraceDataDimensionalityType:3D:",
+ "SeismicDomainTypeID": "srn:opendes:reference-data/SeismicDomainType:Time:",
+ "SeismicMigrationTypeID": "srn:opendes:reference-data/SeismicMigrationType:Prestack Depth - Kirchhoff:",
+ "SeismicStackingTypeID": "srn:opendes:reference-data/SeismicStackingType:Full:",
+ "SeismicFilteringTypeID": "srn:opendes:reference-data/SeismicFilteringType:Tau-P:",
+ "Phase": "0",
+ "Polarity": "Normal",
+ "SampleInterval": 4.0,
+ "SampleCount": 1126,
+ "Difference": false,
+ "StartTime": 0.0,
+ "EndTime": 4500.0,
+ "TraceCount": 58479,
+ "TraceLength": 4500.0,
+ "TraceDomainUOM": "srn:opendes:reference-data/UnitOfMeasure:ms:",
+ "InlineMin": 9985,
+ "InlineMax": 10369,
+ "CrosslineMin": 1932,
+ "CrosslineMax": 2536,
+ "InlineIncrement": 2,
+ "CrosslineIncrement": 2,
+ "Precision": {
+ "WordFormat": "srn:opendes:reference-data/WordFormatType:IBM_FLOAT:",
+ "WordWidth": 4
+ },
+ "ProcessingParameters": [
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:REFORMAT:"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:NAVIGATION MERGE:"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:TRACE EDITING:"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:DESIGNATURE/ZEROPHASE:"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:TIDAL STATICS:"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:RESAMPLE:",
+ "ProcessingParameterValue": "4 MS"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:GEOPHONE MATCHING CORRECTION:"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:GEOPHONE ORIENTATION CORRECTION:"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:ROTATION TO RADIAL:"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:NOISE BAND EDITING:"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:RECEIVER SHEAR STATIC CORRECTION:"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:TAUP DECONVOLUTION:"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:3D KIRCHHOFF DEPTH MIGRATION:"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:VEL ANALYSIS AND RMO CORRECTION:",
+ "ProcessingParameterValue": "200M X 200M"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:CONVERSION TO PP TWT:"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:PRE-RADON MILD DIP FILTER:"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:PARABOLIC RADON TRANSFORM DEMULTIPLE:"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:ANGLE MUTE:",
+ "ProcessingParameterValue": "3-37 DEGREES"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:STACK:"
+ },
+ {
+ "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:OUTPUT SEGY:"
+ }
],
- "SpatialParameterTypeID": "srn:opendes:reference-data/SpatialParameterType:Outline:",
- "SpatialGeometryTypeID": "srn:opendes:reference-data/SpatialGeometryType:Polygon:"
- },
- "PrincipalAcquisitionProjectID": "srn:opendes:master-data/SeismicAcquisitionProject:ST0202R08:",
- "ProcessingProjectID": "srn:opendes:master-data/SeismicProcessingProject:ST0202R08:",
- "SeismicTraceDataDimensionalityTypeID": "srn:opendes:reference-data/SeismicTraceDataDimensionalityType:3D:",
- "SeismicDomainTypeID": "srn:opendes:reference-data/SeismicDomainType:Time:",
- "SeismicMigrationTypeID": "srn:opendes:reference-data/SeismicMigrationType:Prestack Depth - Kirchhoff:",
- "SeismicStackingTypeID": "srn:opendes:reference-data/SeismicStackingType:Full:",
- "SeismicFilteringTypeID": "srn:opendes:reference-data/SeismicFilteringType:Tau-P:",
- "Phase": "0",
- "Polarity": "Normal",
- "SampleInterval": 4.0,
- "SampleCount": 1126,
- "Difference": false,
- "StartTime": 0.0,
- "EndTime": 4500.0,
- "TraceCount": 58479,
- "TraceLength": 4500.0,
- "TraceDomainUOM": "srn:opendes:reference-data/UnitOfMeasure:ms:",
- "InlineMin": 9985,
- "InlineMax": 10369,
- "CrosslineMin": 1932,
- "CrosslineMax": 2536,
- "InlineIncrement": 2,
- "CrosslineIncrement": 2,
- "Precision": {
- "WordFormat": "srn:opendes:reference-data/WordFormatType:IBM_FLOAT:",
- "WordWidth": 4
+ "TextualFileHeader": [
+ "C 1 CLIENT=STATOIL, PROCESSED BY=WESTERNGECO C 2 LINE ST0202D04-9985 C 3 SURVEY:ST0202 , AREA: VOLVE 15/9 C 4 DATAFORMAT: SEG-Y, DATE:02012008 C 5 DATATYPE: RAW PS PSDM STACK IN PP TIME C 6 DATA SHOT BY/VESSEL:GECO ANGLER, CABLE LENGTH:6000 M NO OF GROUPS: 240x4 C 7 NO OF CABLES 2, SOURCE:2, POP INTERVAL 25.0 M (FLIP-FLOP) C 8 RCV LINE SPACING: 400 M, SRC LINE SPACING: 100 M, RECORD LENGTH: 10.2 S C 9 GEODETIC DATUM: ED-50, SPHEROID: INTERNAT., PROJECTION: UTM C10 CENTR. MERID.: 03, UTM ZONE: 31 N, FALSE EASTING: 500000 C11 FIRST SAMPLE= 0, LAST SAMPLE= 4500, SAMPLE INTERVAL= 4 MS C12 DIST. BETWEEN INLINES=25.0M, XLINES=25.0M BIN SIZE (I x X):12.5M x 12.5M C13 INLINE DIRECTION (GRID): 284.0000000 DEGREES (CLOCKWISE FROM NORTH); C14 X-LINE DIRECTION (GRID): 014.0000000 DEGREES (CLOCKWISE FROM NORTH); C15 ORIGO(1.1) UTMX:431955.70E; UTMY:6348582.15N ; C16 DATA RANGE INLINES=9985-10369 (EVERY 2ND), X-LINES=1932-2536 (EVERY 2ND) C17 CORNER1:3D INLINE 9985, 3D XLINE 1932, UTM-X 438727.0, UTM-Y 6475514.4 C18 CORNER2:3D INLINE 9985, 3D XLINE 2536, UTM-X 431401.3, UTM-Y 6477341.0 C19 CORNER3:3D INLINE 10369, 3D XLINE 2536, UTM-X 432562.5, UTM-Y 6481998.4 C20 CORNER4:3D INLINE 10369, 3D XLINE 1932, UTM-X 439888.3, UTM-Y 6480171.9 C21 LIVE DATA POLYGON: (9985,1932);(9985,2536);(10369,2536);(10369,1932); C22 NAVIGATION SOURCE: P1/90 UKOOA BIN CENTER CELL GRID, C23 PROCESSING SEQUENCE: C24 1) REFORMAT 2) NAVIGATION MERGE 3) TRACE EDITING 4) DESIGNATURE/ZEROPHASE C25 5) TIDAL STATICS 6) RESAMPLE 4MS 7) GEOPHONE MATCHING CORRECTION C26 8) GEOPHONE ORIENTATION CORRECTION 9) ROTATION TO RADIAL C27 10) NOISE BAND EDITING 11) RECEIVER SHEAR STATIC CORRECTION C28 12) TAUP DECONVOLUTION 13) 3D KIRCHHOFF DEPTH MIGRATION C29 14) VEL ANALYSIS AND RMO CORRECTION 200M X 200M 15) CONVERSION TO PP TWT C30 16) PRE-RADON MILD DIP FILTER 17) PARABOLIC RADON TRANSFORM DEMULTIPLE C31 18) 3-37 DEGREE ANGLE MUTE 19) STACK 20) OUTPUT TO SEGY FORMAT C32 C33 C34 C35 HEADER WORD POSITIONS: C36 INLINE: 189-192 ; X-LINE: 193-196; C37 BINX (CDPX): 181-184, BINY (CDPY): 185-188, C38 MERID.: 3.0E, SPHEROID: INT.; ROTATION (AMS): 1245600000, C39 A POSITIVE SAMPLE CORRESPONDS TO A INCREASE IN ACOUSTIC IMPEDANCE. C40 END EBCDIC. "
+ ],
+ "RangeAmplitudeMax": 0.07441109418869019,
+ "RangeAmplitudeMin": -0.10446560382843018
},
- "ProcessingParameters": [
- {
- "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:REFORMAT:"
- },
- {
- "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:NAVIGATION MERGE:"
- },
- {
- "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:TRACE EDITING:"
- },
- {
- "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:DESIGNATURE/ZEROPHASE:"
- },
- {
- "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:TIDAL STATICS:"
- },
- {
- "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:RESAMPLE:",
- "ProcessingParameterValue": "4 MS"
- },
- {
- "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:GEOPHONE MATCHING CORRECTION:"
- },
- {
- "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:GEOPHONE ORIENTATION CORRECTION:"
- },
- {
- "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:ROTATION TO RADIAL:"
- },
- {
- "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:NOISE BAND EDITING:"
- },
- {
- "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:RECEIVER SHEAR STATIC CORRECTION:"
- },
- {
- "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:TAUP DECONVOLUTION:"
- },
- {
- "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:3D KIRCHHOFF DEPTH MIGRATION:"
- },
- {
- "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:VEL ANALYSIS AND RMO CORRECTION:",
- "ProcessingParameterValue": "200M X 200M"
- },
- {
- "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:CONVERSION TO PP TWT:"
- },
- {
- "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:PRE-RADON MILD DIP FILTER:"
- },
- {
- "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:PARABOLIC RADON TRANSFORM DEMULTIPLE:"
- },
- {
- "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:ANGLE MUTE:",
- "ProcessingParameterValue": "3-37 DEGREES"
- },
- {
- "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:STACK:"
- },
- {
- "ProcessingParameterTypeID": "srn:opendes:reference-data/ProcessingParameterType:OUTPUT SEGY:"
- }
- ],
- "TextualFileHeader": [
- "C 1 CLIENT=STATOIL, PROCESSED BY=WESTERNGECO C 2 LINE ST0202D04-9985 C 3 SURVEY:ST0202 , AREA: VOLVE 15/9 C 4 DATAFORMAT: SEG-Y, DATE:02012008 C 5 DATATYPE: RAW PS PSDM STACK IN PP TIME C 6 DATA SHOT BY/VESSEL:GECO ANGLER, CABLE LENGTH:6000 M NO OF GROUPS: 240x4 C 7 NO OF CABLES 2, SOURCE:2, POP INTERVAL 25.0 M (FLIP-FLOP) C 8 RCV LINE SPACING: 400 M, SRC LINE SPACING: 100 M, RECORD LENGTH: 10.2 S C 9 GEODETIC DATUM: ED-50, SPHEROID: INTERNAT., PROJECTION: UTM C10 CENTR. MERID.: 03, UTM ZONE: 31 N, FALSE EASTING: 500000 C11 FIRST SAMPLE= 0, LAST SAMPLE= 4500, SAMPLE INTERVAL= 4 MS C12 DIST. BETWEEN INLINES=25.0M, XLINES=25.0M BIN SIZE (I x X):12.5M x 12.5M C13 INLINE DIRECTION (GRID): 284.0000000 DEGREES (CLOCKWISE FROM NORTH); C14 X-LINE DIRECTION (GRID): 014.0000000 DEGREES (CLOCKWISE FROM NORTH); C15 ORIGO(1.1) UTMX:431955.70E; UTMY:6348582.15N ; C16 DATA RANGE INLINES=9985-10369 (EVERY 2ND), X-LINES=1932-2536 (EVERY 2ND) C17 CORNER1:3D INLINE 9985, 3D XLINE 1932, UTM-X 438727.0, UTM-Y 6475514.4 C18 CORNER2:3D INLINE 9985, 3D XLINE 2536, UTM-X 431401.3, UTM-Y 6477341.0 C19 CORNER3:3D INLINE 10369, 3D XLINE 2536, UTM-X 432562.5, UTM-Y 6481998.4 C20 CORNER4:3D INLINE 10369, 3D XLINE 1932, UTM-X 439888.3, UTM-Y 6480171.9 C21 LIVE DATA POLYGON: (9985,1932);(9985,2536);(10369,2536);(10369,1932); C22 NAVIGATION SOURCE: P1/90 UKOOA BIN CENTER CELL GRID, C23 PROCESSING SEQUENCE: C24 1) REFORMAT 2) NAVIGATION MERGE 3) TRACE EDITING 4) DESIGNATURE/ZEROPHASE C25 5) TIDAL STATICS 6) RESAMPLE 4MS 7) GEOPHONE MATCHING CORRECTION C26 8) GEOPHONE ORIENTATION CORRECTION 9) ROTATION TO RADIAL C27 10) NOISE BAND EDITING 11) RECEIVER SHEAR STATIC CORRECTION C28 12) TAUP DECONVOLUTION 13) 3D KIRCHHOFF DEPTH MIGRATION C29 14) VEL ANALYSIS AND RMO CORRECTION 200M X 200M 15) CONVERSION TO PP TWT C30 16) PRE-RADON MILD DIP FILTER 17) PARABOLIC RADON TRANSFORM DEMULTIPLE C31 18) 3-37 DEGREE ANGLE MUTE 19) STACK 20) OUTPUT TO SEGY FORMAT C32 C33 C34 C35 HEADER WORD POSITIONS: C36 INLINE: 189-192 ; X-LINE: 193-196; C37 BINX (CDPX): 181-184, BINY (CDPY): 185-188, C38 MERID.: 3.0E, SPHEROID: INT.; ROTATION (AMS): 1245600000, C39 A POSITIVE SAMPLE CORRESPONDS TO A INCREASE IN ACOUSTIC IMPEDANCE. C40 END EBCDIC. "
- ],
- "RangeAmplitudeMax": 0.07441109418869019,
- "RangeAmplitudeMin": -0.10446560382843018
- },
- "AssociativeID": "wpc-1",
- "FileAssociativeIDs": [
- "f-1"
- ]
- }
- ],
- "Datasets": [
- {
- "kind": "opendes:osdu:File:1.0.0",
- "groupType": "file",
- "acl": {
- "owners": [
- "ownergroup@testcompany.com"
- ],
- "viewers": [
- "viewgroup@testcompany.com"
- ]
- },
- "legal": {
- "legaltags": [
- "legaltag"
- ],
- "otherRelevantDataCountries": [
- "NO",
- "US"
+ "AssociativeID": "wpc-1",
+ "FileAssociativeIDs": [
+ "f-1"
]
- },
- "resourceSecurityClassification": "srn:opendes:reference-data/ResourceSecurityClassification:RESTRICTED:",
- "data": {
- "Name": "RequiredName",
- "SchemaFormatTypeID": "srn:opendes:reference-data/SchemaFormatType:SEG-Y Seismic Trace Data:",
- "DatasetProperties": {
- "FileSourceInfo": {
- "PreloadFilePath": "test",
- "FileSource": ""
- }
+ }
+ ],
+ "Datasets": [
+ {
+ "kind": "opendes:osdu:File:1.0.0",
+ "groupType": "file",
+ "acl": {
+ "owners": [
+ "ownergroup@testcompany.com"
+ ],
+ "viewers": [
+ "viewgroup@testcompany.com"
+ ]
},
- "FileSize": 277427976,
- "EncodingFormatTypeID": "srn:opendes:reference-data/EncodingFormatType:segy:",
- "Endian": "BIG",
- "Checksum": "c9df7234d5d0a7a2c2676ee2e2aa48b6",
- "VectorHeaderMapping": [
- {
- "KeyName": "srn:opendes:reference-data/HeaderKeyName:INLINE:",
- "WordFormat": "srn:opendes:reference-data/WordFormatType:INT:",
- "WordWidth": 4,
- "Position": 189
- },
- {
- "KeyName": "srn:opendes:reference-data/HeaderKeyName:CROSSLINE:",
- "WordFormat": "srn:opendes:reference-data/WordFormatType:INT:",
- "WordWidth": 4,
- "Position": 193
- },
- {
- "KeyName": "srn:opendes:reference-data/HeaderKeyName:CMPX:",
- "WordFormat": "srn:opendes:reference-data/WordFormatType:INT:",
- "WordWidth": 4,
- "Position": 181,
- "UoM": "srn:opendes:reference-data/UnitOfMeasure:M:",
- "ScalarIndicator": "OVERRIDE",
- "ScalarOverride": 100.0
+ "legal": {
+ "legaltags": [
+ "legaltag"
+ ],
+ "otherRelevantDataCountries": [
+ "NO",
+ "US"
+ ]
+ },
+ "resourceSecurityClassification": "srn:opendes:reference-data/ResourceSecurityClassification:RESTRICTED:",
+ "data": {
+ "Name": "RequiredName",
+ "SchemaFormatTypeID": "srn:opendes:reference-data/SchemaFormatType:SEG-Y Seismic Trace Data:",
+ "DatasetProperties": {
+ "FileSourceInfo": {
+ "PreloadFilePath": "test",
+ "FileSource": ""
+ }
},
- {
- "KeyName": "srn:opendes:reference-data/HeaderKeyName:CMPY:",
- "WordFormat": "srn:opendes:reference-data/WordFormatType:INT:",
- "WordWidth": 4,
- "Position": 185,
- "UoM": "srn:opendes:reference-data/UnitOfMeasure:M:",
- "ScalarIndicator": "OVERRIDE",
- "ScalarOverride": 100.0
- }
- ]
- },
- "AssociativeID": "f-1"
- }
- ]
+ "FileSize": 277427976,
+ "EncodingFormatTypeID": "srn:opendes:reference-data/EncodingFormatType:segy:",
+ "Endian": "BIG",
+ "Checksum": "c9df7234d5d0a7a2c2676ee2e2aa48b6",
+ "VectorHeaderMapping": [
+ {
+ "KeyName": "srn:opendes:reference-data/HeaderKeyName:INLINE:",
+ "WordFormat": "srn:opendes:reference-data/WordFormatType:INT:",
+ "WordWidth": 4,
+ "Position": 189
+ },
+ {
+ "KeyName": "srn:opendes:reference-data/HeaderKeyName:CROSSLINE:",
+ "WordFormat": "srn:opendes:reference-data/WordFormatType:INT:",
+ "WordWidth": 4,
+ "Position": 193
+ },
+ {
+ "KeyName": "srn:opendes:reference-data/HeaderKeyName:CMPX:",
+ "WordFormat": "srn:opendes:reference-data/WordFormatType:INT:",
+ "WordWidth": 4,
+ "Position": 181,
+ "UoM": "srn:opendes:reference-data/UnitOfMeasure:M:",
+ "ScalarIndicator": "OVERRIDE",
+ "ScalarOverride": 100.0
+ },
+ {
+ "KeyName": "srn:opendes:reference-data/HeaderKeyName:CMPY:",
+ "WordFormat": "srn:opendes:reference-data/WordFormatType:INT:",
+ "WordWidth": 4,
+ "Position": 185,
+ "UoM": "srn:opendes:reference-data/UnitOfMeasure:M:",
+ "ScalarIndicator": "OVERRIDE",
+ "ScalarOverride": 100.0
+ }
+ ]
+ },
+ "AssociativeID": "f-1"
+ }
+ ]
+ }
}
},
- "WorkflowID": "foo"
+ "workflow_name": "osdu_ingest",
+ "run_id": "foo"
}
diff --git a/tests/plugin-unit-tests/test_manifest_processor_r3.py b/tests/plugin-unit-tests/test_manifest_processor_r3.py
index e3bd248..8fbab18 100644
--- a/tests/plugin-unit-tests/test_manifest_processor_r3.py
+++ b/tests/plugin-unit-tests/test_manifest_processor_r3.py
@@ -84,7 +84,7 @@ class TestManifestProcessor:
with open(traversal_manifest_file) as f:
manifest_file = json.load(f)
manifest_records = manifest_file
- context = Context.populate(conf)
+ context = Context.populate(conf["execution_context"])
token_refresher = AirflowTokenRefresher()
file_handler = FileHandler("test", token_refresher, context)
source_file_checker = SourceFileChecker()
diff --git a/tests/plugin-unit-tests/test_manifest_traversal.py b/tests/plugin-unit-tests/test_manifest_traversal.py
index 0dd5653..2b0c392 100644
--- a/tests/plugin-unit-tests/test_manifest_traversal.py
+++ b/tests/plugin-unit-tests/test_manifest_traversal.py
@@ -36,7 +36,7 @@ class TestManifestTraversal:
with open(manifest_schema_file) as f:
manifest_schema = json.load(f)
traversal = ManifestTraversal(
- conf_manifest_file,
+ conf_manifest_file["execution_context"],
manifest_schema
)
return traversal
diff --git a/tests/plugin-unit-tests/test_operators_r3.py b/tests/plugin-unit-tests/test_operators_r3.py
index 6919db0..0bada1d 100644
--- a/tests/plugin-unit-tests/test_operators_r3.py
+++ b/tests/plugin-unit-tests/test_operators_r3.py
@@ -84,37 +84,10 @@ class TestOperators(object):
task.pre_execute(context)
task.execute(context)
- @pytest.mark.parametrize(
- "record_ids,search_response_path",
- [
- pytest.param(
- ["test"],
- SEARCH_VALID_RESPONSE_PATH
- ),
- pytest.param(
- ["test", "test", "test"],
- SEARCH_VALID_RESPONSE_PATH
- )
- ]
- )
- def test_search_id_operator(self, monkeypatch, record_ids: list, search_response_path: str):
- # make validator pass
- monkeypatch.setattr(SchemaValidator, "validate_manifest", lambda obj: None)
- task, context = self._create_task(SearchRecordIdOperator)
-
- monkeypatch.setattr(context["ti"], "xcom_pull", lambda **kwargs: record_ids)
- mock_search_response = MockSearchResponse(body_path=search_response_path,
- status_code=http.HTTPStatus.OK,
- total_count=len(record_ids))
- monkeypatch.setattr(requests, "post", lambda *args, **kwargs: mock_search_response)
-
- task.pre_execute(context)
- task.execute(context)
-
def _test_update_status_operator(self, monkeypatch, status: UpdateStatusOperator.prev_ti_state):
monkeypatch.setattr(UpdateStatusOperator, "get_previous_ti_statuses",
lambda obj, context: status)
- monkeypatch.setattr(requests, "post", lambda *args, **kwargs: MockWorkflowResponse(
+ monkeypatch.setattr(requests, "put", lambda *args, **kwargs: MockWorkflowResponse(
status_code=http.HTTPStatus.OK, json="test"))
task, context = self._create_task(UpdateStatusOperator)
diff --git a/tests/plugin-unit-tests/test_update_status_r3.py b/tests/plugin-unit-tests/test_update_status_r3.py
index 34fdfda..e08571f 100644
--- a/tests/plugin-unit-tests/test_update_status_r3.py
+++ b/tests/plugin-unit-tests/test_update_status_r3.py
@@ -40,11 +40,14 @@ class TestUpdateStatus:
def status_updater(self, status: str, conf_path: str):
with open(conf_path) as f:
conf = json.load(f)
- context = Context.populate(conf)
- workflow_id = conf["WorkflowID"]
+ context = Context.populate(conf["execution_context"])
+ workflow_name = conf["workflow_name"]
+ run_id = conf["run_id"]
status_updater = UpdateStatus(
- workflow_url = "http://test",
- workflow_id=workflow_id,
+ workflow_name=workflow_name,
+ workflow_url="http://test",
+ workflow_id="",
+ run_id=run_id,
token_refresher=AirflowTokenRefresher(get_test_credentials()),
context=context,
status=status
@@ -63,4 +66,5 @@ class TestUpdateStatus:
def test_update_workflow_status(self, monkeypatch, status_updater: UpdateStatus, conf_path: str,
status: str):
monkeypatch.setattr(requests, "post", lambda *args, **kwargs: MockWorkflowResponse())
+ monkeypatch.setattr(requests, "put", lambda *args, **kwargs: MockWorkflowResponse())
status_updater.update_workflow_status()
--
GitLab