Commit c60268fe authored by Mykola Zamkovyi (EPAM)'s avatar Mykola Zamkovyi (EPAM)
Browse files

Merge remote-tracking branch 'origin/master' into GONRG-2066_release_wellbore_pipe_k8s

# Conflicts:
#	.gitlab-ci.yml
parents 40758161 0ab17f23
import urllib
import dateutil.parser
SEP = "benderseparator"
EMPTY = "benderempty"
BENDINGCONTEXT = "bending_context"
WDMS_FRAGMENT = "wdms"
DELFI_SOURCE = "delfi_source_entity"
class ConverterUtils:
@staticmethod
def kind_transform(wks_kind: str, osdu_kind) -> str:
if wks_kind is None:
return None
kind_as_list = wks_kind.split(sep=":")
kind_as_list[1] = "osdu"
kind_as_list[2] = osdu_kind[0]
kind_as_list[3] = osdu_kind[1]
return ":".join(kind_as_list)
@staticmethod
def wellbore_kind_transform(wks_kind: str) -> str:
return ConverterUtils.kind_transform(wks_kind, ["Wellbore", "1.0.0"])
@staticmethod
def well_kind_transform(wks_kind: str) -> str:
return ConverterUtils.kind_transform(wks_kind, ["Well", "1.0.0"])
@staticmethod
def decode_id(osdu_id: str) -> str:
"""
decode osdu style id to delfi id
"""
if osdu_id is None:
return None
return bytes.fromhex(osdu_id.split(":")[2]).decode()
@staticmethod
def fix_id(delfi_id: str, osdu_type: str) -> str:
if delfi_id is None:
return None
# Id will have the OSDU style but not pointing on an actual osdu item
# it will still be a delfi item - to be used with a get_as api
# TODO find a way to make delfi id match the osdu format
# TODO encode the delfi
id_as_list = delfi_id.split(sep=":")
encoded_str = delfi_id.encode().hex()
res_as_list = []
res_as_list.append(id_as_list[0])
res_as_list.append(osdu_type)
res_as_list.append(encoded_str)
res_as_list.append("")
return ":".join(res_as_list)
@staticmethod
def lookup(input_params: str, osdu_type: str) -> str:
# returns the id of the corresponding osdu type
# TODO implement this lookup with a cache
# Some of the lookup have fixed values such as WellboreTrajectoryType (Vertical, Directional, Horizontal),
# reference-data--VerticalMeasurementType,UnitOfMeasure
# Other have to be found in storage and stored in a cache
if input_params is None:
return None
input_params = input_params.split(SEP)
namespace = input_params[0]
delfi_value = input_params[1]
if delfi_value == EMPTY:
return None
# TODO lookup in catalog, put results in cache, insert if needed
ret = f"{namespace}:{osdu_type}:{urllib.parse.quote(delfi_value)}:"
return ret
@staticmethod
def find_in_meta(metas: [dict], search_attribute: str, search_value: str, returned_attribute: str) -> str:
if metas is None:
return EMPTY
for meta_item in metas:
if meta_item.get(search_attribute, EMPTY) == search_value:
return meta_item.get(returned_attribute, EMPTY)
@staticmethod
def date_to_datetime(in_date: str) -> str:
return (
dateutil.parser.parse(in_date).strftime("%Y-%m-%dT%H:%M:%S.%f")
if in_date
else None
)
@staticmethod
def remove_none_from_dict(in_dict: dict) -> dict:
new_dict = {}
for k, v in in_dict.items():
if isinstance(v, dict):
v = ConverterUtils.remove_none_from_dict(v)
if v is not None:
new_dict[k] = v
return new_dict or None
@staticmethod
def remove_none(in_obj):
# This method can be optimized in python 3.8 with assignment expression PEP572 := https://stackoverflow.com/questions/4097518/intermediate-variable-in-a-list-comprehension-for-simultaneous-filtering-and-tra
#(x or x == False) keep x if x is not not None or if x a non empty container
if isinstance(in_obj, (list, tuple, set)):
return type(in_obj)(ConverterUtils.remove_none(x) for x in in_obj if (x or x == False) and (
ConverterUtils.remove_none(x) or ConverterUtils.remove_none(x) == False))
elif isinstance(in_obj, dict):
return type(in_obj)(
(ConverterUtils.remove_none(k), ConverterUtils.remove_none(v)) for k, v in in_obj.items()
if k is not None and (v or v == False)
and (ConverterUtils.remove_none(k) or ConverterUtils.remove_none(k) == False)
and (ConverterUtils.remove_none(v) or ConverterUtils.remove_none(v) == False)
)
else:
return in_obj
This diff is collapsed.
......@@ -12,8 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from fastapi import APIRouter, Depends, status, Response, Body
from fastapi import APIRouter, Depends, status, Response, Body, HTTPException
import starlette.status
from app.clients.storage_service_client import get_storage_record_service
from odes_storage.models import (
......@@ -27,9 +27,56 @@ from app.utils import Context
from app.utils import get_ctx
from app.utils import load_schema_example
from app.model.model_utils import to_record, from_record
from app.converter.wellbore_converter import WellboreConverter, ConverterUtils
import re
router = APIRouter()
OSDU_WELLBORE_VERSION_REGEX = re.compile(r'^([\w\-\.]+:master-data\-\-Wellbore:[\w\-\.\:\%]+):([0-9]*)$')
OSDU_WELLBORE_REGEX = re.compile(r'^[\w\-\.]+:master-data\-\-Wellbore:[\w\-\.\:\%]+$')
DELFI_REGEX = re.compile(r'^[\w\-\.]+:[\w\-\.]+:[\w\-\.]+$')
router = APIRouter()
def is_osdu_wellbore_id(entity_id:str) -> bool:
return OSDU_WELLBORE_REGEX.match(entity_id) is not None
def is_osdu_versionned_wellbore_id(entity_id:str) -> (bool, str, str):
matches = OSDU_WELLBORE_VERSION_REGEX.match(entity_id)
if matches is None:
return False, None, None
return True, matches.group(1), matches.group(2)
def is_delfi_id(entity_id:str) -> bool:
return DELFI_REGEX.match(entity_id) is not None
def is_osdu_wellbore_fake_id(entity_id:str) -> (bool, str):
try:
delfi_id = ConverterUtils.decode_id(entity_id)
return is_delfi_id(delfi_id), delfi_id
except ValueError as e:
return False, None
async def get_wellbore_as_osdu(wellboreid: str, ctx: Context) -> Wellbore:
storage_client = await get_storage_record_service(ctx)
wellbore_record = await storage_client.get_record(
id=wellboreid, data_partition_id=ctx.partition_id
)
res_as_dict = wellbore_record.dict(
exclude_unset=True, exclude_none=True, by_alias=True
)
wellbore = Wellbore.parse_obj(WellboreConverter.convert_wks_to_osdu(res_as_dict,
context={"namespace": ctx.partition_id}))
return wellbore
async def get_osdu_wellbore(wellboreid: str, ctx: Context) -> Wellbore:
storage_client = await get_storage_record_service(ctx)
wellbore_record = await storage_client.get_record(
id=wellboreid, data_partition_id=ctx.partition_id
)
return from_record(Wellbore, wellbore_record)
@router.get(
......@@ -37,7 +84,9 @@ router = APIRouter()
response_model=Wellbore,
response_model_exclude_unset=True,
summary="Get the Wellbore using osdu schema",
description="""Get the Wellbore object using its **id**. {}""".format(REQUIRED_ROLES_READ),
description="""Get the Wellbore object using its **id**.
<p>If the **id** is a Delfi Wellbore Id, tries to convert it on the fly to return the Wellbore as an osdu Wellbore.</p>
{}""".format(REQUIRED_ROLES_READ),
operation_id="get_wellbore_osdu",
responses={
status.HTTP_404_NOT_FOUND: {"description": "Wellbore not found"}
......@@ -46,20 +95,20 @@ router = APIRouter()
async def get_wellbore_osdu(
wellboreid: str, ctx: Context = Depends(get_ctx)
) -> Wellbore:
"""
Regarding to the storage those fields are not stored and will be lost/not
retrieved:
- tags
- createTime
- createUser
- modifyTime
- modifyUser
"""
storage_client = await get_storage_record_service(ctx)
wellbore_record = await storage_client.get_record(
id=wellboreid, data_partition_id=ctx.partition_id
)
return from_record(Wellbore, wellbore_record)
delfi_convertion, delfi_id = is_osdu_wellbore_fake_id(wellboreid)
if delfi_convertion:
return await get_wellbore_as_osdu(delfi_id, ctx)
is_osdu_versionned, osdu_id, version = is_osdu_versionned_wellbore_id(wellboreid)
if is_osdu_versionned:
return await get_osdu_wellbore(osdu_id, ctx)
if is_osdu_wellbore_id(wellboreid):
return await get_osdu_wellbore(wellboreid, ctx)
if is_delfi_id(wellboreid):
return await get_wellbore_as_osdu(wellboreid, ctx)
raise HTTPException(status_code=status.HTTP_417_EXPECTATION_FAILED, detail="Id is not a wellbore")
@router.delete(
......
......@@ -33,7 +33,7 @@ class ProcessorItem:
creation_date: datetime
FIXED_RECORD_ID = ":doc:REMOVED_FOR_CICD_SCAN"
FIXED_RECORD_ID = ":doc:" + "WDMS catalog".encode('utf-8').hex()
# This name should not match any existing partition id
DEFAULT_CATALOG_NAME = "default_WDMS_catalog"
......
......@@ -10,6 +10,7 @@ structlog
python-rapidjson
python-multipart
jsonpath-ng # maintenance of 'jsonpath-rw' lib it's bit abandoned
jsonbender==0.9.3
opencensus
opencensus-ext-stackdriver
......
......@@ -15,8 +15,7 @@ The `spec` directory contains the OpenAPI specification files for Wellbore DMS.
Under `spec/generated`, the OpenAPI in JSON format is saved as-is.
## Publishing to Developer Portal
API products are grouped in families as described in the table, and link below.
https://wiki.slb.com/display/mptctransformation/Subscriptions+and+Developer+Portal
API products are grouped in families as described in the table below.
API reference/Swagger | API Product | Path | Objects/services
--- | --- | --- | ---
......
......@@ -50,13 +50,14 @@
}
],
"url": {
"raw": "{{base_url}}/entitlements/v1/groups",
"raw": "{{base_url}}/api/entitlements/v2/groups",
"host": [
"{{base_url}}"
],
"path": [
"api",
"entitlements",
"v1",
"v2",
"groups"
]
}
......@@ -1091,4 +1092,4 @@
"value": "data.default.viewers@{{data_partition}}.{{acl_domain}}"
}
]
}
\ No newline at end of file
}
......@@ -74,3 +74,43 @@ def test_crud_record_versions(with_wdms_env, kind):
def test_crud_delete_record(with_wdms_env, kind):
result = build_request(f'crud.{kind}.delete_{kind}').call(with_wdms_env)
result.assert_status_code(204)
@pytest.fixture()
def wellbore_delfi_id(with_wdms_env):
# Create a delfi wellbore
result = build_request("crud.wellbore.create_wellbore").call(with_wdms_env)
result.assert_ok()
resobj = result.get_response_obj()
assert resobj.recordCount == 1
assert len(resobj.recordIds) == 1
delfi_record_id = resobj.recordIds[0]
with_wdms_env.set('wellbore_record_id', delfi_record_id)
yield delfi_record_id
#Cleanup
result = build_request("crud.wellbore.delete_wellbore").call(with_wdms_env)
result.assert_ok()
@pytest.mark.tag('basic', 'crud', 'smoke')
def test_crud_get_as_record(wellbore_delfi_id, with_wdms_env):
delfi_record_id = wellbore_delfi_id
with_wdms_env.set('osdu_wellbore_record_id', delfi_record_id)
# Get it as osdu wellbore with delfi id
result = build_request('crud.osdu_wellbore.get_osdu_wellbore').call(with_wdms_env)
result.assert_ok()
# Get it as osdu wellbore with fake osdu id
id_as_list = delfi_record_id.split(sep=":")
encoded_str = delfi_record_id.encode().hex()
res_as_list = []
res_as_list.append(id_as_list[0])
res_as_list.append("master-data--Wellbore")
res_as_list.append(encoded_str)
res_as_list.append("")
fakeid = ":".join(res_as_list)
with_wdms_env.set('osdu_wellbore_record_id', fakeid) # stored the record id for the following tests
result = build_request('crud.osdu_wellbore.get_osdu_wellbore').call(with_wdms_env)
result.assert_ok()
......@@ -14,6 +14,8 @@
import requests
import pytest
import datetime
import jwt
payload = {}
......@@ -40,8 +42,9 @@ def skip_if_gcp_environment(base_url):
# Test for expired token
def test_expired_token_returns_40X(base_url, check_cert, token):
url = build_url(base_url, "/about")
token_expired = jwt.encode({"email":"nobody@example.com", "exp":datetime.datetime.utcnow() - datetime.timedelta(seconds=300)}, key="secret", algorithm="HS256")
headers = {
'Authorization': 'Bearer REMOVED_FOR_CICD_SCAN'
'Authorization': f"Bearer {token_expired}"
}
response = requests.request("GET", url, headers=headers, data=payload, verify=check_cert)
assert response.status_code == 401
......@@ -81,7 +84,7 @@ def test_invalid_token_returns_40X(base_url, check_cert, token):
blank = {}
token_invalid = token[0:len(token) - 10]
headers = {
'Authorization': 'Bearer REMOVED_FOR_CICD_SCAN'
'Authorization': f"Bearer {token_invalid}"
}
response = requests.request("GET", url, headers=headers, data=blank, verify=check_cert)
......@@ -92,8 +95,9 @@ def test_invalid_token_returns_40X(base_url, check_cert, token):
def test_invalid_issuer_token_returns_40X(base_url, check_cert, token):
url = build_url(base_url, "/about")
blank = {}
token_no_iss = jwt.encode({"email": "nobody@example.com"}, key="secret", algorithm="HS256")
headers = {
'Authorization': 'Bearer REMOVED_FOR_CICD_SCAN'
'Authorization': f"Bearer {token_no_iss}"
}
response = requests.request("GET", url, headers=headers, data=blank, verify=check_cert)
assert response.status_code == 401
import json
import os
import pytest
from odes_storage.models import Record
from app.converter.wellbore_converter import WellboreConverter
from app.model.osdu_model import Wellbore as OsduWellbore
RECORD_SAMPLE = [
({
"acl": {
"viewers": [
"vieweremail@domain.com"
],
"owners": [
"owneremail@domain.com"
]
},
"data": {},
"meta": [{
"kind": "Unit",
"name": "Measure depth default unit",
"persistableReference": "persistableReference",
"propertyNames": ["symbol"],
"propertyValues": ["ft"]
}],
"id": "datapartitionid:wellbore:myWellbore",
"kind": "datapartitionid:wks:wellbore:1.0.6",
"legal": {
"legaltags": [
"{{legaltags}}"
],
"otherRelevantDataCountries": [
"FR",
"US"
],
"status": "compliant"
},
"version": 1245,
"ancestry": {
"parents": [
"BP:ihs:7a336d62-ffc4-5a91-b550-f04218579828:1543413435034177",
"common:welldb:6e32f18e-2e51-512e-b982-09c7b6a11663:1543414413403142",
"BP:corp:249edfff-4c6c-538e-8c03-b3b142e45f33:1285161040815393"
]},
"tags": {"key": "value", "simple": "dict"},
"createUser": "fserin",
"modifyUser": "fserin2",
"createTime": "1961-08-25T03:55:42.109Z",
"modifyTime": "1962-11-02T15:39:57.25Z",
})
]
@pytest.mark.parametrize("input_record", RECORD_SAMPLE)
def test_record_conversion(input_record: dict):
# The transformation should let the record fields unchanged (except for data, kind)
record: Record = Record.parse_obj(input_record)
res: dict = WellboreConverter.convert_wks_to_osdu(record.dict(by_alias=True, exclude_none=True, exclude_unset=True),
context={"namespace": "test_namespace"})
# Let's ignore the data part and the kind
res["id"] = input_record["id"]
res["data"] = input_record["data"]
res["kind"] = input_record["kind"]
res_record: Record = Record.parse_obj(res)
assert record == res_record
def test_conversion():
dir_path = os.path.dirname(os.path.realpath(__file__))
with open(os.path.join(dir_path, r"wellbore_wks.json")) as f:
source_wellbore_dict = json.load(f)
source_wellbore: Record = Record.parse_obj(source_wellbore_dict)
res: dict = WellboreConverter.convert_wks_to_osdu(
source_wellbore.dict(by_alias=True, exclude_none=True, exclude_unset=True),
context={"namespace": "test_namespace"})
# Uncomment those lines to dump the actual result of the conversion
# with open("dumpsresdict.json", 'w') as fp:
# json.dump(res, fp, indent=2, default=str)
OsduWellbore.validate(res)
def test_conversion_mini():
dir_path = os.path.dirname(os.path.realpath(__file__))
with open(os.path.join(dir_path, r"wellbore_wks_mini.json")) as f:
source_wellbore_dict = json.load(f)
source_wellbore: Record = Record.parse_obj(source_wellbore_dict)
res: dict = WellboreConverter.convert_wks_to_osdu(
source_wellbore.dict(by_alias=True, exclude_none=True, exclude_unset=True),
context={"namespace": "test_namespace"})
# Uncomment those lines to dump the actual result of the conversion
# with open("dumpsresdict_mini.json", 'w') as fp:
# json.dump(res, fp, indent=2, default=str)
OsduWellbore.validate(res)
{
"data": {
"airGap": {
"unitKey": "ft",
"value": 30.0
},
"block": "Block 11/8",
"country": "United States of America",
"county": "Stark",
"dateCreated": "2013-03-22T11:16:03.123000+00:00",
"dateModified": "2013-03-22T11:16:03.123000+00:00",
"drillingDaysTarget": {
"unitKey": "days",
"value": 12.5
},
"elevationReference": {
"elevationFromMsl": {
"unitKey": "ft",
"value": 2650.5
},
"name": "GL"
},
"externalIds": null,
"field": "Bell",
"formationAtTd": "Bakken",
"formationProjected": "Bakken",
"hasAchievedTotalDepth": true,
"isActive": true,
"kickOffMd": {
"unitKey": "ft",
"value": 5324.9
},
"kickOffTvd": {
"unitKey": "ft",
"value": 5103.8
},
"locationWGS84": {
"bbox": null,
"features": [
{
"bbox": null,
"geometry": {
"bbox": null,
"coordinates": [
[
-103.2380248,
46.8925081,
5301.0
],
[
-103.2380248,
46.8925081,
2801.0
],
[
-103.2378748,
46.892608100000004,
301.0
],
[
-103.23742477750001,
46.89270811,
-2199.0
],
[
-103.23667470999663,
46.892808120001,
-4699.0
],
[
-103.2356245974865,
46.892908130002,
-7199.0
]
],
"type": "LineString"
},
"properties": {
"name": "Newton 2-31-Lat-1"
},
"type": "Feature"
}
],
"type": "FeatureCollection"
},
"name": "Newton 2-31-Lat-1",
"operator": "Don E. Beckert",
"permitDate": "2013-01-15",
"permitNumber": "608020",
"plssLocation": {
"aliquotPart": null,
"range": "99W",
"section": 31,
"township": "140N"
},
"propertyDictionary": {
"API Number": "33003000080000",
"Activity Code": "E",
"Basin": "WILLISTON BASIN",
"Basin Code": "713200",
"Class Initial Code": "WF",
"Class Initial Name": "NEW FIELD WILDCAT",
"Country Name": "UNITED STATES",
"County Name": "BARNES",
"Current Operator City": "BILLINGS",
"Current Operator Name": "NYVATEX MONTANA",
"Date First Report": "11-12-1982",
"Date Last Activity": "06-03-2016",
"Depth Total Projected": "1800",
"Elevation Reference Datum": "GR",
"Elevation Reference Value": "1407",
"Field Name": "WILDCAT",
"Final Status": "ABANDON LOCATION",