Commit 7f53e8c2 authored by ethiraj krishnamanaidu's avatar ethiraj krishnamanaidu
Browse files

Merge branch '38-schema-bootstrap-refresh' into 'master'

Resolve "[Schema Service] Refresh R3 schemas from DD Feb 25/reference-data Mar 16"

Closes #38

See merge request !96
parents 36f8d3ee 65b5b610
Pipeline #35345 failed with stages
in 72 minutes and 53 seconds
......@@ -4,93 +4,134 @@ import time
import requests
import argparse
from Utility import Utility, RunEnv
from typing import Tuple
class DeploySharedSchemas:
SCHEMA_AUTHORITY_TO_REPLACE = '{{schema-authority}}'
SCHEMA_EXISTS = 'Schema Id is already present'
PUBLISHED_SCHEMA_ERROR = ['Only schema in development stage can be updated',
'Only schema in developement stage can be updated']
ALREADY_PUBLISHED = 'AlreadyPublished'
SUCCESS = 'Success'
def __init__(self):
parser = argparse.ArgumentParser(
description="Given a path to an load sequence file, load/update the schemas "
"listed in the load sequence file.")
parser.add_argument('-l', type=str,
help='The path to the load sequence file, e.g. load_sequence.?.?.?',
default=None)
parser.add_argument('-u', help='The complete URL to the Schema Service.',
default=None)
arguments = parser.parse_args()
if arguments.l is not None:
RunEnv.LOAD_SEQUENCE = arguments.l
if arguments.u is not None:
RunEnv.SCHEMA_SERVICE_URL = arguments.u
if RunEnv.SCHEMA_SERVICE_URL is None:
exit('The schema service URL is not specified')
self.url = RunEnv.SCHEMA_SERVICE_URL
self.schema_registered = None
self.token = RunEnv.BEARER_TOKEN
self.schema_info_registered = None
self.headers = {
'data-partition-id': RunEnv.DATA_PARTITION,
'Content-Type': 'application/json',
'AppKey': RunEnv.APP_KEY,
'Authorization': RunEnv.BEARER_TOKEN
}
print('Current data-partition-id: {}'.format(RunEnv.DATA_PARTITION))
ok, error_mess = RunEnv().is_ok()
if not ok:
exit('Error: environment setting incomplete: {}'.format(error_mess))
def create_schema(self):
messages = list()
tenant = RunEnv.DATA_PARTITION
deployments = Utility.path_to_deployments()
start = time.time()
headers = {
'data-partition-id': tenant,
'Content-Type': 'application/json',
'AppKey': RunEnv.APP_KEY,
'Authorization': self.token
}
bootstrap_options = json.loads(RunEnv.BOOTSTRAP_OPTIONS)
for option in bootstrap_options:
try:
schema_path = option['folder']
schema_authority = option['authority']
load_sequence = option['load-sequence']
except KeyError as e:
exit('Key missing in bootstrap-options::{}'.format(str(e)))
sequence = Utility.load_json(os.path.join(deployments, RunEnv.SCHEMAS_FOLDER, schema_path, load_sequence))
for item in sequence:
self.schema_registered = None
schema_file = os.path.join(deployments, item['relativePath'])
schema = open(schema_file, 'r').read()
schema = schema.replace(self.SCHEMA_AUTHORITY_TO_REPLACE, schema_authority)
kind = self.__kind_from_schema_info(schema)
self.__register_one(kind, schema, messages)
sequence = Utility.load_json(os.path.join(deployments, *RunEnv.OSDU, RunEnv.LOAD_SEQUENCE))
for item in sequence:
self.schema_registered = None
schema_file = os.path.join(deployments, item['relativePath'])
kind = item['kind']
schema = open(schema_file, 'r').read()
# attempt to load this kind
exists = self.__does_kind_exist(kind, headers)
method = 'POST'
if exists:
if schema_file.endswith('.dev.json'):
method = 'PUT'
response = requests.request(method, self.url, headers=headers, data=schema)
else:
message = 'Error: The published kind {} cannot be updated; it already exists.'.format(kind)
print(message)
messages.append(message)
response = None
elapsed = time.time() - start
print('This update took {:.2f} seconds.'.format(elapsed))
if len(messages) != 0:
print('Following schemas failed:')
print('\n'.join(messages))
exit(1)
else:
response = requests.request(method, self.url, headers=headers, data=schema)
print('All {} schemas registered, updated or '
'left unchanged because of status PUBLISHED.'.format(str(len(sequence))))
if response is not None:
print('Success: kind {} submitted with method {} schema.'.format(kind, method))
code = response.status_code
if code not in range(200,300):
# response_message = json.loads(response.text)
# if response_message.get('error'):
messages.append(
'Error with kind {}: Message: {}'.format(kind, response.text))
@staticmethod
def __kind_from_schema_info(schema_as_str: str) -> str:
kind = 'Error'
try:
schema = json.loads(schema_as_str)
si = schema.get('schemaInfo', dict()).get('schemaIdentity', dict())
authority = si.get('authority', '')
source = si.get('source', '')
entity = si.get('entityType', '')
major = str(si.get('schemaVersionMajor', 0))
minor = str(si.get('schemaVersionMinor', 0))
patch = str(si.get('schemaVersionPatch', 0))
kind = '{}:{}:{}:{}.{}.{}'.format(authority, source, entity, major, minor, patch)
except Exception as e:
exit('Invalid JSON in payload {}'.format(str(e)))
return kind
elapsed = time.time() - start
print('This update took {:.2f} seconds.'.format(elapsed))
if len(messages) != 0:
print('Following schemas failed:')
print('\n'.join(messages))
exit(1)
else:
print('All {} schemas registered or updated.'.format(str(len(sequence))))
def __register_one(self, kind, schema, messages):
method = 'POST'
try_it = 'Try {} for id: {}'
print(try_it.format(method, kind))
response = requests.request(method, self.url, headers=self.headers, data=schema)
is_error, message, method = self.__evaluate_response(response)
if method == 'PUT': # try again
print(try_it.format(method, kind))
response = requests.request(method, self.url, headers=self.headers, data=schema)
is_error, message, method = self.__evaluate_response(response)
if is_error:
message = 'Error with kind {}: Message: {}'.format(kind, message)
print(message)
messages.append(message)
elif method == self.SUCCESS:
print('The kind {} was registered successfully.'.format(kind))
elif method == self.ALREADY_PUBLISHED:
print('The kind {} was already registered with status PUBLISHED '
'and was not updated.'.format(kind))
def __does_kind_exist(self, kind, headers):
url = '{}/{}'.format(self.url, kind)
response = requests.request("GET", url, headers=headers)
if response.status_code in range(200,300):
self.schema_registered = json.loads(response.text, encoding='utf-8')
return response.status_code in range(200,300)
def __evaluate_response(self, response: requests.Response) -> Tuple[bool, str, str]:
code = response.status_code
message = ''
method = 'Give up'
error = code not in range(200, 300)
if error:
# further test:
try:
js_err = json.loads(response.text)
message = js_err.get('error', dict()).get('message', '')
if message == self.SCHEMA_EXISTS:
method = 'PUT' # try PUT, it might have been DEVELOPMENT, than we can overwrite
elif message in self.PUBLISHED_SCHEMA_ERROR: # already PUBLISHED, no bootstrap required
method = self.ALREADY_PUBLISHED
error = False # this is not considered an error
# everything else is an error
except Exception as e:
message = str(e)
else:
method = self.SUCCESS
return error, message, method
if __name__ == '__main__':
......
......@@ -13,8 +13,10 @@ class RunEnv(object):
SCHEMA_SERVICE_URL = None
STORAGE_SERVICE_URL = None
DATA_PARTITION = os.environ.get('DATA_PARTITION')
OSDU = ['shared-schemas', 'osdu']
LOAD_SEQUENCE = 'load_sequence.0.2.0.json'
SCHEMA_AUTHORITY = os.environ.get('SCHEMA_AUTHORITY')
SCHEMAS_FOLDER = 'shared-schemas'
DEFAULT_BOOTSTRAP_OPTIONS = '[{"authority": "osdu", "folder": "osdu", "load-sequence": "load_sequence.1.0.0.json"}]'
BOOTSTRAP_OPTIONS = os.environ.get('BOOTSTRAP_OPTIONS', DEFAULT_BOOTSTRAP_OPTIONS)
def __init__(self):
"""Empty constructor"""
......@@ -118,9 +120,12 @@ class Utility(object):
@staticmethod
def load_json(path):
"""Load a JSON file"""
with open(path, "r", encoding='utf-8') as text_file:
j_obj = json.load(text_file)
return j_obj
try:
with open(path, "r", encoding='utf-8') as text_file:
j_obj = json.load(text_file)
return j_obj
except FileNotFoundError as e:
exit("Given File path not found::{}".format(str(e)))
@staticmethod
def save_json(schema, path, sort_keys=False):
......@@ -152,7 +157,7 @@ class StorageService(object):
url = '{}/schemas/{}'.format(self.url, urllib.parse.quote_plus(kind))
response = requests.request("GET", url, headers=self.headers)
if response.status_code in range(200, 300):
schema = json.loads(response.text, encoding='utf-8')
schema = json.loads(response.text)
else:
messages.append('Error: Storage Service GET schema {}, {}'.format(response.status_code, response.text))
return response.status_code in range(200, 300), schema
......@@ -167,7 +172,7 @@ class StorageService(object):
url += '&cursor={}'.format(cursor)
response = requests.request("GET", url, headers=self.headers)
if response.status_code in range(200, 300):
rs = json.loads(response.text, encoding='utf-8')
rs = json.loads(response.text)
cursor = rs['cursor']
carry_on = len(rs['results']) > 0
for kind in rs['results']:
......@@ -188,7 +193,7 @@ class SchemaService(object):
url = '{}/{}'.format(self.url, kind)
response = requests.request("GET", url, headers=self.headers)
if response.status_code in range(200,300):
schema = json.loads(response.text, encoding='utf-8')
schema = json.loads(response.text)
return response.status_code in range(200,300), schema
def post_or_put_schema(self, kind: str, schema: dict, schema_status: str, messages: list):
......@@ -242,11 +247,9 @@ class SchemaService(object):
response = requests.request("GET", url, headers=self.headers)
schema_infos = list()
if response.status_code in range(200,300):
r = json.loads(response.text, encoding='utf-8')
r = json.loads(response.text)
schema_infos = r['schemaInfos']
for info in schema_infos:
if self.__match(info['schemaIdentity'], si):
return response.status_code in range(200,300), info
return response.status_code in range(200,300), None
return response.status_code in range(200,300), None
\ No newline at end of file
......@@ -19,13 +19,13 @@ POST/PUT requests:
{
"schemaInfo": {
"schemaIdentity": {
"authority": "osdu",
"authority": "{{schema-authority}}",
"source": "wks",
"entity": "work-product-component.WellLog",
"entityType": "work-product-component--WellLog",
"schemaVersionMajor": 1,
"schemaVersionMinor": 0,
"schemaVersionPatch": 0,
"id": "osdu:wks:work-product-component.WellLog:1.0.0"
"id": "{{schema-authority}}:wks:work-product-component--WellLog:1.0.0"
},
"createdBy": "OSDU Data Definition Group",
"scope": "SHARED",
......@@ -51,19 +51,21 @@ done via the [DeploySharedSchemas.py](../scripts/DeploySharedSchemas.py):
```shell script
python deployments\scripts\DeploySharedSchemas.py -h
usage: DeploySharedSchemas.py [-h] [-l L] [-u U]
usage: DeploySharedSchemas.py [-h] [-a A] [-l L] [-u U]
Given a path to an load sequence file, load/update the schemas listed in the
load sequence file.
optional arguments:
-h, --help show this help message and exit
-a A The schema authority or partition-id to replace (default via
Env)
-l L The path to the load sequence file, e.g. load_sequence.?.?.?
-u U The complete URL to the Schema Service.
example:
python deployments\scripts\DeploySharedSchemas.py -l load_sequence.1.0.0.json -u https://opengroup.test.org/api/schema-service/v1/schema
python deployments\scripts\DeploySharedSchemas.py -l load_sequence.1.0.0.json -a osdu -u https://opengroup.test.org/api/schema-service/v1/schema
```
......
{
"schemaInfo": {
"schemaIdentity": {
"authority": "opendes",
"authority": "{{schema-authority}}",
"source": "wks",
"entityType": "AbstractAccessControlList",
"schemaVersionMajor": 1,
"schemaVersionMinor": 0,
"schemaVersionPatch": 0,
"id": "opendes:wks:AbstractAccessControlList:1.0.0"
"id": "{{schema-authority}}:wks:AbstractAccessControlList:1.0.0"
},
"createdBy": "OSDU Data Definition Group",
"scope": "SHARED",
......
{
"schemaInfo": {
"schemaIdentity": {
"authority": "opendes",
"authority": "{{schema-authority}}",
"source": "wks",
"entityType": "AbstractAliasNames",
"schemaVersionMajor": 1,
"schemaVersionMinor": 0,
"schemaVersionPatch": 0,
"id": "opendes:wks:AbstractAliasNames:1.0.0"
"id": "{{schema-authority}}:wks:AbstractAliasNames:1.0.0"
},
"createdBy": "OSDU Data Definition Group",
"scope": "SHARED",
......
{
"schemaInfo": {
"schemaIdentity": {
"authority": "opendes",
"authority": "{{schema-authority}}",
"source": "wks",
"entityType": "AbstractAnyCrsFeatureCollection",
"schemaVersionMajor": 1,
"schemaVersionMinor": 0,
"schemaVersionPatch": 0,
"id": "opendes:wks:AbstractAnyCrsFeatureCollection:1.0.0"
"id": "{{schema-authority}}:wks:AbstractAnyCrsFeatureCollection:1.0.0"
},
"createdBy": "OSDU Data Definition Group",
"scope": "SHARED",
......
{
"schemaInfo": {
"schemaIdentity": {
"authority": "opendes",
"authority": "{{schema-authority}}",
"source": "wks",
"entityType": "AbstractBinGrid",
"schemaVersionMajor": 1,
"schemaVersionMinor": 0,
"schemaVersionPatch": 0,
"id": "opendes:wks:AbstractBinGrid:1.0.0"
"id": "{{schema-authority}}:wks:AbstractBinGrid:1.0.0"
},
"createdBy": "OSDU Data Definition Group",
"scope": "SHARED",
......@@ -63,13 +63,13 @@
"ABCDBinGridLocalCoordinates": {
"type": "array",
"items": {
"$ref": "opendes:wks:AbstractCoordinates:1.0.0"
"$ref": "{{schema-authority}}:wks:AbstractCoordinates:1.0.0"
},
"description": "Array of 4 corner points for bin grid in local coordinates: Point A (min inline, min crossline); Point B (min inline, max crossline); Point C (max inline, min crossline); Point D (max inline, max crossline). If Point D is not given and BinGridDefinitionMethodTypeID=4, it must be supplied, with its spatial location, before ingestion to create a parallelogram in map coordinate space. Note correspondence of inline=x, crossline=y."
},
"ABCDBinGridSpatialLocation": {
"description": "Bin Grid ABCD points containing the projected coordinates, projected CRS and quality metadata. This attribute is required also for the P6 definition method to define the projected CRS, even if the ABCD coordinates would be optional (recommended to be always calculated).",
"$ref": "opendes:wks:AbstractSpatialLocation:1.0.0"
"$ref": "{{schema-authority}}:wks:AbstractSpatialLocation:1.0.0"
},
"P6TransformationMethod": {
"description": "EPSG code: 9666 for right-handed, 1049 for left-handed. See IOGP Guidance Note 373-07-2 and 483-6.",
......
{
"schemaInfo": {
"schemaIdentity": {
"authority": "opendes",
"authority": "{{schema-authority}}",
"source": "wks",
"entityType": "AbstractBusinessRule",
"schemaVersionMajor": 1,
"schemaVersionMinor": 0,
"schemaVersionPatch": 0,
"id": "opendes:wks:AbstractBusinessRule:1.0.0"
"id": "{{schema-authority}}:wks:AbstractBusinessRule:1.0.0"
},
"createdBy": "OSDU Data Definition Group",
"scope": "SHARED",
......
{
"schemaInfo": {
"schemaIdentity": {
"authority": "opendes",
"authority": "{{schema-authority}}",
"source": "wks",
"entityType": "AbstractCommonResources",
"schemaVersionMajor": 1,
"schemaVersionMinor": 0,
"schemaVersionPatch": 0,
"id": "opendes:wks:AbstractCommonResources:1.0.0"
"id": "{{schema-authority}}:wks:AbstractCommonResources:1.0.0"
},
"createdBy": "OSDU Data Definition Group",
"scope": "SHARED",
......
{
"schemaInfo": {
"schemaIdentity": {
"authority": "opendes",
"authority": "{{schema-authority}}",
"source": "wks",
"entityType": "AbstractCompressionInfo",
"schemaVersionMajor": 1,
"schemaVersionMinor": 0,
"schemaVersionPatch": 0,
"id": "opendes:wks:AbstractCompressionInfo:1.0.0"
"id": "{{schema-authority}}:wks:AbstractCompressionInfo:1.0.0"
},
"createdBy": "OSDU Data Definition Group",
"scope": "SHARED",
......
{
"schemaInfo": {
"schemaIdentity": {
"authority": "opendes",
"authority": "{{schema-authority}}",
"source": "wks",
"entityType": "AbstractCoordinates",
"schemaVersionMajor": 1,
"schemaVersionMinor": 0,
"schemaVersionPatch": 0,
"id": "opendes:wks:AbstractCoordinates:1.0.0"
"id": "{{schema-authority}}:wks:AbstractCoordinates:1.0.0"
},
"createdBy": "OSDU Data Definition Group",
"scope": "SHARED",
......
{
"schemaInfo": {
"schemaIdentity": {
"authority": "opendes",
"authority": "{{schema-authority}}",
"source": "wks",
"entityType": "AbstractDataset",
"schemaVersionMajor": 1,
"schemaVersionMinor": 0,
"schemaVersionPatch": 0,
"id": "opendes:wks:AbstractDataset:1.0.0"
"id": "{{schema-authority}}:wks:AbstractDataset:1.0.0"
},
"createdBy": "OSDU Data Definition Group",
"scope": "SHARED",
......
{
"schemaInfo": {
"schemaIdentity": {
"authority": "opendes",
"authority": "{{schema-authority}}",
"source": "wks",
"entityType": "AbstractFacility",
"schemaVersionMajor": 1,
"schemaVersionMinor": 0,
"schemaVersionPatch": 0,
"id": "opendes:wks:AbstractFacility:1.0.0"
"id": "{{schema-authority}}:wks:AbstractFacility:1.0.0"
},
"createdBy": "OSDU Data Definition Group",
"scope": "SHARED",
......@@ -41,7 +41,7 @@
"description": "The history of operator organizations of the facility.",
"type": "array",
"items": {
"$ref": "opendes:wks:AbstractFacilityOperator:1.0.0"
"$ref": "{{schema-authority}}:wks:AbstractFacilityOperator:1.0.0"
}
},
"InitialOperatorID": {
......@@ -95,31 +95,31 @@
"type": "string"
},
"FacilityNameAliases": {
"description": "Alternative names, including historical, by which this facility is/has been known.",
"description": "DEPRECATED: please use data.NameAliases. Alternative names, including historical, by which this facility is/has been known.",
"type": "array",
"items": {
"$ref": "opendes:wks:AbstractAliasNames:1.0.0"
"$ref": "{{schema-authority}}:wks:AbstractAliasNames:1.0.0"
}
},
"FacilityStates": {
"description": "The history of life cycle states the facility has been through.",
"type": "array",
"items": {
"$ref": "opendes:wks:AbstractFacilityState:1.0.0"
"$ref": "{{schema-authority}}:wks:AbstractFacilityState:1.0.0"
}
},
"FacilityEvents": {
"description": "A list of key facility events.",
"type": "array",
"items": {
"$ref": "opendes:wks:AbstractFacilityEvent:1.0.0"
"$ref": "{{schema-authority}}:wks:AbstractFacilityEvent:1.0.0"
}
},
"FacilitySpecifications": {
"description": "facilitySpecification maintains the specification like slot name, wellbore drilling permit number, rig name etc.",
"type": "array",
"items": {
"$ref": "opendes:wks:AbstractFacilitySpecification:1.0.0"
"$ref": "{{schema-authority}}:wks:AbstractFacilitySpecification:1.0.0"
}
}
},
......
{
"schemaInfo": {
"schemaIdentity": {
"authority": "opendes",
"authority": "{{schema-authority}}",
"source": "wks",
"entityType": "AbstractFacilityEvent",
"schemaVersionMajor": 1,
"schemaVersionMinor": 0,
"schemaVersionPatch": 0,
"id": "opendes:wks:AbstractFacilityEvent:1.0.0"
"id": "{{schema-authority}}:wks:AbstractFacilityEvent:1.0.0"
},
"createdBy": "OSDU Data Definition Group",
"scope": "SHARED",
......
{
"schemaInfo": {
"schemaIdentity": {
"authority": "opendes",
"authority": "{{schema-authority}}",
"source": "wks",
"entityType": "AbstractFacilityOperator",
"schemaVersionMajor": 1,
"schemaVersionMinor": 0,
"schemaVersionPatch": 0,
"id": "opendes:wks:AbstractFacilityOperator:1.0.0"
"id": "{{schema-authority}}:wks:AbstractFacilityOperator:1.0.0"
},
"createdBy": "OSDU Data Definition Group",
"scope": "SHARED",
......