Commit 1b80fc41 authored by Thomas Gehrmann [SLB]'s avatar Thomas Gehrmann [SLB]
Browse files

Re-import and generate loading sequence

parent 145aaa08
Pipeline #23299 failed with stages
in 88 minutes and 23 seconds
......@@ -37,12 +37,6 @@ class ImportFromOSDU(object):
else:
print('Unrecognized universe: {}'.format(arguments.u))
@staticmethod
def __generated_target(file):
if file.endswith('abstractResources.json'):
return None
return file.replace('Authoring', 'Generated')
def discover_schemas(self, deployments, folder_parts):
files = Utility.find_files(folder_parts, deployments)
files = sorted(files) # this brings abstract to the front, less work for dependency chasing.
......@@ -64,6 +58,9 @@ class ImportFromOSDU(object):
}
if group_type is not None and group_type not in self.IGNORE_GROUP_TYPES:
schema_file['group-type'] = group_type
elif group_type is not None and group_type == 'manifest':
schema_file['group-type'] = self.__get_group_type_from_schema_kind(file)
if version is not None:
schema_file['version'] = version
else:
......@@ -82,6 +79,16 @@ class ImportFromOSDU(object):
if len(infos) > 0:
self.info = Utility.load_json(infos[-1])
@staticmethod
def __get_group_type_from_schema_kind(file: str):
msc = Utility.load_json(file)
kind = msc.get('x-osdu-schema-source')
et = kind.split(':')[2]
gt = None
if ImportFromOSDU.SEPARATOR in et:
gt = et.split(ImportFromOSDU.SEPARATOR)[0]
return gt
def copy_and_record_dependencies(self):
remove_me = list()
if self.__status() == 'DEVELOPMENT':
......@@ -96,7 +103,8 @@ class ImportFromOSDU(object):
path = os.path.join(schema_file['target'], kind_file)
schema = Utility.load_json(schema_file['source'])
schema_file['dependencies'] = self.find_references(schema)
schema_info = self.__make_schema_info(schema_file, schema)
entity_info = copy.deepcopy(schema_file)
schema_info = self.__make_schema_info(entity_info, schema)
to_load = {'schemaInfo': schema_info, 'schema': schema}
Utility.save_json(to_load, path)
else:
......@@ -249,6 +257,13 @@ class ImportFromOSDU(object):
if isinstance(entity_file, dict):
version = entity_file['version']
entity = entity_file['entity']
if not is_file:
parts = list()
gt = entity_file.get('group-type')
if gt is not None:
parts.append(gt)
parts.append(entity)
entity = self.SEPARATOR.join(parts)
else:
version = '.'.join([self.__major(), self.__minor(), self.__patch()])
entity = entity_file
......@@ -298,4 +313,4 @@ class ImportFromOSDU(object):
if __name__ == '__main__':
ImportFromOSDU()
\ No newline at end of file
ImportFromOSDU()
......@@ -91,9 +91,9 @@ class Utility(object):
entity = parts[1].replace('.json', '')
if '.' in entity: # OSDU R3 contains version in file name
vps = entity.split('.') # filename: <entityType>.major.minor.patch - 4 parts
if len(vps) == 4:
entity = vps[0]
version = '.'.join(vps[1:])
if len(vps) >= 4:
version = '.'.join([vps[-3], vps[-2], vps[-1]])
entity = entity.replace('.'+version, '')
else:
exit('Error in entity name/version: {} expected <entityType>.major.minor.patch.json'.format(entity))
parts = parts[0].split(os.sep)
......
{
"schemaInfo": {
"schemaIdentity": {
"authority": "osdu",
"source": "wks",
"entityType": "AbstractCommonResources",
"schemaVersionMajor": 0,
"schemaVersionMinor": 0,
"schemaVersionPatch": 0,
"id": "schema-namespace:abstract:AbstractCommonResources:0.0.0"
},
"createdBy": "OSDU Data Definition Group",
"scope": "SHARED",
"status": "DEVELOPMENT"
},
"schema": {
"x-osdu-license": "Copyright 2021, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
"$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractCommonResources.1.0.0.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"x-osdu-schema-source": "schema-namespace:abstract:AbstractCommonResources:0.0.0",
"title": "OSDU Common Resources",
"description": "Common resources to be injected at root 'data' level for every entity, which is persistable in Storage. The insertion is performed by the OsduSchemaComposer script.",
"type": "object",
"properties": {
"ResourceHomeRegionID": {
"description": "The name of the home [cloud environment] region for this OSDU resource object.",
"title": "Resource Home Region ID",
"type": "string",
"pattern": "^[\\w\\-\\.]+:reference-data\\-\\-OSDURegion:.+:[0-9]*$",
"x-osdu-relationship": [
{
"GroupType": "reference-data",
"EntityType": "OSDURegion"
}
]
},
"ResourceHostRegionIDs": {
"description": "The name of the host [cloud environment] region(s) for this OSDU resource object.",
"title": "Resource Host Region ID",
"type": "array",
"items": {
"type": "string",
"pattern": "^[\\w\\-\\.]+:reference-data\\-\\-OSDURegion:.+:[0-9]*$",
"x-osdu-relationship": [
{
"GroupType": "reference-data",
"EntityType": "OSDURegion"
}
]
}
},
"ResourceCurationStatus": {
"description": "Describes the current Curation status.",
"title": "Resource Curation Status",
"type": "string",
"pattern": "^[\\w\\-\\.]+:reference-data\\-\\-ResourceCurationStatus:.+:[0-9]*$",
"x-osdu-relationship": [
{
"GroupType": "reference-data",
"EntityType": "ResourceCurationStatus"
}
]
},
"ResourceLifecycleStatus": {
"description": "Describes the current Resource Lifecycle status.",
"title": "Resource Lifecycle Status",
"type": "string",
"pattern": "^[\\w\\-\\.]+:reference-data\\-\\-ResourceLifecycleStatus:.+:[0-9]*$",
"x-osdu-relationship": [
{
"GroupType": "reference-data",
"EntityType": "ResourceLifecycleStatus"
}
]
},
"ResourceSecurityClassification": {
"description": "Classifies the security level of the resource.",
"title": "Resource Security Classification",
"type": "string",
"pattern": "^[\\w\\-\\.]+:reference-data\\-\\-ResourceSecurityClassification:.+:[0-9]*$",
"x-osdu-relationship": [
{
"GroupType": "reference-data",
"EntityType": "ResourceSecurityClassification"
}
]
},
"Source": {
"description": "The entity that produced the record, or from which it is received; could be an organization, agency, system, internal team, or individual. For informational purposes only, the list of sources is not governed.",
"title": "Data Source",
"type": "string"
},
"ExistenceKind": {
"description": "Where does this data resource sit in the cradle-to-grave span of its existence?",
"title": "Existence Kind",
"type": "string",
"pattern": "^[\\w\\-\\.]+:reference-data\\-\\-ExistenceKind:.+:[0-9]*$",
"x-osdu-relationship": [
{
"GroupType": "reference-data",
"EntityType": "ExistenceKind"
}
]
}
},
"x-osdu-inheriting-from-kind": []
}
}
\ No newline at end of file
{
"schemaInfo": {
"schemaIdentity": {
"authority": "osdu",
"source": "wks",
"entityType": "AbstractCompressionInfo",
"schemaVersionMajor": 1,
"schemaVersionMinor": 0,
"schemaVersionPatch": 0,
"id": "osdu:wks:AbstractCompressionInfo:1.0.0"
},
"createdBy": "OSDU Data Definition Group",
"scope": "SHARED",
"status": "DEVELOPMENT"
},
"schema": {
"x-osdu-license": "Copyright 2021, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
"x-osdu-review-status": "Accepted",
"$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractCompressionInfo.1.0.0.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"x-osdu-schema-source": "osdu:wks:AbstractCompressionInfo:1.0.0",
"title": "AbstractCompressionInfo",
"description": "A schema fragment containing information about the compression applied. This is typically included for types supporting compression in the dataset group-type.",
"type": "object",
"properties": {
"LossyCompressionIndicator": {
"type": "boolean",
"description": "Boolean that warns that an imperfect compression algorithm has been applied to the bulk binary data. Details of the compression method need to be discovered from the format properties and file access methods."
},
"CompressionMethodTypeID": {
"type": "string",
"description": "Name of a compression algorithm applied to the data as stored.",
"pattern": "^[\\w\\-\\.]+:reference-data\\-\\-CompressionMethodType:.+:[0-9]*$",
"x-osdu-relationship": [
{
"GroupType": "reference-data",
"EntityType": "CompressionMethodType"
}
]
},
"CompressionLevel": {
"type": "number",
"description": "Number indicating degree of fidelity present in bulk data resulting from compression. Meaning of number depends on algorithm."
}
},
"x-osdu-inheriting-from-kind": []
}
}
\ No newline at end of file
{
"schemaInfo": {
"schemaIdentity": {
"authority": "osdu",
"source": "wks",
"entityType": "AbstractDataset",
"schemaVersionMajor": 1,
"schemaVersionMinor": 0,
"schemaVersionPatch": 0,
"id": "osdu:wks:AbstractDataset:1.0.0"
},
"createdBy": "OSDU Data Definition Group",
"scope": "SHARED",
"status": "DEVELOPMENT"
},
"schema": {
"x-osdu-license": "Copyright 2021, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
"x-osdu-review-status": "Accepted",
"$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractDataset.1.0.0.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"x-osdu-schema-source": "osdu:wks:AbstractDataset:1.0.0",
"title": "AbstractDataset",
"description": "Schema fragment holding properties common for all datasets.",
"type": "object",
"properties": {
"Name": {
"type": "string",
"title": "Name",
"description": "An optional name of the dataset, e.g. a user friendly file or file collection name.",
"example": "Dataset X221/15"
},
"Description": {
"type": "string",
"title": "Description",
"description": "An optional, textual description of the dataset.",
"example": "As originally delivered by ACME.com."
},
"TotalSize": {
"type": "string",
"title": "Total Size",
"description": "Total size of the dataset in bytes; for files it is the same as declared in FileSourceInfo.FileSize or the sum of all individual files. Implemented as string. The value must be convertible to a long integer (sizes can become very large).",
"format": "integer",
"pattern": "^[0-9]+$",
"example": 13245217273
},
"EncodingFormatTypeID": {
"type": "string",
"title": "Encoding Format Type ID",
"description": "EncodingFormatType ID reference value relationship. It can me a mime-type or media-type.",
"example": "namespace:reference-data--EncodingFormatType:text/csv:",
"pattern": "^[\\w\\-\\.]+:reference-data\\-\\-EncodingFormatType:.+:[0-9]*$",
"x-osdu-relationship": [
{
"GroupType": "reference-data",
"EntityType": "EncodingFormatType"
}
]
},
"SchemaFormatTypeID": {
"type": "string",
"title": "Schema Format Type ID",
"description": "Relationship to the SchemaFormatType reference value.",
"example": "namespace:reference-data--SchemaFormatType:CWLS LAS3:",
"pattern": "^[\\w\\-\\.]+:reference-data\\-\\-SchemaFormatType:.+:[0-9]*$",
"x-osdu-relationship": [
{
"GroupType": "reference-data",
"EntityType": "SchemaFormatType"
}
]
},
"Endian": {
"type": "string",
"description": "Endianness of binary value. Enumeration: \"BIG\", \"LITTLE\". If absent, applications will need to interpret from context indicators.",
"enum": [
"BIG",
"LITTLE"
]
},
"DatasetProperties": {
"type": "object",
"title": "Dataset Properties",
"description": "Placeholder for a specialization.",
"example": {}
}
},
"required": [
"DatasetProperties"
],
"x-osdu-inheriting-from-kind": []
}
}
\ No newline at end of file
{
"schemaInfo": {
"schemaIdentity": {
"authority": "osdu",
"source": "wks",
"entityType": "AbstractFile",
"schemaVersionMajor": 1,
"schemaVersionMinor": 0,
"schemaVersionPatch": 0,
"id": "osdu:wks:AbstractFile:1.0.0"
},
"createdBy": "OSDU Data Definition Group",
"scope": "SHARED",
"status": "DEVELOPMENT"
},
"schema": {
"x-osdu-license": "Copyright 2021, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
"x-osdu-review-status": "Accepted",
"$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractFile.1.0.0.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"x-osdu-schema-source": "osdu:wks:AbstractFile:1.0.0",
"title": "AbstractFile",
"description": "The schema fragment representing single files",
"type": "object",
"properties": {
"DatasetProperties": {
"type": "object",
"title": "Dataset Properties",
"description": "The dataset properties for a single file.",
"properties": {
"FileSourceInfo": {
"$ref": "osdu:wks:AbstractFileSourceInfo:1.0.0",
"title": "File Source Info",
"description": "Information about the single file."
}
}
},
"Checksum": {
"type": "string",
"title": "MD5 Checksum",
"description": "MD5 checksum of file bytes - a 32 byte hexadecimal number.",
"example": "d41d8cd98f00b204e9800998ecf8427e",
"pattern": "^[0-9a-fA-F]{32}"
}
},
"required": [
"DatasetProperties"
],
"x-osdu-inheriting-from-kind": []
}
}
\ No newline at end of file
{
"schemaInfo": {
"schemaIdentity": {
"authority": "osdu",
"source": "wks",
"entityType": "AbstractFileCollection",
"schemaVersionMajor": 1,
"schemaVersionMinor": 0,
"schemaVersionPatch": 0,
"id": "osdu:wks:AbstractFileCollection:1.0.0"
},
"createdBy": "OSDU Data Definition Group",
"scope": "SHARED",
"status": "DEVELOPMENT"
},
"schema": {
"x-osdu-license": "Copyright 2021, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
"x-osdu-review-status": "Accepted",
"$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractFileCollection.1.0.0.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"x-osdu-schema-source": "osdu:wks:AbstractFileCollection:1.0.0",
"title": "AbstractFileCollection",
"description": "The schema fragment representing file collections.",
"type": "object",
"properties": {
"DatasetProperties": {
"type": "object",
"title": "Dataset Properties",
"description": "The dataset properties for a file collection.",
"properties": {
"FileCollectionPath": {
"type": "string",
"title": "File Collection Path",
"description": "An optional path to the file collection.",
"example": "s3://default_bucket/opendes/data/vds-dataset/"
},
"IndexFilePath": {
"type": "string",
"title": "Index File Path",
"description": "An optional path to an index file.",
"example": "s3://default_bucket/opendes/data/vds-dataset/vds-dataset.index"
},
"FileSourceInfos": {
"type": "array",
"title": "File Source Info Array",
"description": "Array of file collection members as FileSourceInfo.",
"example": [
{
"FileSource": "s3://default_bucket/opendes/data/vds-dataset/vds-file-1",
"PreloadFilePath": "s3://staging-area/r7/raw-data/provided/documents/vds-file-1",
"Name": "vds-file-1",
"FileSize": "439452464"
},
{
"FileSource": "s3://default_bucket/opendes/data/vds-dataset/vds-file-2",
"PreloadFilePath": "s3://staging-area/r7/raw-data/provided/documents/vds-file-2",
"Name": "vds-file-2",
"FileSize": "433645463"
}
],
"items": {
"$ref": "osdu:wks:AbstractFileSourceInfo:1.0.0"
}
},
"Checksum": {
"type": "string",
"title": "MD5 Checksum",
"description": "MD5 checksum of file bytes - a 32 byte hexadecimal number.",
"example": "d41d8cd98f00b204e9800998ecf8427e",
"pattern": "^[0-9a-fA-F]{32}"
}
}
}
},
"required": [
"DatasetProperties"
],
"x-osdu-inheriting-from-kind": []
}
}
\ No newline at end of file
{
"schemaInfo": {
"schemaIdentity": {
"authority": "osdu",
"source": "wks",
"entityType": "AbstractFileSourceInfo",
"schemaVersionMajor": 1,
"schemaVersionMinor": 0,
"schemaVersionPatch": 0,
"id": "osdu:wks:AbstractFileSourceInfo:1.0.0"
},
"createdBy": "OSDU Data Definition Group",
"scope": "SHARED",
"status": "DEVELOPMENT"
},
"schema": {
"x-osdu-license": "Copyright 2021, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
"x-osdu-review-status": "Accepted",
"$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractFileSourceInfo.1.0.0.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"x-osdu-schema-source": "osdu:wks:AbstractFileSourceInfo:1.0.0",
"title": "AbstractFileSourceInfo",
"description": "A schema fragment to describe file source information.",
"type": "object",
"properties": {
"FileSource": {
"type": "string",
"title": "File Source",
"description": "The location of the file. It can be a relative path. The actual access is provided via the File Service. This property is required.",
"example": "s3://default_bucket/r1/data/provided/documents/1000.witsml"
},
"PreloadFilePath": {
"type": "string",
"title": "Preload File Path",
"description": "File system path to the data file as it existed before loading to the data platform.",
"example": "s3://staging-area/r7/raw-data/provided/documents/1000.witsml"
},
"PreloadFileCreateUser": {
"type": "string",
"title": "Preload File Create User",
"description": "Optional user name or reference, who created the file prior to up-loading to the platform.",
"example": "somebody@acme.org"
},
"PreloadFileCreateDate": {
"type": "string",
"title": "Preload File Create Date",
"description": "Optional create date and time of the file prior to up-loading to the platform.",
"format": "date-time",
"example": "2019-12-16T11:46:20.163Z",
"x-osdu-frame-of-reference": "DateTime"
},
"PreloadFileModifyUser": {
"type": "string",
"title": "Preload File Modify User",
"description": "Optional user name or reference, who last modified the file prior to up-loading to the platform.",
"example": "somebody.else@acme.org"
},
"PreloadFileModifyDate": {
"type": "string",
"title": "Preload File Modify Date",
"description": "Optional last modified date and time of the file prior to up-loading to the platform.",
"format": "date-time",
"example": "2019-12-20T17:20:05.356Z",
"x-osdu-frame-of-reference": "DateTime"
},
"Name": {