Commit 078a4f75 authored by ethiraj krishnamanaidu's avatar ethiraj krishnamanaidu
Browse files

OpenDES ADO code merge

parent 4d414d4e
......@@ -2,14 +2,22 @@ variables:
GCP_BUILD_SUBDIR: provider/schema-gcp
GCP_INT_TEST_SUBDIR: testing/schema-test-gcp
GCP_APPLICATION_NAME: os-schema
GCP_ENVIRONMENT: dev
GCP_PROJECT: opendes
GCP_TENANT_NAME: opendes
GCP_ENVIRONMENT: testing
GCP_PROJECT: opendes-evt
GCP_TENANT_NAME: opendesevt
GCP_DEPLOY_ENV: p4d
GCP_DOMAIN: cloud.slb-ds.com
GCP_STORAGE_URL: https://os-schema-dot-opendes.appspot.com/api/storage/v2/
include:
- project: 'osdu/platform/ci-cd-pipelines'
ref: 'master'
file: 'service.gitlab-ci.yml'
- project: "osdu/platform/ci-cd-pipelines"
file: "standard-setup.yml"
- project: "osdu/platform/ci-cd-pipelines"
file: "build/maven.yml"
- project: "osdu/platform/ci-cd-pipelines"
file: "scanners/fossa.yml"
- project: "osdu/platform/ci-cd-pipelines"
file: "scanners/gitlab-ultimate.yml"
......@@ -72,7 +72,7 @@ stages:
popd
sleep 10
OUTPUT="Alive"
ENDPOINT=https://open.opendes.cloud.slb-ds.com/api/schema-service/v1/health
ENDPOINT=$(SCHEMA_DEV_URL)/health
echo $ENDPOINT
while [ -z "$STATUS" ]; do
......@@ -101,6 +101,34 @@ stages:
mavenAuthenticateFeed: false
effectivePomSkip: false
sonarQubeRunAnalysis: false
env:
INTEGRATION_TEST_AUDIENCE: $(INTEGRATION_TEST_AUDIENCE)
INTEGRATION_TESTER : $(INTEGRATION_TESTER)
- task: UsePythonVersion@0
inputs:
versionSpec: '3.x'
addToPath: true
architecture: 'x64'
- task: Bash@3
displayName: 'Deploying shared schemas'
inputs:
targetType: 'inline'
script: |
#!/bin/bash
pip install -r deployments/scripts/google/requirements.txt
export JSON_KEY=$(INTEGRATION_TESTER)
export AUDIENCE=$(INTEGRATION_TEST_AUDIENCE)
BEARER_TOKEN=`python deployments/scripts/google/Token.py`
export BEARER_TOKEN=$BEARER_TOKEN
export APP_KEY=""
export DATA_PARTITION=$(DATA_PARTITION)
python deployments/scripts/DeploySharedSchemas.py -u $(SCHEMA_DEV_URL)/schema
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: drop'
......
# Open Subsurface Data Universe JSON Schemas
Used to maintain the OSDU resorce type schemas.
## The repository has the following structure
repo-root/
{
"$schema": "http://json-schema.org/draft-07/schema#",
"properties": {
"Files": {
"items": {
"additionalProperties": false,
"properties": {
"AssociativeID": {
"pattern": "f-[0-9]+",
"type": "string"
},
"Data": {
"additionalProperties": false,
"properties": {
"ExtensionProperties": {
"properties": {},
"type": "object"
},
"GroupTypeProperties": {
"properties": {
"PreLoadFilePath": {
"type": "string"
}
},
"required": [
"PreLoadFilePath"
],
"type": "object"
},
"IndividualTypeProperties": {
"properties": {},
"type": "object"
}
},
"required": [
"GroupTypeProperties"
],
"type": "object"
},
"ResourceSecurityClassification": {
"pattern": "srn:reference-data/ResourceSecurityClassification:RESTRICTED|CLASSIFIED|CONFIDENTIAL|MOST-CONFIDENTIAL:[0-9]*",
"type": "string"
},
"ResourceTypeID": {
"pattern": "srn:type:file/[A-Za-z0-9-]+:[0-9]*",
"type": "string"
}
},
"required": [
"ResourceTypeID",
"ResourceSecurityClassification",
"Data",
"AssociativeID"
],
"type": "object"
},
"type": "array"
},
"WorkProduct": {
"additionalProperties": false,
"properties": {
"ComponentsAssociativeIDs": {
"items": {
"pattern": "wpc-[0-9]+",
"type": "string"
},
"type": "array"
},
"Data": {
"additionalProperties": false,
"properties": {
"ExtensionProperties": {
"properties": {},
"type": "object"
},
"GroupTypeProperties": {
"properties": {},
"type": "object"
},
"IndividualTypeProperties": {
"properties": {
"Description": {
"type": "string"
},
"Name": {
"type": "string"
}
},
"required": [
"Name",
"Description"
],
"type": "object"
}
},
"required": [
"IndividualTypeProperties"
],
"type": "object"
},
"ResourceSecurityClassification": {
"pattern": "srn:reference-data/ResourceSecurityClassification:RESTRICTED|CLASSIFIED|CONFIDENTIAL|MOST-CONFIDENTIAL:[0-9]*",
"type": "string"
},
"ResourceTypeID": {
"pattern": "srn:type:work-product/[A-Za-z]+:[0-9]*",
"type": "string"
}
},
"required": [
"ResourceTypeID",
"ResourceSecurityClassification",
"ComponentsAssociativeIDs",
"Data"
],
"type": "object"
},
"WorkProductComponents": {
"items": {
"additionalProperties": false,
"properties": {
"AssociativeID": {
"pattern": "wpc-[0-9]+",
"type": "string"
},
"Data": {
"additionalProperties": false,
"properties": {
"ExtensionProperties": {
"properties": {},
"type": "object"
},
"GroupTypeProperties": {
"properties": {},
"type": "object"
},
"IndividualTypeProperties": {
"properties": {
"Description": {
"type": "string"
},
"Name": {
"type": "string"
}
},
"required": [
"Name",
"Description"
],
"type": "object"
}
},
"required": [
"IndividualTypeProperties"
],
"type": "object"
},
"FileAssociativeIDs": {
"items": {
"pattern": "f-[0-9]+",
"type": "string"
},
"type": "array"
},
"ResourceSecurityClassification": {
"pattern": "srn:reference-data/ResourceSecurityClassification:RESTRICTED|CLASSIFIED|CONFIDENTIAL|MOST-CONFIDENTIAL:[0-9]*",
"type": "string"
},
"ResourceTypeID": {
"pattern": "srn:type:work-product-component/[A-Za-z]+:[0-9]*",
"type": "string"
}
},
"required": [
"ResourceTypeID",
"ResourceSecurityClassification",
"AssociativeID",
"FileAssociativeIDs",
"Data"
],
"type": "object"
},
"type": "array"
}
},
"required": [
"WorkProduct",
"WorkProductComponents",
"Files"
],
"title": "WorkProductLoadManifestStagedFiles",
"type": "object"
}
\ No newline at end of file
{
"$id":"https://schema.osdu.opengroup.org/json/data-collection/DataCollection/1",
"$schema":"http://json-schema.org/draft-07/schema#",
"title":"DataCollection",
"type":"object",
"properties":{
"ResourceTypeID":{
"description":"The SRN of the resource's resource type.",
"type":"string",
"pattern":"srn:type:data-collection/DataCollection:[0-9]+$"
},
"ResourceID":{
"description":"The SRN which identifies this OSDU resource object at the version level.",
"type":"string",
"pattern":"srn:data-collection/DataCollection:[^/:]+:[0-9]+$"
},
"ResourceHomeRegionID": {
"description": "The name of the home [cloud environment] region for this OSDU resource object.",
"type": "string",
"pattern": "srn::reference-data/OSDURegion:[A-Za-z0-9-]+:[0-9]*$"
},
"ResourceHostRegionIDs": {
"description": "The name of the host [cloud environment] region(s) for this OSDU resource object.",
"type": "array",
"items": {
"type": "string",
"pattern": "srn::reference-data/OSDURegion:[A-Za-z0-9-]+:[0-9]*$"
}
},
"ResourceObjectCreationDateTime": {
"description": "Timestamp of the time at which Version 1 of this OSDU resource object was originated.",
"type": "string",
"format": "date-time"
},
"ResourceVersionCreationDateTime": {
"description": "Timestamp of the time when the current version of this resource entered the OSDU.",
"type": "string",
"format": "date-time"
},
"ResourceCurationStatus": {
"description": "Describes the current Curation status. Possible values - CREATED, CURATING, CURATED.",
"type": "string",
"pattern": "srn::reference-data/ResourceCurationStatus:[^/:]+:[0-9]*$"
},
"ResourceLifecycleStatus": {
"description": "Describes the current Resource Lifecycle status. Possible values - LOADING, RECIEVED, ACCEPTED, RESCINDED, DELETED",
"type": "string",
"pattern": "srn::reference-data/ResourceLifecycleStatus:[^/:]+:[0-9]*$"
},
"ResourceSecurityClassification": {
"description": "Classifies the security level of the resourcse. Possible values = RESTRICTED, CLASSIFIED, CONFIDENTIAL, MOST-CONFIDENTIAL ",
"type": "string",
"pattern": "srn::reference-data/ResourceSecurityClassification:[^/:]+:[0-9]*$"
},
"Data": {
"type": "object",
"properties": {
"GroupTypeProperties": {
"type": "object",
"properties": {
"Resources": {
"description":"List of Resources",
"type":"array",
"items": {
"type": "string",
"pattern": "srn::(?:work-product(?:-component)?|data-collection)/[A-Za-z]+:[^/:]+:[0-9]*$"
}
}
},
"required": [
"Resources"
],
"additionalProperties": false
},
"IndividualTypeProperties": {
"type": "object",
"properties": {
"Name": {
"type": "string",
"description": "Name"
},
"Description": {
"type": "string",
"description": "Description"
},
"CreationDateTime": {
"type": "string",
"format": "date-time",
"description": "Creation DateTime"
},
"Tags": {
"type": "array",
"description": "Array of Tag Names",
"items": {
"type": "string"
}
},
"SubmitterName": {
"type": "string",
"description": "Submitter Name"
},
"AuthorIDs": {
"type": "array",
"description": "Array of Author IDs",
"items": {
"type": "string"
}
},
"OwnerID": {
"description":"ID of the User who owns the Collection",
"type":"string"
},
"WorkSpaceID": {
"description":"Collection Workspace",
"type":"string",
"pattern": "srn::workspace/[A-Za-z]+:[^/:]+:[0-9]*$"
},
"FilterSpecification": {
"description":"Collection Filter Specification",
"type":"object",
"properties":{}
}
},
"required": [
"Name",
"OwnerID"
],
"additionalProperties": false
},
"ExtensionProperties":{
"type":"object",
"properties":{
}
}
},
"required": [
"GroupTypeProperties",
"IndividualTypeProperties"
]
}
},
"required":[
"ResourceTypeID",
"ResourceID",
"ResourceHomeRegionID",
"ResourceHostRegionIDs",
"ResourceObjectCreationDateTime",
"ResourceVersionCreationDateTime",
"ResourceCurationStatus",
"ResourceLifecycleStatus",
"ResourceSecurityClassification",
"Data"
],
"additionalProperties":false
}
{
"$id": "https://schema.osdu.opengroup.org/json/swps/common/File/1",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "File",
"type": "object",
"properties": {
"ResourceTypeID": {
"description": "The SRN of the resource's resource type.",
"type": "string",
"pattern": "srn:type:file/[A-Za-z0-9-]+:[0-9]+$"
},
"ResourceID": {
"description": "The SRN which identifies this OSDU resource object at the version level.",
"type": "string",
"pattern": "srn:file/[A-Za-z0-9-]+:[^/:]+:[0-9]+$"
},
"ResourceHomeRegionID": {
"description": "The name of the home [cloud environment] region for this OSDU resource object.",
"type": "string",
"pattern": "srn:reference-data/OSDURegion:[A-Za-z0-9-]+:[0-9]*$"
},
"ResourceHostRegionIDs": {
"description": "The name of the host [cloud environment] region(s) for this OSDU resource object.",
"type": "array",
"items": {
"type": "string",
"pattern": "srn:reference-data/OSDURegion:[A-Za-z0-9-]+:[0-9]*$"
}
},
"ResourceObjectCreationDateTime": {
"description": "Timestamp of the time at which Version 1 of this OSDU resource object was originated.",
"type": "string",
"format": "date-time"
},
"ResourceVersionCreationDateTime": {
"description": "Timestamp of the time when the current version of this resource entered the OSDU.",
"type": "string",
"format": "date-time"
},
"ResourceCurationStatus": {
"description": "Describes the current Curation status. Possible values - CREATED, CURATING, CURATED.",
"type": "string",
"pattern": "srn:reference-data/ResourceCurationStatus:[^/:]+:[0-9]*$"
},
"ResourceLifecycleStatus": {
"description": "Describes the current Resource Lifecycle status. Possible values - LOADING, RECIEVED, ACCEPTED, RESCINDED, DELETED,",
"type": "string",
"pattern": "srn:reference-data/ResourceLifecycleStatus:[^/:]+:[0-9]*$"
},
"ResourceSecurityClassification": {
"description": "Classifies the security level of the resourcse. Possible values = RESTRICTED, CLASSIFIED, CONFIDENTIAL, MOST-CONFIDENTIAL ",
"type": "string",
"pattern": "srn:reference-data/ResourceSecurityClassification:[^/:]+:[0-9]*$"
},
"Data": {
"type": "object",
"properties": {
"GroupTypeProperties": {
"type": "object",
"properties": {
"SchemaFormatTypeID": {
"type": "string",
"description": "Schema Format Type ID",
"pattern": "srn:reference-data/SchemaFormatType:[^/:]+:[0-9]*$"
},
"PreLoadFilePath": {
"description": "File system path to the data file as it existed before loading to the data platform",
"type": "string"
},
"FileSource": {
"description": "URL or file path for the data in the file",
"type": "string"
},
"FileSize": {
"description": "Length of file in bytes",
"type": "integer"
},
"EncodingFormatTypeID": {
"type": "string",
"description": "Encoding Format Type ID",
"pattern": "srn:reference-data/EncodingFormatType:[^/:]+:[0-9]*$"
},
"Checksum": {
"description": "MD5 checksum of file bytes - a 32 byte hexadecimal number",
"type": "string",
"pattern": "[0-9a-fA-F]{32}"
}
}
},
"IndividualTypeProperties": {
"type": "object",
"properties": {
}
},
"ExtensionProperties": {
"type": "object",
"properties": {
}
}
}
}
},
"required":[
"ResourceID",
"ResourceTypeID",
"ResourceHomeRegionID",
"ResourceHostRegionIDs",
"ResourceObjectCreationDateTime",
"ResourceVersionCreationDateTime",
"ResourceCurationStatus",
"ResourceLifecycleStatus",
"ResourceSecurityClassification"
],
"additionalProperties": false
}
{
"$id": "https://schema.osdu.opengroup.org/json/smds/master-data/Agreement/1",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Agreement",
"description": "A contract or other covenant between Company and counterparties which is relevant to the data universe because it includes terms governing use of data.",
"type": "object",
"properties": {
"ResourceTypeID": {
"description": "The SRN of the resource's resource type.",
"type": "string",
"pattern": "srn:type:master-data/Agreement:[0-9]+"
},
"ResourceID": {
"description": "The SRN which identifies this OSDU resource object at the version level.",