Commit a33c56a1 authored by Aman Verma's avatar Aman Verma
Browse files

Merge remote-tracking branch 'origin/master' into users/amaverma/UpdateBootStrappingScript

parents 48fc4298 32158265
Pipeline #24777 passed with stages
in 24 minutes and 54 seconds
......@@ -37,3 +37,12 @@ target/
# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
hs_err_pid*
*local.properties
# Python virtualenv
/venv/
dist/
# Environment configuration
*.env
.envrc
......@@ -16,14 +16,17 @@ variables:
AZURE_BUILD_SUBDIR: provider/schema-azure
AZURE_TEST_SUBDIR: testing/schema-test-core
AZURE_DEPLOYMENTS_SUBDIR: deployments/scripts/azure
INTEGRATION_TESTER: $AZURE_PRINCIPAL_ID
AZURE_TESTER_SERVICEPRINCIPAL_SECRET: $AZURE_PRINCIPAL_SECRET
AZURE_DEPLOYMENTS_SCRIPTS_SUBDIR: deployments/scripts
AZURE_CLIENT_ID: $AZURE_PRINCIPAL_ID
AZURE_CLIENT_SECRET: $AZURE_PRINCIPAL_SECRET
DATA_PARTITION: opendes
AZURE_AD_APP_RESOURCE_ID: $AZURE_APP_ID
AZURE_STORAGE_CONNECTION_STRING: DefaultEndpointsProtocol=https;AccountName=${AZURE_BASE}data;AccountKey=${AZURE_STORAGE_KEY};EndpointSuffix=core.windows.net
AWS_BUILD_SUBDIR: provider/schema-aws/build-aws
AWS_TEST_SUBDIR: testing/schema-test-core
AWS_DEPLOYMENTS_SUBDIR: deployments/scripts/aws
AWS_SERVICE: schema
AWS_ENVIRONMENT: dev
AWS_MAVEN_TEST_COMMAND_OVERRIDE: verify
......@@ -31,7 +34,7 @@ variables:
OSDU_GCP_SERVICE: schema
OSDU_GCP_VENDOR: gcp
OSDU_GCP_APPLICATION_NAME: os-schema
OSDU_GCP_ENV_VARS: AUTHORIZE_API=$OSDU_GCP_ENTITLEMENTS_URL,AUTHORIZE_API_KEY=$OSDU_GCP_AUTHORIZE_API_KEY,LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_WEB=DEBUG,ACCOUNT_ID_COMMON_PROJECT=$TENANT --vpc-connector=$OSDU_GCP_VPC_CONNECTOR
OSDU_GCP_ENV_VARS: AUTHORIZE_API=$OSDU_GCP_ENTITLEMENTS_URL,AUTHORIZE_API_KEY=$OSDU_GCP_AUTHORIZE_API_KEY,LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_WEB=DEBUG,SHARED_TENANT_NAME=$TENANT --vpc-connector=$OSDU_GCP_VPC_CONNECTOR
include:
- project: "osdu/platform/ci-cd-pipelines"
......@@ -62,4 +65,5 @@ include:
ref: 'master'
file: 'cloud-providers/osdu-gcp-cloudrun.yml'
- local: "/devops/azure/bootstrap.yaml"
- local: "/devops/azure/gitlab-bootstrap.yml"
- local: "/devops/aws/bootstrap.yaml"
......@@ -246,6 +246,7 @@ The following software have components provided under the terms of this license:
- Google Cloud Core HTTP (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-core-http)
- Google Cloud Core gRPC (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-core-grpc)
- Google Cloud Datastore (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-datastore)
- Google Cloud Key Management Service (KMS) API v1-rev22-1.23.0 (from )
- Google Cloud Logging (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-logging)
- Google Cloud Pub/Sub (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-pubsub)
- Google Cloud Storage (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-storage)
......@@ -435,11 +436,13 @@ The following software have components provided under the terms of this license:
- javatuples (from http://www.javatuples.org)
- javax.inject (from http://code.google.com/p/atinject/)
- jose4j (from https://bitbucket.org/b_c/jose4j/)
- kotlin-stdlib (from )
- lang-mustache (from https://github.com/elastic/elasticsearch)
- lettuce (from http://github.com/mp911de/lettuce/wiki)
- micrometer-core (from https://github.com/micrometer-metrics/micrometer)
- micrometer-registry-azure-monitor (from https://github.com/micrometer-metrics/micrometer)
- org.apiguardian:apiguardian-api (from https://github.com/apiguardian-team/apiguardian)
- org.jetbrains.kotlin:kotlin-stdlib-common (from https://kotlinlang.org/)
- org.opentest4j:opentest4j (from https://github.com/ota4j-team/opentest4j)
- org.xmlunit:xmlunit-core (from http://www.xmlunit.org/)
- parent-join (from https://github.com/elastic/elasticsearch)
......@@ -565,11 +568,17 @@ The following software have components provided under the terms of this license:
- tomcat-embed-core (from http://tomcat.apache.org/)
========================================================================
EPL-1.0
CPL-1.0
========================================================================
The following software have components provided under the terms of this license:
- JUnit (from http://junit.org)
========================================================================
EPL-1.0
========================================================================
The following software have components provided under the terms of this license:
- JUnit Jupiter (Aggregator) (from https://junit.org/junit5/)
- Logback Classic Module (from )
- Logback Core Module (from )
......@@ -821,6 +830,7 @@ The following software have components provided under the terms of this license:
- Byte Buddy (without dependencies) (from )
- JSON in Java (from https://github.com/douglascrockford/JSON-java)
- JUnit (from http://junit.org)
- JUnit Jupiter (Aggregator) (from https://junit.org/junit5/)
- JavaBeans Activation Framework API jar (from )
- JavaMail API (from )
......
# Open Subsurface Data Universe JSON Schemas
# R2 Open Subsurface Data Universe JSON Schemas
Used to maintain the OSDU resorce type schemas.
The schemas have been removed as their original definition did
not conform to the Storage service expectations. There was no
Core Schema service available in R2.
## The repository has the following structure
repo-root/
The first Storage service compliant schemas are introduced in
R3. See [../R3-json-schema](../R3-json-schema).
\ No newline at end of file
{
"$schema": "http://json-schema.org/draft-07/schema#",
"properties": {
"Files": {
"items": {
"additionalProperties": false,
"properties": {
"AssociativeID": {
"pattern": "f-[0-9]+",
"type": "string"
},
"Data": {
"additionalProperties": false,
"properties": {
"ExtensionProperties": {
"properties": {},
"type": "object"
},
"GroupTypeProperties": {
"properties": {
"PreLoadFilePath": {
"type": "string"
}
},
"required": [
"PreLoadFilePath"
],
"type": "object"
},
"IndividualTypeProperties": {
"properties": {},
"type": "object"
}
},
"required": [
"GroupTypeProperties"
],
"type": "object"
},
"ResourceSecurityClassification": {
"pattern": "srn:reference-data/ResourceSecurityClassification:RESTRICTED|CLASSIFIED|CONFIDENTIAL|MOST-CONFIDENTIAL:[0-9]*",
"type": "string"
},
"ResourceTypeID": {
"pattern": "srn:type:file/[A-Za-z0-9-]+:[0-9]*",
"type": "string"
}
},
"required": [
"ResourceTypeID",
"ResourceSecurityClassification",
"Data",
"AssociativeID"
],
"type": "object"
},
"type": "array"
},
"WorkProduct": {
"additionalProperties": false,
"properties": {
"ComponentsAssociativeIDs": {
"items": {
"pattern": "wpc-[0-9]+",
"type": "string"
},
"type": "array"
},
"Data": {
"additionalProperties": false,
"properties": {
"ExtensionProperties": {
"properties": {},
"type": "object"
},
"GroupTypeProperties": {
"properties": {},
"type": "object"
},
"IndividualTypeProperties": {
"properties": {
"Description": {
"type": "string"
},
"Name": {
"type": "string"
}
},
"required": [
"Name",
"Description"
],
"type": "object"
}
},
"required": [
"IndividualTypeProperties"
],
"type": "object"
},
"ResourceSecurityClassification": {
"pattern": "srn:reference-data/ResourceSecurityClassification:RESTRICTED|CLASSIFIED|CONFIDENTIAL|MOST-CONFIDENTIAL:[0-9]*",
"type": "string"
},
"ResourceTypeID": {
"pattern": "srn:type:work-product/[A-Za-z]+:[0-9]*",
"type": "string"
}
},
"required": [
"ResourceTypeID",
"ResourceSecurityClassification",
"ComponentsAssociativeIDs",
"Data"
],
"type": "object"
},
"WorkProductComponents": {
"items": {
"additionalProperties": false,
"properties": {
"AssociativeID": {
"pattern": "wpc-[0-9]+",
"type": "string"
},
"Data": {
"additionalProperties": false,
"properties": {
"ExtensionProperties": {
"properties": {},
"type": "object"
},
"GroupTypeProperties": {
"properties": {},
"type": "object"
},
"IndividualTypeProperties": {
"properties": {
"Description": {
"type": "string"
},
"Name": {
"type": "string"
}
},
"required": [
"Name",
"Description"
],
"type": "object"
}
},
"required": [
"IndividualTypeProperties"
],
"type": "object"
},
"FileAssociativeIDs": {
"items": {
"pattern": "f-[0-9]+",
"type": "string"
},
"type": "array"
},
"ResourceSecurityClassification": {
"pattern": "srn:reference-data/ResourceSecurityClassification:RESTRICTED|CLASSIFIED|CONFIDENTIAL|MOST-CONFIDENTIAL:[0-9]*",
"type": "string"
},
"ResourceTypeID": {
"pattern": "srn:type:work-product-component/[A-Za-z]+:[0-9]*",
"type": "string"
}
},
"required": [
"ResourceTypeID",
"ResourceSecurityClassification",
"AssociativeID",
"FileAssociativeIDs",
"Data"
],
"type": "object"
},
"type": "array"
}
},
"required": [
"WorkProduct",
"WorkProductComponents",
"Files"
],
"title": "WorkProductLoadManifestStagedFiles",
"type": "object"
}
\ No newline at end of file
{
"$id":"https://schema.osdu.opengroup.org/json/data-collection/DataCollection/1",
"$schema":"http://json-schema.org/draft-07/schema#",
"title":"DataCollection",
"type":"object",
"properties":{
"ResourceTypeID":{
"description":"The SRN of the resource's resource type.",
"type":"string",
"pattern":"srn:type:data-collection/DataCollection:[0-9]+$"
},
"ResourceID":{
"description":"The SRN which identifies this OSDU resource object at the version level.",
"type":"string",
"pattern":"srn:data-collection/DataCollection:[^/:]+:[0-9]+$"
},
"ResourceHomeRegionID": {
"description": "The name of the home [cloud environment] region for this OSDU resource object.",
"type": "string",
"pattern": "srn::reference-data/OSDURegion:[A-Za-z0-9-]+:[0-9]*$"
},
"ResourceHostRegionIDs": {
"description": "The name of the host [cloud environment] region(s) for this OSDU resource object.",
"type": "array",
"items": {
"type": "string",
"pattern": "srn::reference-data/OSDURegion:[A-Za-z0-9-]+:[0-9]*$"
}
},
"ResourceObjectCreationDateTime": {
"description": "Timestamp of the time at which Version 1 of this OSDU resource object was originated.",
"type": "string",
"format": "date-time"
},
"ResourceVersionCreationDateTime": {
"description": "Timestamp of the time when the current version of this resource entered the OSDU.",
"type": "string",
"format": "date-time"
},
"ResourceCurationStatus": {
"description": "Describes the current Curation status. Possible values - CREATED, CURATING, CURATED.",
"type": "string",
"pattern": "srn::reference-data/ResourceCurationStatus:[^/:]+:[0-9]*$"
},
"ResourceLifecycleStatus": {
"description": "Describes the current Resource Lifecycle status. Possible values - LOADING, RECIEVED, ACCEPTED, RESCINDED, DELETED",
"type": "string",
"pattern": "srn::reference-data/ResourceLifecycleStatus:[^/:]+:[0-9]*$"
},
"ResourceSecurityClassification": {
"description": "Classifies the security level of the resourcse. Possible values = RESTRICTED, CLASSIFIED, CONFIDENTIAL, MOST-CONFIDENTIAL ",
"type": "string",
"pattern": "srn::reference-data/ResourceSecurityClassification:[^/:]+:[0-9]*$"
},
"Data": {
"type": "object",
"properties": {
"GroupTypeProperties": {
"type": "object",
"properties": {
"Resources": {
"description":"List of Resources",
"type":"array",
"items": {
"type": "string",
"pattern": "srn::(?:work-product(?:-component)?|data-collection)/[A-Za-z]+:[^/:]+:[0-9]*$"
}
}
},
"required": [
"Resources"
],
"additionalProperties": false
},
"IndividualTypeProperties": {
"type": "object",
"properties": {
"Name": {
"type": "string",
"description": "Name"
},
"Description": {
"type": "string",
"description": "Description"
},
"CreationDateTime": {
"type": "string",
"format": "date-time",
"description": "Creation DateTime"
},
"Tags": {
"type": "array",
"description": "Array of Tag Names",
"items": {
"type": "string"
}
},
"SubmitterName": {
"type": "string",
"description": "Submitter Name"
},
"AuthorIDs": {
"type": "array",
"description": "Array of Author IDs",
"items": {
"type": "string"
}
},
"OwnerID": {
"description":"ID of the User who owns the Collection",
"type":"string"
},
"WorkSpaceID": {
"description":"Collection Workspace",
"type":"string",
"pattern": "srn::workspace/[A-Za-z]+:[^/:]+:[0-9]*$"
},
"FilterSpecification": {
"description":"Collection Filter Specification",
"type":"object",
"properties":{}
}
},
"required": [
"Name",
"OwnerID"
],
"additionalProperties": false
},
"ExtensionProperties":{
"type":"object",
"properties":{
}
}
},
"required": [
"GroupTypeProperties",
"IndividualTypeProperties"
]
}
},
"required":[
"ResourceTypeID",
"ResourceID",
"ResourceHomeRegionID",
"ResourceHostRegionIDs",
"ResourceObjectCreationDateTime",
"ResourceVersionCreationDateTime",
"ResourceCurationStatus",
"ResourceLifecycleStatus",
"ResourceSecurityClassification",
"Data"
],
"additionalProperties":false
}
{
"$id": "https://schema.osdu.opengroup.org/json/swps/common/File/1",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "File",
"type": "object",
"properties": {
"ResourceTypeID": {
"description": "The SRN of the resource's resource type.",
"type": "string",
"pattern": "srn:type:file/[A-Za-z0-9-]+:[0-9]+$"
},
"ResourceID": {
"description": "The SRN which identifies this OSDU resource object at the version level.",
"type": "string",
"pattern": "srn:file/[A-Za-z0-9-]+:[^/:]+:[0-9]+$"
},
"ResourceHomeRegionID": {
"description": "The name of the home [cloud environment] region for this OSDU resource object.",
"type": "string",
"pattern": "srn:reference-data/OSDURegion:[A-Za-z0-9-]+:[0-9]*$"
},
"ResourceHostRegionIDs": {
"description": "The name of the host [cloud environment] region(s) for this OSDU resource object.",
"type": "array",
"items": {
"type": "string",
"pattern": "srn:reference-data/OSDURegion:[A-Za-z0-9-]+:[0-9]*$"
}
},
"ResourceObjectCreationDateTime": {
"description": "Timestamp of the time at which Version 1 of this OSDU resource object was originated.",
"type": "string",
"format": "date-time"
},
"ResourceVersionCreationDateTime": {
"description": "Timestamp of the time when the current version of this resource entered the OSDU.",
"type": "string",
"format": "date-time"
},
"ResourceCurationStatus": {
"description": "Describes the current Curation status. Possible values - CREATED, CURATING, CURATED.",
"type": "string",
"pattern": "srn:reference-data/ResourceCurationStatus:[^/:]+:[0-9]*$"
},
"ResourceLifecycleStatus": {
"description": "Describes the current Resource Lifecycle status. Possible values - LOADING, RECIEVED, ACCEPTED, RESCINDED, DELETED,",
"type": "string",
"pattern": "srn:reference-data/ResourceLifecycleStatus:[^/:]+:[0-9]*$"
},
"ResourceSecurityClassification": {
"description": "Classifies the security level of the resourcse. Possible values = RESTRICTED, CLASSIFIED, CONFIDENTIAL, MOST-CONFIDENTIAL ",
"type": "string",
"pattern": "srn:reference-data/ResourceSecurityClassification:[^/:]+:[0-9]*$"
},
"Data": {
"type": "object",
"properties": {
"GroupTypeProperties": {
"type": "object",
"properties": {
"SchemaFormatTypeID": {
"type": "string",
"description": "Schema Format Type ID",
"pattern": "srn:reference-data/SchemaFormatType:[^/:]+:[0-9]*$"
},
"PreLoadFilePath": {
"description": "File system path to the data file as it existed before loading to the data platform",
"type": "string"
},
"FileSource": {
"description": "URL or file path for the data in the file",
"type": "string"
},
"FileSize": {
"description": "Length of file in bytes",
"type": "integer"
},
"EncodingFormatTypeID": {
"type": "string",
"description": "Encoding Format Type ID",
"pattern": "srn:reference-data/EncodingFormatType:[^/:]+:[0-9]*$"
},
"Checksum": {
"description": "MD5 checksum of file bytes - a 32 byte hexadecimal number",
"type": "string",
"pattern": "[0-9a-fA-F]{32}"
}
}
},
"IndividualTypeProperties": {
"type": "object",
"properties": {
}
},
"ExtensionProperties": {
"type": "object",
"properties": {
}
}
}
}
},
"required":[
"ResourceID",
"ResourceTypeID",
"ResourceHomeRegionID",
"ResourceHostRegionIDs",
"ResourceObjectCreationDateTime",