Merge branch 'opendes-ado-sync' into 'master'

OpenDES ADO code merge

See merge request !3

.gitlab-ci.yml

@@ -2,14 +2,22 @@ variables:
   GCP_BUILD_SUBDIR: provider/schema-gcp
   GCP_INT_TEST_SUBDIR: testing/schema-test-gcp
   GCP_APPLICATION_NAME: os-schema
-  GCP_ENVIRONMENT: dev
-  GCP_PROJECT: opendes
-  GCP_TENANT_NAME: opendes
+  GCP_ENVIRONMENT: testing
+  GCP_PROJECT: opendes-evt
+  GCP_TENANT_NAME: opendesevt
   GCP_DEPLOY_ENV: p4d
   GCP_DOMAIN: cloud.slb-ds.com
-  GCP_STORAGE_URL: https://os-schema-dot-opendes.appspot.com/api/storage/v2/
 
 include:
-  - project: 'osdu/platform/ci-cd-pipelines'
-    ref: 'master'
-    file: 'service.gitlab-ci.yml'
+  - project: "osdu/platform/ci-cd-pipelines"
+    file: "standard-setup.yml"
+
+  - project: "osdu/platform/ci-cd-pipelines"
+    file: "build/maven.yml"
+
+  - project: "osdu/platform/ci-cd-pipelines"
+    file: "scanners/fossa.yml"
+
+  - project: "osdu/platform/ci-cd-pipelines"
+    file: "scanners/gitlab-ultimate.yml"
+    

azure-pipelines.yml

@@ -72,7 +72,7 @@ stages:
               popd
               sleep 10
               OUTPUT="Alive"
-              ENDPOINT=https://open.opendes.cloud.slb-ds.com/api/schema-service/v1/health
+              ENDPOINT=$(SCHEMA_DEV_URL)/health
               echo $ENDPOINT
 
               while [ -z "$STATUS" ]; do
@@ -101,7 +101,35 @@ stages:
         mavenAuthenticateFeed: false
         effectivePomSkip: false
         sonarQubeRunAnalysis: false
-      
+      env:
+        INTEGRATION_TEST_AUDIENCE: $(INTEGRATION_TEST_AUDIENCE)
+        INTEGRATION_TESTER : $(INTEGRATION_TESTER)
+
+    - task: UsePythonVersion@0
+      inputs:
+        versionSpec: '3.x'
+        addToPath: true
+        architecture: 'x64'
+    - task: Bash@3
+      displayName: 'Deploying shared schemas'
+      inputs:
+        targetType: 'inline'
+        script: |
+          #!/bin/bash
+
+          pip install -r deployments/scripts/google/requirements.txt
+
+          export JSON_KEY=$(INTEGRATION_TESTER)
+          export AUDIENCE=$(INTEGRATION_TEST_AUDIENCE)
+
+          BEARER_TOKEN=`python deployments/scripts/google/Token.py`
+
+          export BEARER_TOKEN=$BEARER_TOKEN
+          export APP_KEY=""
+          export DATA_PARTITION=$(DATA_PARTITION)
+
+          python deployments/scripts/DeploySharedSchemas.py -u $(SCHEMA_DEV_URL)/schema
+
     - task: PublishBuildArtifacts@1
       displayName: 'Publish Artifact: drop'
       inputs:

deployments/osdu-source/R2-json-schema/README.md

+# Open Subsurface Data Universe JSON Schemas
+
+Used to maintain the OSDU resorce type schemas.
+
+## The repository has the following structure
+
+repo-root/

deployments/osdu-source/R2-json-schema/WorkProductLoadManifestStagedFiles.json

+{
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "properties": {
+        "Files": {
+            "items": {
+                "additionalProperties": false,
+                "properties": {
+                    "AssociativeID": {
+                        "pattern": "f-[0-9]+",
+                        "type": "string"
+                    },
+                    "Data": {
+                        "additionalProperties": false,
+                        "properties": {
+                            "ExtensionProperties": {
+                                "properties": {},
+                                "type": "object"
+                            },
+                            "GroupTypeProperties": {
+                                "properties": {
+                                    "PreLoadFilePath": {
+                                        "type": "string"
+                                    }
+                                },
+                                "required": [
+                                    "PreLoadFilePath"
+                                ],
+                                "type": "object"
+                            },
+                            "IndividualTypeProperties": {
+                                "properties": {},
+                                "type": "object"
+                            }
+                        },
+                        "required": [
+                            "GroupTypeProperties"
+                        ],
+                        "type": "object"
+                    },
+                    "ResourceSecurityClassification": {
+                        "pattern": "srn:reference-data/ResourceSecurityClassification:RESTRICTED|CLASSIFIED|CONFIDENTIAL|MOST-CONFIDENTIAL:[0-9]*",
+                        "type": "string"
+                    },
+                    "ResourceTypeID": {
+                        "pattern": "srn:type:file/[A-Za-z0-9-]+:[0-9]*",
+                        "type": "string"
+                    }
+                },
+                "required": [
+                    "ResourceTypeID",
+                    "ResourceSecurityClassification",
+                    "Data",
+                    "AssociativeID"
+                ],
+                "type": "object"
+            },
+            "type": "array"
+        },
+        "WorkProduct": {
+            "additionalProperties": false,
+            "properties": {
+                "ComponentsAssociativeIDs": {
+                    "items": {
+                        "pattern": "wpc-[0-9]+",
+                        "type": "string"
+                    },
+                    "type": "array"
+                },
+                "Data": {
+                    "additionalProperties": false,
+                    "properties": {
+                        "ExtensionProperties": {
+                            "properties": {},
+                            "type": "object"
+                        },
+                        "GroupTypeProperties": {
+                            "properties": {},
+                            "type": "object"
+                        },
+                        "IndividualTypeProperties": {
+                            "properties": {
+                                "Description": {
+                                    "type": "string"
+                                },
+                                "Name": {
+                                    "type": "string"
+                                }
+                            },
+                            "required": [
+                                "Name",
+                                "Description"
+                            ],
+                            "type": "object"
+                        }
+                    },
+                    "required": [
+                        "IndividualTypeProperties"
+                    ],
+                    "type": "object"
+                },
+                "ResourceSecurityClassification": {
+                    "pattern": "srn:reference-data/ResourceSecurityClassification:RESTRICTED|CLASSIFIED|CONFIDENTIAL|MOST-CONFIDENTIAL:[0-9]*",
+                    "type": "string"
+                },
+                "ResourceTypeID": {
+                    "pattern": "srn:type:work-product/[A-Za-z]+:[0-9]*",
+                    "type": "string"
+                }
+            },
+            "required": [
+                "ResourceTypeID",
+                "ResourceSecurityClassification",
+                "ComponentsAssociativeIDs",
+                "Data"
+            ],
+            "type": "object"
+        },
+        "WorkProductComponents": {
+            "items": {
+                "additionalProperties": false,
+                "properties": {
+                    "AssociativeID": {
+                        "pattern": "wpc-[0-9]+",
+                        "type": "string"
+                    },
+                    "Data": {
+                        "additionalProperties": false,
+                        "properties": {
+                            "ExtensionProperties": {
+                                "properties": {},
+                                "type": "object"
+                            },
+                            "GroupTypeProperties": {
+                                "properties": {},
+                                "type": "object"
+                            },
+                            "IndividualTypeProperties": {
+                                "properties": {
+                                    "Description": {
+                                        "type": "string"
+                                    },
+                                    "Name": {
+                                        "type": "string"
+                                    }
+                                },
+                                "required": [
+                                    "Name",
+                                    "Description"
+                                ],
+                                "type": "object"
+                            }
+                        },
+                        "required": [
+                            "IndividualTypeProperties"
+                        ],
+                        "type": "object"
+                    },
+                    "FileAssociativeIDs": {
+                        "items": {
+                            "pattern": "f-[0-9]+",
+                            "type": "string"
+                        },
+                        "type": "array"
+                    },
+                    "ResourceSecurityClassification": {
+                        "pattern": "srn:reference-data/ResourceSecurityClassification:RESTRICTED|CLASSIFIED|CONFIDENTIAL|MOST-CONFIDENTIAL:[0-9]*",
+                        "type": "string"
+                    },
+                    "ResourceTypeID": {
+                        "pattern": "srn:type:work-product-component/[A-Za-z]+:[0-9]*",
+                        "type": "string"
+                    }
+                },
+                "required": [
+                    "ResourceTypeID",
+                    "ResourceSecurityClassification",
+                    "AssociativeID",
+                    "FileAssociativeIDs",
+                    "Data"
+                ],
+                "type": "object"
+            },
+            "type": "array"
+        }
+    },
+    "required": [
+        "WorkProduct",
+        "WorkProductComponents",
+        "Files"
+    ],
+    "title": "WorkProductLoadManifestStagedFiles",
+    "type": "object"
+}
\ No newline at end of file

deployments/osdu-source/R2-json-schema/data-collection/DataCollection.json

+{
+    "$id":"https://schema.osdu.opengroup.org/json/data-collection/DataCollection/1",
+    "$schema":"http://json-schema.org/draft-07/schema#",
+    "title":"DataCollection",
+    "type":"object",
+    "properties":{
+        "ResourceTypeID":{
+            "description":"The SRN of the resource's resource type.",
+            "type":"string",
+            "pattern":"srn:type:data-collection/DataCollection:[0-9]+$"
+        },
+        "ResourceID":{
+            "description":"The SRN which identifies this OSDU resource object at the version level.",
+            "type":"string",
+            "pattern":"srn:data-collection/DataCollection:[^/:]+:[0-9]+$"
+        },
+        "ResourceHomeRegionID": {
+            "description": "The name of the home [cloud environment] region for this OSDU resource object.",
+            "type": "string",
+            "pattern": "srn::reference-data/OSDURegion:[A-Za-z0-9-]+:[0-9]*$"
+        },
+        "ResourceHostRegionIDs": {
+            "description": "The name of the host [cloud environment] region(s) for this OSDU resource object.",
+            "type": "array",
+            "items": {
+              "type": "string",
+              "pattern": "srn::reference-data/OSDURegion:[A-Za-z0-9-]+:[0-9]*$"
+            }
+        },
+        "ResourceObjectCreationDateTime": {
+            "description": "Timestamp of the time at which Version 1 of this OSDU resource object was originated.",
+            "type": "string",
+            "format": "date-time"
+        },
+        "ResourceVersionCreationDateTime": {
+            "description": "Timestamp of the time when the current version of this resource entered the OSDU.",
+            "type": "string",
+            "format": "date-time"
+        },
+        "ResourceCurationStatus": {
+            "description": "Describes the current Curation status. Possible values - CREATED, CURATING, CURATED.",
+            "type": "string",
+            "pattern": "srn::reference-data/ResourceCurationStatus:[^/:]+:[0-9]*$"
+        },
+        "ResourceLifecycleStatus": {
+            "description": "Describes the current Resource Lifecycle status. Possible values - LOADING, RECIEVED, ACCEPTED, RESCINDED, DELETED",
+            "type": "string",
+            "pattern": "srn::reference-data/ResourceLifecycleStatus:[^/:]+:[0-9]*$"
+        },
+        "ResourceSecurityClassification": {
+            "description": "Classifies the security level of the resourcse. Possible values = RESTRICTED, CLASSIFIED, CONFIDENTIAL, MOST-CONFIDENTIAL  ",
+            "type": "string",
+            "pattern": "srn::reference-data/ResourceSecurityClassification:[^/:]+:[0-9]*$"
+        },
+        "Data": {
+            "type": "object",
+            "properties": {
+                "GroupTypeProperties": {
+                    "type": "object",
+                    "properties": {
+                        "Resources": {
+                            "description":"List of Resources",
+                            "type":"array",
+                            "items": {
+                                "type": "string",
+                                "pattern": "srn::(?:work-product(?:-component)?|data-collection)/[A-Za-z]+:[^/:]+:[0-9]*$"
+                            }
+                        }
+                    },
+                    "required": [
+                        "Resources"
+                    ],
+                    "additionalProperties": false
+                },
+                "IndividualTypeProperties": {
+                    "type": "object",
+                    "properties": {
+                        "Name": {
+                            "type": "string",
+                            "description": "Name"
+                        },
+                        "Description": {
+                            "type": "string",
+                            "description": "Description"
+                        },
+                        "CreationDateTime": {
+                            "type": "string",
+                            "format": "date-time",
+                            "description": "Creation DateTime"
+                        },
+                        "Tags": {
+                            "type": "array",
+                            "description": "Array of Tag Names",
+                            "items": {
+                                "type": "string"
+                            }
+                        },
+                        "SubmitterName": {
+                            "type": "string",
+                            "description": "Submitter Name"
+                        },
+                        "AuthorIDs": {
+                            "type": "array",
+                            "description": "Array of Author IDs",
+                            "items": {
+                                "type": "string"
+                            }
+                        },
+                        "OwnerID": {
+                            "description":"ID of the User who owns the Collection",
+                            "type":"string"
+                        },
+                        "WorkSpaceID": {
+                            "description":"Collection Workspace",
+                            "type":"string",
+                            "pattern": "srn::workspace/[A-Za-z]+:[^/:]+:[0-9]*$"
+                        },
+                        "FilterSpecification": {
+                            "description":"Collection Filter Specification",
+                            "type":"object",
+                            "properties":{}
+                        }
+                    },
+                    "required": [
+                        "Name",
+                        "OwnerID"
+                    ],
+                    "additionalProperties": false
+                },
+                "ExtensionProperties":{
+                    "type":"object",
+                    "properties":{
+
+                    }
+                }
+            },
+            "required": [
+                "GroupTypeProperties",
+                "IndividualTypeProperties"
+            ]
+        }
+    },
+    "required":[
+        "ResourceTypeID",
+        "ResourceID",
+        "ResourceHomeRegionID",
+        "ResourceHostRegionIDs",
+        "ResourceObjectCreationDateTime",
+        "ResourceVersionCreationDateTime",
+        "ResourceCurationStatus",
+        "ResourceLifecycleStatus",
+        "ResourceSecurityClassification",
+        "Data"
+    ],
+    "additionalProperties":false
+}

deployments/osdu-source/R2-json-schema/file/File.json

+{
+    "$id": "https://schema.osdu.opengroup.org/json/swps/common/File/1",
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "File",
+    "type": "object",
+    "properties": {
+        "ResourceTypeID": {
+          "description": "The SRN of the resource's resource type.",
+          "type": "string",
+          "pattern": "srn:type:file/[A-Za-z0-9-]+:[0-9]+$"
+        },
+        "ResourceID": {
+          "description": "The SRN which identifies this OSDU resource object at the version level.",
+          "type": "string",
+          "pattern": "srn:file/[A-Za-z0-9-]+:[^/:]+:[0-9]+$"
+        },
+        "ResourceHomeRegionID": {
+          "description": "The name of the home [cloud environment] region for this OSDU resource object.",
+          "type": "string",
+          "pattern": "srn:reference-data/OSDURegion:[A-Za-z0-9-]+:[0-9]*$"
+        },
+        "ResourceHostRegionIDs": {
+          "description": "The name of the host [cloud environment] region(s) for this OSDU resource object.",
+          "type": "array",
+          "items": {
+            "type": "string",
+            "pattern": "srn:reference-data/OSDURegion:[A-Za-z0-9-]+:[0-9]*$"
+          }
+        },
+        "ResourceObjectCreationDateTime": {
+          "description": "Timestamp of the time at which Version 1 of this OSDU resource object was originated.",
+          "type": "string",
+          "format": "date-time"
+        },
+        "ResourceVersionCreationDateTime": {
+          "description": "Timestamp of the time when the current version of this resource entered the OSDU.",
+          "type": "string",
+          "format": "date-time"
+        },
+        "ResourceCurationStatus": {
+          "description": "Describes the current Curation status. Possible values - CREATED, CURATING, CURATED.",
+          "type": "string",
+          "pattern": "srn:reference-data/ResourceCurationStatus:[^/:]+:[0-9]*$"
+        },
+        "ResourceLifecycleStatus": {
+          "description": "Describes the current Resource Lifecycle status. Possible values - LOADING, RECIEVED, ACCEPTED, RESCINDED, DELETED,",
+          "type": "string",
+          "pattern": "srn:reference-data/ResourceLifecycleStatus:[^/:]+:[0-9]*$"
+        },
+        "ResourceSecurityClassification": {
+          "description": "Classifies the security level of the resourcse. Possible values = RESTRICTED, CLASSIFIED, CONFIDENTIAL, MOST-CONFIDENTIAL  ",
+          "type": "string",
+          "pattern": "srn:reference-data/ResourceSecurityClassification:[^/:]+:[0-9]*$"
+        },
+        "Data": {
+            "type": "object",
+            "properties": {
+                "GroupTypeProperties": {
+                    "type": "object",
+                    "properties": {
+                        "SchemaFormatTypeID": {
+                            "type": "string",
+                            "description": "Schema Format Type ID",
+                            "pattern": "srn:reference-data/SchemaFormatType:[^/:]+:[0-9]*$"
+                        },
+  					            "PreLoadFilePath": {
+                            "description": "File system path to the data file as it existed before loading to the data platform",
+                            "type": "string"
+                        },
+                        "FileSource": {
+                            "description": "URL or file path for the data in the file",
+                            "type": "string"
+                        },
+                        "FileSize": {
+                            "description": "Length of file in bytes",
+                            "type": "integer"
+                        },
+  					            "EncodingFormatTypeID": {
+                            "type": "string",
+                            "description": "Encoding Format Type ID",
+                            "pattern": "srn:reference-data/EncodingFormatType:[^/:]+:[0-9]*$"
+                        },
+                        "Checksum": {
+                            "description": "MD5 checksum of file bytes - a 32 byte hexadecimal number",
+                            "type": "string",
+                            "pattern": "[0-9a-fA-F]{32}"
+                        }
+                    }
+                },
+                "IndividualTypeProperties": {
+                    "type": "object",
+                    "properties": {
+                    }
+                },
+                "ExtensionProperties": {
+                    "type": "object",
+                    "properties": {
+                    }
+                }
+            }
+        }
+    },
+    "required":[
+        "ResourceID",
+        "ResourceTypeID",
+        "ResourceHomeRegionID",
+        "ResourceHostRegionIDs",
+        "ResourceObjectCreationDateTime",
+        "ResourceVersionCreationDateTime",
+        "ResourceCurationStatus",
+        "ResourceLifecycleStatus",
+        "ResourceSecurityClassification"
+    ],
+    "additionalProperties": false
+}

deployments/osdu-source/R2-json-schema/master-data/Agreement.json

+{
+  "$id": "https://schema.osdu.opengroup.org/json/smds/master-data/Agreement/1",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "Agreement",
+  "description": "A contract or other covenant between Company and counterparties which is relevant to the data universe because it includes terms governing use of data.",
+  "type": "object",
+  "properties": {
+    "ResourceTypeID": {
+      "description": "The SRN of the resource's resource type.",