merging from master

deployments/scripts/GcpDatastoreCleanUp.py

+import json
+import os
+from google.api_core.retry import Retry
+from google.cloud import datastore
+
+from Utility import Utility, RunEnv
+
+schema_namespace = os.environ.get("SCHEMA_NAMESPACE")
+schema_kind = os.environ.get("SCHEMA_KIND")
+default_namespace = "dataecosystem"
+default_kind = "schema"
+
+
+def cleanup_datastore():
+  if schema_namespace is None:
+    datastore_client = datastore.Client(namespace=default_namespace)
+    print(
+        "SCHEMA_NAMESPACE is empty, using default namespace: " + default_namespace)
+  else:
+    datastore_client = datastore.Client(namespace=schema_namespace)
+    print("SCHEMA_NAMESPACE not empty, using var value: " + schema_namespace)
+
+  if schema_kind is None:
+    print("SCHEMA_KIND is empty, using default kind: " + default_kind)
+    kind_to_use = default_kind
+  else:
+    print("SCHEMA_KIND not empty, using var value: " + schema_kind)
+    kind_to_use = schema_kind
+
+  deployments = Utility.path_to_deployments()
+  bootstrap_options = json.loads(RunEnv.BOOTSTRAP_OPTIONS)
+
+  for option in bootstrap_options:
+    schema_path = option['folder']
+    load_sequence = option['load-sequence']
+    path = os.path.join(deployments, RunEnv.SCHEMAS_FOLDER, schema_path,
+                        load_sequence)
+    print("Schemas sequence location: " + path)
+    sequence = Utility.load_json(path)
+    for item in sequence:
+      complete_key = datastore_client.key(kind_to_use, item['kind'].replace(
+          '{{schema-authority}}', RunEnv.DATA_PARTITION))
+      print("Key to delete: " + complete_key.__str__())
+      range = Retry(initial=5, maximum=20, multiplier=2, deadline=120)
+      datastore_client.delete(key=complete_key, retry=range)
+
+
+if __name__ == '__main__':
+  cleanup_datastore()

deployments/scripts/gcp-deployment-requirements.txt

+google-cloud-datastore
+google.api_core
+grpcio
\ No newline at end of file

deployments/shared-schemas/README.md

@@ -141,3 +141,19 @@ All 120 schemas registered or updated.
 ```
 
 In case of errors, the list of failed creations/updates are summarized at the end.
+
+### Environment clean up (GCP)
+Schema bootstrapping used during new platform configuration, creates schema records in Datastore, that cannot be cleaned up during deinstallation.
+If platform deployment must be re-installed, the cleanup script must be runned.
+Scripts for cleanup schemas can be found in [DatastoreCleanUp.py](../scripts/DatastoreCleanUp.py) 
+
+```bash
+pip install -r gcp-deployment-requirements.txt
+```
+You will need to have the following environment variables defined to run scripts.
+| name | value | description | sensitive? | source |
+| ---  | ---   | ---         | ---        | ---    |
+| `DATA_PARTITION` | ex `osdu`| Data partition id| no | - |
+| `SCHEMA_NAMESPACE` | ex `dataecosystem`| If not specified default `dataecosystem` will be used | no | - |
+| `SCHEMA_KIND` | ex `schema`| If not specified default `schema` will be used  | no | - |
+| `GOOGLE_APPLICATION_CREDENTIALS` | ex`usr/key.json` | Google Service account credentials with delete access to Datastore | yes | - |
\ No newline at end of file

devops/gcp/bootstrap-osdu-module/Dockerfile

+FROM google/cloud-sdk:slim
+
+COPY ./devops/gcp/bootstrap-osdu-module/bootstrap_schema.sh /opt 
+COPY ./deployments /opt/
+
+RUN chmod 775 /opt/bootstrap_schema.sh
+
+RUN pip3 install -r /opt/scripts/requirements.txt && pip3 install -r /opt/scripts/gcp-deployment-requirements.txt
+
+CMD ["/bin/bash", "-c", "source /opt/bootstrap_schema.sh"]

devops/gcp/bootstrap-osdu-module/README.md

+## Prerequisites
+* create workload identity gke service account (it is mandatory that bootstrap script will be running from SA)
+* kubernetes job should be create in namespace that is free from istio-injection (job will be in RUNNING state indefenetly with side-car container)
+* set all required ENV variables, they are listed in job.yml under env directive
+
+## Verify
+Schema service bootstrap is based on python bootstrap scripts at schema service repository -> `https://community.opengroup.org/osdu/platform/system/schema-service/-/tree/master/deployments/scripts`
+
+Boostrap scripts contains python script which executes clean-up on Datastore to prevent incorect bootstrap for schema service.
+
+After execution of bootstrap-script, you could go to **GCP console** and look for logs under `Kubernetes Engine -> Workloads -> bootstrap-schema pod`
+Successful execution will lead to similar output (disclaimer: output could be different due to changes at python3 bootstrap scripts)
+```
+The kind osdu:wks:work-product-component--WellboreTrajectory:1.0.0 was registered successfully.
+Try POST for id: osdu:wks:work-product-component--WellboreTrajectory:1.1.0
+The kind osdu:wks:work-product-component--WellboreTrajectory:1.1.0 was registered successfully.
+Try POST for id: osdu:wks:reference-data--WellboreTrajectoryType:1.0.0
+The kind osdu:wks:reference-data--WellboreTrajectoryType:1.0.0 was registered successfully.
+Try POST for id: osdu:wks:reference-data--WordFormatType:1.0.0
+The kind osdu:wks:reference-data--WordFormatType:1.0.0 was registered successfully.
+Try POST for id: osdu:wks:work-product--WorkProduct:1.0.0
+The kind osdu:wks:work-product--WorkProduct:1.0.0 was registered successfully.
+This update took 156.52 seconds.
+All 216 schemas registered, updated or left unchanged because of status PUBLISHED.
+```
+
+Additionally new **Datastore Entities** should be created:
+Go to `Datastore -> Entitites -> Namespace (dataecosystem) -> Kind (schema)`, this kind should be populated with schema records.
+ 

devops/gcp/bootstrap-osdu-module/bootstrap_schema.sh

+#!/usr/bin/env bash
+
+set -ex
+
+export BEARER_TOKEN=`gcloud auth print-identity-token --audiences=${AUDIENCES}`
+
+echo "Clean-up for Datastore schemas"
+python3 /opt/scripts/GcpDatastoreCleanUp.py
+
+sleep 5
+
+echo "Bootstrap Schema Service"
+python3 /opt/scripts/DeploySharedSchemas.py -u ${SCHEMA_URL}/api/schema-service/v1/schema

devops/gcp/bootstrap.yaml

@@ -22,3 +22,24 @@ osdu-gcp-test:
   only:
     variables:
       - $OSDU_GCP == 'true'
+
+containerize_osdu_gcp_schema_bootstrap:
+  stage: containerize
+  image: docker:19.03.15
+  tags: ["osdu-medium"]
+  services:
+    - docker:20.10.7-dind
+  script:
+    - export EXTRA_DOCKER_TAG=""; if [ "$CI_COMMIT_TAG" != "" ] ; then EXTRA_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/osdu-gcp-bootstrap:$CI_COMMIT_TAG" ; elif [ "$CI_COMMIT_REF_NAME" = "master" ] ; then EXTRA_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/osdu-gcp-bootstrap:latest" ; fi
+    - docker build -t $CI_REGISTRY_IMAGE/osdu-gcp-bootstrap:$CI_COMMIT_SHORT_SHA $EXTRA_DOCKER_TAG --file devops/$OSDU_GCP_VENDOR/bootstrap-osdu-module/Dockerfile .
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - docker push $CI_REGISTRY_IMAGE/osdu-gcp-bootstrap
+  only:
+    refs:
+      - master
+      - /.*bootstrap.*/
+      - tags
+    variables:
+      - $OSDU_GCP == 'true'
+
+

provider/schema-azure/src/main/java/org/opengroup/osdu/schema/azure/impl/schemainfostore/AzureSchemaInfoStore.java

@@ -434,6 +434,11 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore {
 
 			String id = tenant + ":" + schemaId;
 
+			// don't check in the erstwhile common tenant
+			if (tenant.equalsIgnoreCase(sharedTenant)) {
+				continue;
+			}
+
 			try {
 				Boolean existsInTenant = findItemInCosmosStore(tenant, cosmosDBName, schemaInfoContainer, id, partitionKey, SchemaInfoDoc.class).isPresent();
 				if (existsInTenant) {

schema-core/src/test/resources/test_schema/resolvedSchemaWithDefinition.json

@@ -486,6 +486,16 @@
                   }
                 ]
               },
+              "id": {
+                "oneOf": [
+                  {
+                    "type": "number"
+                  },
+                  {
+                    "type": "string"
+                  }
+                ]
+              },
               "type": {
                 "type": "string",
                 "enum": [

schema-core/src/test/resources/test_schema/resolvedSchemaWithRefInsideDefinationBlock.json

@@ -491,6 +491,16 @@
                            }
                         ]
                      },
+                     "id": {
+                        "oneOf": [
+                           {
+                              "type": "number"
+                           },
+                           {
+                              "type": "string"
+                           }
+                        ]
+                     },
                      "type":{
                         "type":"string",
                         "enum":[

testing/schema-test-azure/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>org.opengroup.osdu</groupId>
 		<artifactId>os-schema-test</artifactId>
-		<version>0.11.0-SNAPSHOT</version>
+		<version>0.12.0-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>