Commit fa532172 authored by Aman Verma's avatar Aman Verma
Browse files

merging from master

parents c4ba335e e9680eb6
Pipeline #65796 failed with stages
in 3 minutes and 16 seconds
This diff is collapsed.
import json
import os
from google.api_core.retry import Retry
from google.cloud import datastore
from Utility import Utility, RunEnv
schema_namespace = os.environ.get("SCHEMA_NAMESPACE")
schema_kind = os.environ.get("SCHEMA_KIND")
default_namespace = "dataecosystem"
default_kind = "schema"
def cleanup_datastore():
if schema_namespace is None:
datastore_client = datastore.Client(namespace=default_namespace)
print(
"SCHEMA_NAMESPACE is empty, using default namespace: " + default_namespace)
else:
datastore_client = datastore.Client(namespace=schema_namespace)
print("SCHEMA_NAMESPACE not empty, using var value: " + schema_namespace)
if schema_kind is None:
print("SCHEMA_KIND is empty, using default kind: " + default_kind)
kind_to_use = default_kind
else:
print("SCHEMA_KIND not empty, using var value: " + schema_kind)
kind_to_use = schema_kind
deployments = Utility.path_to_deployments()
bootstrap_options = json.loads(RunEnv.BOOTSTRAP_OPTIONS)
for option in bootstrap_options:
schema_path = option['folder']
load_sequence = option['load-sequence']
path = os.path.join(deployments, RunEnv.SCHEMAS_FOLDER, schema_path,
load_sequence)
print("Schemas sequence location: " + path)
sequence = Utility.load_json(path)
for item in sequence:
complete_key = datastore_client.key(kind_to_use, item['kind'].replace(
'{{schema-authority}}', RunEnv.DATA_PARTITION))
print("Key to delete: " + complete_key.__str__())
range = Retry(initial=5, maximum=20, multiplier=2, deadline=120)
datastore_client.delete(key=complete_key, retry=range)
if __name__ == '__main__':
cleanup_datastore()
google-cloud-datastore
google.api_core
grpcio
\ No newline at end of file
......@@ -141,3 +141,19 @@ All 120 schemas registered or updated.
```
In case of errors, the list of failed creations/updates are summarized at the end.
### Environment clean up (GCP)
Schema bootstrapping used during new platform configuration, creates schema records in Datastore, that cannot be cleaned up during deinstallation.
If platform deployment must be re-installed, the cleanup script must be runned.
Scripts for cleanup schemas can be found in [DatastoreCleanUp.py](../scripts/DatastoreCleanUp.py)
```bash
pip install -r gcp-deployment-requirements.txt
```
You will need to have the following environment variables defined to run scripts.
| name | value | description | sensitive? | source |
| --- | --- | --- | --- | --- |
| `DATA_PARTITION` | ex `osdu`| Data partition id| no | - |
| `SCHEMA_NAMESPACE` | ex `dataecosystem`| If not specified default `dataecosystem` will be used | no | - |
| `SCHEMA_KIND` | ex `schema`| If not specified default `schema` will be used | no | - |
| `GOOGLE_APPLICATION_CREDENTIALS` | ex`usr/key.json` | Google Service account credentials with delete access to Datastore | yes | - |
\ No newline at end of file
FROM google/cloud-sdk:slim
COPY ./devops/gcp/bootstrap-osdu-module/bootstrap_schema.sh /opt
COPY ./deployments /opt/
RUN chmod 775 /opt/bootstrap_schema.sh
RUN pip3 install -r /opt/scripts/requirements.txt && pip3 install -r /opt/scripts/gcp-deployment-requirements.txt
CMD ["/bin/bash", "-c", "source /opt/bootstrap_schema.sh"]
## Prerequisites
* create workload identity gke service account (it is mandatory that bootstrap script will be running from SA)
* kubernetes job should be create in namespace that is free from istio-injection (job will be in RUNNING state indefenetly with side-car container)
* set all required ENV variables, they are listed in job.yml under env directive
## Verify
Schema service bootstrap is based on python bootstrap scripts at schema service repository -> `https://community.opengroup.org/osdu/platform/system/schema-service/-/tree/master/deployments/scripts`
Boostrap scripts contains python script which executes clean-up on Datastore to prevent incorect bootstrap for schema service.
After execution of bootstrap-script, you could go to **GCP console** and look for logs under `Kubernetes Engine -> Workloads -> bootstrap-schema pod`
Successful execution will lead to similar output (disclaimer: output could be different due to changes at python3 bootstrap scripts)
```
The kind osdu:wks:work-product-component--WellboreTrajectory:1.0.0 was registered successfully.
Try POST for id: osdu:wks:work-product-component--WellboreTrajectory:1.1.0
The kind osdu:wks:work-product-component--WellboreTrajectory:1.1.0 was registered successfully.
Try POST for id: osdu:wks:reference-data--WellboreTrajectoryType:1.0.0
The kind osdu:wks:reference-data--WellboreTrajectoryType:1.0.0 was registered successfully.
Try POST for id: osdu:wks:reference-data--WordFormatType:1.0.0
The kind osdu:wks:reference-data--WordFormatType:1.0.0 was registered successfully.
Try POST for id: osdu:wks:work-product--WorkProduct:1.0.0
The kind osdu:wks:work-product--WorkProduct:1.0.0 was registered successfully.
This update took 156.52 seconds.
All 216 schemas registered, updated or left unchanged because of status PUBLISHED.
```
Additionally new **Datastore Entities** should be created:
Go to `Datastore -> Entitites -> Namespace (dataecosystem) -> Kind (schema)`, this kind should be populated with schema records.
#!/usr/bin/env bash
set -ex
export BEARER_TOKEN=`gcloud auth print-identity-token --audiences=${AUDIENCES}`
echo "Clean-up for Datastore schemas"
python3 /opt/scripts/GcpDatastoreCleanUp.py
sleep 5
echo "Bootstrap Schema Service"
python3 /opt/scripts/DeploySharedSchemas.py -u ${SCHEMA_URL}/api/schema-service/v1/schema
......@@ -22,3 +22,24 @@ osdu-gcp-test:
only:
variables:
- $OSDU_GCP == 'true'
containerize_osdu_gcp_schema_bootstrap:
stage: containerize
image: docker:19.03.15
tags: ["osdu-medium"]
services:
- docker:20.10.7-dind
script:
- export EXTRA_DOCKER_TAG=""; if [ "$CI_COMMIT_TAG" != "" ] ; then EXTRA_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/osdu-gcp-bootstrap:$CI_COMMIT_TAG" ; elif [ "$CI_COMMIT_REF_NAME" = "master" ] ; then EXTRA_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/osdu-gcp-bootstrap:latest" ; fi
- docker build -t $CI_REGISTRY_IMAGE/osdu-gcp-bootstrap:$CI_COMMIT_SHORT_SHA $EXTRA_DOCKER_TAG --file devops/$OSDU_GCP_VENDOR/bootstrap-osdu-module/Dockerfile .
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- docker push $CI_REGISTRY_IMAGE/osdu-gcp-bootstrap
only:
refs:
- master
- /.*bootstrap.*/
- tags
variables:
- $OSDU_GCP == 'true'
......@@ -434,6 +434,11 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore {
String id = tenant + ":" + schemaId;
// don't check in the erstwhile common tenant
if (tenant.equalsIgnoreCase(sharedTenant)) {
continue;
}
try {
Boolean existsInTenant = findItemInCosmosStore(tenant, cosmosDBName, schemaInfoContainer, id, partitionKey, SchemaInfoDoc.class).isPresent();
if (existsInTenant) {
......
......@@ -486,6 +486,16 @@
}
]
},
"id": {
"oneOf": [
{
"type": "number"
},
{
"type": "string"
}
]
},
"type": {
"type": "string",
"enum": [
......
......@@ -491,6 +491,16 @@
}
]
},
"id": {
"oneOf": [
{
"type": "number"
},
{
"type": "string"
}
]
},
"type":{
"type":"string",
"enum":[
......
......@@ -4,7 +4,7 @@
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-schema-test</artifactId>
<version>0.11.0-SNAPSHOT</version>
<version>0.12.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment