Commit 790382e0 authored by Aman Verma's avatar Aman Verma
Browse files

resolving merge conflics with master

parents 4873419f dc8b60ce
Pipeline #64344 failed with stages
in 45 seconds
......@@ -355,15 +355,15 @@ The following software have components provided under the terms of this license:
- Jackson dataformat: CBOR (from http://github.com/FasterXML/jackson-dataformats-binary)
- Jackson dataformat: CBOR (from http://github.com/FasterXML/jackson-dataformats-binary)
- Jackson datatype: JSR310 (from http://wiki.fasterxml.com/JacksonModuleJSR310)
- Jackson datatype: Joda (from https://github.com/FasterXML/jackson-datatype-joda)
- Jackson datatype: jdk8 (from https://repo1.maven.org/maven2/com/fasterxml/jackson/datatype/jackson-datatype-jdk8)
- Jackson module: Afterburner (from https://github.com/FasterXML/jackson-modules-base)
- Jackson module: Old JAXB Annotations (javax.xml.bind) (from https://github.com/FasterXML/jackson-modules-base)
- Jackson module: JAXB-annotations (from http://github.com/FasterXML/jackson-module-jaxb-annotations)
- Jackson-annotations (from http://github.com/FasterXML/jackson)
- Jackson-core (from https://github.com/FasterXML/jackson-core)
- Jackson-dataformat-Smile (from http://wiki.fasterxml.com/JacksonForSmile)
- Jackson-dataformat-XML (from https://github.com/FasterXML/jackson-dataformat-xml)
- Jackson-dataformat-XML (from http://wiki.fasterxml.com/JacksonExtensionXmlDataBinding)
- Jackson-dataformat-YAML (from https://github.com/FasterXML/jackson-dataformats-text)
- Jackson-datatype-JODA (from http://wiki.fasterxml.com/JacksonModuleJoda)
- Jackson-module-parameter-names (from https://repo1.maven.org/maven2/com/fasterxml/jackson/module/jackson-module-parameter-names)
- Jakarta Bean Validation API (from https://beanvalidation.org)
- Jakarta Expression Language Implementation (from https://projects.eclipse.org/projects/ee4j.el)
......@@ -392,8 +392,8 @@ The following software have components provided under the terms of this license:
- Lucene Grouping (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-grouping)
- Lucene Highlighter (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-highlighter)
- Lucene Join (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-join)
- Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-memory)
- Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-backward-codecs)
- Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-memory)
- Lucene Miscellaneous (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-misc)
- Lucene Queries (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-queries)
- Lucene QueryParsers (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-queryparser)
......@@ -430,11 +430,12 @@ The following software have components provided under the terms of this license:
- Nimbus JOSE+JWT (from https://bitbucket.org/connect2id/nimbus-jose-jwt)
- Nimbus JOSE+JWT (from https://bitbucket.org/connect2id/nimbus-jose-jwt)
- Nimbus LangTag (from https://bitbucket.org/connect2id/nimbus-language-tags)
- Non-Blocking Reactive Foundation for the JVM (from https://github.com/reactor/reactor-core)
- Non-Blocking Reactive Foundation for the JVM (from https://github.com/reactor/reactor)
- OAuth 2.0 SDK with OpenID Connect extensions (from https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions)
- OAuth 2.0 SDK with OpenID Connect extensions (from https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions)
- Objenesis (from http://objenesis.org)
- Objenesis (from http://objenesis.org)
- OkHttp Logging Interceptor (from https://repo1.maven.org/maven2/com/squareup/okhttp3/logging-interceptor)
- Okio (from https://github.com/square/okio/)
- Okio (from https://github.com/square/okio/)
- OpenCensus (from https://github.com/census-instrumentation/opencensus-java)
......@@ -453,7 +454,6 @@ The following software have components provided under the terms of this license:
- Spring AOP (from https://github.com/spring-projects/spring-framework)
- Spring Beans (from https://github.com/spring-projects/spring-framework)
- Spring Beans (from https://github.com/spring-projects/spring-framework)
- Spring Boot Log4j 2 Starter (from http://projects.spring.io/spring-boot/)
- Spring Boot Tomcat Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-tomcat)
- Spring Boot Tomcat Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-tomcat)
- Spring Commons Logging Bridge (from https://github.com/spring-projects/spring-framework)
......@@ -483,9 +483,9 @@ The following software have components provided under the terms of this license:
- Vavr (from http://vavr.io)
- Vavr Match (from http://vavr.io)
- Woodstox (from https://github.com/FasterXML/woodstox)
- Zipkin Core Library (from https://repo1.maven.org/maven2/io/zipkin/zipkin2/zipkin)
- Zipkin Reporter Brave (from https://repo1.maven.org/maven2/io/zipkin/reporter2/zipkin-reporter-brave)
- Zipkin Reporter: Core (from https://repo1.maven.org/maven2/io/zipkin/reporter2/zipkin-reporter)
- Zipkin v2 (from https://repo1.maven.org/maven2/io/zipkin/zipkin2/zipkin)
- aalto-xml (from )
- aggs-matrix-stats (from https://github.com/elastic/elasticsearch)
- asm (from http://asm.ow2.io/)
......@@ -496,6 +496,7 @@ The following software have components provided under the terms of this license:
- com.google.api.grpc:proto-google-cloud-datastore-v1 (from https://github.com/googleapis/googleapis)
- compiler (from http://github.com/spullara/mustache.java)
- core (from https://github.com/elastic/elasticsearch)
- core (from https://github.com/elastic/elasticsearch)
- datastore-v1-proto-client (from https://repo1.maven.org/maven2/com/google/cloud/datastore/datastore-v1-proto-client)
- elasticsearch-geo (from https://github.com/elastic/elasticsearch)
- error-prone annotations (from https://repo1.maven.org/maven2/com/google/errorprone/error_prone_annotations)
......@@ -523,7 +524,6 @@ The following software have components provided under the terms of this license:
- jose4j (from https://bitbucket.org/b_c/jose4j/)
- lang-mustache (from https://github.com/elastic/elasticsearch)
- lettuce (from http://github.com/mp911de/lettuce/wiki)
- logging-interceptor (from https://github.com/square/okhttp)
- mapper-extras (from https://github.com/elastic/elasticsearch)
- micrometer-core (from https://github.com/micrometer-metrics/micrometer)
- micrometer-core (from https://github.com/micrometer-metrics/micrometer)
......@@ -539,24 +539,23 @@ The following software have components provided under the terms of this license:
- project ':json-path' (from https://github.com/jayway/JsonPath)
- proto-google-cloud-logging-v2 (from https://repo1.maven.org/maven2/com/google/api/grpc/proto-google-cloud-logging-v2)
- proto-google-cloud-pubsub-v1 (from https://github.com/googleapis/java-pubsub/proto-google-cloud-pubsub-v1)
- proto-google-common-protos (from https://github.com/googleapis/api-client-staging)
- proto-google-common-protos (from https://github.com/googleapis/java-iam/proto-google-common-protos)
- proto-google-iam-v1 (from https://github.com/googleapis/java-iam/proto-google-iam-v1)
- rank-eval (from https://github.com/elastic/elasticsearch)
- requests (from https://requests.readthedocs.io)
- resilience4j (from https://resilience4j.readme.io)
- resilience4j (from https://resilience4j.readme.io)
- resilience4j (from https://resilience4j.readme.io)
- resilience4j (from https://github.com/resilience4j/resilience4j)
- resilience4j (from https://resilience4j.readme.io)
- resilience4j (from https://github.com/resilience4j/resilience4j)
- resilience4j (from https://resilience4j.readme.io)
- resilience4j (from https://resilience4j.readme.io)
- resilience4j (from https://resilience4j.readme.io)
- resilience4j (from https://resilience4j.readme.io)
- rest (from https://github.com/elastic/elasticsearch)
- rest (from https://github.com/elastic/elasticsearch)
- rest-high-level (from https://github.com/elastic/elasticsearch)
- rxjava (from https://github.com/ReactiveX/RxJava)
- secure-sm (from https://github.com/elastic/elasticsearch)
- server (from https://github.com/elastic/elasticsearch)
- spring-boot (from https://spring.io/projects/spring-boot)
- spring-boot (from https://spring.io/projects/spring-boot)
- spring-boot-actuator (from https://spring.io/projects/spring-boot)
......@@ -574,6 +573,7 @@ The following software have components provided under the terms of this license:
- spring-boot-starter-aop (from https://spring.io/projects/spring-boot)
- spring-boot-starter-json (from https://spring.io/projects/spring-boot)
- spring-boot-starter-json (from https://spring.io/projects/spring-boot)
- spring-boot-starter-log4j2 (from https://spring.io/projects/spring-boot)
- spring-boot-starter-logging (from https://spring.io/projects/spring-boot)
- spring-boot-starter-reactor-netty (from https://spring.io/projects/spring-boot)
- spring-boot-starter-security (from https://spring.io/projects/spring-boot)
......@@ -589,15 +589,15 @@ The following software have components provided under the terms of this license:
- spring-boot-test (from https://spring.io/projects/spring-boot)
- spring-boot-test-autoconfigure (from https://spring.io/projects/spring-boot)
- spring-boot-test-autoconfigure (from https://spring.io/projects/spring-boot)
- spring-security-config (from http://spring.io/spring-security)
- spring-security-config (from http://spring.io/spring-security)
- spring-security-core (from http://spring.io/spring-security)
- spring-security-core (from http://spring.io/spring-security)
- spring-security-oauth2-core (from http://spring.io/spring-security)
- spring-security-oauth2-jose (from http://spring.io/spring-security)
- spring-security-oauth2-resource-server (from http://spring.io/spring-security)
- spring-security-web (from http://spring.io/spring-security)
- spring-security-web (from http://spring.io/spring-security)
- spring-security-config (from https://spring.io/spring-security)
- spring-security-config (from https://spring.io/spring-security)
- spring-security-core (from https://spring.io/spring-security)
- spring-security-core (from https://spring.io/spring-security)
- spring-security-oauth2-core (from https://spring.io/spring-security)
- spring-security-oauth2-jose (from https://spring.io/spring-security)
- spring-security-oauth2-resource-server (from https://spring.io/spring-security)
- spring-security-web (from https://spring.io/spring-security)
- spring-security-web (from https://spring.io/spring-security)
- springfox-core (from https://github.com/springfox/springfox)
- springfox-schema (from https://github.com/springfox/springfox)
- springfox-spi (from https://github.com/springfox/springfox)
......@@ -609,7 +609,7 @@ The following software have components provided under the terms of this license:
- swagger-jaxrs (from )
- swagger-models (from https://repo1.maven.org/maven2/io/swagger/swagger-models)
- tomcat-embed-core (from http://tomcat.apache.org/)
- tomcat-embed-websocket (from https://tomcat.apache.org/)
- tomcat-embed-websocket (from http://tomcat.apache.org/)
- x-content (from https://github.com/elastic/elasticsearch)
========================================================================
......@@ -626,7 +626,7 @@ The following software have components provided under the terms of this license:
- HdrHistogram (from http://hdrhistogram.github.io/HdrHistogram/)
- Lucene Common Analyzers (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-analyzers-common)
- Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core)
- Reflections (from http://github.com/ronmamo/reflections)
- Reflections (from http://code.google.com/p/reflections/)
- Stax2 API (from http://github.com/FasterXML/stax2-api)
- ThreeTen backport (from https://www.threeten.org/threetenbp)
......@@ -659,7 +659,7 @@ The following software have components provided under the terms of this license:
- Netty/Codec/HTTP (from https://repo1.maven.org/maven2/io/netty/netty-codec-http)
- Protocol Buffers [Core] (from https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java)
- Protocol Buffers [Util] (from https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java-util)
- Reflections (from http://github.com/ronmamo/reflections)
- Reflections (from http://code.google.com/p/reflections/)
- SnakeYAML (from http://www.snakeyaml.org)
- Spring Core (from https://github.com/spring-projects/spring-framework)
- Spring Core (from https://github.com/spring-projects/spring-framework)
......@@ -740,7 +740,6 @@ The following software have components provided under the terms of this license:
- JUnit Platform Commons (from https://junit.org/junit5/)
- JUnit Platform Engine API (from https://junit.org/junit5/)
- JUnit Platform Engine API (from https://junit.org/junit5/)
- JUnit Vintage Engine (from https://junit.org/junit5/)
- Jakarta Annotations API (from https://projects.eclipse.org/projects/ee4j.ca)
- Jakarta Expression Language Implementation (from https://projects.eclipse.org/projects/ee4j.el)
- Logback Classic Module (from https://repo1.maven.org/maven2/ch/qos/logback/logback-classic)
......@@ -753,6 +752,7 @@ The following software have components provided under the terms of this license:
- Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
- SnakeYAML (from http://www.snakeyaml.org)
- org.junit.vintage:junit-vintage-engine (from http://junit.org/junit5/)
========================================================================
EPL-2.0
......@@ -771,9 +771,9 @@ The following software have components provided under the terms of this license:
- JUnit Platform Commons (from https://junit.org/junit5/)
- JUnit Platform Engine API (from https://junit.org/junit5/)
- JUnit Platform Engine API (from https://junit.org/junit5/)
- JUnit Vintage Engine (from https://junit.org/junit5/)
- Jakarta Annotations API (from https://projects.eclipse.org/projects/ee4j.ca)
- Jakarta Expression Language Implementation (from https://projects.eclipse.org/projects/ee4j.el)
- org.junit.vintage:junit-vintage-engine (from http://junit.org/junit5/)
========================================================================
GPL-2.0-only
......@@ -941,8 +941,8 @@ The following software have components provided under the terms of this license:
- msal (from https://github.com/AzureAD/microsoft-authentication-library-for-python)
- msal4j (from https://github.com/AzureAD/microsoft-authentication-library-for-java)
- msal4j-persistence-extension (from https://github.com/AzureAD/microsoft-authentication-extensions-for-java)
- spring-security-core (from http://spring.io/spring-security)
- spring-security-core (from http://spring.io/spring-security)
- spring-security-core (from https://spring.io/spring-security)
- spring-security-core (from https://spring.io/spring-security)
========================================================================
MPL-1.1
......@@ -1000,7 +1000,7 @@ WTFPL
========================================================================
The following software have components provided under the terms of this license:
- Reflections (from http://github.com/ronmamo/reflections)
- Reflections (from http://code.google.com/p/reflections/)
========================================================================
X11
......@@ -1060,9 +1060,9 @@ The following software have components provided under the terms of this license:
- JUnit Platform Commons (from https://junit.org/junit5/)
- JUnit Platform Engine API (from https://junit.org/junit5/)
- JUnit Platform Engine API (from https://junit.org/junit5/)
- JUnit Vintage Engine (from https://junit.org/junit5/)
- Jakarta Activation API jar (from https://repo1.maven.org/maven2/jakarta/activation/jakarta.activation-api)
- Jakarta XML Binding API (from https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api)
- Spongy Castle (from http://rtyley.github.io/spongycastle/)
- org.junit.vintage:junit-vintage-engine (from http://junit.org/junit5/)
import json
import os
from google.api_core.retry import Retry
from google.cloud import datastore
from Utility import Utility, RunEnv
schema_namespace = os.environ.get("SCHEMA_NAMESPACE")
schema_kind = os.environ.get("SCHEMA_KIND")
default_namespace = "dataecosystem"
default_kind = "schema"
def cleanup_datastore():
if schema_namespace is None:
datastore_client = datastore.Client(namespace=default_namespace)
print(
"SCHEMA_NAMESPACE is empty, using default namespace: " + default_namespace)
else:
datastore_client = datastore.Client(namespace=schema_namespace)
print("SCHEMA_NAMESPACE not empty, using var value: " + schema_namespace)
if schema_kind is None:
print("SCHEMA_KIND is empty, using default kind: " + default_kind)
kind_to_use = default_kind
else:
print("SCHEMA_KIND not empty, using var value: " + schema_kind)
kind_to_use = schema_kind
deployments = Utility.path_to_deployments()
bootstrap_options = json.loads(RunEnv.BOOTSTRAP_OPTIONS)
for option in bootstrap_options:
schema_path = option['folder']
load_sequence = option['load-sequence']
path = os.path.join(deployments, RunEnv.SCHEMAS_FOLDER, schema_path,
load_sequence)
print("Schemas sequence location: " + path)
sequence = Utility.load_json(path)
for item in sequence:
complete_key = datastore_client.key(kind_to_use, item['kind'].replace(
'{{schema-authority}}', RunEnv.DATA_PARTITION))
print("Key to delete: " + complete_key.__str__())
range = Retry(initial=5, maximum=20, multiplier=2, deadline=120)
datastore_client.delete(key=complete_key, retry=range)
if __name__ == '__main__':
cleanup_datastore()
google-cloud-datastore
google.api_core
grpcio
\ No newline at end of file
......@@ -141,3 +141,19 @@ All 120 schemas registered or updated.
```
In case of errors, the list of failed creations/updates are summarized at the end.
### Environment clean up (GCP)
Schema bootstrapping used during new platform configuration, creates schema records in Datastore, that cannot be cleaned up during deinstallation.
If platform deployment must be re-installed, the cleanup script must be runned.
Scripts for cleanup schemas can be found in [DatastoreCleanUp.py](../scripts/DatastoreCleanUp.py)
```bash
pip install -r gcp-deployment-requirements.txt
```
You will need to have the following environment variables defined to run scripts.
| name | value | description | sensitive? | source |
| --- | --- | --- | --- | --- |
| `DATA_PARTITION` | ex `osdu`| Data partition id| no | - |
| `SCHEMA_NAMESPACE` | ex `dataecosystem`| If not specified default `dataecosystem` will be used | no | - |
| `SCHEMA_KIND` | ex `schema`| If not specified default `schema` will be used | no | - |
| `GOOGLE_APPLICATION_CREDENTIALS` | ex`usr/key.json` | Google Service account credentials with delete access to Datastore | yes | - |
\ No newline at end of file
FROM google/cloud-sdk:slim
COPY ./devops/gcp/bootstrap-osdu-module/bootstrap_schema.sh /opt
COPY ./deployments /opt/
RUN chmod 775 /opt/bootstrap_schema.sh
RUN pip3 install -r /opt/scripts/requirements.txt && pip3 install -r /opt/scripts/gcp-deployment-requirements.txt
CMD ["/bin/bash", "-c", "source /opt/bootstrap_schema.sh"]
## Prerequisites
* create workload identity gke service account (it is mandatory that bootstrap script will be running from SA)
* kubernetes job should be create in namespace that is free from istio-injection (job will be in RUNNING state indefenetly with side-car container)
* set all required ENV variables, they are listed in job.yml under env directive
## Verify
Schema service bootstrap is based on python bootstrap scripts at schema service repository -> `https://community.opengroup.org/osdu/platform/system/schema-service/-/tree/master/deployments/scripts`
Boostrap scripts contains python script which executes clean-up on Datastore to prevent incorect bootstrap for schema service.
After execution of bootstrap-script, you could go to **GCP console** and look for logs under `Kubernetes Engine -> Workloads -> bootstrap-schema pod`
Successful execution will lead to similar output (disclaimer: output could be different due to changes at python3 bootstrap scripts)
```
The kind osdu:wks:work-product-component--WellboreTrajectory:1.0.0 was registered successfully.
Try POST for id: osdu:wks:work-product-component--WellboreTrajectory:1.1.0
The kind osdu:wks:work-product-component--WellboreTrajectory:1.1.0 was registered successfully.
Try POST for id: osdu:wks:reference-data--WellboreTrajectoryType:1.0.0
The kind osdu:wks:reference-data--WellboreTrajectoryType:1.0.0 was registered successfully.
Try POST for id: osdu:wks:reference-data--WordFormatType:1.0.0
The kind osdu:wks:reference-data--WordFormatType:1.0.0 was registered successfully.
Try POST for id: osdu:wks:work-product--WorkProduct:1.0.0
The kind osdu:wks:work-product--WorkProduct:1.0.0 was registered successfully.
This update took 156.52 seconds.
All 216 schemas registered, updated or left unchanged because of status PUBLISHED.
```
Additionally new **Datastore Entities** should be created:
Go to `Datastore -> Entitites -> Namespace (dataecosystem) -> Kind (schema)`, this kind should be populated with schema records.
#!/usr/bin/env bash
set -ex
export BEARER_TOKEN=`gcloud auth print-identity-token --audiences=${AUDIENCES}`
echo "Clean-up for Datastore schemas"
python3 /opt/scripts/GcpDatastoreCleanUp.py
sleep 5
echo "Bootstrap Schema Service"
python3 /opt/scripts/DeploySharedSchemas.py -u ${SCHEMA_URL}/api/schema-service/v1/schema
......@@ -22,3 +22,24 @@ osdu-gcp-test:
only:
variables:
- $OSDU_GCP == 'true'
containerize_osdu_gcp_schema_bootstrap:
stage: containerize
image: docker:19.03.15
tags: ["osdu-medium"]
services:
- docker:20.10.7-dind
script:
- export EXTRA_DOCKER_TAG=""; if [ "$CI_COMMIT_TAG" != "" ] ; then EXTRA_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/osdu-gcp-bootstrap:$CI_COMMIT_TAG" ; elif [ "$CI_COMMIT_REF_NAME" = "master" ] ; then EXTRA_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/osdu-gcp-bootstrap:latest" ; fi
- docker build -t $CI_REGISTRY_IMAGE/osdu-gcp-bootstrap:$CI_COMMIT_SHORT_SHA $EXTRA_DOCKER_TAG --file devops/$OSDU_GCP_VENDOR/bootstrap-osdu-module/Dockerfile .
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- docker push $CI_REGISTRY_IMAGE/osdu-gcp-bootstrap
only:
refs:
- master
- /.*bootstrap.*/
- tags
variables:
- $OSDU_GCP == 'true'
......@@ -429,6 +429,11 @@ public class AzureSchemaInfoStore implements ISchemaInfoStore {
for (String tenant : tenantList) {
String id = tenant + ":" + schemaId;
// don't check in the erstwhile common tenant
if (tenant.equalsIgnoreCase(sharedTenant)) {
continue;
}
try {
Boolean existsInTenant = findItemInCosmosStore(tenant, cosmosDBName, schemaInfoContainer, id, partitionKey, SchemaInfoDoc.class).isPresent();
if (existsInTenant) {
......
......@@ -4,7 +4,7 @@
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-schema-test</artifactId>
<version>0.11.0-SNAPSHOT</version>
<version>0.12.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment