diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c207063c02f963139326acd7c9f9b1f2030ddfeb..15f4038e15cb89d98a9d7c1b63e235ea6183bb1f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -47,7 +47,10 @@ include:
file: "scanners/gitlab-ultimate.yml"
- project: "osdu/platform/ci-cd-pipelines"
- file: "cloud-providers/aws.yml"
+ file: "cloud-providers/aws-global.yml"
+
+ - project: "osdu/platform/ci-cd-pipelines"
+ file: "cloud-providers/aws-maven.yml"
- project: "osdu/platform/ci-cd-pipelines"
file: "cloud-providers/azure.yml"
diff --git a/NOTICE b/NOTICE
index 59a48b5e482b5b78e502db43b6357c4e9017a420..efc25f4175db2c582c97a31de26de967d3a553e6 100644
--- a/NOTICE
+++ b/NOTICE
@@ -16,7 +16,7 @@ Apache-1.1
The following software have components provided under the terms of this license:
- Apache Commons CLI (from https://commons.apache.org/proper/commons-cli/, https://repo1.maven.org/maven2/commons-cli/commons-cli)
-- Cobertura (from http://cobertura.sourceforge.net)
+- Cobertura code coverage (from http://cobertura.sourceforge.net)
- Default Plexus Container (from https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-container-default)
- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils, https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-utils)
- oro (from https://repo1.maven.org/maven2/oro/oro)
@@ -290,7 +290,6 @@ The following software have components provided under the terms of this license:
- Apache Ant + JUnit (from http://ant.apache.org/, https://ant.apache.org/)
- Apache Ant Core
- Apache Ant Launcher (from http://ant.apache.org/)
-- Apache Commons BeanUtils (from http://commons.apache.org/proper/commons-beanutils/)
- Apache Commons CLI (from https://commons.apache.org/proper/commons-cli/, https://repo1.maven.org/maven2/commons-cli/commons-cli)
- Apache Commons Codec (from https://commons.apache.org/proper/commons-codec/)
- Apache Commons Collections (from http://commons.apache.org/proper/commons-collections/)
@@ -299,9 +298,9 @@ The following software have components provided under the terms of this license:
- Apache Commons Logging (from http://commons.apache.org/logging/, http://commons.apache.org/proper/commons-logging/)
- Apache Commons Validator (from http://commons.apache.org/proper/commons-validator/, http://jakarta.apache.org/commons/${pom.artifactId.substring(8)}/, https://repo1.maven.org/maven2/commons-validator/commons-validator)
- Apache Geronimo JMS Spec 2.0 (from http://geronimo.apache.org/maven/${siteId}/${version})
+- Apache Groovy (from http://groovy-lang.org, http://groovy.codehaus.org/, https://groovy-lang.org)
- Apache HTTP transport v2 for the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-apache-v2)
- Apache HttpAsyncClient (from http://hc.apache.org/httpcomponents-asyncclient)
-- Apache HttpClient Cache (from http://hc.apache.org/httpcomponents-client)
- Apache HttpCore (from http://hc.apache.org/httpcomponents-core-ga, http://hc.apache.org/httpcomponents-core-ga/, http://hc.apache.org/httpcomponents-core/)
- Apache HttpCore NIO (from http://hc.apache.org/httpcomponents-core-ga, http://hc.apache.org/httpcomponents-core/)
- Apache Log4j API (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-api)
@@ -331,10 +330,10 @@ The following software have components provided under the terms of this license:
- Byte Buddy (without dependencies) (from https://repo1.maven.org/maven2/net/bytebuddy/byte-buddy)
- Byte Buddy Java agent (from https://repo1.maven.org/maven2/net/bytebuddy/byte-buddy-agent)
- ClassMate (from http://github.com/cowtowncoder/java-classmate)
-- Cloud Key Management Service (KMS) API (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-cloudkms)
-- Cloud Storage JSON API (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-storage)
+- Cloud Key Management Service (KMS) API v1-rev81-1.25.0 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-cloudkms)
+- Cloud Storage API v1-rev40-1.20.0 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-storage)
- CloudWatch Metrics for AWS Java SDK (from https://aws.amazon.com/sdkforjava)
-- Cobertura (from http://cobertura.sourceforge.net)
+- Cobertura code coverage (from http://cobertura.sourceforge.net)
- Collections (from https://repo1.maven.org/maven2/commons-collections/commons-collections)
- Converter: Jackson (from https://github.com/square/retrofit, https://repo1.maven.org/maven2/com/squareup/retrofit2/converter-jackson)
- Core functionality for the Reactor Netty library (from https://github.com/reactor/reactor-netty)
@@ -353,16 +352,15 @@ The following software have components provided under the terms of this license:
- GSON extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-gson)
- Google APIs Client Library for Java (from https://repo1.maven.org/maven2/com/google/api-client/google-api-client)
- Google App Engine extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-appengine)
-- Google Cloud Core (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-core, https://github.com/googleapis/java-core)
+- Google Cloud Core (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-core, https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-core, https://github.com/googleapis/java-core)
- Google Cloud Core HTTP (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-core-http, https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-core-http, https://github.com/googleapis/java-core)
-- Google Cloud Core gRPC (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-core-grpc, https://github.com/googleapis/java-core)
+- Google Cloud Core gRPC (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-core-grpc, https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-core-grpc, https://github.com/googleapis/java-core)
- Google Cloud Datastore (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-datastore, https://github.com/googleapis/java-datastore)
- Google Cloud IAM Service Account Credentials (from https://github.com/googleapis/java-iamcredentials)
- Google Cloud Logging (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-logging, https://github.com/googleapis/java-logging)
-- Google Cloud Pub/Sub (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-pubsub, https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-pubsub, https://github.com/googleapis/java-pubsub)
+- Google Cloud Pub/Sub (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-pubsub, https://github.com/googleapis/java-pubsub)
- Google Cloud Storage (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-storage, https://github.com/googleapis/java-storage)
- Google HTTP Client Library for Java (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client)
-- Google Identity and Access Management (IAM) API (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-iam)
- Google OAuth Client Library for Java (from https://repo1.maven.org/maven2/com/google/oauth-client/google-oauth-client)
- Gson (from http://code.google.com/p/google-gson/, https://repo1.maven.org/maven2/com/google/code/gson/gson)
- Guava InternalFutureFailureAccess and InternalFutures (from https://repo1.maven.org/maven2/com/google/guava/failureaccess)
@@ -370,11 +368,15 @@ The following software have components provided under the terms of this license:
- HPPC Collections (from https://repo1.maven.org/maven2/com/carrotsearch/hppc)
- HTTP functionality for the Reactor Netty library (from https://github.com/reactor/reactor-netty)
- Hibernate Validator (from https://repo1.maven.org/maven2/org/hibernate/hibernate-validator, https://repo1.maven.org/maven2/org/hibernate/validator/hibernate-validator)
+- HikariCP (from https://github.com/brettwooldridge/HikariCP)
+- Hop (from https://github.com/rabbitmq/hop, https://www.rabbitmq.com)
- HttpClient (from http://hc.apache.org/httpcomponents-client)
+- HttpClient Cache (from http://hc.apache.org/httpcomponents-client)
- IBM COS Java SDK for Amazon S3 (from https://github.com/ibm/ibm-cos-sdk-java)
- IBM COS Java SDK for COS KMS (from https://github.com/ibm/ibm-cos-sdk-java)
- IBM COS SDK For Java (from https://github.com/ibm/ibm-cos-sdk-java)
- IBM COS SDK for Java - Core (from https://github.com/ibm/ibm-cos-sdk-java)
+- Identity and Access Management (IAM) API v1-rev289-1.25.0 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-iam)
- J2ObjC Annotations (from https://github.com/google/j2objc/)
- JBoss Logging 3 (from http://www.jboss.org)
- JCIP Annotations under Apache License (from http://stephenc.github.com/jcip-annotations)
@@ -399,7 +401,7 @@ The following software have components provided under the terms of this license:
- Jackson-annotations (from http://github.com/FasterXML/jackson, http://wiki.fasterxml.com/JacksonHome)
- Jackson-core (from http://wiki.fasterxml.com/JacksonHome, https://github.com/FasterXML/jackson-core)
- Jackson-dataformat-XML (from http://wiki.fasterxml.com/JacksonExtensionXmlDataBinding, https://github.com/FasterXML/jackson-dataformat-xml)
-- Jackson-dataformat-YAML (from https://github.com/FasterXML/jackson, https://github.com/FasterXML/jackson-dataformats-text)
+- Jackson-dataformat-YAML (from http://wiki.fasterxml.com/JacksonExtensionYAML, https://github.com/FasterXML/jackson, https://github.com/FasterXML/jackson-dataformats-text)
- Jackson-module-parameter-names (from https://repo1.maven.org/maven2/com/fasterxml/jackson/module/jackson-module-parameter-names)
- Jakarta Bean Validation API (from https://beanvalidation.org)
- Jakarta Expression Language Implementation (from https://projects.eclipse.org/projects/ee4j.el)
@@ -419,6 +421,8 @@ The following software have components provided under the terms of this license:
- KeePassJava2 :: KDB (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2-kdb)
- KeePassJava2 :: KDBX (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2-kdbx)
- KeePassJava2 :: Simple (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2-simple)
+- Kotlin Stdlib (from https://kotlinlang.org/, https://repo1.maven.org/maven2/org/jetbrains/kotlin/kotlin-stdlib)
+- Kotlin Stdlib Common (from https://kotlinlang.org/)
- Lang (from https://repo1.maven.org/maven2/commons-lang/commons-lang)
- Logback Contrib :: JSON :: Classic (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-json-classic)
- Logback Contrib :: JSON :: Core (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-json-core)
@@ -496,6 +500,7 @@ The following software have components provided under the terms of this license:
- PWDB :: Database (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/database)
- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils, https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-utils)
- Plexus Velocity Component (from https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-velocity)
+- PostgreSQL JDBC Driver
- PowerMock (from http://www.powermock.org, https://repo1.maven.org/maven2/org/powermock/powermock-api-mockito)
- Protocol Buffer extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-protobuf)
- Proton-J (from https://repo1.maven.org/maven2/org/apache/qpid/proton-j)
@@ -504,21 +509,23 @@ The following software have components provided under the terms of this license:
- Reactive Streams Netty driver (from https://github.com/reactor/reactor-netty)
- Retrofit (from https://github.com/square/retrofit, https://repo1.maven.org/maven2/com/squareup/retrofit2/retrofit)
- Servlet Specification 2.5 API (from http://jetty.mortbay.org, https://repo1.maven.org/maven2/org/mortbay/jetty/servlet-api-2.5)
+- Simple XML (safe) (from https://github.com/dweiss/simplexml)
- SnakeYAML (from http://code.google.com/p/snakeyaml/, http://www.snakeyaml.org)
- Spatial4J (from https://projects.eclipse.org/projects/locationtech.spatial4j)
-- Spring AMQP Core (from https://github.com/spring-projects/spring-amqp, https://projects.spring.io/spring-amqp)
+- Spring AMQP Core (from http://www.springsource.org/spring-amqp, https://github.com/spring-projects/spring-amqp, https://projects.spring.io/spring-amqp)
- Spring AOP (from http://www.springframework.org, https://github.com/spring-projects/spring-framework, https://repo1.maven.org/maven2/org/springframework/spring-aop)
- Spring Beans (from http://www.springframework.org, https://github.com/SpringSource/spring-framework, https://github.com/spring-projects/spring-framework, https://repo1.maven.org/maven2/org/springframework/spring-beans)
-- Spring Boot (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot, https://spring.io/projects/spring-boot)
+- Spring Boot (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot, https://spring.io/projects/spring-boot)
- Spring Boot AMPQ Starter (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-amqp, https://spring.io/projects/spring-boot)
- Spring Boot AOP Starter (from http://projects.spring.io/spring-boot/, https://spring.io/projects/spring-boot)
-- Spring Boot Actuator (from http://projects.spring.io/spring-boot/, https://spring.io/projects/spring-boot)
+- Spring Boot Actuator (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-actuator, https://spring.io/projects/spring-boot)
- Spring Boot Actuator AutoConfigure (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-actuator-autoconfigure, https://spring.io/projects/spring-boot)
-- Spring Boot Actuator Starter (from http://projects.spring.io/spring-boot/, https://spring.io/projects/spring-boot)
-- Spring Boot AutoConfigure (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-autoconfigure, https://spring.io/projects/spring-boot)
+- Spring Boot Actuator Starter (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-actuator, https://spring.io/projects/spring-boot)
+- Spring Boot AutoConfigure (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-autoconfigure, https://spring.io/projects/spring-boot)
- Spring Boot Configuration Processor (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-tools/spring-boot-configuration-processor, https://spring.io/projects/spring-boot)
-- Spring Boot Data MongoDB Starter (from http://projects.spring.io/spring-boot/, https://spring.io/projects/spring-boot)
-- Spring Boot Dependencies (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#, https://spring.io/projects/spring-boot)
+- Spring Boot Data MongoDB Starter (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-data-mongodb, https://spring.io/projects/spring-boot)
+- Spring Boot Dependencies (from http://projects.spring.io/spring-boot/, https://spring.io/projects/spring-boot)
+- Spring Boot JDBC Starter (from http://projects.spring.io/spring-boot/, https://spring.io/projects/spring-boot)
- Spring Boot Jersey Starter (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-jersey, https://spring.io/projects/spring-boot)
- Spring Boot Json Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-json, https://spring.io/projects/spring-boot)
- Spring Boot Log4j 2 Starter (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-log4j2, https://spring.io/projects/spring-boot)
@@ -527,10 +534,10 @@ The following software have components provided under the terms of this license:
- Spring Boot Security Starter (from http://projects.spring.io/spring-boot/, https://spring.io/projects/spring-boot)
- Spring Boot Starter (from http://projects.spring.io/spring-boot/, https://spring.io/projects/spring-boot)
- Spring Boot Test (from http://projects.spring.io/spring-boot/, https://spring.io/projects/spring-boot)
-- Spring Boot Test Auto-Configure (from http://projects.spring.io/spring-boot/, https://spring.io/projects/spring-boot)
+- Spring Boot Test Auto-Configure (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-test-autoconfigure, https://spring.io/projects/spring-boot)
- Spring Boot Test Starter (from http://projects.spring.io/spring-boot/, https://spring.io/projects/spring-boot)
- Spring Boot Tomcat Starter (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-tomcat, https://spring.io/projects/spring-boot)
-- Spring Boot Validation Starter (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/, https://spring.io/projects/spring-boot)
+- Spring Boot Validation Starter (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-validation, https://spring.io/projects/spring-boot)
- Spring Boot Web Starter (from http://projects.spring.io/spring-boot/, https://spring.io/projects/spring-boot)
- Spring Boot WebFlux Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-webflux, https://spring.io/projects/spring-boot)
- Spring Commons Logging Bridge (from https://github.com/spring-projects/spring-framework)
@@ -541,9 +548,9 @@ The following software have components provided under the terms of this license:
- Spring Expression Language (SpEL) (from https://github.com/SpringSource/spring-framework, https://github.com/spring-projects/spring-framework, https://repo1.maven.org/maven2/org/springframework/spring-expression)
- Spring JMS (from https://github.com/SpringSource/spring-framework, https://github.com/spring-projects/spring-framework, https://repo1.maven.org/maven2/org/springframework/spring-jms)
- Spring Messaging (from https://github.com/spring-projects/spring-framework)
+- Spring Plugin - Core (from https://repo1.maven.org/maven2/org/springframework/plugin/spring-plugin-core)
- Spring Plugin - Metadata Extension (from https://repo1.maven.org/maven2/org/springframework/plugin/spring-plugin-metadata)
-- Spring Plugin Core (from https://repo1.maven.org/maven2/org/springframework/plugin/spring-plugin-core)
-- Spring RabbitMQ Support (from https://github.com/spring-projects/spring-amqp, https://projects.spring.io/spring-amqp)
+- Spring RabbitMQ Support (from http://www.springsource.org/spring-amqp, https://github.com/spring-projects/spring-amqp, https://projects.spring.io/spring-amqp)
- Spring Retry (from http://www.springsource.org)
- Spring Security - Core (from http://spring.io/spring-security, https://repo1.maven.org/maven2/org/springframework/security/spring-security-core, https://spring.io/projects/spring-security, https://spring.io/spring-security)
- Spring Security - Namespace Configuration Module (from http://spring.io/spring-security, https://repo1.maven.org/maven2/org/springframework/security/spring-security-config, https://spring.io/spring-security)
@@ -564,6 +571,8 @@ The following software have components provided under the terms of this license:
- Zipkin v2 (from https://repo1.maven.org/maven2/io/zipkin/zipkin2/zipkin)
- aalto-xml (from https://github.com/FasterXML/aalto-xml, https://repo1.maven.org/maven2/com/fasterxml/aalto-xml)
- aggs-matrix-stats (from https://github.com/elastic/elasticsearch)
+- com.google.api.grpc:grpc-google-cloud-pubsub-v1 (from https://github.com/googleapis/googleapis)
+- commons-beanutils (from https://repo1.maven.org/maven2/commons-beanutils/commons-beanutils)
- compiler (from http://github.com/spullara/mustache.java)
- datastore-v1-proto-client (from https://repo1.maven.org/maven2/com/google/cloud/datastore/datastore-v1-proto-client)
- elasticsearch (from https://github.com/elastic/elasticsearch, https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch)
@@ -573,7 +582,6 @@ The following software have components provided under the terms of this license:
- elasticsearch-secure-sm (from https://github.com/elastic/elasticsearch)
- elasticsearch-x-content (from https://github.com/elastic/elasticsearch)
- error-prone annotations (from https://repo1.maven.org/maven2/com/google/errorprone/error_prone_annotations)
-- grpc-google-cloud-pubsub-v1 (from https://repo1.maven.org/maven2/com/google/api/grpc/grpc-google-cloud-pubsub-v1)
- io.grpc:grpc-alts (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-api (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-auth (from https://github.com/grpc/grpc-java)
@@ -610,6 +618,7 @@ The following software have components provided under the terms of this license:
- mapper-extras (from https://github.com/elastic/elasticsearch)
- micrometer-core (from https://github.com/micrometer-metrics/micrometer)
- micrometer-registry-azure-monitor (from https://github.com/micrometer-metrics/micrometer)
+- minio (from https://github.com/minio/minio-java)
- org.apiguardian:apiguardian-api (from https://github.com/apiguardian-team/apiguardian)
- org.conscrypt:conscrypt-openjdk-uber (from https://conscrypt.org/)
- org.opentest4j:opentest4j (from https://github.com/ota4j-team/opentest4j)
@@ -618,23 +627,22 @@ The following software have components provided under the terms of this license:
- perfmark:perfmark-api (from https://github.com/perfmark/perfmark)
- powermock-api-mockito2 (from https://repo1.maven.org/maven2/org/powermock/powermock-api-mockito2)
- powermock-api-support (from https://repo1.maven.org/maven2/org/powermock/powermock-api-support)
-- powermock-module-junit4 (from http://www.powermock.org)
-- powermock-module-junit4-common (from https://repo1.maven.org/maven2/org/powermock/powermock-module-junit4-common)
- powermock-reflect (from https://repo1.maven.org/maven2/org/powermock/powermock-reflect)
-- proto-google-cloud-datastore-v1 (from https://github.com/googleapis/googleapis, https://github.com/googleapis/java-datastore/proto-google-cloud-datastore-v1)
+- proto-google-cloud-datastore-v1 (from https://github.com/googleapis/googleapis, https://github.com/googleapis/java-datastore/proto-google-cloud-datastore-v1, https://repo1.maven.org/maven2/com/google/api/grpc/proto-google-cloud-datastore-v1)
- proto-google-cloud-iamcredentials-v1 (from https://github.com/googleapis/java-iamcredentials/proto-google-cloud-iamcredentials-v1, https://repo1.maven.org/maven2/com/google/api/grpc/proto-google-cloud-iamcredentials-v1)
- proto-google-cloud-logging-v2 (from https://github.com/googleapis/java-logging/proto-google-cloud-logging-v2, https://repo1.maven.org/maven2/com/google/api/grpc/proto-google-cloud-logging-v2)
- proto-google-cloud-pubsub-v1 (from https://github.com/googleapis/googleapis, https://github.com/googleapis/java-pubsub/proto-google-cloud-pubsub-v1)
- proto-google-common-protos (from https://github.com/googleapis/api-client-staging, https://github.com/googleapis/googleapis, https://github.com/googleapis/java-iam/proto-google-common-protos)
- proto-google-iam-v1 (from https://github.com/googleapis/googleapis, https://github.com/googleapis/java-iam/proto-google-iam-v1)
- rank-eval (from https://github.com/elastic/elasticsearch)
-- resilience4j (from https://github.com/resilience4j/resilience4j, ttps://resilience4j.readme.io)
+- resilience4j (from https://github.com/resilience4j/resilience4j, https://resilience4j.readme.io, ttps://resilience4j.readme.io)
- rest (from https://github.com/elastic/elasticsearch)
- rest-high-level (from https://github.com/elastic/elasticsearch)
- rxjava (from https://github.com/ReactiveX/RxJava)
-- spring-security-oauth2-client (from http://spring.io/spring-security, https://spring.io/spring-security)
+- spring-jdbc (from https://repo1.maven.org/maven2/org/springframework/spring-jdbc)
+- spring-security-oauth2-client (from http://spring.io/spring-security, https://spring.io/projects/spring-security, https://spring.io/spring-security)
- spring-security-oauth2-core (from http://spring.io/spring-security, https://spring.io/spring-security)
-- spring-security-oauth2-jose (from http://spring.io/spring-security, https://spring.io/spring-security)
+- spring-security-oauth2-jose (from http://spring.io/spring-security, https://spring.io/projects/spring-security, https://spring.io/spring-security)
- spring-security-test (from http://spring.io/spring-security, https://spring.io/projects/spring-security, https://spring.io/spring-security)
- springfox-bean-validators (from https://github.com/springfox/springfox)
- springfox-boot-starter (from https://github.com/springfox/springfox)
@@ -670,6 +678,7 @@ The following software have components provided under the terms of this license:
- Hamcrest Core (from https://repo1.maven.org/maven2/org/hamcrest/hamcrest-core)
- Lucene Common Analyzers (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-analyzers-common)
- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils, https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-utils)
+- PostgreSQL JDBC Driver
- Reflections (from http://code.google.com/p/reflections/, http://github.com/ronmamo/reflections)
- Stax2 API (from http://github.com/FasterXML/stax2-api)
- ThreeTen backport (from https://github.com/ThreeTen/threetenbp, https://www.threeten.org/threetenbp)
@@ -691,11 +700,11 @@ BSD-3-Clause
The following software have components provided under the terms of this license:
- API Common (from https://github.com/googleapis, https://github.com/googleapis/api-common-java)
-- ASM Analysis (from http://asm.ow2.io/, http://asm.ow2.org/)
-- ASM Commons (from http://asm.ow2.io/, http://asm.ow2.org/, https://repo1.maven.org/maven2/org/ow2/asm/asm-commons)
+- ASM Analysis (from http://asm.ow2.io/)
+- ASM Commons (from http://asm.ow2.io/, https://repo1.maven.org/maven2/org/ow2/asm/asm-commons)
- ASM Core (from http://asm.ow2.io/, http://asm.ow2.org/)
-- ASM Tree (from http://asm.ow2.io/, http://asm.ow2.org/)
-- ASM Util (from http://asm.ow2.io/, https://repo1.maven.org/maven2/org/ow2/asm/asm-util)
+- ASM Tree (from http://asm.ow2.io/)
+- ASM Util (from http://asm.ow2.io/)
- ASM library repackaged as OSGi bundle (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/asm-all-repackaged)
- Apache Commons Codec (from https://commons.apache.org/proper/commons-codec/)
- AspectJ Weaver (from http://www.aspectj.org, https://www.eclipse.org/aspectj/)
@@ -719,6 +728,7 @@ The following software have components provided under the terms of this license:
- Netty/Codec/HTTP (from https://repo1.maven.org/maven2/io/netty/netty-codec-http)
- Netty/Transport/Classes/KQueue (from https://repo1.maven.org/maven2/io/netty/netty-transport-classes-kqueue)
- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils, https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-utils)
+- PostgreSQL JDBC Driver
- Protocol Buffer Java API (from http://code.google.com/p/protobuf, https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java)
- Protocol Buffers [Util] (from https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java-util)
- Reflections (from http://code.google.com/p/reflections/, http://github.com/ronmamo/reflections)
@@ -811,7 +821,7 @@ CPL-1.0
The following software have components provided under the terms of this license:
- JUnit (from http://junit.org)
-- System Rules (from http://stefanbirkner.github.io/system-rules/)
+- System Rules (from http://stefanbirkner.github.com/system-rules/)
========================================================================
EPL-1.0
@@ -904,8 +914,8 @@ The following software have components provided under the terms of this license:
- ASM library repackaged as OSGi bundle (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/asm-all-repackaged)
- Class Model for Hk2 (from https://repo1.maven.org/maven2/org/glassfish/hk2/class-model)
-- Cobertura (from http://cobertura.sourceforge.net)
- Cobertura Limited Runtime (from http://cobertura.sourceforge.net)
+- Cobertura code coverage (from http://cobertura.sourceforge.net)
- HK2 Implementation Utilities (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-utils)
- HK2 Spring Bridge (from https://repo1.maven.org/maven2/org/glassfish/hk2/spring-bridge)
- HK2 config types (from https://repo1.maven.org/maven2/org/glassfish/hk2/config-types)
@@ -951,7 +961,7 @@ The following software have components provided under the terms of this license:
- Checker Qual (from https://checkerframework.org)
- Class Model for Hk2 (from https://repo1.maven.org/maven2/org/glassfish/hk2/class-model)
-- Cobertura (from http://cobertura.sourceforge.net)
+- Cobertura code coverage (from http://cobertura.sourceforge.net)
- HK2 Implementation Utilities (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-utils)
- HK2 Spring Bridge (from https://repo1.maven.org/maven2/org/glassfish/hk2/spring-bridge)
- HK2 core module (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-core)
@@ -1006,7 +1016,7 @@ LGPL-2.1-only
========================================================================
The following software have components provided under the terms of this license:
-- Cobertura (from http://cobertura.sourceforge.net)
+- Cobertura code coverage (from http://cobertura.sourceforge.net)
- Elastic JNA Distribution (from https://github.com/java-native-access/jna)
- Java Native Access (from https://github.com/java-native-access/jna, https://github.com/twall/jna)
- Java Native Access Platform (from https://github.com/java-native-access/jna)
@@ -1108,7 +1118,7 @@ MPL-1.1
========================================================================
The following software have components provided under the terms of this license:
-- Cobertura (from http://cobertura.sourceforge.net)
+- Cobertura code coverage (from http://cobertura.sourceforge.net)
- Javassist (from http://www.javassist.org/)
- RabbitMQ Java Client (from http://www.rabbitmq.com, https://www.rabbitmq.com)
@@ -1193,6 +1203,7 @@ public-domain
The following software have components provided under the terms of this license:
- AWS SDK for Java - Models (from https://aws.amazon.com/sdkforjava)
+- Apache Groovy (from http://groovy-lang.org, http://groovy.codehaus.org/, https://groovy-lang.org)
- Asynchronous Http Client (from https://repo1.maven.org/maven2/org/asynchttpclient/async-http-client)
- Bouncy Castle PKIX, CMS, EAC, TSP, PKCS, OCSP, CMP, and CRMF APIs (from https://www.bouncycastle.org/java.html)
- Bouncy Castle Provider (from http://www.bouncycastle.org/java.html, https://www.bouncycastle.org/java.html)
@@ -1205,6 +1216,7 @@ The following software have components provided under the terms of this license:
- Microsoft Azure SDK for EventGrid Management (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure client library for Blob Storage (from https://github.com/Azure/azure-sdk-for-java)
+- PostgreSQL JDBC Driver
- Project Lombok (from http://projectlombok.org, https://projectlombok.org)
- RabbitMQ Java Client (from http://www.rabbitmq.com, https://www.rabbitmq.com)
- Spring Security JWT Library (from http://github.com/spring-projects/spring-security-oauth)
@@ -1236,7 +1248,7 @@ The following software have components provided under the terms of this license:
- Jakarta Activation API jar (from https://repo1.maven.org/maven2/jakarta/activation/jakarta.activation-api)
- Jakarta XML Binding API (from https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api, https://repo1.maven.org/maven2/org/jboss/spec/javax/xml/bind/jboss-jaxb-api_2.3_spec)
- Spongy Castle (from http://rtyley.github.io/spongycastle/)
-- System Rules (from http://stefanbirkner.github.io/system-rules/)
+- System Rules (from http://stefanbirkner.github.com/system-rules/)
- jaxen (from http://jaxen.codehaus.org/, https://repo1.maven.org/maven2/jaxen/jaxen)
- jersey-container-servlet (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet)
- jersey-container-servlet-core (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet-core)
diff --git a/devops/gcp/configmap/templates/indexer-variables.yml b/devops/gcp/configmap/templates/indexer-variables.yml
index 3db35220adcd84c70c91f585bce3cd4f33963833..d9b96cdcea4089f64637a1634e3fab3730348adc 100644
--- a/devops/gcp/configmap/templates/indexer-variables.yml
+++ b/devops/gcp/configmap/templates/indexer-variables.yml
@@ -6,23 +6,18 @@ metadata:
name: "{{ .Values.conf.configmap }}"
namespace: "{{ .Release.Namespace }}"
data:
- KEY_RING: "{{ .Values.data.key_ring }}"
- KMS_KEY: "{{ .Values.data.kms_key }}"
LOG_LEVEL: "{{ .Values.data.log_level }}"
SCHEMA_HOST: "{{ .Values.data.schema_host }}"
AUTHORIZE_API: "{{ .Values.data.authorize_api }}"
- GOOGLE_CLOUD_PROJECT: "{{ .Values.data.google_cloud_project }}"
REDIS_SEARCH_HOST: "{{ .Values.data.redis_search_host }}"
REDIS_GROUP_HOST: "{{ .Values.data.redis_group_host }}"
SECURITY_HTTPS_CERTIFICATE_TRUST: "{{ .Values.data.security_https_certificate_trust }}"
STORAGE_QUERY_RECORD_HOST: "{{ .Values.data.storage_query_record_host }}"
STORAGE_SCHEMA_HOST: "{{ .Values.data.storage_schema_host }}"
STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST: "{{ .Values.data.storage_query_record_for_conversion_host }}"
- STORAGE_HOSTNAME: "{{ .Values.data.storage_hostname }}"
STORAGE_RECORDS_BATCH_SIZE: "{{ .Values.data.storage_records_batch_size }}"
STORAGE_QUERY_KINDS_HOST: "{{ .Values.data.storage_query_kinds_host }}"
INDEXER_QUEUE_HOST: "{{ .Values.data.indexer_queue_host }}"
- LEGALTAG_API: "{{ .Values.data.legaltag_api }}"
- CRS_API: "{{ .Values.data.crs_api }}"
PARTITION_API: "{{ .Values.data.partition_api }}"
GOOGLE_AUDIENCES: "{{ .Values.data.google_audiences }}"
+ SPRING_PROFILES_ACTIVE: "{{ .Values.data.spring_profiles_active }}"
diff --git a/devops/gcp/configmap/values.yaml b/devops/gcp/configmap/values.yaml
index a423c75f0818b2e638467c912ba3f83a4c509d37..21247d9775534523bd1a1390374a9eb7d2cf1a0f 100644
--- a/devops/gcp/configmap/values.yaml
+++ b/devops/gcp/configmap/values.yaml
@@ -3,26 +3,21 @@
# Declare variables to be passed into your templates.
data:
- key_ring: "csqp"
- kms_key: "searchService"
log_level: "INFO"
schema_host: ""
authorize_api: "http://entitlements/api/entitlements/v2/"
- google_cloud_project: ""
redis_search_host: ""
redis_group_host: ""
security_https_certificate_trust: "true"
storage_query_record_host: "http://storage/api/storage/v2/query/records"
storage_schema_host: "http://storage/api/storage/v2/schemas"
storage_query_record_for_conversion_host: "http://storage/api/storage/v2/query/records:batch"
- storage_hostname: "storage"
storage_records_batch_size: "20"
storage_query_kinds_host: "http://storage/api/storage/v2/query/kinds"
indexer_queue_host: ""
- legaltag_api: "http://legal/api/legal/v1"
- crs_api: ""
partition_api: "http://partition/api/partition/v1/"
google_audiences: ""
+ spring_profiles_active: "gcp"
conf:
diff --git a/indexer-core/pom.xml b/indexer-core/pom.xml
index ff9f4431d30c5bbffdcfd419d59374ce6a91f78c..84e8a9e69da845786e9df5d6f2ac181c6d5975b6 100644
--- a/indexer-core/pom.xml
+++ b/indexer-core/pom.xml
@@ -16,7 +16,7 @@
<properties>
<commons-beanutils.version>1.9.4</commons-beanutils.version>
- <osdu.oscorecommon.version>0.13.0-rc6</osdu.oscorecommon.version>
+ <osdu.oscorecommon.version>0.13.0</osdu.oscorecommon.version>
<spring-security-web.version>5.4.9</spring-security-web.version>
<gson.version>2.8.9</gson.version>
<netty.version>4.1.70.Final</netty.version>
diff --git a/pom.xml b/pom.xml
index fae8e6ab1723d7b6c0e2a0e8a1baaa8505356b4a..157543a4672741449bc5b59340a52a8423a863ed 100644
--- a/pom.xml
+++ b/pom.xml
@@ -13,7 +13,7 @@
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
<spring-cloud.version>Greenwich.SR2</spring-cloud.version>
- <os-core-common.version>0.13.0-rc6</os-core-common.version>
+ <os-core-common.version>0.13.0</os-core-common.version>
<snakeyaml.version>1.26</snakeyaml.version>
<hibernate-validator.version>6.1.5.Final</hibernate-validator.version>
<jackson.version>2.11.4</jackson.version>
@@ -23,7 +23,7 @@
<netty.version>4.1.51.Final</netty.version>
<reactor-netty.version>0.8.20.RELEASE</reactor-netty.version>
<woodstox-core.version>6.2.3</woodstox-core.version>
- <log4j2.version>2.17.0</log4j2.version>
+ <log4j2.version>2.17.1</log4j2.version>
<spring-boot.version>2.1.18.RELEASE</spring-boot.version>
<json-smart.version>2.4.7</json-smart.version>
<!-- <maven.war.plugin>2.6</maven.war.plugin>-->
diff --git a/provider/indexer-aws/pom.xml b/provider/indexer-aws/pom.xml
index 080a5384b150c05978aec404a204fb1ded56ef17..1a8ba1dba23253591437777f227794591c9b5fb0 100644
--- a/provider/indexer-aws/pom.xml
+++ b/provider/indexer-aws/pom.xml
@@ -68,7 +68,7 @@
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
<artifactId>os-core-lib-aws</artifactId>
- <version>0.13.0-rc3</version>
+ <version>0.13.0</version>
</dependency>
<!-- AWS managed packages -->
diff --git a/provider/indexer-azure/pom.xml b/provider/indexer-azure/pom.xml
index 131a6fc32781fad3ca19a130ecb9a130c8b3619c..fa94c668f3d2f2ccb599785d8fbf347b540bb865 100644
--- a/provider/indexer-azure/pom.xml
+++ b/provider/indexer-azure/pom.xml
@@ -37,12 +37,12 @@
<azure.appservice.plan />
<azure.appservice.appname />
<azure.appservice.subscription />
- <log4j.version>2.17.0</log4j.version>
+ <log4j.version>2.17.1</log4j.version>
<nimbus-jose-jwt.version>8.2</nimbus-jose-jwt.version>
<indexer-core.version>0.14.0-SNAPSHOT</indexer-core.version>
<spring-security-jwt.version>1.1.1.RELEASE</spring-security-jwt.version>
<osdu.corelibazure.version>0.13.0-rc6</osdu.corelibazure.version>
- <osdu.oscorecommon.version>0.13.0-rc6</osdu.oscorecommon.version>
+ <osdu.oscorecommon.version>0.13.0</osdu.oscorecommon.version>
<reactor-netty.version>0.9.12.RELEASE</reactor-netty.version>
<java-jwt.version>3.8.1</java-jwt.version>
<powermock.version>2.0.2</powermock.version>
diff --git a/provider/indexer-gcp/README.md b/provider/indexer-gcp/README.md
index 49996510dd57276dc47416eb09d17907d16a2491..c4ea917bc3ffe5d2887485c75dd0bd0b3542bed3 100644
--- a/provider/indexer-gcp/README.md
+++ b/provider/indexer-gcp/README.md
@@ -1,44 +1,49 @@
# Indexer Service
os-indexer-gcp is a [Spring Boot](https://spring.io/projects/spring-boot) service that is responsible for indexing Records that enable the `os-search` service to execute OSDU R2 domain searches against Elasticsearch.
+## Table of Contents <a name="TOC"></a>
+* [Getting started](#Getting-started)
+* [Mappers](#Mappers)
+* [Settings and Configuration](#Settings-and-Configuration)
+* [Run service](#Run-service)
+* [Testing](#Testing)
+* [Deployment](#Deployment)
+* [Entitlements groups](#Entitlements-groups)
+* [Licence](#License)
+
## Getting Started
These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system.
-### Prerequisites
-Pre-requisites
+## Mappers
-* GCloud SDK with java (latest version)
-* JDK 8
-* Lombok 1.16 or later
-* Maven
+This is a universal solution created using EPAM OQM mappers technology. It allows you to work with various
+implementations of message brokers.
-### Installation
-In order to run the service locally or remotely, you will need to have the following environment variables defined.
+For more information about mappers:
+- [OQM Readme](https://community.opengroup.org/osdu/platform/system/lib/cloud/gcp/oqm/-/blob/master/README.md)
-| name | value | description | sensitive? | source |
-| --- | --- | --- | --- | --- |
-| `LOG_PREFIX` | `service` | Logging prefix | no | - |
-| `SERVER_SERVLET_CONTEXPATH` | `/api/indexer/v2` | Servlet context path | no | - |
-| `AUTHORIZE_API` | ex `https://entitlements.com/entitlements/v1` | Entitlements API endpoint | no | output of infrastructure deployment |
-| `LEGALTAG_API` | ex `https://legal.com/api/legal/v1` | Legal API endpoint | no | output of infrastructure deployment |
-| `INDEXER_QUEUE_HOST` | ex `https://os-indexer-queue-dot-opendes.appspot.com/_dps/task-handlers/enqueue` | Indexer-Queue API endpoint | no | output of infrastructure deployment |
-| `CRS_API` | ex `https://crs-converter-gae-dot-opendes.appspot.com/api/crs/v1` | CRS API endpoint | no | https://console.cloud.google.com/memorystore/redis/instances |
-| `STORAGE_HOSTNAME` | ex `os-storage-dot-opendes.appspot.com` | Storage Host | no | output of infrastructure deployment |
-| `STORAGE_SCHEMA_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/schemas` | Storage API endpoint 'schemas' | no | https://console.cloud.google.com/apis/credentials |
-| `STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records:batch` | Storage API endpoint 'records' | no | https://console.cloud.google.com/iam-admin/serviceaccounts |
-| `STORAGE_QUERY_RECORD_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records` | Storage API endpoint 'query/records' | no | https://console.cloud.google.com/iam-admin/serviceaccounts |
-| `STORAGE_RECORDS_BATCH_SIZE` | 20 | Batch size for storage API `POST {endpoint}/query/records:batch` | no | - |
-| `STORAGE_RECORDS_BY_KIND_BATCH_SIZE` | - | Batch size for storage API `GET {endpoint}/query/records`. If this is not present, defaults to value of `STORAGE_RECORDS_BATCH_SIZE` | no | - |
-| `REDIS_SEARCH_HOST` | ex `127.0.0.1` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances |
-| `REDIS_GROUP_HOST` | ex `127.0.0.1` | Redis host for groups | no | https://console.cloud.google.com/memorystore/redis/instances |
-| `REDIS_SEARCH_PORT` | ex `6379` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances |
-| `GOOGLE_CLOUD_PROJECT` | ex `opendes` | Google Cloud Project Id| no | output of infrastructure deployment |
-| `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials |
-| `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
-| `security.https.certificate.trust` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment |
-| `indexer.que.service.mail` | ex `default@iam.gserviceaccount.com` | Indexer Que environment service account mail, required if Indexer Que deployed in cloud task mode, to validate token from it | yes | - |
-| `SCHEMA_HOST` | ex `https://os-schema-dot-opendes.appspot.com/api/schema-service/v1/schema` | Schema API endpoint | no | output of infrastructure deployment |
-| `PARTITION_API` | ex `https://localhost:8081/api/partition/v1` | Partition API endpoint | no | output of infrastructure deployment |
+### Limitations of the current version
+
+In the current version, the mappers are equipped with several drivers to the stores and the message broker:
+
+- OQM (mapper to message brokers): Google PubSub; RabbitMQ
+
+## Settings and Configuration
+
+### Prerequisites
+1. Mandatory
+ - JDK 8
+ - Lombok 1.16 or later
+ - Maven
+2. For Google Cloud only
+ - GCloud SDK with java (latest version)
+
+### Anthos Service Configuration:
+[Anthos service configuration ](docs/anthos/README.md)
+### GCP Service Configuration:
+[Gcp service configuration ](docs/gcp/README.md)
+
+## Run service
### Run Locally
Check that maven is installed:
@@ -183,26 +188,6 @@ $ (cd testing/indexer-test-gcp/ && mvn clean test)
or
* Google Documentation: https://cloud.google.com/cloud-build/docs/deploying-builds/deploy-appengine
-#### Cloud KMS Setup
-
-Enable cloud KMS on master project
-
-Create king ring and key in the ***master project***
-
-```bash
- gcloud services enable cloudkms.googleapis.com
- export KEYRING_NAME="csqp"
- export CRYPTOKEY_NAME="searchService"
- gcloud kms keyrings create $KEYRING_NAME --location global
- gcloud kms keys create $CRYPTOKEY_NAME --location global \
- --keyring $KEYRING_NAME \
- --purpose encryption
-```
-
-Add **Cloud KMS CryptoKey Encrypter/Decrypter** role to the **default service account** of the ***master project*** through IAM - Role tab
-
-Add **Cloud KMS Encrypt/Decrypt** role to the **default service account** of ***master project*** through IAM - Role tab
-
#### Memory Store (Redis Instance) Setup
Create a new Standard tier Redis instance on the ***service project***
@@ -213,6 +198,14 @@ The Redis instance must be created under the same region with the App Engine app
gcloud beta redis instances create redis-cache-search --size=10 --region=<service-deployment-region> --zone=<service-deployment-zone> --tier=STANDARD
```
+## Entitlements groups
+Storage service account should have entitlements groups listed below:
+- service.entitlements.user
+- users
+- service.storage.viewer
+- service.schema-service.viewers
+- data.default.viewers
+
## Licence
Copyright © Google LLC
Copyright © EPAM Systems
diff --git a/provider/indexer-gcp/docs/anthos/README.md b/provider/indexer-gcp/docs/anthos/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..90cb1a8b9ed4e39f7e9f80be0ad42524b77d7ace
--- /dev/null
+++ b/provider/indexer-gcp/docs/anthos/README.md
@@ -0,0 +1,188 @@
+## Service Configuration for Anthos
+
+## Environment variables:
+
+Define the following environment variables.
+
+Must have:
+
+| name | value | description | sensitive? | source |
+| --- | --- | --- | --- | --- |
+| `SPRING_PROFILES_ACTIVE` | ex `anthos` | Spring profile that activate default configuration for GCP environment | false | - |
+| `OPENID_PROVIDER_CLIENT_ID` | `*****` | Client id that represents this service and serves to request tokens, example `workload-identity-legal` |yes| - |
+| `OPENID_PROVIDER_CLIENT_SECRET` | `*****` | This client secret that serves to request tokens| yes | - |
+| `OPENID_PROVIDER_URL` | `https://keycloack.com/auth/realms/master` | URL of OpenID Connect provider, it will be used as `<OpenID URL> + /.well-known/openid-configuration` to auto configure endpoint for token request | no | - |
+
+Defined in default application property file but possible to override:
+
+| name | value | description | sensitive? | source |
+| --- | --- | --- | --- | --- |
+| `LOG_PREFIX` | `service` | Logging prefix | no | - |
+| `LOG_LEVEL` | `****` | Logging level | no | - |
+| `SECURITY_HTTPS_CERTIFICATE_TRUST` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment |
+| `REDIS_SEARCH_HOST` | ex `127.0.0.1` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances |
+| `REDIS_SEARCH_PORT` | ex `6379` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances |
+| `REDIS_GROUP_HOST` | ex `127.0.0.1` | Redis host for groups | no | https://console.cloud.google.com/memorystore/redis/instances |
+| `REDIS_GROUP_PORT` | ex `6379` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances |
+| `PARTITION_HOST` | ex `https://partition.com` | Partition host | no | output of infrastructure deployment |
+| `ENTITLEMENTS_HOST` | ex `https://entitlements.com` | Entitlements host | no | output of infrastructure deployment |
+| `STORAGE_HOST` | ex `https://storage.com` | Storage host | no | output of infrastructure deployment |
+| `INDEXER_QUEUE_HOST` | ex `http://indexer-queue` | Indexer-Queue host | no | output of infrastructure deployment |
+| `SCHEMA_BASE_HOST` | ex `https://schema.com` | Schema service host | no | output of infrastructure deployment |
+
+These variables define service behavior, and are used to switch between `anthos` or `gcp` environments, their overriding and usage in mixed mode was not tested.
+Usage of spring profiles is preferred.
+
+| name | value | description | sensitive? | source |
+| --- | --- | --- | --- | --- |
+| `PARTITION_AUTH_ENABLED` | ex `true` or `false` | Disable or enable auth token provisioning for requests to Partition service | no | - |
+| `OQMDRIVER` | `rabbitmq` or `pubsub` | Oqm driver mode that defines which message broker will be used | no | - |
+| `SERVICE_TOKEN_PROVIDER` | `GCP` or `OPENID` |Service account token provider, `GCP` means use Google service account `OPEIND` means use OpenId provider like `Keycloak` | no | - |
+
+## Elasticsearch configuration
+
+### Properties set in Partition service:
+
+**prefix:** `elasticsearch`
+
+It can be overridden by:
+
+- through the Spring Boot property `elastic-search-properties-prefix`
+- environment variable `ELASTIC_SEARCH_PROPERTIES_PREFIX`
+
+**Propertyset:**
+
+| Property | Description |
+| --- | --- |
+| elasticsearch.host | server URL |
+| elasticsearch.port | server port |
+| elasticsearch.configuration | username and password |
+
+<details><summary>Example of a definition for a single tenant</summary></details>
+
+```
+
+curl -L -X PATCH 'http://partition.com/api/partition/v1/partitions/opendes' -H 'data-partition-id: opendes' -H 'Authorization: Bearer ...' -H 'Content-Type: application/json' --data-raw '{
+ "properties": {
+ "elasticsearch.host": {
+ "sensitive": false,
+ "value": "elastic.us-central1.gcp.cloud.es.io"
+ },
+ "elasticsearch.port": {
+ "sensitive": false,
+ "value": "9243"
+ },
+ "elasticsearch.configuration": {
+ "sensitive": true,
+ "value": "elasticuser:elasticpassword"
+ }
+ }
+}'
+
+```
+
+### For OQM RabbitMQ
+
+**prefix:** `oqm.rabbitmq`
+It can be overridden by:
+
+- through the Spring Boot property `oqm.rabbitmq.partition-properties-prefix`
+- environment variable `OQM_RABBITMQ_PARTITION_PROPERTIES_PREFIX``
+
+**Propertyset** (for two types of connection: messaging and admin operations):
+
+| Property | Description |
+| --- | --- |
+| oqm.rabbitmq.amqp.host | messaging hostnameorIP |
+| oqm.rabbitmq.amqp.port | - port |
+| oqm.rabbitmq.amqp.path | - path |
+| oqm.rabbitmq.amqp.username | - username |
+| oqm.rabbitmq.amqp.password | - password |
+| oqm.rabbitmq.admin.schema | admin host schema |
+| oqm.rabbitmq.admin.host | - host name |
+| oqm.rabbitmq.admin.port | - port |
+| oqm.rabbitmq.admin.path | - path |
+| oqm.rabbitmq.admin.username | - username |
+| oqm.rabbitmq.admin.password | - password |
+
+<details><summary>Example of a single tenant definition</summary>
+
+```
+
+curl -L -X PATCH 'https://dev.osdu.club/api/partition/v1/partitions/opendes' -H 'data-partition-id: opendes' -H 'Authorization: Bearer ...' -H 'Content-Type: application/json' --data-raw '{
+ "properties": {
+ "oqm.rabbitmq.amqp.host": {
+ "sensitive": false,
+ "value": "localhost"
+ },
+ "oqm.rabbitmq.amqp.port": {
+ "sensitive": false,
+ "value": "5672"
+ },
+ "oqm.rabbitmq.amqp.path": {
+ "sensitive": false,
+ "value": ""
+ },
+ "oqm.rabbitmq.amqp.username": {
+ "sensitive": false,
+ "value": "guest"
+ },
+ "oqm.rabbitmq.amqp.password": {
+ "sensitive": true,
+ "value": "guest"
+ },
+
+ "oqm.rabbitmq.admin.schema": {
+ "sensitive": false,
+ "value": "http"
+ },
+ "oqm.rabbitmq.admin.host": {
+ "sensitive": false,
+ "value": "localhost"
+ },
+ "oqm.rabbitmq.admin.port": {
+ "sensitive": false,
+ "value": "9002"
+ },
+ "oqm.rabbitmq.admin.path": {
+ "sensitive": false,
+ "value": "/api"
+ },
+ "oqm.rabbitmq.admin.username": {
+ "sensitive": false,
+ "value": "guest"
+ },
+ "oqm.rabbitmq.admin.password": {
+ "sensitive": true,
+ "value": "guest"
+ }
+ }
+}'
+
+```
+
+</details>
+
+#### Exchanges and queues configuration
+
+At RabbitMq should be created exchange with name:
+
+**name:** `indexing-progress`
+
+
+
+## Keycloak configuration
+
+[Keycloak service accounts setup](https://www.keycloak.org/docs/latest/server_admin/#_service_accounts)
+
+Configure Clients. One Client per OSDU service. Set them “confidentialâ€.
+
+
+
+Each Client has embedded Service Account (SA) option. Enable SAs for Clients, make “Authorization enabledâ€:
+
+
+
+Add `partition-and-entitlements` scope to `Default Client Scopes` and generate Keys.
+
+Give `client-id` and `client-secret` to services, which should be authorized within the platform.
\ No newline at end of file
diff --git a/provider/indexer-gcp/docs/anthos/pics/client.png b/provider/indexer-gcp/docs/anthos/pics/client.png
new file mode 100644
index 0000000000000000000000000000000000000000..8a2014e672b65ddf644558e2c9e8cb8823f549dd
Binary files /dev/null and b/provider/indexer-gcp/docs/anthos/pics/client.png differ
diff --git a/provider/indexer-gcp/docs/anthos/pics/rabbit.PNG b/provider/indexer-gcp/docs/anthos/pics/rabbit.PNG
new file mode 100644
index 0000000000000000000000000000000000000000..e1c67734b91a536659fdf0b940a93208d749e3c8
Binary files /dev/null and b/provider/indexer-gcp/docs/anthos/pics/rabbit.PNG differ
diff --git a/provider/indexer-gcp/docs/anthos/pics/sa.png b/provider/indexer-gcp/docs/anthos/pics/sa.png
new file mode 100644
index 0000000000000000000000000000000000000000..b9c8124047b5834a2cf79baecbf751dc106feeff
Binary files /dev/null and b/provider/indexer-gcp/docs/anthos/pics/sa.png differ
diff --git a/provider/indexer-gcp/docs/gcp/README.md b/provider/indexer-gcp/docs/gcp/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..494169b1634af2e093c39038688987e78de6d726
--- /dev/null
+++ b/provider/indexer-gcp/docs/gcp/README.md
@@ -0,0 +1,94 @@
+## Service Configuration for GCP
+
+## Environment variables:
+
+Define the following environment variables.
+
+Must have:
+
+| name | value | description | sensitive? | source |
+| --- | --- | --- | --- | --- |
+| `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials |
+| `SPRING_PROFILES_ACTIVE` | ex `gcp` | Spring profile that activate default configuration for GCP environment | false | - |
+
+Defined in default application property file but possible to override:
+
+| name | value | description | sensitive? | source |
+| --- | --- | --- | --- | --- |
+| `LOG_PREFIX` | `service` | Logging prefix | no | - |
+| `LOG_LEVEL` | `****` | Logging level | no | - |
+| `SECURITY_HTTPS_CERTIFICATE_TRUST` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment |
+| `REDIS_SEARCH_HOST` | ex `127.0.0.1` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances |
+| `REDIS_SEARCH_PORT` | ex `6379` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances |
+| `REDIS_GROUP_HOST` | ex `127.0.0.1` | Redis host for groups | no | https://console.cloud.google.com/memorystore/redis/instances |
+| `REDIS_GROUP_PORT` | ex `6379` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances |
+| `PARTITION_HOST` | ex `https://partition.com` | Partition host | no | output of infrastructure deployment |
+| `ENTITLEMENTS_HOST` | ex `https://entitlements.com` | Entitlements host | no | output of infrastructure deployment |
+| `STORAGE_HOST` | ex `https://storage.com` | Storage host | no | output of infrastructure deployment |
+| `INDEXER_QUEUE_HOST` | ex `http://indexer-queue` | Indexer-Queue host | no | output of infrastructure deployment |
+| `SCHEMA_BASE_HOST` | ex `https://schema.com` | Schema service host | no | output of infrastructure deployment |
+| `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
+
+These variables define service behavior, and are used to switch between `anthos` or `gcp` environments, their overriding and usage in mixed mode was not tested.
+Usage of spring profiles is preferred.
+
+| name | value | description | sensitive? | source |
+| --- | --- | --- | --- | --- |
+| `PARTITION_AUTH_ENABLED` | ex `true` or `false` | Disable or enable auth token provisioning for requests to Partition service | no | - |
+| `OQMDRIVER` | `rabbitmq` or `pubsub` | Oqm driver mode that defines which message broker will be used | no | - |
+| `SERVICE_TOKEN_PROVIDER` | `GCP` or `OPENID` |Service account token provider, `GCP` means use Google service account `OPEIND` means use OpenId provider like `Keycloak` | no | - |
+
+## Pubsub configuration:
+
+At Pubsub should be created topic with name:
+
+**name:** `indexing-progress`
+
+## Elasticsearch configuration
+
+### Properties set in Partition service:
+
+**prefix:** `elasticsearch`
+
+It can be overridden by:
+
+- through the Spring Boot property `elastic-search-properties-prefix`
+- environment variable `ELASTIC_SEARCH_PROPERTIES_PREFIX`
+
+**Propertyset:**
+
+| Property | Description |
+| --- | --- |
+| elasticsearch.host | server URL |
+| elasticsearch.port | server port |
+| elasticsearch.configuration | username and password |
+
+<details><summary>Example of a definition for a single tenant</summary></details>
+
+```
+
+curl -L -X PATCH 'http://partition.com/api/partition/v1/partitions/opendes' -H 'data-partition-id: opendes' -H 'Authorization: Bearer ...' -H 'Content-Type: application/json' --data-raw '{
+ "properties": {
+ "elasticsearch.host": {
+ "sensitive": false,
+ "value": "elastic.us-central1.gcp.cloud.es.io"
+ },
+ "elasticsearch.port": {
+ "sensitive": false,
+ "value": "9243"
+ },
+ "elasticsearch.configuration": {
+ "sensitive": true,
+ "value": "elasticuser:elasticpassword"
+ }
+ }
+}'
+
+```
+
+## Google cloud service account configuration :
+TBD
+
+| Required roles |
+| --- |
+| - |
diff --git a/provider/indexer-gcp/pom.xml b/provider/indexer-gcp/pom.xml
index 71fcc527f0b52f88be1e31f7a5f6bfa2976462d5..3f7bb4987478522f50a8d113ae6094c2997c4583 100644
--- a/provider/indexer-gcp/pom.xml
+++ b/provider/indexer-gcp/pom.xml
@@ -19,7 +19,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-gcp</artifactId>
- <version>0.13.0-rc4</version>
+ <version>0.14.0-rc1</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
@@ -179,6 +179,4 @@
</plugin>
</plugins>
</build>
-
-
</project>
diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java
deleted file mode 100644
index 6a3b330aad0dcc5a7ba09687a82e0c5e1f53548c..0000000000000000000000000000000000000000
--- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java
+++ /dev/null
@@ -1,30 +0,0 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.cache;
-
-import com.google.auth.oauth2.AccessToken;
-import org.opengroup.osdu.core.common.cache.RedisCache;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-@Component
-public class DatastoreCredentialCache extends RedisCache<String, AccessToken> {
-
- @Autowired
- public DatastoreCredentialCache(final IndexerConfigurationProperties configurationProperties) {
- super(configurationProperties.getRedisSearchHost(), Integer.parseInt(configurationProperties.getRedisSearchPort()), 58 * 60, String.class, AccessToken.class);
- }
-}
\ No newline at end of file
diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java
index 206787c8d06dd825dabf1b165c9c2f52f742a735..83ee124a964a148dcaba28b7b0e88d7c7a111b2f 100644
--- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java
+++ b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java
@@ -18,29 +18,23 @@
package org.opengroup.osdu.indexer.cache;
import com.google.gson.Gson;
-import java.io.IOException;
import java.util.Objects;
import javax.inject.Inject;
-import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.cache.RedisCache;
-import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.search.ClusterSettings;
import org.opengroup.osdu.core.common.provider.interfaces.IElasticCredentialsCache;
-import org.opengroup.osdu.core.common.provider.interfaces.IKmsClient;
import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
import org.springframework.stereotype.Component;
@Component
public class ElasticCredentialsCache implements IElasticCredentialsCache<String, ClusterSettings>, AutoCloseable {
- private IKmsClient kmsClient;
private RedisCache<String, String> cache;
@Inject
- public ElasticCredentialsCache(final IndexerConfigurationProperties properties, final IKmsClient kmsClient) {
+ public ElasticCredentialsCache(final IndexerConfigurationProperties properties) {
this.cache = new RedisCache<>(properties.getRedisSearchHost(), Integer.parseInt(properties.getRedisSearchPort()),
properties.getElasticCacheExpiration() * 60, String.class, String.class);
- this.kmsClient = kmsClient;
}
@Override
@@ -50,27 +44,17 @@ public class ElasticCredentialsCache implements IElasticCredentialsCache<String,
@Override
public void put(String s, ClusterSettings o) {
- try {
- String jsonSettings = new Gson().toJson(o);
- String encryptString = kmsClient.encryptString(jsonSettings);
- this.cache.put(s, encryptString);
- } catch (IOException e) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Internal server error", "Unable to encrypt settings before being cached", e);
- }
+ String jsonSettings = new Gson().toJson(o);
+ this.cache.put(s, jsonSettings);
}
@Override
public ClusterSettings get(String s) {
- try {
- String encryptedSettings = this.cache.get(s);
- if (Objects.isNull(encryptedSettings) || encryptedSettings.isEmpty()) {
- return null;
- }
- String jsonSettings = this.kmsClient.decryptString(encryptedSettings);
- return new Gson().fromJson(jsonSettings, ClusterSettings.class);
- } catch (IOException e) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Internal server error", "Unable to decrypt settings from cache", e);
+ String jsonSettings = this.cache.get(s);
+ if (Objects.isNull(jsonSettings) || jsonSettings.isEmpty()) {
+ return null;
}
+ return new Gson().fromJson(jsonSettings, ClusterSettings.class);
}
@Override
diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/DatastoreCredentialsCacheFactory.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/DatastoreCredentialsCacheFactory.java
deleted file mode 100644
index 6f4d9b7505504f259ceca3d3b8e1834dc53f91ba..0000000000000000000000000000000000000000
--- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/DatastoreCredentialsCacheFactory.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- Copyright 2020 Google LLC
- Copyright 2020 EPAM Systems, Inc
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.di;
-
-import org.opengroup.osdu.core.common.cache.ICache;
-import org.opengroup.osdu.core.common.cache.VmCache;
-import org.opengroup.osdu.core.gcp.multitenancy.credentials.DatastoreCredential;
-import org.springframework.beans.factory.config.AbstractFactoryBean;
-import org.springframework.stereotype.Component;
-
-@Component
-public class DatastoreCredentialsCacheFactory extends
- AbstractFactoryBean<ICache<String, DatastoreCredential>> {
-
- @Override
- public Class<?> getObjectType() {
- return ICache.class;
- }
-
- @Override
- protected ICache<String, DatastoreCredential> createInstance() throws Exception {
- return new VmCache<>(5 * 60, 20);
- }
-}
diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/ElasticSearchConfig.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/ElasticSearchConfig.java
new file mode 100644
index 0000000000000000000000000000000000000000..72cb1f6a3a1cedfc3f408853d99d10a9c80cca74
--- /dev/null
+++ b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/ElasticSearchConfig.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2020-2022 Google LLC
+ * Copyright 2020-2022 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.di;
+
+import org.opengroup.osdu.core.common.partition.IPartitionProvider;
+import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository;
+import org.opengroup.osdu.core.destination.elastic.ElasticSearchDestinationResolver;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class ElasticSearchConfig {
+
+ @Bean
+ public IElasticRepository elasticRepository(ElasticSearchConfigurationProperties properties, IPartitionProvider partitionProvider) {
+ return new ElasticSearchDestinationResolver(properties.getElasticsearchPropertiesPrefix(), partitionProvider);
+ }
+}
diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/ElasticSearchConfigurationProperties.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/ElasticSearchConfigurationProperties.java
new file mode 100644
index 0000000000000000000000000000000000000000..586ce1ab9e9127282066a2e0fddf651836d12f18
--- /dev/null
+++ b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/ElasticSearchConfigurationProperties.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2020-2022 Google LLC
+ * Copyright 2020-2022 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.di;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@ConfigurationProperties
+@Getter
+@Setter
+public class ElasticSearchConfigurationProperties {
+
+ private String elasticsearchPropertiesPrefix = "elasticsearch";
+}
diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/PubSubExtensionsFactory.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/PubSubExtensionsFactory.java
deleted file mode 100644
index c0995677c674dc1192b4c9533903454af441cfeb..0000000000000000000000000000000000000000
--- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/PubSubExtensionsFactory.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- Copyright 2020 Google LLC
- Copyright 2020 EPAM Systems, Inc
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.di;
-
-import org.opengroup.osdu.core.gcp.PubSub.PubSubExtensions;
-import org.springframework.beans.factory.config.AbstractFactoryBean;
-import org.springframework.stereotype.Component;
-import org.springframework.web.context.annotation.RequestScope;
-
-@Component
-@RequestScope
-public class PubSubExtensionsFactory extends AbstractFactoryBean<PubSubExtensions> {
-
-
- @Override
- public Class<?> getObjectType() {
- return PubSubExtensions.class;
- }
-
- @Override
- protected PubSubExtensions createInstance() throws Exception {
- return new PubSubExtensions();
- }
-}
diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryService.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryService.java
deleted file mode 100644
index fa51ef212c0427bd21e4a522a99891d38163ccc4..0000000000000000000000000000000000000000
--- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryService.java
+++ /dev/null
@@ -1,38 +0,0 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.di;
-
-import lombok.extern.java.Log;
-import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
-import org.opengroup.osdu.core.gcp.multitenancy.TenantFactory;
-import org.springframework.beans.factory.config.AbstractFactoryBean;
-import org.springframework.stereotype.Component;
-import org.springframework.web.context.annotation.RequestScope;
-
-@Log
-@Component
-@RequestScope
-public class TenantFactoryService extends AbstractFactoryBean<ITenantFactory> {
-
- @Override
- protected ITenantFactory createInstance() throws Exception {
- return new TenantFactory();
- }
-
- @Override
- public Class<?> getObjectType() {
- return ITenantFactory.class;
- }
-}
\ No newline at end of file
diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java
deleted file mode 100644
index 9a6dc9ef7e3920668aa2155c1c39dc8531ed3529..0000000000000000000000000000000000000000
--- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.persistence;
-
-import com.google.api.gax.retrying.RetrySettings;
-import com.google.cloud.TransportOptions;
-import com.google.cloud.datastore.Datastore;
-import com.google.cloud.datastore.DatastoreOptions;
-import com.google.cloud.http.HttpTransportOptions;
-import java.util.HashMap;
-import java.util.Map;
-import javax.inject.Inject;
-import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
-import org.opengroup.osdu.core.gcp.multitenancy.credentials.DatastoreCredential;
-import org.opengroup.osdu.indexer.cache.DatastoreCredentialCache;
-import org.springframework.stereotype.Component;
-import org.threeten.bp.Duration;
-
-@Component("indexerDatastoreFactory")
-public class DatastoreFactory {
-
- @Inject
- private DatastoreCredentialCache cache;
-
- private static Map<String, Datastore> datastoreClients = new HashMap<>();
-
- private static final RetrySettings RETRY_SETTINGS = RetrySettings.newBuilder()
- .setMaxAttempts(6)
- .setInitialRetryDelay(Duration.ofSeconds(10))
- .setMaxRetryDelay(Duration.ofSeconds(32))
- .setRetryDelayMultiplier(2.0)
- .setTotalTimeout(Duration.ofSeconds(50))
- .setInitialRpcTimeout(Duration.ofSeconds(50))
- .setRpcTimeoutMultiplier(1.0)
- .setMaxRpcTimeout(Duration.ofSeconds(50))
- .build();
-
- private static final TransportOptions TRANSPORT_OPTIONS = HttpTransportOptions.newBuilder()
- .setReadTimeout(30000)
- .build();
-
- public Datastore getDatastoreInstance(TenantInfo tenantInfo) {
- if (datastoreClients.get(tenantInfo.getName()) == null) {
- Datastore googleDatastore = DatastoreOptions.newBuilder()
- .setCredentials(new DatastoreCredential(tenantInfo))
- .setRetrySettings(RETRY_SETTINGS)
- .setTransportOptions(TRANSPORT_OPTIONS)
- .setNamespace(tenantInfo.getName())
- .setProjectId(tenantInfo.getProjectId())
- .build().getService();
- datastoreClients.put(tenantInfo.getName(), googleDatastore);
- }
- return datastoreClients.get(tenantInfo.getName());
- }
-}
diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryDatastore.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryDatastore.java
deleted file mode 100644
index a8e3373f44926527a271d4d4390b5f4d5e647177..0000000000000000000000000000000000000000
--- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryDatastore.java
+++ /dev/null
@@ -1,89 +0,0 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.persistence;
-
-import com.google.api.client.googleapis.json.GoogleJsonResponseException;
-import com.google.cloud.datastore.Datastore;
-import com.google.cloud.datastore.Entity;
-import com.google.cloud.datastore.Key;
-import lombok.extern.java.Log;
-import org.apache.http.HttpStatus;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.search.ClusterSettings;
-import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
-import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository;
-import org.opengroup.osdu.core.common.provider.interfaces.IKmsClient;
-import org.opengroup.osdu.core.common.search.Preconditions;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.springframework.stereotype.Component;
-
-import javax.inject.Inject;
-
-@Log
-@Component
-public class ElasticRepositoryDatastore implements IElasticRepository {
-
- static final String HOST = "host";
- static final String PORT = "port";
- static final String XPACK_RESTCLIENT_CONFIGURATION = "configuration";
-
- @Inject
- private IKmsClient kmsClient;
- @Inject
- private DatastoreFactory datastoreFactory;
-
- @Inject
- private IndexerConfigurationProperties properties;
-
-
- @Override
- public ClusterSettings getElasticClusterSettings(TenantInfo tenantInfo) {
-
- Datastore googleDatastore = this.datastoreFactory.getDatastoreInstance(tenantInfo);
- Key key = googleDatastore.newKeyFactory().setKind(properties.getElasticDatastoreKind()).newKey(properties.getElasticDatastoreId());
- Entity datastoreEntity = googleDatastore.get(key);
-
- if (datastoreEntity == null) {
- throw new AppException(HttpStatus.SC_NOT_FOUND, "Cluster setting not found", "The requested cluster setting was not found in datastore.", String.format("Cluster setting with key: '%s' does not exist in datastore.", key.getName()));
- }
-
- String encryptedHost = null;
- String encryptedPort = null;
- String encryptedConfiguration = null;
-
- try {
- encryptedHost = datastoreEntity.getString(HOST);
- encryptedPort = datastoreEntity.getString(PORT);
- encryptedConfiguration = datastoreEntity.getString(XPACK_RESTCLIENT_CONFIGURATION);
-
- String host = this.kmsClient.decryptString(encryptedHost);
- String portString = this.kmsClient.decryptString(encryptedPort);
- String usernameAndPassword = this.kmsClient.decryptString(encryptedConfiguration);
-
- Preconditions.checkNotNullOrEmpty(host, "host cannot be null");
- Preconditions.checkNotNullOrEmpty(portString, "port cannot be null");
- Preconditions.checkNotNullOrEmpty(usernameAndPassword, "configuration cannot be null");
-
- int port = Integer.parseInt(portString);
-
- return new ClusterSettings(host, port, usernameAndPassword);
- } catch (GoogleJsonResponseException e) {
- String debuggingInfo = String.format("Host: %s | port: %s | configuration: %s", encryptedHost, encryptedPort, encryptedConfiguration);
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Cluster setting decryption error", "An error has occurred decrypting cluster settings.", debuggingInfo, e);
- } catch (Exception e) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Cluster setting fetch error", "An error has occurred fetching cluster settings from the datastore.", e);
- }
- }
-}
\ No newline at end of file
diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java
index 31d242cd46515f9b37907e458b510dd7a37557bd..2a35a96f5684f9e0c1227d549c0419da067c8118 100644
--- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java
+++ b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java
@@ -14,129 +14,58 @@
package org.opengroup.osdu.indexer.publish;
-import com.google.api.gax.retrying.RetrySettings;
-import com.google.cloud.pubsub.v1.Publisher;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonElement;
-import com.google.gson.reflect.TypeToken;
-import com.google.protobuf.ByteString;
-import com.google.pubsub.v1.ProjectTopicName;
-import com.google.pubsub.v1.PubsubMessage;
+import java.util.HashMap;
+import java.util.Map;
+import lombok.RequiredArgsConstructor;
import lombok.extern.java.Log;
-import org.apache.http.HttpStatus;
-import org.elasticsearch.common.Strings;
-import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.indexer.JobStatus;
-import org.opengroup.osdu.core.common.model.indexer.RecordStatus;
-import org.opengroup.osdu.core.common.model.search.DeploymentEnvironment;
-import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
-import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
-import org.opengroup.osdu.core.gcp.PubSub.PubSubExtensions;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.opengroup.osdu.core.gcp.oqm.driver.OqmDriver;
+import org.opengroup.osdu.core.gcp.oqm.model.OqmDestination;
+import org.opengroup.osdu.core.gcp.oqm.model.OqmMessage;
+import org.opengroup.osdu.core.gcp.oqm.model.OqmTopic;
import org.opengroup.osdu.indexer.provider.interfaces.IPublisher;
import org.springframework.stereotype.Component;
import org.springframework.web.context.annotation.RequestScope;
-import org.threeten.bp.Duration;
-
-import javax.inject.Inject;
-import java.io.IOException;
-import java.lang.reflect.Type;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
@Log
@Component
@RequestScope
+@RequiredArgsConstructor
public class PublisherImpl implements IPublisher {
- private static final Map<String, Publisher> PUBSUB_CLIENTS = new HashMap<>();
-
private static final String TOPIC_ID = "indexing-progress";
+ private final OqmDriver driver;
- @Inject
- private ITenantFactory tenantStorageFactory;
-
- @Inject
- private PubSubExtensions pubSubExtensions;
-
- @Inject
- private IndexerConfigurationProperties properties;
+ private final OqmTopic oqmTopic = OqmTopic.builder().name(TOPIC_ID).build();
@Override
- public void publishStatusChangedTagsToTopic(DpsHeaders headers, JobStatus indexerBatchStatus) throws Exception {
-
- // Don't publish to pubsub when testing locally
- if (properties.getDeploymentEnvironment() == DeploymentEnvironment.LOCAL) {
- return;
- }
-
- String tenant = headers.getPartitionId();
- if(Strings.isNullOrEmpty(tenant))
- tenant = headers.getAccountId();
-
- Publisher publisher = this.getPublisher(tenant);
- if (publisher == null) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Internal error", "A fatal internal error occurred creating publisher client.");
- }
- PubsubMessage pubsubMessage = getPubsubMessage(headers, indexerBatchStatus);
-
- pubSubExtensions.publishAndCreateTopicIfNotExist(publisher, pubsubMessage);
-
- }
-
- private static final RetrySettings RETRY_SETTINGS = RetrySettings.newBuilder()
- .setTotalTimeout(Duration.ofSeconds(30))
- .setInitialRetryDelay(Duration.ofSeconds(2))
- .setRetryDelayMultiplier(2)
- .setMaxRetryDelay(Duration.ofSeconds(5))
- .setInitialRpcTimeout(Duration.ofSeconds(10))
- .setRpcTimeoutMultiplier(2)
- .setMaxRpcTimeout(Duration.ofSeconds(10))
+ public void publishStatusChangedTagsToTopic(DpsHeaders headers, JobStatus indexerBatchStatus) {
+ OqmDestination oqmDestination = OqmDestination.builder().partitionId(headers.getPartitionId())
.build();
+ String json = generatePubSubMessage(indexerBatchStatus);
-
- private PubsubMessage getPubsubMessage(DpsHeaders headers, JobStatus indexerBatchStatus) {
-
- Gson gson = new GsonBuilder().create();
- Type listType = new TypeToken<List<RecordStatus>>() {}.getType();
- JsonElement statusChangedTagsJson = gson.toJsonTree(indexerBatchStatus.getStatusesList(), listType);
- ByteString statusChangedTagsData = ByteString.copyFromUtf8(statusChangedTagsJson.toString());
-
- PubsubMessage.Builder builder = PubsubMessage.newBuilder();
- String tenant = headers.getPartitionId();
- //This code it to provide backward compatibility to slb-account-id
- if(!Strings.isNullOrEmpty(tenant)) {
- builder.putAttributes(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionId());
- } else {
- builder.putAttributes(DpsHeaders.ACCOUNT_ID, headers.getAccountId());
- }
-
- builder.putAttributes(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
- // TODO: uncomment when deploying to production
-// builder.putAttributes( AppEngineHeaders.CLOUD_TRACE_CONTEXT, headers.getHeaders().get(AppEngineHeaders.CLOUD_TRACE_CONTEXT));
- builder.setData(statusChangedTagsData);
-
- return builder.build();
+ Map<String, String> attributes = getAttributes(headers);
+ OqmMessage oqmMessage = OqmMessage.builder().data(json).attributes(attributes).build();
+ driver.publish(oqmMessage, oqmTopic, oqmDestination);
}
- private Publisher getPublisher(String tenantName) throws IOException {
- TenantInfo info = this.tenantStorageFactory.getTenantInfo(tenantName);
- if (info == null) {
- return null;
- } else {
- if (PUBSUB_CLIENTS.containsKey(tenantName)) return PUBSUB_CLIENTS.get(tenantName);
-
- ProjectTopicName topicName = ProjectTopicName.newBuilder().setProject(info.getProjectId()).setTopic(TOPIC_ID).build();
- Publisher publisher = Publisher.newBuilder(topicName).setRetrySettings(RETRY_SETTINGS).build();
-
- if (publisher == null) return null;
+ private Map<String, String> getAttributes(DpsHeaders headers) {
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(DpsHeaders.ACCOUNT_ID, headers.getPartitionIdWithFallbackToAccountId());
+ attributes.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
+ headers.addCorrelationIdIfMissing();
+ attributes.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
+ return attributes;
+ }
- PUBSUB_CLIENTS.put(tenantName, publisher);
- return publisher;
- }
+ private String generatePubSubMessage(JobStatus jobStatus) {
+ Gson gson = new GsonBuilder().create();
+ JsonElement statusChangedTagsJson = gson.toJsonTree(jobStatus, JobStatus.class);
+ return statusChangedTagsJson.toString();
}
}
\ No newline at end of file
diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/security/GcpSecurityConfig.java
similarity index 95%
rename from provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java
rename to provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/security/GcpSecurityConfig.java
index 0f70a40bc444ae9fd7e1553c319684df1286273b..47b42552ccdef36a19103027cdde783bd4d50359 100644
--- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java
+++ b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/security/GcpSecurityConfig.java
@@ -27,7 +27,7 @@ import org.springframework.security.config.annotation.web.configuration.WebSecur
@Configuration
@EnableWebSecurity
@EnableGlobalMethodSecurity(prePostEnabled = true)
-public class GSuiteSecurityConfig extends WebSecurityConfigurerAdapter {
+public class GcpSecurityConfig extends WebSecurityConfigurerAdapter {
@Override
protected void configure(HttpSecurity http) throws Exception {
diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java
index 9950bb31d3eb14da96c466aeb283eea7b2f2110b..360f576f68021a2fdb398db4d5c368284069bc2e 100644
--- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java
+++ b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java
@@ -25,6 +25,7 @@ import java.util.logging.Level;
import javax.inject.Inject;
import lombok.extern.java.Log;
import org.apache.http.HttpStatus;
+import org.opengroup.osdu.core.auth.TokenProvider;
import org.opengroup.osdu.core.common.Constants;
import org.opengroup.osdu.core.common.model.entitlements.AuthorizationResponse;
import org.opengroup.osdu.core.common.model.http.AppException;
@@ -34,7 +35,6 @@ import org.opengroup.osdu.core.common.model.search.SearchServiceRole;
import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
import org.opengroup.osdu.core.common.provider.interfaces.IAuthorizationService;
import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
-import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient;
import org.opengroup.osdu.core.gcp.model.CloudTaskHeaders;
import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
import org.springframework.stereotype.Component;
@@ -50,7 +50,7 @@ public class RequestInfoImpl implements IRequestInfo {
private DpsHeaders dpsHeaders;
@Inject
- private IServiceAccountJwtClient serviceAccountJwtClient;
+ private TokenProvider tokenProvider;
@Inject
private TenantInfo tenantInfo;
@@ -99,12 +99,12 @@ public class RequestInfoImpl implements IRequestInfo {
@Override
public boolean isTaskQueueRequest() {
- if(this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.CLOUD_TASK_QUEUE_NAME)){
- log.log(Level.INFO,"Request confirmed as cloud task, token validation in progress");
+ if (this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.CLOUD_TASK_QUEUE_NAME)) {
+ log.log(Level.INFO, "Request confirmed as cloud task, token validation in progress");
return isCloudTaskRequest();
}
- if(this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.APPENGINE_TASK_QUEUE_NAME)){
- log.log(Level.INFO,"Request confirmed as AppEngine, headers validation in progress");
+ if (this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.APPENGINE_TASK_QUEUE_NAME)) {
+ log.log(Level.INFO, "Request confirmed as AppEngine, headers validation in progress");
return isAppEngineTaskRequest();
}
return false;
@@ -116,7 +116,7 @@ public class RequestInfoImpl implements IRequestInfo {
return true;
}
- private boolean isAppEngineTaskRequest(){
+ private boolean isAppEngineTaskRequest() {
if (!this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.APPENGINE_TASK_QUEUE_NAME)) {
return false;
}
@@ -136,7 +136,7 @@ public class RequestInfoImpl implements IRequestInfo {
}
return authHeader;
} else {
- return "Bearer " + this.serviceAccountJwtClient.getIdToken(tenantInfo.getName());
+ return "Bearer " + this.tokenProvider.getIdToken();
}
}
}
diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java
deleted file mode 100644
index 6c57527219c328d75ab5acaffbee7d85f7c558e8..0000000000000000000000000000000000000000
--- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java
+++ /dev/null
@@ -1,93 +0,0 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.util;
-
-import com.auth0.jwt.JWT;
-import com.google.cloud.iam.credentials.v1.GenerateIdTokenResponse;
-import com.google.cloud.iam.credentials.v1.IamCredentialsClient;
-import com.google.cloud.iam.credentials.v1.ServiceAccountName;
-import java.util.Collections;
-import javax.inject.Inject;
-import org.apache.http.HttpStatus;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.search.IdToken;
-import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
-import org.opengroup.osdu.core.common.provider.interfaces.IJwtCache;
-import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
-import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient;
-import org.opengroup.osdu.core.gcp.multitenancy.credentials.IamCredentialsProvider;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.context.annotation.Primary;
-import org.springframework.stereotype.Component;
-import org.springframework.web.context.annotation.RequestScope;
-
-@Primary
-@Component
-@RequestScope
-public class ServiceAccountJwtGcpClientImpl implements IServiceAccountJwtClient {
-
- private static final String SERVICE_ACCOUNT_NAME_FORMAT = "projects/-/serviceAccounts/%s";
-
- private final IamCredentialsProvider iamCredentialsProvider = new IamCredentialsProvider();
-
- @Value("${GOOGLE_AUDIENCES}")
- private String audiences;
-
- @Inject
- private ITenantFactory tenantInfoServiceProvider;
- @Inject
- private IJwtCache cacheService;
- @Inject
- private JaxRsDpsLog log;
- @Inject
- private DpsHeaders dpsHeaders;
- @Inject
- private IndexerConfigurationProperties properties;
-
- public String getIdToken(String tenantName) {
- this.log.info("Tenant name received for auth token is: " + tenantName);
- TenantInfo tenant = this.tenantInfoServiceProvider.getTenantInfo(tenantName);
- if (tenant == null) {
- this.log.error("Invalid tenant name receiving from pubsub");
- throw new AppException(HttpStatus.SC_BAD_REQUEST, "Invalid tenant Name", "Invalid tenant Name from pubsub");
- }
- try {
-
- IdToken cachedToken = (IdToken) this.cacheService.get(tenant.getServiceAccount());
- // Add the user to DpsHeaders directly
- this.dpsHeaders.put(DpsHeaders.USER_EMAIL, tenant.getServiceAccount());
-
- if (!IdToken.refreshToken(cachedToken)) {
- return cachedToken.getTokenValue();
- }
-
- try (IamCredentialsClient iamCredentialsClient = iamCredentialsProvider.getIamCredentialsClient()) {
- ServiceAccountName serviceAccountName = ServiceAccountName.parse(String.format(SERVICE_ACCOUNT_NAME_FORMAT, tenant.getServiceAccount()));
- GenerateIdTokenResponse idTokenResponse = iamCredentialsClient.generateIdToken(serviceAccountName, Collections.emptyList(), audiences, true);
- String token = idTokenResponse.getToken();
- IdToken idToken = IdToken.builder().tokenValue(token).expirationTimeMillis(JWT.decode(token).getExpiresAt().getTime()).build();
- this.cacheService.put(tenant.getServiceAccount(), idToken);
- return token;
- }
- } catch (AppException e) {
- throw e;
- } catch (Exception e) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Persistence error", "Error generating token", e);
- }
- }
-}
diff --git a/provider/indexer-gcp/src/main/resources/application-anthos.properties b/provider/indexer-gcp/src/main/resources/application-anthos.properties
new file mode 100644
index 0000000000000000000000000000000000000000..9e553661404fae2ae946a221d506b5b56d16f8a0
--- /dev/null
+++ b/provider/indexer-gcp/src/main/resources/application-anthos.properties
@@ -0,0 +1,6 @@
+oqmDriver=rabbitmq
+service.token.provider=OPENID
+partition-auth-enabled=false
+openid.provider-url=
+openid.provider-client-id=
+openid.provider-client-secret=
\ No newline at end of file
diff --git a/provider/indexer-gcp/src/main/resources/application-dev.properties b/provider/indexer-gcp/src/main/resources/application-dev.properties
deleted file mode 100644
index 6c49ae23e07fddb7668a52d645d70268579ce9a9..0000000000000000000000000000000000000000
--- a/provider/indexer-gcp/src/main/resources/application-dev.properties
+++ /dev/null
@@ -1,21 +0,0 @@
-google-cloud-project=opendes
-
-indexer-host=indexer-jvmvia5dea-uc.a.run.app
-STORAGE_HOSTNAME=storage-jvmvia5dea-uc.a.run.app
-
-storage-schema-host=http://127.0.0.1:8081/api/storage/v2/schemas
-storage-query-record-host=http://127.0.0.1:8081/api/storage/v2/query/records
-storage-query-record-for-conversion-host=https://storage-jvmvia5dea-uc.a.run.app/api/storage/v2/query/records:batch
-storage-records-batch-size=20
-
-indexer-queue-host=https://indexer-queue-jvmvia5dea-uc.a.run.app/_dps/task-handlers/enqueue
-
-AUTHORIZE_API=https://os-entitlements-gcp-jvmvia5dea-uc.a.run.app/entitlements/v1
-LEGALTAG_API=https://os-legal-jvmvia5dea-uc.a.run.app/api/legal/v1
-CRS_API=https://crs-converter-jvmvia5dea-uc.a.run.app/api/crs/v1
-
-## use below values for gcp: opendes
-REDIS_GROUP_HOST=127.0.0.1
-redis-search-host=127.0.0.1
-
-google-audiences=519000754840-09v7ssbpku7sevhvbtq3bdoi70es16p5.apps.googleusercontent.com
\ No newline at end of file
diff --git a/provider/indexer-gcp/src/main/resources/application-gcp.properties b/provider/indexer-gcp/src/main/resources/application-gcp.properties
new file mode 100644
index 0000000000000000000000000000000000000000..fd738d7f484bf44fc3b2f2043191259dd0b3a1b8
--- /dev/null
+++ b/provider/indexer-gcp/src/main/resources/application-gcp.properties
@@ -0,0 +1,3 @@
+oqmDriver=pubsub
+service.token.provider=GCP
+partition-auth-enabled=true
\ No newline at end of file
diff --git a/provider/indexer-gcp/src/main/resources/application-kuber.properties b/provider/indexer-gcp/src/main/resources/application-kuber.properties
deleted file mode 100644
index 22d3ba0253112794113a96f90fa094f5468902e1..0000000000000000000000000000000000000000
--- a/provider/indexer-gcp/src/main/resources/application-kuber.properties
+++ /dev/null
@@ -1,24 +0,0 @@
-google-cloud-project=${GOOGLE_CLOUD_PROJECT}
-
-indexer-host=os-indexer-service
-STORAGE_HOSTNAME=os-storage-service
-
-storage-schema-host=http://os-storage-service/api/storage/v2/schemas
-storage-query-record-host=http://os-storage-service/api/storage/v2/query/records
-storage-query-record-for-conversion-host=http://os-storage-service/api/storage/v2/query/records:batch
-storage-records-batch-size=20
-
-indexer-queue-host=http://os-storage-service/_dps/task-handlers/enqueue
-
-AUTHORIZE_API=http://os-entitlement-service/entitlements/v1
-LEGALTAG_API=http://os-legal-service/api/legal/v1
-CRS_API=${CRS_API}
-
-## use below values for gcp: opendes
-REDIS_GROUP_HOST=${REDIS_GROUP_HOST}
-redis-search-host=${REDIS_SEARCH_HOST}
-
-google-audiences=${GOOGLE_AUDIENCES}
-
-deployment-environment=cloud
-disable.appengine.log.factory=true
\ No newline at end of file
diff --git a/provider/indexer-gcp/src/main/resources/application-local.properties b/provider/indexer-gcp/src/main/resources/application-local.properties
deleted file mode 100644
index a5511b15c0564250bca41b67c6c39646f9512e79..0000000000000000000000000000000000000000
--- a/provider/indexer-gcp/src/main/resources/application-local.properties
+++ /dev/null
@@ -1,21 +0,0 @@
-google-cloud-project=osdu
-
-indexer-host=os-indexer-dot-opendes.appspot.com
-STORAGE_HOSTNAME=os-storage-dot-opendes.appspot.com
-
-storage-schema-host=https://os-storage-dot-opendes.appspot.com/api/storage/v2/schemas
-storage-query-record-host=https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records
-storage-query-record-for-conversion-host=https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records:batch
-storage-records-batch-size=20
-
-indexer-queue-host=https://os-indexer-queue-dot-opendes.appspot.com/_dps/task-handlers/enqueue
-
-AUTHORIZE_API=https://entitlements-dot-opendes.appspot.com/entitlements/v1
-LEGALTAG_API=https://os-legal-dot-opendes.appspot.com/api/legal/v1
-CRS_API=https://crs-converter-gae-dot-opendes.appspot.com/api/crs/v1
-
-## use below values for gcp: opendes
-REDIS_GROUP_HOST=127.0.0.1
-redis-search-host=127.0.0.1
-
-google-audiences=245464679631-ktfdfpl147m1mjpbutl00b3cmffissgq.apps.googleusercontent.com
diff --git a/provider/indexer-gcp/src/main/resources/application-testing.properties b/provider/indexer-gcp/src/main/resources/application-testing.properties
deleted file mode 100644
index 67941ce47b5fd27c99e487fa59c935d0bcacd827..0000000000000000000000000000000000000000
--- a/provider/indexer-gcp/src/main/resources/application-testing.properties
+++ /dev/null
@@ -1,21 +0,0 @@
-google-cloud-project=opendes-evt
-
-indexer-host=os-indexer-dot-opendes-evt.appspot.com
-STORAGE_HOSTNAME=os-storage-dot-opendes-evt.appspot.com
-
-storage-schema-host=https://os-storage-dot-opendes-evt.appspot.com/api/storage/v2/schemas
-storage-query-record-host=https://os-storage-dot-opendes-evt.appspot.com/api/storage/v2/query/records
-storage-query-record-for-conversion-host=https://os-storage-dot-opendes-evt.appspot.com/api/storage/v2/query/records:batch
-storage-records-batch-size=20
-
-indexer-queue-host=https://os-indexer-queue-dot-opendes-evt.appspot.com/_dps/task-handlers/enqueue
-
-AUTHORIZE_API=https://entitlements-dot-opendes-evt.appspot.com/entitlements/v1
-LEGALTAG_API=https://os-legal-dot-opendes-evt.appspot.com/api/legal/v1
-CRS_API=https://crs-converter-gae-dot-opendes-evt.appspot.com/api/crs/v1
-
-## use below values for gcp: opendes
-REDIS_GROUP_HOST=10.253.209.196
-redis-search-host=10.118.2.140
-
-google-audiences=833591776864-oobhqvmtdg9rpreubjvn44m5f8revglk.apps.googleusercontent.com
\ No newline at end of file
diff --git a/provider/indexer-gcp/src/main/resources/application.properties b/provider/indexer-gcp/src/main/resources/application.properties
index e18440679858d1208f1f6f683220e78a7e372978..f8ffcf5aea0314dd8077f87179ceac92a173160d 100644
--- a/provider/indexer-gcp/src/main/resources/application.properties
+++ b/provider/indexer-gcp/src/main/resources/application.properties
@@ -1,18 +1,15 @@
-LOG_PREFIX=indexer
-
+#server config
+server.port=8080
server.servlet.contextPath=/api/indexer/v2
+
+#log config
+LOG_PREFIX=indexer
logging.level.org.springframework.web=${LOG_LEVEL:DEBUG}
-server.port=8080
+
+#jvm config
JAVA_OPTS=-Xms3072m -Xmx3072m
JAVA_GC_OPTS=-XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45
-deployment-environment=CLOUD
-
-REDIS_GROUP_PORT=6379
-DEFAULT_DATA_COUNTRY=US
-
-redis-search-port=6379
-
#Default Cache Settings
schema-cache-expiration=60
index-cache-expiration=60
@@ -22,26 +19,41 @@ cursor-cache-expiration=60
kinds-cache-expiration=2880
# Attributes Cache expiration 2*24*60
attributes-cache-expiration=2880
-
kinds-redis-database=1
cron-index-cleanup-threshold-days=3
cron-empty-index-cleanup-threshold-days=7
+#indexer service config
+DEFAULT_DATA_COUNTRY=US
gae-service=indexer
-key-ring=csqp
-kms-key=searchService
+security.https.certificate.trust=false
+storage-records-by-kind-batch-size=20
+storage-records-batch-size=20
-KEY_RING=${key-ring}
-KMS_KEY=${kms-key}
-GOOGLE_CLOUD_PROJECT=${google-cloud-project}
+REDIS_GROUP_PORT=6379
+REDIS_GROUP_HOST=redis-cache-groups
-elastic-datastore-kind=SearchSettings
-elastic-datastore-id=indexer-service
+REDIS_SEARCH_PORT=6379
+REDIS_SEARCH_HOST=redis-cache-search
-security.https.certificate.trust=false
-indexer.que.service.mail=default@iam.gserviceaccount.com
-SCHEMA_HOST=${HOST}/api/schema-service/v1/schema
-storage-query-kinds-host=https://${STORAGE_HOSTNAME}/api/storage/v2/query/kinds
+PARTITION_HOST=http://partition
+PARTITION_PATH=/api/partition/v1
+PARTITION_API=${PARTITION_HOST}${PARTITION_PATH}
+
+ENTITLEMENTS_HOST=http://entitlements
+ENTITLEMENTS_PATH=/api/entitlements/v2
+AUTHORIZE_API=${ENTITLEMENTS_HOST}${ENTITLEMENTS_PATH}
+
+STORAGE_HOST=http://storage
+STORAGE_PATH=/api/storage/v2
+STORAGE_API=${STORAGE_HOST}${STORAGE_PATH}
+STORAGE_QUERY_KINDS_HOST=${STORAGE_API}/query/kinds
+STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=${STORAGE_API}/query/records:batch
+STORAGE_QUERY_RECORD_HOST=${STORAGE_API}/query/records
+STORAGE_SCHEMA_HOST=${STORAGE_API}/schemas
+
+SCHEMA_BASE_HOST=http://schema
+SCHEMA_PATH=/api/schema-service/v1/schema
+SCHEMA_HOST=${SCHEMA_BASE_HOST}${SCHEMA_PATH}
-google.audiences=123.apps.googleusercontent.com
-partition.api=http://localhost:8081/api/partition/v1
\ No newline at end of file
+INDEXER_QUEUE_HOST=http://indexer-queue
\ No newline at end of file
diff --git a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceTest.java b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceTest.java
deleted file mode 100644
index c716cb1a637f5a5d843ebccd73656f2fa1c0752f..0000000000000000000000000000000000000000
--- a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceTest.java
+++ /dev/null
@@ -1,337 +0,0 @@
-package org.opengroup.osdu.indexer.service;//// Copyright 2017-2019, Schlumberger
-////
-//// Licensed under the Apache License, Version 2.0 (the "License");
-//// you may not use this file except in compliance with the License.
-//// You may obtain a copy of the License at
-////
-//// http://www.apache.org/licenses/LICENSE-2.0
-////
-//// Unless required by applicable law or agreed to in writing, software
-//// distributed under the License is distributed on an "AS IS" BASIS,
-//// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//// See the License for the specific language governing permissions and
-//// limitations under the License.
-//
-//package org.opendes.indexer.service;
-//
-//import com.google.gson.Gson;
-//import com.google.gson.reflect.TypeToken;
-//import org.elasticsearch.action.bulk.BulkItemResponse;
-//import org.elasticsearch.action.bulk.BulkResponse;
-//import org.elasticsearch.client.RequestOptions;
-//import org.elasticsearch.client.RestHighLevelClient;
-//import org.junit.Before;
-//import org.junit.Ignore;
-//import org.junit.Test;
-//import org.junit.runner.RunWith;
-//import org.mockito.InjectMocks;
-//import org.mockito.Mock;
-//import org.mockito.Spy;
-//import org.opendes.client.api.DpsHeaders;
-//import org.opendes.core.logging.JaxRsDpsLog;
-//import org.opendes.core.model.DeploymentEnvironment;
-//import org.opendes.core.model.RecordChangedMessages;
-//import org.opendes.core.service.IndicesService;
-//import org.opendes.core.util.Config;
-//import org.opendes.core.util.ElasticClientHandler;
-//import org.opendes.core.util.ElasticIndexNameResolver;
-//import org.opendes.core.util.HeadersUtil;
-//import org.opendes.indexer.logging.AuditLogger;
-//import org.opendes.indexer.model.*;
-//import org.opendes.indexer.publish.IPublisher;
-//import org.opendes.indexer.util.IRequestInfo;
-//import org.opendes.indexer.util.IndexerQueueTaskBuilder;
-//import org.opendes.indexer.util.JobStatus;
-//import org.opendes.indexer.util.RecordInfo;
-//import org.powermock.core.classloader.annotations.PrepareForTest;
-//import javax.inject.Inject;
-//import org.springframework.context.annotation.Lazy;
-//import org.springframework.test.context.junit4.SpringRunner;
-//
-//import java.io.IOException;
-//import java.lang.reflect.Type;
-//import java.util.*;
-//
-//import static java.util.Collections.singletonList;
-//import static org.junit.Assert.*;
-//import static org.mockito.Matchers.any;
-//import static org.mockito.Mockito.verify;
-//import static org.mockito.Mockito.when;
-//import static org.powermock.api.mockito.PowerMockito.mock;
-//import static org.powermock.api.mockito.PowerMockito.mockStatic;
-//
-//@Ignore
-//@RunWith(SpringRunner.class)
-//@PrepareForTest({RestHighLevelClient.class, BulkResponse.class, StorageAcl.class, HeadersUtil.class, Config.class})
-//public class IndexerServiceTest {
-//
-// private final String pubsubMsg = "[{\"id\":\"tenant1:doc:test1\",\"kind\":\"tenant1:testindexer1:well:1.0.0\",\"op\":\"update\"}," +
-// "{\"id\":\"tenant1:doc:test2\",\"kind\":\"tenant1:testindexer2:well:1.0.0\",\"op\":\"create\"}]";
-// private final String kind1 = "tenant1:testindexer1:well:1.0.0";
-// private final String kind2 = "tenant1:testindexer2:well:1.0.0";
-// private final String recordId1 = "tenant1:doc:test1";
-// private final String recordId2 = "tenant1:doc:test2";
-// private final String failureMassage = "test failure";
-//
-// @Mock
-// private IndexSchemaService indexSchemaService;
-// @Mock
-// private IndicesService indicesService;
-// @Mock
-// private IndexerMappingService indexerMappingService;
-// @Mock
-// private StorageService storageService;
-// @Mock
-// private IPublisher publisherImpl;
-// @Mock
-// private RestHighLevelClient restHighLevelClient;
-// @Mock
-// private ElasticClientHandler elasticClientHandler;
-// @Mock
-// private BulkResponse bulkResponse;
-// @Mock
-// private IRequestInfo requestInfo;
-// @Mock
-// private ElasticIndexNameResolver elasticIndexNameResolver;
-// @Mock
-// private AttributeParsingServiceImpl attributeParsingServiceImpl;
-// @Mock
-// private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
-// @Mock
-// private JaxRsDpsLog log;
-// @Mock
-// private AuditLogger auditLogger;
-// @InjectMocks
-// private IndexerServiceImpl sut;
-// @InjectMocks @Spy
-// private JobStatus jobStatus = new JobStatus();
-//
-// @Inject
-// @Lazy
-// private DpsHeaders dpsHeaders;
-// private RecordChangedMessages recordChangedMessages;
-// private List<RecordInfo> recordInfos;
-//
-// @Before
-// public void setup() throws IOException {
-//
-// mockStatic(StorageAcl.class);
-// mockStatic(Config.class);
-//
-// when(Config.getDeploymentEnvironment()).thenReturn(DeploymentEnvironment.LOCAL);
-// when(Config.getElasticClusterName()).thenReturn("CLUSTER");
-// when(Config.getElasticServerAddress()).thenReturn("testsite");
-//
-// dpsHeaders = new DpsHeaders();
-// dpsHeaders.put(AppEngineHeaders.TASK_QUEUE_RETRY_COUNT, "1");
-// dpsHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
-// when(requestInfo.getHeaders()).thenReturn(dpsHeaders);
-// when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(dpsHeaders.getHeaders());
-//
-// Type listType = new TypeToken<List<RecordInfo>>() {}.getType();
-// recordInfos = (new Gson()).fromJson(pubsubMsg, listType);
-//
-// when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
-// when(restHighLevelClient.bulk(any(), any(RequestOptions.class))).thenReturn(bulkResponse);
-//
-// BulkItemResponse[] responses = new BulkItemResponse[]{prepareResponseFail(), prepareResponseSuccess()};
-// when(bulkResponse.getItems()).thenReturn(responses);
-// Map<String, String> attr = new HashMap<>();
-// attr.put(DpsHeaders.ACCOUNT_ID, "slb");
-// recordChangedMessages = RecordChangedMessages.builder().attributes(attr).messageId("xxxx").publishTime("2000-01-02T10:10:44+0000").data("{}").build();
-// when(StorageAcl.flattenAcl(any())).thenReturn(null);
-// }
-//
-// @Test
-// public void should_returnNull_givenEmptyJobSubInfo_processRecordChangedMessageTest() throws Exception {
-// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, new ArrayList<>());
-//
-// assertNull(jobStatus);
-// }
-//
-// @Test
-// public void should_returnValidJobStatus_givenNullSchema_processRecordChangedMessageTest() {
-// try {
-// indexSchemaServiceMock(kind1, null);
-// indexSchemaServiceMock(kind2, null);
-// List<ConversionStatus> conversionStatus = new LinkedList<>();
-// List<Records.Entity> validRecords = new ArrayList<>();
-// Map<String, Object> storageData = new HashMap<>();
-// storageData.put("schema1", "test-value");
-// storageData.put("schema2", "test-value");
-// storageData.put("schema3", "test-value");
-// storageData.put("schema4", "test-value");
-// storageData.put("schema5", "test-value");
-// storageData.put("schema6", "test-value");
-// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build());
-// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build();
-//
-// when(storageService.getStorageRecords(any())).thenReturn(storageRecords);
-// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true);
-//
-// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
-//
-// assertEquals(2, jobStatus.getStatusesList().size());
-// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
-// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).size());
-// } catch (Exception e) {
-// fail("Should not throw this exception" + e.getMessage());
-// }
-// }
-//
-// @Test
-// public void should_returnValidJobStatus_givenFailedUnitsConversion_processRecordChangedMessageTest() {
-// try {
-// indexSchemaServiceMock(kind1, null);
-// indexSchemaServiceMock(kind2, null);
-// List<ConversionStatus> conversionStatuses = new LinkedList<>();
-// List<String> status=new ArrayList<>();
-// status.add("crs bla bla");
-// ConversionStatus conversionStatus=ConversionStatus.builder().status("ERROR").errors(status).id(recordId2).build();
-// conversionStatuses.add(conversionStatus);
-// List<Records.Entity> validRecords = new ArrayList<>();
-// Map<String, Object> storageData = new HashMap<>();
-// storageData.put("schema1", "test-value");
-// storageData.put("schema2", "test-value");
-// storageData.put("schema3", "test-value");
-// storageData.put("schema4", "test-value");
-// storageData.put("schema5", "test-value");
-// storageData.put("schema6", "test-value");
-// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build());
-// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatuses).build();
-//
-// when(storageService.getStorageRecords(any())).thenReturn(storageRecords);
-// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true);
-//
-// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
-//
-// assertEquals(2, jobStatus.getStatusesList().size());
-// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
-// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).size());
-// assertTrue(jobStatus.getJobStatusByRecordId(jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).get(0)).getIndexProgress().getTrace().contains("crs bla bla"));
-// } catch (Exception e) {
-// fail("Should not throw this exception" + e.getMessage());
-// }
-// }
-//
-// @Test
-// public void should_returnValidJobStatus_givenNullSchemaForARecord_processRecordChangedMessageTest() {
-// try {
-// List<Records.Entity> validRecords = new ArrayList<>();
-// List<ConversionStatus> conversionStatus = new LinkedList<>();
-// Map<String, Object> storageData = new HashMap<>();
-// storageData.put("schema1", "test-value");
-// storageData.put("schema2", "test-value");
-// storageData.put("schema3", "test-value");
-// storageData.put("schema4", "test-value");
-// storageData.put("schema5", "test-value");
-// storageData.put("schema6", "test-value");
-// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build());
-// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build();
-// when(storageService.getStorageRecords(any())).thenReturn(storageRecords);
-//
-// Map<String, String> schema = createSchema();
-// indexSchemaServiceMock(kind1, schema);
-// indexSchemaServiceMock(kind2, null);
-// when(elasticIndexNameResolver.getIndexNameFromKind(kind2)).thenReturn("tenant1-testindexer2-well-1.0.0");
-// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true);
-// JobStatus jobStatus = sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
-//
-// assertEquals(2, jobStatus.getStatusesList().size());
-// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
-// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).size());
-// assertEquals("Indexed Successfully", jobStatus.getStatusesList().get(1).getIndexProgress().getTrace().pop());
-// assertEquals("schema not found", jobStatus.getStatusesList().get(1).getIndexProgress().getTrace().pop());
-// } catch (Exception e) {
-// fail("Should not throw this exception" + e.getMessage());
-// }
-// }
-//
-// @Test
-// public void should_returnValidJobStatus_givenValidCreateAndUpdateRecords_processRecordChangedMessagesTest() {
-// try {
-// Map<String, Object> storageData = new HashMap<>();
-// storageData.put("schema1", "test-value");
-// List<ConversionStatus> conversionStatus = new LinkedList<>();
-// List<Records.Entity> validRecords = new ArrayList<>();
-// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build());
-// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build();
-//
-// when(storageService.getStorageRecords(any())).thenReturn(storageRecords);
-// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true);
-// Map<String, String> schema = createSchema();
-// indexSchemaServiceMock(kind2, schema);
-// indexSchemaServiceMock(kind1, null);
-// JobStatus jobStatus = sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
-//
-// assertEquals(2, jobStatus.getStatusesList().size());
-// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
-// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.SUCCESS).size());
-// } catch (Exception e) {
-// fail("Should not throw this exception" + e.getMessage());
-// }
-// }
-//
-// @Test
-// public void should_properlyUpdateAuditLogs_givenValidCreateAndUpdateRecords() {
-// try {
-// Map<String, Object> storageData = new HashMap<>();
-// List<ConversionStatus> conversionStatus = new LinkedList<>();
-//
-// storageData.put("schema1", "test-value");
-// List<Records.Entity> validRecords = new ArrayList<>();
-// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build());
-// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build();
-//
-// when(this.storageService.getStorageRecords(any())).thenReturn(storageRecords);
-// when(this.indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true);
-// Map<String, String> schema = createSchema();
-// indexSchemaServiceMock(kind2, schema);
-// indexSchemaServiceMock(kind1, null);
-// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
-//
-// assertEquals(2, jobStatus.getStatusesList().size());
-// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
-// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.SUCCESS).size());
-//
-// verify(this.auditLogger).indexCreateRecordSuccess(singletonList("RecordStatus(id=tenant1:doc:test2, kind=tenant1:testindexer2:well:1.0.0, operationType=create, status=SUCCESS)"));
-// verify(this.auditLogger).indexUpdateRecordFail(singletonList("RecordStatus(id=tenant1:doc:test1, kind=tenant1:testindexer1:well:1.0.0, operationType=update, status=FAIL)"));
-// } catch (Exception e) {
-// fail("Should not throw this exception" + e.getMessage());
-// }
-// }
-//
-// private BulkItemResponse prepareResponseFail() {
-// BulkItemResponse responseFail = mock(BulkItemResponse.class);
-// when(responseFail.isFailed()).thenReturn(true);
-// when(responseFail.getFailureMessage()).thenReturn(failureMassage);
-// when(responseFail.getId()).thenReturn(recordId1);
-// when(responseFail.getFailure()).thenReturn(new BulkItemResponse.Failure("failure index", "failure type", "failure id", new Exception("test failure")));
-// return responseFail;
-// }
-//
-// private BulkItemResponse prepareResponseSuccess() {
-// BulkItemResponse responseSuccess = mock(BulkItemResponse.class);
-// when(responseSuccess.getId()).thenReturn(recordId2);
-// return responseSuccess;
-// }
-//
-// private void indexSchemaServiceMock(String kind, Map<String, String> schema) {
-// if (schema == null) {
-// IndexSchema indexSchema = IndexSchema.builder().kind(kind).dataSchema(null).build();
-// when(indexSchemaService.getIndexerInputSchema(kind)).thenReturn(indexSchema);
-// } else {
-// IndexSchema indexSchema = IndexSchema.builder().kind(kind).dataSchema(schema).build();
-// when(indexSchemaService.getIndexerInputSchema(kind)).thenReturn(indexSchema);
-// }
-// }
-//
-// private Map<String, String> createSchema() {
-// Map<String, String> schema = new HashMap<>();
-// schema.put("schema1", "keyword");
-// schema.put("schema2", "boolean");
-// schema.put("schema3", "date");
-// schema.put("schema6", "object");
-// return schema;
-// }
-//}
diff --git a/provider/indexer-ibm/pom.xml b/provider/indexer-ibm/pom.xml
index 58ec11957a5948fbeddede08e79c388ca9c4d95f..378bf7dace7c028b53b55110d81fced5a63d35e3 100644
--- a/provider/indexer-ibm/pom.xml
+++ b/provider/indexer-ibm/pom.xml
@@ -31,7 +31,7 @@
<packaging>jar</packaging>
<properties>
- <os-core-lib-ibm.version>0.13.0-rc6</os-core-lib-ibm.version>
+ <os-core-lib-ibm.version>0.13.0</os-core-lib-ibm.version>
</properties>
<profiles>
diff --git a/testing/indexer-test-ibm/pom.xml b/testing/indexer-test-ibm/pom.xml
index fc48b0c9e03091a576d04730701fe914e01db78b..4531fdba9cfa2a310062e6161c6a32b359282d7f 100644
--- a/testing/indexer-test-ibm/pom.xml
+++ b/testing/indexer-test-ibm/pom.xml
@@ -19,7 +19,7 @@
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
<cucumber.version>1.2.5</cucumber.version>
- <os-core-lib-ibm.version>0.13.0-rc5</os-core-lib-ibm.version>
+ <os-core-lib-ibm.version>0.13.0</os-core-lib-ibm.version>
</properties>
<dependencies>