diff --git a/devops/gcp/configmap/templates/indexer-variables.yml b/devops/gcp/configmap/templates/indexer-variables.yml
index 3db35220adcd84c70c91f585bce3cd4f33963833..d9b96cdcea4089f64637a1634e3fab3730348adc 100644
--- a/devops/gcp/configmap/templates/indexer-variables.yml
+++ b/devops/gcp/configmap/templates/indexer-variables.yml
@@ -6,23 +6,18 @@ metadata:
name: "{{ .Values.conf.configmap }}"
namespace: "{{ .Release.Namespace }}"
data:
- KEY_RING: "{{ .Values.data.key_ring }}"
- KMS_KEY: "{{ .Values.data.kms_key }}"
LOG_LEVEL: "{{ .Values.data.log_level }}"
SCHEMA_HOST: "{{ .Values.data.schema_host }}"
AUTHORIZE_API: "{{ .Values.data.authorize_api }}"
- GOOGLE_CLOUD_PROJECT: "{{ .Values.data.google_cloud_project }}"
REDIS_SEARCH_HOST: "{{ .Values.data.redis_search_host }}"
REDIS_GROUP_HOST: "{{ .Values.data.redis_group_host }}"
SECURITY_HTTPS_CERTIFICATE_TRUST: "{{ .Values.data.security_https_certificate_trust }}"
STORAGE_QUERY_RECORD_HOST: "{{ .Values.data.storage_query_record_host }}"
STORAGE_SCHEMA_HOST: "{{ .Values.data.storage_schema_host }}"
STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST: "{{ .Values.data.storage_query_record_for_conversion_host }}"
- STORAGE_HOSTNAME: "{{ .Values.data.storage_hostname }}"
STORAGE_RECORDS_BATCH_SIZE: "{{ .Values.data.storage_records_batch_size }}"
STORAGE_QUERY_KINDS_HOST: "{{ .Values.data.storage_query_kinds_host }}"
INDEXER_QUEUE_HOST: "{{ .Values.data.indexer_queue_host }}"
- LEGALTAG_API: "{{ .Values.data.legaltag_api }}"
- CRS_API: "{{ .Values.data.crs_api }}"
PARTITION_API: "{{ .Values.data.partition_api }}"
GOOGLE_AUDIENCES: "{{ .Values.data.google_audiences }}"
+ SPRING_PROFILES_ACTIVE: "{{ .Values.data.spring_profiles_active }}"
diff --git a/devops/gcp/configmap/values.yaml b/devops/gcp/configmap/values.yaml
index a423c75f0818b2e638467c912ba3f83a4c509d37..21247d9775534523bd1a1390374a9eb7d2cf1a0f 100644
--- a/devops/gcp/configmap/values.yaml
+++ b/devops/gcp/configmap/values.yaml
@@ -3,26 +3,21 @@
# Declare variables to be passed into your templates.
data:
- key_ring: "csqp"
- kms_key: "searchService"
log_level: "INFO"
schema_host: ""
authorize_api: "http://entitlements/api/entitlements/v2/"
- google_cloud_project: ""
redis_search_host: ""
redis_group_host: ""
security_https_certificate_trust: "true"
storage_query_record_host: "http://storage/api/storage/v2/query/records"
storage_schema_host: "http://storage/api/storage/v2/schemas"
storage_query_record_for_conversion_host: "http://storage/api/storage/v2/query/records:batch"
- storage_hostname: "storage"
storage_records_batch_size: "20"
storage_query_kinds_host: "http://storage/api/storage/v2/query/kinds"
indexer_queue_host: ""
- legaltag_api: "http://legal/api/legal/v1"
- crs_api: ""
partition_api: "http://partition/api/partition/v1/"
google_audiences: ""
+ spring_profiles_active: "gcp"
conf:
diff --git a/indexer-core/pom.xml b/indexer-core/pom.xml
index 5e09b3896648e6c5ffd1b240440b2558a86bb264..9d314a96fe6b504b68f32f900327e8605dfcaaad 100644
--- a/indexer-core/pom.xml
+++ b/indexer-core/pom.xml
@@ -4,12 +4,12 @@
<parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-service</artifactId>
- <version>0.13.0-SNAPSHOT</version>
+ <version>0.14.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>indexer-core</artifactId>
- <version>0.13.0-SNAPSHOT</version>
+ <version>0.14.0-SNAPSHOT</version>
<name>indexer-core</name>
<description>Indexer Service Core</description>
<packaging>jar</packaging>
diff --git a/pom.xml b/pom.xml
index 18d403ff957d1556a889c89d6089255aeeef0143..7901b446aef4599ec86876fd3005d58136180163 100644
--- a/pom.xml
+++ b/pom.xml
@@ -5,7 +5,7 @@
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-service</artifactId>
<packaging>pom</packaging>
- <version>0.13.0-SNAPSHOT</version>
+ <version>0.14.0-SNAPSHOT</version>
<description>Indexer Service</description>
<properties>
diff --git a/provider/indexer-aws/pom.xml b/provider/indexer-aws/pom.xml
index f4d7b03cfa9683bcf644f59ec2845a789263a2eb..cdd56d0e7664e8a0a810973d284c956b86472d94 100644
--- a/provider/indexer-aws/pom.xml
+++ b/provider/indexer-aws/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-service</artifactId>
- <version>0.13.0-SNAPSHOT</version>
+ <version>0.14.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
@@ -26,7 +26,7 @@
<artifactId>indexer-aws</artifactId>
<description>Indexer service on AWS</description>
<packaging>jar</packaging>
- <version>0.13.0-SNAPSHOT</version>
+ <version>0.14.0-SNAPSHOT</version>
<properties>
<aws.version>1.11.1018</aws.version>
@@ -63,7 +63,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-core</artifactId>
- <version>0.13.0-SNAPSHOT</version>
+ <version>0.14.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
diff --git a/provider/indexer-azure/pom.xml b/provider/indexer-azure/pom.xml
index bd52386bb4b22d534757e175d435f8b2ba2d75df..9cfb253a282648fd028aca0bf051b25a35f22964 100644
--- a/provider/indexer-azure/pom.xml
+++ b/provider/indexer-azure/pom.xml
@@ -21,12 +21,12 @@
<parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-service</artifactId>
- <version>0.13.0-SNAPSHOT</version>
+ <version>0.14.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>indexer-azure</artifactId>
- <version>0.13.0-SNAPSHOT</version>
+ <version>0.14.0-SNAPSHOT</version>
<name>indexer-azure</name>
<description>Indexer Service Azure</description>
<packaging>jar</packaging>
@@ -39,7 +39,7 @@
<azure.appservice.subscription />
<log4j.version>2.17.1</log4j.version>
<nimbus-jose-jwt.version>8.2</nimbus-jose-jwt.version>
- <indexer-core.version>0.13.0-SNAPSHOT</indexer-core.version>
+ <indexer-core.version>0.14.0-SNAPSHOT</indexer-core.version>
<spring-security-jwt.version>1.1.1.RELEASE</spring-security-jwt.version>
<osdu.corelibazure.version>0.13.0-rc6</osdu.corelibazure.version>
<osdu.oscorecommon.version>0.13.0-rc7</osdu.oscorecommon.version>
diff --git a/provider/indexer-gcp/README.md b/provider/indexer-gcp/README.md
index 49996510dd57276dc47416eb09d17907d16a2491..c4ea917bc3ffe5d2887485c75dd0bd0b3542bed3 100644
--- a/provider/indexer-gcp/README.md
+++ b/provider/indexer-gcp/README.md
@@ -1,44 +1,49 @@
# Indexer Service
os-indexer-gcp is a [Spring Boot](https://spring.io/projects/spring-boot) service that is responsible for indexing Records that enable the `os-search` service to execute OSDU R2 domain searches against Elasticsearch.
+## Table of Contents <a name="TOC"></a>
+* [Getting started](#Getting-started)
+* [Mappers](#Mappers)
+* [Settings and Configuration](#Settings-and-Configuration)
+* [Run service](#Run-service)
+* [Testing](#Testing)
+* [Deployment](#Deployment)
+* [Entitlements groups](#Entitlements-groups)
+* [Licence](#License)
+
## Getting Started
These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system.
-### Prerequisites
-Pre-requisites
+## Mappers
-* GCloud SDK with java (latest version)
-* JDK 8
-* Lombok 1.16 or later
-* Maven
+This is a universal solution created using EPAM OQM mappers technology. It allows you to work with various
+implementations of message brokers.
-### Installation
-In order to run the service locally or remotely, you will need to have the following environment variables defined.
+For more information about mappers:
+- [OQM Readme](https://community.opengroup.org/osdu/platform/system/lib/cloud/gcp/oqm/-/blob/master/README.md)
-| name | value | description | sensitive? | source |
-| --- | --- | --- | --- | --- |
-| `LOG_PREFIX` | `service` | Logging prefix | no | - |
-| `SERVER_SERVLET_CONTEXPATH` | `/api/indexer/v2` | Servlet context path | no | - |
-| `AUTHORIZE_API` | ex `https://entitlements.com/entitlements/v1` | Entitlements API endpoint | no | output of infrastructure deployment |
-| `LEGALTAG_API` | ex `https://legal.com/api/legal/v1` | Legal API endpoint | no | output of infrastructure deployment |
-| `INDEXER_QUEUE_HOST` | ex `https://os-indexer-queue-dot-opendes.appspot.com/_dps/task-handlers/enqueue` | Indexer-Queue API endpoint | no | output of infrastructure deployment |
-| `CRS_API` | ex `https://crs-converter-gae-dot-opendes.appspot.com/api/crs/v1` | CRS API endpoint | no | https://console.cloud.google.com/memorystore/redis/instances |
-| `STORAGE_HOSTNAME` | ex `os-storage-dot-opendes.appspot.com` | Storage Host | no | output of infrastructure deployment |
-| `STORAGE_SCHEMA_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/schemas` | Storage API endpoint 'schemas' | no | https://console.cloud.google.com/apis/credentials |
-| `STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records:batch` | Storage API endpoint 'records' | no | https://console.cloud.google.com/iam-admin/serviceaccounts |
-| `STORAGE_QUERY_RECORD_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records` | Storage API endpoint 'query/records' | no | https://console.cloud.google.com/iam-admin/serviceaccounts |
-| `STORAGE_RECORDS_BATCH_SIZE` | 20 | Batch size for storage API `POST {endpoint}/query/records:batch` | no | - |
-| `STORAGE_RECORDS_BY_KIND_BATCH_SIZE` | - | Batch size for storage API `GET {endpoint}/query/records`. If this is not present, defaults to value of `STORAGE_RECORDS_BATCH_SIZE` | no | - |
-| `REDIS_SEARCH_HOST` | ex `127.0.0.1` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances |
-| `REDIS_GROUP_HOST` | ex `127.0.0.1` | Redis host for groups | no | https://console.cloud.google.com/memorystore/redis/instances |
-| `REDIS_SEARCH_PORT` | ex `6379` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances |
-| `GOOGLE_CLOUD_PROJECT` | ex `opendes` | Google Cloud Project Id| no | output of infrastructure deployment |
-| `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials |
-| `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
-| `security.https.certificate.trust` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment |
-| `indexer.que.service.mail` | ex `default@iam.gserviceaccount.com` | Indexer Que environment service account mail, required if Indexer Que deployed in cloud task mode, to validate token from it | yes | - |
-| `SCHEMA_HOST` | ex `https://os-schema-dot-opendes.appspot.com/api/schema-service/v1/schema` | Schema API endpoint | no | output of infrastructure deployment |
-| `PARTITION_API` | ex `https://localhost:8081/api/partition/v1` | Partition API endpoint | no | output of infrastructure deployment |
+### Limitations of the current version
+
+In the current version, the mappers are equipped with several drivers to the stores and the message broker:
+
+- OQM (mapper to message brokers): Google PubSub; RabbitMQ
+
+## Settings and Configuration
+
+### Prerequisites
+1. Mandatory
+ - JDK 8
+ - Lombok 1.16 or later
+ - Maven
+2. For Google Cloud only
+ - GCloud SDK with java (latest version)
+
+### Anthos Service Configuration:
+[Anthos service configuration ](docs/anthos/README.md)
+### GCP Service Configuration:
+[Gcp service configuration ](docs/gcp/README.md)
+
+## Run service
### Run Locally
Check that maven is installed:
@@ -183,26 +188,6 @@ $ (cd testing/indexer-test-gcp/ && mvn clean test)
or
* Google Documentation: https://cloud.google.com/cloud-build/docs/deploying-builds/deploy-appengine
-#### Cloud KMS Setup
-
-Enable cloud KMS on master project
-
-Create king ring and key in the ***master project***
-
-```bash
- gcloud services enable cloudkms.googleapis.com
- export KEYRING_NAME="csqp"
- export CRYPTOKEY_NAME="searchService"
- gcloud kms keyrings create $KEYRING_NAME --location global
- gcloud kms keys create $CRYPTOKEY_NAME --location global \
- --keyring $KEYRING_NAME \
- --purpose encryption
-```
-
-Add **Cloud KMS CryptoKey Encrypter/Decrypter** role to the **default service account** of the ***master project*** through IAM - Role tab
-
-Add **Cloud KMS Encrypt/Decrypt** role to the **default service account** of ***master project*** through IAM - Role tab
-
#### Memory Store (Redis Instance) Setup
Create a new Standard tier Redis instance on the ***service project***
@@ -213,6 +198,14 @@ The Redis instance must be created under the same region with the App Engine app
gcloud beta redis instances create redis-cache-search --size=10 --region=<service-deployment-region> --zone=<service-deployment-zone> --tier=STANDARD
```
+## Entitlements groups
+Storage service account should have entitlements groups listed below:
+- service.entitlements.user
+- users
+- service.storage.viewer
+- service.schema-service.viewers
+- data.default.viewers
+
## Licence
Copyright © Google LLC
Copyright © EPAM Systems
diff --git a/provider/indexer-gcp/docs/anthos/README.md b/provider/indexer-gcp/docs/anthos/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..90cb1a8b9ed4e39f7e9f80be0ad42524b77d7ace
--- /dev/null
+++ b/provider/indexer-gcp/docs/anthos/README.md
@@ -0,0 +1,188 @@
+## Service Configuration for Anthos
+
+## Environment variables:
+
+Define the following environment variables.
+
+Must have:
+
+| name | value | description | sensitive? | source |
+| --- | --- | --- | --- | --- |
+| `SPRING_PROFILES_ACTIVE` | ex `anthos` | Spring profile that activate default configuration for GCP environment | false | - |
+| `OPENID_PROVIDER_CLIENT_ID` | `*****` | Client id that represents this service and serves to request tokens, example `workload-identity-legal` |yes| - |
+| `OPENID_PROVIDER_CLIENT_SECRET` | `*****` | This client secret that serves to request tokens| yes | - |
+| `OPENID_PROVIDER_URL` | `https://keycloack.com/auth/realms/master` | URL of OpenID Connect provider, it will be used as `<OpenID URL> + /.well-known/openid-configuration` to auto configure endpoint for token request | no | - |
+
+Defined in default application property file but possible to override:
+
+| name | value | description | sensitive? | source |
+| --- | --- | --- | --- | --- |
+| `LOG_PREFIX` | `service` | Logging prefix | no | - |
+| `LOG_LEVEL` | `****` | Logging level | no | - |
+| `SECURITY_HTTPS_CERTIFICATE_TRUST` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment |
+| `REDIS_SEARCH_HOST` | ex `127.0.0.1` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances |
+| `REDIS_SEARCH_PORT` | ex `6379` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances |
+| `REDIS_GROUP_HOST` | ex `127.0.0.1` | Redis host for groups | no | https://console.cloud.google.com/memorystore/redis/instances |
+| `REDIS_GROUP_PORT` | ex `6379` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances |
+| `PARTITION_HOST` | ex `https://partition.com` | Partition host | no | output of infrastructure deployment |
+| `ENTITLEMENTS_HOST` | ex `https://entitlements.com` | Entitlements host | no | output of infrastructure deployment |
+| `STORAGE_HOST` | ex `https://storage.com` | Storage host | no | output of infrastructure deployment |
+| `INDEXER_QUEUE_HOST` | ex `http://indexer-queue` | Indexer-Queue host | no | output of infrastructure deployment |
+| `SCHEMA_BASE_HOST` | ex `https://schema.com` | Schema service host | no | output of infrastructure deployment |
+
+These variables define service behavior, and are used to switch between `anthos` or `gcp` environments, their overriding and usage in mixed mode was not tested.
+Usage of spring profiles is preferred.
+
+| name | value | description | sensitive? | source |
+| --- | --- | --- | --- | --- |
+| `PARTITION_AUTH_ENABLED` | ex `true` or `false` | Disable or enable auth token provisioning for requests to Partition service | no | - |
+| `OQMDRIVER` | `rabbitmq` or `pubsub` | Oqm driver mode that defines which message broker will be used | no | - |
+| `SERVICE_TOKEN_PROVIDER` | `GCP` or `OPENID` |Service account token provider, `GCP` means use Google service account `OPEIND` means use OpenId provider like `Keycloak` | no | - |
+
+## Elasticsearch configuration
+
+### Properties set in Partition service:
+
+**prefix:** `elasticsearch`
+
+It can be overridden by:
+
+- through the Spring Boot property `elastic-search-properties-prefix`
+- environment variable `ELASTIC_SEARCH_PROPERTIES_PREFIX`
+
+**Propertyset:**
+
+| Property | Description |
+| --- | --- |
+| elasticsearch.host | server URL |
+| elasticsearch.port | server port |
+| elasticsearch.configuration | username and password |
+
+<details><summary>Example of a definition for a single tenant</summary></details>
+
+```
+
+curl -L -X PATCH 'http://partition.com/api/partition/v1/partitions/opendes' -H 'data-partition-id: opendes' -H 'Authorization: Bearer ...' -H 'Content-Type: application/json' --data-raw '{
+ "properties": {
+ "elasticsearch.host": {
+ "sensitive": false,
+ "value": "elastic.us-central1.gcp.cloud.es.io"
+ },
+ "elasticsearch.port": {
+ "sensitive": false,
+ "value": "9243"
+ },
+ "elasticsearch.configuration": {
+ "sensitive": true,
+ "value": "elasticuser:elasticpassword"
+ }
+ }
+}'
+
+```
+
+### For OQM RabbitMQ
+
+**prefix:** `oqm.rabbitmq`
+It can be overridden by:
+
+- through the Spring Boot property `oqm.rabbitmq.partition-properties-prefix`
+- environment variable `OQM_RABBITMQ_PARTITION_PROPERTIES_PREFIX``
+
+**Propertyset** (for two types of connection: messaging and admin operations):
+
+| Property | Description |
+| --- | --- |
+| oqm.rabbitmq.amqp.host | messaging hostnameorIP |
+| oqm.rabbitmq.amqp.port | - port |
+| oqm.rabbitmq.amqp.path | - path |
+| oqm.rabbitmq.amqp.username | - username |
+| oqm.rabbitmq.amqp.password | - password |
+| oqm.rabbitmq.admin.schema | admin host schema |
+| oqm.rabbitmq.admin.host | - host name |
+| oqm.rabbitmq.admin.port | - port |
+| oqm.rabbitmq.admin.path | - path |
+| oqm.rabbitmq.admin.username | - username |
+| oqm.rabbitmq.admin.password | - password |
+
+<details><summary>Example of a single tenant definition</summary>
+
+```
+
+curl -L -X PATCH 'https://dev.osdu.club/api/partition/v1/partitions/opendes' -H 'data-partition-id: opendes' -H 'Authorization: Bearer ...' -H 'Content-Type: application/json' --data-raw '{
+ "properties": {
+ "oqm.rabbitmq.amqp.host": {
+ "sensitive": false,
+ "value": "localhost"
+ },
+ "oqm.rabbitmq.amqp.port": {
+ "sensitive": false,
+ "value": "5672"
+ },
+ "oqm.rabbitmq.amqp.path": {
+ "sensitive": false,
+ "value": ""
+ },
+ "oqm.rabbitmq.amqp.username": {
+ "sensitive": false,
+ "value": "guest"
+ },
+ "oqm.rabbitmq.amqp.password": {
+ "sensitive": true,
+ "value": "guest"
+ },
+
+ "oqm.rabbitmq.admin.schema": {
+ "sensitive": false,
+ "value": "http"
+ },
+ "oqm.rabbitmq.admin.host": {
+ "sensitive": false,
+ "value": "localhost"
+ },
+ "oqm.rabbitmq.admin.port": {
+ "sensitive": false,
+ "value": "9002"
+ },
+ "oqm.rabbitmq.admin.path": {
+ "sensitive": false,
+ "value": "/api"
+ },
+ "oqm.rabbitmq.admin.username": {
+ "sensitive": false,
+ "value": "guest"
+ },
+ "oqm.rabbitmq.admin.password": {
+ "sensitive": true,
+ "value": "guest"
+ }
+ }
+}'
+
+```
+
+</details>
+
+#### Exchanges and queues configuration
+
+At RabbitMq should be created exchange with name:
+
+**name:** `indexing-progress`
+
+
+
+## Keycloak configuration
+
+[Keycloak service accounts setup](https://www.keycloak.org/docs/latest/server_admin/#_service_accounts)
+
+Configure Clients. One Client per OSDU service. Set them “confidential”.
+
+
+
+Each Client has embedded Service Account (SA) option. Enable SAs for Clients, make “Authorization enabled”:
+
+
+
+Add `partition-and-entitlements` scope to `Default Client Scopes` and generate Keys.
+
+Give `client-id` and `client-secret` to services, which should be authorized within the platform.
\ No newline at end of file
diff --git a/provider/indexer-gcp/docs/anthos/pics/client.png b/provider/indexer-gcp/docs/anthos/pics/client.png
new file mode 100644
index 0000000000000000000000000000000000000000..8a2014e672b65ddf644558e2c9e8cb8823f549dd
Binary files /dev/null and b/provider/indexer-gcp/docs/anthos/pics/client.png differ
diff --git a/provider/indexer-gcp/docs/anthos/pics/rabbit.PNG b/provider/indexer-gcp/docs/anthos/pics/rabbit.PNG
new file mode 100644
index 0000000000000000000000000000000000000000..e1c67734b91a536659fdf0b940a93208d749e3c8
Binary files /dev/null and b/provider/indexer-gcp/docs/anthos/pics/rabbit.PNG differ
diff --git a/provider/indexer-gcp/docs/anthos/pics/sa.png b/provider/indexer-gcp/docs/anthos/pics/sa.png
new file mode 100644
index 0000000000000000000000000000000000000000..b9c8124047b5834a2cf79baecbf751dc106feeff
Binary files /dev/null and b/provider/indexer-gcp/docs/anthos/pics/sa.png differ
diff --git a/provider/indexer-gcp/docs/gcp/README.md b/provider/indexer-gcp/docs/gcp/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..494169b1634af2e093c39038688987e78de6d726
--- /dev/null
+++ b/provider/indexer-gcp/docs/gcp/README.md
@@ -0,0 +1,94 @@
+## Service Configuration for GCP
+
+## Environment variables:
+
+Define the following environment variables.
+
+Must have:
+
+| name | value | description | sensitive? | source |
+| --- | --- | --- | --- | --- |
+| `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials |
+| `SPRING_PROFILES_ACTIVE` | ex `gcp` | Spring profile that activate default configuration for GCP environment | false | - |
+
+Defined in default application property file but possible to override:
+
+| name | value | description | sensitive? | source |
+| --- | --- | --- | --- | --- |
+| `LOG_PREFIX` | `service` | Logging prefix | no | - |
+| `LOG_LEVEL` | `****` | Logging level | no | - |
+| `SECURITY_HTTPS_CERTIFICATE_TRUST` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment |
+| `REDIS_SEARCH_HOST` | ex `127.0.0.1` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances |
+| `REDIS_SEARCH_PORT` | ex `6379` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances |
+| `REDIS_GROUP_HOST` | ex `127.0.0.1` | Redis host for groups | no | https://console.cloud.google.com/memorystore/redis/instances |
+| `REDIS_GROUP_PORT` | ex `6379` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances |
+| `PARTITION_HOST` | ex `https://partition.com` | Partition host | no | output of infrastructure deployment |
+| `ENTITLEMENTS_HOST` | ex `https://entitlements.com` | Entitlements host | no | output of infrastructure deployment |
+| `STORAGE_HOST` | ex `https://storage.com` | Storage host | no | output of infrastructure deployment |
+| `INDEXER_QUEUE_HOST` | ex `http://indexer-queue` | Indexer-Queue host | no | output of infrastructure deployment |
+| `SCHEMA_BASE_HOST` | ex `https://schema.com` | Schema service host | no | output of infrastructure deployment |
+| `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
+
+These variables define service behavior, and are used to switch between `anthos` or `gcp` environments, their overriding and usage in mixed mode was not tested.
+Usage of spring profiles is preferred.
+
+| name | value | description | sensitive? | source |
+| --- | --- | --- | --- | --- |
+| `PARTITION_AUTH_ENABLED` | ex `true` or `false` | Disable or enable auth token provisioning for requests to Partition service | no | - |
+| `OQMDRIVER` | `rabbitmq` or `pubsub` | Oqm driver mode that defines which message broker will be used | no | - |
+| `SERVICE_TOKEN_PROVIDER` | `GCP` or `OPENID` |Service account token provider, `GCP` means use Google service account `OPEIND` means use OpenId provider like `Keycloak` | no | - |
+
+## Pubsub configuration:
+
+At Pubsub should be created topic with name:
+
+**name:** `indexing-progress`
+
+## Elasticsearch configuration
+
+### Properties set in Partition service:
+
+**prefix:** `elasticsearch`
+
+It can be overridden by:
+
+- through the Spring Boot property `elastic-search-properties-prefix`
+- environment variable `ELASTIC_SEARCH_PROPERTIES_PREFIX`
+
+**Propertyset:**
+
+| Property | Description |
+| --- | --- |
+| elasticsearch.host | server URL |
+| elasticsearch.port | server port |
+| elasticsearch.configuration | username and password |
+
+<details><summary>Example of a definition for a single tenant</summary></details>
+
+```
+
+curl -L -X PATCH 'http://partition.com/api/partition/v1/partitions/opendes' -H 'data-partition-id: opendes' -H 'Authorization: Bearer ...' -H 'Content-Type: application/json' --data-raw '{
+ "properties": {
+ "elasticsearch.host": {
+ "sensitive": false,
+ "value": "elastic.us-central1.gcp.cloud.es.io"
+ },
+ "elasticsearch.port": {
+ "sensitive": false,
+ "value": "9243"
+ },
+ "elasticsearch.configuration": {
+ "sensitive": true,
+ "value": "elasticuser:elasticpassword"
+ }
+ }
+}'
+
+```
+
+## Google cloud service account configuration :
+TBD
+
+| Required roles |
+| --- |
+| - |
diff --git a/provider/indexer-gcp/pom.xml b/provider/indexer-gcp/pom.xml
index d5f0df645c4dec672c806196d3fdb97724ae3438..3f7bb4987478522f50a8d113ae6094c2997c4583 100644
--- a/provider/indexer-gcp/pom.xml
+++ b/provider/indexer-gcp/pom.xml
@@ -5,12 +5,12 @@
<parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-service</artifactId>
- <version>0.13.0-SNAPSHOT</version>
+ <version>0.14.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>indexer-gcp</artifactId>
- <version>0.13.0-SNAPSHOT</version>
+ <version>0.14.0-SNAPSHOT</version>
<name>indexer-gcp</name>
<description>Indexer Service GCP App Engine</description>
<packaging>jar</packaging>
@@ -19,12 +19,12 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-gcp</artifactId>
- <version>0.13.0-rc5</version>
+ <version>0.14.0-rc1</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-core</artifactId>
- <version>0.13.0-SNAPSHOT</version>
+ <version>0.14.0-SNAPSHOT</version>
<!-- excluded due to runtime conflict with latest core-lib-gcp transient dependencies -->
<exclusions>
<exclusion>
@@ -179,6 +179,4 @@
</plugin>
</plugins>
</build>
-
-
</project>
diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java
deleted file mode 100644
index 6a3b330aad0dcc5a7ba09687a82e0c5e1f53548c..0000000000000000000000000000000000000000
--- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java
+++ /dev/null
@@ -1,30 +0,0 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.cache;
-
-import com.google.auth.oauth2.AccessToken;
-import org.opengroup.osdu.core.common.cache.RedisCache;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-@Component
-public class DatastoreCredentialCache extends RedisCache<String, AccessToken> {
-
- @Autowired
- public DatastoreCredentialCache(final IndexerConfigurationProperties configurationProperties) {
- super(configurationProperties.getRedisSearchHost(), Integer.parseInt(configurationProperties.getRedisSearchPort()), 58 * 60, String.class, AccessToken.class);
- }
-}
\ No newline at end of file
diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java
index 206787c8d06dd825dabf1b165c9c2f52f742a735..83ee124a964a148dcaba28b7b0e88d7c7a111b2f 100644
--- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java
+++ b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java
@@ -18,29 +18,23 @@
package org.opengroup.osdu.indexer.cache;
import com.google.gson.Gson;
-import java.io.IOException;
import java.util.Objects;
import javax.inject.Inject;
-import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.cache.RedisCache;
-import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.search.ClusterSettings;
import org.opengroup.osdu.core.common.provider.interfaces.IElasticCredentialsCache;
-import org.opengroup.osdu.core.common.provider.interfaces.IKmsClient;
import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
import org.springframework.stereotype.Component;
@Component
public class ElasticCredentialsCache implements IElasticCredentialsCache<String, ClusterSettings>, AutoCloseable {
- private IKmsClient kmsClient;
private RedisCache<String, String> cache;
@Inject
- public ElasticCredentialsCache(final IndexerConfigurationProperties properties, final IKmsClient kmsClient) {
+ public ElasticCredentialsCache(final IndexerConfigurationProperties properties) {
this.cache = new RedisCache<>(properties.getRedisSearchHost(), Integer.parseInt(properties.getRedisSearchPort()),
properties.getElasticCacheExpiration() * 60, String.class, String.class);
- this.kmsClient = kmsClient;
}
@Override
@@ -50,27 +44,17 @@ public class ElasticCredentialsCache implements IElasticCredentialsCache<String,
@Override
public void put(String s, ClusterSettings o) {
- try {
- String jsonSettings = new Gson().toJson(o);
- String encryptString = kmsClient.encryptString(jsonSettings);
- this.cache.put(s, encryptString);
- } catch (IOException e) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Internal server error", "Unable to encrypt settings before being cached", e);
- }
+ String jsonSettings = new Gson().toJson(o);
+ this.cache.put(s, jsonSettings);
}
@Override
public ClusterSettings get(String s) {
- try {
- String encryptedSettings = this.cache.get(s);
- if (Objects.isNull(encryptedSettings) || encryptedSettings.isEmpty()) {
- return null;
- }
- String jsonSettings = this.kmsClient.decryptString(encryptedSettings);
- return new Gson().fromJson(jsonSettings, ClusterSettings.class);
- } catch (IOException e) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Internal server error", "Unable to decrypt settings from cache", e);
+ String jsonSettings = this.cache.get(s);
+ if (Objects.isNull(jsonSettings) || jsonSettings.isEmpty()) {
+ return null;
}
+ return new Gson().fromJson(jsonSettings, ClusterSettings.class);
}
@Override
diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/DatastoreCredentialsCacheFactory.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/DatastoreCredentialsCacheFactory.java
deleted file mode 100644
index 6f4d9b7505504f259ceca3d3b8e1834dc53f91ba..0000000000000000000000000000000000000000
--- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/DatastoreCredentialsCacheFactory.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- Copyright 2020 Google LLC
- Copyright 2020 EPAM Systems, Inc
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.di;
-
-import org.opengroup.osdu.core.common.cache.ICache;
-import org.opengroup.osdu.core.common.cache.VmCache;
-import org.opengroup.osdu.core.gcp.multitenancy.credentials.DatastoreCredential;
-import org.springframework.beans.factory.config.AbstractFactoryBean;
-import org.springframework.stereotype.Component;
-
-@Component
-public class DatastoreCredentialsCacheFactory extends
- AbstractFactoryBean<ICache<String, DatastoreCredential>> {
-
- @Override
- public Class<?> getObjectType() {
- return ICache.class;
- }
-
- @Override
- protected ICache<String, DatastoreCredential> createInstance() throws Exception {
- return new VmCache<>(5 * 60, 20);
- }
-}
diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/ElasticSearchConfig.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/ElasticSearchConfig.java
new file mode 100644
index 0000000000000000000000000000000000000000..72cb1f6a3a1cedfc3f408853d99d10a9c80cca74
--- /dev/null
+++ b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/ElasticSearchConfig.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2020-2022 Google LLC
+ * Copyright 2020-2022 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.di;
+
+import org.opengroup.osdu.core.common.partition.IPartitionProvider;
+import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository;
+import org.opengroup.osdu.core.destination.elastic.ElasticSearchDestinationResolver;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class ElasticSearchConfig {
+
+ @Bean
+ public IElasticRepository elasticRepository(ElasticSearchConfigurationProperties properties, IPartitionProvider partitionProvider) {
+ return new ElasticSearchDestinationResolver(properties.getElasticsearchPropertiesPrefix(), partitionProvider);
+ }
+}
diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/ElasticSearchConfigurationProperties.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/ElasticSearchConfigurationProperties.java
new file mode 100644
index 0000000000000000000000000000000000000000..586ce1ab9e9127282066a2e0fddf651836d12f18
--- /dev/null
+++ b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/ElasticSearchConfigurationProperties.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2020-2022 Google LLC
+ * Copyright 2020-2022 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.di;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@ConfigurationProperties
+@Getter
+@Setter
+public class ElasticSearchConfigurationProperties {
+
+ private String elasticsearchPropertiesPrefix = "elasticsearch";
+}
diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/PubSubExtensionsFactory.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/PubSubExtensionsFactory.java
deleted file mode 100644
index c0995677c674dc1192b4c9533903454af441cfeb..0000000000000000000000000000000000000000
--- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/PubSubExtensionsFactory.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- Copyright 2020 Google LLC
- Copyright 2020 EPAM Systems, Inc
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.di;
-
-import org.opengroup.osdu.core.gcp.PubSub.PubSubExtensions;
-import org.springframework.beans.factory.config.AbstractFactoryBean;
-import org.springframework.stereotype.Component;
-import org.springframework.web.context.annotation.RequestScope;
-
-@Component
-@RequestScope
-public class PubSubExtensionsFactory extends AbstractFactoryBean<PubSubExtensions> {
-
-
- @Override
- public Class<?> getObjectType() {
- return PubSubExtensions.class;
- }
-
- @Override
- protected PubSubExtensions createInstance() throws Exception {
- return new PubSubExtensions();
- }
-}
diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryService.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryService.java
deleted file mode 100644
index fa51ef212c0427bd21e4a522a99891d38163ccc4..0000000000000000000000000000000000000000
--- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryService.java
+++ /dev/null
@@ -1,38 +0,0 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.di;
-
-import lombok.extern.java.Log;
-import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
-import org.opengroup.osdu.core.gcp.multitenancy.TenantFactory;
-import org.springframework.beans.factory.config.AbstractFactoryBean;
-import org.springframework.stereotype.Component;
-import org.springframework.web.context.annotation.RequestScope;
-
-@Log
-@Component
-@RequestScope
-public class TenantFactoryService extends AbstractFactoryBean<ITenantFactory> {
-
- @Override
- protected ITenantFactory createInstance() throws Exception {
- return new TenantFactory();
- }
-
- @Override
- public Class<?> getObjectType() {
- return ITenantFactory.class;
- }
-}
\ No newline at end of file
diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java
deleted file mode 100644
index 9a6dc9ef7e3920668aa2155c1c39dc8531ed3529..0000000000000000000000000000000000000000
--- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.persistence;
-
-import com.google.api.gax.retrying.RetrySettings;
-import com.google.cloud.TransportOptions;
-import com.google.cloud.datastore.Datastore;
-import com.google.cloud.datastore.DatastoreOptions;
-import com.google.cloud.http.HttpTransportOptions;
-import java.util.HashMap;
-import java.util.Map;
-import javax.inject.Inject;
-import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
-import org.opengroup.osdu.core.gcp.multitenancy.credentials.DatastoreCredential;
-import org.opengroup.osdu.indexer.cache.DatastoreCredentialCache;
-import org.springframework.stereotype.Component;
-import org.threeten.bp.Duration;
-
-@Component("indexerDatastoreFactory")
-public class DatastoreFactory {
-
- @Inject
- private DatastoreCredentialCache cache;
-
- private static Map<String, Datastore> datastoreClients = new HashMap<>();
-
- private static final RetrySettings RETRY_SETTINGS = RetrySettings.newBuilder()
- .setMaxAttempts(6)
- .setInitialRetryDelay(Duration.ofSeconds(10))
- .setMaxRetryDelay(Duration.ofSeconds(32))
- .setRetryDelayMultiplier(2.0)
- .setTotalTimeout(Duration.ofSeconds(50))
- .setInitialRpcTimeout(Duration.ofSeconds(50))
- .setRpcTimeoutMultiplier(1.0)
- .setMaxRpcTimeout(Duration.ofSeconds(50))
- .build();
-
- private static final TransportOptions TRANSPORT_OPTIONS = HttpTransportOptions.newBuilder()
- .setReadTimeout(30000)
- .build();
-
- public Datastore getDatastoreInstance(TenantInfo tenantInfo) {
- if (datastoreClients.get(tenantInfo.getName()) == null) {
- Datastore googleDatastore = DatastoreOptions.newBuilder()
- .setCredentials(new DatastoreCredential(tenantInfo))
- .setRetrySettings(RETRY_SETTINGS)
- .setTransportOptions(TRANSPORT_OPTIONS)
- .setNamespace(tenantInfo.getName())
- .setProjectId(tenantInfo.getProjectId())
- .build().getService();
- datastoreClients.put(tenantInfo.getName(), googleDatastore);
- }
- return datastoreClients.get(tenantInfo.getName());
- }
-}
diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryDatastore.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryDatastore.java
deleted file mode 100644
index a8e3373f44926527a271d4d4390b5f4d5e647177..0000000000000000000000000000000000000000
--- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryDatastore.java
+++ /dev/null
@@ -1,89 +0,0 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.persistence;
-
-import com.google.api.client.googleapis.json.GoogleJsonResponseException;
-import com.google.cloud.datastore.Datastore;
-import com.google.cloud.datastore.Entity;
-import com.google.cloud.datastore.Key;
-import lombok.extern.java.Log;
-import org.apache.http.HttpStatus;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.search.ClusterSettings;
-import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
-import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository;
-import org.opengroup.osdu.core.common.provider.interfaces.IKmsClient;
-import org.opengroup.osdu.core.common.search.Preconditions;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.springframework.stereotype.Component;
-
-import javax.inject.Inject;
-
-@Log
-@Component
-public class ElasticRepositoryDatastore implements IElasticRepository {
-
- static final String HOST = "host";
- static final String PORT = "port";
- static final String XPACK_RESTCLIENT_CONFIGURATION = "configuration";
-
- @Inject
- private IKmsClient kmsClient;
- @Inject
- private DatastoreFactory datastoreFactory;
-
- @Inject
- private IndexerConfigurationProperties properties;
-
-
- @Override
- public ClusterSettings getElasticClusterSettings(TenantInfo tenantInfo) {
-
- Datastore googleDatastore = this.datastoreFactory.getDatastoreInstance(tenantInfo);
- Key key = googleDatastore.newKeyFactory().setKind(properties.getElasticDatastoreKind()).newKey(properties.getElasticDatastoreId());
- Entity datastoreEntity = googleDatastore.get(key);
-
- if (datastoreEntity == null) {
- throw new AppException(HttpStatus.SC_NOT_FOUND, "Cluster setting not found", "The requested cluster setting was not found in datastore.", String.format("Cluster setting with key: '%s' does not exist in datastore.", key.getName()));
- }
-
- String encryptedHost = null;
- String encryptedPort = null;
- String encryptedConfiguration = null;
-
- try {
- encryptedHost = datastoreEntity.getString(HOST);
- encryptedPort = datastoreEntity.getString(PORT);
- encryptedConfiguration = datastoreEntity.getString(XPACK_RESTCLIENT_CONFIGURATION);
-
- String host = this.kmsClient.decryptString(encryptedHost);
- String portString = this.kmsClient.decryptString(encryptedPort);
- String usernameAndPassword = this.kmsClient.decryptString(encryptedConfiguration);
-
- Preconditions.checkNotNullOrEmpty(host, "host cannot be null");
- Preconditions.checkNotNullOrEmpty(portString, "port cannot be null");
- Preconditions.checkNotNullOrEmpty(usernameAndPassword, "configuration cannot be null");
-
- int port = Integer.parseInt(portString);
-
- return new ClusterSettings(host, port, usernameAndPassword);
- } catch (GoogleJsonResponseException e) {
- String debuggingInfo = String.format("Host: %s | port: %s | configuration: %s", encryptedHost, encryptedPort, encryptedConfiguration);
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Cluster setting decryption error", "An error has occurred decrypting cluster settings.", debuggingInfo, e);
- } catch (Exception e) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Cluster setting fetch error", "An error has occurred fetching cluster settings from the datastore.", e);
- }
- }
-}
\ No newline at end of file
diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java
index 31d242cd46515f9b37907e458b510dd7a37557bd..2a35a96f5684f9e0c1227d549c0419da067c8118 100644
--- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java
+++ b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java
@@ -14,129 +14,58 @@
package org.opengroup.osdu.indexer.publish;
-import com.google.api.gax.retrying.RetrySettings;
-import com.google.cloud.pubsub.v1.Publisher;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonElement;
-import com.google.gson.reflect.TypeToken;
-import com.google.protobuf.ByteString;
-import com.google.pubsub.v1.ProjectTopicName;
-import com.google.pubsub.v1.PubsubMessage;
+import java.util.HashMap;
+import java.util.Map;
+import lombok.RequiredArgsConstructor;
import lombok.extern.java.Log;
-import org.apache.http.HttpStatus;
-import org.elasticsearch.common.Strings;
-import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.indexer.JobStatus;
-import org.opengroup.osdu.core.common.model.indexer.RecordStatus;
-import org.opengroup.osdu.core.common.model.search.DeploymentEnvironment;
-import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
-import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
-import org.opengroup.osdu.core.gcp.PubSub.PubSubExtensions;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.opengroup.osdu.core.gcp.oqm.driver.OqmDriver;
+import org.opengroup.osdu.core.gcp.oqm.model.OqmDestination;
+import org.opengroup.osdu.core.gcp.oqm.model.OqmMessage;
+import org.opengroup.osdu.core.gcp.oqm.model.OqmTopic;
import org.opengroup.osdu.indexer.provider.interfaces.IPublisher;
import org.springframework.stereotype.Component;
import org.springframework.web.context.annotation.RequestScope;
-import org.threeten.bp.Duration;
-
-import javax.inject.Inject;
-import java.io.IOException;
-import java.lang.reflect.Type;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
@Log
@Component
@RequestScope
+@RequiredArgsConstructor
public class PublisherImpl implements IPublisher {
- private static final Map<String, Publisher> PUBSUB_CLIENTS = new HashMap<>();
-
private static final String TOPIC_ID = "indexing-progress";
+ private final OqmDriver driver;
- @Inject
- private ITenantFactory tenantStorageFactory;
-
- @Inject
- private PubSubExtensions pubSubExtensions;
-
- @Inject
- private IndexerConfigurationProperties properties;
+ private final OqmTopic oqmTopic = OqmTopic.builder().name(TOPIC_ID).build();
@Override
- public void publishStatusChangedTagsToTopic(DpsHeaders headers, JobStatus indexerBatchStatus) throws Exception {
-
- // Don't publish to pubsub when testing locally
- if (properties.getDeploymentEnvironment() == DeploymentEnvironment.LOCAL) {
- return;
- }
-
- String tenant = headers.getPartitionId();
- if(Strings.isNullOrEmpty(tenant))
- tenant = headers.getAccountId();
-
- Publisher publisher = this.getPublisher(tenant);
- if (publisher == null) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Internal error", "A fatal internal error occurred creating publisher client.");
- }
- PubsubMessage pubsubMessage = getPubsubMessage(headers, indexerBatchStatus);
-
- pubSubExtensions.publishAndCreateTopicIfNotExist(publisher, pubsubMessage);
-
- }
-
- private static final RetrySettings RETRY_SETTINGS = RetrySettings.newBuilder()
- .setTotalTimeout(Duration.ofSeconds(30))
- .setInitialRetryDelay(Duration.ofSeconds(2))
- .setRetryDelayMultiplier(2)
- .setMaxRetryDelay(Duration.ofSeconds(5))
- .setInitialRpcTimeout(Duration.ofSeconds(10))
- .setRpcTimeoutMultiplier(2)
- .setMaxRpcTimeout(Duration.ofSeconds(10))
+ public void publishStatusChangedTagsToTopic(DpsHeaders headers, JobStatus indexerBatchStatus) {
+ OqmDestination oqmDestination = OqmDestination.builder().partitionId(headers.getPartitionId())
.build();
+ String json = generatePubSubMessage(indexerBatchStatus);
-
- private PubsubMessage getPubsubMessage(DpsHeaders headers, JobStatus indexerBatchStatus) {
-
- Gson gson = new GsonBuilder().create();
- Type listType = new TypeToken<List<RecordStatus>>() {}.getType();
- JsonElement statusChangedTagsJson = gson.toJsonTree(indexerBatchStatus.getStatusesList(), listType);
- ByteString statusChangedTagsData = ByteString.copyFromUtf8(statusChangedTagsJson.toString());
-
- PubsubMessage.Builder builder = PubsubMessage.newBuilder();
- String tenant = headers.getPartitionId();
- //This code it to provide backward compatibility to slb-account-id
- if(!Strings.isNullOrEmpty(tenant)) {
- builder.putAttributes(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionId());
- } else {
- builder.putAttributes(DpsHeaders.ACCOUNT_ID, headers.getAccountId());
- }
-
- builder.putAttributes(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
- // TODO: uncomment when deploying to production
-// builder.putAttributes( AppEngineHeaders.CLOUD_TRACE_CONTEXT, headers.getHeaders().get(AppEngineHeaders.CLOUD_TRACE_CONTEXT));
- builder.setData(statusChangedTagsData);
-
- return builder.build();
+ Map<String, String> attributes = getAttributes(headers);
+ OqmMessage oqmMessage = OqmMessage.builder().data(json).attributes(attributes).build();
+ driver.publish(oqmMessage, oqmTopic, oqmDestination);
}
- private Publisher getPublisher(String tenantName) throws IOException {
- TenantInfo info = this.tenantStorageFactory.getTenantInfo(tenantName);
- if (info == null) {
- return null;
- } else {
- if (PUBSUB_CLIENTS.containsKey(tenantName)) return PUBSUB_CLIENTS.get(tenantName);
-
- ProjectTopicName topicName = ProjectTopicName.newBuilder().setProject(info.getProjectId()).setTopic(TOPIC_ID).build();
- Publisher publisher = Publisher.newBuilder(topicName).setRetrySettings(RETRY_SETTINGS).build();
-
- if (publisher == null) return null;
+ private Map<String, String> getAttributes(DpsHeaders headers) {
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(DpsHeaders.ACCOUNT_ID, headers.getPartitionIdWithFallbackToAccountId());
+ attributes.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
+ headers.addCorrelationIdIfMissing();
+ attributes.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
+ return attributes;
+ }
- PUBSUB_CLIENTS.put(tenantName, publisher);
- return publisher;
- }
+ private String generatePubSubMessage(JobStatus jobStatus) {
+ Gson gson = new GsonBuilder().create();
+ JsonElement statusChangedTagsJson = gson.toJsonTree(jobStatus, JobStatus.class);
+ return statusChangedTagsJson.toString();
}
}
\ No newline at end of file
diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/security/GcpSecurityConfig.java
similarity index 95%
rename from provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java
rename to provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/security/GcpSecurityConfig.java
index 0f70a40bc444ae9fd7e1553c319684df1286273b..47b42552ccdef36a19103027cdde783bd4d50359 100644
--- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java
+++ b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/security/GcpSecurityConfig.java
@@ -27,7 +27,7 @@ import org.springframework.security.config.annotation.web.configuration.WebSecur
@Configuration
@EnableWebSecurity
@EnableGlobalMethodSecurity(prePostEnabled = true)
-public class GSuiteSecurityConfig extends WebSecurityConfigurerAdapter {
+public class GcpSecurityConfig extends WebSecurityConfigurerAdapter {
@Override
protected void configure(HttpSecurity http) throws Exception {
diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java
index 9950bb31d3eb14da96c466aeb283eea7b2f2110b..360f576f68021a2fdb398db4d5c368284069bc2e 100644
--- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java
+++ b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java
@@ -25,6 +25,7 @@ import java.util.logging.Level;
import javax.inject.Inject;
import lombok.extern.java.Log;
import org.apache.http.HttpStatus;
+import org.opengroup.osdu.core.auth.TokenProvider;
import org.opengroup.osdu.core.common.Constants;
import org.opengroup.osdu.core.common.model.entitlements.AuthorizationResponse;
import org.opengroup.osdu.core.common.model.http.AppException;
@@ -34,7 +35,6 @@ import org.opengroup.osdu.core.common.model.search.SearchServiceRole;
import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
import org.opengroup.osdu.core.common.provider.interfaces.IAuthorizationService;
import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
-import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient;
import org.opengroup.osdu.core.gcp.model.CloudTaskHeaders;
import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
import org.springframework.stereotype.Component;
@@ -50,7 +50,7 @@ public class RequestInfoImpl implements IRequestInfo {
private DpsHeaders dpsHeaders;
@Inject
- private IServiceAccountJwtClient serviceAccountJwtClient;
+ private TokenProvider tokenProvider;
@Inject
private TenantInfo tenantInfo;
@@ -99,12 +99,12 @@ public class RequestInfoImpl implements IRequestInfo {
@Override
public boolean isTaskQueueRequest() {
- if(this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.CLOUD_TASK_QUEUE_NAME)){
- log.log(Level.INFO,"Request confirmed as cloud task, token validation in progress");
+ if (this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.CLOUD_TASK_QUEUE_NAME)) {
+ log.log(Level.INFO, "Request confirmed as cloud task, token validation in progress");
return isCloudTaskRequest();
}
- if(this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.APPENGINE_TASK_QUEUE_NAME)){
- log.log(Level.INFO,"Request confirmed as AppEngine, headers validation in progress");
+ if (this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.APPENGINE_TASK_QUEUE_NAME)) {
+ log.log(Level.INFO, "Request confirmed as AppEngine, headers validation in progress");
return isAppEngineTaskRequest();
}
return false;
@@ -116,7 +116,7 @@ public class RequestInfoImpl implements IRequestInfo {
return true;
}
- private boolean isAppEngineTaskRequest(){
+ private boolean isAppEngineTaskRequest() {
if (!this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.APPENGINE_TASK_QUEUE_NAME)) {
return false;
}
@@ -136,7 +136,7 @@ public class RequestInfoImpl implements IRequestInfo {
}
return authHeader;
} else {
- return "Bearer " + this.serviceAccountJwtClient.getIdToken(tenantInfo.getName());
+ return "Bearer " + this.tokenProvider.getIdToken();
}
}
}
diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java
deleted file mode 100644
index 6c57527219c328d75ab5acaffbee7d85f7c558e8..0000000000000000000000000000000000000000
--- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java
+++ /dev/null
@@ -1,93 +0,0 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.util;
-
-import com.auth0.jwt.JWT;
-import com.google.cloud.iam.credentials.v1.GenerateIdTokenResponse;
-import com.google.cloud.iam.credentials.v1.IamCredentialsClient;
-import com.google.cloud.iam.credentials.v1.ServiceAccountName;
-import java.util.Collections;
-import javax.inject.Inject;
-import org.apache.http.HttpStatus;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.search.IdToken;
-import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
-import org.opengroup.osdu.core.common.provider.interfaces.IJwtCache;
-import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
-import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient;
-import org.opengroup.osdu.core.gcp.multitenancy.credentials.IamCredentialsProvider;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.context.annotation.Primary;
-import org.springframework.stereotype.Component;
-import org.springframework.web.context.annotation.RequestScope;
-
-@Primary
-@Component
-@RequestScope
-public class ServiceAccountJwtGcpClientImpl implements IServiceAccountJwtClient {
-
- private static final String SERVICE_ACCOUNT_NAME_FORMAT = "projects/-/serviceAccounts/%s";
-
- private final IamCredentialsProvider iamCredentialsProvider = new IamCredentialsProvider();
-
- @Value("${GOOGLE_AUDIENCES}")
- private String audiences;
-
- @Inject
- private ITenantFactory tenantInfoServiceProvider;
- @Inject
- private IJwtCache cacheService;
- @Inject
- private JaxRsDpsLog log;
- @Inject
- private DpsHeaders dpsHeaders;
- @Inject
- private IndexerConfigurationProperties properties;
-
- public String getIdToken(String tenantName) {
- this.log.info("Tenant name received for auth token is: " + tenantName);
- TenantInfo tenant = this.tenantInfoServiceProvider.getTenantInfo(tenantName);
- if (tenant == null) {
- this.log.error("Invalid tenant name receiving from pubsub");
- throw new AppException(HttpStatus.SC_BAD_REQUEST, "Invalid tenant Name", "Invalid tenant Name from pubsub");
- }
- try {
-
- IdToken cachedToken = (IdToken) this.cacheService.get(tenant.getServiceAccount());
- // Add the user to DpsHeaders directly
- this.dpsHeaders.put(DpsHeaders.USER_EMAIL, tenant.getServiceAccount());
-
- if (!IdToken.refreshToken(cachedToken)) {
- return cachedToken.getTokenValue();
- }
-
- try (IamCredentialsClient iamCredentialsClient = iamCredentialsProvider.getIamCredentialsClient()) {
- ServiceAccountName serviceAccountName = ServiceAccountName.parse(String.format(SERVICE_ACCOUNT_NAME_FORMAT, tenant.getServiceAccount()));
- GenerateIdTokenResponse idTokenResponse = iamCredentialsClient.generateIdToken(serviceAccountName, Collections.emptyList(), audiences, true);
- String token = idTokenResponse.getToken();
- IdToken idToken = IdToken.builder().tokenValue(token).expirationTimeMillis(JWT.decode(token).getExpiresAt().getTime()).build();
- this.cacheService.put(tenant.getServiceAccount(), idToken);
- return token;
- }
- } catch (AppException e) {
- throw e;
- } catch (Exception e) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Persistence error", "Error generating token", e);
- }
- }
-}
diff --git a/provider/indexer-gcp/src/main/resources/application-anthos.properties b/provider/indexer-gcp/src/main/resources/application-anthos.properties
new file mode 100644
index 0000000000000000000000000000000000000000..9e553661404fae2ae946a221d506b5b56d16f8a0
--- /dev/null
+++ b/provider/indexer-gcp/src/main/resources/application-anthos.properties
@@ -0,0 +1,6 @@
+oqmDriver=rabbitmq
+service.token.provider=OPENID
+partition-auth-enabled=false
+openid.provider-url=
+openid.provider-client-id=
+openid.provider-client-secret=
\ No newline at end of file
diff --git a/provider/indexer-gcp/src/main/resources/application-dev.properties b/provider/indexer-gcp/src/main/resources/application-dev.properties
deleted file mode 100644
index 6c49ae23e07fddb7668a52d645d70268579ce9a9..0000000000000000000000000000000000000000
--- a/provider/indexer-gcp/src/main/resources/application-dev.properties
+++ /dev/null
@@ -1,21 +0,0 @@
-google-cloud-project=opendes
-
-indexer-host=indexer-jvmvia5dea-uc.a.run.app
-STORAGE_HOSTNAME=storage-jvmvia5dea-uc.a.run.app
-
-storage-schema-host=http://127.0.0.1:8081/api/storage/v2/schemas
-storage-query-record-host=http://127.0.0.1:8081/api/storage/v2/query/records
-storage-query-record-for-conversion-host=https://storage-jvmvia5dea-uc.a.run.app/api/storage/v2/query/records:batch
-storage-records-batch-size=20
-
-indexer-queue-host=https://indexer-queue-jvmvia5dea-uc.a.run.app/_dps/task-handlers/enqueue
-
-AUTHORIZE_API=https://os-entitlements-gcp-jvmvia5dea-uc.a.run.app/entitlements/v1
-LEGALTAG_API=https://os-legal-jvmvia5dea-uc.a.run.app/api/legal/v1
-CRS_API=https://crs-converter-jvmvia5dea-uc.a.run.app/api/crs/v1
-
-## use below values for gcp: opendes
-REDIS_GROUP_HOST=127.0.0.1
-redis-search-host=127.0.0.1
-
-google-audiences=519000754840-09v7ssbpku7sevhvbtq3bdoi70es16p5.apps.googleusercontent.com
\ No newline at end of file
diff --git a/provider/indexer-gcp/src/main/resources/application-gcp.properties b/provider/indexer-gcp/src/main/resources/application-gcp.properties
new file mode 100644
index 0000000000000000000000000000000000000000..fd738d7f484bf44fc3b2f2043191259dd0b3a1b8
--- /dev/null
+++ b/provider/indexer-gcp/src/main/resources/application-gcp.properties
@@ -0,0 +1,3 @@
+oqmDriver=pubsub
+service.token.provider=GCP
+partition-auth-enabled=true
\ No newline at end of file
diff --git a/provider/indexer-gcp/src/main/resources/application-kuber.properties b/provider/indexer-gcp/src/main/resources/application-kuber.properties
deleted file mode 100644
index 22d3ba0253112794113a96f90fa094f5468902e1..0000000000000000000000000000000000000000
--- a/provider/indexer-gcp/src/main/resources/application-kuber.properties
+++ /dev/null
@@ -1,24 +0,0 @@
-google-cloud-project=${GOOGLE_CLOUD_PROJECT}
-
-indexer-host=os-indexer-service
-STORAGE_HOSTNAME=os-storage-service
-
-storage-schema-host=http://os-storage-service/api/storage/v2/schemas
-storage-query-record-host=http://os-storage-service/api/storage/v2/query/records
-storage-query-record-for-conversion-host=http://os-storage-service/api/storage/v2/query/records:batch
-storage-records-batch-size=20
-
-indexer-queue-host=http://os-storage-service/_dps/task-handlers/enqueue
-
-AUTHORIZE_API=http://os-entitlement-service/entitlements/v1
-LEGALTAG_API=http://os-legal-service/api/legal/v1
-CRS_API=${CRS_API}
-
-## use below values for gcp: opendes
-REDIS_GROUP_HOST=${REDIS_GROUP_HOST}
-redis-search-host=${REDIS_SEARCH_HOST}
-
-google-audiences=${GOOGLE_AUDIENCES}
-
-deployment-environment=cloud
-disable.appengine.log.factory=true
\ No newline at end of file
diff --git a/provider/indexer-gcp/src/main/resources/application-local.properties b/provider/indexer-gcp/src/main/resources/application-local.properties
deleted file mode 100644
index a5511b15c0564250bca41b67c6c39646f9512e79..0000000000000000000000000000000000000000
--- a/provider/indexer-gcp/src/main/resources/application-local.properties
+++ /dev/null
@@ -1,21 +0,0 @@
-google-cloud-project=osdu
-
-indexer-host=os-indexer-dot-opendes.appspot.com
-STORAGE_HOSTNAME=os-storage-dot-opendes.appspot.com
-
-storage-schema-host=https://os-storage-dot-opendes.appspot.com/api/storage/v2/schemas
-storage-query-record-host=https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records
-storage-query-record-for-conversion-host=https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records:batch
-storage-records-batch-size=20
-
-indexer-queue-host=https://os-indexer-queue-dot-opendes.appspot.com/_dps/task-handlers/enqueue
-
-AUTHORIZE_API=https://entitlements-dot-opendes.appspot.com/entitlements/v1
-LEGALTAG_API=https://os-legal-dot-opendes.appspot.com/api/legal/v1
-CRS_API=https://crs-converter-gae-dot-opendes.appspot.com/api/crs/v1
-
-## use below values for gcp: opendes
-REDIS_GROUP_HOST=127.0.0.1
-redis-search-host=127.0.0.1
-
-google-audiences=245464679631-ktfdfpl147m1mjpbutl00b3cmffissgq.apps.googleusercontent.com
diff --git a/provider/indexer-gcp/src/main/resources/application-testing.properties b/provider/indexer-gcp/src/main/resources/application-testing.properties
deleted file mode 100644
index 67941ce47b5fd27c99e487fa59c935d0bcacd827..0000000000000000000000000000000000000000
--- a/provider/indexer-gcp/src/main/resources/application-testing.properties
+++ /dev/null
@@ -1,21 +0,0 @@
-google-cloud-project=opendes-evt
-
-indexer-host=os-indexer-dot-opendes-evt.appspot.com
-STORAGE_HOSTNAME=os-storage-dot-opendes-evt.appspot.com
-
-storage-schema-host=https://os-storage-dot-opendes-evt.appspot.com/api/storage/v2/schemas
-storage-query-record-host=https://os-storage-dot-opendes-evt.appspot.com/api/storage/v2/query/records
-storage-query-record-for-conversion-host=https://os-storage-dot-opendes-evt.appspot.com/api/storage/v2/query/records:batch
-storage-records-batch-size=20
-
-indexer-queue-host=https://os-indexer-queue-dot-opendes-evt.appspot.com/_dps/task-handlers/enqueue
-
-AUTHORIZE_API=https://entitlements-dot-opendes-evt.appspot.com/entitlements/v1
-LEGALTAG_API=https://os-legal-dot-opendes-evt.appspot.com/api/legal/v1
-CRS_API=https://crs-converter-gae-dot-opendes-evt.appspot.com/api/crs/v1
-
-## use below values for gcp: opendes
-REDIS_GROUP_HOST=10.253.209.196
-redis-search-host=10.118.2.140
-
-google-audiences=833591776864-oobhqvmtdg9rpreubjvn44m5f8revglk.apps.googleusercontent.com
\ No newline at end of file
diff --git a/provider/indexer-gcp/src/main/resources/application.properties b/provider/indexer-gcp/src/main/resources/application.properties
index e18440679858d1208f1f6f683220e78a7e372978..f8ffcf5aea0314dd8077f87179ceac92a173160d 100644
--- a/provider/indexer-gcp/src/main/resources/application.properties
+++ b/provider/indexer-gcp/src/main/resources/application.properties
@@ -1,18 +1,15 @@
-LOG_PREFIX=indexer
-
+#server config
+server.port=8080
server.servlet.contextPath=/api/indexer/v2
+
+#log config
+LOG_PREFIX=indexer
logging.level.org.springframework.web=${LOG_LEVEL:DEBUG}
-server.port=8080
+
+#jvm config
JAVA_OPTS=-Xms3072m -Xmx3072m
JAVA_GC_OPTS=-XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45
-deployment-environment=CLOUD
-
-REDIS_GROUP_PORT=6379
-DEFAULT_DATA_COUNTRY=US
-
-redis-search-port=6379
-
#Default Cache Settings
schema-cache-expiration=60
index-cache-expiration=60
@@ -22,26 +19,41 @@ cursor-cache-expiration=60
kinds-cache-expiration=2880
# Attributes Cache expiration 2*24*60
attributes-cache-expiration=2880
-
kinds-redis-database=1
cron-index-cleanup-threshold-days=3
cron-empty-index-cleanup-threshold-days=7
+#indexer service config
+DEFAULT_DATA_COUNTRY=US
gae-service=indexer
-key-ring=csqp
-kms-key=searchService
+security.https.certificate.trust=false
+storage-records-by-kind-batch-size=20
+storage-records-batch-size=20
-KEY_RING=${key-ring}
-KMS_KEY=${kms-key}
-GOOGLE_CLOUD_PROJECT=${google-cloud-project}
+REDIS_GROUP_PORT=6379
+REDIS_GROUP_HOST=redis-cache-groups
-elastic-datastore-kind=SearchSettings
-elastic-datastore-id=indexer-service
+REDIS_SEARCH_PORT=6379
+REDIS_SEARCH_HOST=redis-cache-search
-security.https.certificate.trust=false
-indexer.que.service.mail=default@iam.gserviceaccount.com
-SCHEMA_HOST=${HOST}/api/schema-service/v1/schema
-storage-query-kinds-host=https://${STORAGE_HOSTNAME}/api/storage/v2/query/kinds
+PARTITION_HOST=http://partition
+PARTITION_PATH=/api/partition/v1
+PARTITION_API=${PARTITION_HOST}${PARTITION_PATH}
+
+ENTITLEMENTS_HOST=http://entitlements
+ENTITLEMENTS_PATH=/api/entitlements/v2
+AUTHORIZE_API=${ENTITLEMENTS_HOST}${ENTITLEMENTS_PATH}
+
+STORAGE_HOST=http://storage
+STORAGE_PATH=/api/storage/v2
+STORAGE_API=${STORAGE_HOST}${STORAGE_PATH}
+STORAGE_QUERY_KINDS_HOST=${STORAGE_API}/query/kinds
+STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=${STORAGE_API}/query/records:batch
+STORAGE_QUERY_RECORD_HOST=${STORAGE_API}/query/records
+STORAGE_SCHEMA_HOST=${STORAGE_API}/schemas
+
+SCHEMA_BASE_HOST=http://schema
+SCHEMA_PATH=/api/schema-service/v1/schema
+SCHEMA_HOST=${SCHEMA_BASE_HOST}${SCHEMA_PATH}
-google.audiences=123.apps.googleusercontent.com
-partition.api=http://localhost:8081/api/partition/v1
\ No newline at end of file
+INDEXER_QUEUE_HOST=http://indexer-queue
\ No newline at end of file
diff --git a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceTest.java b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceTest.java
deleted file mode 100644
index c716cb1a637f5a5d843ebccd73656f2fa1c0752f..0000000000000000000000000000000000000000
--- a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceTest.java
+++ /dev/null
@@ -1,337 +0,0 @@
-package org.opengroup.osdu.indexer.service;//// Copyright 2017-2019, Schlumberger
-////
-//// Licensed under the Apache License, Version 2.0 (the "License");
-//// you may not use this file except in compliance with the License.
-//// You may obtain a copy of the License at
-////
-//// http://www.apache.org/licenses/LICENSE-2.0
-////
-//// Unless required by applicable law or agreed to in writing, software
-//// distributed under the License is distributed on an "AS IS" BASIS,
-//// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//// See the License for the specific language governing permissions and
-//// limitations under the License.
-//
-//package org.opendes.indexer.service;
-//
-//import com.google.gson.Gson;
-//import com.google.gson.reflect.TypeToken;
-//import org.elasticsearch.action.bulk.BulkItemResponse;
-//import org.elasticsearch.action.bulk.BulkResponse;
-//import org.elasticsearch.client.RequestOptions;
-//import org.elasticsearch.client.RestHighLevelClient;
-//import org.junit.Before;
-//import org.junit.Ignore;
-//import org.junit.Test;
-//import org.junit.runner.RunWith;
-//import org.mockito.InjectMocks;
-//import org.mockito.Mock;
-//import org.mockito.Spy;
-//import org.opendes.client.api.DpsHeaders;
-//import org.opendes.core.logging.JaxRsDpsLog;
-//import org.opendes.core.model.DeploymentEnvironment;
-//import org.opendes.core.model.RecordChangedMessages;
-//import org.opendes.core.service.IndicesService;
-//import org.opendes.core.util.Config;
-//import org.opendes.core.util.ElasticClientHandler;
-//import org.opendes.core.util.ElasticIndexNameResolver;
-//import org.opendes.core.util.HeadersUtil;
-//import org.opendes.indexer.logging.AuditLogger;
-//import org.opendes.indexer.model.*;
-//import org.opendes.indexer.publish.IPublisher;
-//import org.opendes.indexer.util.IRequestInfo;
-//import org.opendes.indexer.util.IndexerQueueTaskBuilder;
-//import org.opendes.indexer.util.JobStatus;
-//import org.opendes.indexer.util.RecordInfo;
-//import org.powermock.core.classloader.annotations.PrepareForTest;
-//import javax.inject.Inject;
-//import org.springframework.context.annotation.Lazy;
-//import org.springframework.test.context.junit4.SpringRunner;
-//
-//import java.io.IOException;
-//import java.lang.reflect.Type;
-//import java.util.*;
-//
-//import static java.util.Collections.singletonList;
-//import static org.junit.Assert.*;
-//import static org.mockito.Matchers.any;
-//import static org.mockito.Mockito.verify;
-//import static org.mockito.Mockito.when;
-//import static org.powermock.api.mockito.PowerMockito.mock;
-//import static org.powermock.api.mockito.PowerMockito.mockStatic;
-//
-//@Ignore
-//@RunWith(SpringRunner.class)
-//@PrepareForTest({RestHighLevelClient.class, BulkResponse.class, StorageAcl.class, HeadersUtil.class, Config.class})
-//public class IndexerServiceTest {
-//
-// private final String pubsubMsg = "[{\"id\":\"tenant1:doc:test1\",\"kind\":\"tenant1:testindexer1:well:1.0.0\",\"op\":\"update\"}," +
-// "{\"id\":\"tenant1:doc:test2\",\"kind\":\"tenant1:testindexer2:well:1.0.0\",\"op\":\"create\"}]";
-// private final String kind1 = "tenant1:testindexer1:well:1.0.0";
-// private final String kind2 = "tenant1:testindexer2:well:1.0.0";
-// private final String recordId1 = "tenant1:doc:test1";
-// private final String recordId2 = "tenant1:doc:test2";
-// private final String failureMassage = "test failure";
-//
-// @Mock
-// private IndexSchemaService indexSchemaService;
-// @Mock
-// private IndicesService indicesService;
-// @Mock
-// private IndexerMappingService indexerMappingService;
-// @Mock
-// private StorageService storageService;
-// @Mock
-// private IPublisher publisherImpl;
-// @Mock
-// private RestHighLevelClient restHighLevelClient;
-// @Mock
-// private ElasticClientHandler elasticClientHandler;
-// @Mock
-// private BulkResponse bulkResponse;
-// @Mock
-// private IRequestInfo requestInfo;
-// @Mock
-// private ElasticIndexNameResolver elasticIndexNameResolver;
-// @Mock
-// private AttributeParsingServiceImpl attributeParsingServiceImpl;
-// @Mock
-// private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
-// @Mock
-// private JaxRsDpsLog log;
-// @Mock
-// private AuditLogger auditLogger;
-// @InjectMocks
-// private IndexerServiceImpl sut;
-// @InjectMocks @Spy
-// private JobStatus jobStatus = new JobStatus();
-//
-// @Inject
-// @Lazy
-// private DpsHeaders dpsHeaders;
-// private RecordChangedMessages recordChangedMessages;
-// private List<RecordInfo> recordInfos;
-//
-// @Before
-// public void setup() throws IOException {
-//
-// mockStatic(StorageAcl.class);
-// mockStatic(Config.class);
-//
-// when(Config.getDeploymentEnvironment()).thenReturn(DeploymentEnvironment.LOCAL);
-// when(Config.getElasticClusterName()).thenReturn("CLUSTER");
-// when(Config.getElasticServerAddress()).thenReturn("testsite");
-//
-// dpsHeaders = new DpsHeaders();
-// dpsHeaders.put(AppEngineHeaders.TASK_QUEUE_RETRY_COUNT, "1");
-// dpsHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
-// when(requestInfo.getHeaders()).thenReturn(dpsHeaders);
-// when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(dpsHeaders.getHeaders());
-//
-// Type listType = new TypeToken<List<RecordInfo>>() {}.getType();
-// recordInfos = (new Gson()).fromJson(pubsubMsg, listType);
-//
-// when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
-// when(restHighLevelClient.bulk(any(), any(RequestOptions.class))).thenReturn(bulkResponse);
-//
-// BulkItemResponse[] responses = new BulkItemResponse[]{prepareResponseFail(), prepareResponseSuccess()};
-// when(bulkResponse.getItems()).thenReturn(responses);
-// Map<String, String> attr = new HashMap<>();
-// attr.put(DpsHeaders.ACCOUNT_ID, "slb");
-// recordChangedMessages = RecordChangedMessages.builder().attributes(attr).messageId("xxxx").publishTime("2000-01-02T10:10:44+0000").data("{}").build();
-// when(StorageAcl.flattenAcl(any())).thenReturn(null);
-// }
-//
-// @Test
-// public void should_returnNull_givenEmptyJobSubInfo_processRecordChangedMessageTest() throws Exception {
-// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, new ArrayList<>());
-//
-// assertNull(jobStatus);
-// }
-//
-// @Test
-// public void should_returnValidJobStatus_givenNullSchema_processRecordChangedMessageTest() {
-// try {
-// indexSchemaServiceMock(kind1, null);
-// indexSchemaServiceMock(kind2, null);
-// List<ConversionStatus> conversionStatus = new LinkedList<>();
-// List<Records.Entity> validRecords = new ArrayList<>();
-// Map<String, Object> storageData = new HashMap<>();
-// storageData.put("schema1", "test-value");
-// storageData.put("schema2", "test-value");
-// storageData.put("schema3", "test-value");
-// storageData.put("schema4", "test-value");
-// storageData.put("schema5", "test-value");
-// storageData.put("schema6", "test-value");
-// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build());
-// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build();
-//
-// when(storageService.getStorageRecords(any())).thenReturn(storageRecords);
-// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true);
-//
-// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
-//
-// assertEquals(2, jobStatus.getStatusesList().size());
-// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
-// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).size());
-// } catch (Exception e) {
-// fail("Should not throw this exception" + e.getMessage());
-// }
-// }
-//
-// @Test
-// public void should_returnValidJobStatus_givenFailedUnitsConversion_processRecordChangedMessageTest() {
-// try {
-// indexSchemaServiceMock(kind1, null);
-// indexSchemaServiceMock(kind2, null);
-// List<ConversionStatus> conversionStatuses = new LinkedList<>();
-// List<String> status=new ArrayList<>();
-// status.add("crs bla bla");
-// ConversionStatus conversionStatus=ConversionStatus.builder().status("ERROR").errors(status).id(recordId2).build();
-// conversionStatuses.add(conversionStatus);
-// List<Records.Entity> validRecords = new ArrayList<>();
-// Map<String, Object> storageData = new HashMap<>();
-// storageData.put("schema1", "test-value");
-// storageData.put("schema2", "test-value");
-// storageData.put("schema3", "test-value");
-// storageData.put("schema4", "test-value");
-// storageData.put("schema5", "test-value");
-// storageData.put("schema6", "test-value");
-// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build());
-// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatuses).build();
-//
-// when(storageService.getStorageRecords(any())).thenReturn(storageRecords);
-// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true);
-//
-// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
-//
-// assertEquals(2, jobStatus.getStatusesList().size());
-// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
-// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).size());
-// assertTrue(jobStatus.getJobStatusByRecordId(jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).get(0)).getIndexProgress().getTrace().contains("crs bla bla"));
-// } catch (Exception e) {
-// fail("Should not throw this exception" + e.getMessage());
-// }
-// }
-//
-// @Test
-// public void should_returnValidJobStatus_givenNullSchemaForARecord_processRecordChangedMessageTest() {
-// try {
-// List<Records.Entity> validRecords = new ArrayList<>();
-// List<ConversionStatus> conversionStatus = new LinkedList<>();
-// Map<String, Object> storageData = new HashMap<>();
-// storageData.put("schema1", "test-value");
-// storageData.put("schema2", "test-value");
-// storageData.put("schema3", "test-value");
-// storageData.put("schema4", "test-value");
-// storageData.put("schema5", "test-value");
-// storageData.put("schema6", "test-value");
-// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build());
-// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build();
-// when(storageService.getStorageRecords(any())).thenReturn(storageRecords);
-//
-// Map<String, String> schema = createSchema();
-// indexSchemaServiceMock(kind1, schema);
-// indexSchemaServiceMock(kind2, null);
-// when(elasticIndexNameResolver.getIndexNameFromKind(kind2)).thenReturn("tenant1-testindexer2-well-1.0.0");
-// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true);
-// JobStatus jobStatus = sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
-//
-// assertEquals(2, jobStatus.getStatusesList().size());
-// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
-// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).size());
-// assertEquals("Indexed Successfully", jobStatus.getStatusesList().get(1).getIndexProgress().getTrace().pop());
-// assertEquals("schema not found", jobStatus.getStatusesList().get(1).getIndexProgress().getTrace().pop());
-// } catch (Exception e) {
-// fail("Should not throw this exception" + e.getMessage());
-// }
-// }
-//
-// @Test
-// public void should_returnValidJobStatus_givenValidCreateAndUpdateRecords_processRecordChangedMessagesTest() {
-// try {
-// Map<String, Object> storageData = new HashMap<>();
-// storageData.put("schema1", "test-value");
-// List<ConversionStatus> conversionStatus = new LinkedList<>();
-// List<Records.Entity> validRecords = new ArrayList<>();
-// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build());
-// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build();
-//
-// when(storageService.getStorageRecords(any())).thenReturn(storageRecords);
-// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true);
-// Map<String, String> schema = createSchema();
-// indexSchemaServiceMock(kind2, schema);
-// indexSchemaServiceMock(kind1, null);
-// JobStatus jobStatus = sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
-//
-// assertEquals(2, jobStatus.getStatusesList().size());
-// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
-// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.SUCCESS).size());
-// } catch (Exception e) {
-// fail("Should not throw this exception" + e.getMessage());
-// }
-// }
-//
-// @Test
-// public void should_properlyUpdateAuditLogs_givenValidCreateAndUpdateRecords() {
-// try {
-// Map<String, Object> storageData = new HashMap<>();
-// List<ConversionStatus> conversionStatus = new LinkedList<>();
-//
-// storageData.put("schema1", "test-value");
-// List<Records.Entity> validRecords = new ArrayList<>();
-// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build());
-// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build();
-//
-// when(this.storageService.getStorageRecords(any())).thenReturn(storageRecords);
-// when(this.indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true);
-// Map<String, String> schema = createSchema();
-// indexSchemaServiceMock(kind2, schema);
-// indexSchemaServiceMock(kind1, null);
-// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
-//
-// assertEquals(2, jobStatus.getStatusesList().size());
-// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
-// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.SUCCESS).size());
-//
-// verify(this.auditLogger).indexCreateRecordSuccess(singletonList("RecordStatus(id=tenant1:doc:test2, kind=tenant1:testindexer2:well:1.0.0, operationType=create, status=SUCCESS)"));
-// verify(this.auditLogger).indexUpdateRecordFail(singletonList("RecordStatus(id=tenant1:doc:test1, kind=tenant1:testindexer1:well:1.0.0, operationType=update, status=FAIL)"));
-// } catch (Exception e) {
-// fail("Should not throw this exception" + e.getMessage());
-// }
-// }
-//
-// private BulkItemResponse prepareResponseFail() {
-// BulkItemResponse responseFail = mock(BulkItemResponse.class);
-// when(responseFail.isFailed()).thenReturn(true);
-// when(responseFail.getFailureMessage()).thenReturn(failureMassage);
-// when(responseFail.getId()).thenReturn(recordId1);
-// when(responseFail.getFailure()).thenReturn(new BulkItemResponse.Failure("failure index", "failure type", "failure id", new Exception("test failure")));
-// return responseFail;
-// }
-//
-// private BulkItemResponse prepareResponseSuccess() {
-// BulkItemResponse responseSuccess = mock(BulkItemResponse.class);
-// when(responseSuccess.getId()).thenReturn(recordId2);
-// return responseSuccess;
-// }
-//
-// private void indexSchemaServiceMock(String kind, Map<String, String> schema) {
-// if (schema == null) {
-// IndexSchema indexSchema = IndexSchema.builder().kind(kind).dataSchema(null).build();
-// when(indexSchemaService.getIndexerInputSchema(kind)).thenReturn(indexSchema);
-// } else {
-// IndexSchema indexSchema = IndexSchema.builder().kind(kind).dataSchema(schema).build();
-// when(indexSchemaService.getIndexerInputSchema(kind)).thenReturn(indexSchema);
-// }
-// }
-//
-// private Map<String, String> createSchema() {
-// Map<String, String> schema = new HashMap<>();
-// schema.put("schema1", "keyword");
-// schema.put("schema2", "boolean");
-// schema.put("schema3", "date");
-// schema.put("schema6", "object");
-// return schema;
-// }
-//}
diff --git a/provider/indexer-ibm/pom.xml b/provider/indexer-ibm/pom.xml
index 7360337792ed8a9c59c1735a37dd614f0b3f687b..0c922e0415b7aa243287a578ebe7ba802d126b63 100644
--- a/provider/indexer-ibm/pom.xml
+++ b/provider/indexer-ibm/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-service</artifactId>
- <version>0.13.0-SNAPSHOT</version>
+ <version>0.14.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
@@ -53,7 +53,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-core</artifactId>
- <version>0.13.0-SNAPSHOT</version>
+ <version>0.14.0-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>io.netty</groupId>
diff --git a/provider/indexer-reference/pom.xml b/provider/indexer-reference/pom.xml
index 0a0e1d0964271cc66be97fbf0c528956036e818a..3e1359fa94dae1c660471bfd797f0dc482d8b961 100644
--- a/provider/indexer-reference/pom.xml
+++ b/provider/indexer-reference/pom.xml
@@ -22,12 +22,12 @@
<parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-service</artifactId>
- <version>0.13.0-SNAPSHOT</version>
+ <version>0.14.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>indexer-reference</artifactId>
- <version>0.13.0-SNAPSHOT</version>
+ <version>0.14.0-SNAPSHOT</version>
<name>indexer-reference</name>
<description>Indexer Service GCP Anthos</description>
<packaging>jar</packaging>
@@ -36,7 +36,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-core</artifactId>
- <version>0.13.0-SNAPSHOT</version>
+ <version>0.14.0-SNAPSHOT</version>
</dependency>
<dependency>
diff --git a/testing/indexer-test-aws/pom.xml b/testing/indexer-test-aws/pom.xml
index 1fd89ff029cac32e6c9580dd231add4c010c9ec3..538651fb4407d563794d76bc9f5259b443484991 100644
--- a/testing/indexer-test-aws/pom.xml
+++ b/testing/indexer-test-aws/pom.xml
@@ -21,13 +21,13 @@
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>indexer-test</artifactId>
- <version>0.13.0-SNAPSHOT</version>
+ <version>0.14.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-aws</artifactId>
- <version>0.13.0-SNAPSHOT</version>
+ <version>0.14.0-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
@@ -42,7 +42,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
- <version>0.13.0-SNAPSHOT</version>
+ <version>0.14.0-SNAPSHOT</version>
</dependency>
<!-- AWS specific packages -->
diff --git a/testing/indexer-test-azure/pom.xml b/testing/indexer-test-azure/pom.xml
index 2b157b63a7ad882852ac5e43e61e56e8ba39cbc7..4f4c6e50ac203625417c450c7fab210604ec13cb 100644
--- a/testing/indexer-test-azure/pom.xml
+++ b/testing/indexer-test-azure/pom.xml
@@ -21,13 +21,13 @@
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>indexer-test</artifactId>
- <version>0.13.0-SNAPSHOT</version>
+ <version>0.14.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-azure</artifactId>
- <version>0.13.0-SNAPSHOT</version>
+ <version>0.14.0-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
@@ -45,7 +45,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
- <version>0.13.0-SNAPSHOT</version>
+ <version>0.14.0-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
diff --git a/testing/indexer-test-core/pom.xml b/testing/indexer-test-core/pom.xml
index fcc2b5e81bc27d8f33b10d1ac5a46f1fcbd33e90..51bf9d386095dbcf92745592525ebaa1b360f105 100644
--- a/testing/indexer-test-core/pom.xml
+++ b/testing/indexer-test-core/pom.xml
@@ -5,13 +5,13 @@
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>indexer-test</artifactId>
- <version>0.13.0-SNAPSHOT</version>
+ <version>0.14.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
- <version>0.13.0-SNAPSHOT</version>
+ <version>0.14.0-SNAPSHOT</version>
<properties>
<maven.compiler.target>1.8</maven.compiler.target>
diff --git a/testing/indexer-test-gcp/pom.xml b/testing/indexer-test-gcp/pom.xml
index e86a33cbd846f37586836b850bc340f176093c5b..c32245324cbf5bf9290df99dab3b0ff2886c3bce 100644
--- a/testing/indexer-test-gcp/pom.xml
+++ b/testing/indexer-test-gcp/pom.xml
@@ -6,13 +6,13 @@
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>indexer-test</artifactId>
- <version>0.13.0-SNAPSHOT</version>
+ <version>0.14.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-gcp</artifactId>
- <version>0.13.0-SNAPSHOT</version>
+ <version>0.14.0-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
@@ -37,7 +37,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
- <version>0.13.0-SNAPSHOT</version>
+ <version>0.14.0-SNAPSHOT</version>
</dependency>
<!-- Cucumber -->
diff --git a/testing/indexer-test-ibm/pom.xml b/testing/indexer-test-ibm/pom.xml
index 4e763a5c100f05688ad3036dbce3edf503a1a97b..54cf68bcbb6d03ad7d170990550517ac6ceb1a1a 100644
--- a/testing/indexer-test-ibm/pom.xml
+++ b/testing/indexer-test-ibm/pom.xml
@@ -6,13 +6,13 @@
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>indexer-test</artifactId>
- <version>0.13.0-SNAPSHOT</version>
+ <version>0.14.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-ibm</artifactId>
- <version>0.13.0-SNAPSHOT</version>
+ <version>0.14.0-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
@@ -38,7 +38,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
- <version>0.13.0-SNAPSHOT</version>
+ <version>0.14.0-SNAPSHOT</version>
</dependency>
<dependency>
diff --git a/testing/pom.xml b/testing/pom.xml
index 679e6dcc19bb2beca2e96243c9cbe5105db0dc47..08218f0dd1511159b46ec16efe599ca5d66ca7e2 100644
--- a/testing/pom.xml
+++ b/testing/pom.xml
@@ -18,7 +18,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.opengroup.osdu</groupId>
<artifactId>indexer-test</artifactId>
- <version>0.13.0-SNAPSHOT</version>
+ <version>0.14.0-SNAPSHOT</version>
<description>Indexer Service Integration Test Root Project</description>
<properties>
<spring.version>5.1.19.RELEASE</spring.version>