diff --git a/devops/gcp/deploy/templates/indexer-deploy.yml b/devops/gcp/deploy/templates/indexer-deploy.yml
index fcbe1a8708b76eb20517324fe46cec9bf1eaa6b0..2cd5d863e8075527279c5e4a26fab4d7471b78ad 100644
--- a/devops/gcp/deploy/templates/indexer-deploy.yml
+++ b/devops/gcp/deploy/templates/indexer-deploy.yml
@@ -23,7 +23,9 @@ spec:
imagePullPolicy: "{{ .Values.data.imagePullPolicy }}"
envFrom:
- configMapRef:
- name: "{{ .Values.conf.configmap }}"
+ name: "{{ .Values.conf.configmap }}"
+ - secretRef:
+ name: "{{ .Values.conf.elastic_secret_name }}"
securityContext:
allowPrivilegeEscalation: false
runAsUser: 0
diff --git a/devops/gcp/deploy/values.yaml b/devops/gcp/deploy/values.yaml
index cb2dc074295ad64cccccaf0fa60382016736fb9b..3ae46b413512e04ad6f574c68a9bbd0f9bfe557c 100644
--- a/devops/gcp/deploy/values.yaml
+++ b/devops/gcp/deploy/values.yaml
@@ -13,4 +13,5 @@ data:
conf:
configmap: "indexer-config"
+ elastic_secret_name: "elastic-secret"
app_name: "indexer"
diff --git a/docs/tutorial/IndexerService.md b/docs/tutorial/IndexerService.md
index 350af65429f6e8a8c82f1888e1e37c13e2c2e227..5208cc9b30936fdc9c2ebc75fab57e2f47f9fb82 100644
--- a/docs/tutorial/IndexerService.md
+++ b/docs/tutorial/IndexerService.md
@@ -1,130 +1,107 @@
## Indexer service
### Table of contents <a name="TOC"></a>
+
- [Indexer service](#indexer-service)
- [Introduction](#introduction)
- [Indexer API access](#indexer-api-access)
-- [Get indexing status <a name="get-indexing-status"></a>](#get-indexing-status)
-- [Reindex <a name="reindex"></a>](#reindex)
-- [Schema Service adoption <a name="schema-service-adoption"></a>](#schema-service-adoption)
- - [R3 Schema Support <a name="r3-schema-support"></a>](#r3-schema-support)
- [Version info endpoint](#version-info-endpoint)
-
+- [Reindex](#reindex)
+- [Data Partition provision](#data-partition-provision)
+- [Schema Service adoption](#schema-service-adoption)
+ * [R3 Schema Support](#r3-schema-support)
+- [Troubleshoot Indexing Issues](#troubleshoot-indexing-issues)
+ * [Get indexing status](#get-indexing-status)
## Introduction <a name="introduction"></a>
-The Indexer API provides a mechanism for indexing documents that contain structured or unstructured data. Documents and indices are saved in a separate persistent store optimized for search operations. The indexer API can index any number of documents.
-The indexer is indexes attributes defined in the schema. Schema can be created at the time of record ingestion in Data Ecosystem via Storage Service. The Indexer service also adds number of Data Ecosystem meta attributes such as id, kind, parent, acl, namespace, type, version, legaltags, index to each record at the time of indexing.
+The Indexer API provides a mechanism for indexing documents that contain structured or unstructured data. Documents and
+indices are saved in a separate persistent store optimized for search operations. The indexer API can index any number
+of documents.
+
+The indexer is indexes attributes defined in the schema. Schema can be created at the time of record ingestion in OSDU Data Platform
+via Schema Service. The Indexer service also adds number of OSDU Data Platform meta attributes such as id, kind,
+parent, acl, namespace, type, version, legaltags, index to each record at the time of indexing.
## Indexer API access <a name="indexer-api-access"></a>
* Required roles
- Indexer service requires that users (and service accounts) have dedicated roles in order to use it. The following roles should be created and assigned using the entitlements service:
-
- __users.OSDU.viewers__
+ Indexer service requires that users (and service accounts) have dedicated roles in order to use it. Users must be a member of `users.datalake.viewers` or `users.datalake.editors` or `users.datalake.admins`, roles can be assigned using the [Entitlements Service](/solutions/osdu/tutorials/core-services/entitlementsservice). Please look at the API documentation for specific requirements.
- __users.OSDU.editors__
-
- __users.OSDU.admin__
-
-* Required headers
+ In addition to service roles, users __must__ be a member of data groups to access the data.
- The Data Ecosystem stores data in different data partitions depending on the different accounts in the OSDU system.
+* Required headers
- A user may belong to many partitions in OSDU e.g. a OSDU user may belong to both the OSDU partition and a customer's partition. When a user logs into the OSDU portal they choose which data partition they currently want to be active.
+ The OSDU Data Platform stores data in different partitions, depending on the different accounts in the OSDU system.
- When using the Indexer APIs you need to specify which data partition they currently have active and send it in the OSDU-Data-Partition-Id header. e.g.
+ A user may belong to more than one account. As a user, after logging into the OSDU portal, you need to select the account you wish to be active.
+ Likewise, when using the Search APIs, you need to specify the active account in the header called `data-partition-id`. The correct `data-partition-id` can be obtained from the CFS services. The `data-partition-id` enables the search within the mapped partition. e.g.
```
- OSDU-Data-Partition-Id: OSDU
+ data-partition-id: opendes
```
- The correct values can be obtained from CFS services.
- We use this value to work out which partition to use. There is also a special data partition known as common
- ```
- OSDU-Data-Partition-Id: common
- ```
- This has all public data in the Data Ecosystem. Users always have access to this as well as their current active data partition.
+* Optional headers
- You should also send a correlation id as a header so that a single request can be tracked throughout all the services it passes through. This can be a GUID on the header with a key
+ The `correlation-id` is a traceable ID to track the journey of a single request. The `correlation-id` can be a GUID on the header with a key. It is best practice to provide the correlation-id so the request can be tracked through all the services.
```
- OSDU-Correlation-Id: 1e0fef08-22fd-49b1-a5cc-dffa21bc0b70
+ correlation-id: 1e0fef08-22fd-49b1-a5cc-dffa21bc0b70
```
- If you are the service initiating the request you should generate the id, otherwise you should just forward it on in the request.
-
+If the service is initiating the request, an ID should be generated. If the `correlation-id` is not provided, then a new ID will be generated by the service so that the request would be traceable.
+
[Back to table of contents](#TOC)
-## Get indexing status <a name="get-indexing-status"></a>
+## Version info endpoint
-Indexer service adds internal metadata to each record which registers the status of the indexing. The meta data includes the status and the last indexing date and time. This additional meta block helps to see the details of indexing. The format of the index meta block is as follows:
+For deployment available public `/info` endpoint, which provides build and git related information.
+
+#### Example response:
```json
{
- "index": {
- "trace": [
- String,
- String
- ],
- "statusCode": Integer,
- "lastUpdateTime": Datetime
- }
+ "groupId": "org.opengroup.osdu",
+ "artifactId": "indexer-gcp",
+ "version": "0.10.0-SNAPSHOT",
+ "buildTime": "2021-07-09T14:29:51.584Z",
+ "branch": "feature/GONRG-2681_Build_info",
+ "commitId": "7777",
+ "commitMessage": "Added copyright to version info properties file",
+ "connectedOuterServices": [
+ {
+ "name": "elasticSearch",
+ "version": "..."
+ },
+ {
+ "name": "redis",
+ "version": "..."
+ }
+ ]
}
```
-Example:
-```json
-{
- "results": [
- {
- "index": {
- "trace": [
- "datetime parsing error: unknown format for attribute: endDate | value: 9000-01-01T00:00:00.0000000",
- "datetime parsing error: unknown format for attribute: startDate | value: 1990-01-01T00:00:00.0000000"
- ],
- "statusCode": 400,
- "lastUpdateTime": "2018-11-16T01:44:08.687Z"
- }
- }
- ],
- "totalCount": 31895
-}
-```
-
-Details of the index block:
-1) trace: This field collects all the issues related to the indexing and concatinates using '|'. This is a String field.
-2) statusCode: This field determines the category of the error. This is integer field. It can have the following values:
- * 200 - All OK
- * 404 - Schema is missing in Storage
- * 400 - Some fields were not properly mapped with the schema defined
-3) lastUpdateTime: This field captures the last time the record was updated by by the indexer service. This is datetime field so you can do range queries on this field.
-You can query the index status using the following example query:
-
-```bash
-curl --request POST \
- --url /api/search/v2/query \
- --header 'Authorization: Token' \
- --header 'Content-Type: application/json' \
- --header 'OSDU-Data-Partition-Id: Data partition id' \
- --data '{"kind": "*:*:*:*","query": "index.statusCode:404","returnedFields": ["index"]}'
-
-NOTE: By default, the API response excludes the 'index' attribute block. The user must specify 'index' as the 'returnedFields" in order to see it in the response.
-```
-The above query will return all records which had problems due to fields mismatch.
+This endpoint takes information from files, generated by `spring-boot-maven-plugin`, `git-commit-id-plugin` plugins. Need to specify paths for generated files to matching properties:
-Please refer to the [Search service](searchservice#query) documentation for examples on different kinds of search queries.
+- `version.info.buildPropertiesPath`
+- `version.info.gitPropertiesPath`
[Back to table of contents](#TOC)
-
+
## Reindex <a name="reindex"></a>
-Reindex API allows users to re-index a `kind` without re-ingesting the records via storage API. Reindexing a kind is an asynchronous operation and when a user calls this API, it will respond with HTTP 200 if it can launch the re-indexing or appropriate error code if it cannot. The current status of the indexing can be tracked by calling search API and making query with this particular kind. Please be advised, it may take few seconds to few hours to finish the re-indexing as multiple factors contribute to latency, such as number of records in the kind, current load at the indexer service etc.
+Reindex API allows users to re-index a `kind` without re-ingesting the records via storage API. Reindexing a kind is an
+asynchronous operation and when a user calls this API, it will respond with HTTP 200 if it can launch the re-indexing or
+appropriate error code if it cannot. The current status of the indexing can be tracked by calling search API and making
+query with this particular kind. Please be advised, it may take few seconds to few hours to finish the re-indexing as
+multiple factors contribute to latency, such as number of records in the kind, current load at the indexer service etc.
-__Note__: If a kind has been previously indexed with particular schema and if you wish to apply the schema changes during re-indexing, previous kind index has to be deleted via Index Delete API. In absence of this clean-up, reindex API will use the same schema and overwrite records with the same ids.
+__Note__: If a kind has been previously indexed with particular schema and if you wish to apply the schema changes
+during re-indexing, previous kind index has to be deleted via Index Delete API. In absence of this clean-up, reindex API
+will use the same schema and overwrite records with the same ids.
```
POST /api/indexer/v2/reindex
{
- "kind": "common:welldb:wellbore:1.0.0"
+ "kind": "opendes:welldb:wellbore:1.0.0"
}
```
@@ -136,32 +113,61 @@ curl --request POST \
--header 'accept: application/json' \
--header 'authorization: Bearer <JWT>' \
--header 'content-type: application/json' \
- --header 'OSDU-Data-Partition-Id: common' \
+ --header 'data-partition-id: opendes' \
--data '{
- "kind": "common:welldb:wellbore:1.0.0"
+ "kind": "opendes:welldb:wellbore:1.0.0"
}'
```
+
</details>
[Back to table of contents](#TOC)
-##Schema Service adoption <a name="schema-service-adoption"></a>
+## Data Partition provision <a name="data-partition-provision"></a>
+
+Configures Search backend for a data partition.
-Indexer service is in adaptation process to use schemas from the Schema service instead of Storage Service.
-The Indexer Service retrieves a schema from the Schema Service if the schema is not found on the Storage Service.
-Change affects only Azure implementation so far. Later call to the Storage Service will be deprecated and then removed (after the end of the deprecation period).
+```
+PUT /api/indexer/v2/partitions/provision
+```
+
+<details><summary>**Curl**</summary>
+
+```bash
+curl --request PUT \
+ --url '/api/indexer/v2/partitions/provision' \
+ --header 'accept: application/json' \
+ --header 'authorization: Bearer <JWT>' \
+ --header 'content-type: application/json' \
+ --header 'data-partition-id: opendes''
+```
+</details>
+
+> __NOTE__: API should be run at-least once at the data partition provisioning to configure required resources/settings.
+
+[Back to table of contents](#TOC)
+
+## Schema Service adoption <a name="schema-service-adoption"></a>
+
+Indexer service is in adaptation process to use schemas from the Schema service instead of Storage Service. The Indexer
+Service retrieves a schema from the Schema Service if the schema is not found on the Storage Service. Change affects
+only Azure implementation so far. Later call to the Storage Service will be deprecated and then removed (after the end
+of the deprecation period).
[Back to table of contents](#TOC)
-###R3 Schema Support <a name="r3-schema-support"></a>
+### R3 Schema Support <a name="r3-schema-support"></a>
-Indexer service support r3 schema. These schemas are created via Schema service.
+Indexer service support r3 schema. These schemas are created via Schema service.
Here is an example following end-to-end workflow can be exercised (please update the schema based on your environment):
-* Ingest r3 schema for `opendes:wks:master-data--Wellbore:1.0.0`. Schema service payload can be found [here](https://community.opengroup.org/osdu/platform/system/indexer-service/-/blob/master/testing/indexer-test-core/src/main/resources/testData/r3-index_record_wks_master.schema.json).
+* Ingest r3 schema for `opendes:wks:master-data--Wellbore:1.0.0`. Schema service payload can be
+ found [here](https://community.opengroup.org/osdu/platform/system/indexer-service/-/blob/master/testing/indexer-test-core/src/main/resources/testData/r3-index_record_wks_master.schema.json)
+ .
-* Ingest r3 master-data Wellbore record. Storage service payload can be found [here](https://community.opengroup.org/osdu/platform/system/indexer-service/-/blob/master/testing/indexer-test-core/src/main/resources/testData/r3-index_record_wks_master.json)
+* Ingest r3 master-data Wellbore record. Storage service payload can be
+ found [here](https://community.opengroup.org/osdu/platform/system/indexer-service/-/blob/master/testing/indexer-test-core/src/main/resources/testData/r3-index_record_wks_master.json)
* Records can be searched via Search service. Here is sample payload:
@@ -186,40 +192,76 @@ data-partition-id: opendes
}
}
```
+
[Back to table of contents](#TOC)
-## Version info endpoint
-For deployment available public `/info` endpoint, which provides build and git related information.
-#### Example response:
+# Troubleshoot Indexing Issues <a name="troubleshoot-indexing-issues"></a>
+
+## Get indexing status <a name="get-indexing-status"></a>
+
+Indexer service adds internal metadata to each record which registers the status of the indexing. The meta data includes
+the status and the last indexing date and time. This additional meta block helps to see the details of indexing. The
+format of the index meta block is as follows:
+
```json
{
- "groupId": "org.opengroup.osdu",
- "artifactId": "storage-gcp",
- "version": "0.10.0-SNAPSHOT",
- "buildTime": "2021-07-09T14:29:51.584Z",
- "branch": "feature/GONRG-2681_Build_info",
- "commitId": "7777",
- "commitMessage": "Added copyright to version info properties file",
- "connectedOuterServices": [
- {
- "name": "elasticSearch",
- "version":"..."
- },
- {
- "name": "postgresSql",
- "version":"..."
- },
- {
- "name": "redis",
- "version":"..."
- }
- ]
+ "index": {
+ "trace": [
+ String,
+ String
+ ],
+ "statusCode": Integer,
+ "lastUpdateTime": Datetime
+ }
}
```
-This endpoint takes information from files, generated by `spring-boot-maven-plugin`,
-`git-commit-id-plugin` plugins. Need to specify paths for generated files to matching
-properties:
-- `version.info.buildPropertiesPath`
-- `version.info.gitPropertiesPath`
-[Back to table of contents](#TOC)
+Example:
+
+```json
+{
+ "results": [
+ {
+ "index": {
+ "trace": [
+ "datetime parsing error: unknown format for attribute: endDate | value: 9000-01-01T00:00:00.0000000",
+ "datetime parsing error: unknown format for attribute: startDate | value: 1990-01-01T00:00:00.0000000"
+ ],
+ "statusCode": 400,
+ "lastUpdateTime": "2018-11-16T01:44:08.687Z"
+ }
+ }
+ ],
+ "totalCount": 31895
+}
+```
+
+Details of the index block:
+
+1) trace: This field collects all the issues related to the indexing and concatenates using '|'. This is a String field.
+2) statusCode: This field determines the category of the error. This is integer field. It can have the following values:
+ * 200 - All OK
+ * 404 - Schema is missing in Storage
+ * 400 - Some fields were not properly mapped with the schema defined
+3) lastUpdateTime: This field captures the last time the record was updated by the indexer service. This is datetime
+ field so you can do range queries on this field.
+
+You can query the index status using the following example query:
+
+```bash
+curl --request POST \
+ --url /api/search/v2/query \
+ --header 'Authorization: Token' \
+ --header 'Content-Type: application/json' \
+ --header 'data-partition-id: Data partition id' \
+ --data '{"kind": "*:*:*:*","query": "index.statusCode:404","returnedFields": ["index"]}'
+
+NOTE: By default, the API response excludes the 'index' attribute block. The user must specify 'index' as the 'returnedFields" in order to see it in the response.
+```
+
+The above query will return all records which had problems due to fields mismatch.
+
+Please refer to the [Search service](https://community.opengroup.org/osdu/platform/system/search-service/-/blob/master/docs/api/search_openapi.yaml#L28) documentation for examples on different kinds of search
+queries.
+
+[Back to table of contents](#TOC)
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/PartitionSetupApi.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/PartitionSetupApi.java
new file mode 100644
index 0000000000000000000000000000000000000000..26435bfa27a8f90ff6d66d4721088b916a216f4c
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/PartitionSetupApi.java
@@ -0,0 +1,52 @@
+// Copyright © Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.api;
+
+import org.opengroup.osdu.indexer.logging.AuditLogger;
+import org.opengroup.osdu.indexer.service.IClusterConfigurationService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.ResponseEntity;
+import org.springframework.security.access.prepost.PreAuthorize;
+import org.springframework.web.bind.annotation.PutMapping;
+import org.springframework.web.bind.annotation.RequestHeader;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.context.annotation.RequestScope;
+
+import java.io.IOException;
+
+import static java.util.Collections.singletonList;
+import static org.opengroup.osdu.core.common.model.http.DpsHeaders.DATA_PARTITION_ID;
+
+@RestController
+@RequestMapping("/partitions")
+@RequestScope
+public class PartitionSetupApi {
+
+ private static final String OPS = "users.datalake.ops";
+
+ @Autowired
+ private IClusterConfigurationService clusterConfigurationService;
+ @Autowired
+ private AuditLogger auditLogger;
+
+ @PreAuthorize("@authorizationFilter.hasPermission('" + OPS + "')")
+ @PutMapping(path = "/provision", consumes = "application/json")
+ public ResponseEntity<?> provisionPartition(@RequestHeader(DATA_PARTITION_ID) String dataPartitionId) throws IOException {
+ this.clusterConfigurationService.updateClusterConfiguration();
+ this.auditLogger.getConfigurePartition(singletonList(dataPartitionId));
+ return new ResponseEntity<>(org.springframework.http.HttpStatus.OK);
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafeIndexCache.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafeIndexCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..0942b5ecbc5decfc6d9902f96080505b7659f662
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafeIndexCache.java
@@ -0,0 +1,51 @@
+// Copyright © Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.provider.interfaces.IIndexCache;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+@Component
+@RequestScope
+public class PartitionSafeIndexCache {
+
+ @Autowired
+ private IIndexCache indexCache;
+ @Autowired
+ private IRequestInfo requestInfo;
+
+ public void put(String s, Boolean o) {
+ this.indexCache.put(cacheKey(s), o);
+ }
+
+ public Boolean get(String s) {
+ return (Boolean) this.indexCache.get(cacheKey(s));
+ }
+
+ public void delete(String s) {
+ this.indexCache.delete(cacheKey(s));
+ }
+
+ public void clearAll() {
+ this.indexCache.clearAll();
+ }
+
+ private String cacheKey(String s) {
+ return this.requestInfo.getPartitionId() + "-" + s;
+ }
+}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditEvents.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditEvents.java
index 1f0970f1db6b0a629b2dbfa7a34d1b3d54f11a04..703d6747e2510d71ad0b27d1fe9843c697ea5ea8 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditEvents.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditEvents.java
@@ -57,6 +57,9 @@ public class AuditEvents {
private static final String INDEX_MAPPING_UPDATE_SUCCESS = "Successfully updated index mapping";
private static final String INDEX_MAPPING_UPDATE_FAILURE = "Failed updating index mapping";
+ private static final String CONFIGURE_PARTITION_ACTION_ID = "IN0012";
+ private static final String CONFIGURE_PARTITION_OPERATION = "Data partition cluster configuration update";
+
private final String user;
public AuditEvents(String user) {
@@ -209,6 +212,17 @@ public class AuditEvents {
.build();
}
+ public AuditPayload getConfigurePartitionEvent(List<String> resources) {
+ return AuditPayload.builder()
+ .action(AuditAction.UPDATE)
+ .status(AuditStatus.SUCCESS)
+ .actionId(CONFIGURE_PARTITION_ACTION_ID)
+ .message(CONFIGURE_PARTITION_OPERATION)
+ .resources(resources)
+ .user(this.user)
+ .build();
+ }
+
public AuditPayload getIndexMappingUpdateEvent(List<String> resources, boolean isSuccess) {
if (isSuccess) {
return AuditPayload.builder()
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditLogger.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditLogger.java
index 431e99afc29e4f4714ce3121057ecae2a81891ed..1a37d76881a1bbc49bb2b4c97cc037f8a8af2a32 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditLogger.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditLogger.java
@@ -100,6 +100,10 @@ public class AuditLogger {
this.writeLog(this.getAuditEvents().getIndexMappingUpdateEvent(resources,false));
}
+ public void getConfigurePartition(List<String> resources) {
+ this.writeLog(this.getAuditEvents().getConfigurePartitionEvent(resources));
+ }
+
private void writeLog(AuditPayload log) {
this.logger.audit(log);
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ClusterConfigurationServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ClusterConfigurationServiceImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..63543864722f29546604a2dcd8df76faa78a5a4f
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ClusterConfigurationServiceImpl.java
@@ -0,0 +1,50 @@
+// Copyright © Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.service;
+
+import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
+import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.opengroup.osdu.indexer.util.ElasticClientHandler;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.io.IOException;
+
+@Service
+public class ClusterConfigurationServiceImpl implements IClusterConfigurationService {
+
+ @Autowired
+ private ElasticClientHandler elasticClientHandler;
+
+ @Override
+ public boolean updateClusterConfiguration() throws IOException {
+ ClusterUpdateSettingsRequest request = new ClusterUpdateSettingsRequest();
+
+ Settings persistentSettings =
+ Settings.builder()
+ .put("action.auto_create_index", "false")
+ .build();
+ request.persistentSettings(persistentSettings);
+ request.timeout(TimeValue.timeValueMinutes(1));
+ try (RestHighLevelClient client = this.elasticClientHandler.createRestClient()) {
+ ClusterUpdateSettingsResponse response = client.cluster().putSettings(request, RequestOptions.DEFAULT);
+ return response.isAcknowledged();
+ }
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IClusterConfigurationService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IClusterConfigurationService.java
new file mode 100644
index 0000000000000000000000000000000000000000..25255be14b13acb645e3849599fa15cb744608c3
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IClusterConfigurationService.java
@@ -0,0 +1,22 @@
+// Copyright © Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.service;
+
+import java.io.IOException;
+
+public interface IClusterConfigurationService {
+
+ boolean updateClusterConfiguration() throws IOException;
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
index 329ebd3245eef95724ac38a84c3cd15de49a89bc..6acd078e913b9733f3d95e335461f397a6966525 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
@@ -35,8 +35,8 @@ import org.opengroup.osdu.core.common.Constants;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
-import org.opengroup.osdu.core.common.provider.interfaces.IIndexCache;
import org.opengroup.osdu.core.common.search.Preconditions;
+import org.opengroup.osdu.indexer.cache.PartitionSafeIndexCache;
import org.opengroup.osdu.indexer.util.ElasticClientHandler;
import org.opengroup.osdu.indexer.util.TypeMapper;
import org.springframework.beans.factory.annotation.Autowired;
@@ -55,7 +55,7 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements IMa
@Inject
private ElasticClientHandler elasticClientHandler;
@Autowired
- private IIndexCache indexCache;
+ private PartitionSafeIndexCache indexCache;
@Autowired
private IMappingService mappingService;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
index e5ffb3b7aedf3474537977cb1d313dd506e9482d..b0a565847e31101e905940cfcd4f389c785f38c2 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
@@ -19,6 +19,7 @@ import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import org.apache.http.HttpStatus;
import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.action.DocWriteRequest;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
@@ -430,6 +431,8 @@ public class IndexerServiceImpl implements IndexerService {
List<String> failureRecordIds = new LinkedList<>();
if (bulkRequest.numberOfActions() == 0) return failureRecordIds;
+ int failedRequestStatus = 500;
+ Exception failedRequestCause = null;
try {
BulkResponse bulkResponse = restClient.bulk(bulkRequest, RequestOptions.DEFAULT);
@@ -442,11 +445,17 @@ public class IndexerServiceImpl implements IndexerService {
for (BulkItemResponse bulkItemResponse : bulkResponse.getItems()) {
if (bulkItemResponse.isFailed()) {
BulkItemResponse.Failure failure = bulkItemResponse.getFailure();
- bulkFailures.add(String.format("elasticsearch bulk service status: %s id: %s message: %s", failure.getStatus(), failure.getId(), failure.getMessage()));
+ bulkFailures.add(String.format("elasticsearch bulk service status: %s | id: %s | message: %s", failure.getStatus(), failure.getId(), failure.getMessage()));
this.jobStatus.addOrUpdateRecordStatus(bulkItemResponse.getId(), IndexingStatus.FAIL, failure.getStatus().getStatus(), bulkItemResponse.getFailureMessage());
- if (RETRY_ELASTIC_EXCEPTION.contains(bulkItemResponse.status())) {
+ if (canIndexerRetry(bulkItemResponse)) {
failureRecordIds.add(bulkItemResponse.getId());
+
+ if (failedRequestCause == null) {
+ failedRequestCause = failure.getCause();
+ failedRequestStatus = failure.getStatus().getStatus();
+ }
}
+
failedResponses++;
} else {
succeededResponses++;
@@ -456,12 +465,18 @@ public class IndexerServiceImpl implements IndexerService {
if (!bulkFailures.isEmpty()) this.jaxRsDpsLog.warning(bulkFailures);
jaxRsDpsLog.info(String.format("records in elasticsearch service bulk request: %s | successful: %s | failed: %s", bulkRequest.numberOfActions(), succeededResponses, failedResponses));
+
+ // retry entire message if all records are failing
+ if (bulkRequest.numberOfActions() == failureRecordIds.size()) throw new AppException(failedRequestStatus, "Elastic error", failedRequestCause.getMessage(), failedRequestCause);
} catch (IOException e) {
// throw explicit 504 for IOException
throw new AppException(HttpStatus.SC_GATEWAY_TIMEOUT, "Elastic error", "Request cannot be completed in specified time.", e);
} catch (ElasticsearchStatusException e) {
throw new AppException(e.status().getStatus(), "Elastic error", e.getMessage(), e);
} catch (Exception e) {
+ if (e instanceof AppException) {
+ throw e;
+ }
throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error", "Error indexing records.", e);
}
return failureRecordIds;
@@ -506,6 +521,17 @@ public class IndexerServiceImpl implements IndexerService {
return indexerPayload;
}
+ private boolean canIndexerRetry(BulkItemResponse bulkItemResponse) {
+ if (RETRY_ELASTIC_EXCEPTION.contains(bulkItemResponse.status())) return true;
+
+ if ((bulkItemResponse.getOpType() == DocWriteRequest.OpType.CREATE || bulkItemResponse.getOpType() == DocWriteRequest.OpType.UPDATE)
+ && bulkItemResponse.status() == RestStatus.NOT_FOUND) {
+ return true;
+ }
+
+ return false;
+ }
+
private void retryAndEnqueueFailedRecords(List<RecordInfo> recordInfos, List<String> failuresRecordIds, RecordChangedMessages message) throws IOException {
jaxRsDpsLog.info(String.format("queuing bulk failed records back to task-queue for retry | count: %s | records: %s", failuresRecordIds.size(), failuresRecordIds));
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java
index 50f58e9d9c38bc0ccd4e155904b868d8ae5d3cb9..ad1d20f43d6c72f8371331e70779f1e7dd18a791 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java
@@ -35,13 +35,13 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.rest.RestStatus;
-import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.provider.interfaces.IIndexCache;
+import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.search.IndexInfo;
-import org.opengroup.osdu.indexer.util.ElasticClientHandler;
import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
import org.opengroup.osdu.core.common.search.Preconditions;
+import org.opengroup.osdu.indexer.cache.PartitionSafeIndexCache;
+import org.opengroup.osdu.indexer.util.ElasticClientHandler;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.web.context.annotation.RequestScope;
@@ -61,7 +61,7 @@ public class IndicesServiceImpl implements IndicesService {
@Autowired
private ElasticIndexNameResolver elasticIndexNameResolver;
@Autowired
- private IIndexCache indexCache;
+ private PartitionSafeIndexCache indexCache;
@Autowired
private JaxRsDpsLog log;
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/PartitionSetupApiTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/PartitionSetupApiTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..03c6d718f261f8e4d7b9f7fa7120d5a11fff86be
--- /dev/null
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/PartitionSetupApiTest.java
@@ -0,0 +1,57 @@
+// Copyright © Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.api;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.indexer.logging.AuditLogger;
+import org.opengroup.osdu.indexer.service.IClusterConfigurationService;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.when;
+
+@RunWith(SpringRunner.class)
+public class PartitionSetupApiTest {
+
+ @Mock
+ private AuditLogger auditLogger;
+ @Mock
+ private IClusterConfigurationService clusterConfigurationService;
+ @InjectMocks
+ private PartitionSetupApi sut;
+
+ @Test
+ public void should_return200_when_valid_kind_provided() throws IOException {
+ ResponseEntity<?> response = this.sut.provisionPartition("opendes");
+
+ assertEquals(HttpStatus.OK, response.getStatusCode());
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throwAppException_ifUnknownExceptionCaught_reindexTest() throws IOException {
+ when(this.clusterConfigurationService.updateClusterConfiguration()).thenThrow(new AppException(500, "", ""));
+
+ this.sut.provisionPartition("opendes");
+ }
+}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/cache/PartitionSafeIndexCacheTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/cache/PartitionSafeIndexCacheTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..106aad0b910d85fc818b307f264a7218fb6cae63
--- /dev/null
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/cache/PartitionSafeIndexCacheTest.java
@@ -0,0 +1,70 @@
+// Copyright © Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.provider.interfaces.IIndexCache;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import static org.mockito.Mockito.*;
+
+@RunWith(SpringRunner.class)
+public class PartitionSafeIndexCacheTest {
+
+ @Mock
+ private IIndexCache cache;
+ @Mock
+ private IRequestInfo requestInfo;
+ @InjectMocks
+ private PartitionSafeIndexCache sut;
+
+ @Test
+ public void should_addopendesNamToKey_when_addingToCache() {
+ when(this.requestInfo.getPartitionId()).thenReturn("opendes");
+
+ this.sut.put("key", true);
+
+ verify(this.cache, times(1)).put("opendes-key", true);
+ }
+
+ @Test
+ public void should_addopendesNamToKey_when_deletingFromCache() {
+ when(this.requestInfo.getPartitionId()).thenReturn("opendes");
+
+ this.sut.delete("key");
+
+ verify(this.cache, times(1)).delete("opendes-key");
+ }
+
+ @Test
+ public void should_addopendesNamToKey_when_retrievingfromCache() {
+ when(this.requestInfo.getPartitionId()).thenReturn("opendes");
+
+ this.sut.get("key");
+
+ verify(this.cache, times(1)).get("opendes-key");
+ }
+
+ @Test
+ public void should_callWrappedClearCache() {
+ this.sut.clearAll();
+
+ verify(this.cache, times(1)).clearAll();
+ }
+}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/logging/AuditEventsTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/logging/AuditEventsTest.java
index 9f9b8d7123f5cd0243d9190f73df5a9b393d335b..685482e2833c2bfb02509b27c8c29b07b73691a8 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/logging/AuditEventsTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/logging/AuditEventsTest.java
@@ -227,4 +227,18 @@ public class AuditEventsTest {
assertEquals("IN0011", payload.get("actionId"));
assertEquals("testUser", payload.get("user"));
}
+
+ @Test
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ public void should_getConfigurePartitionEvent() {
+ AuditEvents auditEvent = new AuditEvents("testUser");
+ Map<String, String> payload = (Map) auditEvent.getConfigurePartitionEvent(Lists.newArrayList("anything"))
+ .get("auditLog");
+ assertEquals(Lists.newArrayList("anything"), payload.get("resources"));
+ assertEquals(AuditStatus.SUCCESS, payload.get("status"));
+ assertEquals("Data partition cluster configuration update", payload.get("message"));
+ assertEquals(AuditAction.UPDATE, payload.get("action"));
+ assertEquals("IN0012", payload.get("actionId"));
+ assertEquals("testUser", payload.get("user"));
+ }
}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/logging/AuditLoggerTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/logging/AuditLoggerTest.java
index d6e0cf1a06705e47e08f82918a59b2f8ba2fb6b5..63906cfdc636439514809aded529e17524b14a23 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/logging/AuditLoggerTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/logging/AuditLoggerTest.java
@@ -196,6 +196,7 @@ public class AuditLoggerTest {
assertEquals("IN0011", ((Map) payload.get("auditLog")).get("actionId"));
assertEquals("testUser", ((Map) payload.get("auditLog")).get("user"));
}
+
@Test
@SuppressWarnings("rawtypes")
public void should_createAuditLogEvent_when_indexMappingUpdateSuccess() {
@@ -206,4 +207,15 @@ public class AuditLoggerTest {
assertEquals("IN0011", ((Map) payload.get("auditLog")).get("actionId"));
assertEquals("testUser", ((Map) payload.get("auditLog")).get("user"));
}
+
+ @Test
+ @SuppressWarnings("rawtypes")
+ public void should_createAuditLogEvent_when_configurePartitionSuccess() {
+ this.sut.getConfigurePartition(Lists.newArrayList("anything"));
+ ArgumentCaptor<AuditPayload> payloadCaptor = ArgumentCaptor.forClass(AuditPayload.class);
+ verify(this.logger).audit(payloadCaptor.capture());
+ AuditPayload payload = payloadCaptor.getValue();
+ assertEquals("IN0012", ((Map) payload.get("auditLog")).get("actionId"));
+ assertEquals("testUser", ((Map) payload.get("auditLog")).get("user"));
+ }
}
\ No newline at end of file
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/ClusterConfigurationServiceTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/ClusterConfigurationServiceTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..08bf634fa5875d5b4ec26010ffcf296b414e0f65
--- /dev/null
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/ClusterConfigurationServiceTest.java
@@ -0,0 +1,72 @@
+// Copyright © Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.service;
+
+import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
+import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse;
+import org.elasticsearch.client.ClusterClient;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.indexer.util.ElasticClientHandler;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.io.IOException;
+
+import static junit.framework.TestCase.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.mockito.MockitoAnnotations.initMocks;
+
+@RunWith(SpringRunner.class)
+@PrepareForTest({RestHighLevelClient.class, ClusterClient.class})
+public class ClusterConfigurationServiceTest {
+
+ @Mock
+ private ElasticClientHandler elasticClientHandler;
+ @InjectMocks
+ private ClusterConfigurationServiceImpl sut;
+
+ private RestHighLevelClient restHighLevelClient;
+ private ClusterClient clusterClient;
+
+ @Before
+ public void setup() {
+ initMocks(this);
+ clusterClient = PowerMockito.mock(ClusterClient.class);
+ restHighLevelClient = PowerMockito.mock(RestHighLevelClient.class);
+ }
+
+ @Test
+ public void should_updateClusterConfiguration() throws IOException {
+ ClusterUpdateSettingsResponse clusterUpdateSettingsResponse = mock(ClusterUpdateSettingsResponse.class);
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ when(clusterUpdateSettingsResponse.isAcknowledged()).thenReturn(true);
+ doReturn(clusterClient).when(restHighLevelClient).cluster();
+ doReturn(clusterUpdateSettingsResponse).when(clusterClient).putSettings(any(ClusterUpdateSettingsRequest.class), any(RequestOptions.class));
+
+ boolean result = this.sut.updateClusterConfiguration();
+
+ assertTrue(result);
+ }
+}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java
index b52abb7c8598bb3a7e8ed3a1c064cf8c445ee1b1..c3ff4bf968ffbf21d41dd516f340bd92cb3e6a41 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java
@@ -27,7 +27,7 @@ import org.mockito.Mock;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
-import org.opengroup.osdu.core.common.provider.interfaces.IIndexCache;
+import org.opengroup.osdu.indexer.cache.PartitionSafeIndexCache;
import org.opengroup.osdu.indexer.util.ElasticClientHandler;
import org.opengroup.osdu.indexer.util.TypeMapper;
import org.powermock.api.mockito.PowerMockito;
@@ -70,7 +70,7 @@ public class IndexerMappingServiceTest {
@Mock
private ElasticClientHandler elasticClientHandler;
@Mock
- private IIndexCache indexCache;
+ private PartitionSafeIndexCache indexCache;
@Mock
private IMappingService mappingService;
@InjectMocks
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceImplTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceImplTest.java
index 026ff32cb2e04a3da21b765baefc282d056915bf..285b384a017b4eefd044dc3e58e1434b1ad0543c 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceImplTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceImplTest.java
@@ -1,55 +1,206 @@
package org.opengroup.osdu.indexer.service;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.MockitoAnnotations.initMocks;
-import java.util.ArrayList;
-import java.util.List;
+import com.google.common.reflect.TypeToken;
+import com.google.gson.Gson;
+import org.elasticsearch.action.bulk.BulkItemResponse;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.RestHighLevelClient;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
-import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
+import org.mockito.Spy;
+import org.opengroup.osdu.core.common.http.HeadersUtil;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.entitlements.Acl;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.indexer.*;
+import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
+import org.opengroup.osdu.core.common.model.storage.ConversionStatus;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
+import org.opengroup.osdu.indexer.logging.AuditLogger;
+import org.opengroup.osdu.indexer.provider.interfaces.IPublisher;
import org.opengroup.osdu.indexer.util.ElasticClientHandler;
-import org.springframework.test.context.junit4.SpringRunner;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
-@RunWith(SpringRunner.class)
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.lang.reflect.Type;
+import java.net.URISyntaxException;
+import java.util.*;
+
+import static java.util.Collections.singletonList;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.MockitoAnnotations.initMocks;
+import static org.powermock.api.mockito.PowerMockito.*;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({RestHighLevelClient.class, BulkResponse.class, Acl.class, HeadersUtil.class})
public class IndexerServiceImplTest {
- @InjectMocks
- private IndexerServiceImpl indexerService;
+ @InjectMocks
+ private IndexerServiceImpl sut;
+ @Mock
+ private ElasticClientHandler elasticClientHandler;
+ @Mock
+ private ElasticIndexNameResolver elasticIndexNameResolver;
+ @Mock
+ private IndicesService indicesService;
+ @Mock
+ private StorageService storageService;
+ @Mock
+ private StorageIndexerPayloadMapper storageIndexerPayloadMapper;
+ @InjectMocks
+ @Spy
+ private JobStatus jobStatus = new JobStatus();
+ @Mock
+ private AuditLogger auditLogger;
+ @Mock
+ private BulkResponse bulkResponse;
+ @Mock
+ private IRequestInfo requestInfo;
+ @Mock
+ private RestHighLevelClient restHighLevelClient;
+ @Mock
+ private IndexSchemaService schemaService;
+ @Mock
+ private JaxRsDpsLog jaxRsDpsLog;
+ @Mock
+ private IMappingService mappingService;
+ @Mock
+ private IPublisher progressPublisher;
+
+ private List<RecordInfo> recordInfos = new ArrayList<>();
+
+ private final String pubsubMsg = "[{\"id\":\"opendes:doc:test1\",\"kind\":\"opendes:testindexer1:well:1.0.0\",\"op\":\"update\"}," +
+ "{\"id\":\"opendes:doc:test2\",\"kind\":\"opendes:testindexer2:well:1.0.0\",\"op\":\"create\"}]";
+ private final String kind1 = "opendes:testindexer1:well:1.0.0";
+ private final String kind2 = "opendes:testindexer2:well:1.0.0";
+ private final String recordId1 = "opendes:doc:test1";
+ private final String recordId2 = "opendes:doc:test2";
+ private final String failureMassage = "test failure";
+
+ private DpsHeaders dpsHeaders;
+ private RecordChangedMessages recordChangedMessages;
+
+ @Before
+ public void setup() throws IOException {
+ }
+
+ @Test
+ public void processSchemaMessagesTest() throws Exception {
+ RecordInfo recordInfo = new RecordInfo();
+ recordInfo.setId("opendes:ds:mytest3-d9033ae1-fb15-496c-9ba0-880fd1d2b2qf");
+ recordInfo.setKind("opendes:ds:mytest2:1.0.0");
+ recordInfo.setOp("purge_schema");
+ this.recordInfos.add(recordInfo);
+
+ initMocks(this);
+
+ this.sut.processSchemaMessages(recordInfos);
+
+ verify(this.elasticClientHandler, times(1)).createRestClient();
+ verify(this.elasticIndexNameResolver, times(1)).getIndexNameFromKind(any());
+ verify(this.indicesService, times(1)).isIndexExist(any(), any());
+ }
+
+ @Test
+ public void should_properlyUpdateAuditLogs_givenValidCreateAndUpdateRecords() {
+ try {
+ mockStatic(Acl.class);
+
+ // setup headers
+ this.dpsHeaders = new DpsHeaders();
+ this.dpsHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
+ when(this.requestInfo.getHeaders()).thenReturn(dpsHeaders);
+ when(this.requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(dpsHeaders.getHeaders());
+
+ // setup message
+ Type listType = new TypeToken<List<RecordInfo>>() {}.getType();
+ this.recordInfos = (new Gson()).fromJson(this.pubsubMsg, listType);
+ Map<String, String> messageAttributes = new HashMap<>();
+ messageAttributes.put(DpsHeaders.DATA_PARTITION_ID, "opendes");
+ this.recordChangedMessages = RecordChangedMessages.builder().attributes(messageAttributes).messageId("xxxx").publishTime("2000-01-02T10:10:44+0000").data("{}").build();
+
+ // setup schema
+ Map<String, Object> schema = createSchema();
+ indexSchemaServiceMock(kind2, schema);
+ indexSchemaServiceMock(kind1, null);
+
+ // setup storage records
+ Map<String, Object> storageData = new HashMap<>();
+ storageData.put("schema1", "test-value");
+ List<Records.Entity> validRecords = new ArrayList<>();
+ validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build());
+ List<ConversionStatus> conversionStatus = new LinkedList<>();
+ Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build();
+ when(this.storageService.getStorageRecords(any())).thenReturn(storageRecords);
+
+ // setup elastic, index and mapped document
+ when(this.indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true);
+ when(this.mappingService.getIndexMappingFromRecordSchema(any())).thenReturn(new HashMap<>());
+
+ when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient);
+ when(this.restHighLevelClient.bulk(any(), any(RequestOptions.class))).thenReturn(this.bulkResponse);
+
+ Map<String, Object> indexerMappedPayload = new HashMap<>();
+ indexerMappedPayload.put("id", "keyword");
+ when(this.storageIndexerPayloadMapper.mapDataPayload(any(), any(), any())).thenReturn(indexerMappedPayload);
- @Mock
- private ElasticClientHandler elasticClientHandler;
+ BulkItemResponse[] responses = new BulkItemResponse[]{prepareFailedResponse(), prepareSuccessfulResponse()};
+ when(this.bulkResponse.getItems()).thenReturn(responses);
- @Mock
- private ElasticIndexNameResolver elasticIndexNameResolver;
+ // test
+ JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
- @Mock
- private IndicesService indicesService;
+ // validate
+ assertEquals(2, jobStatus.getStatusesList().size());
+ assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
+ assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.SUCCESS).size());
- private List<RecordInfo> recordInfos = new ArrayList<>();
+ verify(this.auditLogger).indexCreateRecordSuccess(singletonList("RecordStatus(id=opendes:doc:test2, kind=opendes:testindexer2:well:1.0.0, operationType=create, status=SUCCESS)"));
+ verify(this.auditLogger).indexUpdateRecordFail(singletonList("RecordStatus(id=opendes:doc:test1, kind=opendes:testindexer1:well:1.0.0, operationType=update, status=FAIL, message=test failure)"));
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
- @Before
- public void setup() {
- RecordInfo recordInfo = new RecordInfo();
- recordInfo.setId("opendes:ds:mytest3-d9033ae1-fb15-496c-9ba0-880fd1d2b2qf");
- recordInfo.setKind("opendes:ds:mytest2:1.0.0");
- recordInfo.setOp("purge_schema");
- recordInfos.add(recordInfo);
+ private BulkItemResponse prepareFailedResponse() {
+ BulkItemResponse responseFail = mock(BulkItemResponse.class);
+ when(responseFail.isFailed()).thenReturn(true);
+ when(responseFail.getFailureMessage()).thenReturn(failureMassage);
+ when(responseFail.getId()).thenReturn(recordId1);
+ when(responseFail.getFailure()).thenReturn(new BulkItemResponse.Failure("failure index", "failure type", "failure id", new Exception("test failure")));
+ return responseFail;
+ }
- initMocks(this);
- }
+ private BulkItemResponse prepareSuccessfulResponse() {
+ BulkItemResponse responseSuccess = mock(BulkItemResponse.class);
+ when(responseSuccess.getId()).thenReturn(recordId2);
+ return responseSuccess;
+ }
- @Test
- public void processSchemaMessagesTest() throws Exception {
- indexerService.processSchemaMessages(recordInfos);
+ private void indexSchemaServiceMock(String kind, Map<String, Object> schema) throws UnsupportedEncodingException, URISyntaxException {
+ IndexSchema indexSchema = schema == null
+ ? IndexSchema.builder().kind(kind).dataSchema(new HashMap<>()).build()
+ : IndexSchema.builder().kind(kind).dataSchema(schema).build();
+ when(schemaService.getIndexerInputSchema(kind, new ArrayList<>())).thenReturn(indexSchema);
+ }
- verify(elasticClientHandler, times(1)).createRestClient();
- verify(elasticIndexNameResolver, times(1)).getIndexNameFromKind(any());
- verify(indicesService, times(1)).isIndexExist(any(), any());
- }
+ private Map<String, Object> createSchema() {
+ Map<String, Object> schema = new HashMap<>();
+ schema.put("schema1", "keyword");
+ schema.put("schema2", "boolean");
+ schema.put("schema3", "date");
+ schema.put("schema6", "object");
+ return schema;
+ }
}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndicesServiceTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndicesServiceTest.java
index 0051012ef572f075e91df197310e2557786c69ce..38c7a4f682e39182beaac68bac21ebd8ee34d3d9 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndicesServiceTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndicesServiceTest.java
@@ -16,7 +16,6 @@ package org.opengroup.osdu.indexer.service;
import com.google.common.reflect.TypeToken;
import com.google.gson.Gson;
-
import org.apache.http.HttpEntity;
import org.apache.http.HttpStatus;
import org.apache.http.entity.ContentType;
@@ -31,11 +30,11 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
-import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.provider.interfaces.IIndexCache;
+import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.search.IndexInfo;
import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
+import org.opengroup.osdu.indexer.cache.PartitionSafeIndexCache;
import org.opengroup.osdu.indexer.util.ElasticClientHandler;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
@@ -59,7 +58,7 @@ public class IndicesServiceTest {
@Mock
private ElasticIndexNameResolver elasticIndexNameResolver;
@Mock
- private IIndexCache indicesExistCache;
+ private PartitionSafeIndexCache indicesExistCache;
@Mock
@Lazy
private JaxRsDpsLog log;
diff --git a/provider/indexer-azure/pom.xml b/provider/indexer-azure/pom.xml
index fa94c668f3d2f2ccb599785d8fbf347b540bb865..109f4eea0d2af31a8a849da4876421483fb9bfaa 100644
--- a/provider/indexer-azure/pom.xml
+++ b/provider/indexer-azure/pom.xml
@@ -41,7 +41,7 @@
<nimbus-jose-jwt.version>8.2</nimbus-jose-jwt.version>
<indexer-core.version>0.14.0-SNAPSHOT</indexer-core.version>
<spring-security-jwt.version>1.1.1.RELEASE</spring-security-jwt.version>
- <osdu.corelibazure.version>0.13.0-rc6</osdu.corelibazure.version>
+ <osdu.corelibazure.version>0.14.0-rc2</osdu.corelibazure.version>
<osdu.oscorecommon.version>0.13.0</osdu.oscorecommon.version>
<reactor-netty.version>0.9.12.RELEASE</reactor-netty.version>
<java-jwt.version>3.8.1</java-jwt.version>
diff --git a/provider/indexer-gcp/README.md b/provider/indexer-gcp/README.md
index c4ea917bc3ffe5d2887485c75dd0bd0b3542bed3..c7593de2753b57138443b03c4fd1042dc399e4cf 100644
--- a/provider/indexer-gcp/README.md
+++ b/provider/indexer-gcp/README.md
@@ -14,6 +14,14 @@ os-indexer-gcp is a [Spring Boot](https://spring.io/projects/spring-boot) servic
## Getting Started
These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system.
+# Configuration
+
+## Service Configuration
+### Anthos:
+[Anthos service configuration ](docs/anthos/README.md)
+### GCP:
+[Gcp service configuration ](docs/gcp/README.md)
+
## Mappers
This is a universal solution created using EPAM OQM mappers technology. It allows you to work with various
@@ -205,6 +213,8 @@ Storage service account should have entitlements groups listed below:
- service.storage.viewer
- service.schema-service.viewers
- data.default.viewers
+- service.storage.admin
+- service.search.admin
## Licence
Copyright © Google LLC
diff --git a/provider/indexer-gcp/docs/anthos/README.md b/provider/indexer-gcp/docs/anthos/README.md
index 90cb1a8b9ed4e39f7e9f80be0ad42524b77d7ace..880cce5011fff28c9737475189f3f96b8b6d17e3 100644
--- a/provider/indexer-gcp/docs/anthos/README.md
+++ b/provider/indexer-gcp/docs/anthos/README.md
@@ -12,6 +12,8 @@ Must have:
| `OPENID_PROVIDER_CLIENT_ID` | `*****` | Client id that represents this service and serves to request tokens, example `workload-identity-legal` |yes| - |
| `OPENID_PROVIDER_CLIENT_SECRET` | `*****` | This client secret that serves to request tokens| yes | - |
| `OPENID_PROVIDER_URL` | `https://keycloack.com/auth/realms/master` | URL of OpenID Connect provider, it will be used as `<OpenID URL> + /.well-known/openid-configuration` to auto configure endpoint for token request | no | - |
+| `<ELASTICSEARCH_USER_ENV_VARIABLE_NAME>` | ex `user` | Elasticsearch user, name of that variable not defined at the service level, the name will be received through partition service. Each tenant can have it's own ENV name value, and it must be present in ENV of Indexer service, see [Partition properties set](#Properties-set-in-Partition-service) | yes | - |
+| `<ELASTICSEARCH_PASSWORD_ENV_VARIABLE_NAME>` | ex `password` | Elasticsearch password, name of that variable not defined at the service level, the name will be received through partition service. Each tenant can have it's own ENV name value, and it must be present in ENV of Indexer service, see [Partition properties set](#Properties-set-in-Partition-service) | false | - |
Defined in default application property file but possible to override:
@@ -27,7 +29,7 @@ Defined in default application property file but possible to override:
| `PARTITION_HOST` | ex `https://partition.com` | Partition host | no | output of infrastructure deployment |
| `ENTITLEMENTS_HOST` | ex `https://entitlements.com` | Entitlements host | no | output of infrastructure deployment |
| `STORAGE_HOST` | ex `https://storage.com` | Storage host | no | output of infrastructure deployment |
-| `INDEXER_QUEUE_HOST` | ex `http://indexer-queue` | Indexer-Queue host | no | output of infrastructure deployment |
+| `INDEXER_QUEUE_HOST` | ex `http://indexer-queue/api/indexer-queue/v1/_dps/task-handlers/enqueue` | Indexer-Queue host endpoint used for reprocessing tasks| no | output of infrastructure deployment |
| `SCHEMA_BASE_HOST` | ex `https://schema.com` | Schema service host | no | output of infrastructure deployment |
These variables define service behavior, and are used to switch between `anthos` or `gcp` environments, their overriding and usage in mixed mode was not tested.
@@ -39,10 +41,26 @@ Usage of spring profiles is preferred.
| `OQMDRIVER` | `rabbitmq` or `pubsub` | Oqm driver mode that defines which message broker will be used | no | - |
| `SERVICE_TOKEN_PROVIDER` | `GCP` or `OPENID` |Service account token provider, `GCP` means use Google service account `OPEIND` means use OpenId provider like `Keycloak` | no | - |
-## Elasticsearch configuration
-
### Properties set in Partition service:
+Note that properties can be set in Partition as `sensitive` in that case in property `value` should be present not value itself, but ENV variable name.
+This variable should be present in environment of service that need that variable.
+
+Example:
+```
+ "elasticsearch.port": {
+ "sensitive": false, <- value not sensitive
+ "value": "9243" <- will be used as is.
+ },
+ "elasticsearch.password": {
+ "sensitive": true, <- value is sensitive
+ "value": "ELASTIC_SEARCH_PASSWORD_OSDU" <- service consumer should have env variable ELASTIC_SEARCH_PASSWORD_OSDU with elastic search password
+ }
+```
+
+
+## Elasticsearch configuration
+
**prefix:** `elasticsearch`
It can be overridden by:
@@ -56,7 +74,8 @@ It can be overridden by:
| --- | --- |
| elasticsearch.host | server URL |
| elasticsearch.port | server port |
-| elasticsearch.configuration | username and password |
+| elasticsearch.user | username |
+| elasticsearch.password | password |
<details><summary>Example of a definition for a single tenant</summary></details>
@@ -72,9 +91,13 @@ curl -L -X PATCH 'http://partition.com/api/partition/v1/partitions/opendes' -H '
"sensitive": false,
"value": "9243"
},
- "elasticsearch.configuration": {
+ "elasticsearch.user": {
+ "sensitive": true,
+ "value": "<ELASTICSEARCH_USER_ENV_VARIABLE_NAME>" <- (Not actual value, just name of env variable)
+ },
+ "elasticsearch.password": {
"sensitive": true,
- "value": "elasticuser:elasticpassword"
+ "value": "<ELASTICSEARCH_PASSWORD_ENV_VARIABLE_NAME>" <- (Not actual value, just name of env variable)
}
}
}'
diff --git a/provider/indexer-gcp/docs/gcp/README.md b/provider/indexer-gcp/docs/gcp/README.md
index 494169b1634af2e093c39038688987e78de6d726..d583f831a53e5738cdf913fb0b0669974b8f26a8 100644
--- a/provider/indexer-gcp/docs/gcp/README.md
+++ b/provider/indexer-gcp/docs/gcp/README.md
@@ -10,6 +10,8 @@ Must have:
| --- | --- | --- | --- | --- |
| `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials |
| `SPRING_PROFILES_ACTIVE` | ex `gcp` | Spring profile that activate default configuration for GCP environment | false | - |
+| `<ELASTICSEARCH_USER_ENV_VARIABLE_NAME>` | ex `user` | Elasticsearch user, name of that variable not defined at the service level, the name will be received through partition service. Each tenant can have it's own ENV name value, and it must be present in ENV of Indexer service, see [Partition properties set](#Properties-set-in-Partition-service) | yes | - |
+| `<ELASTICSEARCH_PASSWORD_ENV_VARIABLE_NAME>` | ex `password` | Elasticsearch password, name of that variable not defined at the service level, the name will be received through partition service. Each tenant can have it's own ENV name value, and it must be present in ENV of Indexer service, see [Partition properties set](#Properties-set-in-Partition-service) | false | - |
Defined in default application property file but possible to override:
@@ -25,7 +27,7 @@ Defined in default application property file but possible to override:
| `PARTITION_HOST` | ex `https://partition.com` | Partition host | no | output of infrastructure deployment |
| `ENTITLEMENTS_HOST` | ex `https://entitlements.com` | Entitlements host | no | output of infrastructure deployment |
| `STORAGE_HOST` | ex `https://storage.com` | Storage host | no | output of infrastructure deployment |
-| `INDEXER_QUEUE_HOST` | ex `http://indexer-queue` | Indexer-Queue host | no | output of infrastructure deployment |
+| `INDEXER_QUEUE_HOST` | ex `http://indexer-queue/api/indexer-queue/v1/_dps/task-handlers/enqueue` | Indexer-Queue host endpoint used for reprocessing tasks | no | output of infrastructure deployment |
| `SCHEMA_BASE_HOST` | ex `https://schema.com` | Schema service host | no | output of infrastructure deployment |
| `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
@@ -44,10 +46,27 @@ At Pubsub should be created topic with name:
**name:** `indexing-progress`
-## Elasticsearch configuration
-
### Properties set in Partition service:
+Note that properties can be set in Partition as `sensitive` in that case in property `value` should be present not value itself, but ENV variable name.
+This variable should be present in environment of service that need that variable.
+
+Example:
+```
+ "elasticsearch.port": {
+ "sensitive": false, <- value not sensitive
+ "value": "9243" <- will be used as is.
+ },
+ "elasticsearch.password": {
+ "sensitive": true, <- value is sensitive
+ "value": "ELASTIC_SEARCH_PASSWORD_OSDU" <- service consumer should have env variable ELASTIC_SEARCH_PASSWORD_OSDU with elastic search password
+ }
+```
+
+There is no hardcode in services, all behaviour defined by sensitivity of property.
+
+## Elasticsearch configuration
+
**prefix:** `elasticsearch`
It can be overridden by:
@@ -61,7 +80,8 @@ It can be overridden by:
| --- | --- |
| elasticsearch.host | server URL |
| elasticsearch.port | server port |
-| elasticsearch.configuration | username and password |
+| elasticsearch.user | username |
+| elasticsearch.password | password |
<details><summary>Example of a definition for a single tenant</summary></details>
@@ -77,9 +97,13 @@ curl -L -X PATCH 'http://partition.com/api/partition/v1/partitions/opendes' -H '
"sensitive": false,
"value": "9243"
},
- "elasticsearch.configuration": {
+ "elasticsearch.user": {
+ "sensitive": true,
+ "value": "<USER_ENV_VARIABLE_NAME>" <- (Not actual value, just name of env variable)
+ },
+ "elasticsearch.password": {
"sensitive": true,
- "value": "elasticuser:elasticpassword"
+ "value": "<PASSWORD_ENV_VARIABLE_NAME>" <- (Not actual value, just name of env variable)
}
}
}'
diff --git a/provider/indexer-gcp/pom.xml b/provider/indexer-gcp/pom.xml
index 3f7bb4987478522f50a8d113ae6094c2997c4583..311aa52ab24e9f07619c5db489bf3d1d2a78785f 100644
--- a/provider/indexer-gcp/pom.xml
+++ b/provider/indexer-gcp/pom.xml
@@ -19,7 +19,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-gcp</artifactId>
- <version>0.14.0-rc1</version>
+ <version>0.14.0-rc2</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/ElasticSearchConfig.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/ElasticSearchConfig.java
index 72cb1f6a3a1cedfc3f408853d99d10a9c80cca74..0f9cc1f4d3ea4319c729af434fd5710203b30141 100644
--- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/ElasticSearchConfig.java
+++ b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/di/ElasticSearchConfig.java
@@ -20,6 +20,7 @@ package org.opengroup.osdu.indexer.di;
import org.opengroup.osdu.core.common.partition.IPartitionProvider;
import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository;
import org.opengroup.osdu.core.destination.elastic.ElasticSearchDestinationResolver;
+import org.opengroup.osdu.core.destination.util.IPartitionPropertyResolver;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@@ -27,7 +28,8 @@ import org.springframework.context.annotation.Configuration;
public class ElasticSearchConfig {
@Bean
- public IElasticRepository elasticRepository(ElasticSearchConfigurationProperties properties, IPartitionProvider partitionProvider) {
- return new ElasticSearchDestinationResolver(properties.getElasticsearchPropertiesPrefix(), partitionProvider);
+ public IElasticRepository elasticRepository(ElasticSearchConfigurationProperties properties,
+ IPartitionProvider partitionProvider, IPartitionPropertyResolver propertyResolver) {
+ return new ElasticSearchDestinationResolver(properties.getElasticsearchPropertiesPrefix(), partitionProvider, propertyResolver);
}
}