From 6bfba9814bc98b93e7c870b792381c0de6bb1de9 Mon Sep 17 00:00:00 2001
From: Krisztian Molnar <krisztian_molnar@epam.com>
Date: Mon, 27 Jun 2022 13:39:01 +0200
Subject: [PATCH] changed all java files to LF (approved by dev team)
---
.gitattributes | 2 +-
.../osdu/indexer/api/RecordIndexerApi.java | 272 ++--
.../osdu/indexer/api/ReindexApi.java | 128 +-
.../service/AttributeParsingServiceImpl.java | 466 +++----
.../indexer/service/IndexSchemaService.java | 86 +-
.../service/IndexerMappingServiceImpl.java | 802 ++++++------
.../indexer/service/IndexerServiceImpl.java | 1156 ++++++++---------
.../service/StorageIndexerPayloadMapper.java | 378 +++---
.../indexer/service/StorageServiceImpl.java | 466 +++----
.../indexer/util/IndexerQueueTaskBuilder.java | 170 +--
.../indexer/util/parser/BooleanParser.java | 30 +-
.../osdu/indexer/api/ReindexApiTest.java | 154 +--
.../util/parser/BooleanParserTest.java | 52 +-
.../indexer/azure/service/RetryPolicy.java | 228 ++--
.../service/UrlFetchServiceAzureImpl.java | 196 +--
.../azure/service/RetryPolicyTest.java | 326 ++---
.../service/UrlFetchServiceAzureImplTest.java | 262 ++--
.../indexer/middleware/IndexFilterTest.java | 106 +-
.../service/IndexerMappingServiceTest.java | 606 ++++-----
.../indexer/service/ReindexServiceTest.java | 300 ++---
.../indexer/service/StorageServiceTest.java | 508 ++++----
.../ibm/util/GlobalExceptionMapper.java | 142 +-
.../ibm/util/IndexerQueueTaskBuilderIbm.java | 316 ++---
.../ibm/service/ReindexServiceTest.java | 288 ++--
.../ibm/service/StorageServiceTest.java | 438 +++----
25 files changed, 3939 insertions(+), 3939 deletions(-)
diff --git a/.gitattributes b/.gitattributes
index 8e66562ab..ce3bfb293 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,2 +1,2 @@
* text=auto eol=lf
-*.java binary
+*.sh eol=lf
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/RecordIndexerApi.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/RecordIndexerApi.java
index 1f8df5cf5..4cb1ef238 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/RecordIndexerApi.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/RecordIndexerApi.java
@@ -1,136 +1,136 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.api;
-
-import com.google.common.reflect.TypeToken;
-import com.google.gson.Gson;
-
-import com.google.gson.JsonParseException;
-import io.swagger.annotations.ApiOperation;
-import lombok.extern.java.Log;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.indexer.JobStatus;
-import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
-import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
-import org.opengroup.osdu.core.common.model.indexer.SchemaChangedMessages;
-import org.opengroup.osdu.core.common.model.indexer.SchemaInfo;
-import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
-import org.opengroup.osdu.indexer.SwaggerDoc;
-import org.opengroup.osdu.indexer.service.IndexerService;
-import org.opengroup.osdu.indexer.service.ReindexService;
-import org.opengroup.osdu.indexer.service.SchemaService;
-import org.springframework.http.HttpStatus;
-import org.springframework.http.ResponseEntity;
-import org.springframework.web.bind.annotation.*;
-import org.springframework.web.context.annotation.RequestScope;
-
-import javax.inject.Inject;
-import javax.validation.Valid;
-import javax.validation.constraints.NotNull;
-import java.io.IOException;
-import java.lang.reflect.Type;
-import java.util.List;
-
-@Log
-@RestController
-@RequestMapping("/_dps/task-handlers")
-@RequestScope
-public class RecordIndexerApi {
-
- @Inject
- private IndexerService indexerService;
- @Inject
- private ReindexService reIndexService;
- @Inject
- private SchemaService schemaService;
-
- // THIS IS AN INTERNAL USE API ONLY
- // THAT MEANS WE DON'T DOCUMENT IT IN SWAGGER, ACCESS IS LIMITED TO CLOUD TASK QUEUE CALLS ONLY
- @PostMapping(path = "/index-worker", consumes = "application/json")
- @ApiOperation(hidden = true, value = "", notes = "")
- public ResponseEntity<JobStatus> indexWorker (
- @NotNull(message = SwaggerDoc.REQUEST_VALIDATION_NOT_NULL_BODY)
- @Valid @RequestBody RecordChangedMessages recordChangedMessages) throws Exception {
-
- if (recordChangedMessages.missingAccountId()) {
- throw new AppException(org.apache.http.HttpStatus.SC_BAD_REQUEST, "Invalid tenant",
- String.format("Required header: '%s' not found", DpsHeaders.DATA_PARTITION_ID));
- }
- try {
- if (recordChangedMessages == null) {
- log.info("record change messages is null");
- }
-
- Type listType = new TypeToken<List<RecordInfo>>() {
- }.getType();
- List<RecordInfo> recordInfos = new Gson().fromJson(recordChangedMessages.getData(), listType);
-
- if (recordInfos.size() == 0) {
- log.info("none of record-change message can be deserialized");
- return new ResponseEntity(HttpStatus.OK);
- }
- this.indexerService.processRecordChangedMessages(recordChangedMessages, recordInfos);
- return new ResponseEntity(HttpStatus.OK);
- } catch (AppException e) {
- throw e;
- } catch (JsonParseException e) {
- throw new AppException(org.apache.http.HttpStatus.SC_BAD_REQUEST, "Request payload parsing error", "Unable to parse request payload.", e);
- } catch (Exception e) {
- throw new AppException(org.apache.http.HttpStatus.SC_BAD_REQUEST, "Unknown error", "An unknown error has occurred.", e);
- }
- }
-
- // THIS IS AN INTERNAL USE API ONLY
- // THAT MEANS WE DON'T DOCUMENT IT IN SWAGGER, ACCESS IS LIMITED TO CLOUD TASK QUEUE CALLS ONLY
- @PostMapping("/reindex-worker")
- @ApiOperation(hidden = true, value = "", notes = "")
- public ResponseEntity<?> reindex(
- @RequestBody @NotNull(message = SwaggerDoc.REQUEST_VALIDATION_NOT_NULL_BODY)
- @Valid RecordReindexRequest recordReindexRequest) {
- return new ResponseEntity<>(reIndexService.reindexRecords(recordReindexRequest, false), HttpStatus.OK);
- }
-
- // THIS IS AN INTERNAL USE API ONLY
- // THAT MEANS WE DON'T DOCUMENT IT IN SWAGGER, ACCESS IS LIMITED TO CLOUD TASK QUEUE CALLS ONLY
- @PostMapping("/schema-worker")
- @ApiOperation(hidden = true, value = "", notes = "")
- public ResponseEntity<?> schemaWorker(
- @NotNull(message = SwaggerDoc.REQUEST_VALIDATION_NOT_NULL_BODY)
- @Valid @RequestBody SchemaChangedMessages schemaChangedMessage) throws IOException {
- if (schemaChangedMessage == null) {
- log.warning("schema change messages is null");
- }
-
- if (schemaChangedMessage.missingAccountId()) {
- throw new AppException(org.apache.http.HttpStatus.SC_BAD_REQUEST, "Invalid tenant",
- String.format("Required header: '%s' not found", DpsHeaders.DATA_PARTITION_ID));
- }
-
- try {
- Type listType = new TypeToken<List<SchemaInfo>>() {}.getType();
- List<SchemaInfo> schemaInfos = new Gson().fromJson(schemaChangedMessage.getData(), listType);
-
- if (schemaInfos.size() == 0) {
- log.warning("none of schema-change message can be deserialized");
- return new ResponseEntity(org.springframework.http.HttpStatus.OK);
- }
- this.schemaService.processSchemaMessages(schemaInfos);
- return new ResponseEntity(HttpStatus.OK);
- } catch (JsonParseException e) {
- throw new AppException(org.apache.http.HttpStatus.SC_BAD_REQUEST, "Request payload parsing error", "Unable to parse request payload.", e);
- }
- }
-}
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.api;
+
+import com.google.common.reflect.TypeToken;
+import com.google.gson.Gson;
+
+import com.google.gson.JsonParseException;
+import io.swagger.annotations.ApiOperation;
+import lombok.extern.java.Log;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.indexer.JobStatus;
+import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
+import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
+import org.opengroup.osdu.core.common.model.indexer.SchemaChangedMessages;
+import org.opengroup.osdu.core.common.model.indexer.SchemaInfo;
+import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
+import org.opengroup.osdu.indexer.SwaggerDoc;
+import org.opengroup.osdu.indexer.service.IndexerService;
+import org.opengroup.osdu.indexer.service.ReindexService;
+import org.opengroup.osdu.indexer.service.SchemaService;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.*;
+import org.springframework.web.context.annotation.RequestScope;
+
+import javax.inject.Inject;
+import javax.validation.Valid;
+import javax.validation.constraints.NotNull;
+import java.io.IOException;
+import java.lang.reflect.Type;
+import java.util.List;
+
+@Log
+@RestController
+@RequestMapping("/_dps/task-handlers")
+@RequestScope
+public class RecordIndexerApi {
+
+ @Inject
+ private IndexerService indexerService;
+ @Inject
+ private ReindexService reIndexService;
+ @Inject
+ private SchemaService schemaService;
+
+ // THIS IS AN INTERNAL USE API ONLY
+ // THAT MEANS WE DON'T DOCUMENT IT IN SWAGGER, ACCESS IS LIMITED TO CLOUD TASK QUEUE CALLS ONLY
+ @PostMapping(path = "/index-worker", consumes = "application/json")
+ @ApiOperation(hidden = true, value = "", notes = "")
+ public ResponseEntity<JobStatus> indexWorker (
+ @NotNull(message = SwaggerDoc.REQUEST_VALIDATION_NOT_NULL_BODY)
+ @Valid @RequestBody RecordChangedMessages recordChangedMessages) throws Exception {
+
+ if (recordChangedMessages.missingAccountId()) {
+ throw new AppException(org.apache.http.HttpStatus.SC_BAD_REQUEST, "Invalid tenant",
+ String.format("Required header: '%s' not found", DpsHeaders.DATA_PARTITION_ID));
+ }
+ try {
+ if (recordChangedMessages == null) {
+ log.info("record change messages is null");
+ }
+
+ Type listType = new TypeToken<List<RecordInfo>>() {
+ }.getType();
+ List<RecordInfo> recordInfos = new Gson().fromJson(recordChangedMessages.getData(), listType);
+
+ if (recordInfos.size() == 0) {
+ log.info("none of record-change message can be deserialized");
+ return new ResponseEntity(HttpStatus.OK);
+ }
+ this.indexerService.processRecordChangedMessages(recordChangedMessages, recordInfos);
+ return new ResponseEntity(HttpStatus.OK);
+ } catch (AppException e) {
+ throw e;
+ } catch (JsonParseException e) {
+ throw new AppException(org.apache.http.HttpStatus.SC_BAD_REQUEST, "Request payload parsing error", "Unable to parse request payload.", e);
+ } catch (Exception e) {
+ throw new AppException(org.apache.http.HttpStatus.SC_BAD_REQUEST, "Unknown error", "An unknown error has occurred.", e);
+ }
+ }
+
+ // THIS IS AN INTERNAL USE API ONLY
+ // THAT MEANS WE DON'T DOCUMENT IT IN SWAGGER, ACCESS IS LIMITED TO CLOUD TASK QUEUE CALLS ONLY
+ @PostMapping("/reindex-worker")
+ @ApiOperation(hidden = true, value = "", notes = "")
+ public ResponseEntity<?> reindex(
+ @RequestBody @NotNull(message = SwaggerDoc.REQUEST_VALIDATION_NOT_NULL_BODY)
+ @Valid RecordReindexRequest recordReindexRequest) {
+ return new ResponseEntity<>(reIndexService.reindexRecords(recordReindexRequest, false), HttpStatus.OK);
+ }
+
+ // THIS IS AN INTERNAL USE API ONLY
+ // THAT MEANS WE DON'T DOCUMENT IT IN SWAGGER, ACCESS IS LIMITED TO CLOUD TASK QUEUE CALLS ONLY
+ @PostMapping("/schema-worker")
+ @ApiOperation(hidden = true, value = "", notes = "")
+ public ResponseEntity<?> schemaWorker(
+ @NotNull(message = SwaggerDoc.REQUEST_VALIDATION_NOT_NULL_BODY)
+ @Valid @RequestBody SchemaChangedMessages schemaChangedMessage) throws IOException {
+ if (schemaChangedMessage == null) {
+ log.warning("schema change messages is null");
+ }
+
+ if (schemaChangedMessage.missingAccountId()) {
+ throw new AppException(org.apache.http.HttpStatus.SC_BAD_REQUEST, "Invalid tenant",
+ String.format("Required header: '%s' not found", DpsHeaders.DATA_PARTITION_ID));
+ }
+
+ try {
+ Type listType = new TypeToken<List<SchemaInfo>>() {}.getType();
+ List<SchemaInfo> schemaInfos = new Gson().fromJson(schemaChangedMessage.getData(), listType);
+
+ if (schemaInfos.size() == 0) {
+ log.warning("none of schema-change message can be deserialized");
+ return new ResponseEntity(org.springframework.http.HttpStatus.OK);
+ }
+ this.schemaService.processSchemaMessages(schemaInfos);
+ return new ResponseEntity(HttpStatus.OK);
+ } catch (JsonParseException e) {
+ throw new AppException(org.apache.http.HttpStatus.SC_BAD_REQUEST, "Request payload parsing error", "Unable to parse request payload.", e);
+ }
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java
index 4f6a1f666..a4f38e31d 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java
@@ -1,64 +1,64 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.api;
-
-import org.opengroup.osdu.core.common.model.search.SearchServiceRole;
-import org.opengroup.osdu.indexer.logging.AuditLogger;
-import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
-import org.opengroup.osdu.indexer.service.IndexSchemaService;
-import org.opengroup.osdu.indexer.service.ReindexService;
-import org.springframework.http.ResponseEntity;
-import org.springframework.security.access.prepost.PreAuthorize;
-
-import org.springframework.web.bind.annotation.*;
-import org.springframework.web.context.annotation.RequestScope;
-
-import javax.inject.Inject;
-import javax.validation.Valid;
-import javax.validation.constraints.NotNull;
-
-import java.io.IOException;
-
-import static java.util.Collections.singletonList;
-
-@RestController
-@RequestMapping("/reindex")
-@RequestScope
-public class ReindexApi {
-
- @Inject
- private ReindexService reIndexService;
- @Inject
- private IndexSchemaService indexSchemaService;
- @Inject
- private AuditLogger auditLogger;
-
- @PreAuthorize("@authorizationFilter.hasPermission('" + SearchServiceRole.ADMIN + "')")
- @PostMapping
- public ResponseEntity<?> reindex(
- @NotNull @Valid @RequestBody RecordReindexRequest recordReindexRequest,
- @RequestParam(value = "force_clean", defaultValue = "false") boolean forceClean) throws IOException {
- this.reIndexService.reindexRecords(recordReindexRequest, this.indexSchemaService.isStorageSchemaSyncRequired(recordReindexRequest.getKind(), forceClean));
- this.auditLogger.getReindex(singletonList(recordReindexRequest.getKind()));
- return new ResponseEntity<>(org.springframework.http.HttpStatus.OK);
- }
-
- @PreAuthorize("@authorizationFilter.hasPermission('" + SearchServiceRole.ADMIN + "')")
- @PatchMapping
- public ResponseEntity<String> fullReindex(@RequestParam(value = "force_clean", defaultValue = "false") boolean forceClean) throws IOException {
- this.reIndexService.fullReindex(forceClean);
- return new ResponseEntity<>(org.springframework.http.HttpStatus.OK);
- }
-}
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.api;
+
+import org.opengroup.osdu.core.common.model.search.SearchServiceRole;
+import org.opengroup.osdu.indexer.logging.AuditLogger;
+import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
+import org.opengroup.osdu.indexer.service.IndexSchemaService;
+import org.opengroup.osdu.indexer.service.ReindexService;
+import org.springframework.http.ResponseEntity;
+import org.springframework.security.access.prepost.PreAuthorize;
+
+import org.springframework.web.bind.annotation.*;
+import org.springframework.web.context.annotation.RequestScope;
+
+import javax.inject.Inject;
+import javax.validation.Valid;
+import javax.validation.constraints.NotNull;
+
+import java.io.IOException;
+
+import static java.util.Collections.singletonList;
+
+@RestController
+@RequestMapping("/reindex")
+@RequestScope
+public class ReindexApi {
+
+ @Inject
+ private ReindexService reIndexService;
+ @Inject
+ private IndexSchemaService indexSchemaService;
+ @Inject
+ private AuditLogger auditLogger;
+
+ @PreAuthorize("@authorizationFilter.hasPermission('" + SearchServiceRole.ADMIN + "')")
+ @PostMapping
+ public ResponseEntity<?> reindex(
+ @NotNull @Valid @RequestBody RecordReindexRequest recordReindexRequest,
+ @RequestParam(value = "force_clean", defaultValue = "false") boolean forceClean) throws IOException {
+ this.reIndexService.reindexRecords(recordReindexRequest, this.indexSchemaService.isStorageSchemaSyncRequired(recordReindexRequest.getKind(), forceClean));
+ this.auditLogger.getReindex(singletonList(recordReindexRequest.getKind()));
+ return new ResponseEntity<>(org.springframework.http.HttpStatus.OK);
+ }
+
+ @PreAuthorize("@authorizationFilter.hasPermission('" + SearchServiceRole.ADMIN + "')")
+ @PatchMapping
+ public ResponseEntity<String> fullReindex(@RequestParam(value = "force_clean", defaultValue = "false") boolean forceClean) throws IOException {
+ this.reIndexService.fullReindex(forceClean);
+ return new ResponseEntity<>(org.springframework.http.HttpStatus.OK);
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java
index b91282586..017a12608 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java
@@ -1,233 +1,233 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.service;
-
-import com.google.common.base.Strings;
-import com.google.gson.Gson;
-import com.google.gson.JsonSyntaxException;
-import com.google.gson.reflect.TypeToken;
-import java.lang.reflect.Array;
-import java.lang.reflect.Type;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.function.BiFunction;
-import javax.inject.Inject;
-import org.apache.http.HttpStatus;
-import org.opengroup.osdu.core.common.model.indexer.ElasticType;
-import org.opengroup.osdu.core.common.model.indexer.IndexingStatus;
-import org.opengroup.osdu.core.common.model.indexer.JobStatus;
-import org.opengroup.osdu.indexer.util.parser.BooleanParser;
-import org.opengroup.osdu.indexer.util.parser.DateTimeParser;
-import org.opengroup.osdu.indexer.util.parser.GeoShapeParser;
-import org.opengroup.osdu.indexer.util.parser.NumberParser;
-import org.springframework.stereotype.Service;
-import org.springframework.web.context.annotation.RequestScope;
-
-@Service
-@RequestScope
-public class AttributeParsingServiceImpl implements IAttributeParsingService {
-
- @Inject
- private NumberParser numberParser;
- @Inject
- private BooleanParser booleanParser;
- @Inject
- private DateTimeParser dateTimeParser;
- @Inject
- private GeoShapeParser geoShapeParser;
- @Inject
- private GeometryConversionService geometryConversionService;
- @Inject
- private JobStatus jobStatus;
-
- @Override
- public void tryParseValueArray(Class<?> attributeClass, String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
- BiFunction<String, Object, ?> parser;
- ElasticType elasticType = ElasticType.forValue(attributeClass.getSimpleName());
- switch (elasticType) {
- case DOUBLE:
- parser = this.numberParser::parseDouble;
- break;
- case FLOAT:
- parser = this.numberParser::parseFloat;
- break;
- case INTEGER:
- parser = this.numberParser::parseInteger;
- break;
- case LONG:
- parser = this.numberParser::parseLong;
- break;
- case BOOLEAN:
- parser = this.booleanParser::parseBoolean;
- break;
- case DATE:
- parser = this.dateTimeParser::parseDate;
- // DateTime parser output is String
- attributeClass = String.class;
- break;
- default:
- throw new IllegalArgumentException("Invalid array attribute type");
- }
-
- try {
- List<String> parsedStringList = isArrayType(attributeVal);
- List out = new ArrayList<>();
- for (Object o : parsedStringList) {
- out.add(parser.apply(attributeName, o));
- }
- Object parsedAttribute = toTypeArray(attributeClass, out);
- dataMap.put(attributeName, parsedAttribute);
- } catch (IllegalArgumentException e) {
- this.jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, e.getMessage(), String.format("record-id: %s | %s", recordId, e.getMessage()));
- }
- }
-
- @Override
- public void tryParseInteger(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
- try {
- int parsedInteger = this.numberParser.parseInteger(attributeName, attributeVal);
- dataMap.put(attributeName, parsedInteger);
- } catch (IllegalArgumentException e) {
- jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, e.getMessage(), String.format("record-id: %s | %s", recordId, e.getMessage()));
- }
- }
-
- @Override
- public void tryParseLong(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
- try {
- long parsedLong = this.numberParser.parseLong(attributeName, attributeVal);
- dataMap.put(attributeName, parsedLong);
- } catch (IllegalArgumentException e) {
- jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, e.getMessage(), String.format("record-id: %s | %s", recordId, e.getMessage()));
- }
- }
-
- @Override
- public void tryParseFloat(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
- try {
- float parsedFloat = this.numberParser.parseFloat(attributeName, attributeVal);
- dataMap.put(attributeName, parsedFloat);
- } catch (IllegalArgumentException e) {
- jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, e.getMessage(), String.format("record-id: %s | %s", recordId, e.getMessage()));
- }
- }
-
- @Override
- public void tryParseDouble(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
- try {
- double parsedDouble = this.numberParser.parseDouble(attributeName, attributeVal);
- dataMap.put(attributeName, parsedDouble);
- } catch (IllegalArgumentException e) {
- jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, e.getMessage(), String.format("record-id: %s | %s", recordId, e.getMessage()));
- }
- }
-
- @Override
- public void tryParseBoolean(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
- boolean parsedBoolean = this.booleanParser.parseBoolean(attributeName, attributeVal);
- dataMap.put(attributeName, parsedBoolean);
- }
-
- @Override
- public void tryParseDate(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
- try {
- String parsedDate = this.dateTimeParser.parseDate(attributeName, attributeVal);
- if (parsedDate == null) return;
- dataMap.put(attributeName, parsedDate);
- } catch (IllegalArgumentException e) {
- jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, e.getMessage(), String.format("record-id: %s | %s", recordId, e.getMessage()));
- }
- }
-
- @Override
- public void tryParseGeopoint(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
-
- try {
- Type type = new TypeToken<Map<String, Double>>() {}.getType();
- Map<String, Double> positionMap = new Gson().fromJson(attributeVal.toString(), type);
-
- if (positionMap == null || positionMap.isEmpty()) return;
-
- Map<String, Double> position = this.geometryConversionService.tryGetGeopoint(positionMap);
-
- if (position == null || position.isEmpty()) return;
-
- dataMap.put(attributeName, position);
- } catch (JsonSyntaxException | IllegalArgumentException e) {
- String parsingError = String.format("geo-point parsing error: %s attribute: %s | value: %s", e.getMessage(), attributeName, attributeVal);
- jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, parsingError, String.format("record-id: %s | %s", recordId, parsingError));
- }
- }
-
- @Override
- public void tryParseGeojson(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
-
- try {
- Type type = new TypeToken<Map<String, Object>>() {}.getType();
- Map<String, Object> geoJsonMap = new Gson().fromJson(attributeVal.toString(), type);
-
- if (geoJsonMap == null || geoJsonMap.isEmpty()) return;
-
- Map<String, Object> parsedShape = this.geoShapeParser.parseGeoJson(geoJsonMap);
-
- dataMap.put(attributeName, parsedShape);
- } catch (JsonSyntaxException | IllegalArgumentException e) {
- String parsingError = String.format("geo-json shape parsing error: %s attribute: %s", e.getMessage(), attributeName);
- jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, parsingError, String.format("record-id: %s | %s", recordId, parsingError));
- }
- }
-
- @Override
- public void tryParseNested(String recordId, String name, Object value, Map<String, Object> dataMap) {
- dataMap.put(name,value);
- }
-
- @Override
- public void tryParseObject(String recordId, String name, Object value, Map<String, Object> dataMap) {
- dataMap.put(name,value);
- }
-
- @Override
- public void tryParseFlattened(String recordId, String name, Object value, Map<String, Object> dataMap) {
- dataMap.put(name,value);
- }
-
-
- private List<String> isArrayType(Object attributeVal) {
- try {
- String value = attributeVal == null ? null : String.valueOf(attributeVal);
- if (attributeVal == null || Strings.isNullOrEmpty(value)) {
- return Collections.EMPTY_LIST;
- }
-
- Gson converter = new Gson();
- Type type = new TypeToken<List<String>>() {}.getType();
- return converter.fromJson(value, type);
- } catch (JsonSyntaxException e) {
- throw new IllegalArgumentException("array parsing error, not a valid array");
- }
- }
-
- private <N> N toTypeArray(Class<N> fieldClass, List<?> list) {
- Object array = Array.newInstance(fieldClass, list.size());
- for (int i = 0; i < list.size(); i++) {
- Array.set(array, i, list.get(i));
- }
- return (N) array;
- }
-}
-
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.service;
+
+import com.google.common.base.Strings;
+import com.google.gson.Gson;
+import com.google.gson.JsonSyntaxException;
+import com.google.gson.reflect.TypeToken;
+import java.lang.reflect.Array;
+import java.lang.reflect.Type;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.function.BiFunction;
+import javax.inject.Inject;
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.core.common.model.indexer.ElasticType;
+import org.opengroup.osdu.core.common.model.indexer.IndexingStatus;
+import org.opengroup.osdu.core.common.model.indexer.JobStatus;
+import org.opengroup.osdu.indexer.util.parser.BooleanParser;
+import org.opengroup.osdu.indexer.util.parser.DateTimeParser;
+import org.opengroup.osdu.indexer.util.parser.GeoShapeParser;
+import org.opengroup.osdu.indexer.util.parser.NumberParser;
+import org.springframework.stereotype.Service;
+import org.springframework.web.context.annotation.RequestScope;
+
+@Service
+@RequestScope
+public class AttributeParsingServiceImpl implements IAttributeParsingService {
+
+ @Inject
+ private NumberParser numberParser;
+ @Inject
+ private BooleanParser booleanParser;
+ @Inject
+ private DateTimeParser dateTimeParser;
+ @Inject
+ private GeoShapeParser geoShapeParser;
+ @Inject
+ private GeometryConversionService geometryConversionService;
+ @Inject
+ private JobStatus jobStatus;
+
+ @Override
+ public void tryParseValueArray(Class<?> attributeClass, String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
+ BiFunction<String, Object, ?> parser;
+ ElasticType elasticType = ElasticType.forValue(attributeClass.getSimpleName());
+ switch (elasticType) {
+ case DOUBLE:
+ parser = this.numberParser::parseDouble;
+ break;
+ case FLOAT:
+ parser = this.numberParser::parseFloat;
+ break;
+ case INTEGER:
+ parser = this.numberParser::parseInteger;
+ break;
+ case LONG:
+ parser = this.numberParser::parseLong;
+ break;
+ case BOOLEAN:
+ parser = this.booleanParser::parseBoolean;
+ break;
+ case DATE:
+ parser = this.dateTimeParser::parseDate;
+ // DateTime parser output is String
+ attributeClass = String.class;
+ break;
+ default:
+ throw new IllegalArgumentException("Invalid array attribute type");
+ }
+
+ try {
+ List<String> parsedStringList = isArrayType(attributeVal);
+ List out = new ArrayList<>();
+ for (Object o : parsedStringList) {
+ out.add(parser.apply(attributeName, o));
+ }
+ Object parsedAttribute = toTypeArray(attributeClass, out);
+ dataMap.put(attributeName, parsedAttribute);
+ } catch (IllegalArgumentException e) {
+ this.jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, e.getMessage(), String.format("record-id: %s | %s", recordId, e.getMessage()));
+ }
+ }
+
+ @Override
+ public void tryParseInteger(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
+ try {
+ int parsedInteger = this.numberParser.parseInteger(attributeName, attributeVal);
+ dataMap.put(attributeName, parsedInteger);
+ } catch (IllegalArgumentException e) {
+ jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, e.getMessage(), String.format("record-id: %s | %s", recordId, e.getMessage()));
+ }
+ }
+
+ @Override
+ public void tryParseLong(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
+ try {
+ long parsedLong = this.numberParser.parseLong(attributeName, attributeVal);
+ dataMap.put(attributeName, parsedLong);
+ } catch (IllegalArgumentException e) {
+ jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, e.getMessage(), String.format("record-id: %s | %s", recordId, e.getMessage()));
+ }
+ }
+
+ @Override
+ public void tryParseFloat(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
+ try {
+ float parsedFloat = this.numberParser.parseFloat(attributeName, attributeVal);
+ dataMap.put(attributeName, parsedFloat);
+ } catch (IllegalArgumentException e) {
+ jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, e.getMessage(), String.format("record-id: %s | %s", recordId, e.getMessage()));
+ }
+ }
+
+ @Override
+ public void tryParseDouble(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
+ try {
+ double parsedDouble = this.numberParser.parseDouble(attributeName, attributeVal);
+ dataMap.put(attributeName, parsedDouble);
+ } catch (IllegalArgumentException e) {
+ jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, e.getMessage(), String.format("record-id: %s | %s", recordId, e.getMessage()));
+ }
+ }
+
+ @Override
+ public void tryParseBoolean(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
+ boolean parsedBoolean = this.booleanParser.parseBoolean(attributeName, attributeVal);
+ dataMap.put(attributeName, parsedBoolean);
+ }
+
+ @Override
+ public void tryParseDate(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
+ try {
+ String parsedDate = this.dateTimeParser.parseDate(attributeName, attributeVal);
+ if (parsedDate == null) return;
+ dataMap.put(attributeName, parsedDate);
+ } catch (IllegalArgumentException e) {
+ jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, e.getMessage(), String.format("record-id: %s | %s", recordId, e.getMessage()));
+ }
+ }
+
+ @Override
+ public void tryParseGeopoint(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
+
+ try {
+ Type type = new TypeToken<Map<String, Double>>() {}.getType();
+ Map<String, Double> positionMap = new Gson().fromJson(attributeVal.toString(), type);
+
+ if (positionMap == null || positionMap.isEmpty()) return;
+
+ Map<String, Double> position = this.geometryConversionService.tryGetGeopoint(positionMap);
+
+ if (position == null || position.isEmpty()) return;
+
+ dataMap.put(attributeName, position);
+ } catch (JsonSyntaxException | IllegalArgumentException e) {
+ String parsingError = String.format("geo-point parsing error: %s attribute: %s | value: %s", e.getMessage(), attributeName, attributeVal);
+ jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, parsingError, String.format("record-id: %s | %s", recordId, parsingError));
+ }
+ }
+
+ @Override
+ public void tryParseGeojson(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
+
+ try {
+ Type type = new TypeToken<Map<String, Object>>() {}.getType();
+ Map<String, Object> geoJsonMap = new Gson().fromJson(attributeVal.toString(), type);
+
+ if (geoJsonMap == null || geoJsonMap.isEmpty()) return;
+
+ Map<String, Object> parsedShape = this.geoShapeParser.parseGeoJson(geoJsonMap);
+
+ dataMap.put(attributeName, parsedShape);
+ } catch (JsonSyntaxException | IllegalArgumentException e) {
+ String parsingError = String.format("geo-json shape parsing error: %s attribute: %s", e.getMessage(), attributeName);
+ jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, parsingError, String.format("record-id: %s | %s", recordId, parsingError));
+ }
+ }
+
+ @Override
+ public void tryParseNested(String recordId, String name, Object value, Map<String, Object> dataMap) {
+ dataMap.put(name,value);
+ }
+
+ @Override
+ public void tryParseObject(String recordId, String name, Object value, Map<String, Object> dataMap) {
+ dataMap.put(name,value);
+ }
+
+ @Override
+ public void tryParseFlattened(String recordId, String name, Object value, Map<String, Object> dataMap) {
+ dataMap.put(name,value);
+ }
+
+
+ private List<String> isArrayType(Object attributeVal) {
+ try {
+ String value = attributeVal == null ? null : String.valueOf(attributeVal);
+ if (attributeVal == null || Strings.isNullOrEmpty(value)) {
+ return Collections.EMPTY_LIST;
+ }
+
+ Gson converter = new Gson();
+ Type type = new TypeToken<List<String>>() {}.getType();
+ return converter.fromJson(value, type);
+ } catch (JsonSyntaxException e) {
+ throw new IllegalArgumentException("array parsing error, not a valid array");
+ }
+ }
+
+ private <N> N toTypeArray(Class<N> fieldClass, List<?> list) {
+ Object array = Array.newInstance(fieldClass, list.size());
+ for (int i = 0; i < list.size(); i++) {
+ Array.set(array, i, list.get(i));
+ }
+ return (N) array;
+ }
+}
+
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaService.java
index e474feef6..d19833f25 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaService.java
@@ -1,43 +1,43 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.service;
-
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.ElasticsearchStatusException;
-import org.elasticsearch.client.RestHighLevelClient;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
-import org.opengroup.osdu.core.common.model.indexer.OperationType;
-
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.net.URISyntaxException;
-import java.util.List;
-import java.util.Map;
-
-public interface IndexSchemaService {
-
- IndexSchema getIndexerInputSchema(String kind, List<String> errors) throws AppException, UnsupportedEncodingException, URISyntaxException;
-
- IndexSchema getIndexerInputSchema(String kind, boolean invalidateCached) throws AppException, UnsupportedEncodingException, URISyntaxException;
-
- void processSchemaMessages(Map<String, OperationType> schemaMsgs) throws IOException;
-
- void processSchemaUpsertEvent(RestHighLevelClient restClient, String kind) throws IOException, ElasticsearchStatusException, URISyntaxException;
-
- void syncIndexMappingWithStorageSchema(String kind) throws ElasticsearchException, IOException, AppException, URISyntaxException;
-
- boolean isStorageSchemaSyncRequired(String kind, boolean forceClean) throws IOException;
-}
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.service;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
+import org.opengroup.osdu.core.common.model.indexer.OperationType;
+
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URISyntaxException;
+import java.util.List;
+import java.util.Map;
+
+public interface IndexSchemaService {
+
+ IndexSchema getIndexerInputSchema(String kind, List<String> errors) throws AppException, UnsupportedEncodingException, URISyntaxException;
+
+ IndexSchema getIndexerInputSchema(String kind, boolean invalidateCached) throws AppException, UnsupportedEncodingException, URISyntaxException;
+
+ void processSchemaMessages(Map<String, OperationType> schemaMsgs) throws IOException;
+
+ void processSchemaUpsertEvent(RestHighLevelClient restClient, String kind) throws IOException, ElasticsearchStatusException, URISyntaxException;
+
+ void syncIndexMappingWithStorageSchema(String kind) throws ElasticsearchException, IOException, AppException, URISyntaxException;
+
+ boolean isStorageSchemaSyncRequired(String kind, boolean forceClean) throws IOException;
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
index 13738aefa..c10da49c0 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
@@ -1,402 +1,402 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.service;
-
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
-import com.lambdaworks.redis.RedisException;
-import org.apache.http.HttpStatus;
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.action.support.master.AcknowledgedResponse;
-import org.elasticsearch.client.Request;
-import org.elasticsearch.client.RequestOptions;
-import org.elasticsearch.client.Response;
-import org.elasticsearch.client.RestHighLevelClient;
-import org.elasticsearch.client.indices.GetFieldMappingsRequest;
-import org.elasticsearch.client.indices.GetFieldMappingsResponse;
-import org.elasticsearch.client.indices.PutMappingRequest;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.reindex.BulkByScrollResponse;
-import org.elasticsearch.index.reindex.UpdateByQueryRequest;
-import org.opengroup.osdu.core.common.Constants;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
-import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
-import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
-import org.opengroup.osdu.core.common.search.Preconditions;
-import org.opengroup.osdu.indexer.cache.PartitionSafeIndexCache;
-import org.opengroup.osdu.indexer.model.Kind;
-import org.opengroup.osdu.indexer.util.ElasticClientHandler;
-import org.opengroup.osdu.indexer.util.TypeMapper;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Service;
-
-import javax.inject.Inject;
-import java.io.IOException;
-import java.lang.reflect.Type;
-import java.util.*;
-
-@Service
-public class IndexerMappingServiceImpl extends MappingServiceImpl implements IMappingService {
-
- @Inject
- private JaxRsDpsLog log;
- @Inject
- private ElasticClientHandler elasticClientHandler;
- @Autowired
- private PartitionSafeIndexCache indexCache;
- @Autowired
- private IMappingService mappingService;
- @Autowired
- private ElasticIndexNameResolver elasticIndexNameResolver;
-
- private static TimeValue REQUEST_TIMEOUT = TimeValue.timeValueMinutes(1);
-
-
- /**
- * Create a new type in Elasticsearch
- *
- * @param client Elasticsearch client
- * @param index Index name
- * @param merge Try to merge mapping if type already exists
- * @throws IOException if cannot create mapping
- */
- public String createMapping(RestHighLevelClient client, IndexSchema schema, String index, boolean merge) throws IOException {
-
- Map<String, Object> mappingMap = this.getIndexMappingFromRecordSchema(schema);
- String mapping = new Gson().toJson(mappingMap, Map.class);
- this.createMappingWithJson(client, index, schema.getType(), mapping, merge);
- return mapping;
- }
-
- /*
- * Read schema mapping
- *
- * @param schema Index schema
- * @param type Mapping type
- * @return String JSON representation of type and elastic type
- *
- * sample index mapping:
- * "properties": {
- * all meta attributes
- * "acl": {
- * "properties": {
- mapping of all roles
- * }
- * },
- * "legal": {
- * "properties": {
- * mapping of all legal properties
- * }
- * }
- * "data": {
- * "properties": {
- * all data-source attributes
- * }
- * }
- * }
- * */
- public Map<String, Object> getIndexMappingFromRecordSchema(IndexSchema schema) {
-
- // entire property block
- Map<String, Object> properties = new HashMap<>();
-
- // meta attribute
- Map<String, Object> metaMapping = this.getMetaMapping(schema);
-
- // data-source attributes
- Map<String, Object> dataMapping = this.getDataMapping(schema);
- if (!dataMapping.isEmpty()) {
- // inner properties.data.properties block
- Map<String, Object> dataProperties = new HashMap<>();
- dataProperties.put(Constants.PROPERTIES, dataMapping);
-
- // data & meta block
- properties.put(Constants.DATA, dataProperties);
- }
- properties.putAll(metaMapping);
-
- // entire document properties block
- Map<String, Object> documentMapping = new HashMap<>();
- documentMapping.put(Constants.PROPERTIES, properties);
-
- // don't add dynamic mapping
- documentMapping.put("dynamic", false);
- return documentMapping;
- }
-
- private Map<String, Object> getMetaMapping(IndexSchema schema) {
- Map<String, Object> metaMapping = new HashMap<>();
- Kind kind = new Kind(schema.getKind());
-
- for (Map.Entry<String, Object> entry : schema.getMetaSchema().entrySet()) {
- if (entry.getKey() == RecordMetaAttribute.AUTHORITY.getValue()) {
- metaMapping.put(entry.getKey(), TypeMapper.getMetaAttributeIndexerMapping(entry.getKey(), kind.getAuthority()));
- } else if (entry.getKey() == RecordMetaAttribute.SOURCE.getValue()) {
- metaMapping.put(entry.getKey(), TypeMapper.getMetaAttributeIndexerMapping(entry.getKey(), kind.getSource()));
- } else {
- metaMapping.put(entry.getKey(), TypeMapper.getMetaAttributeIndexerMapping(entry.getKey(), null));
- }
- }
- return metaMapping;
- }
-
- private Map<String, Object> getDataMapping(IndexSchema schema) {
- Map<String, Object> dataMapping = new HashMap<>();
- if (schema.getDataSchema() == null || schema.getDataSchema().isEmpty()) return dataMapping;
-
- for (Map.Entry<String, Object> entry : schema.getDataSchema().entrySet()) {
- dataMapping.put(entry.getKey(), TypeMapper.getDataAttributeIndexerMapping(entry.getValue()));
- }
- return dataMapping;
- }
-
- @Override
- public void updateIndexMappingForIndicesOfSameType(Set<String> indices, String fieldName) throws Exception {
- try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
- if(!updateMappingToEnableKeywordIndexingForField(restClient,indices,fieldName)){
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error", "Error updating index mapping.", String.format("Failed to get confirmation from elastic server mapping update for indices: %s", indices));
- }
- }
- }
-
- @Override
- public void syncIndexMappingIfRequired(RestHighLevelClient restClient, IndexSchema schema) throws Exception {
- String index = this.elasticIndexNameResolver.getIndexNameFromKind(schema.getKind());
- final String cacheKey = String.format("metaAttributeMappingSynced-%s", index);
-
- try {
- Boolean mappingSynced = (Boolean) this.indexCache.get(cacheKey);
- if (mappingSynced != null && mappingSynced) return;
- } catch (RedisException ex) {
- //In case the format of cache changes then clean the cache
- this.indexCache.delete(cacheKey);
- }
-
- String jsonResponse = this.mappingService.getIndexMapping(restClient, index);
- Type type = new TypeToken<Map<String, Object>>() {}.getType();
- Map<String, Object> mappings = new Gson().fromJson(jsonResponse, type);
-
- if (mappings == null || mappings.isEmpty()) return;
-
- Map<String, Object> props = (Map<String, Object>) mappings.get("properties");
-
- if (props == null || props.isEmpty()) return;
-
- List<String> missing = new ArrayList<>();
- for (String attribute : TypeMapper.getMetaAttributesKeys()) {
- if (props.containsKey(attribute)) continue;
- missing.add(attribute);
- }
-
- if (missing.isEmpty()) {
- this.indexCache.put(cacheKey, true);
- return;
- }
-
- Map<String, Object> properties = new HashMap<>();
- Kind kind = new Kind(schema.getKind());
- for (String attribute : missing) {
- if (attribute == RecordMetaAttribute.AUTHORITY.getValue()) {
- properties.put(attribute, TypeMapper.getMetaAttributeIndexerMapping(attribute, kind.getAuthority()));
- } else if (attribute == RecordMetaAttribute.SOURCE.getValue()) {
- properties.put(attribute, TypeMapper.getMetaAttributeIndexerMapping(attribute, kind.getSource()));
- } else {
- properties.put(attribute, TypeMapper.getMetaAttributeIndexerMapping(attribute, null));
- }
- }
-
- Map<String, Object> documentMapping = new HashMap<>();
- documentMapping.put(Constants.PROPERTIES, properties);
-
- String mapping = new Gson().toJson(documentMapping, Map.class);
- this.createMappingWithJson(restClient, index, "_doc", mapping, true);
-
- this.indexCache.put(cacheKey, true);
- }
-
- private boolean updateMappingToEnableKeywordIndexingForField(RestHighLevelClient client, Set<String> indicesSet, String fieldName) throws IOException {
- String[] indices = indicesSet.toArray(new String[indicesSet.size()]);
- Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> indexMappingMap = getIndexFieldMap(new String[]{"data."+fieldName}, client, indices);
- boolean failure = false;
- for (String index : indicesSet) {
- if (indexMappingMap.get(index)!=null && updateMappingForAllIndicesOfSameTypeToEnableKeywordIndexingForField(client, index, indexMappingMap.get(index), fieldName)) {
- log.info(String.format("Updated field: %s | index: %s", fieldName, index));
- } else {
- failure=true;
- log.warning(String.format("Failed to update field: %s | index %s", fieldName, index));
- }
- }
- return !failure;
- }
-
- private Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> getIndexFieldMap(String[] fieldNames, RestHighLevelClient client, String[] indices) throws IOException {
- Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> indexMappingMap = new HashMap<>();
- GetFieldMappingsRequest request = new GetFieldMappingsRequest();
- request.indices(indices);
- request.fields(fieldNames);
- try {
- GetFieldMappingsResponse response = client.indices().getFieldMapping(request, RequestOptions.DEFAULT);
- if (response != null && !response.mappings().isEmpty()) {
- final Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappings = response.mappings();
- for (String index : indices) {
- if (mappings != null && !mappings.isEmpty()) {
- indexMappingMap.put(index, mappings);
- }
- }
- }
-
- return indexMappingMap;
- } catch (ElasticsearchException e) {
- log.error(String.format("Failed to get indices: %s. | Error: %s", Arrays.toString(indices), e));
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error", "Error getting indices.", String.format("Failed to get indices error: %s", Arrays.toString(indices)));
- }
- }
-
- private boolean updateMappingForAllIndicesOfSameTypeToEnableKeywordIndexingForField(
- RestHighLevelClient client, String index, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> indexMapping, String fieldName) throws IOException {
-
- PutMappingRequest request = new PutMappingRequest(index);
- String type = indexMapping.keySet().iterator().next();
- if(type.isEmpty()) {
- log.error(String.format("Could not find type of the mappings for index: %s.", index));
- return false;
- }
-
- request.setTimeout(REQUEST_TIMEOUT);
- Map<String, GetFieldMappingsResponse.FieldMappingMetadata> metaData = indexMapping.get(type);
- if(metaData==null || metaData.get("data." + fieldName)==null) {
- log.error(String.format("Could not find field: %s in the mapping of index: %s.", fieldName, index));
- return false;
- }
-
- GetFieldMappingsResponse.FieldMappingMetadata fieldMetaData = metaData.get("data." + fieldName);
- Map<String, Object> source = fieldMetaData.sourceAsMap();
- if(!source.containsKey(fieldName)){
- log.error(String.format("Could not find field: %s in the mapping of index: %s.", fieldName, index));
- return false;
- }
-
- //Index the field with additional keyword type
- Map<String, Object> keywordMap = new HashMap<>();
- keywordMap.put(Constants.TYPE, "keyword");
- Map<String, Object> fieldIndexTypeMap = new HashMap<>();
- fieldIndexTypeMap.put("keyword", keywordMap);
- Map<String, Object> dataFieldMap = (Map<String, Object>) source.get(fieldName);
- dataFieldMap.put("fields", fieldIndexTypeMap);
- Map<String, Object> dataProperties = new HashMap<>();
- dataProperties.put(fieldName, dataFieldMap);
- Map<String, Object> mapping = new HashMap<>();
- mapping.put(Constants.PROPERTIES, dataProperties);
- Map<String, Object> data = new HashMap<>();
- data.put(Constants.DATA,mapping);
- Map<String, Object> properties = new HashMap<>();
- properties.put(Constants.PROPERTIES, data);
-
- request.source(new Gson().toJson(properties), XContentType.JSON);
-
- try {
- AcknowledgedResponse response = client.indices().putMapping(request, RequestOptions.DEFAULT);
- boolean isIndicesUpdated = updateIndices(client, index);
- return response.isAcknowledged() && isIndicesUpdated;
-
- } catch (Exception e) {
- log.error(String.format("Could not update mapping of index: %s. | Error: %s", index, e));
- return false;
- }
- }
-
- private boolean updateIndices(RestHighLevelClient client, String index) throws IOException {
- UpdateByQueryRequest request = new UpdateByQueryRequest(index);
- request.setConflicts("proceed");
- BulkByScrollResponse response = client.updateByQuery(request, RequestOptions.DEFAULT);
- if(!response.getBulkFailures().isEmpty()) {
- log.error(String.format("Could not update index: %s.",index));
- return false;
- }
- return true;
- }
-
- /**
- * Create a new type in Elasticsearch
- *
- * @param client Elasticsearch client
- * @param index Index name
- * @param type Type name
- * @param mapping Mapping if any, null if no specific mapping
- * @param merge Try to merge mapping if type already exists
- * @throws IOException if cannot create index mapping with input json
- */
- private void createMappingWithJson(RestHighLevelClient client, String index, String type, String mapping, boolean merge)
- throws IOException {
-
- boolean mappingExist = isTypeExist(client, index, type);
- if (merge || !mappingExist) {
- createTypeWithMappingInElasticsearch(client, index, type, mapping);
- }
- }
-
- /**
- * Check if a type already exists
- *
- * @param client Elasticsearch client
- * @param index Index name
- * @param type Type name
- * @return true if type already exists
- * @throws IOException in case Elasticsearch responded with a status code that indicated an error
- */
- public boolean isTypeExist(RestHighLevelClient client, String index, String type) throws IOException {
-
- Request request = new Request("HEAD", "/" + index + "/_mapping/" + type);
- Response response = client.getLowLevelClient().performRequest(request);
- return response.getStatusLine().getStatusCode() == 200;
- }
-
- /**
- * Create a new type in Elasticsearch
- *
- * @param client Elasticsearch client
- * @param index Index name
- * @param type Type name
- * @param mapping Mapping if any, null if no specific mapping
- * @throws IOException if mapping cannot be created
- */
- private Boolean createTypeWithMappingInElasticsearch(RestHighLevelClient client, String index, String type, String mapping) throws IOException {
-
- Preconditions.checkNotNull(client, "client cannot be null");
- Preconditions.checkNotNull(index, "index cannot be null");
- Preconditions.checkNotNull(type, "type cannot be null");
-
- try {
- if (mapping != null) {
- PutMappingRequest request = new PutMappingRequest(index);
- request.source(mapping, XContentType.JSON);
- request.setTimeout(REQUEST_TIMEOUT);
- AcknowledgedResponse response = client.indices().putMapping(request, RequestOptions.DEFAULT);
- return response.isAcknowledged();
- }
- } catch (ElasticsearchException e) {
- throw new AppException(
- e.status().getStatus(),
- e.getMessage(),
- String.format("Could not create type mapping %s/%s.", index, type),
- String.format("Failed creating mapping: %s", mapping),
- e);
- }
- return false;
- }
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.service;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import com.lambdaworks.redis.RedisException;
+import org.apache.http.HttpStatus;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.client.indices.GetFieldMappingsRequest;
+import org.elasticsearch.client.indices.GetFieldMappingsResponse;
+import org.elasticsearch.client.indices.PutMappingRequest;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.reindex.BulkByScrollResponse;
+import org.elasticsearch.index.reindex.UpdateByQueryRequest;
+import org.opengroup.osdu.core.common.Constants;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
+import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
+import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
+import org.opengroup.osdu.core.common.search.Preconditions;
+import org.opengroup.osdu.indexer.cache.PartitionSafeIndexCache;
+import org.opengroup.osdu.indexer.model.Kind;
+import org.opengroup.osdu.indexer.util.ElasticClientHandler;
+import org.opengroup.osdu.indexer.util.TypeMapper;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import javax.inject.Inject;
+import java.io.IOException;
+import java.lang.reflect.Type;
+import java.util.*;
+
+@Service
+public class IndexerMappingServiceImpl extends MappingServiceImpl implements IMappingService {
+
+ @Inject
+ private JaxRsDpsLog log;
+ @Inject
+ private ElasticClientHandler elasticClientHandler;
+ @Autowired
+ private PartitionSafeIndexCache indexCache;
+ @Autowired
+ private IMappingService mappingService;
+ @Autowired
+ private ElasticIndexNameResolver elasticIndexNameResolver;
+
+ private static TimeValue REQUEST_TIMEOUT = TimeValue.timeValueMinutes(1);
+
+
+ /**
+ * Create a new type in Elasticsearch
+ *
+ * @param client Elasticsearch client
+ * @param index Index name
+ * @param merge Try to merge mapping if type already exists
+ * @throws IOException if cannot create mapping
+ */
+ public String createMapping(RestHighLevelClient client, IndexSchema schema, String index, boolean merge) throws IOException {
+
+ Map<String, Object> mappingMap = this.getIndexMappingFromRecordSchema(schema);
+ String mapping = new Gson().toJson(mappingMap, Map.class);
+ this.createMappingWithJson(client, index, schema.getType(), mapping, merge);
+ return mapping;
+ }
+
+ /*
+ * Read schema mapping
+ *
+ * @param schema Index schema
+ * @param type Mapping type
+ * @return String JSON representation of type and elastic type
+ *
+ * sample index mapping:
+ * "properties": {
+ * all meta attributes
+ * "acl": {
+ * "properties": {
+ mapping of all roles
+ * }
+ * },
+ * "legal": {
+ * "properties": {
+ * mapping of all legal properties
+ * }
+ * }
+ * "data": {
+ * "properties": {
+ * all data-source attributes
+ * }
+ * }
+ * }
+ * */
+ public Map<String, Object> getIndexMappingFromRecordSchema(IndexSchema schema) {
+
+ // entire property block
+ Map<String, Object> properties = new HashMap<>();
+
+ // meta attribute
+ Map<String, Object> metaMapping = this.getMetaMapping(schema);
+
+ // data-source attributes
+ Map<String, Object> dataMapping = this.getDataMapping(schema);
+ if (!dataMapping.isEmpty()) {
+ // inner properties.data.properties block
+ Map<String, Object> dataProperties = new HashMap<>();
+ dataProperties.put(Constants.PROPERTIES, dataMapping);
+
+ // data & meta block
+ properties.put(Constants.DATA, dataProperties);
+ }
+ properties.putAll(metaMapping);
+
+ // entire document properties block
+ Map<String, Object> documentMapping = new HashMap<>();
+ documentMapping.put(Constants.PROPERTIES, properties);
+
+ // don't add dynamic mapping
+ documentMapping.put("dynamic", false);
+ return documentMapping;
+ }
+
+ private Map<String, Object> getMetaMapping(IndexSchema schema) {
+ Map<String, Object> metaMapping = new HashMap<>();
+ Kind kind = new Kind(schema.getKind());
+
+ for (Map.Entry<String, Object> entry : schema.getMetaSchema().entrySet()) {
+ if (entry.getKey() == RecordMetaAttribute.AUTHORITY.getValue()) {
+ metaMapping.put(entry.getKey(), TypeMapper.getMetaAttributeIndexerMapping(entry.getKey(), kind.getAuthority()));
+ } else if (entry.getKey() == RecordMetaAttribute.SOURCE.getValue()) {
+ metaMapping.put(entry.getKey(), TypeMapper.getMetaAttributeIndexerMapping(entry.getKey(), kind.getSource()));
+ } else {
+ metaMapping.put(entry.getKey(), TypeMapper.getMetaAttributeIndexerMapping(entry.getKey(), null));
+ }
+ }
+ return metaMapping;
+ }
+
+ private Map<String, Object> getDataMapping(IndexSchema schema) {
+ Map<String, Object> dataMapping = new HashMap<>();
+ if (schema.getDataSchema() == null || schema.getDataSchema().isEmpty()) return dataMapping;
+
+ for (Map.Entry<String, Object> entry : schema.getDataSchema().entrySet()) {
+ dataMapping.put(entry.getKey(), TypeMapper.getDataAttributeIndexerMapping(entry.getValue()));
+ }
+ return dataMapping;
+ }
+
+ @Override
+ public void updateIndexMappingForIndicesOfSameType(Set<String> indices, String fieldName) throws Exception {
+ try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
+ if(!updateMappingToEnableKeywordIndexingForField(restClient,indices,fieldName)){
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error", "Error updating index mapping.", String.format("Failed to get confirmation from elastic server mapping update for indices: %s", indices));
+ }
+ }
+ }
+
+ @Override
+ public void syncIndexMappingIfRequired(RestHighLevelClient restClient, IndexSchema schema) throws Exception {
+ String index = this.elasticIndexNameResolver.getIndexNameFromKind(schema.getKind());
+ final String cacheKey = String.format("metaAttributeMappingSynced-%s", index);
+
+ try {
+ Boolean mappingSynced = (Boolean) this.indexCache.get(cacheKey);
+ if (mappingSynced != null && mappingSynced) return;
+ } catch (RedisException ex) {
+ //In case the format of cache changes then clean the cache
+ this.indexCache.delete(cacheKey);
+ }
+
+ String jsonResponse = this.mappingService.getIndexMapping(restClient, index);
+ Type type = new TypeToken<Map<String, Object>>() {}.getType();
+ Map<String, Object> mappings = new Gson().fromJson(jsonResponse, type);
+
+ if (mappings == null || mappings.isEmpty()) return;
+
+ Map<String, Object> props = (Map<String, Object>) mappings.get("properties");
+
+ if (props == null || props.isEmpty()) return;
+
+ List<String> missing = new ArrayList<>();
+ for (String attribute : TypeMapper.getMetaAttributesKeys()) {
+ if (props.containsKey(attribute)) continue;
+ missing.add(attribute);
+ }
+
+ if (missing.isEmpty()) {
+ this.indexCache.put(cacheKey, true);
+ return;
+ }
+
+ Map<String, Object> properties = new HashMap<>();
+ Kind kind = new Kind(schema.getKind());
+ for (String attribute : missing) {
+ if (attribute == RecordMetaAttribute.AUTHORITY.getValue()) {
+ properties.put(attribute, TypeMapper.getMetaAttributeIndexerMapping(attribute, kind.getAuthority()));
+ } else if (attribute == RecordMetaAttribute.SOURCE.getValue()) {
+ properties.put(attribute, TypeMapper.getMetaAttributeIndexerMapping(attribute, kind.getSource()));
+ } else {
+ properties.put(attribute, TypeMapper.getMetaAttributeIndexerMapping(attribute, null));
+ }
+ }
+
+ Map<String, Object> documentMapping = new HashMap<>();
+ documentMapping.put(Constants.PROPERTIES, properties);
+
+ String mapping = new Gson().toJson(documentMapping, Map.class);
+ this.createMappingWithJson(restClient, index, "_doc", mapping, true);
+
+ this.indexCache.put(cacheKey, true);
+ }
+
+ private boolean updateMappingToEnableKeywordIndexingForField(RestHighLevelClient client, Set<String> indicesSet, String fieldName) throws IOException {
+ String[] indices = indicesSet.toArray(new String[indicesSet.size()]);
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> indexMappingMap = getIndexFieldMap(new String[]{"data."+fieldName}, client, indices);
+ boolean failure = false;
+ for (String index : indicesSet) {
+ if (indexMappingMap.get(index)!=null && updateMappingForAllIndicesOfSameTypeToEnableKeywordIndexingForField(client, index, indexMappingMap.get(index), fieldName)) {
+ log.info(String.format("Updated field: %s | index: %s", fieldName, index));
+ } else {
+ failure=true;
+ log.warning(String.format("Failed to update field: %s | index %s", fieldName, index));
+ }
+ }
+ return !failure;
+ }
+
+ private Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> getIndexFieldMap(String[] fieldNames, RestHighLevelClient client, String[] indices) throws IOException {
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> indexMappingMap = new HashMap<>();
+ GetFieldMappingsRequest request = new GetFieldMappingsRequest();
+ request.indices(indices);
+ request.fields(fieldNames);
+ try {
+ GetFieldMappingsResponse response = client.indices().getFieldMapping(request, RequestOptions.DEFAULT);
+ if (response != null && !response.mappings().isEmpty()) {
+ final Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappings = response.mappings();
+ for (String index : indices) {
+ if (mappings != null && !mappings.isEmpty()) {
+ indexMappingMap.put(index, mappings);
+ }
+ }
+ }
+
+ return indexMappingMap;
+ } catch (ElasticsearchException e) {
+ log.error(String.format("Failed to get indices: %s. | Error: %s", Arrays.toString(indices), e));
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error", "Error getting indices.", String.format("Failed to get indices error: %s", Arrays.toString(indices)));
+ }
+ }
+
+ private boolean updateMappingForAllIndicesOfSameTypeToEnableKeywordIndexingForField(
+ RestHighLevelClient client, String index, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> indexMapping, String fieldName) throws IOException {
+
+ PutMappingRequest request = new PutMappingRequest(index);
+ String type = indexMapping.keySet().iterator().next();
+ if(type.isEmpty()) {
+ log.error(String.format("Could not find type of the mappings for index: %s.", index));
+ return false;
+ }
+
+ request.setTimeout(REQUEST_TIMEOUT);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> metaData = indexMapping.get(type);
+ if(metaData==null || metaData.get("data." + fieldName)==null) {
+ log.error(String.format("Could not find field: %s in the mapping of index: %s.", fieldName, index));
+ return false;
+ }
+
+ GetFieldMappingsResponse.FieldMappingMetadata fieldMetaData = metaData.get("data." + fieldName);
+ Map<String, Object> source = fieldMetaData.sourceAsMap();
+ if(!source.containsKey(fieldName)){
+ log.error(String.format("Could not find field: %s in the mapping of index: %s.", fieldName, index));
+ return false;
+ }
+
+ //Index the field with additional keyword type
+ Map<String, Object> keywordMap = new HashMap<>();
+ keywordMap.put(Constants.TYPE, "keyword");
+ Map<String, Object> fieldIndexTypeMap = new HashMap<>();
+ fieldIndexTypeMap.put("keyword", keywordMap);
+ Map<String, Object> dataFieldMap = (Map<String, Object>) source.get(fieldName);
+ dataFieldMap.put("fields", fieldIndexTypeMap);
+ Map<String, Object> dataProperties = new HashMap<>();
+ dataProperties.put(fieldName, dataFieldMap);
+ Map<String, Object> mapping = new HashMap<>();
+ mapping.put(Constants.PROPERTIES, dataProperties);
+ Map<String, Object> data = new HashMap<>();
+ data.put(Constants.DATA,mapping);
+ Map<String, Object> properties = new HashMap<>();
+ properties.put(Constants.PROPERTIES, data);
+
+ request.source(new Gson().toJson(properties), XContentType.JSON);
+
+ try {
+ AcknowledgedResponse response = client.indices().putMapping(request, RequestOptions.DEFAULT);
+ boolean isIndicesUpdated = updateIndices(client, index);
+ return response.isAcknowledged() && isIndicesUpdated;
+
+ } catch (Exception e) {
+ log.error(String.format("Could not update mapping of index: %s. | Error: %s", index, e));
+ return false;
+ }
+ }
+
+ private boolean updateIndices(RestHighLevelClient client, String index) throws IOException {
+ UpdateByQueryRequest request = new UpdateByQueryRequest(index);
+ request.setConflicts("proceed");
+ BulkByScrollResponse response = client.updateByQuery(request, RequestOptions.DEFAULT);
+ if(!response.getBulkFailures().isEmpty()) {
+ log.error(String.format("Could not update index: %s.",index));
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Create a new type in Elasticsearch
+ *
+ * @param client Elasticsearch client
+ * @param index Index name
+ * @param type Type name
+ * @param mapping Mapping if any, null if no specific mapping
+ * @param merge Try to merge mapping if type already exists
+ * @throws IOException if cannot create index mapping with input json
+ */
+ private void createMappingWithJson(RestHighLevelClient client, String index, String type, String mapping, boolean merge)
+ throws IOException {
+
+ boolean mappingExist = isTypeExist(client, index, type);
+ if (merge || !mappingExist) {
+ createTypeWithMappingInElasticsearch(client, index, type, mapping);
+ }
+ }
+
+ /**
+ * Check if a type already exists
+ *
+ * @param client Elasticsearch client
+ * @param index Index name
+ * @param type Type name
+ * @return true if type already exists
+ * @throws IOException in case Elasticsearch responded with a status code that indicated an error
+ */
+ public boolean isTypeExist(RestHighLevelClient client, String index, String type) throws IOException {
+
+ Request request = new Request("HEAD", "/" + index + "/_mapping/" + type);
+ Response response = client.getLowLevelClient().performRequest(request);
+ return response.getStatusLine().getStatusCode() == 200;
+ }
+
+ /**
+ * Create a new type in Elasticsearch
+ *
+ * @param client Elasticsearch client
+ * @param index Index name
+ * @param type Type name
+ * @param mapping Mapping if any, null if no specific mapping
+ * @throws IOException if mapping cannot be created
+ */
+ private Boolean createTypeWithMappingInElasticsearch(RestHighLevelClient client, String index, String type, String mapping) throws IOException {
+
+ Preconditions.checkNotNull(client, "client cannot be null");
+ Preconditions.checkNotNull(index, "index cannot be null");
+ Preconditions.checkNotNull(type, "type cannot be null");
+
+ try {
+ if (mapping != null) {
+ PutMappingRequest request = new PutMappingRequest(index);
+ request.source(mapping, XContentType.JSON);
+ request.setTimeout(REQUEST_TIMEOUT);
+ AcknowledgedResponse response = client.indices().putMapping(request, RequestOptions.DEFAULT);
+ return response.isAcknowledged();
+ }
+ } catch (ElasticsearchException e) {
+ throw new AppException(
+ e.status().getStatus(),
+ e.getMessage(),
+ String.format("Could not create type mapping %s/%s.", index, type),
+ String.format("Failed creating mapping: %s", mapping),
+ e);
+ }
+ return false;
+ }
}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
index 6bec8964f..63ea7062d 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
@@ -1,578 +1,578 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.service;
-
-import com.google.common.base.Strings;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import org.apache.http.HttpStatus;
-import org.elasticsearch.ElasticsearchStatusException;
-import org.elasticsearch.action.DocWriteRequest;
-import org.elasticsearch.action.bulk.BulkItemResponse;
-import org.elasticsearch.action.bulk.BulkRequest;
-import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.delete.DeleteRequest;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.update.UpdateRequest;
-import org.elasticsearch.client.RequestOptions;
-import org.elasticsearch.client.RestHighLevelClient;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.rest.RestStatus;
-import org.opengroup.osdu.core.common.Constants;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.entitlements.Acl;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.http.RequestStatus;
-import org.opengroup.osdu.core.common.model.indexer.*;
-import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
-import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
-import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
-import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
-import org.opengroup.osdu.indexer.logging.AuditLogger;
-import org.opengroup.osdu.indexer.provider.interfaces.IPublisher;
-import org.opengroup.osdu.indexer.util.ElasticClientHandler;
-import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
-import org.springframework.context.annotation.Primary;
-import org.springframework.stereotype.Service;
-
-import javax.inject.Inject;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.function.Consumer;
-import java.util.stream.Collectors;
-
-@Service
-@Primary
-public class IndexerServiceImpl implements IndexerService {
-
- private static final TimeValue BULK_REQUEST_TIMEOUT = TimeValue.timeValueMinutes(1);
-
- private static final List<RestStatus> RETRY_ELASTIC_EXCEPTION = new ArrayList<>(Arrays.asList(RestStatus.TOO_MANY_REQUESTS, RestStatus.BAD_GATEWAY, RestStatus.SERVICE_UNAVAILABLE));
-
- private final Gson gson = new GsonBuilder().serializeNulls().create();
-
- @Inject
- private JaxRsDpsLog jaxRsDpsLog;
- @Inject
- private AuditLogger auditLogger;
- @Inject
- private StorageService storageService;
- @Inject
- private IndexSchemaService schemaService;
- @Inject
- private IndicesService indicesService;
- @Inject
- private IMappingService mappingService;
- @Inject
- private IPublisher progressPublisher;
- @Inject
- private ElasticClientHandler elasticClientHandler;
- @Inject
- private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
- @Inject
- private ElasticIndexNameResolver elasticIndexNameResolver;
- @Inject
- private StorageIndexerPayloadMapper storageIndexerPayloadMapper;
- @Inject
- private IRequestInfo requestInfo;
- @Inject
- private JobStatus jobStatus;
-
- private DpsHeaders headers;
-
- @Override
- public JobStatus processRecordChangedMessages(RecordChangedMessages message, List<RecordInfo> recordInfos) throws Exception {
-
- // this should not happen
- if (recordInfos.size() == 0) return null;
-
- String errorMessage = "";
- List<String> retryRecordIds = new LinkedList<>();
-
- // get auth header with service account Authorization
- this.headers = this.requestInfo.getHeadersWithDwdAuthZ();
-
- // initialize status for all messages.
- this.jobStatus.initialize(recordInfos);
-
- try {
- auditLogger.indexStarted(recordInfos.stream()
- .map(RecordInfo::getKind)
- .collect(Collectors.toList()));
-
- // get upsert records
- Map<String, Map<String, OperationType>> upsertRecordMap = RecordInfo.getUpsertRecordIds(recordInfos);
- if (upsertRecordMap != null && !upsertRecordMap.isEmpty()) {
- List<String> upsertFailureRecordIds = processUpsertRecords(upsertRecordMap);
- retryRecordIds.addAll(upsertFailureRecordIds);
- }
-
- // get delete records
- Map<String, List<String>> deleteRecordMap = RecordInfo.getDeleteRecordIds(recordInfos);
- if (deleteRecordMap != null && !deleteRecordMap.isEmpty()) {
- List<String> deleteFailureRecordIds = processDeleteRecords(deleteRecordMap);
- retryRecordIds.addAll(deleteFailureRecordIds);
- }
-
- // process legacy storage schema change messages
- Map<String, OperationType> schemaMsgs = RecordInfo.getSchemaMsgs(recordInfos);
- if (schemaMsgs != null && !schemaMsgs.isEmpty()) {
- this.schemaService.processSchemaMessages(schemaMsgs);
- }
-
- // process failed records
- if (retryRecordIds.size() > 0) {
- retryAndEnqueueFailedRecords(recordInfos, retryRecordIds, message);
- }
- } catch (IOException e) {
- errorMessage = e.getMessage();
- throw new AppException(HttpStatus.SC_GATEWAY_TIMEOUT, "Internal communication failure", errorMessage, e);
- } catch (AppException e) {
- errorMessage = e.getMessage();
- throw e;
- } catch (Exception e) {
- errorMessage = "error indexing records";
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Unknown error", "An unknown error has occurred.", e);
- } finally {
- this.jobStatus.finalizeRecordStatus(errorMessage);
- this.updateAuditLog();
- this.progressPublisher.publishStatusChangedTagsToTopic(this.headers, this.jobStatus);
- }
-
- return jobStatus;
- }
-
- @Override
- public void processSchemaMessages(List<RecordInfo> recordInfos) throws IOException {
- Map<String, OperationType> schemaMsgs = RecordInfo.getSchemaMsgs(recordInfos);
- if (schemaMsgs != null && !schemaMsgs.isEmpty()) {
- try (RestHighLevelClient restClient = elasticClientHandler.createRestClient()) {
- schemaMsgs.entrySet().forEach(msg -> {
- try {
- processSchemaEvents(restClient, msg);
- } catch (IOException | ElasticsearchStatusException e) {
- throw new AppException(org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR.value(), "unable to process schema delete", e.getMessage());
- }
- });
- }
- }
- }
-
- private void processSchemaEvents(RestHighLevelClient restClient,
- Map.Entry<String, OperationType> msg) throws IOException, ElasticsearchStatusException {
- String kind = msg.getKey();
- String index = elasticIndexNameResolver.getIndexNameFromKind(kind);
-
- boolean indexExist = indicesService.isIndexExist(restClient, index);
- if (indexExist && msg.getValue() == OperationType.purge_schema) {
- indicesService.deleteIndex(restClient, index);
- }
- }
-
- private List<String> processUpsertRecords(Map<String, Map<String, OperationType>> upsertRecordMap) throws Exception {
- // get schema for kind
- Map<String, IndexSchema> schemas = this.getSchema(upsertRecordMap);
-
- if (schemas.isEmpty()) return new LinkedList<>();
-
- // get recordIds with valid upsert index-status
- List<String> recordIds = this.jobStatus.getIdsByValidUpsertIndexingStatus();
-
- if (recordIds.isEmpty()) return new LinkedList<>();
-
- // get records via storage api
- Records storageRecords = this.storageService.getStorageRecords(recordIds);
- List<String> failedOrRetryRecordIds = new LinkedList<>(storageRecords.getMissingRetryRecords());
-
- // map storage records to indexer payload
- RecordIndexerPayload recordIndexerPayload = this.getIndexerPayload(upsertRecordMap, schemas, storageRecords);
-
- jaxRsDpsLog.info(String.format("records change messages received : %s | valid storage bulk records: %s | valid index payload: %s", recordIds.size(), storageRecords.getRecords().size(), recordIndexerPayload.getRecords().size()));
-
- // index records
- failedOrRetryRecordIds.addAll(processElasticMappingAndUpsertRecords(recordIndexerPayload));
-
- return failedOrRetryRecordIds;
- }
-
- private Map<String, IndexSchema> getSchema(Map<String, Map<String, OperationType>> upsertRecordMap) {
-
- Map<String, IndexSchema> schemas = new HashMap<>();
-
- try {
- for (Map.Entry<String, Map<String, OperationType>> entry : upsertRecordMap.entrySet()) {
-
- String kind = entry.getKey();
- List<String> errors = new ArrayList<>();
- IndexSchema schemaObj = this.schemaService.getIndexerInputSchema(kind, errors);
- if (!errors.isEmpty()) {
- this.jobStatus.addOrUpdateRecordStatus(entry.getValue().keySet(), IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, String.join("|", errors), String.format("error | kind: %s", kind));
- } else if (schemaObj.isDataSchemaMissing()) {
- this.jobStatus.addOrUpdateRecordStatus(entry.getValue().keySet(), IndexingStatus.WARN, HttpStatus.SC_NOT_FOUND, "schema not found", String.format("schema not found | kind: %s", kind));
- }
-
- schemas.put(kind, schemaObj);
- }
- } catch (AppException e) {
- throw e;
- } catch (Exception e) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Get schema error", "An error has occurred while getting schema", e);
- }
-
- return schemas;
- }
-
- private RecordIndexerPayload getIndexerPayload(Map<String, Map<String, OperationType>> upsertRecordMap, Map<String, IndexSchema> kindSchemaMap, Records records) {
- List<Records.Entity> storageValidRecords = records.getRecords();
- List<RecordIndexerPayload.Record> indexerPayload = new ArrayList<>();
- List<IndexSchema> schemas = new ArrayList<>();
-
- for (Records.Entity storageRecord : storageValidRecords) {
-
- Map<String, OperationType> idOperationMap = upsertRecordMap.get(storageRecord.getKind());
-
- // skip if storage returned record with same id but different kind
- if (idOperationMap == null) {
- String message = String.format("storage service returned incorrect record | requested kind: %s | received kind: %s", this.jobStatus.getRecordKindById(storageRecord.getId()), storageRecord.getKind());
- this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.SKIP, RequestStatus.STORAGE_CONFLICT, message, String.format("%s | record-id: %s", message, storageRecord.getId()));
- continue;
- }
-
- IndexSchema schema = kindSchemaMap.get(storageRecord.getKind());
- schemas.add(schema);
-
- // skip indexing of records if data block is empty
- RecordIndexerPayload.Record document = prepareIndexerPayload(schema, storageRecord, idOperationMap);
- if (document != null) {
- indexerPayload.add(document);
- }
- }
-
- // this should only happen if storage service returned WRONG records with kind for all the records in the messages
- if (indexerPayload.isEmpty()) {
- throw new AppException(RequestStatus.STORAGE_CONFLICT, "Indexer error", "upsert record failed, storage service returned incorrect records");
- }
-
- return RecordIndexerPayload.builder().records(indexerPayload).schemas(schemas).build();
- }
-
- private RecordIndexerPayload.Record prepareIndexerPayload(IndexSchema schemaObj, Records.Entity storageRecord, Map<String, OperationType> idToOperationMap) {
-
- RecordIndexerPayload.Record document = null;
-
- try {
- Map<String, Object> storageRecordData = storageRecord.getData();
- document = new RecordIndexerPayload.Record();
- if (storageRecordData == null || storageRecordData.isEmpty()) {
- String message = "empty or null data block found in the storage record";
- this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.WARN, HttpStatus.SC_NOT_FOUND, message, String.format("record-id: %s | %s", storageRecord.getId(), message));
- } else if (schemaObj.isDataSchemaMissing()) {
- document.setSchemaMissing(true);
- } else {
- Map<String, Object> dataMap = this.storageIndexerPayloadMapper.mapDataPayload(schemaObj, storageRecordData, storageRecord.getId());
- if (dataMap.isEmpty()) {
- document.setMappingMismatch(true);
- String message = String.format("complete schema mismatch: none of the data attribute can be mapped | data: %s", storageRecordData);
- this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.WARN, HttpStatus.SC_NOT_FOUND, message, String.format("record-id: %s | %s", storageRecord.getId(), message));
- }
- document.setData(dataMap);
- }
- } catch (AppException e) {
- this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL, HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getMessage());
- jaxRsDpsLog.warning(String.format("record-id: %s | %s", storageRecord.getId(), e.getMessage()), e);
- } catch (Exception e) {
- this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL, HttpStatus.SC_INTERNAL_SERVER_ERROR, String.format("error parsing records against schema, error-message: %s", e.getMessage()));
- jaxRsDpsLog.error(String.format("record-id: %s | error parsing records against schema, error-message: %s", storageRecord.getId(), e.getMessage()), e);
- }
-
- try {
- // index individual parts of kind
- String[] kindParts = storageRecord.getKind().split(":");
- String authority = kindParts[0];
- String source = kindParts[1];
- String type = kindParts[2];
- document.setKind(storageRecord.getKind());
- document.setNamespace(authority + ":" + source);
- document.setAuthority(authority);
- document.setSource(source);
- document.setType(type);
- document.setId(storageRecord.getId());
- document.setVersion(storageRecord.getVersion());
- document.setAcl(storageRecord.getAcl());
- document.setLegal(storageRecord.getLegal());
- if (storageRecord.getTags() != null) {
- document.setTags(storageRecord.getTags());
- }
- document.setCreateUser(storageRecord.getCreateUser());
- document.setCreateTime(storageRecord.getCreateTime());
- if (!Strings.isNullOrEmpty(storageRecord.getModifyUser())) {
- document.setModifyUser(storageRecord.getModifyUser());
- }
- if (!Strings.isNullOrEmpty(storageRecord.getModifyTime())) {
- document.setModifyTime(storageRecord.getModifyTime());
- }
- RecordStatus recordStatus = this.jobStatus.getJobStatusByRecordId(storageRecord.getId());
- if (recordStatus.getIndexProgress().getStatusCode() == 0) {
- recordStatus.getIndexProgress().setStatusCode(HttpStatus.SC_OK);
- }
- document.setIndexProgress(recordStatus.getIndexProgress());
- if (storageRecord.getAncestry() != null) document.setAncestry(storageRecord.getAncestry());
- document.setOperationType(idToOperationMap.get(storageRecord.getId()));
- } catch (Exception e) {
- this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL, HttpStatus.SC_INTERNAL_SERVER_ERROR, String.format("error parsing meta data, error-message: %s", e.getMessage()));
- jaxRsDpsLog.error(String.format("record-id: %s | error parsing meta data, error-message: %s", storageRecord.getId(), e.getMessage()), e);
- }
- return document;
- }
-
- private List<String> processElasticMappingAndUpsertRecords(RecordIndexerPayload recordIndexerPayload) throws Exception {
-
- try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
- List<IndexSchema> schemas = recordIndexerPayload.getSchemas();
- if (schemas == null || schemas.isEmpty()) {
- return new LinkedList<>();
- }
-
- // process the schema
- this.cacheOrCreateElasticMapping(schemas, restClient);
-
- // process the records
- return this.upsertRecords(recordIndexerPayload.getRecords(), restClient);
- }
- }
-
- private void cacheOrCreateElasticMapping(List<IndexSchema> schemas, RestHighLevelClient restClient) throws Exception {
-
- for (IndexSchema schema : schemas) {
- String index = this.elasticIndexNameResolver.getIndexNameFromKind(schema.getKind());
-
- // check if index exist and sync meta attribute schema if required
- if (this.indicesService.isIndexReady(restClient, index)) {
- this.mappingService.syncIndexMappingIfRequired(restClient, schema);
- continue;
- }
-
- // create index
- Map<String, Object> mapping = this.mappingService.getIndexMappingFromRecordSchema(schema);
- if (!this.indicesService.createIndex(restClient, index, null, schema.getType(), mapping)) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error", "Error creating index.", String.format("Failed to get confirmation from elastic server for index: %s", index));
- }
- }
- }
-
- private List<String> upsertRecords(List<RecordIndexerPayload.Record> records, RestHighLevelClient restClient) throws AppException {
- if (records == null || records.isEmpty()) return new LinkedList<>();
-
- BulkRequest bulkRequest = new BulkRequest();
- bulkRequest.timeout(BULK_REQUEST_TIMEOUT);
-
- for (RecordIndexerPayload.Record record : records) {
- if ((record.getData() == null || record.getData().isEmpty()) && !record.skippedDataIndexing()) {
- // it will come here when schema is missing
- // TODO: rollback once we know what is causing the problem
- jaxRsDpsLog.warning(String.format("data not found for record: %s", record));
- }
-
- OperationType operation = record.getOperationType();
- Map<String, Object> sourceMap = getSourceMap(record);
- String index = this.elasticIndexNameResolver.getIndexNameFromKind(record.getKind());
-
- if (operation == OperationType.create) {
- IndexRequest indexRequest = new IndexRequest(index).id(record.getId()).source(this.gson.toJson(sourceMap), XContentType.JSON);
- bulkRequest.add(indexRequest);
- } else if (operation == OperationType.update) {
- UpdateRequest updateRequest = new UpdateRequest(index, "_doc", record.getId()).doc(this.gson.toJson(sourceMap), XContentType.JSON).docAsUpsert(true);
- bulkRequest.add(updateRequest);
- }
- }
-
- return processBulkRequest(restClient, bulkRequest);
- }
-
- private List<String> processDeleteRecords(Map<String, List<String>> deleteRecordMap) throws Exception {
- BulkRequest bulkRequest = new BulkRequest();
- bulkRequest.timeout(BULK_REQUEST_TIMEOUT);
-
- for (Map.Entry<String, List<String>> record : deleteRecordMap.entrySet()) {
-
- String index = this.elasticIndexNameResolver.getIndexNameFromKind(record.getKey());
-
- for (String id : record.getValue()) {
- DeleteRequest deleteRequest = new DeleteRequest(index, id);
- bulkRequest.add(deleteRequest);
- }
- }
-
- try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
- return processBulkRequest(restClient, bulkRequest);
- }
- }
-
- private List<String> processBulkRequest(RestHighLevelClient restClient, BulkRequest bulkRequest) throws AppException {
-
- List<String> failureRecordIds = new LinkedList<>();
- if (bulkRequest.numberOfActions() == 0) return failureRecordIds;
- int failedRequestStatus = 500;
- Exception failedRequestCause = null;
-
- try {
- long startTime = System.currentTimeMillis();
- BulkResponse bulkResponse = restClient.bulk(bulkRequest, RequestOptions.DEFAULT);
- long stopTime = System.currentTimeMillis();
-
- // log failed bulk requests
- ArrayList<String> bulkFailures = new ArrayList<>();
- int succeededResponses = 0;
- int failedResponses = 0;
-
- for (BulkItemResponse bulkItemResponse : bulkResponse.getItems()) {
- if (bulkItemResponse.isFailed()) {
- BulkItemResponse.Failure failure = bulkItemResponse.getFailure();
- bulkFailures.add(String.format("elasticsearch bulk service status: %s | id: %s | message: %s", failure.getStatus(), failure.getId(), failure.getMessage()));
- this.jobStatus.addOrUpdateRecordStatus(bulkItemResponse.getId(), IndexingStatus.FAIL, failure.getStatus().getStatus(), bulkItemResponse.getFailureMessage());
- if (canIndexerRetry(bulkItemResponse)) {
- failureRecordIds.add(bulkItemResponse.getId());
-
- if (failedRequestCause == null) {
- failedRequestCause = failure.getCause();
- failedRequestStatus = failure.getStatus().getStatus();
- }
- }
-
- failedResponses++;
- } else {
- succeededResponses++;
- this.jobStatus.addOrUpdateRecordStatus(bulkItemResponse.getId(), IndexingStatus.SUCCESS, HttpStatus.SC_OK, "Indexed Successfully");
- }
- }
- if (!bulkFailures.isEmpty()) this.jaxRsDpsLog.warning(bulkFailures);
-
- jaxRsDpsLog.info(String.format("records in elasticsearch service bulk request: %s | successful: %s | failed: %s | time taken for bulk request: %d milliseconds", bulkRequest.numberOfActions(), succeededResponses, failedResponses, stopTime-startTime));
-
- // retry entire message if all records are failing
- if (bulkRequest.numberOfActions() == failureRecordIds.size()) throw new AppException(failedRequestStatus, "Elastic error", failedRequestCause.getMessage(), failedRequestCause);
- } catch (IOException e) {
- // throw explicit 504 for IOException
- throw new AppException(HttpStatus.SC_GATEWAY_TIMEOUT, "Elastic error", "Request cannot be completed in specified time.", e);
- } catch (ElasticsearchStatusException e) {
- throw new AppException(e.status().getStatus(), "Elastic error", e.getMessage(), e);
- } catch (Exception e) {
- if (e instanceof AppException) {
- throw e;
- }
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error", "Error indexing records.", e);
- }
- return failureRecordIds;
- }
-
- private Map<String, Object> getSourceMap(RecordIndexerPayload.Record record) {
-
- Map<String, Object> indexerPayload = new HashMap<>();
-
- // get the key and get the corresponding object from the individualRecord object
- if (record.getData() != null) {
- Map<String, Object> data = new HashMap<>();
- for (Map.Entry<String, Object> entry : record.getData().entrySet()) {
- data.put(entry.getKey(), entry.getValue());
- }
- indexerPayload.put(Constants.DATA, data);
- }
-
- indexerPayload.put(RecordMetaAttribute.ID.getValue(), record.getId());
- indexerPayload.put(RecordMetaAttribute.KIND.getValue(), record.getKind());
- indexerPayload.put(RecordMetaAttribute.AUTHORITY.getValue(), record.getAuthority());
- indexerPayload.put(RecordMetaAttribute.SOURCE.getValue(), record.getSource());
- indexerPayload.put(RecordMetaAttribute.NAMESPACE.getValue(), record.getNamespace());
- indexerPayload.put(RecordMetaAttribute.TYPE.getValue(), record.getType());
- indexerPayload.put(RecordMetaAttribute.VERSION.getValue(), record.getVersion());
- indexerPayload.put(RecordMetaAttribute.ACL.getValue(), record.getAcl());
- indexerPayload.put(RecordMetaAttribute.TAGS.getValue(), record.getTags());
- indexerPayload.put(RecordMetaAttribute.X_ACL.getValue(), Acl.flattenAcl(record.getAcl()));
- indexerPayload.put(RecordMetaAttribute.LEGAL.getValue(), record.getLegal());
- indexerPayload.put(RecordMetaAttribute.INDEX_STATUS.getValue(), record.getIndexProgress());
- if (record.getAncestry() != null) {
- indexerPayload.put(RecordMetaAttribute.ANCESTRY.getValue(), record.getAncestry());
- }
- indexerPayload.put(RecordMetaAttribute.CREATE_USER.getValue(), record.getCreateUser());
- indexerPayload.put(RecordMetaAttribute.CREATE_TIME.getValue(), record.getCreateTime());
- if (!Strings.isNullOrEmpty(record.getModifyUser())) {
- indexerPayload.put(RecordMetaAttribute.MODIFY_USER.getValue(), record.getModifyUser());
- }
- if (!Strings.isNullOrEmpty(record.getModifyTime())) {
- indexerPayload.put(RecordMetaAttribute.MODIFY_TIME.getValue(), record.getModifyTime());
- }
- return indexerPayload;
- }
-
- private boolean canIndexerRetry(BulkItemResponse bulkItemResponse) {
- if (RETRY_ELASTIC_EXCEPTION.contains(bulkItemResponse.status())) return true;
-
- if ((bulkItemResponse.getOpType() == DocWriteRequest.OpType.CREATE || bulkItemResponse.getOpType() == DocWriteRequest.OpType.UPDATE)
- && bulkItemResponse.status() == RestStatus.NOT_FOUND) {
- return true;
- }
-
- return false;
- }
-
- private void retryAndEnqueueFailedRecords(List<RecordInfo> recordInfos, List<String> failuresRecordIds, RecordChangedMessages message) throws IOException {
-
- jaxRsDpsLog.info(String.format("queuing bulk failed records back to task-queue for retry | count: %s | records: %s", failuresRecordIds.size(), failuresRecordIds));
- List<RecordInfo> retryRecordInfos = new LinkedList<>();
- for (String recordId : failuresRecordIds) {
- for (RecordInfo origMessage : recordInfos) {
- if (origMessage.getId().equalsIgnoreCase(recordId)) {
- retryRecordInfos.add(origMessage);
- }
- }
- }
-
- RecordChangedMessages newMessage = RecordChangedMessages.builder()
- .messageId(message.getMessageId())
- .publishTime(message.getPublishTime())
- .data(this.gson.toJson(retryRecordInfos))
- .attributes(message.getAttributes()).build();
-
- String payLoad = this.gson.toJson(newMessage);
- this.indexerQueueTaskBuilder.createWorkerTask(payLoad, this.headers);
- }
-
- private void updateAuditLog() {
- logAuditEvents(OperationType.create, this.auditLogger::indexCreateRecordSuccess, this.auditLogger::indexCreateRecordFail);
- logAuditEvents(OperationType.update, this.auditLogger::indexUpdateRecordSuccess, this.auditLogger::indexUpdateRecordFail);
- logAuditEvents(OperationType.purge, this.auditLogger::indexPurgeRecordSuccess, this.auditLogger::indexPurgeRecordFail);
- logAuditEvents(OperationType.delete, this.auditLogger::indexDeleteRecordSuccess, this.auditLogger::indexDeleteRecordFail);
- }
-
- private void logAuditEvents(OperationType operationType, Consumer<List<String>> successEvent, Consumer<List<String>> failedEvent) {
- List<RecordStatus> succeededRecords = this.jobStatus.getRecordStatuses(IndexingStatus.SUCCESS, operationType);
- if (!succeededRecords.isEmpty()) {
- successEvent.accept(succeededRecords.stream().map(RecordStatus::succeededAuditLogMessage).collect(Collectors.toList()));
- }
- List<RecordStatus> skippedRecords = this.jobStatus.getRecordStatuses(IndexingStatus.SKIP, operationType);
- List<RecordStatus> failedRecords = this.jobStatus.getRecordStatuses(IndexingStatus.FAIL, operationType);
- failedRecords.addAll(skippedRecords);
- if (!failedRecords.isEmpty()) {
- failedEvent.accept(failedRecords.stream().map(RecordStatus::failedAuditLogMessage).collect(Collectors.toList()));
- }
- }
-}
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.service;
+
+import com.google.common.base.Strings;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import org.apache.http.HttpStatus;
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.bulk.BulkItemResponse;
+import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.rest.RestStatus;
+import org.opengroup.osdu.core.common.Constants;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.entitlements.Acl;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.http.RequestStatus;
+import org.opengroup.osdu.core.common.model.indexer.*;
+import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
+import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
+import org.opengroup.osdu.indexer.logging.AuditLogger;
+import org.opengroup.osdu.indexer.provider.interfaces.IPublisher;
+import org.opengroup.osdu.indexer.util.ElasticClientHandler;
+import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
+import org.springframework.context.annotation.Primary;
+import org.springframework.stereotype.Service;
+
+import javax.inject.Inject;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+@Service
+@Primary
+public class IndexerServiceImpl implements IndexerService {
+
+ private static final TimeValue BULK_REQUEST_TIMEOUT = TimeValue.timeValueMinutes(1);
+
+ private static final List<RestStatus> RETRY_ELASTIC_EXCEPTION = new ArrayList<>(Arrays.asList(RestStatus.TOO_MANY_REQUESTS, RestStatus.BAD_GATEWAY, RestStatus.SERVICE_UNAVAILABLE));
+
+ private final Gson gson = new GsonBuilder().serializeNulls().create();
+
+ @Inject
+ private JaxRsDpsLog jaxRsDpsLog;
+ @Inject
+ private AuditLogger auditLogger;
+ @Inject
+ private StorageService storageService;
+ @Inject
+ private IndexSchemaService schemaService;
+ @Inject
+ private IndicesService indicesService;
+ @Inject
+ private IMappingService mappingService;
+ @Inject
+ private IPublisher progressPublisher;
+ @Inject
+ private ElasticClientHandler elasticClientHandler;
+ @Inject
+ private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
+ @Inject
+ private ElasticIndexNameResolver elasticIndexNameResolver;
+ @Inject
+ private StorageIndexerPayloadMapper storageIndexerPayloadMapper;
+ @Inject
+ private IRequestInfo requestInfo;
+ @Inject
+ private JobStatus jobStatus;
+
+ private DpsHeaders headers;
+
+ @Override
+ public JobStatus processRecordChangedMessages(RecordChangedMessages message, List<RecordInfo> recordInfos) throws Exception {
+
+ // this should not happen
+ if (recordInfos.size() == 0) return null;
+
+ String errorMessage = "";
+ List<String> retryRecordIds = new LinkedList<>();
+
+ // get auth header with service account Authorization
+ this.headers = this.requestInfo.getHeadersWithDwdAuthZ();
+
+ // initialize status for all messages.
+ this.jobStatus.initialize(recordInfos);
+
+ try {
+ auditLogger.indexStarted(recordInfos.stream()
+ .map(RecordInfo::getKind)
+ .collect(Collectors.toList()));
+
+ // get upsert records
+ Map<String, Map<String, OperationType>> upsertRecordMap = RecordInfo.getUpsertRecordIds(recordInfos);
+ if (upsertRecordMap != null && !upsertRecordMap.isEmpty()) {
+ List<String> upsertFailureRecordIds = processUpsertRecords(upsertRecordMap);
+ retryRecordIds.addAll(upsertFailureRecordIds);
+ }
+
+ // get delete records
+ Map<String, List<String>> deleteRecordMap = RecordInfo.getDeleteRecordIds(recordInfos);
+ if (deleteRecordMap != null && !deleteRecordMap.isEmpty()) {
+ List<String> deleteFailureRecordIds = processDeleteRecords(deleteRecordMap);
+ retryRecordIds.addAll(deleteFailureRecordIds);
+ }
+
+ // process legacy storage schema change messages
+ Map<String, OperationType> schemaMsgs = RecordInfo.getSchemaMsgs(recordInfos);
+ if (schemaMsgs != null && !schemaMsgs.isEmpty()) {
+ this.schemaService.processSchemaMessages(schemaMsgs);
+ }
+
+ // process failed records
+ if (retryRecordIds.size() > 0) {
+ retryAndEnqueueFailedRecords(recordInfos, retryRecordIds, message);
+ }
+ } catch (IOException e) {
+ errorMessage = e.getMessage();
+ throw new AppException(HttpStatus.SC_GATEWAY_TIMEOUT, "Internal communication failure", errorMessage, e);
+ } catch (AppException e) {
+ errorMessage = e.getMessage();
+ throw e;
+ } catch (Exception e) {
+ errorMessage = "error indexing records";
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Unknown error", "An unknown error has occurred.", e);
+ } finally {
+ this.jobStatus.finalizeRecordStatus(errorMessage);
+ this.updateAuditLog();
+ this.progressPublisher.publishStatusChangedTagsToTopic(this.headers, this.jobStatus);
+ }
+
+ return jobStatus;
+ }
+
+ @Override
+ public void processSchemaMessages(List<RecordInfo> recordInfos) throws IOException {
+ Map<String, OperationType> schemaMsgs = RecordInfo.getSchemaMsgs(recordInfos);
+ if (schemaMsgs != null && !schemaMsgs.isEmpty()) {
+ try (RestHighLevelClient restClient = elasticClientHandler.createRestClient()) {
+ schemaMsgs.entrySet().forEach(msg -> {
+ try {
+ processSchemaEvents(restClient, msg);
+ } catch (IOException | ElasticsearchStatusException e) {
+ throw new AppException(org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR.value(), "unable to process schema delete", e.getMessage());
+ }
+ });
+ }
+ }
+ }
+
+ private void processSchemaEvents(RestHighLevelClient restClient,
+ Map.Entry<String, OperationType> msg) throws IOException, ElasticsearchStatusException {
+ String kind = msg.getKey();
+ String index = elasticIndexNameResolver.getIndexNameFromKind(kind);
+
+ boolean indexExist = indicesService.isIndexExist(restClient, index);
+ if (indexExist && msg.getValue() == OperationType.purge_schema) {
+ indicesService.deleteIndex(restClient, index);
+ }
+ }
+
+ private List<String> processUpsertRecords(Map<String, Map<String, OperationType>> upsertRecordMap) throws Exception {
+ // get schema for kind
+ Map<String, IndexSchema> schemas = this.getSchema(upsertRecordMap);
+
+ if (schemas.isEmpty()) return new LinkedList<>();
+
+ // get recordIds with valid upsert index-status
+ List<String> recordIds = this.jobStatus.getIdsByValidUpsertIndexingStatus();
+
+ if (recordIds.isEmpty()) return new LinkedList<>();
+
+ // get records via storage api
+ Records storageRecords = this.storageService.getStorageRecords(recordIds);
+ List<String> failedOrRetryRecordIds = new LinkedList<>(storageRecords.getMissingRetryRecords());
+
+ // map storage records to indexer payload
+ RecordIndexerPayload recordIndexerPayload = this.getIndexerPayload(upsertRecordMap, schemas, storageRecords);
+
+ jaxRsDpsLog.info(String.format("records change messages received : %s | valid storage bulk records: %s | valid index payload: %s", recordIds.size(), storageRecords.getRecords().size(), recordIndexerPayload.getRecords().size()));
+
+ // index records
+ failedOrRetryRecordIds.addAll(processElasticMappingAndUpsertRecords(recordIndexerPayload));
+
+ return failedOrRetryRecordIds;
+ }
+
+ private Map<String, IndexSchema> getSchema(Map<String, Map<String, OperationType>> upsertRecordMap) {
+
+ Map<String, IndexSchema> schemas = new HashMap<>();
+
+ try {
+ for (Map.Entry<String, Map<String, OperationType>> entry : upsertRecordMap.entrySet()) {
+
+ String kind = entry.getKey();
+ List<String> errors = new ArrayList<>();
+ IndexSchema schemaObj = this.schemaService.getIndexerInputSchema(kind, errors);
+ if (!errors.isEmpty()) {
+ this.jobStatus.addOrUpdateRecordStatus(entry.getValue().keySet(), IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, String.join("|", errors), String.format("error | kind: %s", kind));
+ } else if (schemaObj.isDataSchemaMissing()) {
+ this.jobStatus.addOrUpdateRecordStatus(entry.getValue().keySet(), IndexingStatus.WARN, HttpStatus.SC_NOT_FOUND, "schema not found", String.format("schema not found | kind: %s", kind));
+ }
+
+ schemas.put(kind, schemaObj);
+ }
+ } catch (AppException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Get schema error", "An error has occurred while getting schema", e);
+ }
+
+ return schemas;
+ }
+
+ private RecordIndexerPayload getIndexerPayload(Map<String, Map<String, OperationType>> upsertRecordMap, Map<String, IndexSchema> kindSchemaMap, Records records) {
+ List<Records.Entity> storageValidRecords = records.getRecords();
+ List<RecordIndexerPayload.Record> indexerPayload = new ArrayList<>();
+ List<IndexSchema> schemas = new ArrayList<>();
+
+ for (Records.Entity storageRecord : storageValidRecords) {
+
+ Map<String, OperationType> idOperationMap = upsertRecordMap.get(storageRecord.getKind());
+
+ // skip if storage returned record with same id but different kind
+ if (idOperationMap == null) {
+ String message = String.format("storage service returned incorrect record | requested kind: %s | received kind: %s", this.jobStatus.getRecordKindById(storageRecord.getId()), storageRecord.getKind());
+ this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.SKIP, RequestStatus.STORAGE_CONFLICT, message, String.format("%s | record-id: %s", message, storageRecord.getId()));
+ continue;
+ }
+
+ IndexSchema schema = kindSchemaMap.get(storageRecord.getKind());
+ schemas.add(schema);
+
+ // skip indexing of records if data block is empty
+ RecordIndexerPayload.Record document = prepareIndexerPayload(schema, storageRecord, idOperationMap);
+ if (document != null) {
+ indexerPayload.add(document);
+ }
+ }
+
+ // this should only happen if storage service returned WRONG records with kind for all the records in the messages
+ if (indexerPayload.isEmpty()) {
+ throw new AppException(RequestStatus.STORAGE_CONFLICT, "Indexer error", "upsert record failed, storage service returned incorrect records");
+ }
+
+ return RecordIndexerPayload.builder().records(indexerPayload).schemas(schemas).build();
+ }
+
+ private RecordIndexerPayload.Record prepareIndexerPayload(IndexSchema schemaObj, Records.Entity storageRecord, Map<String, OperationType> idToOperationMap) {
+
+ RecordIndexerPayload.Record document = null;
+
+ try {
+ Map<String, Object> storageRecordData = storageRecord.getData();
+ document = new RecordIndexerPayload.Record();
+ if (storageRecordData == null || storageRecordData.isEmpty()) {
+ String message = "empty or null data block found in the storage record";
+ this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.WARN, HttpStatus.SC_NOT_FOUND, message, String.format("record-id: %s | %s", storageRecord.getId(), message));
+ } else if (schemaObj.isDataSchemaMissing()) {
+ document.setSchemaMissing(true);
+ } else {
+ Map<String, Object> dataMap = this.storageIndexerPayloadMapper.mapDataPayload(schemaObj, storageRecordData, storageRecord.getId());
+ if (dataMap.isEmpty()) {
+ document.setMappingMismatch(true);
+ String message = String.format("complete schema mismatch: none of the data attribute can be mapped | data: %s", storageRecordData);
+ this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.WARN, HttpStatus.SC_NOT_FOUND, message, String.format("record-id: %s | %s", storageRecord.getId(), message));
+ }
+ document.setData(dataMap);
+ }
+ } catch (AppException e) {
+ this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL, HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getMessage());
+ jaxRsDpsLog.warning(String.format("record-id: %s | %s", storageRecord.getId(), e.getMessage()), e);
+ } catch (Exception e) {
+ this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL, HttpStatus.SC_INTERNAL_SERVER_ERROR, String.format("error parsing records against schema, error-message: %s", e.getMessage()));
+ jaxRsDpsLog.error(String.format("record-id: %s | error parsing records against schema, error-message: %s", storageRecord.getId(), e.getMessage()), e);
+ }
+
+ try {
+ // index individual parts of kind
+ String[] kindParts = storageRecord.getKind().split(":");
+ String authority = kindParts[0];
+ String source = kindParts[1];
+ String type = kindParts[2];
+ document.setKind(storageRecord.getKind());
+ document.setNamespace(authority + ":" + source);
+ document.setAuthority(authority);
+ document.setSource(source);
+ document.setType(type);
+ document.setId(storageRecord.getId());
+ document.setVersion(storageRecord.getVersion());
+ document.setAcl(storageRecord.getAcl());
+ document.setLegal(storageRecord.getLegal());
+ if (storageRecord.getTags() != null) {
+ document.setTags(storageRecord.getTags());
+ }
+ document.setCreateUser(storageRecord.getCreateUser());
+ document.setCreateTime(storageRecord.getCreateTime());
+ if (!Strings.isNullOrEmpty(storageRecord.getModifyUser())) {
+ document.setModifyUser(storageRecord.getModifyUser());
+ }
+ if (!Strings.isNullOrEmpty(storageRecord.getModifyTime())) {
+ document.setModifyTime(storageRecord.getModifyTime());
+ }
+ RecordStatus recordStatus = this.jobStatus.getJobStatusByRecordId(storageRecord.getId());
+ if (recordStatus.getIndexProgress().getStatusCode() == 0) {
+ recordStatus.getIndexProgress().setStatusCode(HttpStatus.SC_OK);
+ }
+ document.setIndexProgress(recordStatus.getIndexProgress());
+ if (storageRecord.getAncestry() != null) document.setAncestry(storageRecord.getAncestry());
+ document.setOperationType(idToOperationMap.get(storageRecord.getId()));
+ } catch (Exception e) {
+ this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL, HttpStatus.SC_INTERNAL_SERVER_ERROR, String.format("error parsing meta data, error-message: %s", e.getMessage()));
+ jaxRsDpsLog.error(String.format("record-id: %s | error parsing meta data, error-message: %s", storageRecord.getId(), e.getMessage()), e);
+ }
+ return document;
+ }
+
+ private List<String> processElasticMappingAndUpsertRecords(RecordIndexerPayload recordIndexerPayload) throws Exception {
+
+ try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
+ List<IndexSchema> schemas = recordIndexerPayload.getSchemas();
+ if (schemas == null || schemas.isEmpty()) {
+ return new LinkedList<>();
+ }
+
+ // process the schema
+ this.cacheOrCreateElasticMapping(schemas, restClient);
+
+ // process the records
+ return this.upsertRecords(recordIndexerPayload.getRecords(), restClient);
+ }
+ }
+
+ private void cacheOrCreateElasticMapping(List<IndexSchema> schemas, RestHighLevelClient restClient) throws Exception {
+
+ for (IndexSchema schema : schemas) {
+ String index = this.elasticIndexNameResolver.getIndexNameFromKind(schema.getKind());
+
+ // check if index exist and sync meta attribute schema if required
+ if (this.indicesService.isIndexReady(restClient, index)) {
+ this.mappingService.syncIndexMappingIfRequired(restClient, schema);
+ continue;
+ }
+
+ // create index
+ Map<String, Object> mapping = this.mappingService.getIndexMappingFromRecordSchema(schema);
+ if (!this.indicesService.createIndex(restClient, index, null, schema.getType(), mapping)) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error", "Error creating index.", String.format("Failed to get confirmation from elastic server for index: %s", index));
+ }
+ }
+ }
+
+ private List<String> upsertRecords(List<RecordIndexerPayload.Record> records, RestHighLevelClient restClient) throws AppException {
+ if (records == null || records.isEmpty()) return new LinkedList<>();
+
+ BulkRequest bulkRequest = new BulkRequest();
+ bulkRequest.timeout(BULK_REQUEST_TIMEOUT);
+
+ for (RecordIndexerPayload.Record record : records) {
+ if ((record.getData() == null || record.getData().isEmpty()) && !record.skippedDataIndexing()) {
+ // it will come here when schema is missing
+ // TODO: rollback once we know what is causing the problem
+ jaxRsDpsLog.warning(String.format("data not found for record: %s", record));
+ }
+
+ OperationType operation = record.getOperationType();
+ Map<String, Object> sourceMap = getSourceMap(record);
+ String index = this.elasticIndexNameResolver.getIndexNameFromKind(record.getKind());
+
+ if (operation == OperationType.create) {
+ IndexRequest indexRequest = new IndexRequest(index).id(record.getId()).source(this.gson.toJson(sourceMap), XContentType.JSON);
+ bulkRequest.add(indexRequest);
+ } else if (operation == OperationType.update) {
+ UpdateRequest updateRequest = new UpdateRequest(index, "_doc", record.getId()).doc(this.gson.toJson(sourceMap), XContentType.JSON).docAsUpsert(true);
+ bulkRequest.add(updateRequest);
+ }
+ }
+
+ return processBulkRequest(restClient, bulkRequest);
+ }
+
+ private List<String> processDeleteRecords(Map<String, List<String>> deleteRecordMap) throws Exception {
+ BulkRequest bulkRequest = new BulkRequest();
+ bulkRequest.timeout(BULK_REQUEST_TIMEOUT);
+
+ for (Map.Entry<String, List<String>> record : deleteRecordMap.entrySet()) {
+
+ String index = this.elasticIndexNameResolver.getIndexNameFromKind(record.getKey());
+
+ for (String id : record.getValue()) {
+ DeleteRequest deleteRequest = new DeleteRequest(index, id);
+ bulkRequest.add(deleteRequest);
+ }
+ }
+
+ try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
+ return processBulkRequest(restClient, bulkRequest);
+ }
+ }
+
+ private List<String> processBulkRequest(RestHighLevelClient restClient, BulkRequest bulkRequest) throws AppException {
+
+ List<String> failureRecordIds = new LinkedList<>();
+ if (bulkRequest.numberOfActions() == 0) return failureRecordIds;
+ int failedRequestStatus = 500;
+ Exception failedRequestCause = null;
+
+ try {
+ long startTime = System.currentTimeMillis();
+ BulkResponse bulkResponse = restClient.bulk(bulkRequest, RequestOptions.DEFAULT);
+ long stopTime = System.currentTimeMillis();
+
+ // log failed bulk requests
+ ArrayList<String> bulkFailures = new ArrayList<>();
+ int succeededResponses = 0;
+ int failedResponses = 0;
+
+ for (BulkItemResponse bulkItemResponse : bulkResponse.getItems()) {
+ if (bulkItemResponse.isFailed()) {
+ BulkItemResponse.Failure failure = bulkItemResponse.getFailure();
+ bulkFailures.add(String.format("elasticsearch bulk service status: %s | id: %s | message: %s", failure.getStatus(), failure.getId(), failure.getMessage()));
+ this.jobStatus.addOrUpdateRecordStatus(bulkItemResponse.getId(), IndexingStatus.FAIL, failure.getStatus().getStatus(), bulkItemResponse.getFailureMessage());
+ if (canIndexerRetry(bulkItemResponse)) {
+ failureRecordIds.add(bulkItemResponse.getId());
+
+ if (failedRequestCause == null) {
+ failedRequestCause = failure.getCause();
+ failedRequestStatus = failure.getStatus().getStatus();
+ }
+ }
+
+ failedResponses++;
+ } else {
+ succeededResponses++;
+ this.jobStatus.addOrUpdateRecordStatus(bulkItemResponse.getId(), IndexingStatus.SUCCESS, HttpStatus.SC_OK, "Indexed Successfully");
+ }
+ }
+ if (!bulkFailures.isEmpty()) this.jaxRsDpsLog.warning(bulkFailures);
+
+ jaxRsDpsLog.info(String.format("records in elasticsearch service bulk request: %s | successful: %s | failed: %s | time taken for bulk request: %d milliseconds", bulkRequest.numberOfActions(), succeededResponses, failedResponses, stopTime-startTime));
+
+ // retry entire message if all records are failing
+ if (bulkRequest.numberOfActions() == failureRecordIds.size()) throw new AppException(failedRequestStatus, "Elastic error", failedRequestCause.getMessage(), failedRequestCause);
+ } catch (IOException e) {
+ // throw explicit 504 for IOException
+ throw new AppException(HttpStatus.SC_GATEWAY_TIMEOUT, "Elastic error", "Request cannot be completed in specified time.", e);
+ } catch (ElasticsearchStatusException e) {
+ throw new AppException(e.status().getStatus(), "Elastic error", e.getMessage(), e);
+ } catch (Exception e) {
+ if (e instanceof AppException) {
+ throw e;
+ }
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error", "Error indexing records.", e);
+ }
+ return failureRecordIds;
+ }
+
+ private Map<String, Object> getSourceMap(RecordIndexerPayload.Record record) {
+
+ Map<String, Object> indexerPayload = new HashMap<>();
+
+ // get the key and get the corresponding object from the individualRecord object
+ if (record.getData() != null) {
+ Map<String, Object> data = new HashMap<>();
+ for (Map.Entry<String, Object> entry : record.getData().entrySet()) {
+ data.put(entry.getKey(), entry.getValue());
+ }
+ indexerPayload.put(Constants.DATA, data);
+ }
+
+ indexerPayload.put(RecordMetaAttribute.ID.getValue(), record.getId());
+ indexerPayload.put(RecordMetaAttribute.KIND.getValue(), record.getKind());
+ indexerPayload.put(RecordMetaAttribute.AUTHORITY.getValue(), record.getAuthority());
+ indexerPayload.put(RecordMetaAttribute.SOURCE.getValue(), record.getSource());
+ indexerPayload.put(RecordMetaAttribute.NAMESPACE.getValue(), record.getNamespace());
+ indexerPayload.put(RecordMetaAttribute.TYPE.getValue(), record.getType());
+ indexerPayload.put(RecordMetaAttribute.VERSION.getValue(), record.getVersion());
+ indexerPayload.put(RecordMetaAttribute.ACL.getValue(), record.getAcl());
+ indexerPayload.put(RecordMetaAttribute.TAGS.getValue(), record.getTags());
+ indexerPayload.put(RecordMetaAttribute.X_ACL.getValue(), Acl.flattenAcl(record.getAcl()));
+ indexerPayload.put(RecordMetaAttribute.LEGAL.getValue(), record.getLegal());
+ indexerPayload.put(RecordMetaAttribute.INDEX_STATUS.getValue(), record.getIndexProgress());
+ if (record.getAncestry() != null) {
+ indexerPayload.put(RecordMetaAttribute.ANCESTRY.getValue(), record.getAncestry());
+ }
+ indexerPayload.put(RecordMetaAttribute.CREATE_USER.getValue(), record.getCreateUser());
+ indexerPayload.put(RecordMetaAttribute.CREATE_TIME.getValue(), record.getCreateTime());
+ if (!Strings.isNullOrEmpty(record.getModifyUser())) {
+ indexerPayload.put(RecordMetaAttribute.MODIFY_USER.getValue(), record.getModifyUser());
+ }
+ if (!Strings.isNullOrEmpty(record.getModifyTime())) {
+ indexerPayload.put(RecordMetaAttribute.MODIFY_TIME.getValue(), record.getModifyTime());
+ }
+ return indexerPayload;
+ }
+
+ private boolean canIndexerRetry(BulkItemResponse bulkItemResponse) {
+ if (RETRY_ELASTIC_EXCEPTION.contains(bulkItemResponse.status())) return true;
+
+ if ((bulkItemResponse.getOpType() == DocWriteRequest.OpType.CREATE || bulkItemResponse.getOpType() == DocWriteRequest.OpType.UPDATE)
+ && bulkItemResponse.status() == RestStatus.NOT_FOUND) {
+ return true;
+ }
+
+ return false;
+ }
+
+ private void retryAndEnqueueFailedRecords(List<RecordInfo> recordInfos, List<String> failuresRecordIds, RecordChangedMessages message) throws IOException {
+
+ jaxRsDpsLog.info(String.format("queuing bulk failed records back to task-queue for retry | count: %s | records: %s", failuresRecordIds.size(), failuresRecordIds));
+ List<RecordInfo> retryRecordInfos = new LinkedList<>();
+ for (String recordId : failuresRecordIds) {
+ for (RecordInfo origMessage : recordInfos) {
+ if (origMessage.getId().equalsIgnoreCase(recordId)) {
+ retryRecordInfos.add(origMessage);
+ }
+ }
+ }
+
+ RecordChangedMessages newMessage = RecordChangedMessages.builder()
+ .messageId(message.getMessageId())
+ .publishTime(message.getPublishTime())
+ .data(this.gson.toJson(retryRecordInfos))
+ .attributes(message.getAttributes()).build();
+
+ String payLoad = this.gson.toJson(newMessage);
+ this.indexerQueueTaskBuilder.createWorkerTask(payLoad, this.headers);
+ }
+
+ private void updateAuditLog() {
+ logAuditEvents(OperationType.create, this.auditLogger::indexCreateRecordSuccess, this.auditLogger::indexCreateRecordFail);
+ logAuditEvents(OperationType.update, this.auditLogger::indexUpdateRecordSuccess, this.auditLogger::indexUpdateRecordFail);
+ logAuditEvents(OperationType.purge, this.auditLogger::indexPurgeRecordSuccess, this.auditLogger::indexPurgeRecordFail);
+ logAuditEvents(OperationType.delete, this.auditLogger::indexDeleteRecordSuccess, this.auditLogger::indexDeleteRecordFail);
+ }
+
+ private void logAuditEvents(OperationType operationType, Consumer<List<String>> successEvent, Consumer<List<String>> failedEvent) {
+ List<RecordStatus> succeededRecords = this.jobStatus.getRecordStatuses(IndexingStatus.SUCCESS, operationType);
+ if (!succeededRecords.isEmpty()) {
+ successEvent.accept(succeededRecords.stream().map(RecordStatus::succeededAuditLogMessage).collect(Collectors.toList()));
+ }
+ List<RecordStatus> skippedRecords = this.jobStatus.getRecordStatuses(IndexingStatus.SKIP, operationType);
+ List<RecordStatus> failedRecords = this.jobStatus.getRecordStatuses(IndexingStatus.FAIL, operationType);
+ failedRecords.addAll(skippedRecords);
+ if (!failedRecords.isEmpty()) {
+ failedEvent.accept(failedRecords.stream().map(RecordStatus::failedAuditLogMessage).collect(Collectors.toList()));
+ }
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java
index 55759face..ccedb93d6 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java
@@ -1,190 +1,190 @@
-package org.opengroup.osdu.indexer.service;
-
-import static org.opengroup.osdu.indexer.service.IAttributeParsingService.DATA_GEOJSON_TAG;
-import static org.opengroup.osdu.indexer.service.IAttributeParsingService.RECORD_GEOJSON_TAG;
-
-import java.lang.reflect.InvocationTargetException;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Objects;
-import javax.inject.Inject;
-import org.apache.commons.beanutils.NestedNullException;
-import org.apache.commons.beanutils.PropertyUtils;
-import org.apache.http.HttpStatus;
-import org.opengroup.osdu.core.common.Constants;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.indexer.ElasticType;
-import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
-import org.opengroup.osdu.core.common.model.indexer.IndexingStatus;
-import org.opengroup.osdu.core.common.model.indexer.JobStatus;
-import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterConfig;
-import org.springframework.stereotype.Component;
-
-@Component
-public class StorageIndexerPayloadMapper {
-
- @Inject
- private JaxRsDpsLog log;
- @Inject
- private IAttributeParsingService attributeParsingService;
- @Inject
- private JobStatus jobStatus;
- @Inject
- private SchemaConverterConfig schemaConfig;
-
- public Map<String, Object> mapDataPayload(IndexSchema storageSchema, Map<String, Object> storageRecordData,
- String recordId) {
-
- Map<String, Object> dataCollectorMap = new HashMap<>();
-
- if (storageSchema.isDataSchemaMissing()) {
- this.log.warning(String.format("record-id: %s | schema mismatching: %s ", recordId, storageSchema.getKind()));
- return dataCollectorMap;
- }
-
- mapDataPayload(storageSchema.getDataSchema(), storageRecordData, recordId, dataCollectorMap);
-
- // add these once iterated over the list
- storageSchema.getDataSchema().put(DATA_GEOJSON_TAG, ElasticType.GEO_SHAPE.getValue());
- storageSchema.getDataSchema().remove(RECORD_GEOJSON_TAG);
-
- return dataCollectorMap;
- }
-
- private Map<String, Object> mapDataPayload(Map<String, Object> dataSchema, Map<String, Object> storageRecordData,
- String recordId, Map<String, Object> dataCollectorMap) {
-
- // get the key and get the corresponding object from the storageRecord object
- for (Map.Entry<String, Object> entry : dataSchema.entrySet()) {
- String schemaPropertyName = entry.getKey();
- Object storageRecordValue = getPropertyValue(recordId, storageRecordData, schemaPropertyName);
- ElasticType elasticType = defineElasticType(entry.getValue());
-
- if (Objects.isNull(elasticType)) {
- this.jobStatus
- .addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST,
- String.format("record-id: %s | %s for entry %s", recordId, "Not resolvable elastic type", schemaPropertyName));
- continue;
- }
-
- if (schemaConfig.getProcessedArraysTypes().contains(elasticType.getValue().toLowerCase()) && Objects.nonNull(storageRecordValue)) {
- processInnerProperties(recordId, dataCollectorMap, entry.getValue(), schemaPropertyName, (List<Map>) storageRecordValue);
- }
-
- if (storageRecordValue == null && !nullIndexedValueSupported(elasticType)) {
- continue;
- }
-
- switch (elasticType) {
- case KEYWORD:
- case KEYWORD_ARRAY:
- case TEXT:
- case TEXT_ARRAY:
- dataCollectorMap.put(schemaPropertyName, storageRecordValue);
- break;
- case INTEGER_ARRAY:
- this.attributeParsingService.tryParseValueArray(Integer.class, recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
- break;
- case INTEGER:
- this.attributeParsingService.tryParseInteger(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
- break;
- case LONG_ARRAY:
- this.attributeParsingService.tryParseValueArray(Long.class, recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
- break;
- case LONG:
- this.attributeParsingService.tryParseLong(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
- break;
- case FLOAT_ARRAY:
- this.attributeParsingService.tryParseValueArray(Float.class, recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
- break;
- case FLOAT:
- this.attributeParsingService.tryParseFloat(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
- break;
- case DOUBLE_ARRAY:
- this.attributeParsingService.tryParseValueArray(Double.class, recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
- break;
- case DOUBLE:
- this.attributeParsingService.tryParseDouble(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
- break;
- case BOOLEAN_ARRAY:
- this.attributeParsingService.tryParseValueArray(Boolean.class, recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
- break;
- case BOOLEAN:
- this.attributeParsingService.tryParseBoolean(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
- break;
- case DATE_ARRAY:
- this.attributeParsingService.tryParseValueArray(Date.class, recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
- break;
- case DATE:
- this.attributeParsingService.tryParseDate(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
- break;
- case GEO_POINT:
- this.attributeParsingService.tryParseGeopoint(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
- break;
- case GEO_SHAPE:
- this.attributeParsingService.tryParseGeojson(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
- break;
- case FLATTENED:
- // flattened type inner properties will be added "as is" without parsing as they types not present in schema
- this.attributeParsingService.tryParseFlattened(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
- break;
- case OBJECT:
- // object type inner properties will be added "as is" without parsing as they types not present in schema
- this.attributeParsingService.tryParseObject(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
- break;
- case UNDEFINED:
- // don't do anything for now
- break;
- }
- }
-
- return dataCollectorMap;
- }
-
- private void processInnerProperties(String recordId, Map<String, Object> dataCollectorMap, Object schemaPropertyWithInnerProperties,
- String name, List<Map> storageRecordValue) {
- Map schemaPropertyMap = (Map) schemaPropertyWithInnerProperties;
- Map innerProperties = (Map) schemaPropertyMap.get(Constants.PROPERTIES);
- ArrayList<Map> innerPropertiesMappingCollector = new ArrayList<>();
- storageRecordValue.forEach(recordData -> innerPropertiesMappingCollector.add(mapDataPayload(innerProperties, recordData, recordId, new HashMap<>())));
- dataCollectorMap.put(name, innerPropertiesMappingCollector);
- }
-
- private ElasticType defineElasticType(Object entryValue) {
- ElasticType elasticType = null;
- if (entryValue instanceof String) {
- elasticType = ElasticType.forValue(entryValue.toString());
- } else if (entryValue instanceof Map) {
- Map map = (Map) entryValue;
- elasticType = ElasticType.forValue(map.get(Constants.TYPE).toString());
- }
- return elasticType;
- }
-
- private Object getPropertyValue(String recordId, Map<String, Object> storageRecordData, String propertyKey) {
-
- try {
- // try getting first level property using optimized collection
- Object propertyVal = storageRecordData.get(propertyKey);
- if (propertyVal != null) return propertyVal;
-
- // use apache utils to get nested property
- return PropertyUtils.getProperty(storageRecordData, propertyKey);
- } catch (NestedNullException ignored) {
- // property not found in record
- } catch (NoSuchMethodException e) {
- this.log.warning(String.format("record-id: %s | error fetching property: %s | error: %s", recordId, propertyKey, e.getMessage()));
- } catch (IllegalArgumentException | IllegalAccessException | InvocationTargetException e) {
- this.log.warning(String.format("record-id: %s | error fetching property: %s | error: %s", recordId, propertyKey, e.getMessage()), e);
- }
- return null;
- }
-
- private boolean nullIndexedValueSupported(ElasticType type) {
- return type == ElasticType.TEXT;
- }
+package org.opengroup.osdu.indexer.service;
+
+import static org.opengroup.osdu.indexer.service.IAttributeParsingService.DATA_GEOJSON_TAG;
+import static org.opengroup.osdu.indexer.service.IAttributeParsingService.RECORD_GEOJSON_TAG;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import javax.inject.Inject;
+import org.apache.commons.beanutils.NestedNullException;
+import org.apache.commons.beanutils.PropertyUtils;
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.core.common.Constants;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.indexer.ElasticType;
+import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
+import org.opengroup.osdu.core.common.model.indexer.IndexingStatus;
+import org.opengroup.osdu.core.common.model.indexer.JobStatus;
+import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterConfig;
+import org.springframework.stereotype.Component;
+
+@Component
+public class StorageIndexerPayloadMapper {
+
+ @Inject
+ private JaxRsDpsLog log;
+ @Inject
+ private IAttributeParsingService attributeParsingService;
+ @Inject
+ private JobStatus jobStatus;
+ @Inject
+ private SchemaConverterConfig schemaConfig;
+
+ public Map<String, Object> mapDataPayload(IndexSchema storageSchema, Map<String, Object> storageRecordData,
+ String recordId) {
+
+ Map<String, Object> dataCollectorMap = new HashMap<>();
+
+ if (storageSchema.isDataSchemaMissing()) {
+ this.log.warning(String.format("record-id: %s | schema mismatching: %s ", recordId, storageSchema.getKind()));
+ return dataCollectorMap;
+ }
+
+ mapDataPayload(storageSchema.getDataSchema(), storageRecordData, recordId, dataCollectorMap);
+
+ // add these once iterated over the list
+ storageSchema.getDataSchema().put(DATA_GEOJSON_TAG, ElasticType.GEO_SHAPE.getValue());
+ storageSchema.getDataSchema().remove(RECORD_GEOJSON_TAG);
+
+ return dataCollectorMap;
+ }
+
+ private Map<String, Object> mapDataPayload(Map<String, Object> dataSchema, Map<String, Object> storageRecordData,
+ String recordId, Map<String, Object> dataCollectorMap) {
+
+ // get the key and get the corresponding object from the storageRecord object
+ for (Map.Entry<String, Object> entry : dataSchema.entrySet()) {
+ String schemaPropertyName = entry.getKey();
+ Object storageRecordValue = getPropertyValue(recordId, storageRecordData, schemaPropertyName);
+ ElasticType elasticType = defineElasticType(entry.getValue());
+
+ if (Objects.isNull(elasticType)) {
+ this.jobStatus
+ .addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST,
+ String.format("record-id: %s | %s for entry %s", recordId, "Not resolvable elastic type", schemaPropertyName));
+ continue;
+ }
+
+ if (schemaConfig.getProcessedArraysTypes().contains(elasticType.getValue().toLowerCase()) && Objects.nonNull(storageRecordValue)) {
+ processInnerProperties(recordId, dataCollectorMap, entry.getValue(), schemaPropertyName, (List<Map>) storageRecordValue);
+ }
+
+ if (storageRecordValue == null && !nullIndexedValueSupported(elasticType)) {
+ continue;
+ }
+
+ switch (elasticType) {
+ case KEYWORD:
+ case KEYWORD_ARRAY:
+ case TEXT:
+ case TEXT_ARRAY:
+ dataCollectorMap.put(schemaPropertyName, storageRecordValue);
+ break;
+ case INTEGER_ARRAY:
+ this.attributeParsingService.tryParseValueArray(Integer.class, recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
+ break;
+ case INTEGER:
+ this.attributeParsingService.tryParseInteger(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
+ break;
+ case LONG_ARRAY:
+ this.attributeParsingService.tryParseValueArray(Long.class, recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
+ break;
+ case LONG:
+ this.attributeParsingService.tryParseLong(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
+ break;
+ case FLOAT_ARRAY:
+ this.attributeParsingService.tryParseValueArray(Float.class, recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
+ break;
+ case FLOAT:
+ this.attributeParsingService.tryParseFloat(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
+ break;
+ case DOUBLE_ARRAY:
+ this.attributeParsingService.tryParseValueArray(Double.class, recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
+ break;
+ case DOUBLE:
+ this.attributeParsingService.tryParseDouble(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
+ break;
+ case BOOLEAN_ARRAY:
+ this.attributeParsingService.tryParseValueArray(Boolean.class, recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
+ break;
+ case BOOLEAN:
+ this.attributeParsingService.tryParseBoolean(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
+ break;
+ case DATE_ARRAY:
+ this.attributeParsingService.tryParseValueArray(Date.class, recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
+ break;
+ case DATE:
+ this.attributeParsingService.tryParseDate(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
+ break;
+ case GEO_POINT:
+ this.attributeParsingService.tryParseGeopoint(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
+ break;
+ case GEO_SHAPE:
+ this.attributeParsingService.tryParseGeojson(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
+ break;
+ case FLATTENED:
+ // flattened type inner properties will be added "as is" without parsing as they types not present in schema
+ this.attributeParsingService.tryParseFlattened(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
+ break;
+ case OBJECT:
+ // object type inner properties will be added "as is" without parsing as they types not present in schema
+ this.attributeParsingService.tryParseObject(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
+ break;
+ case UNDEFINED:
+ // don't do anything for now
+ break;
+ }
+ }
+
+ return dataCollectorMap;
+ }
+
+ private void processInnerProperties(String recordId, Map<String, Object> dataCollectorMap, Object schemaPropertyWithInnerProperties,
+ String name, List<Map> storageRecordValue) {
+ Map schemaPropertyMap = (Map) schemaPropertyWithInnerProperties;
+ Map innerProperties = (Map) schemaPropertyMap.get(Constants.PROPERTIES);
+ ArrayList<Map> innerPropertiesMappingCollector = new ArrayList<>();
+ storageRecordValue.forEach(recordData -> innerPropertiesMappingCollector.add(mapDataPayload(innerProperties, recordData, recordId, new HashMap<>())));
+ dataCollectorMap.put(name, innerPropertiesMappingCollector);
+ }
+
+ private ElasticType defineElasticType(Object entryValue) {
+ ElasticType elasticType = null;
+ if (entryValue instanceof String) {
+ elasticType = ElasticType.forValue(entryValue.toString());
+ } else if (entryValue instanceof Map) {
+ Map map = (Map) entryValue;
+ elasticType = ElasticType.forValue(map.get(Constants.TYPE).toString());
+ }
+ return elasticType;
+ }
+
+ private Object getPropertyValue(String recordId, Map<String, Object> storageRecordData, String propertyKey) {
+
+ try {
+ // try getting first level property using optimized collection
+ Object propertyVal = storageRecordData.get(propertyKey);
+ if (propertyVal != null) return propertyVal;
+
+ // use apache utils to get nested property
+ return PropertyUtils.getProperty(storageRecordData, propertyKey);
+ } catch (NestedNullException ignored) {
+ // property not found in record
+ } catch (NoSuchMethodException e) {
+ this.log.warning(String.format("record-id: %s | error fetching property: %s | error: %s", recordId, propertyKey, e.getMessage()));
+ } catch (IllegalArgumentException | IllegalAccessException | InvocationTargetException e) {
+ this.log.warning(String.format("record-id: %s | error fetching property: %s | error: %s", recordId, propertyKey, e.getMessage()), e);
+ }
+ return null;
+ }
+
+ private boolean nullIndexedValueSupported(ElasticType type) {
+ return type == ElasticType.TEXT;
+ }
}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java
index 50363f976..1dcb6a091 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java
@@ -1,234 +1,234 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.service;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.api.client.http.HttpMethods;
-import com.google.common.base.Strings;
-import com.google.common.collect.Lists;
-import com.google.gson.Gson;
-
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParser;
-import org.opengroup.osdu.core.common.http.FetchServiceHttpRequest;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.http.HttpResponse;
-import org.opengroup.osdu.core.common.model.http.RequestStatus;
-import org.opengroup.osdu.core.common.model.indexer.*;
-import org.opengroup.osdu.core.common.model.storage.ConversionStatus;
-import org.opengroup.osdu.core.common.model.storage.RecordIds;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.http.IUrlFetchService;
-import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
-import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
-import org.apache.http.HttpStatus;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.springframework.stereotype.Component;
-
-import javax.inject.Inject;
-import java.io.UnsupportedEncodingException;
-import java.net.URISyntaxException;
-import java.net.URLEncoder;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import static org.opengroup.osdu.core.common.model.http.DpsHeaders.FRAME_OF_REFERENCE;
-import static org.opengroup.osdu.core.common.Constants.SLB_FRAME_OF_REFERENCE_VALUE;
-
-@Component
-public class StorageServiceImpl implements StorageService {
-
- private final Gson gson = new Gson();
-
- @Inject
- private ObjectMapper objectMapper;
- @Inject
- private IUrlFetchService urlFetchService;
- @Inject
- private JobStatus jobStatus;
- @Inject
- private IRequestInfo requestInfo;
- @Inject
- private JaxRsDpsLog jaxRsDpsLog;
- @Inject
- private IndexerConfigurationProperties configurationProperties;
-
- @Override
- public Records getStorageRecords(List<String> ids) throws AppException, URISyntaxException {
- List<Records.Entity> valid = new ArrayList<>();
- List<String> notFound = new ArrayList<>();
- List<ConversionStatus> conversionStatuses = new ArrayList<>();
- List<String> missingRetryRecordIds = new ArrayList<>();
-
- List<List<String>> batch = Lists.partition(ids, configurationProperties.getStorageRecordsBatchSize());
- for (List<String> recordsBatch : batch) {
- Records storageOut = this.getRecords(recordsBatch);
- valid.addAll(storageOut.getRecords());
- notFound.addAll(storageOut.getNotFound());
- conversionStatuses.addAll(storageOut.getConversionStatuses());
- missingRetryRecordIds.addAll(storageOut.getMissingRetryRecords());
- }
- return Records.builder().records(valid).notFound(notFound).conversionStatuses(conversionStatuses).missingRetryRecords(missingRetryRecordIds).build();
- }
-
- protected Records getRecords(List<String> ids) throws URISyntaxException {
- // e.g. {"records":["test:10"]}
- String body = this.gson.toJson(RecordIds.builder().records(ids).build());
-
-// Map<String, String> headers = this.requestInfo.getHeadersMap();
- DpsHeaders headers = this.requestInfo.getHeaders();
- headers.put(FRAME_OF_REFERENCE, SLB_FRAME_OF_REFERENCE_VALUE);
- FetchServiceHttpRequest request = FetchServiceHttpRequest
- .builder()
- .httpMethod(HttpMethods.POST)
- .url(configurationProperties.getStorageQueryRecordForConversionHost())
- .headers(headers)
- .body(body).build();
- HttpResponse response = this.urlFetchService.sendRequest(request);
- return this.validateStorageResponse(response, ids);
- }
-
- private Records validateStorageResponse(HttpResponse response, List<String> ids) {
- String bulkStorageData = response.getBody();
-
- // retry entire payload -- storage service returned empty response
- if (Strings.isNullOrEmpty(bulkStorageData)) {
- throw new AppException(HttpStatus.SC_NOT_FOUND, "Invalid request", "Storage service returned empty response");
- }
-
- Records records = null;
- try {
- records = this.objectMapper.readValue(bulkStorageData, Records.class);
- } catch (JsonProcessingException e) {
- throw new AppException(RequestStatus.INVALID_RECORD, "Invalid request", "Successful Storage service response with wrong json", e);
- }
-
- // no retry possible, update record status as failed -- storage service cannot locate records
- if (!records.getNotFound().isEmpty()) {
- this.jobStatus.addOrUpdateRecordStatus(records.getNotFound(), IndexingStatus.FAIL, RequestStatus.INVALID_RECORD, "Storage service records not found", String.format("Storage service records not found: %s", String.join(",", records.getNotFound())));
- }
-
- List<Records.Entity> validRecords = records.getRecords();
- if (validRecords.isEmpty()) {
- // no need to retry, ack the CloudTask message -- nothing to process from RecordChangeMessage batch
- if (response.isSuccessCode()) {
- throw new AppException(RequestStatus.INVALID_RECORD, "Invalid request", "Successful Storage service response with no valid records");
- }
-
- // retry entire payload -- storage service returned empty valid records with non-success response-code
- jaxRsDpsLog.warning(String.format("unable to proceed, valid storage record not found. | upstream response code: %s | record ids: %s", response.getResponseCode(), String.join(" | ", ids)));
- throw new AppException(HttpStatus.SC_NOT_FOUND, "Invalid request", "Storage service error");
- }
-
- Map<String, List<String>> conversionStatus = getConversionErrors(records.getConversionStatuses());
- for (Records.Entity storageRecord : validRecords) {
- String recordId = storageRecord.getId();
- if (conversionStatus.get(recordId) == null) {
- continue;
- }
- for (String status : conversionStatus.get(recordId)) {
- this.jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, status, String.format("record-id: %s | %s", recordId, status));
- }
- }
-
- // retry missing records -- storage did not return response for all RecordChangeMessage record-ids
- if (records.getTotalRecordCount() != ids.size()) {
- List<String> missingRecords = this.getMissingRecords(records, ids);
- records.setMissingRetryRecords(missingRecords);
- this.jobStatus.addOrUpdateRecordStatus(missingRecords, IndexingStatus.FAIL, HttpStatus.SC_NOT_FOUND, "Partial response received from Storage service - missing records", String.format("Partial response received from Storage service: %s", String.join(",", missingRecords)));
- }
-
- return records;
- }
-
- private List<String> getMissingRecords(Records records, List<String> ids) {
- List<String> validRecordIds = records.getRecords().stream().map(Records.Entity::getId).collect(Collectors.toList());
- List<String> invalidRecordsIds = records.getNotFound();
- List<String> requestedIds = new ArrayList<>(ids);
- requestedIds.removeAll(validRecordIds);
- requestedIds.removeAll(invalidRecordsIds);
- return requestedIds;
- }
-
- private Map<String, List<String>> getConversionErrors(List<ConversionStatus> conversionStatuses) {
- Map<String, List<String>> errorsByRecordId = new HashMap<>();
- for (ConversionStatus conversionStatus : conversionStatuses) {
- if (Strings.isNullOrEmpty(conversionStatus.getStatus())) continue;
- if (conversionStatus.getStatus().equalsIgnoreCase("ERROR")) {
- List<String> statuses = errorsByRecordId.getOrDefault(conversionStatus.getId(), new LinkedList<>());
- statuses.addAll(conversionStatus.getErrors());
- errorsByRecordId.put(conversionStatus.getId(), statuses);
- }
- }
- return errorsByRecordId;
- }
-
- @Override
- public RecordQueryResponse getRecordsByKind(RecordReindexRequest reindexRequest) throws URISyntaxException {
- Map<String, String> queryParams = new HashMap<>();
- queryParams.put(RecordMetaAttribute.KIND.getValue(), reindexRequest.getKind());
- queryParams.put("limit", configurationProperties.getStorageRecordsByKindBatchSize().toString());
- if (!Strings.isNullOrEmpty(reindexRequest.getCursor())) {
- queryParams.put("cursor", reindexRequest.getCursor());
- }
-
- if(requestInfo == null)
- throw new AppException(HttpStatus.SC_NO_CONTENT, "Invalid header", "header can't be null");
-
- FetchServiceHttpRequest request = FetchServiceHttpRequest.builder()
- .httpMethod(HttpMethods.GET)
- .headers(this.requestInfo.getHeadersMap())
- .url(configurationProperties.getStorageQueryRecordHost())
- .queryParams(queryParams)
- .build();
-
- HttpResponse response = this.urlFetchService.sendRequest(request);
- return this.gson.fromJson(response.getBody(), RecordQueryResponse.class);
- }
-
- @Override
- public String getStorageSchema(String kind) throws URISyntaxException, UnsupportedEncodingException {
- String url = String.format("%s/%s", configurationProperties.getStorageSchemaHost(), URLEncoder.encode(kind, StandardCharsets.UTF_8.toString()));
- FetchServiceHttpRequest request = FetchServiceHttpRequest.builder()
- .httpMethod(HttpMethods.GET)
- .headers(this.requestInfo.getHeadersMap())
- .url(url)
- .build();
- HttpResponse response = this.urlFetchService.sendRequest(request);
- return response.getResponseCode() != HttpStatus.SC_OK ? null : response.getBody();
- }
-
- @Override
- public List<String> getAllKinds() throws URISyntaxException {
- String url = configurationProperties.getStorageQueryKindsHost();
- FetchServiceHttpRequest request = FetchServiceHttpRequest.builder()
- .httpMethod(HttpMethods.GET)
- .headers(this.requestInfo.getHeadersMap())
- .url(url)
- .build();
- HttpResponse response = this.urlFetchService.sendRequest(request);
- JsonObject asJsonObject = new JsonParser().parse(response.getBody()).getAsJsonObject();
- JsonElement results = asJsonObject.get("results");
- return response.getResponseCode() != HttpStatus.SC_OK ? null : this.gson.fromJson(results,List.class);
- }
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.service;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.api.client.http.HttpMethods;
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
+import com.google.gson.Gson;
+
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import org.opengroup.osdu.core.common.http.FetchServiceHttpRequest;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.http.HttpResponse;
+import org.opengroup.osdu.core.common.model.http.RequestStatus;
+import org.opengroup.osdu.core.common.model.indexer.*;
+import org.opengroup.osdu.core.common.model.storage.ConversionStatus;
+import org.opengroup.osdu.core.common.model.storage.RecordIds;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.http.IUrlFetchService;
+import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+import javax.inject.Inject;
+import java.io.UnsupportedEncodingException;
+import java.net.URISyntaxException;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.opengroup.osdu.core.common.model.http.DpsHeaders.FRAME_OF_REFERENCE;
+import static org.opengroup.osdu.core.common.Constants.SLB_FRAME_OF_REFERENCE_VALUE;
+
+@Component
+public class StorageServiceImpl implements StorageService {
+
+ private final Gson gson = new Gson();
+
+ @Inject
+ private ObjectMapper objectMapper;
+ @Inject
+ private IUrlFetchService urlFetchService;
+ @Inject
+ private JobStatus jobStatus;
+ @Inject
+ private IRequestInfo requestInfo;
+ @Inject
+ private JaxRsDpsLog jaxRsDpsLog;
+ @Inject
+ private IndexerConfigurationProperties configurationProperties;
+
+ @Override
+ public Records getStorageRecords(List<String> ids) throws AppException, URISyntaxException {
+ List<Records.Entity> valid = new ArrayList<>();
+ List<String> notFound = new ArrayList<>();
+ List<ConversionStatus> conversionStatuses = new ArrayList<>();
+ List<String> missingRetryRecordIds = new ArrayList<>();
+
+ List<List<String>> batch = Lists.partition(ids, configurationProperties.getStorageRecordsBatchSize());
+ for (List<String> recordsBatch : batch) {
+ Records storageOut = this.getRecords(recordsBatch);
+ valid.addAll(storageOut.getRecords());
+ notFound.addAll(storageOut.getNotFound());
+ conversionStatuses.addAll(storageOut.getConversionStatuses());
+ missingRetryRecordIds.addAll(storageOut.getMissingRetryRecords());
+ }
+ return Records.builder().records(valid).notFound(notFound).conversionStatuses(conversionStatuses).missingRetryRecords(missingRetryRecordIds).build();
+ }
+
+ protected Records getRecords(List<String> ids) throws URISyntaxException {
+ // e.g. {"records":["test:10"]}
+ String body = this.gson.toJson(RecordIds.builder().records(ids).build());
+
+// Map<String, String> headers = this.requestInfo.getHeadersMap();
+ DpsHeaders headers = this.requestInfo.getHeaders();
+ headers.put(FRAME_OF_REFERENCE, SLB_FRAME_OF_REFERENCE_VALUE);
+ FetchServiceHttpRequest request = FetchServiceHttpRequest
+ .builder()
+ .httpMethod(HttpMethods.POST)
+ .url(configurationProperties.getStorageQueryRecordForConversionHost())
+ .headers(headers)
+ .body(body).build();
+ HttpResponse response = this.urlFetchService.sendRequest(request);
+ return this.validateStorageResponse(response, ids);
+ }
+
+ private Records validateStorageResponse(HttpResponse response, List<String> ids) {
+ String bulkStorageData = response.getBody();
+
+ // retry entire payload -- storage service returned empty response
+ if (Strings.isNullOrEmpty(bulkStorageData)) {
+ throw new AppException(HttpStatus.SC_NOT_FOUND, "Invalid request", "Storage service returned empty response");
+ }
+
+ Records records = null;
+ try {
+ records = this.objectMapper.readValue(bulkStorageData, Records.class);
+ } catch (JsonProcessingException e) {
+ throw new AppException(RequestStatus.INVALID_RECORD, "Invalid request", "Successful Storage service response with wrong json", e);
+ }
+
+ // no retry possible, update record status as failed -- storage service cannot locate records
+ if (!records.getNotFound().isEmpty()) {
+ this.jobStatus.addOrUpdateRecordStatus(records.getNotFound(), IndexingStatus.FAIL, RequestStatus.INVALID_RECORD, "Storage service records not found", String.format("Storage service records not found: %s", String.join(",", records.getNotFound())));
+ }
+
+ List<Records.Entity> validRecords = records.getRecords();
+ if (validRecords.isEmpty()) {
+ // no need to retry, ack the CloudTask message -- nothing to process from RecordChangeMessage batch
+ if (response.isSuccessCode()) {
+ throw new AppException(RequestStatus.INVALID_RECORD, "Invalid request", "Successful Storage service response with no valid records");
+ }
+
+ // retry entire payload -- storage service returned empty valid records with non-success response-code
+ jaxRsDpsLog.warning(String.format("unable to proceed, valid storage record not found. | upstream response code: %s | record ids: %s", response.getResponseCode(), String.join(" | ", ids)));
+ throw new AppException(HttpStatus.SC_NOT_FOUND, "Invalid request", "Storage service error");
+ }
+
+ Map<String, List<String>> conversionStatus = getConversionErrors(records.getConversionStatuses());
+ for (Records.Entity storageRecord : validRecords) {
+ String recordId = storageRecord.getId();
+ if (conversionStatus.get(recordId) == null) {
+ continue;
+ }
+ for (String status : conversionStatus.get(recordId)) {
+ this.jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, status, String.format("record-id: %s | %s", recordId, status));
+ }
+ }
+
+ // retry missing records -- storage did not return response for all RecordChangeMessage record-ids
+ if (records.getTotalRecordCount() != ids.size()) {
+ List<String> missingRecords = this.getMissingRecords(records, ids);
+ records.setMissingRetryRecords(missingRecords);
+ this.jobStatus.addOrUpdateRecordStatus(missingRecords, IndexingStatus.FAIL, HttpStatus.SC_NOT_FOUND, "Partial response received from Storage service - missing records", String.format("Partial response received from Storage service: %s", String.join(",", missingRecords)));
+ }
+
+ return records;
+ }
+
+ private List<String> getMissingRecords(Records records, List<String> ids) {
+ List<String> validRecordIds = records.getRecords().stream().map(Records.Entity::getId).collect(Collectors.toList());
+ List<String> invalidRecordsIds = records.getNotFound();
+ List<String> requestedIds = new ArrayList<>(ids);
+ requestedIds.removeAll(validRecordIds);
+ requestedIds.removeAll(invalidRecordsIds);
+ return requestedIds;
+ }
+
+ private Map<String, List<String>> getConversionErrors(List<ConversionStatus> conversionStatuses) {
+ Map<String, List<String>> errorsByRecordId = new HashMap<>();
+ for (ConversionStatus conversionStatus : conversionStatuses) {
+ if (Strings.isNullOrEmpty(conversionStatus.getStatus())) continue;
+ if (conversionStatus.getStatus().equalsIgnoreCase("ERROR")) {
+ List<String> statuses = errorsByRecordId.getOrDefault(conversionStatus.getId(), new LinkedList<>());
+ statuses.addAll(conversionStatus.getErrors());
+ errorsByRecordId.put(conversionStatus.getId(), statuses);
+ }
+ }
+ return errorsByRecordId;
+ }
+
+ @Override
+ public RecordQueryResponse getRecordsByKind(RecordReindexRequest reindexRequest) throws URISyntaxException {
+ Map<String, String> queryParams = new HashMap<>();
+ queryParams.put(RecordMetaAttribute.KIND.getValue(), reindexRequest.getKind());
+ queryParams.put("limit", configurationProperties.getStorageRecordsByKindBatchSize().toString());
+ if (!Strings.isNullOrEmpty(reindexRequest.getCursor())) {
+ queryParams.put("cursor", reindexRequest.getCursor());
+ }
+
+ if(requestInfo == null)
+ throw new AppException(HttpStatus.SC_NO_CONTENT, "Invalid header", "header can't be null");
+
+ FetchServiceHttpRequest request = FetchServiceHttpRequest.builder()
+ .httpMethod(HttpMethods.GET)
+ .headers(this.requestInfo.getHeadersMap())
+ .url(configurationProperties.getStorageQueryRecordHost())
+ .queryParams(queryParams)
+ .build();
+
+ HttpResponse response = this.urlFetchService.sendRequest(request);
+ return this.gson.fromJson(response.getBody(), RecordQueryResponse.class);
+ }
+
+ @Override
+ public String getStorageSchema(String kind) throws URISyntaxException, UnsupportedEncodingException {
+ String url = String.format("%s/%s", configurationProperties.getStorageSchemaHost(), URLEncoder.encode(kind, StandardCharsets.UTF_8.toString()));
+ FetchServiceHttpRequest request = FetchServiceHttpRequest.builder()
+ .httpMethod(HttpMethods.GET)
+ .headers(this.requestInfo.getHeadersMap())
+ .url(url)
+ .build();
+ HttpResponse response = this.urlFetchService.sendRequest(request);
+ return response.getResponseCode() != HttpStatus.SC_OK ? null : response.getBody();
+ }
+
+ @Override
+ public List<String> getAllKinds() throws URISyntaxException {
+ String url = configurationProperties.getStorageQueryKindsHost();
+ FetchServiceHttpRequest request = FetchServiceHttpRequest.builder()
+ .httpMethod(HttpMethods.GET)
+ .headers(this.requestInfo.getHeadersMap())
+ .url(url)
+ .build();
+ HttpResponse response = this.urlFetchService.sendRequest(request);
+ JsonObject asJsonObject = new JsonParser().parse(response.getBody()).getAsJsonObject();
+ JsonElement results = asJsonObject.get("results");
+ return response.getResponseCode() != HttpStatus.SC_OK ? null : this.gson.fromJson(results,List.class);
+ }
}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/IndexerQueueTaskBuilder.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/IndexerQueueTaskBuilder.java
index 6d0cf380a..1d94b21ae 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/IndexerQueueTaskBuilder.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/IndexerQueueTaskBuilder.java
@@ -1,86 +1,86 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.util;
-
-import com.google.api.client.http.HttpMethods;
-import com.google.gson.Gson;
-import lombok.extern.java.Log;
-import org.opengroup.osdu.core.common.http.FetchServiceHttpRequest;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.search.CloudTaskRequest;
-import org.opengroup.osdu.core.common.model.http.HttpResponse;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.http.IUrlFetchService;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Component;
-import org.springframework.web.context.annotation.RequestScope;
-
-import java.net.URISyntaxException;
-import java.util.Map;
-import javax.inject.Inject;
-
-import static org.opengroup.osdu.core.common.Constants.REINDEX_RELATIVE_URL;
-import static org.opengroup.osdu.core.common.Constants.WORKER_RELATIVE_URL;
-
- @Log
- @Component
- @RequestScope
- public class IndexerQueueTaskBuilder {
-
- @Inject
- private IUrlFetchService urlFetchService;
- @Inject
- private JaxRsDpsLog jaxRsDpsLog;
- @Inject
- private IndexerConfigurationProperties configurationProperties;
-
- public void createWorkerTask(String payload, DpsHeaders headers) {
- createTask(WORKER_RELATIVE_URL, payload, 0l, headers);
- }
-
- public void createWorkerTask(String payload, Long countdownMillis, DpsHeaders headers) {
- createTask(WORKER_RELATIVE_URL, payload, countdownMillis, headers);
- }
-
- public void createReIndexTask(String payload, DpsHeaders headers) {
- createTask(REINDEX_RELATIVE_URL, payload, 0l, headers);
- }
-
- public void createReIndexTask(String payload, Long countdownMillis, DpsHeaders headers) {
- createTask(REINDEX_RELATIVE_URL, payload, countdownMillis, headers);
- }
-
- private void createTask(String url, String payload, Long countdownMillis, DpsHeaders headers) {
- CloudTaskRequest cloudTaskRequest = CloudTaskRequest.builder()
- .message(payload)
- .url(url)
- .initialDelayMillis(countdownMillis)
- .build();
-
- FetchServiceHttpRequest request = FetchServiceHttpRequest.builder()
- .httpMethod(HttpMethods.POST)
- .url(configurationProperties.getIndexerQueueHost())
- .body(new Gson().toJson(cloudTaskRequest))
- .headers(headers)
- .build();
- try {
- HttpResponse response = this.urlFetchService.sendRequest(request);
- this.jaxRsDpsLog.info(String.format("task enqueuing response: %s", response.getResponseCode()));
- } catch (URISyntaxException e) {
- this.jaxRsDpsLog.warning(String.format("error enqueuing task message: %s | url: %s | task payload: %s", e.getMessage(), url, payload));
- }
- }
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.util;
+
+import com.google.api.client.http.HttpMethods;
+import com.google.gson.Gson;
+import lombok.extern.java.Log;
+import org.opengroup.osdu.core.common.http.FetchServiceHttpRequest;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.search.CloudTaskRequest;
+import org.opengroup.osdu.core.common.model.http.HttpResponse;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.http.IUrlFetchService;
+import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+import java.net.URISyntaxException;
+import java.util.Map;
+import javax.inject.Inject;
+
+import static org.opengroup.osdu.core.common.Constants.REINDEX_RELATIVE_URL;
+import static org.opengroup.osdu.core.common.Constants.WORKER_RELATIVE_URL;
+
+ @Log
+ @Component
+ @RequestScope
+ public class IndexerQueueTaskBuilder {
+
+ @Inject
+ private IUrlFetchService urlFetchService;
+ @Inject
+ private JaxRsDpsLog jaxRsDpsLog;
+ @Inject
+ private IndexerConfigurationProperties configurationProperties;
+
+ public void createWorkerTask(String payload, DpsHeaders headers) {
+ createTask(WORKER_RELATIVE_URL, payload, 0l, headers);
+ }
+
+ public void createWorkerTask(String payload, Long countdownMillis, DpsHeaders headers) {
+ createTask(WORKER_RELATIVE_URL, payload, countdownMillis, headers);
+ }
+
+ public void createReIndexTask(String payload, DpsHeaders headers) {
+ createTask(REINDEX_RELATIVE_URL, payload, 0l, headers);
+ }
+
+ public void createReIndexTask(String payload, Long countdownMillis, DpsHeaders headers) {
+ createTask(REINDEX_RELATIVE_URL, payload, countdownMillis, headers);
+ }
+
+ private void createTask(String url, String payload, Long countdownMillis, DpsHeaders headers) {
+ CloudTaskRequest cloudTaskRequest = CloudTaskRequest.builder()
+ .message(payload)
+ .url(url)
+ .initialDelayMillis(countdownMillis)
+ .build();
+
+ FetchServiceHttpRequest request = FetchServiceHttpRequest.builder()
+ .httpMethod(HttpMethods.POST)
+ .url(configurationProperties.getIndexerQueueHost())
+ .body(new Gson().toJson(cloudTaskRequest))
+ .headers(headers)
+ .build();
+ try {
+ HttpResponse response = this.urlFetchService.sendRequest(request);
+ this.jaxRsDpsLog.info(String.format("task enqueuing response: %s", response.getResponseCode()));
+ } catch (URISyntaxException e) {
+ this.jaxRsDpsLog.warning(String.format("error enqueuing task message: %s | url: %s | task payload: %s", e.getMessage(), url, payload));
+ }
+ }
}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/parser/BooleanParser.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/parser/BooleanParser.java
index 3bb08d3fc..3177f3b65 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/parser/BooleanParser.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/parser/BooleanParser.java
@@ -1,15 +1,15 @@
-package org.opengroup.osdu.indexer.util.parser;
-
-import org.springframework.stereotype.Component;
-import org.springframework.web.context.annotation.RequestScope;
-
-@Component
-@RequestScope
-public class BooleanParser {
-
- public boolean parseBoolean(String attributeName, Object attributeVal) {
- String val = attributeVal == null ? null : String.valueOf(attributeVal);
- return Boolean.parseBoolean(val);
- }
-
-}
+package org.opengroup.osdu.indexer.util.parser;
+
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+@Component
+@RequestScope
+public class BooleanParser {
+
+ public boolean parseBoolean(String attributeName, Object attributeVal) {
+ String val = attributeVal == null ? null : String.valueOf(attributeVal);
+ return Boolean.parseBoolean(val);
+ }
+
+}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/ReindexApiTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/ReindexApiTest.java
index 089f8417a..23513bc5d 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/ReindexApiTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/ReindexApiTest.java
@@ -1,77 +1,77 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.api;
-
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
-import org.opengroup.osdu.indexer.logging.AuditLogger;
-import org.opengroup.osdu.indexer.service.IndexSchemaService;
-import org.opengroup.osdu.indexer.service.ReindexService;
-import org.springframework.http.HttpStatus;
-import org.springframework.http.ResponseEntity;
-import org.springframework.test.context.junit4.SpringRunner;
-
-import java.io.IOException;
-
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.when;
-
-@RunWith(SpringRunner.class)
-public class ReindexApiTest {
-
- private RecordReindexRequest recordReindexRequest;
-
- @Mock
- private ReindexService reIndexService;
- @Mock
- private IndexSchemaService indexSchemaService;
- @Mock
- private AuditLogger auditLogger;
- @InjectMocks
- private ReindexApi sut;
-
- @Before
- public void setup() {
- recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor("100").build();
- }
-
- @Test
- public void should_return200_when_valid_kind_provided() throws IOException {
- when(this.reIndexService.reindexRecords(recordReindexRequest, false)).thenReturn("something");
-
- ResponseEntity<?> response = sut.reindex(recordReindexRequest, false);
-
- assertEquals(HttpStatus.OK, response.getStatusCode());
- }
-
- @Test(expected = AppException.class)
- public void should_throwAppException_ifUnknownExceptionCaught_reindexTest() throws IOException {
- when(this.reIndexService.reindexRecords(recordReindexRequest, false)).thenThrow(new AppException(500, "", ""));
-
- sut.reindex(recordReindexRequest, false);
- }
-
- @Test(expected = NullPointerException.class)
- public void should_throwAppException_ifNullPointerExceptionCaught_ReindexTest() throws IOException {
- when(this.reIndexService.reindexRecords(recordReindexRequest, false)).thenThrow(new NullPointerException(""));
-
- sut.reindex(recordReindexRequest, false);
- }
-}
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.api;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
+import org.opengroup.osdu.indexer.logging.AuditLogger;
+import org.opengroup.osdu.indexer.service.IndexSchemaService;
+import org.opengroup.osdu.indexer.service.ReindexService;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.when;
+
+@RunWith(SpringRunner.class)
+public class ReindexApiTest {
+
+ private RecordReindexRequest recordReindexRequest;
+
+ @Mock
+ private ReindexService reIndexService;
+ @Mock
+ private IndexSchemaService indexSchemaService;
+ @Mock
+ private AuditLogger auditLogger;
+ @InjectMocks
+ private ReindexApi sut;
+
+ @Before
+ public void setup() {
+ recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor("100").build();
+ }
+
+ @Test
+ public void should_return200_when_valid_kind_provided() throws IOException {
+ when(this.reIndexService.reindexRecords(recordReindexRequest, false)).thenReturn("something");
+
+ ResponseEntity<?> response = sut.reindex(recordReindexRequest, false);
+
+ assertEquals(HttpStatus.OK, response.getStatusCode());
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throwAppException_ifUnknownExceptionCaught_reindexTest() throws IOException {
+ when(this.reIndexService.reindexRecords(recordReindexRequest, false)).thenThrow(new AppException(500, "", ""));
+
+ sut.reindex(recordReindexRequest, false);
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void should_throwAppException_ifNullPointerExceptionCaught_ReindexTest() throws IOException {
+ when(this.reIndexService.reindexRecords(recordReindexRequest, false)).thenThrow(new NullPointerException(""));
+
+ sut.reindex(recordReindexRequest, false);
+ }
+}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/parser/BooleanParserTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/parser/BooleanParserTest.java
index b12673e19..98ea59355 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/parser/BooleanParserTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/parser/BooleanParserTest.java
@@ -1,27 +1,27 @@
-package org.opengroup.osdu.indexer.util.parser;
-
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.mockito.runners.MockitoJUnitRunner;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-@RunWith(MockitoJUnitRunner.class)
-public class BooleanParserTest {
-
- @InjectMocks
- private BooleanParser sut;
-
- @Test
- public void should_parseBoolean() {
- assertTrue(this.sut.parseBoolean("testBooleanAttribute", "true"));
- assertTrue(this.sut.parseBoolean("testBooleanAttribute", "TRUE"));
-
- assertFalse(this.sut.parseBoolean("testBooleanAttribute", ""));
- assertFalse(this.sut.parseBoolean("testBooleanAttribute", null));
- assertFalse(this.sut.parseBoolean("testBooleanAttribute", "false"));
- assertFalse(this.sut.parseBoolean("testBooleanAttribute", "truee"));
- }
+package org.opengroup.osdu.indexer.util.parser;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(MockitoJUnitRunner.class)
+public class BooleanParserTest {
+
+ @InjectMocks
+ private BooleanParser sut;
+
+ @Test
+ public void should_parseBoolean() {
+ assertTrue(this.sut.parseBoolean("testBooleanAttribute", "true"));
+ assertTrue(this.sut.parseBoolean("testBooleanAttribute", "TRUE"));
+
+ assertFalse(this.sut.parseBoolean("testBooleanAttribute", ""));
+ assertFalse(this.sut.parseBoolean("testBooleanAttribute", null));
+ assertFalse(this.sut.parseBoolean("testBooleanAttribute", "false"));
+ assertFalse(this.sut.parseBoolean("testBooleanAttribute", "truee"));
+ }
}
\ No newline at end of file
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/service/RetryPolicy.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/service/RetryPolicy.java
index 85aaa1ee1..c3876e4a3 100644
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/service/RetryPolicy.java
+++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/service/RetryPolicy.java
@@ -1,114 +1,114 @@
-// Copyright © Microsoft Corporation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.azure.service;
-
-import com.google.gson.JsonArray;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParser;
-import io.github.resilience4j.core.IntervalFunction;
-import io.github.resilience4j.retry.RetryConfig;
-import lombok.Data;
-import lombok.extern.java.Log;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.http.HttpResponse;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.context.properties.ConfigurationProperties;
-import org.springframework.stereotype.Component;
-
-import java.util.function.Predicate;
-
-/**
- * This class handles retry configuration logic for calls made to <prefix>/storage/v2/query/records:batch
- * to resolve intermittent CosmosDb Not found issue
- */
-
-@Log
-@Component
-@Data
-@ConfigurationProperties(prefix = "azure.storage.client.retry")
-public class RetryPolicy {
-
- @Autowired
- private JaxRsDpsLog logger;
-
- private static int MAX_ATTEMPTS = 5;
- private static int INITIAL_DELAY = 1000;
- private final String RECORD_NOT_FOUND = "notFound";
-
- /**
- * @return RetryConfig with 3 attempts and 1 sec wait time
- */
- public RetryConfig retryConfig(Predicate<HttpResponse> predicate) {
- return RetryConfig.<HttpResponse>custom()
- .maxAttempts(MAX_ATTEMPTS)
- .intervalFunction(IntervalFunction.ofExponentialBackoff(INITIAL_DELAY, 2))
- .retryOnResult(predicate)
- .build();
- }
-
- /**
- * Unfound records get listed under a JsonArray "notFound" in the http json response
- *
- * @param response
- * @return if there are elements in "notFound" returns true, else false
- */
- public boolean batchRetryPolicy(HttpResponse response) {
- if (retryOnEmptyResponse(response)) return false;
-
- if (defaultResponseRetry(response)) return true;
-
- JsonObject jsonObject = new JsonParser().parse(response.getBody()).getAsJsonObject();
- JsonElement notFoundElement = (JsonArray) jsonObject.get(RECORD_NOT_FOUND);
- if (notFoundElement == null ||
- !notFoundElement.isJsonArray() ||
- notFoundElement.getAsJsonArray().size() == 0 ||
- notFoundElement.getAsJsonArray().isJsonNull()) {
- return false;
- }
- log.info("Storage batch API retry");
- return true;
- }
-
- public boolean schemaRetryPolicy(HttpResponse response) {
- if (retryOnEmptyResponse(response)) return false;
-
- if (defaultResponseRetry(response)) return true;
-
- if (response.getResponseCode() == 404) {
- log.info("Schema API retry");
- return true;
- }
-
- return false;
- }
-
- public boolean defaultRetryPolicy(HttpResponse response) {
- if (retryOnEmptyResponse(response)) return false;
-
- return defaultResponseRetry(response);
- }
-
- private boolean retryOnEmptyResponse(HttpResponse response) {
- return response == null || response.getBody().isEmpty();
- }
-
- private boolean defaultResponseRetry(HttpResponse response) {
- if (response.getResponseCode() <= 501) return false;
-
- log.info(String.format("Default retry, response code: %s", response.getResponseCode()));
- return true;
- }
-}
+// Copyright © Microsoft Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.azure.service;
+
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import io.github.resilience4j.core.IntervalFunction;
+import io.github.resilience4j.retry.RetryConfig;
+import lombok.Data;
+import lombok.extern.java.Log;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.HttpResponse;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+import java.util.function.Predicate;
+
+/**
+ * This class handles retry configuration logic for calls made to <prefix>/storage/v2/query/records:batch
+ * to resolve intermittent CosmosDb Not found issue
+ */
+
+@Log
+@Component
+@Data
+@ConfigurationProperties(prefix = "azure.storage.client.retry")
+public class RetryPolicy {
+
+ @Autowired
+ private JaxRsDpsLog logger;
+
+ private static int MAX_ATTEMPTS = 5;
+ private static int INITIAL_DELAY = 1000;
+ private final String RECORD_NOT_FOUND = "notFound";
+
+ /**
+ * @return RetryConfig with 3 attempts and 1 sec wait time
+ */
+ public RetryConfig retryConfig(Predicate<HttpResponse> predicate) {
+ return RetryConfig.<HttpResponse>custom()
+ .maxAttempts(MAX_ATTEMPTS)
+ .intervalFunction(IntervalFunction.ofExponentialBackoff(INITIAL_DELAY, 2))
+ .retryOnResult(predicate)
+ .build();
+ }
+
+ /**
+ * Unfound records get listed under a JsonArray "notFound" in the http json response
+ *
+ * @param response
+ * @return if there are elements in "notFound" returns true, else false
+ */
+ public boolean batchRetryPolicy(HttpResponse response) {
+ if (retryOnEmptyResponse(response)) return false;
+
+ if (defaultResponseRetry(response)) return true;
+
+ JsonObject jsonObject = new JsonParser().parse(response.getBody()).getAsJsonObject();
+ JsonElement notFoundElement = (JsonArray) jsonObject.get(RECORD_NOT_FOUND);
+ if (notFoundElement == null ||
+ !notFoundElement.isJsonArray() ||
+ notFoundElement.getAsJsonArray().size() == 0 ||
+ notFoundElement.getAsJsonArray().isJsonNull()) {
+ return false;
+ }
+ log.info("Storage batch API retry");
+ return true;
+ }
+
+ public boolean schemaRetryPolicy(HttpResponse response) {
+ if (retryOnEmptyResponse(response)) return false;
+
+ if (defaultResponseRetry(response)) return true;
+
+ if (response.getResponseCode() == 404) {
+ log.info("Schema API retry");
+ return true;
+ }
+
+ return false;
+ }
+
+ public boolean defaultRetryPolicy(HttpResponse response) {
+ if (retryOnEmptyResponse(response)) return false;
+
+ return defaultResponseRetry(response);
+ }
+
+ private boolean retryOnEmptyResponse(HttpResponse response) {
+ return response == null || response.getBody().isEmpty();
+ }
+
+ private boolean defaultResponseRetry(HttpResponse response) {
+ if (response.getResponseCode() <= 501) return false;
+
+ log.info(String.format("Default retry, response code: %s", response.getResponseCode()));
+ return true;
+ }
+}
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/service/UrlFetchServiceAzureImpl.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/service/UrlFetchServiceAzureImpl.java
index c0c56ccd8..91c331f25 100644
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/service/UrlFetchServiceAzureImpl.java
+++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/service/UrlFetchServiceAzureImpl.java
@@ -1,98 +1,98 @@
-// Copyright © Microsoft Corporation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.azure.service;
-
-import io.github.resilience4j.retry.Retry;
-import io.github.resilience4j.retry.RetryConfig;
-import io.github.resilience4j.retry.RetryRegistry;
-import org.opengroup.osdu.core.common.http.FetchServiceHttpRequest;
-import org.opengroup.osdu.core.common.http.IUrlFetchService;
-import org.opengroup.osdu.core.common.http.UrlFetchServiceImpl;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.http.HttpResponse;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.context.annotation.Primary;
-import org.springframework.stereotype.Service;
-import org.springframework.web.context.annotation.RequestScope;
-
-import java.net.URISyntaxException;
-import java.util.function.Supplier;
-
-/**
- * This class has same function as that of UrlFetchService except in the case of
- * <prefix>/storage/v2/query/records:batch call for which it enables retry
- */
-
-@Service
-@RequestScope
-@Primary
-public class UrlFetchServiceAzureImpl implements IUrlFetchService {
-
- public static final String STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST_URL = "storage/v2/query/records:batch";
- public static final String SCHEMA_SERVICE_HOST_URL = "api/schema-service/v1/schema";
-
- @Autowired
- private RetryPolicy policy;
-
- @Autowired
- private UrlFetchServiceImpl urlFetchService;
-
- @Autowired
- private JaxRsDpsLog logger;
-
- /**
- * this method invokes retryFunction only for <prefix>/storage/v2/query/records:batch
- * calls otherwise invokes UrlFetchService.sendRequest(FetchServiceHttpRequest request)
- *
- * @param httpRequest
- * @return
- * @throws URISyntaxException
- */
- @Override
- public HttpResponse sendRequest(FetchServiceHttpRequest httpRequest) throws URISyntaxException {
- return this.retryFunction(httpRequest);
- }
-
- /**
- * decorates UrlFetchService.sendRequest(FetchServiceHttpRequest request)
- * with retry configurations in RetryPolicy
- *
- * @param request
- * @return null if URISyntaxException is caught else returns HttpResponse
- */
- private HttpResponse retryFunction(FetchServiceHttpRequest request) {
- RetryConfig config;
- if (request.getUrl().contains(STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST_URL)) {
- config = this.policy.retryConfig(response -> this.policy.batchRetryPolicy(response));
- } else if (request.getUrl().contains(SCHEMA_SERVICE_HOST_URL)) {
- config = this.policy.retryConfig(response -> this.policy.schemaRetryPolicy(response));
- } else {
- config = this.policy.retryConfig(response -> this.policy.defaultRetryPolicy(response));
- }
-
- RetryRegistry registry = RetryRegistry.of(config);
- Retry retry = registry.retry("retryPolicy", config);
-
- Supplier<HttpResponse> urlFetchServiceSupplier = () -> {
- try {
- return this.urlFetchService.sendRequest(request);
- } catch (URISyntaxException e) {
- logger.error("HttpResponse is null due to URISyntaxException. " + e.getReason());
- return null;
- }
- };
- return (urlFetchServiceSupplier == null) ? null : Retry.decorateSupplier(retry, urlFetchServiceSupplier).get();
- }
-}
+// Copyright © Microsoft Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.azure.service;
+
+import io.github.resilience4j.retry.Retry;
+import io.github.resilience4j.retry.RetryConfig;
+import io.github.resilience4j.retry.RetryRegistry;
+import org.opengroup.osdu.core.common.http.FetchServiceHttpRequest;
+import org.opengroup.osdu.core.common.http.IUrlFetchService;
+import org.opengroup.osdu.core.common.http.UrlFetchServiceImpl;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.HttpResponse;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Primary;
+import org.springframework.stereotype.Service;
+import org.springframework.web.context.annotation.RequestScope;
+
+import java.net.URISyntaxException;
+import java.util.function.Supplier;
+
+/**
+ * This class has same function as that of UrlFetchService except in the case of
+ * <prefix>/storage/v2/query/records:batch call for which it enables retry
+ */
+
+@Service
+@RequestScope
+@Primary
+public class UrlFetchServiceAzureImpl implements IUrlFetchService {
+
+ public static final String STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST_URL = "storage/v2/query/records:batch";
+ public static final String SCHEMA_SERVICE_HOST_URL = "api/schema-service/v1/schema";
+
+ @Autowired
+ private RetryPolicy policy;
+
+ @Autowired
+ private UrlFetchServiceImpl urlFetchService;
+
+ @Autowired
+ private JaxRsDpsLog logger;
+
+ /**
+ * this method invokes retryFunction only for <prefix>/storage/v2/query/records:batch
+ * calls otherwise invokes UrlFetchService.sendRequest(FetchServiceHttpRequest request)
+ *
+ * @param httpRequest
+ * @return
+ * @throws URISyntaxException
+ */
+ @Override
+ public HttpResponse sendRequest(FetchServiceHttpRequest httpRequest) throws URISyntaxException {
+ return this.retryFunction(httpRequest);
+ }
+
+ /**
+ * decorates UrlFetchService.sendRequest(FetchServiceHttpRequest request)
+ * with retry configurations in RetryPolicy
+ *
+ * @param request
+ * @return null if URISyntaxException is caught else returns HttpResponse
+ */
+ private HttpResponse retryFunction(FetchServiceHttpRequest request) {
+ RetryConfig config;
+ if (request.getUrl().contains(STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST_URL)) {
+ config = this.policy.retryConfig(response -> this.policy.batchRetryPolicy(response));
+ } else if (request.getUrl().contains(SCHEMA_SERVICE_HOST_URL)) {
+ config = this.policy.retryConfig(response -> this.policy.schemaRetryPolicy(response));
+ } else {
+ config = this.policy.retryConfig(response -> this.policy.defaultRetryPolicy(response));
+ }
+
+ RetryRegistry registry = RetryRegistry.of(config);
+ Retry retry = registry.retry("retryPolicy", config);
+
+ Supplier<HttpResponse> urlFetchServiceSupplier = () -> {
+ try {
+ return this.urlFetchService.sendRequest(request);
+ } catch (URISyntaxException e) {
+ logger.error("HttpResponse is null due to URISyntaxException. " + e.getReason());
+ return null;
+ }
+ };
+ return (urlFetchServiceSupplier == null) ? null : Retry.decorateSupplier(retry, urlFetchServiceSupplier).get();
+ }
+}
diff --git a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/RetryPolicyTest.java b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/RetryPolicyTest.java
index a27c40932..cb891b20b 100644
--- a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/RetryPolicyTest.java
+++ b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/RetryPolicyTest.java
@@ -1,163 +1,163 @@
-// Copyright © Microsoft Corporation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.azure.service;
-
-import io.github.resilience4j.retry.RetryConfig;
-
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.mockito.junit.MockitoJUnitRunner;
-import org.opengroup.osdu.core.common.http.FetchServiceHttpRequest;
-import org.opengroup.osdu.core.common.http.UrlFetchServiceImpl;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.http.HttpResponse;
-
-import java.util.function.Predicate;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-@RunWith(MockitoJUnitRunner.class)
-public class RetryPolicyTest {
-
- private static final String JSON_RESPONSE_WITH_NOT_FOUND = "{\n" +
- " \"records\": [\n" +
- " {\n" +
- " \"data\": {\n" +
- " \"Spuddate\": \"atspud\",\n" +
- " \"UWI\": \"atuwi\",\n" +
- " \"latitude\": \"latitude\",\n" +
- " \"longitude\": \"longitude\"\n" +
- " },\n" +
- " \"meta\": null,\n" +
- " \"id\": \"demo\",\n" +
- " \"version\": demo,\n" +
- " \"kind\": \"demo\",\n" +
- " \"acl\": {\n" +
- " \"viewers\": [\n" +
- " \"demo\"\n" +
- " ],\n" +
- " \"owners\": [\n" +
- " \"demo\"\n" +
- " ]\n" +
- " },\n" +
- " \"legal\": {\n" +
- " \"legaltags\": [\n" +
- " \"opendes-test-tag\"\n" +
- " ],\n" +
- " \"otherRelevantDataCountries\": [\n" +
- " \"BR\"\n" +
- " ],\n" +
- " \"status\": \"compliant\"\n" +
- " },\n" +
- " \"createUser\": \"demo\",\n" +
- " \"createTime\": \"demo\"\n" +
- " }\n" +
- " ],\n" +
- " \"notFound\": [\n" +
- " \"demo\"\n" +
- " ],\n" +
- " \"conversionStatuses\": []\n" +
- "}";
-
- private static final String JSON_RESPONSE1_WITHOUT_NOT_FOUND = "{\n" +
- " \"records\": [\n" +
- " {\n" +
- " \"data\": {\n" +
- " \"Spuddate\": \"atspud\",\n" +
- " \"UWI\": \"atuwi\",\n" +
- " \"latitude\": \"latitude\",\n" +
- " \"longitude\": \"longitude\"\n" +
- " },\n" +
- " \"meta\": null,\n" +
- " \"id\": \"demo\",\n" +
- " \"version\": demo,\n" +
- " \"kind\": \"demo\",\n" +
- " \"acl\": {\n" +
- " \"viewers\": [\n" +
- " \"demo\"\n" +
- " ],\n" +
- " \"owners\": [\n" +
- " \"demo\"\n" +
- " ]\n" +
- " },\n" +
- " \"legal\": {\n" +
- " \"legaltags\": [\n" +
- " \"opendes-test-tag\"\n" +
- " ],\n" +
- " \"otherRelevantDataCountries\": [\n" +
- " \"BR\"\n" +
- " ],\n" +
- " \"status\": \"compliant\"\n" +
- " },\n" +
- " \"createUser\": \"demo\",\n" +
- " \"createTime\": \"demo\"\n" +
- " }\n" +
- " ],\n" +
- " \"notFound\": [],\n" +
- " \"conversionStatuses\": []\n" +
- "}";
-
- private static final String JSON_RESPONSE2_WITHOUT_NOT_FOUND = "{\n" +
- " \"records\" :[],\n" +
- " \"conversionStatuses\":[]\n" +
- "}";
-
- @Mock
- private UrlFetchServiceImpl urlFetchService;
- @Mock
- private FetchServiceHttpRequest httpRequest;
- @InjectMocks
- private HttpResponse response;
- @InjectMocks
- private RetryPolicy retryPolicy;
- @Mock
- private JaxRsDpsLog logger;
-
-
- @Test
- public void retry_should_be_true_for_jsonResponseWithNotFound() {
- RetryConfig config = this.retryPolicy.retryConfig(response -> this.retryPolicy.batchRetryPolicy(response));
- Predicate<HttpResponse> retry = config.getResultPredicate();
- response.setBody(JSON_RESPONSE_WITH_NOT_FOUND);
- assert retry != null;
- boolean value = retry.test(response);
-
- assertTrue(value);
- }
-
- @Test
- public void retry_should_be_false_for_jsonResponse1WithOut_NotFound() {
- RetryConfig config = this.retryPolicy.retryConfig(response -> this.retryPolicy.batchRetryPolicy(response));
- Predicate<HttpResponse> retry = config.getResultPredicate();
- response.setBody(JSON_RESPONSE1_WITHOUT_NOT_FOUND);
- boolean value = retry.test(response);
-
- assertFalse(value);
- }
-
- @Test
- public void retry_should_be_false_for_jsonResponse2WithOut_NotFound() {
- RetryConfig config = this.retryPolicy.retryConfig(response -> this.retryPolicy.batchRetryPolicy(response));
- Predicate<HttpResponse> retry = config.getResultPredicate();
- response.setBody(JSON_RESPONSE2_WITHOUT_NOT_FOUND);
- boolean value = retry.test(response);
-
- assertFalse(value);
- }
-
-}
+// Copyright © Microsoft Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.azure.service;
+
+import io.github.resilience4j.retry.RetryConfig;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+import org.opengroup.osdu.core.common.http.FetchServiceHttpRequest;
+import org.opengroup.osdu.core.common.http.UrlFetchServiceImpl;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.HttpResponse;
+
+import java.util.function.Predicate;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(MockitoJUnitRunner.class)
+public class RetryPolicyTest {
+
+ private static final String JSON_RESPONSE_WITH_NOT_FOUND = "{\n" +
+ " \"records\": [\n" +
+ " {\n" +
+ " \"data\": {\n" +
+ " \"Spuddate\": \"atspud\",\n" +
+ " \"UWI\": \"atuwi\",\n" +
+ " \"latitude\": \"latitude\",\n" +
+ " \"longitude\": \"longitude\"\n" +
+ " },\n" +
+ " \"meta\": null,\n" +
+ " \"id\": \"demo\",\n" +
+ " \"version\": demo,\n" +
+ " \"kind\": \"demo\",\n" +
+ " \"acl\": {\n" +
+ " \"viewers\": [\n" +
+ " \"demo\"\n" +
+ " ],\n" +
+ " \"owners\": [\n" +
+ " \"demo\"\n" +
+ " ]\n" +
+ " },\n" +
+ " \"legal\": {\n" +
+ " \"legaltags\": [\n" +
+ " \"opendes-test-tag\"\n" +
+ " ],\n" +
+ " \"otherRelevantDataCountries\": [\n" +
+ " \"BR\"\n" +
+ " ],\n" +
+ " \"status\": \"compliant\"\n" +
+ " },\n" +
+ " \"createUser\": \"demo\",\n" +
+ " \"createTime\": \"demo\"\n" +
+ " }\n" +
+ " ],\n" +
+ " \"notFound\": [\n" +
+ " \"demo\"\n" +
+ " ],\n" +
+ " \"conversionStatuses\": []\n" +
+ "}";
+
+ private static final String JSON_RESPONSE1_WITHOUT_NOT_FOUND = "{\n" +
+ " \"records\": [\n" +
+ " {\n" +
+ " \"data\": {\n" +
+ " \"Spuddate\": \"atspud\",\n" +
+ " \"UWI\": \"atuwi\",\n" +
+ " \"latitude\": \"latitude\",\n" +
+ " \"longitude\": \"longitude\"\n" +
+ " },\n" +
+ " \"meta\": null,\n" +
+ " \"id\": \"demo\",\n" +
+ " \"version\": demo,\n" +
+ " \"kind\": \"demo\",\n" +
+ " \"acl\": {\n" +
+ " \"viewers\": [\n" +
+ " \"demo\"\n" +
+ " ],\n" +
+ " \"owners\": [\n" +
+ " \"demo\"\n" +
+ " ]\n" +
+ " },\n" +
+ " \"legal\": {\n" +
+ " \"legaltags\": [\n" +
+ " \"opendes-test-tag\"\n" +
+ " ],\n" +
+ " \"otherRelevantDataCountries\": [\n" +
+ " \"BR\"\n" +
+ " ],\n" +
+ " \"status\": \"compliant\"\n" +
+ " },\n" +
+ " \"createUser\": \"demo\",\n" +
+ " \"createTime\": \"demo\"\n" +
+ " }\n" +
+ " ],\n" +
+ " \"notFound\": [],\n" +
+ " \"conversionStatuses\": []\n" +
+ "}";
+
+ private static final String JSON_RESPONSE2_WITHOUT_NOT_FOUND = "{\n" +
+ " \"records\" :[],\n" +
+ " \"conversionStatuses\":[]\n" +
+ "}";
+
+ @Mock
+ private UrlFetchServiceImpl urlFetchService;
+ @Mock
+ private FetchServiceHttpRequest httpRequest;
+ @InjectMocks
+ private HttpResponse response;
+ @InjectMocks
+ private RetryPolicy retryPolicy;
+ @Mock
+ private JaxRsDpsLog logger;
+
+
+ @Test
+ public void retry_should_be_true_for_jsonResponseWithNotFound() {
+ RetryConfig config = this.retryPolicy.retryConfig(response -> this.retryPolicy.batchRetryPolicy(response));
+ Predicate<HttpResponse> retry = config.getResultPredicate();
+ response.setBody(JSON_RESPONSE_WITH_NOT_FOUND);
+ assert retry != null;
+ boolean value = retry.test(response);
+
+ assertTrue(value);
+ }
+
+ @Test
+ public void retry_should_be_false_for_jsonResponse1WithOut_NotFound() {
+ RetryConfig config = this.retryPolicy.retryConfig(response -> this.retryPolicy.batchRetryPolicy(response));
+ Predicate<HttpResponse> retry = config.getResultPredicate();
+ response.setBody(JSON_RESPONSE1_WITHOUT_NOT_FOUND);
+ boolean value = retry.test(response);
+
+ assertFalse(value);
+ }
+
+ @Test
+ public void retry_should_be_false_for_jsonResponse2WithOut_NotFound() {
+ RetryConfig config = this.retryPolicy.retryConfig(response -> this.retryPolicy.batchRetryPolicy(response));
+ Predicate<HttpResponse> retry = config.getResultPredicate();
+ response.setBody(JSON_RESPONSE2_WITHOUT_NOT_FOUND);
+ boolean value = retry.test(response);
+
+ assertFalse(value);
+ }
+
+}
diff --git a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/UrlFetchServiceAzureImplTest.java b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/UrlFetchServiceAzureImplTest.java
index 18adccbfe..255619afc 100644
--- a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/UrlFetchServiceAzureImplTest.java
+++ b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/UrlFetchServiceAzureImplTest.java
@@ -1,131 +1,131 @@
-// Copyright © Microsoft Corporation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.azure.service;
-
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.mockito.junit.MockitoJUnitRunner;
-import org.opengroup.osdu.core.common.http.FetchServiceHttpRequest;
-import org.opengroup.osdu.core.common.http.UrlFetchServiceImpl;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.http.HttpResponse;
-
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.*;
-
-
-@RunWith(MockitoJUnitRunner.class)
-public class UrlFetchServiceAzureImplTest {
-
- @Mock
- private JaxRsDpsLog logger;
- @Mock
- private UrlFetchServiceImpl urlFetchService;
- @InjectMocks
- private FetchServiceHttpRequest httpRequest;
- @InjectMocks
- private HttpResponse response;
- @Mock
- private RetryPolicy retryPolicy;
- @InjectMocks
- private UrlFetchServiceAzureImpl urlFetchServiceAzure;
-
- private static final String JSON1 = "{\n" +
- " \"records\": [\n" +
- " {\n" +
- " \"data\": {\n" +
- " \"Spuddate\": \"atspud\",\n" +
- " \"UWI\": \"atuwi\",\n" +
- " \"latitude\": \"latitude\",\n" +
- " \"longitude\": \"longitude\"\n" +
- " },\n" +
- " \"meta\": null,\n" +
- " \"id\": \"demo\",\n" +
- " \"version\": demo,\n" +
- " \"kind\": \"demo\",\n" +
- " \"acl\": {\n" +
- " \"viewers\": [\n" +
- " \"demo\"\n" +
- " ],\n" +
- " \"owners\": [\n" +
- " \"demo\"\n" +
- " ]\n" +
- " },\n" +
- " \"legal\": {\n" +
- " \"legaltags\": [\n" +
- " \"opendes-test-tag\"\n" +
- " ],\n" +
- " \"otherRelevantDataCountries\": [\n" +
- " \"BR\"\n" +
- " ],\n" +
- " \"status\": \"compliant\"\n" +
- " },\n" +
- " \"createUser\": \"demo\",\n" +
- " \"createTime\": \"demo\"\n" +
- " }\n" +
- " ],\n" +
- " \"notFound\": [\n" +
- " \"demo\"\n" +
- " ],\n" +
- " \"conversionStatuses\": []\n" +
- "}";
-
- private static final String JSON2 = "{\n" +
- " \"records\" :[],\n" +
- " \"conversionStatuses\":[]\n" +
- "}";
-
- private static final String BATCH_API_URL = "https://demo/api/storage/v2/query/records:batch";
- private static final String STORAGE_API_URL = "https://demo/api/storage/v2/schemas";
- private static final String SCHEMA_API_URL = "https://demo/api/schema-service/v1/schema/osdu:file:gom:1.0.0";
-
- @Test
- public void shouldRetry_ForJSON1_when_storageQueryRecordCallIsMade() throws Exception {
- response.setBody(JSON1);
- httpRequest.setUrl(BATCH_API_URL);
- when(this.retryPolicy.retryConfig(any())).thenReturn(new RetryPolicy().retryConfig(response -> this.retryPolicy.batchRetryPolicy(response)));
- when(urlFetchService.sendRequest(httpRequest)).thenReturn(response);
-
- urlFetchServiceAzure.sendRequest(httpRequest);
-
- verify(urlFetchService, atMost(4)).sendRequest(httpRequest);
- }
-
- @Test
- public void shouldRetry_ForJSON1_when_schemaRecordCallIsMade() throws Exception {
- response.setBody(JSON1);
- httpRequest.setUrl(SCHEMA_API_URL);
- when(this.retryPolicy.retryConfig(any())).thenReturn(new RetryPolicy().retryConfig(response -> this.retryPolicy.schemaRetryPolicy(response)));
- when(urlFetchService.sendRequest(httpRequest)).thenReturn(response);
-
- urlFetchServiceAzure.sendRequest(httpRequest);
-
- verify(urlFetchService, atMost(4)).sendRequest(httpRequest);
- }
-
- @Test
- public void shouldRetry_when_anyOtherCallIsMade() throws Exception {
- response.setBody(JSON2);
- httpRequest.setUrl(STORAGE_API_URL);
- when(this.retryPolicy.retryConfig(any())).thenReturn(new RetryPolicy().retryConfig(response -> this.retryPolicy.defaultRetryPolicy(response)));
- when(urlFetchService.sendRequest(httpRequest)).thenReturn(response);
-
- urlFetchServiceAzure.sendRequest(httpRequest);
-
- verify(urlFetchService, atMost(4)).sendRequest(httpRequest);
- }
-}
+// Copyright © Microsoft Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.azure.service;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+import org.opengroup.osdu.core.common.http.FetchServiceHttpRequest;
+import org.opengroup.osdu.core.common.http.UrlFetchServiceImpl;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.HttpResponse;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.*;
+
+
+@RunWith(MockitoJUnitRunner.class)
+public class UrlFetchServiceAzureImplTest {
+
+ @Mock
+ private JaxRsDpsLog logger;
+ @Mock
+ private UrlFetchServiceImpl urlFetchService;
+ @InjectMocks
+ private FetchServiceHttpRequest httpRequest;
+ @InjectMocks
+ private HttpResponse response;
+ @Mock
+ private RetryPolicy retryPolicy;
+ @InjectMocks
+ private UrlFetchServiceAzureImpl urlFetchServiceAzure;
+
+ private static final String JSON1 = "{\n" +
+ " \"records\": [\n" +
+ " {\n" +
+ " \"data\": {\n" +
+ " \"Spuddate\": \"atspud\",\n" +
+ " \"UWI\": \"atuwi\",\n" +
+ " \"latitude\": \"latitude\",\n" +
+ " \"longitude\": \"longitude\"\n" +
+ " },\n" +
+ " \"meta\": null,\n" +
+ " \"id\": \"demo\",\n" +
+ " \"version\": demo,\n" +
+ " \"kind\": \"demo\",\n" +
+ " \"acl\": {\n" +
+ " \"viewers\": [\n" +
+ " \"demo\"\n" +
+ " ],\n" +
+ " \"owners\": [\n" +
+ " \"demo\"\n" +
+ " ]\n" +
+ " },\n" +
+ " \"legal\": {\n" +
+ " \"legaltags\": [\n" +
+ " \"opendes-test-tag\"\n" +
+ " ],\n" +
+ " \"otherRelevantDataCountries\": [\n" +
+ " \"BR\"\n" +
+ " ],\n" +
+ " \"status\": \"compliant\"\n" +
+ " },\n" +
+ " \"createUser\": \"demo\",\n" +
+ " \"createTime\": \"demo\"\n" +
+ " }\n" +
+ " ],\n" +
+ " \"notFound\": [\n" +
+ " \"demo\"\n" +
+ " ],\n" +
+ " \"conversionStatuses\": []\n" +
+ "}";
+
+ private static final String JSON2 = "{\n" +
+ " \"records\" :[],\n" +
+ " \"conversionStatuses\":[]\n" +
+ "}";
+
+ private static final String BATCH_API_URL = "https://demo/api/storage/v2/query/records:batch";
+ private static final String STORAGE_API_URL = "https://demo/api/storage/v2/schemas";
+ private static final String SCHEMA_API_URL = "https://demo/api/schema-service/v1/schema/osdu:file:gom:1.0.0";
+
+ @Test
+ public void shouldRetry_ForJSON1_when_storageQueryRecordCallIsMade() throws Exception {
+ response.setBody(JSON1);
+ httpRequest.setUrl(BATCH_API_URL);
+ when(this.retryPolicy.retryConfig(any())).thenReturn(new RetryPolicy().retryConfig(response -> this.retryPolicy.batchRetryPolicy(response)));
+ when(urlFetchService.sendRequest(httpRequest)).thenReturn(response);
+
+ urlFetchServiceAzure.sendRequest(httpRequest);
+
+ verify(urlFetchService, atMost(4)).sendRequest(httpRequest);
+ }
+
+ @Test
+ public void shouldRetry_ForJSON1_when_schemaRecordCallIsMade() throws Exception {
+ response.setBody(JSON1);
+ httpRequest.setUrl(SCHEMA_API_URL);
+ when(this.retryPolicy.retryConfig(any())).thenReturn(new RetryPolicy().retryConfig(response -> this.retryPolicy.schemaRetryPolicy(response)));
+ when(urlFetchService.sendRequest(httpRequest)).thenReturn(response);
+
+ urlFetchServiceAzure.sendRequest(httpRequest);
+
+ verify(urlFetchService, atMost(4)).sendRequest(httpRequest);
+ }
+
+ @Test
+ public void shouldRetry_when_anyOtherCallIsMade() throws Exception {
+ response.setBody(JSON2);
+ httpRequest.setUrl(STORAGE_API_URL);
+ when(this.retryPolicy.retryConfig(any())).thenReturn(new RetryPolicy().retryConfig(response -> this.retryPolicy.defaultRetryPolicy(response)));
+ when(urlFetchService.sendRequest(httpRequest)).thenReturn(response);
+
+ urlFetchServiceAzure.sendRequest(httpRequest);
+
+ verify(urlFetchService, atMost(4)).sendRequest(httpRequest);
+ }
+}
diff --git a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java
index 50ff8d892..bd698e00c 100644
--- a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java
+++ b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java
@@ -1,53 +1,53 @@
-package org.opengroup.osdu.indexer.middleware;
-
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.mockito.Mockito;
-import org.mockito.junit.MockitoJUnitRunner;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-
-import javax.servlet.FilterChain;
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import java.io.IOException;
-import java.util.Collections;
-
-@RunWith(MockitoJUnitRunner.class)
-public class IndexFilterTest {
-
- @InjectMocks
- private IndexFilter indexFilter;
-
- @Mock
- private DpsHeaders dpsHeaders;
-
- @Test
- public void shouldSetCorrectResponseHeaders() throws IOException, ServletException {
- HttpServletRequest httpServletRequest = Mockito.mock(HttpServletRequest.class);
- HttpServletResponse httpServletResponse = Mockito.mock(HttpServletResponse.class);
- FilterChain filterChain = Mockito.mock(FilterChain.class);
- Mockito.when(httpServletRequest.getRequestURI()).thenReturn("https://test.com");
- Mockito.when(httpServletRequest.getMethod()).thenReturn("POST");
- Mockito.when(dpsHeaders.getCorrelationId()).thenReturn("correlation-id-value");
-
- indexFilter.doFilter(httpServletRequest, httpServletResponse, filterChain);
-
- Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Origin", Collections.singletonList("*").toString());
- Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Headers", Collections.singletonList("origin, content-type, accept, authorization, data-partition-id, correlation-id, appkey").toString());
- Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Methods", Collections.singletonList("GET, POST, PUT, DELETE, OPTIONS, HEAD, PATCH").toString());
- Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Credentials", Collections.singletonList("true").toString());
- Mockito.verify(httpServletResponse).addHeader("X-Frame-Options", Collections.singletonList("DENY").toString());
- Mockito.verify(httpServletResponse).addHeader("X-XSS-Protection", Collections.singletonList("1; mode=block").toString());
- Mockito.verify(httpServletResponse).addHeader("X-Content-Type-Options", Collections.singletonList("nosniff").toString());
- Mockito.verify(httpServletResponse).addHeader("Cache-Control", Collections.singletonList("no-cache, no-store, must-revalidate").toString());
- Mockito.verify(httpServletResponse).addHeader("Content-Security-Policy", Collections.singletonList("default-src 'self'").toString());
- Mockito.verify(httpServletResponse).addHeader("Strict-Transport-Security", Collections.singletonList("max-age=31536000; includeSubDomains").toString());
- Mockito.verify(httpServletResponse).addHeader("Expires", Collections.singletonList("0").toString());
- Mockito.verify(httpServletResponse).addHeader("correlation-id", "correlation-id-value");
- Mockito.verify(filterChain).doFilter(httpServletRequest, httpServletResponse);
- }
-}
+package org.opengroup.osdu.indexer.middleware;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.MockitoJUnitRunner;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+
+import javax.servlet.FilterChain;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import java.io.IOException;
+import java.util.Collections;
+
+@RunWith(MockitoJUnitRunner.class)
+public class IndexFilterTest {
+
+ @InjectMocks
+ private IndexFilter indexFilter;
+
+ @Mock
+ private DpsHeaders dpsHeaders;
+
+ @Test
+ public void shouldSetCorrectResponseHeaders() throws IOException, ServletException {
+ HttpServletRequest httpServletRequest = Mockito.mock(HttpServletRequest.class);
+ HttpServletResponse httpServletResponse = Mockito.mock(HttpServletResponse.class);
+ FilterChain filterChain = Mockito.mock(FilterChain.class);
+ Mockito.when(httpServletRequest.getRequestURI()).thenReturn("https://test.com");
+ Mockito.when(httpServletRequest.getMethod()).thenReturn("POST");
+ Mockito.when(dpsHeaders.getCorrelationId()).thenReturn("correlation-id-value");
+
+ indexFilter.doFilter(httpServletRequest, httpServletResponse, filterChain);
+
+ Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Origin", Collections.singletonList("*").toString());
+ Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Headers", Collections.singletonList("origin, content-type, accept, authorization, data-partition-id, correlation-id, appkey").toString());
+ Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Methods", Collections.singletonList("GET, POST, PUT, DELETE, OPTIONS, HEAD, PATCH").toString());
+ Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Credentials", Collections.singletonList("true").toString());
+ Mockito.verify(httpServletResponse).addHeader("X-Frame-Options", Collections.singletonList("DENY").toString());
+ Mockito.verify(httpServletResponse).addHeader("X-XSS-Protection", Collections.singletonList("1; mode=block").toString());
+ Mockito.verify(httpServletResponse).addHeader("X-Content-Type-Options", Collections.singletonList("nosniff").toString());
+ Mockito.verify(httpServletResponse).addHeader("Cache-Control", Collections.singletonList("no-cache, no-store, must-revalidate").toString());
+ Mockito.verify(httpServletResponse).addHeader("Content-Security-Policy", Collections.singletonList("default-src 'self'").toString());
+ Mockito.verify(httpServletResponse).addHeader("Strict-Transport-Security", Collections.singletonList("max-age=31536000; includeSubDomains").toString());
+ Mockito.verify(httpServletResponse).addHeader("Expires", Collections.singletonList("0").toString());
+ Mockito.verify(httpServletResponse).addHeader("correlation-id", "correlation-id-value");
+ Mockito.verify(filterChain).doFilter(httpServletRequest, httpServletResponse);
+ }
+}
diff --git a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java
index f8debf207..87dea8ec4 100644
--- a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java
+++ b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java
@@ -1,303 +1,303 @@
-package org.opengroup.osdu.indexer.service;
-
-import org.apache.http.StatusLine;
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest;
-import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
-import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
-import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
-import org.elasticsearch.action.support.master.AcknowledgedResponse;
-import org.elasticsearch.client.*;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.index.reindex.BulkByScrollResponse;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.ArgumentMatchers;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
-import org.opengroup.osdu.indexer.service.IndexerMappingServiceImpl;
-import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.indexer.util.ElasticClientHandler;
-import org.powermock.api.mockito.PowerMockito;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.springframework.test.context.junit4.SpringRunner;
-
-import java.io.IOException;
-import java.util.*;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.powermock.api.mockito.PowerMockito.when;
-
-@Ignore
-@RunWith(SpringRunner.class)
-@PrepareForTest({ RestHighLevelClient.class, IndicesClient.class })
-public class IndexerMappingServiceTest {
-
- private final String kind = "tenant:test:test:1.0.0";
- private final String index = "tenant-test-test-1.0.0";
- private final String type = "test";
- private final String mappingValid = "{\"dynamic\":false,\"properties\":{\"data\":{\"properties\":{\"Location\":{\"type\":\"geo_point\"}}},\"id\":{\"type\":\"keyword\"}}}";
-
- @Mock
- private RestClient restClient;
- @Mock
- private Response response;
- @Mock
- private StatusLine statusLine;
-
- @InjectMocks
- private IndexerMappingServiceImpl sut;
-
- @Mock
- private ElasticClientHandler elasticClientHandler;
-
- @InjectMocks
- private RestHighLevelClient restHighLevelClient;
-
- @InjectMocks
- private IndexSchema indexSchema;
- @InjectMocks
- private IndicesClient indicesClient;
-
- @InjectMocks
- private AcknowledgedResponse mappingResponse;
-
- @Before
- public void setup() throws IOException {
- Map<String, Object> dataMapping = new HashMap<>();
- dataMapping.put("Location", "geo_point");
- Map<String, Object> metaMapping = new HashMap<>();
- metaMapping.put(RecordMetaAttribute.ID.getValue(), "keyword");
- this.indexSchema = IndexSchema.builder().kind(kind).type(type).dataSchema(dataMapping).metaSchema(metaMapping)
- .build();
-
- this.indicesClient = PowerMockito.mock(IndicesClient.class);
- this.restHighLevelClient = PowerMockito.mock(RestHighLevelClient.class);
-
- when(this.restHighLevelClient.getLowLevelClient()).thenReturn(restClient);
- when(this.restClient.performRequest(ArgumentMatchers.any())).thenReturn(response);
- when(this.response.getStatusLine()).thenReturn(statusLine);
- when(this.statusLine.getStatusCode()).thenReturn(200);
- }
-
- @Test
- public void should_returnValidMapping_givenFalseMerge_createMappingTest() {
- try {
- String mapping = this.sut.createMapping(restHighLevelClient, indexSchema, index, false);
- assertEquals(mappingValid, mapping);
- } catch (Exception e) {
- fail("Should not throw this exception" + e.getMessage());
- }
- }
-
- @Test
- public void should_returnValidMapping_givenTrueMerge_createMappingTest() {
- try {
- doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
-
- String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true);
- assertEquals(this.mappingValid, mapping);
- } catch (Exception e) {
- fail("Should not throw this exception" + e.getMessage());
- }
- }
-
- @Test
- public void should_returnValidMapping_givenExistType_createMappingTest() {
- try {
- doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
-
- IndexerMappingServiceImpl indexerMappingServiceLocal = PowerMockito.spy(new IndexerMappingServiceImpl());
- doReturn(false).when(indexerMappingServiceLocal).isTypeExist(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any());
- String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true);
- assertEquals(this.mappingValid, mapping);
- } catch (Exception e) {
- fail("Should not throw this exception" + e.getMessage());
- }
- }
-
- @Test
- public void should_update_indices_field_with_keyword_when_valid_indices() throws Exception {
- try {
- Set<String> indices = new HashSet<String>();
- indices.add("indices 1");
- GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
- doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
- XContentBuilder builder = XContentFactory.jsonBuilder();
- builder.startObject();
- builder.field("any field", new HashMap());
- builder.endObject();
- BytesReference bytesReference = BytesReference.bytes(builder);
- GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
- Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
- mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
- mappingBuilder.put("any index 1", mapBuilder);
- mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
- mapping.put("indices 1", mappingBuilder);
- when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
- doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
- BulkByScrollResponse response = mock(BulkByScrollResponse.class);
- doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
- when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
- when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
-
- this.sut.updateIndexMappingForIndicesOfSameType( indices,"any field");
- } catch (Exception e) {
- fail("Should not throw this exception" + e.getMessage());
- }
- }
-
- @Test(expected = AppException.class)
- public void should_throw_exception_if_someIndex_is_invalid_andWeIndexfield_with_keyword() throws Exception {
- try {
- Set<String> indices = new HashSet<String>();
- indices.add("invalid 1");
- GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
- doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
- XContentBuilder builder = XContentFactory.jsonBuilder();
- builder.startObject();
- builder.field("any field", new HashMap());
- builder.endObject();
- BytesReference bytesReference = BytesReference.bytes(builder);
- GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
- Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
- mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
- mappingBuilder.put("any index 1", mapBuilder);
- mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
- mapping.put("indices 1", mappingBuilder);
- when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
- doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
- BulkByScrollResponse response = mock(BulkByScrollResponse.class);
- doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
- when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
- when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
-
- this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
- } catch (Exception e) {
- throw e;
- }
- }
-
- @Test(expected = AppException.class)
- public void should_throw_exception_if_type_of_index_is_invalid_andWeIndexfield_with_keyword() throws Exception {
- try {
- Set<String> indices = new HashSet<String>();
- indices.add("indices 1");
- GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
- doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
- XContentBuilder builder = XContentFactory.jsonBuilder();
- builder.startObject();
- builder.field("any field", new HashMap());
- builder.endObject();
- BytesReference bytesReference = BytesReference.bytes(builder);
- GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
- Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
- mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
- mappingBuilder.put("any index 1", mapBuilder);
- mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
- mapping.put("indices 1", mappingBuilder);
- when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
- doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
- BulkByScrollResponse response = mock(BulkByScrollResponse.class);
- doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
- when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
- when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
- this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field invalid");
- } catch (Exception e) {
- throw e;
- }
- }
-
- @Test(expected = AppException.class)
- public void should_throw_exception_if_elastic_search_failedToFetch_andWeIndexfield_with_keyword() throws Exception {
- try {
-
- Set<String> indices = new HashSet<String>();
- indices.add("indices 1");
- indices.add("indices Invalid");
- GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
- doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenThrow(new ElasticsearchException(""));
- XContentBuilder builder = XContentFactory.jsonBuilder();
- builder.startObject();
- builder.field("any field", new HashMap());
- builder.endObject();
- BytesReference bytesReference = BytesReference.bytes(builder);
- GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
- Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
- mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
- mappingBuilder.put("any index 1", mapBuilder);
- mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
- mapping.put("indices 1", mappingBuilder);
- when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
- doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
- BulkByScrollResponse response = mock(BulkByScrollResponse.class);
- doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
- when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
- when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
- this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
- } catch (AppException e) {
- throw e;
- }
- }
-
- @Test(expected = AppException.class)
- public void should_throw_exception_when_elastic_failedToIndex_indices_field_with_keyword() {
- try {
- Set<String> indices = new HashSet<String>();
- indices.add("indices 1");
- indices.add("indices Invalid");
- GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
- doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
- XContentBuilder builder = XContentFactory.jsonBuilder();
- builder.startObject();
- builder.field("any field", new HashMap());
- builder.endObject();
- BytesReference bytesReference = BytesReference.bytes(builder);
- GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
- Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
- mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
- mappingBuilder.put("any index 1", mapBuilder);
- mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
- mapping.put("indices 1", mappingBuilder);
- when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
- doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
- BulkByScrollResponse response = mock(BulkByScrollResponse.class);
- doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
- when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
- when(this.indicesClient.putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class))).thenThrow(new ElasticsearchException(""));
- when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
- this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
- } catch (AppException e) {
- throw e;
- } catch (Exception e) {
- fail("Should not throw this exception" + e.getMessage());
- }
- }
-}
+package org.opengroup.osdu.indexer.service;
+
+import org.apache.http.StatusLine;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest;
+import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
+import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.*;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.reindex.BulkByScrollResponse;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentMatchers;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
+import org.opengroup.osdu.indexer.service.IndexerMappingServiceImpl;
+import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.indexer.util.ElasticClientHandler;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.io.IOException;
+import java.util.*;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@Ignore
+@RunWith(SpringRunner.class)
+@PrepareForTest({ RestHighLevelClient.class, IndicesClient.class })
+public class IndexerMappingServiceTest {
+
+ private final String kind = "tenant:test:test:1.0.0";
+ private final String index = "tenant-test-test-1.0.0";
+ private final String type = "test";
+ private final String mappingValid = "{\"dynamic\":false,\"properties\":{\"data\":{\"properties\":{\"Location\":{\"type\":\"geo_point\"}}},\"id\":{\"type\":\"keyword\"}}}";
+
+ @Mock
+ private RestClient restClient;
+ @Mock
+ private Response response;
+ @Mock
+ private StatusLine statusLine;
+
+ @InjectMocks
+ private IndexerMappingServiceImpl sut;
+
+ @Mock
+ private ElasticClientHandler elasticClientHandler;
+
+ @InjectMocks
+ private RestHighLevelClient restHighLevelClient;
+
+ @InjectMocks
+ private IndexSchema indexSchema;
+ @InjectMocks
+ private IndicesClient indicesClient;
+
+ @InjectMocks
+ private AcknowledgedResponse mappingResponse;
+
+ @Before
+ public void setup() throws IOException {
+ Map<String, Object> dataMapping = new HashMap<>();
+ dataMapping.put("Location", "geo_point");
+ Map<String, Object> metaMapping = new HashMap<>();
+ metaMapping.put(RecordMetaAttribute.ID.getValue(), "keyword");
+ this.indexSchema = IndexSchema.builder().kind(kind).type(type).dataSchema(dataMapping).metaSchema(metaMapping)
+ .build();
+
+ this.indicesClient = PowerMockito.mock(IndicesClient.class);
+ this.restHighLevelClient = PowerMockito.mock(RestHighLevelClient.class);
+
+ when(this.restHighLevelClient.getLowLevelClient()).thenReturn(restClient);
+ when(this.restClient.performRequest(ArgumentMatchers.any())).thenReturn(response);
+ when(this.response.getStatusLine()).thenReturn(statusLine);
+ when(this.statusLine.getStatusCode()).thenReturn(200);
+ }
+
+ @Test
+ public void should_returnValidMapping_givenFalseMerge_createMappingTest() {
+ try {
+ String mapping = this.sut.createMapping(restHighLevelClient, indexSchema, index, false);
+ assertEquals(mappingValid, mapping);
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_returnValidMapping_givenTrueMerge_createMappingTest() {
+ try {
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
+
+ String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true);
+ assertEquals(this.mappingValid, mapping);
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_returnValidMapping_givenExistType_createMappingTest() {
+ try {
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
+
+ IndexerMappingServiceImpl indexerMappingServiceLocal = PowerMockito.spy(new IndexerMappingServiceImpl());
+ doReturn(false).when(indexerMappingServiceLocal).isTypeExist(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any());
+ String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true);
+ assertEquals(this.mappingValid, mapping);
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_update_indices_field_with_keyword_when_valid_indices() throws Exception {
+ try {
+ Set<String> indices = new HashSet<String>();
+ indices.add("indices 1");
+ GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
+ XContentBuilder builder = XContentFactory.jsonBuilder();
+ builder.startObject();
+ builder.field("any field", new HashMap());
+ builder.endObject();
+ BytesReference bytesReference = BytesReference.bytes(builder);
+ GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
+ mapBuilder.put("data.any field", mappingMetaData);
+ Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
+ mappingBuilder.put("any index 1", mapBuilder);
+ mappingBuilder.put("any index 2", mapBuilder);
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
+ mapping.put("indices 1", mappingBuilder);
+ when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
+ BulkByScrollResponse response = mock(BulkByScrollResponse.class);
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+
+ this.sut.updateIndexMappingForIndicesOfSameType( indices,"any field");
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throw_exception_if_someIndex_is_invalid_andWeIndexfield_with_keyword() throws Exception {
+ try {
+ Set<String> indices = new HashSet<String>();
+ indices.add("invalid 1");
+ GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
+ XContentBuilder builder = XContentFactory.jsonBuilder();
+ builder.startObject();
+ builder.field("any field", new HashMap());
+ builder.endObject();
+ BytesReference bytesReference = BytesReference.bytes(builder);
+ GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
+ mapBuilder.put("data.any field", mappingMetaData);
+ Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
+ mappingBuilder.put("any index 1", mapBuilder);
+ mappingBuilder.put("any index 2", mapBuilder);
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
+ mapping.put("indices 1", mappingBuilder);
+ when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
+ BulkByScrollResponse response = mock(BulkByScrollResponse.class);
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+
+ this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
+ } catch (Exception e) {
+ throw e;
+ }
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throw_exception_if_type_of_index_is_invalid_andWeIndexfield_with_keyword() throws Exception {
+ try {
+ Set<String> indices = new HashSet<String>();
+ indices.add("indices 1");
+ GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
+ XContentBuilder builder = XContentFactory.jsonBuilder();
+ builder.startObject();
+ builder.field("any field", new HashMap());
+ builder.endObject();
+ BytesReference bytesReference = BytesReference.bytes(builder);
+ GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
+ mapBuilder.put("data.any field", mappingMetaData);
+ Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
+ mappingBuilder.put("any index 1", mapBuilder);
+ mappingBuilder.put("any index 2", mapBuilder);
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
+ mapping.put("indices 1", mappingBuilder);
+ when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
+ BulkByScrollResponse response = mock(BulkByScrollResponse.class);
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field invalid");
+ } catch (Exception e) {
+ throw e;
+ }
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throw_exception_if_elastic_search_failedToFetch_andWeIndexfield_with_keyword() throws Exception {
+ try {
+
+ Set<String> indices = new HashSet<String>();
+ indices.add("indices 1");
+ indices.add("indices Invalid");
+ GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenThrow(new ElasticsearchException(""));
+ XContentBuilder builder = XContentFactory.jsonBuilder();
+ builder.startObject();
+ builder.field("any field", new HashMap());
+ builder.endObject();
+ BytesReference bytesReference = BytesReference.bytes(builder);
+ GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
+ mapBuilder.put("data.any field", mappingMetaData);
+ Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
+ mappingBuilder.put("any index 1", mapBuilder);
+ mappingBuilder.put("any index 2", mapBuilder);
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
+ mapping.put("indices 1", mappingBuilder);
+ when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
+ BulkByScrollResponse response = mock(BulkByScrollResponse.class);
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
+ } catch (AppException e) {
+ throw e;
+ }
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throw_exception_when_elastic_failedToIndex_indices_field_with_keyword() {
+ try {
+ Set<String> indices = new HashSet<String>();
+ indices.add("indices 1");
+ indices.add("indices Invalid");
+ GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
+ XContentBuilder builder = XContentFactory.jsonBuilder();
+ builder.startObject();
+ builder.field("any field", new HashMap());
+ builder.endObject();
+ BytesReference bytesReference = BytesReference.bytes(builder);
+ GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
+ mapBuilder.put("data.any field", mappingMetaData);
+ Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
+ mappingBuilder.put("any index 1", mapBuilder);
+ mappingBuilder.put("any index 2", mapBuilder);
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
+ mapping.put("indices 1", mappingBuilder);
+ when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
+ BulkByScrollResponse response = mock(BulkByScrollResponse.class);
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
+ when(this.indicesClient.putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class))).thenThrow(new ElasticsearchException(""));
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
+ } catch (AppException e) {
+ throw e;
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+}
diff --git a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java
index 4ba73dacf..403199cb3 100644
--- a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java
+++ b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java
@@ -1,150 +1,150 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.service;
-
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
-import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
-import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
-import org.powermock.modules.junit4.PowerMockRunner;
-import org.powermock.modules.junit4.PowerMockRunnerDelegate;
-import org.springframework.test.context.junit4.SpringRunner;
-
-import java.util.*;
-
-import static org.junit.Assert.fail;
-import static org.mockito.Matchers.any;
-import static org.mockito.MockitoAnnotations.initMocks;
-import static org.powermock.api.mockito.PowerMockito.mockStatic;
-import static org.powermock.api.mockito.PowerMockito.when;
-
-@RunWith(PowerMockRunner.class)
-@PowerMockRunnerDelegate(SpringRunner.class)
-public class ReindexServiceTest {
-
- private final String cursor = "100";
-
- private final String correlationId = UUID.randomUUID().toString();
-
- @Mock
- private IndexerConfigurationProperties configurationProperties;
- @Mock
- private StorageService storageService;
- @Mock
- private IRequestInfo requestInfo;
- @Mock
- private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
- @Mock
- private JaxRsDpsLog log;
- @InjectMocks
- private ReindexServiceImpl sut;
-
- private RecordReindexRequest recordReindexRequest;
- private RecordQueryResponse recordQueryResponse;
-
- private Map<String, String> httpHeaders;
-
- @Before
- public void setup() {
- initMocks(this);
-
- mockStatic(UUID.class);
-
- recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor(cursor).build();
- recordQueryResponse = new RecordQueryResponse();
-
- httpHeaders = new HashMap<>();
- httpHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
- httpHeaders.put(DpsHeaders.CORRELATION_ID, correlationId);
- DpsHeaders standardHeaders = DpsHeaders.createFromMap(httpHeaders);
- when(requestInfo.getHeaders()).thenReturn(standardHeaders);
- when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(httpHeaders);
- when(requestInfo.getHeadersWithDwdAuthZ()).thenReturn(standardHeaders);
- }
-
- @Test
- public void should_returnNull_givenNullResponseResult_reIndexRecordsTest() {
- try {
- recordQueryResponse.setResults(null);
- when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
-
- String response = sut.reindexRecords(recordReindexRequest, false);
-
- Assert.assertNull(response);
- } catch (Exception e) {
- fail("Should not throw this exception" + e.getMessage());
- }
- }
-
- @Test
- public void should_returnNull_givenEmptyResponseResult_reIndexRecordsTest() {
- try {
- recordQueryResponse.setResults(new ArrayList<>());
- when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
-
- String response = sut.reindexRecords(recordReindexRequest, false);
-
- Assert.assertNull(response);
- } catch (Exception e) {
- fail("Should not throw this exception" + e.getMessage());
- }
- }
- @Ignore
- @Test
- public void should_returnRecordQueryRequestPayload_givenValidResponseResult_reIndexRecordsTest() {
- try {
- recordQueryResponse.setCursor(cursor);
- List<String> results = new ArrayList<>();
- results.add("test1");
- recordQueryResponse.setResults(results);
-
- when(configurationProperties.getStorageRecordsBatchSize()).thenReturn(1);
-
- when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
-
- String taskQueuePayload = sut.reindexRecords(recordReindexRequest, false);
-
- Assert.assertEquals("{\"kind\":\"tenant:test:test:1.0.0\",\"cursor\":\"100\"}", taskQueuePayload);
- } catch (Exception e) {
- fail("Should not throw exception" + e.getMessage());
- }
- }
-
- @Test
- public void should_returnRecordChangedMessage_givenValidResponseResult_reIndexRecordsTest() {
- try {
- List<String> results = new ArrayList<>();
- results.add("test1");
- recordQueryResponse.setResults(results);
- when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
-
- String taskQueuePayload = sut.reindexRecords(recordReindexRequest, false);
-
- Assert.assertEquals(String.format("{\"data\":\"[{\\\"id\\\":\\\"test1\\\",\\\"kind\\\":\\\"tenant:test:test:1.0.0\\\",\\\"op\\\":\\\"create\\\"}]\",\"attributes\":{\"correlation-id\":\"%s\"}}", correlationId), taskQueuePayload);
- } catch (Exception e) {
- fail("Should not throw exception" + e.getMessage());
- }
- }
-}
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.service;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
+import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
+import org.powermock.modules.junit4.PowerMockRunner;
+import org.powermock.modules.junit4.PowerMockRunnerDelegate;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.util.*;
+
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.MockitoAnnotations.initMocks;
+import static org.powermock.api.mockito.PowerMockito.mockStatic;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@RunWith(PowerMockRunner.class)
+@PowerMockRunnerDelegate(SpringRunner.class)
+public class ReindexServiceTest {
+
+ private final String cursor = "100";
+
+ private final String correlationId = UUID.randomUUID().toString();
+
+ @Mock
+ private IndexerConfigurationProperties configurationProperties;
+ @Mock
+ private StorageService storageService;
+ @Mock
+ private IRequestInfo requestInfo;
+ @Mock
+ private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
+ @Mock
+ private JaxRsDpsLog log;
+ @InjectMocks
+ private ReindexServiceImpl sut;
+
+ private RecordReindexRequest recordReindexRequest;
+ private RecordQueryResponse recordQueryResponse;
+
+ private Map<String, String> httpHeaders;
+
+ @Before
+ public void setup() {
+ initMocks(this);
+
+ mockStatic(UUID.class);
+
+ recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor(cursor).build();
+ recordQueryResponse = new RecordQueryResponse();
+
+ httpHeaders = new HashMap<>();
+ httpHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
+ httpHeaders.put(DpsHeaders.CORRELATION_ID, correlationId);
+ DpsHeaders standardHeaders = DpsHeaders.createFromMap(httpHeaders);
+ when(requestInfo.getHeaders()).thenReturn(standardHeaders);
+ when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(httpHeaders);
+ when(requestInfo.getHeadersWithDwdAuthZ()).thenReturn(standardHeaders);
+ }
+
+ @Test
+ public void should_returnNull_givenNullResponseResult_reIndexRecordsTest() {
+ try {
+ recordQueryResponse.setResults(null);
+ when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
+
+ String response = sut.reindexRecords(recordReindexRequest, false);
+
+ Assert.assertNull(response);
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_returnNull_givenEmptyResponseResult_reIndexRecordsTest() {
+ try {
+ recordQueryResponse.setResults(new ArrayList<>());
+ when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
+
+ String response = sut.reindexRecords(recordReindexRequest, false);
+
+ Assert.assertNull(response);
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+ @Ignore
+ @Test
+ public void should_returnRecordQueryRequestPayload_givenValidResponseResult_reIndexRecordsTest() {
+ try {
+ recordQueryResponse.setCursor(cursor);
+ List<String> results = new ArrayList<>();
+ results.add("test1");
+ recordQueryResponse.setResults(results);
+
+ when(configurationProperties.getStorageRecordsBatchSize()).thenReturn(1);
+
+ when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
+
+ String taskQueuePayload = sut.reindexRecords(recordReindexRequest, false);
+
+ Assert.assertEquals("{\"kind\":\"tenant:test:test:1.0.0\",\"cursor\":\"100\"}", taskQueuePayload);
+ } catch (Exception e) {
+ fail("Should not throw exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_returnRecordChangedMessage_givenValidResponseResult_reIndexRecordsTest() {
+ try {
+ List<String> results = new ArrayList<>();
+ results.add("test1");
+ recordQueryResponse.setResults(results);
+ when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
+
+ String taskQueuePayload = sut.reindexRecords(recordReindexRequest, false);
+
+ Assert.assertEquals(String.format("{\"data\":\"[{\\\"id\\\":\\\"test1\\\",\\\"kind\\\":\\\"tenant:test:test:1.0.0\\\",\\\"op\\\":\\\"create\\\"}]\",\"attributes\":{\"correlation-id\":\"%s\"}}", correlationId), taskQueuePayload);
+ } catch (Exception e) {
+ fail("Should not throw exception" + e.getMessage());
+ }
+ }
+}
diff --git a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceTest.java b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceTest.java
index 99bf8b099..3ff767ebe 100644
--- a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceTest.java
+++ b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceTest.java
@@ -1,254 +1,254 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.service;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
-
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.ArgumentMatchers;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.mockito.Mockito;
-import org.mockito.Spy;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.indexer.IndexingStatus;
-import org.opengroup.osdu.core.common.model.indexer.JobStatus;
-import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
-import org.opengroup.osdu.core.common.model.http.HttpResponse;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
-import org.opengroup.osdu.core.common.http.IUrlFetchService;
-import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
-import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
-import org.opengroup.osdu.core.common.model.indexer.Records;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.springframework.http.HttpStatus;
-import org.springframework.test.context.junit4.SpringRunner;
-
-import java.lang.reflect.Type;
-import java.net.URISyntaxException;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-
-import static java.util.Collections.singletonList;
-import static org.junit.Assert.*;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.powermock.api.mockito.PowerMockito.when;
-
-@RunWith(SpringRunner.class)
-public class StorageServiceTest {
-
- @Mock
- private IUrlFetchService urlFetchService;
- @Mock
- private JobStatus jobStatus;
- @Mock
- private JaxRsDpsLog log;
- @Mock
- private IRequestInfo requestInfo;
- @Mock
- private IndexerConfigurationProperties configurationProperties;
- @Spy
- private ObjectMapper objectMapper = new ObjectMapper();
- @InjectMocks
- private StorageServiceImpl sut;
-
- private List<String> ids;
- private static final String RECORD_ID1 = "tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465";
- private static final String RECORDS_ID2 = "tenant1:doc:15e790a69beb4d789b1f979e2af2e813";
-
- @Before
- public void setup() {
-
- String recordChangedMessages = "[{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"purge\"}," +
- "{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"create\"}]";
-
- when(this.requestInfo.getHeadersMap()).thenReturn(new HashMap<>());
- when(this.requestInfo.getHeaders()).thenReturn(new DpsHeaders());
-
- Type listType = new TypeToken<List<RecordInfo>>() {}.getType();
-
- List<RecordInfo> msgs = (new Gson()).fromJson(recordChangedMessages, listType);
- jobStatus.initialize(msgs);
- ids = Arrays.asList(RECORD_ID1, RECORDS_ID2);
-
- when(configurationProperties.getStorageRecordsBatchSize()).thenReturn(20);
- }
-
- @Test
- public void should_return404_givenNullData_getValidStorageRecordsTest() throws URISyntaxException {
-
- HttpResponse httpResponse = mock(HttpResponse.class);
- Mockito.when(httpResponse.getBody()).thenReturn(null);
-
- when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
-
- should_return404_getValidStorageRecordsTest();
- }
-
- @Test
- public void should_return404_givenEmptyData_getValidStorageRecordsTest() throws URISyntaxException {
-
- String emptyDataFromStorage = "{\"records\":[],\"notFound\":[]}";
-
- HttpResponse httpResponse = mock(HttpResponse.class);
- Mockito.when(httpResponse.getBody()).thenReturn(emptyDataFromStorage);
-
- when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
-
- should_return404_getValidStorageRecordsTest();
- }
-
- @Test
- public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsTest() throws URISyntaxException {
-
- String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"], \"conversionStatuses\": []}";
-
- HttpResponse httpResponse = mock(HttpResponse.class);
- Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
-
- when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
- Records storageRecords = this.sut.getStorageRecords(ids);
-
- assertEquals(1, storageRecords.getRecords().size());
- }
-
- @Test
- public void should_logMissingRecord_given_storageMissedRecords() throws URISyntaxException {
-
- String validDataFromStorage = "{\"records\":[{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[]}";
-
- HttpResponse httpResponse = mock(HttpResponse.class);
- Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
-
- when(this.urlFetchService.sendRequest(any())).thenReturn(httpResponse);
- Records storageRecords = this.sut.getStorageRecords(ids);
-
- assertEquals(1, storageRecords.getRecords().size());
- verify(this.jobStatus).addOrUpdateRecordStatus(singletonList(RECORDS_ID2), IndexingStatus.FAIL, HttpStatus.NOT_FOUND.value(), "Partial response received from Storage service - missing records", "Partial response received from Storage service: tenant1:doc:15e790a69beb4d789b1f979e2af2e813");
- }
-
- @Test
- public void should_returnValidJobStatus_givenFailedUnitsConversion_processRecordChangedMessageTest() throws URISyntaxException {
- String validDataFromStorage = "{\"records\":[{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[],\"conversionStatuses\":[{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\",\"status\":\"ERROR\",\"errors\":[\"crs conversion failed\"]}]}";
-
- HttpResponse httpResponse = mock(HttpResponse.class);
- Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
-
- when(this.urlFetchService.sendRequest(any())).thenReturn(httpResponse);
- Records storageRecords = this.sut.getStorageRecords(singletonList(RECORDS_ID2));
-
- assertEquals(1, storageRecords.getRecords().size());
- verify(this.jobStatus).addOrUpdateRecordStatus(RECORDS_ID2, IndexingStatus.WARN, HttpStatus.BAD_REQUEST.value(), "crs conversion failed", String.format("record-id: %s | %s", "tenant1:doc:15e790a69beb4d789b1f979e2af2e813", "crs conversion failed"));
- }
-
- @Test
- public void should_returnValidResponse_givenValidRecordQueryRequest_getRecordListByKind() throws Exception {
-
- RecordReindexRequest recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor("100").build();
-
- HttpResponse httpResponse = new HttpResponse();
- httpResponse.setBody(new Gson().toJson(recordReindexRequest, RecordReindexRequest.class));
-
- when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
-
- RecordQueryResponse recordQueryResponse = this.sut.getRecordsByKind(recordReindexRequest);
-
- assertEquals("100", recordQueryResponse.getCursor());
- assertNull(recordQueryResponse.getResults());
- }
-
- @Test
- public void should_returnValidResponse_givenValidKind_getSchemaByKind() throws Exception {
-
- String validSchemaFromStorage = "{" +
- " \"kind\": \"tenant:test:test:1.0.0\"," +
- " \"schema\": [" +
- " {" +
- " \"path\": \"msg\"," +
- " \"kind\": \"string\"" +
- " }," +
- " {" +
- " \"path\": \"references.entity\"," +
- " \"kind\": \"string\"" +
- " }" +
- " ]," +
- " \"ext\": null" +
- "}";
- String kind = "tenant:test:test:1.0.0";
-
- HttpResponse httpResponse = new HttpResponse();
- httpResponse.setResponseCode(HttpStatus.OK.value());
- httpResponse.setBody(validSchemaFromStorage);
-
- when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
-
- String recordSchemaResponse = this.sut.getStorageSchema(kind);
-
- assertNotNull(recordSchemaResponse);
- }
-
- @Test
- public void should_returnNullResponse_givenAbsentKind_getSchemaByKind() throws Exception {
-
- String kind = "tenant:test:test:1.0.0";
-
- HttpResponse httpResponse = new HttpResponse();
- httpResponse.setResponseCode(HttpStatus.NOT_FOUND.value());
-
- when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
-
- String recordSchemaResponse = this.sut.getStorageSchema(kind);
-
- assertNull(recordSchemaResponse);
- }
-
- @Test
- public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsWithInvalidConversionTest() throws URISyntaxException {
-
- String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"],\"conversionStatuses\": [{\"id\":\"testid\",\"status\":\"ERROR\",\"errors\":[\"conversion error occurred\"] } ]}";
-
- HttpResponse httpResponse = mock(HttpResponse.class);
- Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
-
- when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
- Records storageRecords = this.sut.getStorageRecords(ids);
-
- assertEquals(1, storageRecords.getRecords().size());
-
- assertEquals(1, storageRecords.getConversionStatuses().get(0).getErrors().size());
-
- assertEquals("conversion error occurred", storageRecords.getConversionStatuses().get(0).getErrors().get(0));
- }
-
- private void should_return404_getValidStorageRecordsTest() {
- try {
- this.sut.getStorageRecords(ids);
- fail("Should throw exception");
- } catch (AppException e) {
- assertEquals(HttpStatus.NOT_FOUND.value(), e.getError().getCode());
- } catch (Exception e) {
- fail("Should not throw this exception" + e.getMessage());
- }
- }
-}
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.service;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentMatchers;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.Spy;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.indexer.IndexingStatus;
+import org.opengroup.osdu.core.common.model.indexer.JobStatus;
+import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
+import org.opengroup.osdu.core.common.model.http.HttpResponse;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.opengroup.osdu.core.common.http.IUrlFetchService;
+import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
+import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
+import org.opengroup.osdu.core.common.model.indexer.Records;
+import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.springframework.http.HttpStatus;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.lang.reflect.Type;
+import java.net.URISyntaxException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+
+import static java.util.Collections.singletonList;
+import static org.junit.Assert.*;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@RunWith(SpringRunner.class)
+public class StorageServiceTest {
+
+ @Mock
+ private IUrlFetchService urlFetchService;
+ @Mock
+ private JobStatus jobStatus;
+ @Mock
+ private JaxRsDpsLog log;
+ @Mock
+ private IRequestInfo requestInfo;
+ @Mock
+ private IndexerConfigurationProperties configurationProperties;
+ @Spy
+ private ObjectMapper objectMapper = new ObjectMapper();
+ @InjectMocks
+ private StorageServiceImpl sut;
+
+ private List<String> ids;
+ private static final String RECORD_ID1 = "tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465";
+ private static final String RECORDS_ID2 = "tenant1:doc:15e790a69beb4d789b1f979e2af2e813";
+
+ @Before
+ public void setup() {
+
+ String recordChangedMessages = "[{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"purge\"}," +
+ "{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"create\"}]";
+
+ when(this.requestInfo.getHeadersMap()).thenReturn(new HashMap<>());
+ when(this.requestInfo.getHeaders()).thenReturn(new DpsHeaders());
+
+ Type listType = new TypeToken<List<RecordInfo>>() {}.getType();
+
+ List<RecordInfo> msgs = (new Gson()).fromJson(recordChangedMessages, listType);
+ jobStatus.initialize(msgs);
+ ids = Arrays.asList(RECORD_ID1, RECORDS_ID2);
+
+ when(configurationProperties.getStorageRecordsBatchSize()).thenReturn(20);
+ }
+
+ @Test
+ public void should_return404_givenNullData_getValidStorageRecordsTest() throws URISyntaxException {
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(null);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ should_return404_getValidStorageRecordsTest();
+ }
+
+ @Test
+ public void should_return404_givenEmptyData_getValidStorageRecordsTest() throws URISyntaxException {
+
+ String emptyDataFromStorage = "{\"records\":[],\"notFound\":[]}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(emptyDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ should_return404_getValidStorageRecordsTest();
+ }
+
+ @Test
+ public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsTest() throws URISyntaxException {
+
+ String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"], \"conversionStatuses\": []}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+ Records storageRecords = this.sut.getStorageRecords(ids);
+
+ assertEquals(1, storageRecords.getRecords().size());
+ }
+
+ @Test
+ public void should_logMissingRecord_given_storageMissedRecords() throws URISyntaxException {
+
+ String validDataFromStorage = "{\"records\":[{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[]}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(any())).thenReturn(httpResponse);
+ Records storageRecords = this.sut.getStorageRecords(ids);
+
+ assertEquals(1, storageRecords.getRecords().size());
+ verify(this.jobStatus).addOrUpdateRecordStatus(singletonList(RECORDS_ID2), IndexingStatus.FAIL, HttpStatus.NOT_FOUND.value(), "Partial response received from Storage service - missing records", "Partial response received from Storage service: tenant1:doc:15e790a69beb4d789b1f979e2af2e813");
+ }
+
+ @Test
+ public void should_returnValidJobStatus_givenFailedUnitsConversion_processRecordChangedMessageTest() throws URISyntaxException {
+ String validDataFromStorage = "{\"records\":[{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[],\"conversionStatuses\":[{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\",\"status\":\"ERROR\",\"errors\":[\"crs conversion failed\"]}]}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(any())).thenReturn(httpResponse);
+ Records storageRecords = this.sut.getStorageRecords(singletonList(RECORDS_ID2));
+
+ assertEquals(1, storageRecords.getRecords().size());
+ verify(this.jobStatus).addOrUpdateRecordStatus(RECORDS_ID2, IndexingStatus.WARN, HttpStatus.BAD_REQUEST.value(), "crs conversion failed", String.format("record-id: %s | %s", "tenant1:doc:15e790a69beb4d789b1f979e2af2e813", "crs conversion failed"));
+ }
+
+ @Test
+ public void should_returnValidResponse_givenValidRecordQueryRequest_getRecordListByKind() throws Exception {
+
+ RecordReindexRequest recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor("100").build();
+
+ HttpResponse httpResponse = new HttpResponse();
+ httpResponse.setBody(new Gson().toJson(recordReindexRequest, RecordReindexRequest.class));
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ RecordQueryResponse recordQueryResponse = this.sut.getRecordsByKind(recordReindexRequest);
+
+ assertEquals("100", recordQueryResponse.getCursor());
+ assertNull(recordQueryResponse.getResults());
+ }
+
+ @Test
+ public void should_returnValidResponse_givenValidKind_getSchemaByKind() throws Exception {
+
+ String validSchemaFromStorage = "{" +
+ " \"kind\": \"tenant:test:test:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"msg\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"references.entity\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]," +
+ " \"ext\": null" +
+ "}";
+ String kind = "tenant:test:test:1.0.0";
+
+ HttpResponse httpResponse = new HttpResponse();
+ httpResponse.setResponseCode(HttpStatus.OK.value());
+ httpResponse.setBody(validSchemaFromStorage);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ String recordSchemaResponse = this.sut.getStorageSchema(kind);
+
+ assertNotNull(recordSchemaResponse);
+ }
+
+ @Test
+ public void should_returnNullResponse_givenAbsentKind_getSchemaByKind() throws Exception {
+
+ String kind = "tenant:test:test:1.0.0";
+
+ HttpResponse httpResponse = new HttpResponse();
+ httpResponse.setResponseCode(HttpStatus.NOT_FOUND.value());
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ String recordSchemaResponse = this.sut.getStorageSchema(kind);
+
+ assertNull(recordSchemaResponse);
+ }
+
+ @Test
+ public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsWithInvalidConversionTest() throws URISyntaxException {
+
+ String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"],\"conversionStatuses\": [{\"id\":\"testid\",\"status\":\"ERROR\",\"errors\":[\"conversion error occurred\"] } ]}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+ Records storageRecords = this.sut.getStorageRecords(ids);
+
+ assertEquals(1, storageRecords.getRecords().size());
+
+ assertEquals(1, storageRecords.getConversionStatuses().get(0).getErrors().size());
+
+ assertEquals("conversion error occurred", storageRecords.getConversionStatuses().get(0).getErrors().get(0));
+ }
+
+ private void should_return404_getValidStorageRecordsTest() {
+ try {
+ this.sut.getStorageRecords(ids);
+ fail("Should throw exception");
+ } catch (AppException e) {
+ assertEquals(HttpStatus.NOT_FOUND.value(), e.getError().getCode());
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/GlobalExceptionMapper.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/GlobalExceptionMapper.java
index 092b76bb7..8ceacd2d7 100644
--- a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/GlobalExceptionMapper.java
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/GlobalExceptionMapper.java
@@ -1,72 +1,72 @@
-/* Licensed Materials - Property of IBM */
-/* (c) Copyright IBM Corp. 2020. All Rights Reserved.*/
-
-package org.opengroup.osdu.indexer.ibm.util;
-
-import javax.validation.ValidationException;
-
-import javassist.NotFoundException;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.core.Ordered;
-import org.springframework.core.annotation.Order;
-import org.springframework.http.HttpStatus;
-import org.springframework.http.ResponseEntity;
-import org.springframework.security.access.AccessDeniedException;
-import org.springframework.web.bind.annotation.ControllerAdvice;
-import org.springframework.web.bind.annotation.ExceptionHandler;
-import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler;
-import org.opengroup.osdu.core.common.model.http.AppException;
-
-@Order(Ordered.HIGHEST_PRECEDENCE)
-@ControllerAdvice
-public class GlobalExceptionMapper extends ResponseEntityExceptionHandler {
-
- @Autowired
- private JaxRsDpsLog logger;
-
- @ExceptionHandler(AppException.class)
- protected ResponseEntity<Object> handleAppException(AppException e) {
- return this.getErrorResponse(e);
- }
-
- @ExceptionHandler(ValidationException.class)
- protected ResponseEntity<Object> handleValidationException(ValidationException e) {
- return this.getErrorResponse(
- new AppException(HttpStatus.BAD_REQUEST.value(), "Validation error.", e.getMessage(), e));
- }
-
- @ExceptionHandler(NotFoundException.class)
- protected ResponseEntity<Object> handleNotFoundException(NotFoundException e) {
- return this.getErrorResponse(
- new AppException(HttpStatus.NOT_FOUND.value(), "Resource not found.", e.getMessage(), e));
- }
-
- @ExceptionHandler(AccessDeniedException.class)
- protected ResponseEntity<Object> handleAccessDeniedException(AccessDeniedException e) {
- return this.getErrorResponse(
- new AppException(HttpStatus.FORBIDDEN.value(), "Access denied", e.getMessage(), e));
- }
-
- @ExceptionHandler(Exception.class)
- protected ResponseEntity<Object> handleGeneralException(Exception e) {
- return this.getErrorResponse(
- new AppException(HttpStatus.INTERNAL_SERVER_ERROR.value(), "Server error.",
- "An unknown error has occurred.", e));
- }
-
- private ResponseEntity<Object> getErrorResponse(AppException e) {
-
- String exceptionMsg = e.getOriginalException() != null
- ? e.getOriginalException().getMessage()
- : e.getError().getMessage();
-
- if (e.getError().getCode() > 499) {
- this.logger.error(exceptionMsg, e);
- } else {
- this.logger.warning(exceptionMsg, e);
- }
-
- return new ResponseEntity<Object>(e.getError(), HttpStatus.resolve(e.getError().getCode()));
- }
+/* Licensed Materials - Property of IBM */
+/* (c) Copyright IBM Corp. 2020. All Rights Reserved.*/
+
+package org.opengroup.osdu.indexer.ibm.util;
+
+import javax.validation.ValidationException;
+
+import javassist.NotFoundException;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.Ordered;
+import org.springframework.core.annotation.Order;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.security.access.AccessDeniedException;
+import org.springframework.web.bind.annotation.ControllerAdvice;
+import org.springframework.web.bind.annotation.ExceptionHandler;
+import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler;
+import org.opengroup.osdu.core.common.model.http.AppException;
+
+@Order(Ordered.HIGHEST_PRECEDENCE)
+@ControllerAdvice
+public class GlobalExceptionMapper extends ResponseEntityExceptionHandler {
+
+ @Autowired
+ private JaxRsDpsLog logger;
+
+ @ExceptionHandler(AppException.class)
+ protected ResponseEntity<Object> handleAppException(AppException e) {
+ return this.getErrorResponse(e);
+ }
+
+ @ExceptionHandler(ValidationException.class)
+ protected ResponseEntity<Object> handleValidationException(ValidationException e) {
+ return this.getErrorResponse(
+ new AppException(HttpStatus.BAD_REQUEST.value(), "Validation error.", e.getMessage(), e));
+ }
+
+ @ExceptionHandler(NotFoundException.class)
+ protected ResponseEntity<Object> handleNotFoundException(NotFoundException e) {
+ return this.getErrorResponse(
+ new AppException(HttpStatus.NOT_FOUND.value(), "Resource not found.", e.getMessage(), e));
+ }
+
+ @ExceptionHandler(AccessDeniedException.class)
+ protected ResponseEntity<Object> handleAccessDeniedException(AccessDeniedException e) {
+ return this.getErrorResponse(
+ new AppException(HttpStatus.FORBIDDEN.value(), "Access denied", e.getMessage(), e));
+ }
+
+ @ExceptionHandler(Exception.class)
+ protected ResponseEntity<Object> handleGeneralException(Exception e) {
+ return this.getErrorResponse(
+ new AppException(HttpStatus.INTERNAL_SERVER_ERROR.value(), "Server error.",
+ "An unknown error has occurred.", e));
+ }
+
+ private ResponseEntity<Object> getErrorResponse(AppException e) {
+
+ String exceptionMsg = e.getOriginalException() != null
+ ? e.getOriginalException().getMessage()
+ : e.getError().getMessage();
+
+ if (e.getError().getCode() > 499) {
+ this.logger.error(exceptionMsg, e);
+ } else {
+ this.logger.warning(exceptionMsg, e);
+ }
+
+ return new ResponseEntity<Object>(e.getError(), HttpStatus.resolve(e.getError().getCode()));
+ }
}
\ No newline at end of file
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/IndexerQueueTaskBuilderIbm.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/IndexerQueueTaskBuilderIbm.java
index a3d7805e6..b44ce5753 100644
--- a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/IndexerQueueTaskBuilderIbm.java
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/IndexerQueueTaskBuilderIbm.java
@@ -1,159 +1,159 @@
-/* Licensed Materials - Property of IBM */
-/* (c) Copyright IBM Corp. 2020. All Rights Reserved.*/
-
-package org.opengroup.osdu.indexer.ibm.util;
-
-import static org.opengroup.osdu.core.common.Constants.WORKER_RELATIVE_URL;
-import static org.opengroup.osdu.core.common.model.http.DpsHeaders.AUTHORIZATION;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import javax.inject.Inject;
-
-import org.apache.http.HttpStatus;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.indexer.OperationType;
-import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
-import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
-import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
-import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
-import org.opengroup.osdu.core.ibm.messagebus.IMessageFactory;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.opengroup.osdu.indexer.service.StorageService;
-import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.context.annotation.Primary;
-import org.springframework.stereotype.Component;
-
-import com.google.common.base.Strings;
-import com.google.gson.Gson;
-import com.google.gson.JsonSyntaxException;
-
-@Primary
-@Component
-public class IndexerQueueTaskBuilderIbm extends IndexerQueueTaskBuilder {
- private static final Logger logger = LoggerFactory.getLogger(IndexerQueueTaskBuilderIbm.class);
-
- @Inject
- IMessageFactory mq;
-
- @Inject
- private StorageService storageService;
-
- @Inject
- private RequestInfoImpl requestInfo;
-
- @Inject
- private IndexerConfigurationProperties configurationProperties;
-
- private Gson gson;
- private final static String RETRY_STRING = "retry";
- private final static String ERROR_CODE = "errorCode";
- private final static String ERROR_MESSAGE = "errorMessage";
-
- @Inject
- public void init() {
- gson = new Gson();
- }
-
- @Override
- public void createWorkerTask(String payload, DpsHeaders headers) {
- createTask(payload, headers);
- }
-
- @Override
- public void createReIndexTask(String payload, DpsHeaders headers) {
- createTask(payload, headers);
- }
-
- public void createReIndexTask(String payload, Long countdownMillis, DpsHeaders headers) {
- publishAllRecordsToSubscriber(payload, headers);
- }
-
- //used by reindexer api
- @Override
- public void createWorkerTask(String payload, Long countdownMillis, DpsHeaders headers) {
- createTask(payload, headers);
- }
-
- private void createTask(String payload, DpsHeaders headers) {
-
- try {
- RecordChangedMessages receivedPayload = gson.fromJson(payload, RecordChangedMessages.class);
-
- Map<String, String> attributes = receivedPayload.getAttributes();
- int retryCount = 0;
- if (attributes.containsKey(RETRY_STRING)) {
- retryCount = Integer.parseInt(attributes.get(RETRY_STRING));
- retryCount++;
- } else {
- retryCount = 1;
- }
- attributes.put(RETRY_STRING, String.valueOf(retryCount));
- attributes.put(ERROR_CODE, "999"); //error code TBD
- attributes.put(ERROR_MESSAGE, "Indexer could not process record");
- receivedPayload.setAttributes(attributes);
-
- // incase if we need to shift logic from indexer-queue-ibm/subscriber.java
- /*
- * if(Integer.parseInt(receivedPayload.getAttributes().get(RETRY_STRING))>3) {
- * //add DLQ in IMessageFactory
- *
- * mq.sendMessage("DLQ", gson.toJson(receivedPayload)); }
- */
- logger.info("Message send back to queue : " + receivedPayload);
- mq.sendMessage(IMessageFactory.DEFAULT_QUEUE_NAME, gson.toJson(receivedPayload));
- } catch (JsonSyntaxException e) {
- logger.error("JsonSyntaxException in IndexerQueueTaskBuilderIbm " + e.toString());
- e.printStackTrace();
- } catch (NumberFormatException e) {
- logger.error("NumberFormatException in IndexerQueueTaskBuilderIbm " + e.toString());
- e.printStackTrace();
- } catch (Exception e) {
- logger.error("Exception in IndexerQueueTaskBuilderIbm " + e.toString());
- e.printStackTrace();
- }
-
- }
- private void publishAllRecordsToSubscriber(String payload, DpsHeaders headers) {
- Gson gson = new Gson();
- RecordReindexRequest recordReindexRequest = gson.fromJson(payload, RecordReindexRequest.class);
- final String recordKind = recordReindexRequest.getKind();
- RecordQueryResponse recordQueryResponse = null;
-
- try {
- do {
- headers.put(AUTHORIZATION, this.requestInfo.checkOrGetAuthorizationHeader());
-
- if (recordQueryResponse != null) {
- recordReindexRequest = RecordReindexRequest.builder().cursor(recordQueryResponse.getCursor()).kind(recordKind).build();
- }
- recordQueryResponse = this.storageService.getRecordsByKind(recordReindexRequest);
- if (recordQueryResponse.getResults() != null && recordQueryResponse.getResults().size() != 0) {
-
- List<RecordInfo> records = recordQueryResponse.getResults().stream()
- .map(record -> RecordInfo.builder().id(record).kind(recordKind).op(OperationType.create.name()).build()).collect(Collectors.toList());
-
- Map<String, String> attributes = new HashMap<>();
- attributes.put(DpsHeaders.ACCOUNT_ID, headers.getPartitionIdWithFallbackToAccountId());
- attributes.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
- attributes.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
-
- RecordChangedMessages recordChangedMessages = RecordChangedMessages.builder().data(gson.toJson(records)).attributes(attributes).build();
- String recordChangedMessagePayload = gson.toJson(recordChangedMessages);
- createTask(recordChangedMessagePayload, headers);
- }
- } while (!Strings.isNullOrEmpty(recordQueryResponse.getCursor()) && recordQueryResponse.getResults().size() == configurationProperties.getStorageRecordsBatchSize());
-
- } catch (AppException e) {
- throw e;
- } catch (Exception e) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Unknown error", "An unknown error has occurred.", e);
- }
- }
+/* Licensed Materials - Property of IBM */
+/* (c) Copyright IBM Corp. 2020. All Rights Reserved.*/
+
+package org.opengroup.osdu.indexer.ibm.util;
+
+import static org.opengroup.osdu.core.common.Constants.WORKER_RELATIVE_URL;
+import static org.opengroup.osdu.core.common.model.http.DpsHeaders.AUTHORIZATION;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import javax.inject.Inject;
+
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.indexer.OperationType;
+import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
+import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
+import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
+import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
+import org.opengroup.osdu.core.ibm.messagebus.IMessageFactory;
+import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.opengroup.osdu.indexer.service.StorageService;
+import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.Primary;
+import org.springframework.stereotype.Component;
+
+import com.google.common.base.Strings;
+import com.google.gson.Gson;
+import com.google.gson.JsonSyntaxException;
+
+@Primary
+@Component
+public class IndexerQueueTaskBuilderIbm extends IndexerQueueTaskBuilder {
+ private static final Logger logger = LoggerFactory.getLogger(IndexerQueueTaskBuilderIbm.class);
+
+ @Inject
+ IMessageFactory mq;
+
+ @Inject
+ private StorageService storageService;
+
+ @Inject
+ private RequestInfoImpl requestInfo;
+
+ @Inject
+ private IndexerConfigurationProperties configurationProperties;
+
+ private Gson gson;
+ private final static String RETRY_STRING = "retry";
+ private final static String ERROR_CODE = "errorCode";
+ private final static String ERROR_MESSAGE = "errorMessage";
+
+ @Inject
+ public void init() {
+ gson = new Gson();
+ }
+
+ @Override
+ public void createWorkerTask(String payload, DpsHeaders headers) {
+ createTask(payload, headers);
+ }
+
+ @Override
+ public void createReIndexTask(String payload, DpsHeaders headers) {
+ createTask(payload, headers);
+ }
+
+ public void createReIndexTask(String payload, Long countdownMillis, DpsHeaders headers) {
+ publishAllRecordsToSubscriber(payload, headers);
+ }
+
+ //used by reindexer api
+ @Override
+ public void createWorkerTask(String payload, Long countdownMillis, DpsHeaders headers) {
+ createTask(payload, headers);
+ }
+
+ private void createTask(String payload, DpsHeaders headers) {
+
+ try {
+ RecordChangedMessages receivedPayload = gson.fromJson(payload, RecordChangedMessages.class);
+
+ Map<String, String> attributes = receivedPayload.getAttributes();
+ int retryCount = 0;
+ if (attributes.containsKey(RETRY_STRING)) {
+ retryCount = Integer.parseInt(attributes.get(RETRY_STRING));
+ retryCount++;
+ } else {
+ retryCount = 1;
+ }
+ attributes.put(RETRY_STRING, String.valueOf(retryCount));
+ attributes.put(ERROR_CODE, "999"); //error code TBD
+ attributes.put(ERROR_MESSAGE, "Indexer could not process record");
+ receivedPayload.setAttributes(attributes);
+
+ // incase if we need to shift logic from indexer-queue-ibm/subscriber.java
+ /*
+ * if(Integer.parseInt(receivedPayload.getAttributes().get(RETRY_STRING))>3) {
+ * //add DLQ in IMessageFactory
+ *
+ * mq.sendMessage("DLQ", gson.toJson(receivedPayload)); }
+ */
+ logger.info("Message send back to queue : " + receivedPayload);
+ mq.sendMessage(IMessageFactory.DEFAULT_QUEUE_NAME, gson.toJson(receivedPayload));
+ } catch (JsonSyntaxException e) {
+ logger.error("JsonSyntaxException in IndexerQueueTaskBuilderIbm " + e.toString());
+ e.printStackTrace();
+ } catch (NumberFormatException e) {
+ logger.error("NumberFormatException in IndexerQueueTaskBuilderIbm " + e.toString());
+ e.printStackTrace();
+ } catch (Exception e) {
+ logger.error("Exception in IndexerQueueTaskBuilderIbm " + e.toString());
+ e.printStackTrace();
+ }
+
+ }
+ private void publishAllRecordsToSubscriber(String payload, DpsHeaders headers) {
+ Gson gson = new Gson();
+ RecordReindexRequest recordReindexRequest = gson.fromJson(payload, RecordReindexRequest.class);
+ final String recordKind = recordReindexRequest.getKind();
+ RecordQueryResponse recordQueryResponse = null;
+
+ try {
+ do {
+ headers.put(AUTHORIZATION, this.requestInfo.checkOrGetAuthorizationHeader());
+
+ if (recordQueryResponse != null) {
+ recordReindexRequest = RecordReindexRequest.builder().cursor(recordQueryResponse.getCursor()).kind(recordKind).build();
+ }
+ recordQueryResponse = this.storageService.getRecordsByKind(recordReindexRequest);
+ if (recordQueryResponse.getResults() != null && recordQueryResponse.getResults().size() != 0) {
+
+ List<RecordInfo> records = recordQueryResponse.getResults().stream()
+ .map(record -> RecordInfo.builder().id(record).kind(recordKind).op(OperationType.create.name()).build()).collect(Collectors.toList());
+
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(DpsHeaders.ACCOUNT_ID, headers.getPartitionIdWithFallbackToAccountId());
+ attributes.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
+ attributes.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
+
+ RecordChangedMessages recordChangedMessages = RecordChangedMessages.builder().data(gson.toJson(records)).attributes(attributes).build();
+ String recordChangedMessagePayload = gson.toJson(recordChangedMessages);
+ createTask(recordChangedMessagePayload, headers);
+ }
+ } while (!Strings.isNullOrEmpty(recordQueryResponse.getCursor()) && recordQueryResponse.getResults().size() == configurationProperties.getStorageRecordsBatchSize());
+
+ } catch (AppException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Unknown error", "An unknown error has occurred.", e);
+ }
+ }
}
\ No newline at end of file
diff --git a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/ReindexServiceTest.java b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/ReindexServiceTest.java
index 87dfea951..acb116a76 100644
--- a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/ReindexServiceTest.java
+++ b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/ReindexServiceTest.java
@@ -1,144 +1,144 @@
-/* Licensed Materials - Property of IBM */
-/* (c) Copyright IBM Corp. 2020. All Rights Reserved.*/
-package org.opengroup.osdu.indexer.ibm.service;
-
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.ArgumentMatchers;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
-import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.opengroup.osdu.indexer.service.ReindexServiceImpl;
-import org.opengroup.osdu.indexer.service.StorageService;
-import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
-import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
-import org.powermock.modules.junit4.PowerMockRunnerDelegate;
-import org.springframework.test.context.junit4.SpringRunner;
-
-import java.util.*;
-
-import static org.junit.Assert.fail;
-import static org.mockito.MockitoAnnotations.initMocks;
-import static org.powermock.api.mockito.PowerMockito.mockStatic;
-import static org.powermock.api.mockito.PowerMockito.when;
-
-@Ignore
-@RunWith(PowerMockRunner.class)
-@PowerMockRunnerDelegate(SpringRunner.class)
-@PrepareForTest({IndexerConfigurationProperties.class})
-public class ReindexServiceTest {
-
- private final String cursor = "100";
-
- private final String correlationId = UUID.randomUUID().toString();
-
- @Mock
- private IndexerConfigurationProperties indexerConfigurationProperties;
-
- @Mock
- private StorageService storageService;
-
- @Mock
- private Map<String, String> httpHeaders;
- @Mock
- private IRequestInfo requestInfo;
- @Mock
- private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
- @Mock
- private JaxRsDpsLog log;
- @InjectMocks
- private ReindexServiceImpl sut;
-
- private RecordReindexRequest recordReindexRequest;
- private RecordQueryResponse recordQueryResponse;
-
- @Before
- public void setup() {
- initMocks(this);
-
- mockStatic(UUID.class);
-
- recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor(cursor).build();
- recordQueryResponse = new RecordQueryResponse();
-
- httpHeaders = new HashMap<>();
- httpHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
- httpHeaders.put(DpsHeaders.CORRELATION_ID, correlationId);
- DpsHeaders standardHeaders = DpsHeaders.createFromMap(httpHeaders);
- when(requestInfo.getHeaders()).thenReturn(standardHeaders);
- when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(httpHeaders);
- }
-
- @Test
- public void should_returnNull_givenNullResponseResult_reIndexRecordsTest() {
- try {
- recordQueryResponse.setResults(null);
- when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
-
- String response = sut.reindexRecords(recordReindexRequest, false);
-
- Assert.assertNull(response);
- } catch (Exception e) {
- fail("Should not throw this exception" + e.getMessage());
- }
- }
-
- @Test
- public void should_returnNull_givenEmptyResponseResult_reIndexRecordsTest() {
- try {
- recordQueryResponse.setResults(new ArrayList<>());
- when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
-
- String response = sut.reindexRecords(recordReindexRequest, false);
-
- Assert.assertNull(response);
- } catch (Exception e) {
- fail("Should not throw this exception" + e.getMessage());
- }
- }
-
- @Test
- public void should_returnRecordQueryRequestPayload_givenValidResponseResult_reIndexRecordsTest() {
- try {
- recordQueryResponse.setCursor(cursor);
- List<String> results = new ArrayList<>();
- results.add("test1");
- recordQueryResponse.setResults(results);
-
- when(indexerConfigurationProperties.getStorageRecordsBatchSize()).thenReturn(1);
-
- when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
-
- String taskQueuePayload = sut.reindexRecords(recordReindexRequest, false);
-
- Assert.assertEquals("{\"kind\":\"tenant:test:test:1.0.0\",\"cursor\":\"100\"}", taskQueuePayload);
- } catch (Exception e) {
- fail("Should not throw exception" + e.getMessage());
- }
- }
-
- @Test
- public void should_returnRecordChangedMessage_givenValidResponseResult_reIndexRecordsTest() {
- try {
- List<String> results = new ArrayList<>();
- results.add("test1");
- recordQueryResponse.setResults(results);
- when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
-
- String taskQueuePayload = sut.reindexRecords(recordReindexRequest, false);
-
- Assert.assertEquals(String.format("{\"data\":\"[{\\\"id\\\":\\\"test1\\\",\\\"kind\\\":\\\"tenant:test:test:1.0.0\\\",\\\"op\\\":\\\"create\\\"}]\",\"attributes\":{\"slb-correlation-id\":\"%s\"}}", correlationId), taskQueuePayload);
- } catch (Exception e) {
- fail("Should not throw exception" + e.getMessage());
- }
- }
-}
+/* Licensed Materials - Property of IBM */
+/* (c) Copyright IBM Corp. 2020. All Rights Reserved.*/
+package org.opengroup.osdu.indexer.ibm.service;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentMatchers;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
+import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.opengroup.osdu.indexer.service.ReindexServiceImpl;
+import org.opengroup.osdu.indexer.service.StorageService;
+import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+import org.powermock.modules.junit4.PowerMockRunnerDelegate;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.util.*;
+
+import static org.junit.Assert.fail;
+import static org.mockito.MockitoAnnotations.initMocks;
+import static org.powermock.api.mockito.PowerMockito.mockStatic;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@Ignore
+@RunWith(PowerMockRunner.class)
+@PowerMockRunnerDelegate(SpringRunner.class)
+@PrepareForTest({IndexerConfigurationProperties.class})
+public class ReindexServiceTest {
+
+ private final String cursor = "100";
+
+ private final String correlationId = UUID.randomUUID().toString();
+
+ @Mock
+ private IndexerConfigurationProperties indexerConfigurationProperties;
+
+ @Mock
+ private StorageService storageService;
+
+ @Mock
+ private Map<String, String> httpHeaders;
+ @Mock
+ private IRequestInfo requestInfo;
+ @Mock
+ private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
+ @Mock
+ private JaxRsDpsLog log;
+ @InjectMocks
+ private ReindexServiceImpl sut;
+
+ private RecordReindexRequest recordReindexRequest;
+ private RecordQueryResponse recordQueryResponse;
+
+ @Before
+ public void setup() {
+ initMocks(this);
+
+ mockStatic(UUID.class);
+
+ recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor(cursor).build();
+ recordQueryResponse = new RecordQueryResponse();
+
+ httpHeaders = new HashMap<>();
+ httpHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
+ httpHeaders.put(DpsHeaders.CORRELATION_ID, correlationId);
+ DpsHeaders standardHeaders = DpsHeaders.createFromMap(httpHeaders);
+ when(requestInfo.getHeaders()).thenReturn(standardHeaders);
+ when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(httpHeaders);
+ }
+
+ @Test
+ public void should_returnNull_givenNullResponseResult_reIndexRecordsTest() {
+ try {
+ recordQueryResponse.setResults(null);
+ when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
+
+ String response = sut.reindexRecords(recordReindexRequest, false);
+
+ Assert.assertNull(response);
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_returnNull_givenEmptyResponseResult_reIndexRecordsTest() {
+ try {
+ recordQueryResponse.setResults(new ArrayList<>());
+ when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
+
+ String response = sut.reindexRecords(recordReindexRequest, false);
+
+ Assert.assertNull(response);
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_returnRecordQueryRequestPayload_givenValidResponseResult_reIndexRecordsTest() {
+ try {
+ recordQueryResponse.setCursor(cursor);
+ List<String> results = new ArrayList<>();
+ results.add("test1");
+ recordQueryResponse.setResults(results);
+
+ when(indexerConfigurationProperties.getStorageRecordsBatchSize()).thenReturn(1);
+
+ when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
+
+ String taskQueuePayload = sut.reindexRecords(recordReindexRequest, false);
+
+ Assert.assertEquals("{\"kind\":\"tenant:test:test:1.0.0\",\"cursor\":\"100\"}", taskQueuePayload);
+ } catch (Exception e) {
+ fail("Should not throw exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_returnRecordChangedMessage_givenValidResponseResult_reIndexRecordsTest() {
+ try {
+ List<String> results = new ArrayList<>();
+ results.add("test1");
+ recordQueryResponse.setResults(results);
+ when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
+
+ String taskQueuePayload = sut.reindexRecords(recordReindexRequest, false);
+
+ Assert.assertEquals(String.format("{\"data\":\"[{\\\"id\\\":\\\"test1\\\",\\\"kind\\\":\\\"tenant:test:test:1.0.0\\\",\\\"op\\\":\\\"create\\\"}]\",\"attributes\":{\"slb-correlation-id\":\"%s\"}}", correlationId), taskQueuePayload);
+ } catch (Exception e) {
+ fail("Should not throw exception" + e.getMessage());
+ }
+ }
+}
diff --git a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/StorageServiceTest.java b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/StorageServiceTest.java
index d546e9b65..dfbdb4396 100644
--- a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/StorageServiceTest.java
+++ b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/StorageServiceTest.java
@@ -1,219 +1,219 @@
-/* Licensed Materials - Property of IBM */
-/* (c) Copyright IBM Corp. 2020. All Rights Reserved.*/
-
-package org.opengroup.osdu.indexer.ibm.service;
-
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.ArgumentMatchers;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.mockito.Mockito;
-import org.opengroup.osdu.core.common.model.indexer.IndexingStatus;
-import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
-import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
-import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
-import org.opengroup.osdu.core.common.model.indexer.Records;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.indexer.service.StorageServiceImpl;
-import org.opengroup.osdu.core.common.model.indexer.JobStatus;
-import org.opengroup.osdu.core.common.model.http.HttpResponse;
-import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
-import org.opengroup.osdu.core.common.http.IUrlFetchService;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.springframework.http.HttpStatus;
-import org.springframework.test.context.junit4.SpringRunner;
-
-import java.lang.reflect.Type;
-import java.net.URISyntaxException;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-
-import static java.util.Collections.singletonList;
-import static org.junit.Assert.*;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.powermock.api.mockito.PowerMockito.when;
-
-@Ignore
-@RunWith(SpringRunner.class)
-public class StorageServiceTest {
-
- @Mock
- private IUrlFetchService urlFetchService;
- @Mock
- private JobStatus jobStatus;
- @Mock
- private JaxRsDpsLog log;
- @Mock
- private IRequestInfo requestInfo;
- @InjectMocks
- private StorageServiceImpl sut;
-
- private List<String> ids;
- private static final String RECORD_ID1 = "tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465";
- private static final String RECORDS_ID2 = "tenant1:doc:15e790a69beb4d789b1f979e2af2e813";
-
- @Before
- public void setup() {
-
- String recordChangedMessages = "[{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"purge\"}," +
- "{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"create\"}]";
-
- when(this.requestInfo.getHeadersMap()).thenReturn(new HashMap<>());
-
- Type listType = new TypeToken<List<RecordInfo>>() {}.getType();
-
- List<RecordInfo> msgs = (new Gson()).fromJson(recordChangedMessages, listType);
- jobStatus.initialize(msgs);
- ids = Arrays.asList(RECORD_ID1, RECORDS_ID2);
- }
-
- @Test
- public void should_return404_givenNullData_getValidStorageRecordsTest() throws URISyntaxException {
-
- HttpResponse httpResponse = mock(HttpResponse.class);
- Mockito.when(httpResponse.getBody()).thenReturn(null);
-
- when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
-
- should_return404_getValidStorageRecordsTest();
- }
-
- @Test
- public void should_return404_givenEmptyData_getValidStorageRecordsTest() throws URISyntaxException {
-
- String emptyDataFromStorage = "{\"records\":[],\"notFound\":[]}";
-
- HttpResponse httpResponse = mock(HttpResponse.class);
- Mockito.when(httpResponse.getBody()).thenReturn(emptyDataFromStorage);
-
- when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
-
- should_return404_getValidStorageRecordsTest();
- }
-
- @Test
- public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsTest() throws URISyntaxException {
-
- String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"]}";
-
- HttpResponse httpResponse = mock(HttpResponse.class);
- Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
-
- when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
- Records storageRecords = this.sut.getStorageRecords(ids);
-
- assertEquals(1, storageRecords.getRecords().size());
- }
-
- @Test
- public void should_returnValidResponse_givenValidRecordQueryRequest_getRecordListByKind() throws Exception {
-
- RecordReindexRequest recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor("100").build();
-
- HttpResponse httpResponse = new HttpResponse();
- httpResponse.setBody(new Gson().toJson(recordReindexRequest, RecordReindexRequest.class));
-
- when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
-
- RecordQueryResponse recordQueryResponse = this.sut.getRecordsByKind(recordReindexRequest);
-
- assertEquals("100", recordQueryResponse.getCursor());
- assertNull(recordQueryResponse.getResults());
- }
-
- @Test
- public void should_returnValidJobStatus_givenFailedUnitsConversion_processRecordChangedMessageTest() throws URISyntaxException {
- String validDataFromStorage = "{\"records\":[{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[],\"conversionStatuses\":[{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\",\"status\":\"ERROR\",\"errors\":[\"crs conversion failed\"]}]}";
-
- HttpResponse httpResponse = mock(HttpResponse.class);
- Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
-
- when(this.urlFetchService.sendRequest(any())).thenReturn(httpResponse);
- Records storageRecords = this.sut.getStorageRecords(singletonList("tenant1:doc:15e790a69beb4d789b1f979e2af2e813"));
-
- assertEquals(1, storageRecords.getRecords().size());
- verify(this.jobStatus).addOrUpdateRecordStatus(RECORDS_ID2, IndexingStatus.WARN, HttpStatus.BAD_REQUEST.value(), "crs conversion failed", String.format("record-id: %s | %s", "tenant1:doc:15e790a69beb4d789b1f979e2af2e813", "crs conversion failed"));
- }
-
- @Test
- public void should_returnValidResponse_givenValidKind_getSchemaByKind() throws Exception {
-
- String validSchemaFromStorage = "{" +
- " \"kind\": \"tenant:test:test:1.0.0\"," +
- " \"schema\": [" +
- " {" +
- " \"path\": \"msg\"," +
- " \"kind\": \"string\"" +
- " }," +
- " {" +
- " \"path\": \"references.entity\"," +
- " \"kind\": \"string\"" +
- " }" +
- " ]," +
- " \"ext\": null" +
- "}";
- String kind = "tenant:test:test:1.0.0";
-
- HttpResponse httpResponse = new HttpResponse();
- httpResponse.setResponseCode(HttpStatus.OK.value());
- httpResponse.setBody(validSchemaFromStorage);
-
- when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
-
- String recordSchemaResponse = this.sut.getStorageSchema(kind);
-
- assertNotNull(recordSchemaResponse);
- }
-
- @Test
- public void should_returnNullResponse_givenAbsentKind_getSchemaByKind() throws Exception {
-
- String kind = "tenant:test:test:1.0.0";
-
- HttpResponse httpResponse = new HttpResponse();
- httpResponse.setResponseCode(HttpStatus.NOT_FOUND.value());
-
- when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
-
- String recordSchemaResponse = this.sut.getStorageSchema(kind);
-
- assertNull(recordSchemaResponse);
- }
-
- @Test
- public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsWithInvalidConversionTest() throws URISyntaxException {
-
- String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"],\"conversionStatuses\": [{\"id\":\"testid\",\"status\":\"ERROR\",\"errors\":[\"conversion error occurred\"] } ]}";
-
- HttpResponse httpResponse = mock(HttpResponse.class);
- Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
-
- when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
- Records storageRecords = this.sut.getStorageRecords(ids);
-
- assertEquals(1, storageRecords.getRecords().size());
-
- assertEquals(1, storageRecords.getConversionStatuses().get(0).getErrors().size());
-
- assertEquals("conversion error occurred", storageRecords.getConversionStatuses().get(0).getErrors().get(0));
- }
-
- private void should_return404_getValidStorageRecordsTest() {
- try {
- this.sut.getStorageRecords(ids);
- fail("Should throw exception");
- } catch (AppException e) {
- assertEquals(HttpStatus.NOT_FOUND, e.getError().getCode());
- } catch (Exception e) {
- fail("Should not throw this exception" + e.getMessage());
- }
- }
-}
+/* Licensed Materials - Property of IBM */
+/* (c) Copyright IBM Corp. 2020. All Rights Reserved.*/
+
+package org.opengroup.osdu.indexer.ibm.service;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentMatchers;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.opengroup.osdu.core.common.model.indexer.IndexingStatus;
+import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
+import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
+import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
+import org.opengroup.osdu.core.common.model.indexer.Records;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.indexer.service.StorageServiceImpl;
+import org.opengroup.osdu.core.common.model.indexer.JobStatus;
+import org.opengroup.osdu.core.common.model.http.HttpResponse;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.opengroup.osdu.core.common.http.IUrlFetchService;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.springframework.http.HttpStatus;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.lang.reflect.Type;
+import java.net.URISyntaxException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+
+import static java.util.Collections.singletonList;
+import static org.junit.Assert.*;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@Ignore
+@RunWith(SpringRunner.class)
+public class StorageServiceTest {
+
+ @Mock
+ private IUrlFetchService urlFetchService;
+ @Mock
+ private JobStatus jobStatus;
+ @Mock
+ private JaxRsDpsLog log;
+ @Mock
+ private IRequestInfo requestInfo;
+ @InjectMocks
+ private StorageServiceImpl sut;
+
+ private List<String> ids;
+ private static final String RECORD_ID1 = "tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465";
+ private static final String RECORDS_ID2 = "tenant1:doc:15e790a69beb4d789b1f979e2af2e813";
+
+ @Before
+ public void setup() {
+
+ String recordChangedMessages = "[{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"purge\"}," +
+ "{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"create\"}]";
+
+ when(this.requestInfo.getHeadersMap()).thenReturn(new HashMap<>());
+
+ Type listType = new TypeToken<List<RecordInfo>>() {}.getType();
+
+ List<RecordInfo> msgs = (new Gson()).fromJson(recordChangedMessages, listType);
+ jobStatus.initialize(msgs);
+ ids = Arrays.asList(RECORD_ID1, RECORDS_ID2);
+ }
+
+ @Test
+ public void should_return404_givenNullData_getValidStorageRecordsTest() throws URISyntaxException {
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(null);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ should_return404_getValidStorageRecordsTest();
+ }
+
+ @Test
+ public void should_return404_givenEmptyData_getValidStorageRecordsTest() throws URISyntaxException {
+
+ String emptyDataFromStorage = "{\"records\":[],\"notFound\":[]}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(emptyDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ should_return404_getValidStorageRecordsTest();
+ }
+
+ @Test
+ public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsTest() throws URISyntaxException {
+
+ String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"]}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+ Records storageRecords = this.sut.getStorageRecords(ids);
+
+ assertEquals(1, storageRecords.getRecords().size());
+ }
+
+ @Test
+ public void should_returnValidResponse_givenValidRecordQueryRequest_getRecordListByKind() throws Exception {
+
+ RecordReindexRequest recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor("100").build();
+
+ HttpResponse httpResponse = new HttpResponse();
+ httpResponse.setBody(new Gson().toJson(recordReindexRequest, RecordReindexRequest.class));
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ RecordQueryResponse recordQueryResponse = this.sut.getRecordsByKind(recordReindexRequest);
+
+ assertEquals("100", recordQueryResponse.getCursor());
+ assertNull(recordQueryResponse.getResults());
+ }
+
+ @Test
+ public void should_returnValidJobStatus_givenFailedUnitsConversion_processRecordChangedMessageTest() throws URISyntaxException {
+ String validDataFromStorage = "{\"records\":[{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[],\"conversionStatuses\":[{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\",\"status\":\"ERROR\",\"errors\":[\"crs conversion failed\"]}]}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(any())).thenReturn(httpResponse);
+ Records storageRecords = this.sut.getStorageRecords(singletonList("tenant1:doc:15e790a69beb4d789b1f979e2af2e813"));
+
+ assertEquals(1, storageRecords.getRecords().size());
+ verify(this.jobStatus).addOrUpdateRecordStatus(RECORDS_ID2, IndexingStatus.WARN, HttpStatus.BAD_REQUEST.value(), "crs conversion failed", String.format("record-id: %s | %s", "tenant1:doc:15e790a69beb4d789b1f979e2af2e813", "crs conversion failed"));
+ }
+
+ @Test
+ public void should_returnValidResponse_givenValidKind_getSchemaByKind() throws Exception {
+
+ String validSchemaFromStorage = "{" +
+ " \"kind\": \"tenant:test:test:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"msg\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"references.entity\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]," +
+ " \"ext\": null" +
+ "}";
+ String kind = "tenant:test:test:1.0.0";
+
+ HttpResponse httpResponse = new HttpResponse();
+ httpResponse.setResponseCode(HttpStatus.OK.value());
+ httpResponse.setBody(validSchemaFromStorage);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ String recordSchemaResponse = this.sut.getStorageSchema(kind);
+
+ assertNotNull(recordSchemaResponse);
+ }
+
+ @Test
+ public void should_returnNullResponse_givenAbsentKind_getSchemaByKind() throws Exception {
+
+ String kind = "tenant:test:test:1.0.0";
+
+ HttpResponse httpResponse = new HttpResponse();
+ httpResponse.setResponseCode(HttpStatus.NOT_FOUND.value());
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ String recordSchemaResponse = this.sut.getStorageSchema(kind);
+
+ assertNull(recordSchemaResponse);
+ }
+
+ @Test
+ public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsWithInvalidConversionTest() throws URISyntaxException {
+
+ String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"],\"conversionStatuses\": [{\"id\":\"testid\",\"status\":\"ERROR\",\"errors\":[\"conversion error occurred\"] } ]}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+ Records storageRecords = this.sut.getStorageRecords(ids);
+
+ assertEquals(1, storageRecords.getRecords().size());
+
+ assertEquals(1, storageRecords.getConversionStatuses().get(0).getErrors().size());
+
+ assertEquals("conversion error occurred", storageRecords.getConversionStatuses().get(0).getErrors().get(0));
+ }
+
+ private void should_return404_getValidStorageRecordsTest() {
+ try {
+ this.sut.getStorageRecords(ids);
+ fail("Should throw exception");
+ } catch (AppException e) {
+ assertEquals(HttpStatus.NOT_FOUND, e.getError().getCode());
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+}
--
GitLab