diff --git a/indexer-core/pom.xml b/indexer-core/pom.xml
index 2aefa834bad5cffdb6b39da78b359f4d8fd03cfb..04c37e8b3300ca0cb8bcc526ccf9085307a11df8 100644
--- a/indexer-core/pom.xml
+++ b/indexer-core/pom.xml
@@ -18,7 +18,6 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
- <version>0.0.13</version>
</dependency>
<!-- spring boot dependencies -->
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/RecordIndexerApi.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/RecordIndexerApi.java
index cfa3f23415108db2b88c95386b28de7d7679789f..5962141825b4a3c6bf33ddec01c270a560e27dc9 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/RecordIndexerApi.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/RecordIndexerApi.java
@@ -1,100 +1,99 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.api;
-
-import com.google.common.reflect.TypeToken;
-import com.google.gson.Gson;
-
-import com.google.gson.JsonParseException;
-import io.swagger.annotations.ApiOperation;
-import lombok.extern.java.Log;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
-import org.opengroup.osdu.indexer.SwaggerDoc;
-import org.opengroup.osdu.core.common.model.indexer.JobStatus;
-import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
-import org.opengroup.osdu.indexer.service.IndexerService;
-import org.opengroup.osdu.indexer.service.ReindexService;
-import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
-import org.springframework.http.HttpStatus;
-import org.springframework.http.ResponseEntity;
-import org.springframework.web.bind.annotation.*;
-import org.springframework.web.context.annotation.RequestScope;
-
-import javax.inject.Inject;
-import javax.validation.Valid;
-import javax.validation.constraints.NotNull;
-import java.lang.reflect.Type;
-import java.util.List;
-
-@Log
-@RestController
-@RequestMapping("/_dps/task-handlers")
-@RequestScope
-public class RecordIndexerApi {
-
- @Inject
- private IndexerService indexerService;
- @Inject
- private ReindexService reIndexService;
-
- // THIS IS AN INTERNAL USE API ONLY
- // THAT MEANS WE DON'T DOCUMENT IT IN SWAGGER, ACCESS IS LIMITED TO CLOUD TASK QUEUE CALLS ONLY
- @PostMapping(path = "/index-worker", consumes = "application/json")
- @ApiOperation(hidden = true, value = "", notes = "")
- public ResponseEntity<JobStatus> indexWorker (
- @NotNull(message = SwaggerDoc.REQUEST_VALIDATION_NOT_NULL_BODY)
- @Valid @RequestBody RecordChangedMessages recordChangedMessages) throws Exception {
-
- if (recordChangedMessages.missingAccountId()) {
- throw new AppException(org.apache.http.HttpStatus.SC_BAD_REQUEST, "Invalid tenant",
- String.format("Required header: '%s' not found", DpsHeaders.DATA_PARTITION_ID));
- }
- try {
- if (recordChangedMessages == null) {
- log.info("record change messages is null");
- }
-
- Type listType = new TypeToken<List<RecordInfo>>() {
- }.getType();
- List<RecordInfo> recordInfos = new Gson().fromJson(recordChangedMessages.getData(), listType);
-
- if (recordInfos.size() == 0) {
- log.info("none of record-change message can be deserialized");
- return new ResponseEntity(HttpStatus.OK);
- }
- this.indexerService.processRecordChangedMessages(recordChangedMessages, recordInfos);
- return new ResponseEntity(HttpStatus.OK);
- } catch (AppException e) {
- throw e;
- } catch (JsonParseException e) {
- throw new AppException(org.apache.http.HttpStatus.SC_BAD_REQUEST, "Request payload parsing error", "Unable to parse request payload.", e);
- } catch (Exception e) {
- throw new AppException(org.apache.http.HttpStatus.SC_BAD_REQUEST, "Unknown error", "An unknown error has occurred.", e);
- }
- }
-
- // THIS IS AN INTERNAL USE API ONLY
- // THAT MEANS WE DON'T DOCUMENT IT IN SWAGGER, ACCESS IS LIMITED TO CLOUD TASK QUEUE CALLS ONLY
- @PostMapping("/reindex-worker")
- @ApiOperation(hidden = true, value = "", notes = "")
- public ResponseEntity reindex(
- @RequestBody @NotNull(message = SwaggerDoc.REQUEST_VALIDATION_NOT_NULL_BODY)
- @Valid RecordReindexRequest recordReindexRequest) {
-
- return new ResponseEntity(reIndexService.reindexRecords(recordReindexRequest),HttpStatus.OK);
- }
-}
\ No newline at end of file
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.api;
+
+import com.google.common.reflect.TypeToken;
+import com.google.gson.Gson;
+
+import com.google.gson.JsonParseException;
+import io.swagger.annotations.ApiOperation;
+import lombok.extern.java.Log;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
+import org.opengroup.osdu.indexer.SwaggerDoc;
+import org.opengroup.osdu.core.common.model.indexer.JobStatus;
+import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
+import org.opengroup.osdu.indexer.service.IndexerService;
+import org.opengroup.osdu.indexer.service.ReindexService;
+import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.*;
+import org.springframework.web.context.annotation.RequestScope;
+
+import javax.inject.Inject;
+import javax.validation.Valid;
+import javax.validation.constraints.NotNull;
+import java.lang.reflect.Type;
+import java.util.List;
+
+@Log
+@RestController
+@RequestMapping("/_dps/task-handlers")
+@RequestScope
+public class RecordIndexerApi {
+
+ @Inject
+ private IndexerService indexerService;
+ @Inject
+ private ReindexService reIndexService;
+
+ // THIS IS AN INTERNAL USE API ONLY
+ // THAT MEANS WE DON'T DOCUMENT IT IN SWAGGER, ACCESS IS LIMITED TO CLOUD TASK QUEUE CALLS ONLY
+ @PostMapping(path = "/index-worker", consumes = "application/json")
+ @ApiOperation(hidden = true, value = "", notes = "")
+ public ResponseEntity<JobStatus> indexWorker (
+ @NotNull(message = SwaggerDoc.REQUEST_VALIDATION_NOT_NULL_BODY)
+ @Valid @RequestBody RecordChangedMessages recordChangedMessages) throws Exception {
+
+ if (recordChangedMessages.missingAccountId()) {
+ throw new AppException(org.apache.http.HttpStatus.SC_BAD_REQUEST, "Invalid tenant",
+ String.format("Required header: '%s' not found", DpsHeaders.DATA_PARTITION_ID));
+ }
+ try {
+ if (recordChangedMessages == null) {
+ log.info("record change messages is null");
+ }
+
+ Type listType = new TypeToken<List<RecordInfo>>() {
+ }.getType();
+ List<RecordInfo> recordInfos = new Gson().fromJson(recordChangedMessages.getData(), listType);
+
+ if (recordInfos.size() == 0) {
+ log.info("none of record-change message can be deserialized");
+ return new ResponseEntity(HttpStatus.OK);
+ }
+ this.indexerService.processRecordChangedMessages(recordChangedMessages, recordInfos);
+ return new ResponseEntity(HttpStatus.OK);
+ } catch (AppException e) {
+ throw e;
+ } catch (JsonParseException e) {
+ throw new AppException(org.apache.http.HttpStatus.SC_BAD_REQUEST, "Request payload parsing error", "Unable to parse request payload.", e);
+ } catch (Exception e) {
+ throw new AppException(org.apache.http.HttpStatus.SC_BAD_REQUEST, "Unknown error", "An unknown error has occurred.", e);
+ }
+ }
+
+ // THIS IS AN INTERNAL USE API ONLY
+ // THAT MEANS WE DON'T DOCUMENT IT IN SWAGGER, ACCESS IS LIMITED TO CLOUD TASK QUEUE CALLS ONLY
+ @PostMapping("/reindex-worker")
+ @ApiOperation(hidden = true, value = "", notes = "")
+ public ResponseEntity<?> reindex(
+ @RequestBody @NotNull(message = SwaggerDoc.REQUEST_VALIDATION_NOT_NULL_BODY)
+ @Valid RecordReindexRequest recordReindexRequest) {
+ return new ResponseEntity<>(reIndexService.reindexRecords(recordReindexRequest, false), HttpStatus.OK);
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java
index 5df99479e436aa6a66ef8dd803c549922cdf40ea..0a6f6ba7a14c54fbf7b34f5e4acadf755a75109a 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java
@@ -1,53 +1,57 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.api;
-
-import org.opengroup.osdu.core.common.model.search.SearchServiceRole;
-import org.opengroup.osdu.indexer.logging.AuditLogger;
-import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
-import org.opengroup.osdu.indexer.service.ReindexService;
-import org.springframework.http.ResponseEntity;
-import org.springframework.security.access.prepost.PreAuthorize;
-import org.springframework.web.bind.annotation.PostMapping;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RestController;
-import org.springframework.web.context.annotation.RequestScope;
-
-import javax.inject.Inject;
-import javax.validation.Valid;
-import javax.validation.constraints.NotNull;
-
-import static java.util.Collections.singletonList;
-
-@RestController
-@RequestMapping("/reindex")
-@RequestScope
-public class ReindexApi {
-
- @Inject
- private ReindexService reIndexService;
- @Inject
- private AuditLogger auditLogger;
-
- @PreAuthorize("@authorizationFilter.hasPermission('" + SearchServiceRole.ADMIN + "')")
- @PostMapping
- public ResponseEntity reindex(
- @NotNull @Valid @RequestBody RecordReindexRequest recordReindexRequest) {
- this.reIndexService.reindexRecords(recordReindexRequest);
- this.auditLogger.getReindex(singletonList(recordReindexRequest.getKind()));
- return new ResponseEntity (org.springframework.http.HttpStatus.OK);
- }
-}
\ No newline at end of file
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.api;
+
+import org.opengroup.osdu.core.common.model.search.SearchServiceRole;
+import org.opengroup.osdu.indexer.logging.AuditLogger;
+import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
+import org.opengroup.osdu.indexer.service.IndexSchemaService;
+import org.opengroup.osdu.indexer.service.ReindexService;
+import org.springframework.http.ResponseEntity;
+import org.springframework.security.access.prepost.PreAuthorize;
+
+import org.springframework.web.bind.annotation.*;
+import org.springframework.web.context.annotation.RequestScope;
+
+import javax.inject.Inject;
+import javax.validation.Valid;
+import javax.validation.constraints.NotNull;
+
+import java.io.IOException;
+
+import static java.util.Collections.singletonList;
+
+@RestController
+@RequestMapping("/reindex")
+@RequestScope
+public class ReindexApi {
+
+ @Inject
+ private ReindexService reIndexService;
+ @Inject
+ private IndexSchemaService indexSchemaService;
+ @Inject
+ private AuditLogger auditLogger;
+
+ @PreAuthorize("@authorizationFilter.hasPermission('" + SearchServiceRole.ADMIN + "')")
+ @PostMapping
+ public ResponseEntity<?> reindex(
+ @NotNull @Valid @RequestBody RecordReindexRequest recordReindexRequest,
+ @RequestParam(value = "force_clean", defaultValue = "false") boolean forceClean) throws IOException {
+ this.reIndexService.reindexRecords(recordReindexRequest, this.indexSchemaService.isStorageSchemaSyncRequired(recordReindexRequest.getKind(), forceClean));
+ this.auditLogger.getReindex(singletonList(recordReindexRequest.getKind()));
+ return new ResponseEntity<>(org.springframework.http.HttpStatus.OK);
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java
index 3188710f54680d8e18a9f99542a19b6fa4fdeb0a..9e9026c815055600e3b2a58da8e2c82317e812cd 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java
@@ -1,165 +1,235 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.service;
-
-import com.google.common.base.Strings;
-import com.google.gson.Gson;
-import com.google.gson.JsonSyntaxException;
-import com.google.gson.internal.LinkedTreeMap;
-import com.google.gson.reflect.TypeToken;
-import org.apache.http.HttpStatus;
-import org.opengroup.osdu.core.common.model.indexer.IndexingStatus;
-import org.opengroup.osdu.core.common.model.indexer.JobStatus;
-import org.opengroup.osdu.indexer.util.parser.DateTimeParser;
-import org.opengroup.osdu.indexer.util.parser.GeoShapeParser;
-import org.opengroup.osdu.indexer.util.parser.NumberParser;
-import org.springframework.stereotype.Service;
-import org.springframework.web.context.annotation.RequestScope;
-
-import javax.inject.Inject;
-import java.lang.reflect.Type;
-import java.util.Map;
-
-@Service
-@RequestScope
-public class AttributeParsingServiceImpl implements IAttributeParsingService {
-
- private static final String GEOJSON = "GeoJSON";
-
- @Inject
- private NumberParser numberParser;
- @Inject
- private DateTimeParser dateTimeParser;
- @Inject
- private GeoShapeParser geoShapeParser;
- @Inject
- private GeometryConversionService geometryConversionService;
- @Inject
- private JobStatus jobStatus;
-
- @Override
- public void tryParseInteger(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
- try {
- int parsedInteger = this.numberParser.parseInteger(attributeName, attributeVal);
- dataMap.put(attributeName, parsedInteger);
- } catch (IllegalArgumentException e) {
- jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, e.getMessage(), String.format("record-id: %s | %s", recordId, e.getMessage()));
- }
- }
-
- @Override
- public void tryParseLong(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
- try {
- long parsedLong = this.numberParser.parseLong(attributeName, attributeVal);
- dataMap.put(attributeName, parsedLong);
- } catch (IllegalArgumentException e) {
- jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, e.getMessage(), String.format("record-id: %s | %s", recordId, e.getMessage()));
- }
- }
-
- @Override
- public void tryParseFloat(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
- try {
- float parsedFloat = this.numberParser.parseFloat(attributeName, attributeVal);
- dataMap.put(attributeName, parsedFloat);
- } catch (IllegalArgumentException e) {
- jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, e.getMessage(), String.format("record-id: %s | %s", recordId, e.getMessage()));
- }
- }
-
- @Override
- public void tryParseDouble(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
- try {
- double parsedDouble = this.numberParser.parseDouble(attributeName, attributeVal);
- dataMap.put(attributeName, parsedDouble);
- } catch (IllegalArgumentException e) {
- jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, e.getMessage(), String.format("record-id: %s | %s", recordId, e.getMessage()));
- }
- }
-
- @Override
- public void tryParseBoolean(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
- String val = attributeVal == null ? null : String.valueOf(attributeVal);
- dataMap.put(attributeName, Boolean.parseBoolean(val));
- }
-
- @Override
- public void tryParseDate(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
- String val = attributeVal == null ? null : String.valueOf(attributeVal);
- if (Strings.isNullOrEmpty(val)) {
- // skip indexing
- return;
- }
-
- String utcDate = this.dateTimeParser.convertDateObjectToUtc(val);
- if (Strings.isNullOrEmpty(utcDate)) {
- String parsingError = String.format("datetime parsing error: unknown format for attribute: %s | value: %s", attributeName, attributeVal);
- jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, parsingError, String.format("record-id: %s | %s", recordId, parsingError));
- } else {
- dataMap.put(attributeName, utcDate);
- }
- }
-
- @Override
- public void tryParseGeopoint(String recordId, String attributeName, Map<String, Object> storageRecordData, Map<String, Object> dataMap) {
-
- Object attributeVal = storageRecordData.get(attributeName);
-
- try {
- Type type = new TypeToken<Map<String, Double>>() {}.getType();
- Map<String, Double> positionMap = new Gson().fromJson(attributeVal.toString(), type);
-
- if (positionMap == null || positionMap.isEmpty()) return;
-
- Map<String, Double> position = this.geometryConversionService.tryGetGeopoint(positionMap);
-
- if (position == null || position.isEmpty()) return;
-
- dataMap.put(attributeName, position);
-
- // check if geo shape is not there and if it is not then create it in the schema as well as create the data.
- LinkedTreeMap<String, Object> map = (LinkedTreeMap) storageRecordData.get(GEOJSON);
- if (map == null || map.isEmpty()) {
- Map<String, Object> geometry = this.geometryConversionService.getGeopointGeoJson(positionMap);
-
- if (geometry == null) return;
-
- dataMap.put(DATA_GEOJSON_TAG, geometry);
- }
- } catch (JsonSyntaxException | IllegalArgumentException e) {
- String parsingError = String.format("geopoint parsing error: %s attribute: %s | value: %s", e.getMessage(), attributeName, attributeVal);
- jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, parsingError, String.format("record-id: %s | %s", recordId, parsingError));
- }
- }
-
- @Override
- public void tryParseGeojson(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
-
- try {
- Type type = new TypeToken<Map<String, Object>>() {}.getType();
- Map<String, Object> geoJsonMap = new Gson().fromJson(attributeVal.toString(), type);
-
- if (geoJsonMap == null || geoJsonMap.isEmpty()) return;
-
- this.geoShapeParser.parseGeoJson(geoJsonMap);
-
- dataMap.put(attributeName, geoJsonMap);
- } catch (JsonSyntaxException | IllegalArgumentException e) {
- String parsingError = String.format("geo-json shape parsing error: %s attribute: %s", e.getMessage(), attributeName);
- jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, parsingError, String.format("record-id: %s | %s", recordId, parsingError));
- }
- }
-}
-
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.service;
+
+import com.google.common.base.Strings;
+import com.google.gson.Gson;
+import com.google.gson.JsonSyntaxException;
+import com.google.gson.internal.LinkedTreeMap;
+import com.google.gson.reflect.TypeToken;
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.core.common.model.indexer.ElasticType;
+import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
+import org.opengroup.osdu.core.common.model.indexer.IndexingStatus;
+import org.opengroup.osdu.core.common.model.indexer.JobStatus;
+import org.opengroup.osdu.core.common.Constants;
+import org.opengroup.osdu.indexer.util.parser.BooleanParser;
+import org.opengroup.osdu.indexer.util.parser.DateTimeParser;
+import org.opengroup.osdu.indexer.util.parser.GeoShapeParser;
+import org.opengroup.osdu.indexer.util.parser.NumberParser;
+import org.springframework.stereotype.Service;
+import org.springframework.web.context.annotation.RequestScope;
+
+import javax.inject.Inject;
+import java.lang.reflect.Array;
+import java.lang.reflect.Type;
+import java.util.*;
+import java.util.function.BiFunction;
+
+@Service
+@RequestScope
+public class AttributeParsingServiceImpl implements IAttributeParsingService {
+
+ private static final String GEOJSON = "GeoJSON";
+ private static final String GEOMETRY_COLLECTION = "geometrycollection";
+ private static final String GEOMETRIES = "geometries";
+
+ @Inject
+ private NumberParser numberParser;
+ @Inject
+ private BooleanParser booleanParser;
+ @Inject
+ private DateTimeParser dateTimeParser;
+ @Inject
+ private GeoShapeParser geoShapeParser;
+ @Inject
+ private GeometryConversionService geometryConversionService;
+ @Inject
+ private JobStatus jobStatus;
+
+ @Override
+ public void tryParseValueArray(Class<?> attributeClass, String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
+ BiFunction<String, Object, ?> parser;
+ ElasticType elasticType = ElasticType.forValue(attributeClass.getSimpleName());
+ switch (elasticType) {
+ case DOUBLE:
+ parser = this.numberParser::parseDouble;
+ break;
+ case FLOAT:
+ parser = this.numberParser::parseFloat;
+ break;
+ case INTEGER:
+ parser = this.numberParser::parseInteger;
+ break;
+ case LONG:
+ parser = this.numberParser::parseLong;
+ break;
+ case BOOLEAN:
+ parser = this.booleanParser::parseBoolean;
+ break;
+ case DATE:
+ parser = this.dateTimeParser::parseDate;
+ // DateTime parser output is String
+ attributeClass = String.class;
+ break;
+ default:
+ throw new IllegalArgumentException("Invalid array attribute type");
+ }
+
+ try {
+ List<String> parsedStringList = isArrayType(attributeVal);
+ List out = new ArrayList<>();
+ for (Object o : parsedStringList) {
+ out.add(parser.apply(attributeName, o));
+ }
+ Object parsedAttribute = toTypeArray(attributeClass, out);
+ dataMap.put(attributeName, parsedAttribute);
+ } catch (IllegalArgumentException e) {
+ this.jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, e.getMessage(), String.format("record-id: %s | %s", recordId, e.getMessage()));
+ }
+ }
+
+ @Override
+ public void tryParseInteger(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
+ try {
+ int parsedInteger = this.numberParser.parseInteger(attributeName, attributeVal);
+ dataMap.put(attributeName, parsedInteger);
+ } catch (IllegalArgumentException e) {
+ jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, e.getMessage(), String.format("record-id: %s | %s", recordId, e.getMessage()));
+ }
+ }
+
+ @Override
+ public void tryParseLong(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
+ try {
+ long parsedLong = this.numberParser.parseLong(attributeName, attributeVal);
+ dataMap.put(attributeName, parsedLong);
+ } catch (IllegalArgumentException e) {
+ jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, e.getMessage(), String.format("record-id: %s | %s", recordId, e.getMessage()));
+ }
+ }
+
+ @Override
+ public void tryParseFloat(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
+ try {
+ float parsedFloat = this.numberParser.parseFloat(attributeName, attributeVal);
+ dataMap.put(attributeName, parsedFloat);
+ } catch (IllegalArgumentException e) {
+ jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, e.getMessage(), String.format("record-id: %s | %s", recordId, e.getMessage()));
+ }
+ }
+
+ @Override
+ public void tryParseDouble(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
+ try {
+ double parsedDouble = this.numberParser.parseDouble(attributeName, attributeVal);
+ dataMap.put(attributeName, parsedDouble);
+ } catch (IllegalArgumentException e) {
+ jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, e.getMessage(), String.format("record-id: %s | %s", recordId, e.getMessage()));
+ }
+ }
+
+ @Override
+ public void tryParseBoolean(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
+ boolean parsedBoolean = this.booleanParser.parseBoolean(attributeName, attributeVal);
+ dataMap.put(attributeName, parsedBoolean);
+ }
+
+ @Override
+ public void tryParseDate(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
+ try {
+ String parsedDate = this.dateTimeParser.parseDate(attributeName, attributeVal);
+ if (parsedDate == null) return;
+ dataMap.put(attributeName, parsedDate);
+ } catch (IllegalArgumentException e) {
+ jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, e.getMessage(), String.format("record-id: %s | %s", recordId, e.getMessage()));
+ }
+ }
+
+ @Override
+ public void tryParseGeopoint(String recordId, String attributeName, Map<String, Object> storageRecordData, Map<String, Object> dataMap) {
+
+ Object attributeVal = storageRecordData.get(attributeName);
+
+ try {
+ Type type = new TypeToken<Map<String, Double>>() {}.getType();
+ Map<String, Double> positionMap = new Gson().fromJson(attributeVal.toString(), type);
+
+ if (positionMap == null || positionMap.isEmpty()) return;
+
+ Map<String, Double> position = this.geometryConversionService.tryGetGeopoint(positionMap);
+
+ if (position == null || position.isEmpty()) return;
+
+ dataMap.put(attributeName, position);
+
+ // check if geo shape is not there and if it is not then create it in the schema as well as create the data.
+ LinkedTreeMap<String, Object> map = (LinkedTreeMap) storageRecordData.get(GEOJSON);
+ if (map == null || map.isEmpty()) {
+ Map<String, Object> geometry = this.geometryConversionService.getGeopointGeoJson(positionMap);
+
+ if (geometry == null) return;
+
+ dataMap.put(DATA_GEOJSON_TAG, geometry);
+ }
+ } catch (JsonSyntaxException | IllegalArgumentException e) {
+ String parsingError = String.format("geopoint parsing error: %s attribute: %s | value: %s", e.getMessage(), attributeName, attributeVal);
+ jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, parsingError, String.format("record-id: %s | %s", recordId, parsingError));
+ }
+ }
+
+ @Override
+ public void tryParseGeojson(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
+
+ try {
+ Type type = new TypeToken<Map<String, Object>>() {}.getType();
+ Map<String, Object> geoJsonMap = new Gson().fromJson(attributeVal.toString(), type);
+
+ if (geoJsonMap == null || geoJsonMap.isEmpty()) return;
+
+ this.geoShapeParser.parseGeoJson(geoJsonMap);
+
+ dataMap.put(attributeName, geoJsonMap);
+ } catch (JsonSyntaxException | IllegalArgumentException e) {
+ String parsingError = String.format("geo-json shape parsing error: %s attribute: %s", e.getMessage(), attributeName);
+ jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, parsingError, String.format("record-id: %s | %s", recordId, parsingError));
+ }
+ }
+
+
+ private List<String> isArrayType(Object attributeVal) {
+ try {
+ String value = attributeVal == null ? null : String.valueOf(attributeVal);
+ if (attributeVal == null || Strings.isNullOrEmpty(value)) {
+ return Collections.EMPTY_LIST;
+ }
+
+ Gson converter = new Gson();
+ Type type = new TypeToken<List<String>>() {}.getType();
+ return converter.fromJson(value, type);
+ } catch (JsonSyntaxException e) {
+ throw new IllegalArgumentException("array parsing error, not a valid array");
+ }
+ }
+
+ private <N> N toTypeArray(Class<N> fieldClass, List<?> list) {
+ Object array = Array.newInstance(fieldClass, list.size());
+ for (int i = 0; i < list.size(); i++) {
+ Array.set(array, i, list.get(i));
+ }
+ return (N) array;
+ }
+}
+
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java
index 8d7647d31e4811d5915b0fd73ba7f8c159155aa6..4050444a40b1bef8f184d8b09c80a078d9e73729 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java
@@ -9,6 +9,7 @@ public interface IAttributeParsingService {
public static final String RECORD_GEOJSON_TAG = "GeoJSON.features.geometry";
public static final String DATA_GEOJSON_TAG = "x-geojson";
+ void tryParseValueArray(Class<?> attributeClass, String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap);
void tryParseInteger(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap);
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaService.java
index e0a7f7a649ec2788162fd43c7cb800e021e07983..7e512ae7af32a70650372cfd2a2e48bed6093e27 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaService.java
@@ -1,29 +1,34 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.service;
-
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
-import org.opengroup.osdu.core.common.model.indexer.OperationType;
-
-import java.io.IOException;
-import java.util.Map;
-
-public interface IndexSchemaService {
-
- IndexSchema getIndexerInputSchema(String kind) throws AppException;
-
- void processSchemaMessages(Map<String, OperationType> schemaMsgs) throws IOException;
-}
\ No newline at end of file
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.service;
+
+import org.elasticsearch.ElasticsearchException;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
+import org.opengroup.osdu.core.common.model.indexer.OperationType;
+
+import java.io.IOException;
+import java.util.Map;
+
+public interface IndexSchemaService {
+
+ IndexSchema getIndexerInputSchema(String kind, boolean invalidateCached) throws AppException;
+
+ void processSchemaMessages(Map<String, OperationType> schemaMsgs) throws IOException;
+
+ void syncIndexMappingWithStorageSchema(String kind) throws ElasticsearchException, IOException, AppException;
+
+ boolean isStorageSchemaSyncRequired(String kind, boolean forceClean) throws IOException;
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
index 6148991bd175232a10733fbe1cd0e24daf70b29e..4d0240b652ebe86c92fe65b7f0d472ad3f52e63f 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
@@ -16,6 +16,7 @@ package org.opengroup.osdu.indexer.service;
import com.google.common.base.Strings;
import com.google.gson.Gson;
+import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchStatusException;
import org.elasticsearch.client.RestHighLevelClient;
import org.opengroup.osdu.core.common.model.storage.SchemaItem;
@@ -84,7 +85,7 @@ public class IndexSchemaServiceImpl implements IndexSchemaService {
if (msg.getValue() == OperationType.create_schema) {
// reset cache and get new schema
this.invalidateCache(kind);
- IndexSchema schemaObj = this.getIndexerInputSchema(kind);
+ IndexSchema schemaObj = this.getIndexerInputSchema(kind, true);
if (schemaObj.isDataSchemaMissing()) {
log.warning(String.format("schema not found for kind: %s", kind));
return;
@@ -118,7 +119,11 @@ public class IndexSchemaServiceImpl implements IndexSchemaService {
}
@Override
- public IndexSchema getIndexerInputSchema(String kind) throws AppException {
+ public IndexSchema getIndexerInputSchema(String kind, boolean invalidateCached) throws AppException {
+
+ if (invalidateCached) {
+ this.invalidateCache(kind);
+ }
try {
String schema = (String) this.schemaCache.get(kind);
@@ -154,6 +159,26 @@ public class IndexSchemaServiceImpl implements IndexSchemaService {
}
}
+ public void syncIndexMappingWithStorageSchema(String kind) throws ElasticsearchException, IOException, AppException {
+ String index = this.elasticIndexNameResolver.getIndexNameFromKind(kind);
+ try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
+ if (this.indicesService.isIndexExist(restClient, index)) {
+ this.indicesService.deleteIndex(restClient, index);
+ this.log.info(String.format("deleted index: %s", index));
+ }
+ IndexSchema schemaObj = this.getIndexerInputSchema(kind, true);
+ this.indicesService.createIndex(restClient, index, null, schemaObj.getType(), this.mappingService.getIndexMappingFromRecordSchema(schemaObj));
+ }
+ }
+
+ public boolean isStorageSchemaSyncRequired(String kind, boolean forceClean) throws IOException {
+ String index = this.elasticIndexNameResolver.getIndexNameFromKind(kind);
+ try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
+ boolean indexExist = this.indicesService.isIndexExist(restClient, index);
+ return !indexExist || forceClean;
+ }
+ }
+
private void invalidateCache(String kind) {
String schema = (String) this.schemaCache.get(kind);
if (!Strings.isNullOrEmpty(schema)) this.schemaCache.delete(kind);
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
index 33d34bac44b4a428df4f10d62a5962939f7fbc5f..8882bd76e5f560f708c0b01b0caa69cbe39b192a 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
@@ -1,327 +1,337 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.service;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.http.HttpStatus;
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest;
-import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
-import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData;
-import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
-import org.elasticsearch.action.support.master.AcknowledgedResponse;
-import org.elasticsearch.client.Request;
-import org.elasticsearch.client.RequestOptions;
-import org.elasticsearch.client.Response;
-import org.elasticsearch.client.RestHighLevelClient;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.reindex.BulkByScrollResponse;
-import org.elasticsearch.index.reindex.UpdateByQueryRequest;
-
-import com.google.gson.Gson;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.Constants;
-import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
-import org.opengroup.osdu.core.common.search.Preconditions;
-import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
-import org.opengroup.osdu.core.common.model.indexer.Records;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.indexer.util.ElasticClientHandler;
-import org.springframework.stereotype.Service;
-import javax.inject.Inject;
-
-@Service
-public class IndexerMappingServiceImpl extends MappingServiceImpl implements IndexerMappingService {
-
- @Inject
- private JaxRsDpsLog log;
- @Inject
- private ElasticClientHandler elasticClientHandler;
- private TimeValue REQUEST_TIMEOUT = TimeValue.timeValueMinutes(1);
-
-
- /**
- * Create a new type in Elasticsearch
- *
- * @param client Elasticsearch client
- * @param index Index name
- * @param merge Try to merge mapping if type already exists
- * @throws IOException if cannot create mapping
- */
- public String createMapping(RestHighLevelClient client, IndexSchema schema, String index, boolean merge) throws IOException {
-
- Map<String, Object> mappingMap = this.getIndexMappingFromRecordSchema(schema);
- String mapping = new Gson().toJson(mappingMap, Map.class);
- this.createMappingWithJson(client, index, schema.getType(), mapping, merge);
- return mapping;
- }
-
- /*
- * Read schema mapping
- *
- * @param schema Index schema
- * @param type Mapping type
- * @return String JSON represnetation of type and elastic type
- *
- * sample index mapping:
- * "properties": {
- * all meta attributes
- * "acl": {
- * "properties": {
- mapping of all roles
- * }
- * },
- * "legal": {
- * "properties": {
- * mapping of all legal properties
- * }
- * }
- * "data": {
- * "properties": {
- * all data-source attributes
- * }
- * }
- * }
- * */
- public Map<String, Object> getIndexMappingFromRecordSchema(IndexSchema schema) {
-
- // entire property block
- Map<String, Object> properties = new HashMap<>();
-
- // meta attribute
- Map<String, Object> metaMapping = new HashMap<>();
- for (Map.Entry<String, Object> entry : schema.getMetaSchema().entrySet()) {
- String key = entry.getKey();
- if (key.equals(RecordMetaAttribute.ACL.getValue()) || key.equals(RecordMetaAttribute.LEGAL.getValue()) || key.equals(RecordMetaAttribute.ANCESTRY.getValue()) || key.equals(RecordMetaAttribute.INDEX_STATUS.getValue())) {
- metaMapping.put(key, entry.getValue());
- } else {
- metaMapping.put(key, Records.Type.builder().type(entry.getValue().toString()).build());
- }
- }
-
- // data-source attributes
- Map<String, Object> dataMapping = new HashMap<>();
- if (schema.getDataSchema() != null) {
- for (Map.Entry<String, String> entry : schema.getDataSchema().entrySet()) {
- dataMapping.put(entry.getKey(), Records.Type.builder().type(entry.getValue()).build());
- }
-
- // inner properties.data.properties block
- Map<String, Object> dataProperties = new HashMap<>();
- dataProperties.put(Constants.PROPERTIES, dataMapping);
-
- // data & meta block
- properties.put(Constants.DATA, dataProperties);
- }
- properties.putAll(metaMapping);
-
- // entire document properties block
- Map<String, Object> documentMapping = new HashMap<>();
- documentMapping.put(Constants.PROPERTIES, properties);
-
- // don't add dynamic mapping
- documentMapping.put("dynamic", false);
-
- return documentMapping;
- }
-
- @Override
- public void updateIndexMappingForIndicesOfSameType(Set<String> indices, String fieldName) throws Exception {
- try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
- if(!updateMappingToEnableKeywordIndexingForField(restClient,indices,fieldName)){
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error", "Error updating index mapping.", String.format("Failed to get confirmation from elastic server mapping update for indices: %s", indices));
- }
- }
- }
-
- private boolean updateMappingToEnableKeywordIndexingForField(RestHighLevelClient client, Set<String> indicesSet, String fieldName) throws IOException {
- String[] indices = indicesSet.toArray(new String[indicesSet.size()]);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> indexMappingMap = getIndexFieldMap(new String[]{"data."+fieldName}, client, indices);
- boolean failure = false;
- for (String index : indicesSet) {
- if (indexMappingMap.get(index)!=null && updateMappingForAllIndicesOfSameTypeToEnableKeywordIndexingForField(client, index, indexMappingMap.get(index), fieldName)) {
- log.info(String.format("Updated field: %s | index: %s", fieldName, index));
- } else {
- failure=true;
- log.warning(String.format("Failed to update field: %s | index %s", fieldName, index));
- }
- }
- return !failure;
- }
-
- private Map<String, Map<String, Map<String, FieldMappingMetaData>>> getIndexFieldMap(String[] fieldNames, RestHighLevelClient client, String[] indices) throws IOException {
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> indexMappingMap = new HashMap<>();
- GetFieldMappingsRequest request = new GetFieldMappingsRequest();
- request.indices(indices);
- request.fields(fieldNames);
- try {
- GetFieldMappingsResponse response = client.indices().getFieldMapping(request, RequestOptions.DEFAULT);
- if (response != null && !response.mappings().isEmpty()) {
- final Map<String, Map<String, Map<String, FieldMappingMetaData>>> mappings = response.mappings();
- for (String index : indices) {
- //extract mapping of each index
- final Map<String, Map<String, FieldMappingMetaData>> indexMapping = mappings.get(index);
- if (indexMapping != null && !indexMapping.isEmpty()) {
- indexMappingMap.put(index, indexMapping);
- }
- }
- }
-
- return indexMappingMap;
- } catch (ElasticsearchException e) {
- log.error(String.format("Failed to get indices: %s. | Error: %s", Arrays.toString(indices), e));
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error", "Error getting indices.", String.format("Failed to get indices error: %s", Arrays.toString(indices)));
- }
- }
-
- private boolean updateMappingForAllIndicesOfSameTypeToEnableKeywordIndexingForField(RestHighLevelClient client, String index, Map<String, Map<String, FieldMappingMetaData>> indexMapping, String fieldName) throws IOException {
- PutMappingRequest request = new PutMappingRequest(index);
- String type = indexMapping.keySet().iterator().next();
- if(type.isEmpty()) {
- log.error(String.format("Could not find type of the mappings for index: %s.", index));
- return false;
- }
-
- request.type(type);
- request.timeout(REQUEST_TIMEOUT);
- Map<String, FieldMappingMetaData> metaData = indexMapping.get(type);
- if(metaData==null || metaData.get("data." + fieldName)==null) {
- log.error(String.format("Could not find field: %s in the mapping of index: %s.", fieldName, index));
- return false;
- }
-
- FieldMappingMetaData fieldMetaData = metaData.get("data." + fieldName);
- Map<String, Object> source = fieldMetaData.sourceAsMap();
- if(!source.containsKey(fieldName)){
- log.error(String.format("Could not find field: %s in the mapping of index: %s.", fieldName, index));
- return false;
- }
-
- //Index the field with additional keyword type
- Map<String, Object> keywordMap = new HashMap<>();
- keywordMap.put(Constants.TYPE, "keyword");
- Map<String, Object> fieldIndexTypeMap = new HashMap<>();
- fieldIndexTypeMap.put("keyword", keywordMap);
- Map<String, Object> dataFieldMap = (Map<String, Object>) source.get(fieldName);
- dataFieldMap.put("fields", fieldIndexTypeMap);
- Map<String, Object> dataProperties = new HashMap<>();
- dataProperties.put(fieldName, dataFieldMap);
- Map<String, Object> mapping = new HashMap<>();
- mapping.put(Constants.PROPERTIES, dataProperties);
- Map<String, Object> data = new HashMap<>();
- data.put(Constants.DATA,mapping);
- Map<String, Object> properties = new HashMap<>();
- properties.put(Constants.PROPERTIES, data);
-
- request.source(new Gson().toJson(properties), XContentType.JSON);
-
- try {
- AcknowledgedResponse response = client.indices().putMapping(request, RequestOptions.DEFAULT);
- boolean isIndicesUpdated = updateIndices(client, index);
- return response.isAcknowledged() && isIndicesUpdated;
-
- } catch (Exception e) {
- log.error(String.format("Could not update mapping of index: %s. | Error: %s", index, e));
- return false;
- }
- }
-
- private boolean updateIndices(RestHighLevelClient client, String index) throws IOException {
- UpdateByQueryRequest request = new UpdateByQueryRequest(index);
- request.setConflicts("proceed");
- BulkByScrollResponse response = client.updateByQuery(request, RequestOptions.DEFAULT);
- if(!response.getBulkFailures().isEmpty()) {
- log.error(String.format("Could not update index: %s.",index));
- return false;
- }
- return true;
- }
-
- /**
- * Create a new type in Elasticsearch
- *
- * @param client Elasticsearch client
- * @param index Index name
- * @param type Type name
- * @param mapping Mapping if any, null if no specific mapping
- * @param merge Try to merge mapping if type already exists
- * @throws IOException if cannot create index mapping with input json
- */
- private void createMappingWithJson(RestHighLevelClient client, String index, String type, String mapping, boolean merge)
- throws IOException {
-
- boolean mappingExist = isTypeExist(client, index, type);
- if (merge || !mappingExist) {
- createTypeWithMappingInElasticsearch(client, index, type, mapping);
- }
- }
-
- /**
- * Check if a type already exists
- *
- * @param client Elasticsearch client
- * @param index Index name
- * @param type Type name
- * @return true if type already exists
- * @throws IOException in case Elasticsearch responded with a status code that indicated an error
- */
- public boolean isTypeExist(RestHighLevelClient client, String index, String type) throws IOException {
-
- Request request = new Request("HEAD", "/" + index + "/_mapping/" + type);
- Response response = client.getLowLevelClient().performRequest(request);
- return response.getStatusLine().getStatusCode() == 200;
- }
-
- /**
- * Create a new type in Elasticsearch
- *
- * @param client Elasticsearch client
- * @param index Index name
- * @param type Type name
- * @param mapping Mapping if any, null if no specific mapping
- * @throws IOException if mapping cannot be created
- */
- private Boolean createTypeWithMappingInElasticsearch(RestHighLevelClient client, String index, String type, String mapping) throws IOException {
-
- Preconditions.checkNotNull(client, "client cannot be null");
- Preconditions.checkNotNull(index, "index cannot be null");
- Preconditions.checkNotNull(type, "type cannot be null");
-
- try {
- if (mapping != null) {
- PutMappingRequest request = new PutMappingRequest(index);
- request.type(type);
- request.source(mapping, XContentType.JSON);
- request.timeout(REQUEST_TIMEOUT);
- AcknowledgedResponse response = client.indices().putMapping(request, RequestOptions.DEFAULT);
- return response.isAcknowledged();
- }
- } catch (ElasticsearchException e) {
- throw new AppException(
- e.status().getStatus(),
- e.getMessage(),
- String.format("Could not create type mapping %s/%s.", index, type),
- String.format("Failed creating mapping: %s", mapping),
- e);
- }
- return false;
- }
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.service;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.http.HttpStatus;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest;
+import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
+import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.reindex.BulkByScrollResponse;
+import org.elasticsearch.index.reindex.UpdateByQueryRequest;
+
+import com.google.gson.Gson;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.Constants;
+import org.opengroup.osdu.core.common.model.indexer.DEAnalyzerType;
+import org.opengroup.osdu.core.common.model.indexer.ElasticType;
+import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
+import org.opengroup.osdu.core.common.search.Preconditions;
+import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
+import org.opengroup.osdu.core.common.model.indexer.Records;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.indexer.util.ElasticClientHandler;
+import org.springframework.stereotype.Service;
+import javax.inject.Inject;
+
+import static org.opengroup.osdu.core.common.search.Config.isPreDemo;
+
+@Service
+public class IndexerMappingServiceImpl extends MappingServiceImpl implements IndexerMappingService {
+
+ @Inject
+ private JaxRsDpsLog log;
+ @Inject
+ private ElasticClientHandler elasticClientHandler;
+ private TimeValue REQUEST_TIMEOUT = TimeValue.timeValueMinutes(1);
+
+
+ /**
+ * Create a new type in Elasticsearch
+ *
+ * @param client Elasticsearch client
+ * @param index Index name
+ * @param merge Try to merge mapping if type already exists
+ * @throws IOException if cannot create mapping
+ */
+ public String createMapping(RestHighLevelClient client, IndexSchema schema, String index, boolean merge) throws IOException {
+
+ Map<String, Object> mappingMap = this.getIndexMappingFromRecordSchema(schema);
+ String mapping = new Gson().toJson(mappingMap, Map.class);
+ this.createMappingWithJson(client, index, schema.getType(), mapping, merge);
+ return mapping;
+ }
+
+ /*
+ * Read schema mapping
+ *
+ * @param schema Index schema
+ * @param type Mapping type
+ * @return String JSON represnetation of type and elastic type
+ *
+ * sample index mapping:
+ * "properties": {
+ * all meta attributes
+ * "acl": {
+ * "properties": {
+ mapping of all roles
+ * }
+ * },
+ * "legal": {
+ * "properties": {
+ * mapping of all legal properties
+ * }
+ * }
+ * "data": {
+ * "properties": {
+ * all data-source attributes
+ * }
+ * }
+ * }
+ * */
+ public Map<String, Object> getIndexMappingFromRecordSchema(IndexSchema schema) {
+
+ // entire property block
+ Map<String, Object> properties = new HashMap<>();
+
+ // meta attribute
+ Map<String, Object> metaMapping = new HashMap<>();
+ for (Map.Entry<String, Object> entry : schema.getMetaSchema().entrySet()) {
+ String key = entry.getKey();
+ if (key.equals(RecordMetaAttribute.ACL.getValue()) || key.equals(RecordMetaAttribute.LEGAL.getValue()) || key.equals(RecordMetaAttribute.ANCESTRY.getValue()) || key.equals(RecordMetaAttribute.INDEX_STATUS.getValue())) {
+ metaMapping.put(key, entry.getValue());
+ } else {
+ metaMapping.put(key, Records.Type.builder().type(entry.getValue().toString()).build());
+ }
+ }
+
+ // data-source attributes
+ Map<String, Object> dataMapping = new HashMap<>();
+ if (schema.getDataSchema() != null) {
+ for (Map.Entry<String, String> entry : schema.getDataSchema().entrySet()) {
+ // Apply de_indexer_analyzer and de_search_analyzer to TEXT field
+ if (isPreDemo() && ElasticType.TEXT.getValue().equalsIgnoreCase(entry.getValue())) {
+ log.info(String.format("indexing %s with custom analyzer", entry.getKey()));
+ dataMapping.put(entry.getKey(), Records.Analyzer.builder().type(entry.getValue()).analyzer(DEAnalyzerType.INDEXER_ANALYZER.getValue()).search_analyzer(DEAnalyzerType.SEARCH_ANALYZER.getValue()).build());
+ } else {
+ dataMapping.put(entry.getKey(), Records.Type.builder().type(entry.getValue()).build());
+ }
+ }
+
+ // inner properties.data.properties block
+ Map<String, Object> dataProperties = new HashMap<>();
+ dataProperties.put(Constants.PROPERTIES, dataMapping);
+
+ // data & meta block
+ properties.put(Constants.DATA, dataProperties);
+ }
+ properties.putAll(metaMapping);
+
+ // entire document properties block
+ Map<String, Object> documentMapping = new HashMap<>();
+ documentMapping.put(Constants.PROPERTIES, properties);
+
+ // don't add dynamic mapping
+ documentMapping.put("dynamic", false);
+
+ return documentMapping;
+ }
+
+ @Override
+ public void updateIndexMappingForIndicesOfSameType(Set<String> indices, String fieldName) throws Exception {
+ try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
+ if(!updateMappingToEnableKeywordIndexingForField(restClient,indices,fieldName)){
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error", "Error updating index mapping.", String.format("Failed to get confirmation from elastic server mapping update for indices: %s", indices));
+ }
+ }
+ }
+
+ private boolean updateMappingToEnableKeywordIndexingForField(RestHighLevelClient client, Set<String> indicesSet, String fieldName) throws IOException {
+ String[] indices = indicesSet.toArray(new String[indicesSet.size()]);
+ Map<String, Map<String, Map<String, FieldMappingMetaData>>> indexMappingMap = getIndexFieldMap(new String[]{"data."+fieldName}, client, indices);
+ boolean failure = false;
+ for (String index : indicesSet) {
+ if (indexMappingMap.get(index)!=null && updateMappingForAllIndicesOfSameTypeToEnableKeywordIndexingForField(client, index, indexMappingMap.get(index), fieldName)) {
+ log.info(String.format("Updated field: %s | index: %s", fieldName, index));
+ } else {
+ failure=true;
+ log.warning(String.format("Failed to update field: %s | index %s", fieldName, index));
+ }
+ }
+ return !failure;
+ }
+
+ private Map<String, Map<String, Map<String, FieldMappingMetaData>>> getIndexFieldMap(String[] fieldNames, RestHighLevelClient client, String[] indices) throws IOException {
+ Map<String, Map<String, Map<String, FieldMappingMetaData>>> indexMappingMap = new HashMap<>();
+ GetFieldMappingsRequest request = new GetFieldMappingsRequest();
+ request.indices(indices);
+ request.fields(fieldNames);
+ try {
+ GetFieldMappingsResponse response = client.indices().getFieldMapping(request, RequestOptions.DEFAULT);
+ if (response != null && !response.mappings().isEmpty()) {
+ final Map<String, Map<String, Map<String, FieldMappingMetaData>>> mappings = response.mappings();
+ for (String index : indices) {
+ //extract mapping of each index
+ final Map<String, Map<String, FieldMappingMetaData>> indexMapping = mappings.get(index);
+ if (indexMapping != null && !indexMapping.isEmpty()) {
+ indexMappingMap.put(index, indexMapping);
+ }
+ }
+ }
+
+ return indexMappingMap;
+ } catch (ElasticsearchException e) {
+ log.error(String.format("Failed to get indices: %s. | Error: %s", Arrays.toString(indices), e));
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error", "Error getting indices.", String.format("Failed to get indices error: %s", Arrays.toString(indices)));
+ }
+ }
+
+ private boolean updateMappingForAllIndicesOfSameTypeToEnableKeywordIndexingForField(RestHighLevelClient client, String index, Map<String, Map<String, FieldMappingMetaData>> indexMapping, String fieldName) throws IOException {
+ PutMappingRequest request = new PutMappingRequest(index);
+ String type = indexMapping.keySet().iterator().next();
+ if(type.isEmpty()) {
+ log.error(String.format("Could not find type of the mappings for index: %s.", index));
+ return false;
+ }
+
+ request.type(type);
+ request.timeout(REQUEST_TIMEOUT);
+ Map<String, FieldMappingMetaData> metaData = indexMapping.get(type);
+ if(metaData==null || metaData.get("data." + fieldName)==null) {
+ log.error(String.format("Could not find field: %s in the mapping of index: %s.", fieldName, index));
+ return false;
+ }
+
+ FieldMappingMetaData fieldMetaData = metaData.get("data." + fieldName);
+ Map<String, Object> source = fieldMetaData.sourceAsMap();
+ if(!source.containsKey(fieldName)){
+ log.error(String.format("Could not find field: %s in the mapping of index: %s.", fieldName, index));
+ return false;
+ }
+
+ //Index the field with additional keyword type
+ Map<String, Object> keywordMap = new HashMap<>();
+ keywordMap.put(Constants.TYPE, "keyword");
+ Map<String, Object> fieldIndexTypeMap = new HashMap<>();
+ fieldIndexTypeMap.put("keyword", keywordMap);
+ Map<String, Object> dataFieldMap = (Map<String, Object>) source.get(fieldName);
+ dataFieldMap.put("fields", fieldIndexTypeMap);
+ Map<String, Object> dataProperties = new HashMap<>();
+ dataProperties.put(fieldName, dataFieldMap);
+ Map<String, Object> mapping = new HashMap<>();
+ mapping.put(Constants.PROPERTIES, dataProperties);
+ Map<String, Object> data = new HashMap<>();
+ data.put(Constants.DATA,mapping);
+ Map<String, Object> properties = new HashMap<>();
+ properties.put(Constants.PROPERTIES, data);
+
+ request.source(new Gson().toJson(properties), XContentType.JSON);
+
+ try {
+ AcknowledgedResponse response = client.indices().putMapping(request, RequestOptions.DEFAULT);
+ boolean isIndicesUpdated = updateIndices(client, index);
+ return response.isAcknowledged() && isIndicesUpdated;
+
+ } catch (Exception e) {
+ log.error(String.format("Could not update mapping of index: %s. | Error: %s", index, e));
+ return false;
+ }
+ }
+
+ private boolean updateIndices(RestHighLevelClient client, String index) throws IOException {
+ UpdateByQueryRequest request = new UpdateByQueryRequest(index);
+ request.setConflicts("proceed");
+ BulkByScrollResponse response = client.updateByQuery(request, RequestOptions.DEFAULT);
+ if(!response.getBulkFailures().isEmpty()) {
+ log.error(String.format("Could not update index: %s.",index));
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Create a new type in Elasticsearch
+ *
+ * @param client Elasticsearch client
+ * @param index Index name
+ * @param type Type name
+ * @param mapping Mapping if any, null if no specific mapping
+ * @param merge Try to merge mapping if type already exists
+ * @throws IOException if cannot create index mapping with input json
+ */
+ private void createMappingWithJson(RestHighLevelClient client, String index, String type, String mapping, boolean merge)
+ throws IOException {
+
+ boolean mappingExist = isTypeExist(client, index, type);
+ if (merge || !mappingExist) {
+ createTypeWithMappingInElasticsearch(client, index, type, mapping);
+ }
+ }
+
+ /**
+ * Check if a type already exists
+ *
+ * @param client Elasticsearch client
+ * @param index Index name
+ * @param type Type name
+ * @return true if type already exists
+ * @throws IOException in case Elasticsearch responded with a status code that indicated an error
+ */
+ public boolean isTypeExist(RestHighLevelClient client, String index, String type) throws IOException {
+
+ Request request = new Request("HEAD", "/" + index + "/_mapping/" + type);
+ Response response = client.getLowLevelClient().performRequest(request);
+ return response.getStatusLine().getStatusCode() == 200;
+ }
+
+ /**
+ * Create a new type in Elasticsearch
+ *
+ * @param client Elasticsearch client
+ * @param index Index name
+ * @param type Type name
+ * @param mapping Mapping if any, null if no specific mapping
+ * @throws IOException if mapping cannot be created
+ */
+ private Boolean createTypeWithMappingInElasticsearch(RestHighLevelClient client, String index, String type, String mapping) throws IOException {
+
+ Preconditions.checkNotNull(client, "client cannot be null");
+ Preconditions.checkNotNull(index, "index cannot be null");
+ Preconditions.checkNotNull(type, "type cannot be null");
+
+ try {
+ if (mapping != null) {
+ PutMappingRequest request = new PutMappingRequest(index);
+ request.type(type);
+ request.source(mapping, XContentType.JSON);
+ request.timeout(REQUEST_TIMEOUT);
+ AcknowledgedResponse response = client.indices().putMapping(request, RequestOptions.DEFAULT);
+ return response.isAcknowledged();
+ }
+ } catch (ElasticsearchException e) {
+ throw new AppException(
+ e.status().getStatus(),
+ e.getMessage(),
+ String.format("Could not create type mapping %s/%s.", index, type),
+ String.format("Failed creating mapping: %s", mapping),
+ e);
+ }
+ return false;
+ }
}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
index d346d5c3653b7c7aa9fea7413d2492d944b89298..0fa06829611148c5ed9525dc71a1c4649f995d43 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
@@ -1,583 +1,489 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.service;
-
-import com.google.gson.Gson;
-
-import lombok.extern.java.Log;
-import org.apache.http.HttpStatus;
-import org.elasticsearch.ElasticsearchStatusException;
-import org.elasticsearch.action.bulk.BulkItemResponse;
-import org.elasticsearch.action.bulk.BulkRequest;
-import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.delete.DeleteRequest;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.update.UpdateRequest;
-import org.elasticsearch.client.RequestOptions;
-import org.elasticsearch.client.RestHighLevelClient;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.rest.RestStatus;
-import org.opengroup.osdu.core.common.model.entitlements.Acl;
-import org.opengroup.osdu.core.common.Constants;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.indexer.*;
-import org.opengroup.osdu.core.common.model.storage.ConversionStatus;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.indexer.provider.interfaces.IPublisher;
-import org.opengroup.osdu.indexer.logging.AuditLogger;
-import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
-import org.opengroup.osdu.core.common.model.http.RequestStatus;
-import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
-import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
-import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
-import org.opengroup.osdu.core.common.search.IndicesService;
-import org.opengroup.osdu.indexer.util.ElasticClientHandler;
-import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
-import org.apache.commons.beanutils.PropertyUtils;
-import org.apache.commons.beanutils.NestedNullException;
-import org.springframework.stereotype.Service;
-
-import javax.inject.Inject;
-import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
-import java.util.*;
-import java.util.function.Consumer;
-import java.util.stream.Collectors;
-
-@Log
-@Service
-public class IndexerServiceImpl implements IndexerService {
-
- private final TimeValue BULK_REQUEST_TIMEOUT = TimeValue.timeValueMinutes(1);
-
- private static final List<RestStatus> RETRY_ELASTIC_EXCEPTION = new ArrayList<>(Arrays.asList(RestStatus.TOO_MANY_REQUESTS, RestStatus.BAD_GATEWAY, RestStatus.SERVICE_UNAVAILABLE));
-
- private final Gson gson = new Gson();
-
- @Inject
- private JaxRsDpsLog jaxRsDpsLog;
- @Inject
- private AuditLogger auditLogger;
- @Inject
- private StorageService storageService;
- @Inject
- private IndexSchemaService schemaService;
- @Inject
- private IndicesService indicesService;
- @Inject
- private IndexerMappingService mappingService;
- @Inject
- private IPublisher progressPublisher;
- @Inject
- private ElasticClientHandler elasticClientHandler;
- @Inject
- private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
- @Inject
- private ElasticIndexNameResolver elasticIndexNameResolver;
- @Inject
- private IAttributeParsingService attributeParsingServiceImpl;
- @Inject
- private IRequestInfo requestInfo;
- @Inject
- private JobStatus jobStatus;
-
- private DpsHeaders headers;
-
- @Override
- public JobStatus processRecordChangedMessages(RecordChangedMessages message, List<RecordInfo> recordInfos) throws Exception {
-
- // this should not happen
- if (recordInfos.size() == 0) return null;
-
- String errorMessage = "";
- List<String> retryRecordIds = new LinkedList<>();
-
- // get auth header with service account Authorization
- this.headers = this.requestInfo.getHeadersWithDwdAuthZ();
-
- // initialize status for all messages.
- this.jobStatus.initialize(recordInfos);
-
- try {
- // get upsert records
- Map<String, Map<String, OperationType>> upsertRecordMap = RecordInfo.getUpsertRecordIds(recordInfos);
- if (upsertRecordMap != null && !upsertRecordMap.isEmpty()) {
- List<String> upsertFailureRecordIds = processUpsertRecords(upsertRecordMap);
- retryRecordIds.addAll(upsertFailureRecordIds);
- }
-
- // get delete records
- Map<String, List<String>> deleteRecordMap = RecordInfo.getDeleteRecordIds(recordInfos);
- if (deleteRecordMap != null && !deleteRecordMap.isEmpty()) {
- List<String> deleteFailureRecordIds = processDeleteRecords(deleteRecordMap);
- retryRecordIds.addAll(deleteFailureRecordIds);
- }
-
- // process schema change messages
- Map<String, OperationType> schemaMsgs = RecordInfo.getSchemaMsgs(recordInfos);
- if (schemaMsgs != null && !schemaMsgs.isEmpty()) {
- this.schemaService.processSchemaMessages(schemaMsgs);
- }
-
- // process failed records
- if (retryRecordIds.size() > 0) {
- retryAndEnqueueFailedRecords(recordInfos, retryRecordIds, message);
- }
- } catch (IOException e) {
- errorMessage = e.getMessage();
- throw new AppException(HttpStatus.SC_GATEWAY_TIMEOUT, "Internal communication failure", errorMessage, e);
- } catch (AppException e) {
- errorMessage = e.getMessage();
- throw e;
- } catch (Exception e) {
- errorMessage = "error indexing records";
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Unknown error", "An unknown error has occurred.", e);
- } finally {
- this.jobStatus.finalizeRecordStatus(errorMessage);
- this.progressPublisher.publishStatusChangedTagsToTopic(this.headers, jobStatus);
- this.updateAuditLog();
- }
-
- return jobStatus;
- }
-
- private List<String> processUpsertRecords(Map<String, Map<String, OperationType>> upsertRecordMap) throws Exception {
-
- List<String> failedRecordIds = new LinkedList<>();
-
- // get schema for kind
- Map<String, IndexSchema> schemas = this.getSchema(upsertRecordMap);
-
- if (schemas.isEmpty()) return new LinkedList<>();
-
- List<String> recordIds = this.jobStatus.getIdsByIndexingStatus(IndexingStatus.PROCESSING);
- recordIds.addAll(this.jobStatus.getIdsByIndexingStatus(IndexingStatus.SKIP));
- recordIds.addAll(this.jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN));
-
- if (recordIds.isEmpty()) return new LinkedList<>();
-
- // get records via storage api
- Records storageRecords = this.storageService.getStorageRecords(recordIds);
-
- // map storage records to indexer payload
- RecordIndexerPayload recordIndexerPayload = this.getIndexerPayload(upsertRecordMap, schemas, storageRecords);
-
- // index records
- failedRecordIds.addAll(processElasticMappingAndUpsertRecords(recordIndexerPayload));
-
- return failedRecordIds;
- }
-
- private Map<String, IndexSchema> getSchema(Map<String, Map<String, OperationType>> upsertRecordMap) {
-
- Map<String, IndexSchema> schemas = new HashMap<>();
-
- try {
- for (Map.Entry<String, Map<String, OperationType>> entry : upsertRecordMap.entrySet()) {
-
- String kind = entry.getKey();
- IndexSchema schemaObj = this.schemaService.getIndexerInputSchema(kind);
- if (schemaObj.isDataSchemaMissing()) {
- this.jobStatus.addOrUpdateRecordStatus(entry.getValue().keySet(), IndexingStatus.WARN, HttpStatus.SC_NOT_FOUND, "schema not found", String.format("schema not found | kind: %s", kind));
- }
-
- schemas.put(kind, schemaObj);
- }
- } catch (AppException e) {
- throw e;
- } catch (Exception e) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Get schema error", "An error has occurred while getting schema", e);
- }
-
- return schemas;
- }
-
- private RecordIndexerPayload getIndexerPayload(Map<String, Map<String, OperationType>> upsertRecordMap, Map<String, IndexSchema> kindSchemaMap, Records records) {
- List<Records.Entity> storageValidRecords=records.getRecords();
- Map<String,List<String>> conversionStatus = getConversionErrors(records.getConversionStatuses());
- List<RecordIndexerPayload.Record> indexerPayload = new ArrayList<>();
- List<IndexSchema> schemas = new ArrayList<>();
-
- for (Records.Entity storageRecord : storageValidRecords) {
-
- Map<String, OperationType> idOperationMap = upsertRecordMap.get(storageRecord.getKind());
-
- String recordId=storageRecord.getId();
- if(conversionStatus.get(recordId)!=null){
- for(String status:conversionStatus.get(recordId)) {
- jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST,status, String.format("record-id: %s | %s", recordId, status));
- }
- }
- // skip if storage returned record with same id but different kind
- if (idOperationMap == null) {
- String message = String.format("storage service returned incorrect record | requested kind: %s | received kind: %s", this.jobStatus.getRecordKindById(storageRecord.getId()), storageRecord.getKind());
- this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.SKIP, RequestStatus.STORAGE_CONFLICT, message, String.format("%s | record-id: %s", message, storageRecord.getId()));
- continue;
- }
-
- IndexSchema schema = kindSchemaMap.get(storageRecord.getKind());
- schemas.add(schema);
-
- // skip indexing of records if data block is empty
- RecordIndexerPayload.Record document = prepareIndexerPayload(schema, storageRecord, idOperationMap);
- if (document != null) {
- indexerPayload.add(document);
- }
- }
-
- jaxRsDpsLog.info(String.format("valid upsert records: %s | can be indexed: %s", storageValidRecords.size(), indexerPayload.size()));
-
- // this should only happen if storage service returned WRONG records with kind for all the records in the messages
- if (indexerPayload.isEmpty()) {
- throw new AppException(RequestStatus.STORAGE_CONFLICT, "Indexer error", "upsert record failed, storage service returned incorrect records");
- }
-
- return RecordIndexerPayload.builder().records(indexerPayload).schemas(schemas).build();
- }
-
- private RecordIndexerPayload.Record prepareIndexerPayload(IndexSchema schemaObj, Records.Entity storageRecord, Map<String, OperationType> idToOperationMap) {
-
- RecordIndexerPayload.Record document = null;
-
- try {
- Map<String, Object> storageRecordData = storageRecord.getData();
- document = new RecordIndexerPayload.Record();
- if (storageRecordData == null || storageRecordData.isEmpty()) {
- String message = "empty or null data block found in the storage record";
- this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.SKIP, HttpStatus.SC_NOT_FOUND, message, String.format("record-id: %s | %s", storageRecord.getId(), message));
- } else if (schemaObj.isDataSchemaMissing()) {
- document.setSchemaMissing(true);
- } else {
- Map<String, Object> dataMap = mapDataPayload(schemaObj, storageRecordData, storageRecord.getId());
- if (dataMap.isEmpty()) {
- document.setMappingMismatch(true);
- String message = String.format("complete schema mismatch: none of the data attribute can be mapped | data: %s", storageRecordData);
- this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.SKIP, HttpStatus.SC_NOT_FOUND, message, String.format("record-id: %s | %s", storageRecord.getId(), message));
- }
- document.setData(dataMap);
- }
- } catch (AppException e) {
- this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL, HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getMessage());
- jaxRsDpsLog.warning(String.format("record-id: %s | %s", storageRecord.getId(), e.getMessage()), e);
- } catch (Exception e) {
- this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL, HttpStatus.SC_INTERNAL_SERVER_ERROR, String.format("error parsing records against schema, error-message: %s", e.getMessage()));
- jaxRsDpsLog.error(String.format("record-id: %s | error parsing records against schema, error-message: %s", storageRecord.getId(), e.getMessage()), e);
- }
-
- try {
- // index individual parts of kind
- String[] kindParts = storageRecord.getKind().split(":");
-
- document.setKind(storageRecord.getKind());
- document.setNamespace(kindParts[0] + ":" + kindParts[1]);
- document.setType(kindParts[2]);
- document.setId(storageRecord.getId());
- document.setVersion(storageRecord.getVersion());
- document.setAcl(storageRecord.getAcl());
- document.setLegal(storageRecord.getLegal());
- RecordStatus recordStatus = this.jobStatus.getJobStatusByRecordId(storageRecord.getId());
- if (recordStatus.getIndexProgress().getStatusCode() == 0) {
- recordStatus.getIndexProgress().setStatusCode(HttpStatus.SC_OK);
- }
- document.setIndexProgress(recordStatus.getIndexProgress());
- if (storageRecord.getAncestry() != null) document.setAncestry(storageRecord.getAncestry());
- document.setOperationType(idToOperationMap.get(storageRecord.getId()));
- } catch (Exception e) {
- this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL, HttpStatus.SC_INTERNAL_SERVER_ERROR, String.format("error parsing meta data, error-message: %s", e.getMessage()));
- jaxRsDpsLog.error(String.format("record-id: %s | error parsing meta data, error-message: %s", storageRecord.getId(), e.getMessage()), e);
- }
- return document;
- }
-
- private Map<String, Object> mapDataPayload(IndexSchema schemaObj, Map<String, Object> storageRecordData, String recordId) {
-
- Map<String, Object> dataMap = new HashMap<>();
-
- if (schemaObj.isDataSchemaMissing()) return dataMap;
-
- // get the key and get the corresponding object from the storageRecord object
- for (Map.Entry<String, String> entry : schemaObj.getDataSchema().entrySet()) {
-
- String name = entry.getKey();
-
- Object value = getPropertyValue(recordId, storageRecordData, name);
-
- if (value == null) continue;
-
- ElasticType elasticType = ElasticType.forValue(entry.getValue());
-
- switch (elasticType) {
- case KEYWORD:
- case TEXT:
- dataMap.put(name, value);
- break;
- case INTEGER:
- this.attributeParsingServiceImpl.tryParseInteger(recordId, name, value, dataMap);
- break;
- case LONG:
- this.attributeParsingServiceImpl.tryParseLong(recordId, name, value, dataMap);
- break;
- case FLOAT:
- this.attributeParsingServiceImpl.tryParseFloat(recordId, name, value, dataMap);
- break;
- case DOUBLE:
- this.attributeParsingServiceImpl.tryParseDouble(recordId, name, value, dataMap);
- break;
- case BOOLEAN:
- this.attributeParsingServiceImpl.tryParseBoolean(recordId, name, value, dataMap);
- break;
- case DATE:
- this.attributeParsingServiceImpl.tryParseDate(recordId, name, value, dataMap);
- break;
- case GEO_POINT:
- this.attributeParsingServiceImpl.tryParseGeopoint(recordId, name, storageRecordData, dataMap);
- break;
- case GEO_SHAPE:
- this.attributeParsingServiceImpl.tryParseGeojson(recordId, name, value, dataMap);
- break;
- case NESTED:
- case OBJECT:
- case UNDEFINED:
- // don't do anything for now
- break;
- }
- }
-
- // add these once iterated over the list
- schemaObj.getDataSchema().put(IAttributeParsingService.DATA_GEOJSON_TAG, ElasticType.GEO_SHAPE.getValue());
- schemaObj.getDataSchema().remove(IAttributeParsingService.RECORD_GEOJSON_TAG);
-
- return dataMap;
- }
-
- private List<String> processElasticMappingAndUpsertRecords(RecordIndexerPayload recordIndexerPayload) throws Exception {
-
- try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
- List<IndexSchema> schemas = recordIndexerPayload.getSchemas();
- if (schemas == null || schemas.isEmpty()) {
- return new LinkedList<>();
- }
-
- // process the schema
- this.cacheOrCreateElasticMapping(schemas, restClient);
-
- // process the records
- return this.upsertRecords(recordIndexerPayload.getRecords(), restClient);
- }
- }
-
- private void cacheOrCreateElasticMapping(List<IndexSchema> schemas, RestHighLevelClient restClient) throws Exception {
-
- for (IndexSchema schema : schemas) {
- String index = this.elasticIndexNameResolver.getIndexNameFromKind(schema.getKind());
-
- // check if index exist
- if (this.indicesService.isIndexExist(restClient, index)) continue;
-
- // create index
- Map<String, Object> mapping = this.mappingService.getIndexMappingFromRecordSchema(schema);
- if (!this.indicesService.createIndex(restClient, index, null, schema.getType(), mapping)) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error", "Error creating index.", String.format("Failed to get confirmation from elastic server for index: %s", index));
- }
- }
- }
-
- private List<String> upsertRecords(List<RecordIndexerPayload.Record> records, RestHighLevelClient restClient) throws AppException {
- if (records == null || records.isEmpty()) return new LinkedList<>();
-
- BulkRequest bulkRequest = new BulkRequest();
- bulkRequest.timeout(BULK_REQUEST_TIMEOUT);
-
- for (RecordIndexerPayload.Record record : records) {
- if ((record.getData() == null || record.getData().isEmpty()) && !record.skippedDataIndexing()) {
- // it will come here when schema is missing
- // TODO: rollback once we know what is causing the problem
- jaxRsDpsLog.warning(String.format("data not found for record: %s", record));
- }
-
- OperationType operation = record.getOperationType();
- Map<String, Object> sourceMap = getSourceMap(record);
- String index = this.elasticIndexNameResolver.getIndexNameFromKind(record.getKind());
-
- if (operation == OperationType.create) {
- IndexRequest indexRequest = new IndexRequest(index, record.getType(), record.getId()).source(this.gson.toJson(sourceMap), XContentType.JSON);
- bulkRequest.add(indexRequest);
- } else if (operation == OperationType.update) {
- UpdateRequest updateRequest = new UpdateRequest(index, record.getType(), record.getId()).upsert(this.gson.toJson(sourceMap), XContentType.JSON);
- bulkRequest.add(updateRequest);
- }
- }
-
- return processBulkRequest(restClient, bulkRequest);
- }
-
- private List<String> processDeleteRecords(Map<String, List<String>> deleteRecordMap) throws Exception {
- BulkRequest bulkRequest = new BulkRequest();
- bulkRequest.timeout(BULK_REQUEST_TIMEOUT);
-
- for (Map.Entry<String, List<String>> record : deleteRecordMap.entrySet()) {
-
- String[] kindParts = record.getKey().split(":");
- String type = kindParts[2];
-
- String index = this.elasticIndexNameResolver.getIndexNameFromKind(record.getKey());
-
- for (String id : record.getValue()) {
- DeleteRequest deleteRequest = new DeleteRequest(index, type, id);
- bulkRequest.add(deleteRequest);
- }
- }
-
- try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
- return processBulkRequest(restClient, bulkRequest);
- }
- }
-
- private List<String> processBulkRequest(RestHighLevelClient restClient, BulkRequest bulkRequest) throws AppException {
-
- List<String> failureRecordIds = new LinkedList<>();
- if (bulkRequest.numberOfActions() == 0) return failureRecordIds;
-
- try {
- BulkResponse bulkResponse = restClient.bulk(bulkRequest, RequestOptions.DEFAULT);
- jaxRsDpsLog.info(String.format("records in bulk request: %s | acknowledged in response: %s", bulkRequest.numberOfActions(), bulkResponse.getItems().length));
-
- // log failed bulk requests
- ArrayList<String> bulkFailures = new ArrayList<>();
- for (BulkItemResponse bulkItemResponse : bulkResponse.getItems()) {
- if (bulkItemResponse.isFailed()) {
- BulkItemResponse.Failure failure = bulkItemResponse.getFailure();
- bulkFailures.add(String.format("bulk status: %s id: %s message: %s", failure.getStatus(), failure.getId(), failure.getMessage()));
- this.jobStatus.addOrUpdateRecordStatus(bulkItemResponse.getId(), IndexingStatus.FAIL, failure.getStatus().getStatus(), bulkItemResponse.getFailure().getMessage());
- if (RETRY_ELASTIC_EXCEPTION.contains(bulkItemResponse.status())) {
- failureRecordIds.add(bulkItemResponse.getId());
- }
- } else {
- this.jobStatus.addOrUpdateRecordStatus(bulkItemResponse.getId(), IndexingStatus.SUCCESS, HttpStatus.SC_OK, "Indexed Successfully");
- }
- }
- if (!bulkFailures.isEmpty()) this.jaxRsDpsLog.warning(bulkFailures);
- } catch (IOException e) {
- // throw explicit 504 for IOException
- throw new AppException(HttpStatus.SC_GATEWAY_TIMEOUT, "Elastic error", "Request cannot be completed in specified time.", e);
- } catch (ElasticsearchStatusException e) {
- throw new AppException(e.status().getStatus(), "Elastic error", e.getMessage(), e);
- } catch (Exception e) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error", "Error indexing records.", e);
- }
- return failureRecordIds;
- }
-
- private Map<String, Object> getSourceMap(RecordIndexerPayload.Record record) {
-
- Map<String, Object> indexerPayload = new HashMap<>();
-
- // get the key and get the corresponding object from the individualRecord object
- if (record.getData() != null) {
- Map<String, Object> data = new HashMap<>();
- for (Map.Entry<String, Object> entry : record.getData().entrySet()) {
- data.put(entry.getKey(), entry.getValue());
- }
- indexerPayload.put(Constants.DATA, data);
- }
-
- indexerPayload.put(RecordMetaAttribute.ID.getValue(), record.getId());
- indexerPayload.put(RecordMetaAttribute.KIND.getValue(), record.getKind());
- indexerPayload.put(RecordMetaAttribute.NAMESPACE.getValue(), record.getNamespace());
- indexerPayload.put(RecordMetaAttribute.TYPE.getValue(), record.getType());
- indexerPayload.put(RecordMetaAttribute.VERSION.getValue(), record.getVersion());
- indexerPayload.put(RecordMetaAttribute.ACL.getValue(), record.getAcl());
- indexerPayload.put(RecordMetaAttribute.X_ACL.getValue(), Acl.flattenAcl(record.getAcl()));
- indexerPayload.put(RecordMetaAttribute.LEGAL.getValue(), record.getLegal());
- indexerPayload.put(RecordMetaAttribute.INDEX_STATUS.getValue(), record.getIndexProgress());
- if (record.getAncestry() != null) {
- indexerPayload.put(RecordMetaAttribute.ANCESTRY.getValue(), record.getAncestry());
- }
- return indexerPayload;
- }
-
- private Object getPropertyValue(String recordId, Map<String, Object> storageRecordData, String propertyKey) {
-
- try {
- // try getting first level property using optimized collection
- Object propertyVal = storageRecordData.get(propertyKey);
- if (propertyVal != null) return propertyVal;
-
- // use apache utils to get nested property
- return PropertyUtils.getProperty(storageRecordData, propertyKey);
- } catch (NestedNullException ignored) {
- // property not found in record
- } catch (IllegalArgumentException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
- jaxRsDpsLog.warning(String.format("record-id: %s | error fetching property: %s | error: %s", recordId, propertyKey, e.getMessage()), e);
- }
- return null;
- }
-
- private void retryAndEnqueueFailedRecords(List<RecordInfo> recordInfos, List<String> failuresRecordIds, RecordChangedMessages message) throws IOException {
-
- jaxRsDpsLog.info(String.format("queuing bulk failed records back to task-queue for retry | count: %s | records: %s", failuresRecordIds.size(), failuresRecordIds));
- List<RecordInfo> retryRecordInfos = new LinkedList<>();
- for (String recordId : failuresRecordIds) {
- for (RecordInfo origMessage : recordInfos) {
- if (origMessage.getId().equalsIgnoreCase(recordId)) {
- retryRecordInfos.add(origMessage);
- }
- }
- }
-
- RecordChangedMessages newMessage = RecordChangedMessages.builder()
- .messageId(message.getMessageId())
- .publishTime(message.getPublishTime())
- .data(this.gson.toJson(retryRecordInfos))
- .attributes(message.getAttributes()).build();
-
- String payLoad = this.gson.toJson(newMessage);
- this.indexerQueueTaskBuilder.createWorkerTask(payLoad, this.headers);
- }
-
- private void updateAuditLog() {
- logAuditEvents(OperationType.create, this.auditLogger::indexCreateRecordSuccess, this.auditLogger::indexCreateRecordFail);
- logAuditEvents(OperationType.update, this.auditLogger::indexUpdateRecordSuccess, this.auditLogger::indexUpdateRecordFail);
- logAuditEvents(OperationType.purge, this.auditLogger::indexPurgeRecordSuccess, this.auditLogger::indexPurgeRecordFail);
- logAuditEvents(OperationType.delete, this.auditLogger::indexDeleteRecordSuccess, this.auditLogger::indexDeleteRecordFail);
- }
-
- private void logAuditEvents(OperationType operationType, Consumer<List<String>> successEvent, Consumer<List<String>> failedEvent) {
- List<RecordStatus> succeededRecords = this.jobStatus.getRecordStatuses(IndexingStatus.SUCCESS, operationType);
- if(!succeededRecords.isEmpty()) {
- successEvent.accept(succeededRecords.stream().map(RecordStatus::toString).collect(Collectors.toList()));
- }
- List<RecordStatus> skippedRecords = this.jobStatus.getRecordStatuses(IndexingStatus.SKIP, operationType);
- List<RecordStatus> failedRecords = this.jobStatus.getRecordStatuses(IndexingStatus.FAIL, operationType);
- failedRecords.addAll(skippedRecords);
- if(!failedRecords.isEmpty()) {
- failedEvent.accept(failedRecords.stream().map(RecordStatus::toString).collect(Collectors.toList()));
- }
- }
-
- private Map<String,List<String>> getConversionErrors(List<ConversionStatus> conversionStatuses){
- Map<String,List<String>> errorsByRecordId =new HashMap<>();
- for(ConversionStatus conversionStatus:conversionStatuses){
- if(conversionStatus.getStatus().equalsIgnoreCase("ERROR")){
- List<String> statuses=errorsByRecordId.getOrDefault(conversionStatus.getId(),new LinkedList<>());
- statuses.addAll(conversionStatus.getErrors());
- errorsByRecordId.put(conversionStatus.getId(),statuses);
- }
- }
- return errorsByRecordId;
- }
-}
\ No newline at end of file
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.service;
+
+import com.google.gson.Gson;
+
+import org.apache.http.HttpStatus;
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.action.bulk.BulkItemResponse;
+import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.rest.RestStatus;
+import org.opengroup.osdu.core.common.model.entitlements.Acl;
+import org.opengroup.osdu.core.common.Constants;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.indexer.*;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.indexer.provider.interfaces.IPublisher;
+import org.opengroup.osdu.indexer.logging.AuditLogger;
+import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
+import org.opengroup.osdu.core.common.model.http.RequestStatus;
+import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
+import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.opengroup.osdu.core.common.search.IndicesService;
+import org.opengroup.osdu.indexer.util.ElasticClientHandler;
+import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
+import org.apache.commons.beanutils.PropertyUtils;
+import org.apache.commons.beanutils.NestedNullException;
+import org.springframework.stereotype.Service;
+
+import javax.inject.Inject;
+import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.util.*;
+import java.util.function.Consumer;
+import java.util.logging.Level;
+import java.util.stream.Collectors;
+
+@Service
+public class IndexerServiceImpl implements IndexerService {
+
+ private static final TimeValue BULK_REQUEST_TIMEOUT = TimeValue.timeValueMinutes(1);
+
+ private static final List<RestStatus> RETRY_ELASTIC_EXCEPTION = new ArrayList<>(Arrays.asList(RestStatus.TOO_MANY_REQUESTS, RestStatus.BAD_GATEWAY, RestStatus.SERVICE_UNAVAILABLE));
+
+ private final Gson gson = new Gson();
+
+ @Inject
+ private JaxRsDpsLog jaxRsDpsLog;
+ @Inject
+ private AuditLogger auditLogger;
+ @Inject
+ private StorageService storageService;
+ @Inject
+ private IndexSchemaService schemaService;
+ @Inject
+ private IndicesService indicesService;
+ @Inject
+ private IndexerMappingService mappingService;
+ @Inject
+ private IPublisher progressPublisher;
+ @Inject
+ private ElasticClientHandler elasticClientHandler;
+ @Inject
+ private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
+ @Inject
+ private ElasticIndexNameResolver elasticIndexNameResolver;
+ @Inject
+ private StorageIndexerPayloadMapper storageIndexerPayloadMapper;
+ @Inject
+ private IRequestInfo requestInfo;
+ @Inject
+ private JobStatus jobStatus;
+
+ private DpsHeaders headers;
+
+ @Override
+ public JobStatus processRecordChangedMessages(RecordChangedMessages message, List<RecordInfo> recordInfos) throws Exception {
+
+ // this should not happen
+ if (recordInfos.size() == 0) return null;
+
+ String errorMessage = "";
+ List<String> retryRecordIds = new LinkedList<>();
+
+ // get auth header with service account Authorization
+ this.headers = this.requestInfo.getHeadersWithDwdAuthZ();
+
+ // initialize status for all messages.
+ this.jobStatus.initialize(recordInfos);
+
+ try {
+ // get upsert records
+ Map<String, Map<String, OperationType>> upsertRecordMap = RecordInfo.getUpsertRecordIds(recordInfos);
+ if (upsertRecordMap != null && !upsertRecordMap.isEmpty()) {
+ List<String> upsertFailureRecordIds = processUpsertRecords(upsertRecordMap);
+ retryRecordIds.addAll(upsertFailureRecordIds);
+ }
+
+ // get delete records
+ Map<String, List<String>> deleteRecordMap = RecordInfo.getDeleteRecordIds(recordInfos);
+ if (deleteRecordMap != null && !deleteRecordMap.isEmpty()) {
+ List<String> deleteFailureRecordIds = processDeleteRecords(deleteRecordMap);
+ retryRecordIds.addAll(deleteFailureRecordIds);
+ }
+
+ // process schema change messages
+ Map<String, OperationType> schemaMsgs = RecordInfo.getSchemaMsgs(recordInfos);
+ if (schemaMsgs != null && !schemaMsgs.isEmpty()) {
+ this.schemaService.processSchemaMessages(schemaMsgs);
+ }
+
+ // process failed records
+ if (retryRecordIds.size() > 0) {
+ retryAndEnqueueFailedRecords(recordInfos, retryRecordIds, message);
+ }
+ } catch (IOException e) {
+ errorMessage = e.getMessage();
+ throw new AppException(HttpStatus.SC_GATEWAY_TIMEOUT, "Internal communication failure", errorMessage, e);
+ } catch (AppException e) {
+ errorMessage = e.getMessage();
+ throw e;
+ } catch (Exception e) {
+ errorMessage = "error indexing records";
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Unknown error", "An unknown error has occurred.", e);
+ } finally {
+ this.jobStatus.finalizeRecordStatus(errorMessage);
+ this.updateAuditLog();
+ this.progressPublisher.publishStatusChangedTagsToTopic(this.headers, this.jobStatus);
+ }
+
+ return jobStatus;
+ }
+
+ private List<String> processUpsertRecords(Map<String, Map<String, OperationType>> upsertRecordMap) throws Exception {
+ // get schema for kind
+ Map<String, IndexSchema> schemas = this.getSchema(upsertRecordMap);
+
+ if (schemas.isEmpty()) return new LinkedList<>();
+
+ // get recordIds with valid upsert index-status
+ List<String> recordIds = this.jobStatus.getIdsByValidUpsertIndexingStatus();
+
+ if (recordIds.isEmpty()) return new LinkedList<>();
+
+ // get records via storage api
+ Records storageRecords = this.storageService.getStorageRecords(recordIds);
+ List<String> failedOrRetryRecordIds = new LinkedList<>(storageRecords.getMissingRetryRecords());
+
+ // map storage records to indexer payload
+ RecordIndexerPayload recordIndexerPayload = this.getIndexerPayload(upsertRecordMap, schemas, storageRecords);
+
+ jaxRsDpsLog.info(String.format("records change messages received : %s | valid storage bulk records: %s | valid index payload: %s", recordIds.size(), storageRecords.getRecords().size(), recordIndexerPayload.getRecords().size()));
+
+ // index records
+ failedOrRetryRecordIds.addAll(processElasticMappingAndUpsertRecords(recordIndexerPayload));
+
+ return failedOrRetryRecordIds;
+ }
+
+ private Map<String, IndexSchema> getSchema(Map<String, Map<String, OperationType>> upsertRecordMap) {
+
+ Map<String, IndexSchema> schemas = new HashMap<>();
+
+ try {
+ for (Map.Entry<String, Map<String, OperationType>> entry : upsertRecordMap.entrySet()) {
+
+ String kind = entry.getKey();
+ IndexSchema schemaObj = this.schemaService.getIndexerInputSchema(kind, false);
+ if (schemaObj.isDataSchemaMissing()) {
+ this.jobStatus.addOrUpdateRecordStatus(entry.getValue().keySet(), IndexingStatus.WARN, HttpStatus.SC_NOT_FOUND, "schema not found", String.format("schema not found | kind: %s", kind));
+ }
+
+ schemas.put(kind, schemaObj);
+ }
+ } catch (AppException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Get schema error", "An error has occurred while getting schema", e);
+ }
+
+ return schemas;
+ }
+
+ private RecordIndexerPayload getIndexerPayload(Map<String, Map<String, OperationType>> upsertRecordMap, Map<String, IndexSchema> kindSchemaMap, Records records) {
+ List<Records.Entity> storageValidRecords = records.getRecords();
+ List<RecordIndexerPayload.Record> indexerPayload = new ArrayList<>();
+ List<IndexSchema> schemas = new ArrayList<>();
+
+ for (Records.Entity storageRecord : storageValidRecords) {
+
+ Map<String, OperationType> idOperationMap = upsertRecordMap.get(storageRecord.getKind());
+
+ // skip if storage returned record with same id but different kind
+ if (idOperationMap == null) {
+ String message = String.format("storage service returned incorrect record | requested kind: %s | received kind: %s", this.jobStatus.getRecordKindById(storageRecord.getId()), storageRecord.getKind());
+ this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.SKIP, RequestStatus.STORAGE_CONFLICT, message, String.format("%s | record-id: %s", message, storageRecord.getId()));
+ continue;
+ }
+
+ IndexSchema schema = kindSchemaMap.get(storageRecord.getKind());
+ schemas.add(schema);
+
+ // skip indexing of records if data block is empty
+ RecordIndexerPayload.Record document = prepareIndexerPayload(schema, storageRecord, idOperationMap);
+ if (document != null) {
+ indexerPayload.add(document);
+ }
+ }
+
+ // this should only happen if storage service returned WRONG records with kind for all the records in the messages
+ if (indexerPayload.isEmpty()) {
+ throw new AppException(RequestStatus.STORAGE_CONFLICT, "Indexer error", "upsert record failed, storage service returned incorrect records");
+ }
+
+ return RecordIndexerPayload.builder().records(indexerPayload).schemas(schemas).build();
+ }
+
+ private RecordIndexerPayload.Record prepareIndexerPayload(IndexSchema schemaObj, Records.Entity storageRecord, Map<String, OperationType> idToOperationMap) {
+
+ RecordIndexerPayload.Record document = null;
+
+ try {
+ Map<String, Object> storageRecordData = storageRecord.getData();
+ document = new RecordIndexerPayload.Record();
+ if (storageRecordData == null || storageRecordData.isEmpty()) {
+ String message = "empty or null data block found in the storage record";
+ this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.WARN, HttpStatus.SC_NOT_FOUND, message, String.format("record-id: %s | %s", storageRecord.getId(), message));
+ } else if (schemaObj.isDataSchemaMissing()) {
+ document.setSchemaMissing(true);
+ } else {
+ Map<String, Object> dataMap = this.storageIndexerPayloadMapper.mapDataPayload(schemaObj, storageRecordData, storageRecord.getId());
+ if (dataMap.isEmpty()) {
+ document.setMappingMismatch(true);
+ String message = String.format("complete schema mismatch: none of the data attribute can be mapped | data: %s", storageRecordData);
+ this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.WARN, HttpStatus.SC_NOT_FOUND, message, String.format("record-id: %s | %s", storageRecord.getId(), message));
+ }
+ document.setData(dataMap);
+ }
+ } catch (AppException e) {
+ this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL, HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getMessage());
+ jaxRsDpsLog.warning(String.format("record-id: %s | %s", storageRecord.getId(), e.getMessage()), e);
+ } catch (Exception e) {
+ this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL, HttpStatus.SC_INTERNAL_SERVER_ERROR, String.format("error parsing records against schema, error-message: %s", e.getMessage()));
+ jaxRsDpsLog.error(String.format("record-id: %s | error parsing records against schema, error-message: %s", storageRecord.getId(), e.getMessage()), e);
+ }
+
+ try {
+ // index individual parts of kind
+ String[] kindParts = storageRecord.getKind().split(":");
+
+ document.setKind(storageRecord.getKind());
+ document.setNamespace(kindParts[0] + ":" + kindParts[1]);
+ document.setType(kindParts[2]);
+ document.setId(storageRecord.getId());
+ document.setVersion(storageRecord.getVersion());
+ document.setAcl(storageRecord.getAcl());
+ document.setLegal(storageRecord.getLegal());
+ RecordStatus recordStatus = this.jobStatus.getJobStatusByRecordId(storageRecord.getId());
+ if (recordStatus.getIndexProgress().getStatusCode() == 0) {
+ recordStatus.getIndexProgress().setStatusCode(HttpStatus.SC_OK);
+ }
+ document.setIndexProgress(recordStatus.getIndexProgress());
+ if (storageRecord.getAncestry() != null) document.setAncestry(storageRecord.getAncestry());
+ document.setOperationType(idToOperationMap.get(storageRecord.getId()));
+ } catch (Exception e) {
+ this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL, HttpStatus.SC_INTERNAL_SERVER_ERROR, String.format("error parsing meta data, error-message: %s", e.getMessage()));
+ jaxRsDpsLog.error(String.format("record-id: %s | error parsing meta data, error-message: %s", storageRecord.getId(), e.getMessage()), e);
+ }
+ return document;
+ }
+
+ private List<String> processElasticMappingAndUpsertRecords(RecordIndexerPayload recordIndexerPayload) throws Exception {
+
+ try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
+ List<IndexSchema> schemas = recordIndexerPayload.getSchemas();
+ if (schemas == null || schemas.isEmpty()) {
+ return new LinkedList<>();
+ }
+
+ // process the schema
+ this.cacheOrCreateElasticMapping(schemas, restClient);
+
+ // process the records
+ return this.upsertRecords(recordIndexerPayload.getRecords(), restClient);
+ }
+ }
+
+ private void cacheOrCreateElasticMapping(List<IndexSchema> schemas, RestHighLevelClient restClient) throws Exception {
+
+ for (IndexSchema schema : schemas) {
+ String index = this.elasticIndexNameResolver.getIndexNameFromKind(schema.getKind());
+
+ // check if index exist
+ if (this.indicesService.isIndexExist(restClient, index)) continue;
+
+ // create index
+ Map<String, Object> mapping = this.mappingService.getIndexMappingFromRecordSchema(schema);
+ if (!this.indicesService.createIndex(restClient, index, null, schema.getType(), mapping)) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error", "Error creating index.", String.format("Failed to get confirmation from elastic server for index: %s", index));
+ }
+ }
+ }
+
+ private List<String> upsertRecords(List<RecordIndexerPayload.Record> records, RestHighLevelClient restClient) throws AppException {
+ if (records == null || records.isEmpty()) return new LinkedList<>();
+
+ BulkRequest bulkRequest = new BulkRequest();
+ bulkRequest.timeout(BULK_REQUEST_TIMEOUT);
+
+ for (RecordIndexerPayload.Record record : records) {
+ if ((record.getData() == null || record.getData().isEmpty()) && !record.skippedDataIndexing()) {
+ // it will come here when schema is missing
+ // TODO: rollback once we know what is causing the problem
+ jaxRsDpsLog.warning(String.format("data not found for record: %s", record));
+ }
+
+ OperationType operation = record.getOperationType();
+ Map<String, Object> sourceMap = getSourceMap(record);
+ String index = this.elasticIndexNameResolver.getIndexNameFromKind(record.getKind());
+
+ if (operation == OperationType.create) {
+ IndexRequest indexRequest = new IndexRequest(index, record.getType(), record.getId()).source(this.gson.toJson(sourceMap), XContentType.JSON);
+ bulkRequest.add(indexRequest);
+ } else if (operation == OperationType.update) {
+ UpdateRequest updateRequest = new UpdateRequest(index, record.getType(), record.getId()).upsert(this.gson.toJson(sourceMap), XContentType.JSON);
+ bulkRequest.add(updateRequest);
+ }
+ }
+
+ return processBulkRequest(restClient, bulkRequest);
+ }
+
+ private List<String> processDeleteRecords(Map<String, List<String>> deleteRecordMap) throws Exception {
+ BulkRequest bulkRequest = new BulkRequest();
+ bulkRequest.timeout(BULK_REQUEST_TIMEOUT);
+
+ for (Map.Entry<String, List<String>> record : deleteRecordMap.entrySet()) {
+
+ String[] kindParts = record.getKey().split(":");
+ String type = kindParts[2];
+
+ String index = this.elasticIndexNameResolver.getIndexNameFromKind(record.getKey());
+
+ for (String id : record.getValue()) {
+ DeleteRequest deleteRequest = new DeleteRequest(index, type, id);
+ bulkRequest.add(deleteRequest);
+ }
+ }
+
+ try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
+ return processBulkRequest(restClient, bulkRequest);
+ }
+ }
+
+ private List<String> processBulkRequest(RestHighLevelClient restClient, BulkRequest bulkRequest) throws AppException {
+
+ List<String> failureRecordIds = new LinkedList<>();
+ if (bulkRequest.numberOfActions() == 0) return failureRecordIds;
+
+
+
+ try {
+ BulkResponse bulkResponse = restClient.bulk(bulkRequest, RequestOptions.DEFAULT);
+
+ // log failed bulk requests
+ ArrayList<String> bulkFailures = new ArrayList<>();
+ int succeededResponses = 0;
+ int failedResponses = 0;
+
+ for (BulkItemResponse bulkItemResponse : bulkResponse.getItems()) {
+ if (bulkItemResponse.isFailed()) {
+ BulkItemResponse.Failure failure = bulkItemResponse.getFailure();
+ bulkFailures.add(String.format("elasticsearch bulk service status: %s id: %s message: %s", failure.getStatus(), failure.getId(), failure.getMessage()));
+ this.jobStatus.addOrUpdateRecordStatus(bulkItemResponse.getId(), IndexingStatus.FAIL, failure.getStatus().getStatus(), bulkItemResponse.getFailureMessage());
+ if (RETRY_ELASTIC_EXCEPTION.contains(bulkItemResponse.status())) {
+ failureRecordIds.add(bulkItemResponse.getId());
+ }
+ failedResponses++;
+ } else {
+ succeededResponses++;
+ this.jobStatus.addOrUpdateRecordStatus(bulkItemResponse.getId(), IndexingStatus.SUCCESS, HttpStatus.SC_OK, "Indexed Successfully");
+ }
+ }
+ if (!bulkFailures.isEmpty()) this.jaxRsDpsLog.warning(bulkFailures);
+
+ jaxRsDpsLog.info(String.format("records in elasticsearch service bulk request: %s | successful: %s | failed: %s", bulkRequest.numberOfActions(), succeededResponses, failedResponses));
+ } catch (IOException e) {
+ // throw explicit 504 for IOException
+ throw new AppException(HttpStatus.SC_GATEWAY_TIMEOUT, "Elastic error", "Request cannot be completed in specified time.", e);
+ } catch (ElasticsearchStatusException e) {
+ throw new AppException(e.status().getStatus(), "Elastic error", e.getMessage(), e);
+ } catch (Exception e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error", "Error indexing records.", e);
+ }
+ return failureRecordIds;
+ }
+
+ private Map<String, Object> getSourceMap(RecordIndexerPayload.Record record) {
+
+ Map<String, Object> indexerPayload = new HashMap<>();
+
+ // get the key and get the corresponding object from the individualRecord object
+ if (record.getData() != null) {
+ Map<String, Object> data = new HashMap<>();
+ for (Map.Entry<String, Object> entry : record.getData().entrySet()) {
+ data.put(entry.getKey(), entry.getValue());
+ }
+ indexerPayload.put(Constants.DATA, data);
+ }
+
+ indexerPayload.put(RecordMetaAttribute.ID.getValue(), record.getId());
+ indexerPayload.put(RecordMetaAttribute.KIND.getValue(), record.getKind());
+ indexerPayload.put(RecordMetaAttribute.NAMESPACE.getValue(), record.getNamespace());
+ indexerPayload.put(RecordMetaAttribute.TYPE.getValue(), record.getType());
+ indexerPayload.put(RecordMetaAttribute.VERSION.getValue(), record.getVersion());
+ indexerPayload.put(RecordMetaAttribute.ACL.getValue(), record.getAcl());
+ indexerPayload.put(RecordMetaAttribute.X_ACL.getValue(), Acl.flattenAcl(record.getAcl()));
+ indexerPayload.put(RecordMetaAttribute.LEGAL.getValue(), record.getLegal());
+ indexerPayload.put(RecordMetaAttribute.INDEX_STATUS.getValue(), record.getIndexProgress());
+ if (record.getAncestry() != null) {
+ indexerPayload.put(RecordMetaAttribute.ANCESTRY.getValue(), record.getAncestry());
+ }
+ return indexerPayload;
+ }
+
+ private void retryAndEnqueueFailedRecords(List<RecordInfo> recordInfos, List<String> failuresRecordIds, RecordChangedMessages message) throws IOException {
+
+ jaxRsDpsLog.info(String.format("queuing bulk failed records back to task-queue for retry | count: %s | records: %s", failuresRecordIds.size(), failuresRecordIds));
+ List<RecordInfo> retryRecordInfos = new LinkedList<>();
+ for (String recordId : failuresRecordIds) {
+ for (RecordInfo origMessage : recordInfos) {
+ if (origMessage.getId().equalsIgnoreCase(recordId)) {
+ retryRecordInfos.add(origMessage);
+ }
+ }
+ }
+
+ RecordChangedMessages newMessage = RecordChangedMessages.builder()
+ .messageId(message.getMessageId())
+ .publishTime(message.getPublishTime())
+ .data(this.gson.toJson(retryRecordInfos))
+ .attributes(message.getAttributes()).build();
+
+ String payLoad = this.gson.toJson(newMessage);
+ this.indexerQueueTaskBuilder.createWorkerTask(payLoad, this.headers);
+ }
+
+ private void updateAuditLog() {
+ logAuditEvents(OperationType.create, this.auditLogger::indexCreateRecordSuccess, this.auditLogger::indexCreateRecordFail);
+ logAuditEvents(OperationType.update, this.auditLogger::indexUpdateRecordSuccess, this.auditLogger::indexUpdateRecordFail);
+ logAuditEvents(OperationType.purge, this.auditLogger::indexPurgeRecordSuccess, this.auditLogger::indexPurgeRecordFail);
+ logAuditEvents(OperationType.delete, this.auditLogger::indexDeleteRecordSuccess, this.auditLogger::indexDeleteRecordFail);
+ }
+
+ private void logAuditEvents(OperationType operationType, Consumer<List<String>> successEvent, Consumer<List<String>> failedEvent) {
+ List<RecordStatus> succeededRecords = this.jobStatus.getRecordStatuses(IndexingStatus.SUCCESS, operationType);
+ if(!succeededRecords.isEmpty()) {
+ successEvent.accept(succeededRecords.stream().map(RecordStatus::succeededAuditLogMessage).collect(Collectors.toList()));
+ }
+ List<RecordStatus> skippedRecords = this.jobStatus.getRecordStatuses(IndexingStatus.SKIP, operationType);
+ List<RecordStatus> failedRecords = this.jobStatus.getRecordStatuses(IndexingStatus.FAIL, operationType);
+ failedRecords.addAll(skippedRecords);
+ if(!failedRecords.isEmpty()) {
+ failedEvent.accept(failedRecords.stream().map(RecordStatus::failedAuditLogMessage).collect(Collectors.toList()));
+ }
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java
index 22374fc32075455e31df5dd9e4b912f34d9d3292..0f13dde455b6eb97b8cef6de07d4fde14fbd77cb 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java
@@ -151,7 +151,7 @@ public class IndicesServiceImpl implements IndicesService {
* @param client Elasticsearch client
* @param index Index name
*/
- public boolean deleteIndex(RestHighLevelClient client, String index) throws Exception {
+ public boolean deleteIndex(RestHighLevelClient client, String index) throws ElasticsearchException, IOException, AppException {
boolean responseStatus = removeIndexInElasticsearch(client, index);
if (responseStatus) {
this.indicesExistCache.delete(index);
@@ -164,7 +164,7 @@ public class IndicesServiceImpl implements IndicesService {
*
* @param index Index name
*/
- public boolean deleteIndex(String index) throws Exception {
+ public boolean deleteIndex(String index) throws ElasticsearchException, IOException, AppException {
try (RestHighLevelClient client = this.elasticClientHandler.createRestClient()) {
return deleteIndex(client, index);
}
@@ -177,7 +177,7 @@ public class IndicesServiceImpl implements IndicesService {
* @param index Index name
* @throws Exception Throws {@link AppException} if index is not found or elastic cannot delete the index
*/
- private boolean removeIndexInElasticsearch(RestHighLevelClient client, String index) throws Exception {
+ private boolean removeIndexInElasticsearch(RestHighLevelClient client, String index) throws ElasticsearchException, IOException, AppException {
Preconditions.checkArgument(client, Objects::nonNull, "client cannot be null");
Preconditions.checkArgument(index, Objects::nonNull, "index cannot be null");
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexService.java
index 40f22aaa26f931f6bde8aa8d0d28f33a61cbc16e..39486f1c7498260e7eda5fd83b9de379f79b1d4f 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexService.java
@@ -19,5 +19,5 @@ import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
public interface ReindexService {
- String reindexRecords(RecordReindexRequest recordReindexRequest);
+ String reindexRecords(RecordReindexRequest recordReindexRequest, boolean forceClean);
}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java
index 16a8c0d03d41e2566283c0528adfc87d4001d682..ac471b3946609ec01facd3d23b926140a15a2e4e 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java
@@ -16,7 +16,6 @@ package org.opengroup.osdu.indexer.service;
import com.google.common.base.Strings;
import com.google.gson.Gson;
-import lombok.extern.java.Log;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.http.AppException;
@@ -24,6 +23,7 @@ import org.opengroup.osdu.core.common.model.indexer.OperationType;
import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.search.Config;
import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
@@ -36,7 +36,6 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
-@Log
@Component
public class ReindexServiceImpl implements ReindexService {
@@ -47,14 +46,22 @@ public class ReindexServiceImpl implements ReindexService {
@Inject
private IRequestInfo requestInfo;
@Inject
+ private IndexSchemaService indexSchemaService;
+ @Inject
private JaxRsDpsLog jaxRsDpsLog;
@Override
- public String reindexRecords(RecordReindexRequest recordReindexRequest) {
+ public String reindexRecords(RecordReindexRequest recordReindexRequest, boolean forceClean) {
+ Long initialDelayMillis = 0l;
try {
DpsHeaders headers = this.requestInfo.getHeadersWithDwdAuthZ();
+ if (forceClean) {
+ this.indexSchemaService.syncIndexMappingWithStorageSchema(recordReindexRequest.getKind());
+ initialDelayMillis = 30000l;
+ }
+
RecordQueryResponse recordQueryResponse = this.storageService.getRecordsByKind(recordReindexRequest);
if (recordQueryResponse.getResults() != null && recordQueryResponse.getResults().size() != 0) {
@@ -70,11 +77,13 @@ public class ReindexServiceImpl implements ReindexService {
Gson gson = new Gson();
RecordChangedMessages recordChangedMessages = RecordChangedMessages.builder().data(gson.toJson(msgs)).attributes(attributes).build();
String recordChangedMessagePayload = gson.toJson(recordChangedMessages);
- this.indexerQueueTaskBuilder.createWorkerTask(recordChangedMessagePayload, headers);
+ this.indexerQueueTaskBuilder.createWorkerTask(recordChangedMessagePayload, initialDelayMillis, headers);
- if (!Strings.isNullOrEmpty(recordQueryResponse.getCursor())) {
+ // don't call reindex-worker endpoint if it's the last batch
+ // previous storage query result size will be less then requested (limit param)
+ if (!Strings.isNullOrEmpty(recordQueryResponse.getCursor()) && recordQueryResponse.getResults().size() == Config.getStorageRecordsBatchSize()) {
String newPayLoad = gson.toJson(RecordReindexRequest.builder().cursor(recordQueryResponse.getCursor()).kind(recordReindexRequest.getKind()).build());
- this.indexerQueueTaskBuilder.createReIndexTask(newPayLoad, headers);
+ this.indexerQueueTaskBuilder.createReIndexTask(newPayLoad, initialDelayMillis, headers);
return newPayLoad;
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java
new file mode 100644
index 0000000000000000000000000000000000000000..bef023703b60341eb2c214740e1acb1ef23243df
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java
@@ -0,0 +1,124 @@
+package org.opengroup.osdu.indexer.service;
+
+import org.apache.commons.beanutils.NestedNullException;
+import org.apache.commons.beanutils.PropertyUtils;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.indexer.ElasticType;
+import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
+import org.springframework.stereotype.Component;
+
+import javax.inject.Inject;
+import java.lang.reflect.InvocationTargetException;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.opengroup.osdu.indexer.service.IAttributeParsingService.DATA_GEOJSON_TAG;
+import static org.opengroup.osdu.indexer.service.IAttributeParsingService.RECORD_GEOJSON_TAG;
+
+@Component
+public class StorageIndexerPayloadMapper {
+
+ @Inject
+ private JaxRsDpsLog log;
+ @Inject
+ private IAttributeParsingService attributeParsingService;
+
+ public Map<String, Object> mapDataPayload(IndexSchema storageSchema, Map<String, Object> storageRecordData, String recordId) {
+
+ Map<String, Object> dataMap = new HashMap<>();
+
+ if (storageSchema.isDataSchemaMissing()) return dataMap;
+
+ // get the key and get the corresponding object from the storageRecord object
+ for (Map.Entry<String, String> entry : storageSchema.getDataSchema().entrySet()) {
+
+ String name = entry.getKey();
+
+ Object value = getPropertyValue(recordId, storageRecordData, name);
+
+ if (value == null) continue;
+
+ ElasticType elasticType = ElasticType.forValue(entry.getValue());
+
+ switch (elasticType) {
+ case KEYWORD:
+ case KEYWORD_ARRAY:
+ case TEXT:
+ case TEXT_ARRAY:
+ dataMap.put(name, value);
+ break;
+ case INTEGER_ARRAY:
+ this.attributeParsingService.tryParseValueArray(Integer.class, recordId, name, value, dataMap);
+ break;
+ case INTEGER:
+ this.attributeParsingService.tryParseInteger(recordId, name, value, dataMap);
+ break;
+ case LONG_ARRAY:
+ this.attributeParsingService.tryParseValueArray(Long.class, recordId, name, value, dataMap);
+ break;
+ case LONG:
+ this.attributeParsingService.tryParseLong(recordId, name, value, dataMap);
+ break;
+ case FLOAT_ARRAY:
+ this.attributeParsingService.tryParseValueArray(Float.class, recordId, name, value, dataMap);
+ break;
+ case FLOAT:
+ this.attributeParsingService.tryParseFloat(recordId, name, value, dataMap);
+ break;
+ case DOUBLE_ARRAY:
+ this.attributeParsingService.tryParseValueArray(Double.class, recordId, name, value, dataMap);
+ break;
+ case DOUBLE:
+ this.attributeParsingService.tryParseDouble(recordId, name, value, dataMap);
+ break;
+ case BOOLEAN_ARRAY:
+ this.attributeParsingService.tryParseValueArray(Boolean.class, recordId, name, value, dataMap);
+ break;
+ case BOOLEAN:
+ this.attributeParsingService.tryParseBoolean(recordId, name, value, dataMap);
+ break;
+ case DATE_ARRAY:
+ this.attributeParsingService.tryParseValueArray(Date.class, recordId, name, value, dataMap);
+ break;
+ case DATE:
+ this.attributeParsingService.tryParseDate(recordId, name, value, dataMap);
+ break;
+ case GEO_POINT:
+ this.attributeParsingService.tryParseGeopoint(recordId, name, storageRecordData, dataMap);
+ break;
+ case GEO_SHAPE:
+ this.attributeParsingService.tryParseGeojson(recordId, name, value, dataMap);
+ break;
+ case NESTED:
+ case OBJECT:
+ case UNDEFINED:
+ // don't do anything for now
+ break;
+ }
+ }
+
+ // add these once iterated over the list
+ storageSchema.getDataSchema().put(DATA_GEOJSON_TAG, ElasticType.GEO_SHAPE.getValue());
+ storageSchema.getDataSchema().remove(RECORD_GEOJSON_TAG);
+
+ return dataMap;
+ }
+
+ private Object getPropertyValue(String recordId, Map<String, Object> storageRecordData, String propertyKey) {
+
+ try {
+ // try getting first level property using optimized collection
+ Object propertyVal = storageRecordData.get(propertyKey);
+ if (propertyVal != null) return propertyVal;
+
+ // use apache utils to get nested property
+ return PropertyUtils.getProperty(storageRecordData, propertyKey);
+ } catch (NestedNullException ignored) {
+ // property not found in record
+ } catch (IllegalArgumentException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
+ this.log.warning(String.format("record-id: %s | error fetching property: %s | error: %s", recordId, propertyKey, e.getMessage()), e);
+ }
+ return null;
+ }
+}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java
index ec0fbd948171d8c0990dd60403be5a652a5f1761..4fb392ded7f7900b1df3b966c207506ab56072d2 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java
@@ -1,153 +1,224 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.service;
-
-import com.google.api.client.http.HttpMethods;
-import com.google.common.base.Strings;
-import com.google.common.collect.Lists;
-import com.google.common.reflect.TypeToken;
-import com.google.gson.Gson;
-import lombok.extern.java.Log;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.http.HttpResponse;
-import org.opengroup.osdu.core.common.model.http.RequestStatus;
-import org.opengroup.osdu.core.common.model.indexer.*;
-import org.opengroup.osdu.core.common.model.storage.ConversionStatus;
-import org.opengroup.osdu.core.common.model.storage.RecordIds;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.http.IUrlFetchService;
-import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
-import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
-import org.apache.http.HttpStatus;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Component;
-
-import javax.inject.Inject;
-import java.io.UnsupportedEncodingException;
-import java.lang.reflect.Type;
-import java.net.URISyntaxException;
-import java.net.URLEncoder;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.opengroup.osdu.core.common.model.http.DpsHeaders.FRAME_OF_REFERENCE;
-import static org.opengroup.osdu.core.common.Constants.SLB_FRAME_OF_REFERENCE_VALUE;
-
-@Log
-@Component
-public class StorageServiceImpl implements StorageService {
-
- private final Gson gson = new Gson();
-
- @Inject
- private IUrlFetchService urlFetchService;
- @Inject
- private JobStatus jobStatus;
- @Inject
- private IRequestInfo requestInfo;
- @Inject
- private JaxRsDpsLog jaxRsDpsLog;
-
- @Value("${STORAGE_SCHEMA_HOST}")
- private String STORAGE_SCHEMA_HOST;
-
- @Value("${STORAGE_QUERY_RECORD_HOST}")
- private String STORAGE_QUERY_RECORD_HOST;
-
- @Value("${STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST}")
- private String STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST;
-
- @Value("${STORAGE_RECORDS_BATCH_SIZE}")
- private String STORAGE_RECORDS_BATCH_SIZE;
-
- @Override
- public Records getStorageRecords(List<String> ids) throws AppException, URISyntaxException {
- List<Records.Entity> valid = new ArrayList<>();
- List<String> notFound = new ArrayList<>();
- List<ConversionStatus> conversionStatuses = new ArrayList<>();
-
- List<List<String>> batch = Lists.partition(ids, Integer.parseInt(STORAGE_RECORDS_BATCH_SIZE));
- for (List<String> recordsBatch : batch) {
- Records storageOut = this.getRecords(recordsBatch);
- valid.addAll(storageOut.getRecords());
- notFound.addAll(storageOut.getNotFound());
- conversionStatuses.addAll(storageOut.getConversionStatuses());
- }
- return Records.builder().records(valid).notFound(notFound).conversionStatuses(conversionStatuses).build();
- }
-
- private Records getRecords(List<String> ids) throws URISyntaxException {
- // e.g. {"records":["test:10"]}
- String body = this.gson.toJson(RecordIds.builder().records(ids).build());
-
-// Map<String, String> headers = this.requestInfo.getHeadersMap();
- DpsHeaders headers = this.requestInfo.getHeaders();
- headers.put(FRAME_OF_REFERENCE, SLB_FRAME_OF_REFERENCE_VALUE);
- HttpResponse response = this.urlFetchService.sendRequest(HttpMethods.POST, STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST, headers, null, body);
- String dataFromStorage = response.getBody();
- if (Strings.isNullOrEmpty(dataFromStorage)) {
- throw new AppException(HttpStatus.SC_NOT_FOUND, "Invalid request", "Storage service cannot locate records");
- }
-
- Type recordsListType = new TypeToken<Records>() {}.getType();
- Records records = this.gson.fromJson(dataFromStorage, recordsListType);
-
- // update status for invalid records from storage
- if (records.getNotFound() != null && !records.getNotFound().isEmpty()) {
- this.jobStatus.addOrUpdateRecordStatus(records.getNotFound(), IndexingStatus.FAIL, RequestStatus.INVALID_RECORD, "invalid storage records", String.format("invalid records: %s", String.join(",", records.getNotFound())));
- }
-
- // don't proceed if there is nothing to process
- List<Records.Entity> validRecords = records.getRecords();
- if (validRecords == null || validRecords.isEmpty()) {
- if (response.isSuccessCode()) {
- throw new AppException(RequestStatus.INVALID_RECORD, "Invalid request", "Storage service returned retry or invalid records");
- }
-
- // TODO: returned actual code from storage service
- jaxRsDpsLog.warning(String.format("unable to proceed, valid storage record not found. | upstream response code: %s | record ids: %s", response.getResponseCode(), String.join(" | ", ids)));
- throw new AppException(HttpStatus.SC_NOT_FOUND, "Invalid request", "Storage service cannot locate valid records");
- }
-
- return records;
- }
-
- @Override
- public RecordQueryResponse getRecordsByKind(RecordReindexRequest request) throws URISyntaxException {
- Map<String, String> queryParams = new HashMap<>();
- queryParams.put(RecordMetaAttribute.KIND.getValue(), request.getKind());
- queryParams.put("limit", STORAGE_RECORDS_BATCH_SIZE);
- if (!Strings.isNullOrEmpty(request.getCursor())) {
- queryParams.put("cursor", request.getCursor());
- }
-
- if(requestInfo == null)
- throw new AppException(HttpStatus.SC_NO_CONTENT, "Invalid header", "header can't be null");
-
- HttpResponse response = this.urlFetchService.sendRequest(HttpMethods.GET, STORAGE_QUERY_RECORD_HOST, this.requestInfo.getHeaders(), queryParams, null);
- return this.gson.fromJson(response.getBody(), RecordQueryResponse.class);
- }
-
- @Override
- public String getStorageSchema(String kind) throws URISyntaxException, UnsupportedEncodingException {
- String url = String.format("%s/%s", STORAGE_SCHEMA_HOST, URLEncoder.encode(kind, "UTF-8"));
- HttpResponse response = this.urlFetchService.sendRequest(HttpMethods.GET, url, this.requestInfo.getHeaders(), null, null);
- if (response.getResponseCode() != HttpStatus.SC_OK) return null;
- return response.getBody();
- }
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.service;
+
+import com.google.api.client.http.HttpMethods;
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
+import com.google.common.reflect.TypeToken;
+import com.google.gson.Gson;
+
+import org.opengroup.osdu.core.common.http.FetchServiceHttpRequest;
+import lombok.extern.java.Log;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.http.HttpResponse;
+import org.opengroup.osdu.core.common.model.http.RequestStatus;
+import org.opengroup.osdu.core.common.model.indexer.*;
+import org.opengroup.osdu.core.common.model.storage.ConversionStatus;
+import org.opengroup.osdu.core.common.model.storage.RecordIds;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.http.IUrlFetchService;
+import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.core.common.search.Config;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import javax.inject.Inject;
+import java.io.UnsupportedEncodingException;
+import java.lang.reflect.Type;
+import java.net.URISyntaxException;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.opengroup.osdu.core.common.model.http.DpsHeaders.FRAME_OF_REFERENCE;
+import static org.opengroup.osdu.core.common.Constants.SLB_FRAME_OF_REFERENCE_VALUE;
+
+@Component
+public class StorageServiceImpl implements StorageService {
+
+ private final Gson gson = new Gson();
+
+ @Inject
+ private IUrlFetchService urlFetchService;
+ @Inject
+ private JobStatus jobStatus;
+ @Inject
+ private IRequestInfo requestInfo;
+ @Inject
+ private JaxRsDpsLog jaxRsDpsLog;
+
+ @Value("${STORAGE_SCHEMA_HOST}")
+ private String STORAGE_SCHEMA_HOST;
+
+ @Value("${STORAGE_QUERY_RECORD_HOST}")
+ private String STORAGE_QUERY_RECORD_HOST;
+
+ @Value("${STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST}")
+ private String STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST;
+
+ @Value("${STORAGE_RECORDS_BATCH_SIZE}")
+ private String STORAGE_RECORDS_BATCH_SIZE;
+
+ @Override
+ public Records getStorageRecords(List<String> ids) throws AppException, URISyntaxException {
+ List<Records.Entity> valid = new ArrayList<>();
+ List<String> notFound = new ArrayList<>();
+ List<ConversionStatus> conversionStatuses = new ArrayList<>();
+ List<String> missingRetryRecordIds = new ArrayList<>();
+
+ List<List<String>> batch = Lists.partition(ids, Integer.parseInt(STORAGE_RECORDS_BATCH_SIZE));
+ for (List<String> recordsBatch : batch) {
+ Records storageOut = this.getRecords(recordsBatch);
+ valid.addAll(storageOut.getRecords());
+ notFound.addAll(storageOut.getNotFound());
+ conversionStatuses.addAll(storageOut.getConversionStatuses());
+ missingRetryRecordIds.addAll(storageOut.getMissingRetryRecords());
+ }
+ return Records.builder().records(valid).notFound(notFound).conversionStatuses(conversionStatuses).missingRetryRecords(missingRetryRecordIds).build();
+ }
+
+ private Records getRecords(List<String> ids) throws URISyntaxException {
+ // e.g. {"records":["test:10"]}
+ String body = this.gson.toJson(RecordIds.builder().records(ids).build());
+
+// Map<String, String> headers = this.requestInfo.getHeadersMap();
+ DpsHeaders headers = this.requestInfo.getHeaders();
+ headers.put(FRAME_OF_REFERENCE, SLB_FRAME_OF_REFERENCE_VALUE);
+ FetchServiceHttpRequest request = FetchServiceHttpRequest
+ .builder()
+ .httpMethod(HttpMethods.POST)
+ .url(STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST)
+ .headers(headers)
+ .body(body).build();
+ HttpResponse response = this.urlFetchService.sendRequest(request);
+ return this.validateStorageResponse(response, ids);
+ }
+
+ private Records validateStorageResponse(HttpResponse response, List<String> ids) {
+ String bulkStorageData = response.getBody();
+
+ // retry entire payload -- storage service returned empty response
+ if (Strings.isNullOrEmpty(bulkStorageData)) {
+ throw new AppException(HttpStatus.SC_NOT_FOUND, "Invalid request", "Storage service returned empty response");
+ }
+
+ Type recordsListType = new TypeToken<Records>() {
+ }.getType();
+ Records records = this.gson.fromJson(bulkStorageData, recordsListType);
+
+ // no retry possible, update record status as failed -- storage service cannot locate records
+ if (!records.getNotFound().isEmpty()) {
+ this.jobStatus.addOrUpdateRecordStatus(records.getNotFound(), IndexingStatus.FAIL, RequestStatus.INVALID_RECORD, "Storage service records not found", String.format("Storage service records not found: %s", String.join(",", records.getNotFound())));
+ }
+
+ List<Records.Entity> validRecords = records.getRecords();
+ if (validRecords.isEmpty()) {
+ // no need to retry, ack the CloudTask message -- nothing to process from RecordChangeMessage batch
+ if (response.isSuccessCode()) {
+ throw new AppException(RequestStatus.INVALID_RECORD, "Invalid request", "Successful Storage service response with no valid records");
+ }
+
+ // retry entire payload -- storage service returned empty valid records with non-success response-code
+ jaxRsDpsLog.warning(String.format("unable to proceed, valid storage record not found. | upstream response code: %s | record ids: %s", response.getResponseCode(), String.join(" | ", ids)));
+ throw new AppException(HttpStatus.SC_NOT_FOUND, "Invalid request", "Storage service error");
+ }
+
+ Map<String, List<String>> conversionStatus = getConversionErrors(records.getConversionStatuses());
+ for (Records.Entity storageRecord : validRecords) {
+ String recordId = storageRecord.getId();
+ if (conversionStatus.get(recordId) == null) {
+ continue;
+ }
+ for (String status : conversionStatus.get(recordId)) {
+ this.jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, status, String.format("record-id: %s | %s", recordId, status));
+ }
+ }
+
+ // retry missing records -- storage did not return response for all RecordChangeMessage record-ids
+ if (records.getTotalRecordCount() != ids.size()) {
+ List<String> missingRecords = this.getMissingRecords(records, ids);
+ records.setMissingRetryRecords(missingRecords);
+ this.jobStatus.addOrUpdateRecordStatus(missingRecords, IndexingStatus.FAIL, HttpStatus.SC_NOT_FOUND, "Partial response received from Storage service - missing records", String.format("Partial response received from Storage service: %s", String.join(",", missingRecords)));
+ }
+
+ return records;
+ }
+
+ private List<String> getMissingRecords(Records records, List<String> ids) {
+ List<String> validRecordIds = records.getRecords().stream().map(Records.Entity::getId).collect(Collectors.toList());
+ List<String> invalidRecordsIds = records.getNotFound();
+ List<String> requestedIds = new ArrayList<>(ids);
+ requestedIds.removeAll(validRecordIds);
+ requestedIds.removeAll(invalidRecordsIds);
+ return requestedIds;
+ }
+
+ private Map<String, List<String>> getConversionErrors(List<ConversionStatus> conversionStatuses) {
+ Map<String, List<String>> errorsByRecordId = new HashMap<>();
+ for (ConversionStatus conversionStatus : conversionStatuses) {
+ if (Strings.isNullOrEmpty(conversionStatus.getStatus())) continue;
+ if (conversionStatus.getStatus().equalsIgnoreCase("ERROR")) {
+ List<String> statuses = errorsByRecordId.getOrDefault(conversionStatus.getId(), new LinkedList<>());
+ statuses.addAll(conversionStatus.getErrors());
+ errorsByRecordId.put(conversionStatus.getId(), statuses);
+ }
+ }
+ return errorsByRecordId;
+ }
+
+ @Override
+ public RecordQueryResponse getRecordsByKind(RecordReindexRequest reindexRequest) throws URISyntaxException {
+ Map<String, String> queryParams = new HashMap<>();
+ queryParams.put(RecordMetaAttribute.KIND.getValue(), reindexRequest.getKind());
+ queryParams.put("limit", STORAGE_RECORDS_BATCH_SIZE);
+ if (!Strings.isNullOrEmpty(reindexRequest.getCursor())) {
+ queryParams.put("cursor", reindexRequest.getCursor());
+ }
+
+ if(requestInfo == null)
+ throw new AppException(HttpStatus.SC_NO_CONTENT, "Invalid header", "header can't be null");
+
+ FetchServiceHttpRequest request = FetchServiceHttpRequest.builder()
+ .httpMethod(HttpMethods.GET)
+ .headers(this.requestInfo.getHeadersMap())
+ .url(Config.getStorageQueryRecordHostUrl())
+ .queryParams(queryParams)
+ .build();
+
+ HttpResponse response = this.urlFetchService.sendRequest(request);
+ return this.gson.fromJson(response.getBody(), RecordQueryResponse.class);
+ }
+
+ @Override
+ public String getStorageSchema(String kind) throws URISyntaxException, UnsupportedEncodingException {
+ String url = String.format("%s/%s", STORAGE_SCHEMA_HOST, URLEncoder.encode(kind, StandardCharsets.UTF_8.toString()));
+ FetchServiceHttpRequest request = FetchServiceHttpRequest.builder()
+ .httpMethod(HttpMethods.GET)
+ .headers(this.requestInfo.getHeadersMap())
+ .url(url)
+ .build();
+ HttpResponse response = this.urlFetchService.sendRequest(request);
+ return response.getResponseCode() != HttpStatus.SC_OK ? null : response.getBody();
+ }
}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/ElasticClientHandler.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/ElasticClientHandler.java
index 7035abf3ecb9820e4f09b1c7f3ff89966437152a..1007967178a2d2c4859ebb8f3f50c88d29c80afc 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/ElasticClientHandler.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/ElasticClientHandler.java
@@ -1,108 +1,108 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.util;
-
-import org.apache.http.Header;
-import org.apache.http.HttpHost;
-import org.apache.http.HttpStatus;
-import org.apache.http.message.BasicHeader;
-import org.elasticsearch.client.RestClient;
-import org.elasticsearch.client.RestClientBuilder;
-import org.elasticsearch.client.RestHighLevelClient;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.search.ClusterSettings;
-import org.opengroup.osdu.core.common.model.indexer.IElasticSettingService;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-import java.util.Base64;
-
-@Component
-public class ElasticClientHandler {
-
- // Elastic cluster Rest client settings
- private static final int CLOUD_REST_CLIENT_PORT = 9243;
- private static final int REST_CLIENT_CONNECT_TIMEOUT = 60000;
- private static final int REST_CLIENT_SOCKET_TIMEOUT = 60000;
- private static final int REST_CLIENT_RETRY_TIMEOUT = 60000;
-
- @Autowired
- private IElasticSettingService elasticSettingService;
-
- public RestHighLevelClient createRestClient() {
- return getCloudRestClient(elasticSettingService.getElasticClusterInformation());
- }
- // TODO: Remove this temporary implementation when ECE CCS is utilized
- public RestHighLevelClient createRestClient(final ClusterSettings clusterSettings) {
- return getCloudRestClient(clusterSettings);
- }
-
- private RestHighLevelClient getCloudRestClient(final ClusterSettings clusterSettings) {
-
- String cluster = null;
- String host = null;
- int port = CLOUD_REST_CLIENT_PORT;
- String protocolScheme = "https";
- String tls = "true";
-
- try {
- cluster = clusterSettings.getHost();
- host = clusterSettings.getHost();
- port = clusterSettings.getPort();
- if(!clusterSettings.isHttps()){
- protocolScheme = "http";
- }
-
- if(!clusterSettings.isTls()){
- tls = "false";
- }
- String basicEncoded = Base64.getEncoder().encodeToString(clusterSettings.getUserNameAndPassword().getBytes());
- String basicAuthenticationHeaderVal = String.format("Basic %s", basicEncoded);
-
- RestClientBuilder builder = createClientBuilder(host, basicAuthenticationHeaderVal, port, protocolScheme, tls);
-
- return new RestHighLevelClient(builder);
- } catch (AppException e) {
- throw e;
- } catch (Exception e) {
- throw new AppException(
- HttpStatus.SC_INTERNAL_SERVER_ERROR,
- "search client error",
- "error creating search client",
- String.format("Elastic client connection params, cluster: %s, host: %s, port: %s", cluster, host, port),
- e);
- }
- }
-
- public RestClientBuilder createClientBuilder(String host, String basicAuthenticationHeaderVal, int port, String protocolScheme, String tls) {
- RestClientBuilder builder = RestClient.builder(new HttpHost(host, port, protocolScheme));
- builder.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(REST_CLIENT_CONNECT_TIMEOUT)
- .setSocketTimeout(REST_CLIENT_SOCKET_TIMEOUT));
- builder.setMaxRetryTimeoutMillis(REST_CLIENT_RETRY_TIMEOUT);
-
- Header[] defaultHeaders = new Header[]{
- new BasicHeader("client.transport.nodes_sampler_interval", "30s"),
- new BasicHeader("client.transport.ping_timeout", "30s"),
- new BasicHeader("client.transport.sniff", "false"),
- new BasicHeader("request.headers.X-Found-Cluster", host),
- new BasicHeader("cluster.name", host),
- new BasicHeader("xpack.security.transport.ssl.enabled", tls),
- new BasicHeader("Authorization", basicAuthenticationHeaderVal),
- };
-
- builder.setDefaultHeaders(defaultHeaders);
- return builder;
- }
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.util;
+
+import org.apache.http.Header;
+import org.apache.http.HttpHost;
+import org.apache.http.HttpStatus;
+import org.apache.http.message.BasicHeader;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestClientBuilder;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.search.ClusterSettings;
+import org.opengroup.osdu.core.common.model.indexer.IElasticSettingService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.Base64;
+
+@Component
+public class ElasticClientHandler {
+
+ // Elastic cluster Rest client settings
+ private static final int CLOUD_REST_CLIENT_PORT = 9243;
+ private static final int REST_CLIENT_CONNECT_TIMEOUT = 60000;
+ private static final int REST_CLIENT_SOCKET_TIMEOUT = 60000;
+ private static final int REST_CLIENT_RETRY_TIMEOUT = 60000;
+
+ @Autowired
+ private IElasticSettingService elasticSettingService;
+
+ public RestHighLevelClient createRestClient() {
+ return getCloudRestClient(elasticSettingService.getElasticClusterInformation());
+ }
+ // TODO: Remove this temporary implementation when ECE CCS is utilized
+ public RestHighLevelClient createRestClient(final ClusterSettings clusterSettings) {
+ return getCloudRestClient(clusterSettings);
+ }
+
+ private RestHighLevelClient getCloudRestClient(final ClusterSettings clusterSettings) {
+
+ String cluster = null;
+ String host = null;
+ int port = CLOUD_REST_CLIENT_PORT;
+ String protocolScheme = "https";
+ String tls = "true";
+
+ try {
+ cluster = clusterSettings.getHost();
+ host = clusterSettings.getHost();
+ port = clusterSettings.getPort();
+ if(!clusterSettings.isHttps()){
+ protocolScheme = "http";
+ }
+
+ if(!clusterSettings.isTls()){
+ tls = "false";
+ }
+ String basicEncoded = Base64.getEncoder().encodeToString(clusterSettings.getUserNameAndPassword().getBytes());
+ String basicAuthenticationHeaderVal = String.format("Basic %s", basicEncoded);
+
+ RestClientBuilder builder = createClientBuilder(host, basicAuthenticationHeaderVal, port, protocolScheme, tls);
+
+ return new RestHighLevelClient(builder);
+ } catch (AppException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AppException(
+ HttpStatus.SC_INTERNAL_SERVER_ERROR,
+ "search client error",
+ "error creating search client",
+ String.format("Elastic client connection params, cluster: %s, host: %s, port: %s", cluster, host, port),
+ e);
+ }
+ }
+
+ public RestClientBuilder createClientBuilder(String host, String basicAuthenticationHeaderVal, int port, String protocolScheme, String tls) {
+ RestClientBuilder builder = RestClient.builder(new HttpHost(host, port, protocolScheme));
+ builder.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(REST_CLIENT_CONNECT_TIMEOUT)
+ .setSocketTimeout(REST_CLIENT_SOCKET_TIMEOUT));
+ builder.setMaxRetryTimeoutMillis(REST_CLIENT_RETRY_TIMEOUT);
+
+ Header[] defaultHeaders = new Header[]{
+ new BasicHeader("client.transport.nodes_sampler_interval", "30s"),
+ new BasicHeader("client.transport.ping_timeout", "30s"),
+ new BasicHeader("client.transport.sniff", "false"),
+ new BasicHeader("request.headers.X-Found-Cluster", host),
+ new BasicHeader("cluster.name", host),
+ new BasicHeader("xpack.security.transport.ssl.enabled", tls),
+ new BasicHeader("Authorization", basicAuthenticationHeaderVal),
+ };
+
+ builder.setDefaultHeaders(defaultHeaders);
+ return builder;
+ }
}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/IndexerQueueTaskBuilder.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/IndexerQueueTaskBuilder.java
index 3621691c10e7fb0e62aa153d781302798eb4d7f2..845ec58d784c740017eaf6dfe68468ee00b284e9 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/IndexerQueueTaskBuilder.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/IndexerQueueTaskBuilder.java
@@ -1,72 +1,86 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.util;
-
-import com.google.api.client.http.HttpMethods;
-import com.google.gson.Gson;
-import lombok.extern.java.Log;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.search.CloudTaskRequest;
-import org.opengroup.osdu.core.common.model.http.HttpResponse;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.http.IUrlFetchService;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Component;
-import org.springframework.web.context.annotation.RequestScope;
-
-import java.net.URISyntaxException;
-import javax.inject.Inject;
-
-import static org.opengroup.osdu.core.common.Constants.REINDEX_RELATIVE_URL;
-import static org.opengroup.osdu.core.common.Constants.WORKER_RELATIVE_URL;
-
- @Log
- @Component
- @RequestScope
- public class IndexerQueueTaskBuilder {
-
- @Inject
- private IUrlFetchService urlFetchService;
- @Inject
- private JaxRsDpsLog jaxRsDpsLog;
-
- @Value("${INDEXER_QUEUE_HOST}")
- private String INDEXER_QUEUE_HOST;
-
- public void createWorkerTask(String payload, DpsHeaders headers) {
- createTask(WORKER_RELATIVE_URL, payload, headers);
- }
-
- public void createReIndexTask(String payload,DpsHeaders headers) {
- createTask(REINDEX_RELATIVE_URL, payload, headers);
- }
-
- private void createTask(String url, String payload, DpsHeaders headers) {
-
- CloudTaskRequest cloudTaskRequest = CloudTaskRequest.builder().message(payload).url(url).build();
-
- try {
- HttpResponse response = this.urlFetchService.sendRequest(
- HttpMethods.POST,
- INDEXER_QUEUE_HOST,
- headers,
- null,
- new Gson().toJson(cloudTaskRequest));
- this.jaxRsDpsLog.info(String.format("task enqueuing response: %s", response.getResponseCode()));
- } catch (URISyntaxException e) {
- this.jaxRsDpsLog.warning(String.format("error enqueuing task message: %s | url: %s | task payload: %s", e.getMessage(), url, payload));
- }
- }
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.util;
+
+import com.google.api.client.http.HttpMethods;
+import com.google.gson.Gson;
+import lombok.extern.java.Log;
+import org.opengroup.osdu.core.common.http.FetchServiceHttpRequest;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.search.CloudTaskRequest;
+import org.opengroup.osdu.core.common.model.http.HttpResponse;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.http.IUrlFetchService;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+import java.net.URISyntaxException;
+import java.util.Map;
+import javax.inject.Inject;
+
+import static org.opengroup.osdu.core.common.Constants.REINDEX_RELATIVE_URL;
+import static org.opengroup.osdu.core.common.Constants.WORKER_RELATIVE_URL;
+
+ @Log
+ @Component
+ @RequestScope
+ public class IndexerQueueTaskBuilder {
+
+ @Inject
+ private IUrlFetchService urlFetchService;
+ @Inject
+ private JaxRsDpsLog jaxRsDpsLog;
+
+ @Value("${INDEXER_QUEUE_HOST}")
+ private String INDEXER_QUEUE_HOST;
+
+ public void createWorkerTask(String payload, DpsHeaders headers) {
+ createTask(WORKER_RELATIVE_URL, payload, 0l, headers);
+ }
+
+ public void createWorkerTask(String payload, Long countdownMillis, DpsHeaders headers) {
+ createTask(WORKER_RELATIVE_URL, payload, countdownMillis, headers);
+ }
+
+ public void createReIndexTask(String payload, DpsHeaders headers) {
+ createTask(REINDEX_RELATIVE_URL, payload, 0l, headers);
+ }
+
+ public void createReIndexTask(String payload, Long countdownMillis, DpsHeaders headers) {
+ createTask(REINDEX_RELATIVE_URL, payload, countdownMillis, headers);
+ }
+
+ private void createTask(String url, String payload, Long countdownMillis, DpsHeaders headers) {
+ CloudTaskRequest cloudTaskRequest = CloudTaskRequest.builder()
+ .message(payload)
+ .url(url)
+ .initialDelayMillis(countdownMillis)
+ .build();
+
+ FetchServiceHttpRequest request = FetchServiceHttpRequest.builder()
+ .httpMethod(HttpMethods.POST)
+ .url(INDEXER_QUEUE_HOST)
+ .body(new Gson().toJson(cloudTaskRequest))
+ .headers(headers)
+ .build();
+ try {
+ HttpResponse response = this.urlFetchService.sendRequest(request);
+ this.jaxRsDpsLog.info(String.format("task enqueuing response: %s", response.getResponseCode()));
+ } catch (URISyntaxException e) {
+ this.jaxRsDpsLog.warning(String.format("error enqueuing task message: %s | url: %s | task payload: %s", e.getMessage(), url, payload));
+ }
+ }
}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java
index 11b2935c2066a0f932714003089bd4b2489fb723..2f7c9b7c0e9b3051d6a428fec359be7418f35f93 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java
@@ -21,6 +21,7 @@ import org.opengroup.osdu.core.common.model.indexer.Records;
import org.opengroup.osdu.core.common.model.indexer.StorageType;
import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
+import org.apache.commons.lang3.StringUtils;
import java.util.HashMap;
import java.util.Map;
@@ -44,22 +45,32 @@ public class TypeMapper {
metaAttributeIndexerType.put(RecordMetaAttribute.INDEX_STATUS.getValue(), getIndexStatusMapping());
storageToIndexerType.put(StorageType.LINK.getValue(), ElasticType.KEYWORD.getValue());
- storageToIndexerType.put(StorageType.LINK_ARRAY.getValue(), ElasticType.KEYWORD.getValue());
+ storageToIndexerType.put(StorageType.LINK_ARRAY.getValue(), ElasticType.KEYWORD_ARRAY.getValue());
storageToIndexerType.put(StorageType.BOOLEAN.getValue(), ElasticType.BOOLEAN.getValue());
+ storageToIndexerType.put(StorageType.BOOLEAN_ARRAY.getValue(), ElasticType.BOOLEAN_ARRAY.getValue());
storageToIndexerType.put(StorageType.STRING.getValue(), ElasticType.TEXT.getValue());
+ storageToIndexerType.put(StorageType.STRING_ARRAY.getValue(), ElasticType.TEXT_ARRAY.getValue());
storageToIndexerType.put(StorageType.INT.getValue(), ElasticType.INTEGER.getValue());
+ storageToIndexerType.put(StorageType.INT_ARRAY.getValue(), ElasticType.INTEGER_ARRAY.getValue());
storageToIndexerType.put(StorageType.FLOAT.getValue(), ElasticType.FLOAT.getValue());
+ storageToIndexerType.put(StorageType.FLOAT_ARRAY.getValue(), ElasticType.FLOAT_ARRAY.getValue());
storageToIndexerType.put(StorageType.DOUBLE.getValue(), ElasticType.DOUBLE.getValue());
- storageToIndexerType.put(StorageType.DOUBLE_ARRAY.getValue(), ElasticType.DOUBLE.getValue());
+ storageToIndexerType.put(StorageType.DOUBLE_ARRAY.getValue(), ElasticType.DOUBLE_ARRAY.getValue());
storageToIndexerType.put(StorageType.LONG.getValue(), ElasticType.LONG.getValue());
+ storageToIndexerType.put(StorageType.LONG_ARRAY.getValue(), ElasticType.LONG_ARRAY.getValue());
storageToIndexerType.put(StorageType.DATETIME.getValue(), ElasticType.DATE.getValue());
+ storageToIndexerType.put(StorageType.DATETIME_ARRAY.getValue(), ElasticType.DATE_ARRAY.getValue());
storageToIndexerType.put(StorageType.GEO_POINT.getValue(), ElasticType.GEO_POINT.getValue());
storageToIndexerType.put(StorageType.GEO_SHAPE.getValue(), ElasticType.GEO_SHAPE.getValue());
}
public static String getIndexerType(String storageType) {
- return storageToIndexerType.getOrDefault(storageType, null);
+ String indexedType = storageToIndexerType.getOrDefault(storageType, null);
+ if (indexedType != null && indexedType.endsWith("_array")) {
+ return StringUtils.substringBefore(indexedType, "_");
+ }
+ return indexedType;
}
public static Object getIndexerType(RecordMetaAttribute attribute) {
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/parser/BooleanParser.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/parser/BooleanParser.java
new file mode 100644
index 0000000000000000000000000000000000000000..3bb08d3fc3274f2fab8a22114567ac0637ef3653
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/parser/BooleanParser.java
@@ -0,0 +1,15 @@
+package org.opengroup.osdu.indexer.util.parser;
+
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+@Component
+@RequestScope
+public class BooleanParser {
+
+ public boolean parseBoolean(String attributeName, Object attributeVal) {
+ String val = attributeVal == null ? null : String.valueOf(attributeVal);
+ return Boolean.parseBoolean(val);
+ }
+
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/parser/DateTimeParser.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/parser/DateTimeParser.java
index 28ba4a47b0748b5432ba046e33ac3e3320108bfe..1fcf39368710b04d93768a05a560b5a21f95798e 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/parser/DateTimeParser.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/parser/DateTimeParser.java
@@ -83,4 +83,17 @@ public class DateTimeParser {
}
return null;
}
+
+ public String parseDate(String attributeName, Object attributeVal) {
+ String val = attributeVal == null ? null : String.valueOf(attributeVal);
+ if (Strings.isNullOrEmpty(val)) {
+ // skip indexing
+ return null;
+ }
+ String utcDate = this.convertDateObjectToUtc(val);
+ if (Strings.isNullOrEmpty(utcDate)) {
+ throw new IllegalArgumentException(String.format("datetime parsing error: unknown format for attribute: %s | value: %s", attributeName, attributeVal));
+ }
+ return utcDate;
+ }
}
\ No newline at end of file
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/ReindexApiTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/ReindexApiTest.java
index 88a101b5df8f2e996b0602ac47c7940292ed01d3..089f8417a3a853b966f5f0524e7ee07e364db955 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/ReindexApiTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/ReindexApiTest.java
@@ -1,74 +1,77 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.api;
-
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.indexer.logging.AuditLogger;
-import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
-import org.opengroup.osdu.indexer.service.ReindexService;
-
-import org.springframework.http.HttpStatus;
-import org.springframework.http.ResponseEntity;
-import org.springframework.test.context.junit4.SpringRunner;
-
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.when;
-
-@RunWith(SpringRunner.class)
-public class ReindexApiTest {
-
- private RecordReindexRequest recordReindexRequest;
-
- @Mock
- private ReindexService reIndexService;
- @Mock
- private AuditLogger auditLogger;
- @InjectMocks
- private ReindexApi sut;
-
- @Before
- public void setup() {
- recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor("100").build();
- }
-
- @Test
- public void should_return200_when_valid_kind_provided() {
- when(this.reIndexService.reindexRecords(recordReindexRequest)).thenReturn("something");
-
- ResponseEntity response = sut.reindex(recordReindexRequest);
-
- assertEquals(HttpStatus.OK.value(), response.getStatusCodeValue());
- }
-
- @Test(expected = AppException.class)
- public void should_throwAppException_ifUnknownExceptionCaught_reindexTest() {
- when(this.reIndexService.reindexRecords(any())).thenThrow(new AppException(500, "", ""));
-
- sut.reindex(recordReindexRequest);
- }
-
- @Test(expected = NullPointerException.class)
- public void should_throwAppException_ifNullPointerExceptionCaught_ReindexTest() {
- when(this.reIndexService.reindexRecords(any())).thenThrow(new NullPointerException(""));
-
- sut.reindex(recordReindexRequest);
- }
-}
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.api;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
+import org.opengroup.osdu.indexer.logging.AuditLogger;
+import org.opengroup.osdu.indexer.service.IndexSchemaService;
+import org.opengroup.osdu.indexer.service.ReindexService;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.when;
+
+@RunWith(SpringRunner.class)
+public class ReindexApiTest {
+
+ private RecordReindexRequest recordReindexRequest;
+
+ @Mock
+ private ReindexService reIndexService;
+ @Mock
+ private IndexSchemaService indexSchemaService;
+ @Mock
+ private AuditLogger auditLogger;
+ @InjectMocks
+ private ReindexApi sut;
+
+ @Before
+ public void setup() {
+ recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor("100").build();
+ }
+
+ @Test
+ public void should_return200_when_valid_kind_provided() throws IOException {
+ when(this.reIndexService.reindexRecords(recordReindexRequest, false)).thenReturn("something");
+
+ ResponseEntity<?> response = sut.reindex(recordReindexRequest, false);
+
+ assertEquals(HttpStatus.OK, response.getStatusCode());
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throwAppException_ifUnknownExceptionCaught_reindexTest() throws IOException {
+ when(this.reIndexService.reindexRecords(recordReindexRequest, false)).thenThrow(new AppException(500, "", ""));
+
+ sut.reindex(recordReindexRequest, false);
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void should_throwAppException_ifNullPointerExceptionCaught_ReindexTest() throws IOException {
+ when(this.reIndexService.reindexRecords(recordReindexRequest, false)).thenThrow(new NullPointerException(""));
+
+ sut.reindex(recordReindexRequest, false);
+ }
+}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImplTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImplTest.java
index 73da01c76e4f01de80d4b24ddae63cacb453a584..5db76eb05d6eaffa535320fe4603b83191e40d26 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImplTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImplTest.java
@@ -21,8 +21,10 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
+import org.mockito.Spy;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.indexer.JobStatus;
+import org.opengroup.osdu.indexer.util.parser.BooleanParser;
import org.opengroup.osdu.indexer.util.parser.DateTimeParser;
import org.opengroup.osdu.indexer.util.parser.GeoShapeParser;
import org.opengroup.osdu.indexer.util.parser.NumberParser;
@@ -31,6 +33,7 @@ import org.springframework.test.context.junit4.SpringRunner;
import java.lang.reflect.Type;
import java.util.HashMap;
import java.util.Map;
+import java.util.Date;
import static org.junit.Assert.*;
import static org.mockito.Matchers.any;
@@ -41,10 +44,12 @@ public class AttributeParsingServiceImplTest {
@Mock
private GeometryConversionService geometryConversionService;
- @Mock
- private NumberParser numberParser;
- @Mock
- private DateTimeParser dateTimeParser;
+ @Spy
+ private BooleanParser booleanParser = new BooleanParser();
+ @Spy
+ private NumberParser numberParser = new NumberParser();
+ @Spy
+ private DateTimeParser dateTimeParser = new DateTimeParser();
@Mock
private GeoShapeParser geoShapeParser;
@Mock
@@ -58,17 +63,45 @@ public class AttributeParsingServiceImplTest {
public void should_parseValidInteger() {
Map<String, Object> dataMap = new HashMap<>();
- when(this.numberParser.parseInteger(any(), any())).thenThrow(new IllegalArgumentException("number parsing error, integer out of range: attribute: lat | value: 101959.E1019594E"));
- this.sut.tryParseInteger("common:welldb:wellbore-OGY4ZWQ5", "lat", "101959.E1019594E", dataMap);
- assertEquals(dataMap.size(), 0);
- assertFalse(dataMap.containsKey("lat"));
+ this.sut.tryParseInteger("common:welldb:wellbore-OGY4ZWQ5", "lat", "101959.15", dataMap);
+ assertEquals(dataMap.size(), 1);
+ assertTrue(dataMap.containsKey("lat"));
+ }
+
+ @Test
+ public void should_parseValidIntegerArray() {
+ Map<String, Object> dataMap = new HashMap<>();
+ final Map<Object, Object> inputs = new HashMap<Object, Object>() {
+ {
+ put("[]", new Integer[]{});
+ put("[101959.1]", new Integer[]{101959});
+ put("[139, 20]", new Integer[]{139, 20});
+ put("[\"139.987\", \"20\"]", new Integer[]{139, 20});
+ }
+ };
+
+ this.validateInput(this.sut::tryParseValueArray, Integer.class, Integer.class, inputs, dataMap);
+ }
+
+ @Test
+ public void should_parseValidLongArray() {
+ Map<String, Object> dataMap = new HashMap<>();
+ final Map<Object, Object> inputs = new HashMap<Object, Object>() {
+ {
+ put("[]", new Long[]{});
+ put("[4115420654264075766]", new Long[]{4115420654264075766L});
+ put("[4115420, 20]", new Long[]{4115420L, 20L});
+ put("[\"139\", \"20\"]", new Long[]{139L, 20L});
+ }
+ };
+
+ this.validateInput(this.sut::tryParseValueArray, Long.class, Long.class, inputs, dataMap);
}
@Test
public void should_parseValidLong() {
Map<String, Object> dataMap = new HashMap<>();
- when(this.numberParser.parseLong(any(), any())).thenReturn(0L);
this.sut.tryParseLong("common:welldb:wellbore-OGY4ZWQ5", "reference", "", dataMap);
assertEquals(dataMap.size(), 1);
assertEquals(dataMap.get("reference"), 0L);
@@ -78,22 +111,50 @@ public class AttributeParsingServiceImplTest {
public void should_parseValidFloat() {
Map<String, Object> dataMap = new HashMap<>();
- when(this.numberParser.parseFloat(any(), any())).thenReturn(0f);
this.sut.tryParseFloat("common:welldb:wellbore-MjVhND", "lon", null, dataMap);
assertEquals(dataMap.size(), 1);
assertEquals(dataMap.get("lon"), 0.0f);
}
+ @Test
+ public void should_parseValidFloatArray() {
+ Map<String, Object> dataMap = new HashMap<>();
+ final Map<Object, Object> inputs = new HashMap<Object, Object>() {
+ {
+ put("[]", new Float[]{});
+ put("[101959.1]", new Float[]{101959.1f});
+ put("[139.90, 20.7]", new Float[]{139.90f, 20.7f});
+ put("[\"139.987\", \"20\"]", new Float[]{139.987f, 20.0f});
+ }
+ };
+
+ this.validateInput(this.sut::tryParseValueArray, Float.class, Float.class, inputs, dataMap);
+ }
+
@Test
public void should_parseValidDouble() {
Map<String, Object> dataMap = new HashMap<>();
- when(this.numberParser.parseDouble(any(), any())).thenReturn(20.0);
this.sut.tryParseDouble("common:welldb:wellbore-zMWQtMm", "location", 20.0, dataMap);
assertEquals(dataMap.size(), 1);
assertEquals(dataMap.get("location"), 20.0);
}
+ @Test
+ public void should_parseValidDoubleArray() {
+ Map<String, Object> dataMap = new HashMap<>();
+ final Map<Object, Object> inputs = new HashMap<Object, Object>() {
+ {
+ put("[]", new Double[]{});
+ put("[101959.1]", new Double[]{101959.1});
+ put("[139.1, 20.0]", new Double[]{139.1, 20.0});
+ put("[\"139.9\", \"20.1\"]", new Double[]{139.9, 20.1});
+ }
+ };
+
+ this.validateInput(this.sut::tryParseValueArray, Double.class, Double.class, inputs, dataMap);
+ }
+
@Test
public void should_parseBoolean() {
Map<String, Object> dataMap = new HashMap<>();
@@ -119,6 +180,21 @@ public class AttributeParsingServiceImplTest {
assertEquals(dataMap.get("side"), true);
}
+ @Test
+ public void should_parseValidBooleanArray() {
+ Map<String, Object> dataMap = new HashMap<>();
+ final Map<Object, Object> inputs = new HashMap<Object, Object>() {
+ {
+ put("[]", new Boolean[]{});
+ put("[true]", new Boolean[]{true});
+ put("[false, truee]", new Boolean[]{false, false});
+ put("[\"true\", \"false\"]", new Boolean[]{true, false});
+ }
+ };
+
+ this.validateInput(this.sut::tryParseValueArray, Boolean.class, Boolean.class, inputs, dataMap);
+ }
+
@Test
public void should_parseDate_tryParseDate() {
Map<String, Object> dataMap = new HashMap<>();
@@ -135,13 +211,28 @@ public class AttributeParsingServiceImplTest {
assertEquals(dataMap.size(), 0);
assertFalse(dataMap.containsKey("activatedOn"));
- when(this.dateTimeParser.convertDateObjectToUtc("2018-11-06T19:37:11.128Z")).thenReturn("2018-11-06T19:37:11+0000");
+ when(this.dateTimeParser.parseDate("disabledOn", "2018-11-06T19:37:11.128Z")).thenReturn("2018-11-06T19:37:11+0000");
this.sut.tryParseDate("common:welldb:wellbore-OGY4ZWQ5", "disabledOn", "2018-11-06T19:37:11.128Z", dataMap);
assertEquals(dataMap.size(), 1);
assertTrue(dataMap.containsKey("disabledOn"));
assertEquals(dataMap.get("disabledOn"), "2018-11-06T19:37:11+0000");
}
+ @Test
+ public void should_parseValidDateArray() {
+ Map<String, Object> dataMap = new HashMap<>();
+ final Map<Object, Object> inputs = new HashMap<Object, Object>() {
+ {
+ put("[]", new String[]{});
+ put("[\"2018-11-06T19:37:11.128Z\"]", new String[]{"2018-11-06T19:37:11.128+0000"});
+ put("[20000102, 2000-01-02]", new String[]{"2000-01-02T00:00:00+0000", "2000-01-02T00:00:00+0000"});
+ // TODO: put("[2018-11-06T19:37:11.128Z]", new String[]{"2018-11-06T19:37:11.128+0000"});
+ }
+ };
+
+ this.validateInput(this.sut::tryParseValueArray, Date.class, String.class, inputs, dataMap);
+ }
+
@Test
public void should_notReturnLatLong_given_oneOfTheNullAttribute_tryGetGeopointTest() {
LinkedTreeMap<String, Object> positionTreeMap = new LinkedTreeMap<>();
@@ -258,4 +349,32 @@ public class AttributeParsingServiceImplTest {
Type type = new TypeToken<Map<String, Object>>() {}.getType();
return new Gson().fromJson(json, type);
}
+
+ private <I, O> void validateInput(QuintConsumer<Class<I>, String, String, Object, Map<String, Object>> parser, Class<I> inputType, Class<O> expectedType, Map<Object, Object> inputs, Map<String, Object> outMap) {
+ inputs.forEach((attributeVal, expectedOut) -> {
+ try {
+ parser.accept(inputType, "dummyId", "dummyAttribute", attributeVal, outMap);
+
+ assertEquals(outMap.size(), 1);
+ assertTrue(outMap.containsKey("dummyAttribute"));
+ assertArrayEquals((I[]) outMap.get("dummyAttribute"), (O[]) expectedOut);
+ } catch (IllegalArgumentException e) {
+ fail(String.format("Parsing exception expected for %s with value [ %s ]", inputType.getName(), attributeVal));
+ }
+ });
+ }
+
+ @FunctionalInterface
+ private interface QuintConsumer<T, U, V, W, X> {
+ /**
+ * Applies this function to the given arguments.
+ *
+ * @param t the first function argument
+ * @param u the second function argument
+ * @param v the third function argument
+ * @param w the fourth function argument
+ * * @return the function result
+ */
+ void accept(T t, U u, V v, W w, X x);
+ }
}
\ No newline at end of file
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/JobStatusTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/JobStatusTest.java
index 9e61f1cde0e3b673086cd13a671db65a9bb18d3a..79a0f7f0d504791f7f0c99a18a082d26e50b1b23 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/JobStatusTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/JobStatusTest.java
@@ -33,7 +33,6 @@ import java.lang.reflect.Type;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
-import java.util.stream.Collectors;
import static org.junit.Assert.*;
@@ -196,7 +195,7 @@ public class JobStatusTest {
jobStatus.addOrUpdateRecordStatus("skipped2", IndexingStatus.SKIP, 1, "");
jobStatus.addOrUpdateRecordStatus("skipped3", IndexingStatus.SKIP, 1, "");
jobStatus.addOrUpdateRecordStatus("skipped4", IndexingStatus.SKIP, 1, "");
- jobStatus.getIdsByIndexingStatus(IndexingStatus.SUCCESS);
+ jobStatus.addOrUpdateRecordStatus("warn1", IndexingStatus.WARN, 1, "");
return jobStatus;
}
@@ -217,6 +216,13 @@ public class JobStatusTest {
assertEquals(statuses.size(), 0);
}
+ @Test
+ public void should_get_correctUpdateMessageCount_given_updateRecordStatusTest() {
+ JobStatus jobStatus = insertTestCasesIntoJobStatus();
+
+ assertEquals(6, jobStatus.getIdsByValidUpsertIndexingStatus().size());
+ }
+
@Test
public void should_returnValidList_getRecordStatuses() {
@@ -233,7 +239,10 @@ public class JobStatusTest {
assertNotNull(statuses);
assertEquals(1, statuses.size());
- List<String> toString = statuses.stream().map(RecordStatus::toString).collect(Collectors.toList());
- assertEquals(toString.get(0), "RecordStatus(id=tenant1:doc:test2, kind=tenant1:testindexer12:well:1.0.0, operationType=create, status=SUCCESS)");
+ RecordStatus recordStatus = statuses.get(0);
+ assertEquals("tenant1:doc:test2", recordStatus.getId());
+ assertEquals("tenant1:testindexer12:well:1.0.0", recordStatus.getKind());
+ assertEquals("create", recordStatus.getOperationType());
+ assertEquals(IndexingStatus.SUCCESS, recordStatus.getStatus());
}
}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/parser/BooleanParserTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/parser/BooleanParserTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..b12673e1944a71d36fe586444f061384a36b34a6
--- /dev/null
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/parser/BooleanParserTest.java
@@ -0,0 +1,27 @@
+package org.opengroup.osdu.indexer.util.parser;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(MockitoJUnitRunner.class)
+public class BooleanParserTest {
+
+ @InjectMocks
+ private BooleanParser sut;
+
+ @Test
+ public void should_parseBoolean() {
+ assertTrue(this.sut.parseBoolean("testBooleanAttribute", "true"));
+ assertTrue(this.sut.parseBoolean("testBooleanAttribute", "TRUE"));
+
+ assertFalse(this.sut.parseBoolean("testBooleanAttribute", ""));
+ assertFalse(this.sut.parseBoolean("testBooleanAttribute", null));
+ assertFalse(this.sut.parseBoolean("testBooleanAttribute", "false"));
+ assertFalse(this.sut.parseBoolean("testBooleanAttribute", "truee"));
+ }
+}
\ No newline at end of file
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/parser/DateTimeParserTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/parser/DateTimeParserTest.java
index a4fd49f65eee0a76c22926ae5017284f9b10b7e7..9aa1196043357b199ed24a9b723fd6b398b6aa35 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/parser/DateTimeParserTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/parser/DateTimeParserTest.java
@@ -89,4 +89,15 @@ public class DateTimeParserTest {
assertNull(this.sut.convertDateObjectToUtc(".2190851121908511EE44"));
assertNull(this.sut.convertDateObjectToUtc("E.2131"));
}
+
+ @Test(expected = IllegalArgumentException.class)
+ public void should_throwException_given_invalidDate_parseDateTest() {
+ this.sut.parseDate("testDateTimeAttribute", "N/A");
+ }
+
+ @Test
+ public void should_returnNull_given_emptyOrNull_parseDateTest() {
+ assertNull(this.sut.parseDate("testDateTimeAttribute", ""));
+ assertNull(this.sut.parseDate("testDateTimeAttribute", null));
+ }
}
diff --git a/pom.xml b/pom.xml
index 0ce2c8068920277504c25f3b6557316db50eeec2..15379935ce07282cb0f38e82c8463072332df8ae 100644
--- a/pom.xml
+++ b/pom.xml
@@ -21,6 +21,7 @@
<java.version>1.8</java.version>
<springfox-version>2.7.0</springfox-version>
<spring-cloud.version>Greenwich.SR2</spring-cloud.version>
+ <os-core-common.version>0.0.18</os-core-common.version>
<!-- <maven.compiler.target>1.8</maven.compiler.target>-->
<!-- <maven.compiler.source>1.8</maven.compiler.source>-->
<!-- <maven.war.plugin>2.6</maven.war.plugin>-->
@@ -57,6 +58,16 @@
</snapshotRepository>
</distributionManagement>
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.opengroup.osdu</groupId>
+ <artifactId>os-core-common</artifactId>
+ <version>${os-core-common.version}</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
<modules>
<module>indexer-core</module>
<module>provider/indexer-aws</module>
diff --git a/provider/indexer-azure/pom.xml b/provider/indexer-azure/pom.xml
index 3ad7260a2e4f24e716f197532388004322c1ea53..137dc152137760e152bb1db9938cd044bc7c5995 100644
--- a/provider/indexer-azure/pom.xml
+++ b/provider/indexer-azure/pom.xml
@@ -88,7 +88,6 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
- <version>0.0.13</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
diff --git a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexerSchemaServiceTest.java b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexerSchemaServiceTest.java
index 9961c312f01e4ab7695546afc5f881fd3e15624b..bc640485be50de6f1727e4f9bc16306ec01ea1b0 100644
--- a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexerSchemaServiceTest.java
+++ b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexerSchemaServiceTest.java
@@ -87,7 +87,7 @@ public class IndexerSchemaServiceTest {
public void should_returnNull_givenEmptySchema_getIndexerInputSchemaSchemaTest() throws Exception {
when(storageService.getStorageSchema(any())).thenReturn(emptySchema);
- IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind);
+ IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false);
Assert.assertNotNull(indexSchema);
}
@@ -96,7 +96,7 @@ public class IndexerSchemaServiceTest {
public void should_returnValidResponse_givenValidSchema_getIndexerInputSchemaTest() throws Exception {
when(storageService.getStorageSchema(any())).thenReturn(someSchema);
- IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind);
+ IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false);
Assert.assertEquals(kind, indexSchema.getKind());
}
@@ -106,7 +106,7 @@ public class IndexerSchemaServiceTest {
when(storageService.getStorageSchema(any())).thenReturn(someSchema);
when(this.schemaCache.get(kind + "_flattened")).thenReturn(someSchema);
- IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind);
+ IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false);
Assert.assertEquals(kind, indexSchema.getKind());
}
@@ -117,7 +117,7 @@ public class IndexerSchemaServiceTest {
String invalidSchema = "{}}";
when(storageService.getStorageSchema(any())).thenReturn(invalidSchema);
- this.sut.getIndexerInputSchema(kind);
+ this.sut.getIndexerInputSchema(kind, false);
fail("Should throw exception");
} catch (AppException e) {
Assert.assertEquals(HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getError().getCode());
@@ -129,7 +129,7 @@ public class IndexerSchemaServiceTest {
@Test
public void should_return_basic_schema_when_storage_returns_no_schema() {
- IndexSchema returnedSchema = this.sut.getIndexerInputSchema(kind);
+ IndexSchema returnedSchema = this.sut.getIndexerInputSchema(kind, false);
assertNotNull(returnedSchema.getDataSchema());
assertNotNull(returnedSchema);
@@ -326,4 +326,82 @@ public class IndexerSchemaServiceTest {
verify(this.log).warning(eq(String.format("Kind: %s not found", kind)));
}
+
+ @Test
+ public void should_sync_schema_with_storage() throws Exception {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.indicesService.deleteIndex(any(), any())).thenReturn(true);
+ when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
+
+ this.sut.syncIndexMappingWithStorageSchema(kind);
+
+ verify(this.mappingService, times(1)).getIndexMappingFromRecordSchema(any());
+ verify(this.indicesService, times(1)).isIndexExist(any(), any());
+ verify(this.indicesService, times(1)).deleteIndex(any(), any());
+ verify(this.indicesService, times(1)).createIndex(any(), any(), any(), any(), any());
+ verifyNoMoreInteractions(this.mappingService);
+ }
+
+ @Test
+ public void should_throw_exception_while_snapshot_running_sync_schema_with_storage() throws Exception {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.indicesService.deleteIndex(any(), any())).thenThrow(new AppException(HttpStatus.SC_CONFLICT, "Index deletion error", "blah"));
+
+ try {
+ this.sut.syncIndexMappingWithStorageSchema(kind);
+ } catch (AppException e) {
+ assertEquals(e.getError().getCode(), HttpStatus.SC_CONFLICT);
+ assertEquals(e.getError().getMessage(), "blah");
+ assertEquals(e.getError().getReason(), "Index deletion error");
+ } catch (Exception e) {
+ fail("Should not throw this exception " + e.getMessage());
+ }
+
+ verify(this.indicesService, times(1)).isIndexExist(any(), any());
+ verify(this.indicesService, times(1)).deleteIndex(any(), any());
+ verify(this.mappingService, never()).getIndexMappingFromRecordSchema(any());
+ verify(this.indicesService, never()).createIndex(any(), any(), any(), any(), any());
+ }
+
+ @Test
+ public void should_return_true_while_if_forceClean_requested() throws IOException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+
+ assertTrue(this.sut.isStorageSchemaSyncRequired(kind, true));
+ }
+
+ @Test
+ public void should_return_true_while_if_forceClean_notRequested_and_indexNotFound() throws IOException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(false);
+
+ assertTrue(this.sut.isStorageSchemaSyncRequired(kind, false));
+ }
+
+ @Test
+ public void should_return_false_while_if_forceClean_notRequested_and_indexExist() throws IOException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+
+ assertFalse(this.sut.isStorageSchemaSyncRequired(kind, false));
+ }
}
diff --git a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/ReindexServiceTest.java b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/ReindexServiceTest.java
index 711a3e19ed996ffccb30631702f618f53da4210e..5120f069f072945e85a569cb3d5b49a74251fab8 100644
--- a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/ReindexServiceTest.java
+++ b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/ReindexServiceTest.java
@@ -27,10 +27,14 @@ import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.search.Config;
import org.opengroup.osdu.indexer.service.ReindexServiceImpl;
import org.opengroup.osdu.indexer.service.StorageService;
import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+import org.powermock.modules.junit4.PowerMockRunnerDelegate;
import org.springframework.test.context.junit4.SpringRunner;
import java.util.*;
@@ -41,7 +45,9 @@ import static org.powermock.api.mockito.PowerMockito.mockStatic;
import static org.powermock.api.mockito.PowerMockito.when;
@Ignore
-@RunWith(SpringRunner.class)
+@RunWith(PowerMockRunner.class)
+@PowerMockRunnerDelegate(SpringRunner.class)
+@PrepareForTest({Config.class})
public class ReindexServiceTest {
private final String cursor = "100";
@@ -88,7 +94,7 @@ public class ReindexServiceTest {
recordQueryResponse.setResults(null);
when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
- String response = sut.reindexRecords(recordReindexRequest);
+ String response = sut.reindexRecords(recordReindexRequest, false);
Assert.assertNull(response);
} catch (Exception e) {
@@ -102,7 +108,7 @@ public class ReindexServiceTest {
recordQueryResponse.setResults(new ArrayList<>());
when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
- String response = sut.reindexRecords(recordReindexRequest);
+ String response = sut.reindexRecords(recordReindexRequest, false);
Assert.assertNull(response);
} catch (Exception e) {
@@ -117,9 +123,13 @@ public class ReindexServiceTest {
List<String> results = new ArrayList<>();
results.add("test1");
recordQueryResponse.setResults(results);
+
+ mockStatic(Config.class);
+ when(Config.getStorageRecordsBatchSize()).thenReturn(1);
+
when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
- String taskQueuePayload = sut.reindexRecords(recordReindexRequest);
+ String taskQueuePayload = sut.reindexRecords(recordReindexRequest, false);
Assert.assertEquals("{\"kind\":\"tenant:test:test:1.0.0\",\"cursor\":\"100\"}", taskQueuePayload);
} catch (Exception e) {
@@ -135,7 +145,7 @@ public class ReindexServiceTest {
recordQueryResponse.setResults(results);
when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
- String taskQueuePayload = sut.reindexRecords(recordReindexRequest);
+ String taskQueuePayload = sut.reindexRecords(recordReindexRequest, false);
Assert.assertEquals(String.format("{\"data\":\"[{\\\"id\\\":\\\"test1\\\",\\\"kind\\\":\\\"tenant:test:test:1.0.0\\\",\\\"op\\\":\\\"create\\\"}]\",\"attributes\":{\"slb-correlation-id\":\"%s\"}}", correlationId), taskQueuePayload);
} catch (Exception e) {
diff --git a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/StorageServiceTest.java b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/StorageServiceTest.java
index aedf8960acca1ecf4d3807234d6c048dcaa8ff55..d32955169190c779486a55fe6b549b7ef7b20c09 100644
--- a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/StorageServiceTest.java
+++ b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/StorageServiceTest.java
@@ -24,6 +24,7 @@ import org.mockito.ArgumentMatchers;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
+import org.opengroup.osdu.core.common.model.indexer.IndexingStatus;
import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
@@ -44,9 +45,11 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
+import static java.util.Collections.singletonList;
import static org.junit.Assert.*;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
import static org.powermock.api.mockito.PowerMockito.when;
@Ignore
@@ -65,12 +68,14 @@ public class StorageServiceTest {
private StorageServiceImpl sut;
private List<String> ids;
+ private static final String RECORD_ID1 = "tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465";
+ private static final String RECORDS_ID2 = "tenant1:doc:15e790a69beb4d789b1f979e2af2e813";
@Before
public void setup() {
String recordChangedMessages = "[{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"purge\"}," +
- "{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"create\"}]";
+ "{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"create\"}]";
when(this.requestInfo.getHeadersMap()).thenReturn(new HashMap<>());
@@ -78,7 +83,7 @@ public class StorageServiceTest {
List<RecordInfo> msgs = (new Gson()).fromJson(recordChangedMessages, listType);
jobStatus.initialize(msgs);
- ids = Arrays.asList("tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465", "tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465");
+ ids = Arrays.asList(RECORD_ID1, RECORDS_ID2);
}
@Test
@@ -87,7 +92,7 @@ public class StorageServiceTest {
HttpResponse httpResponse = mock(HttpResponse.class);
Mockito.when(httpResponse.getBody()).thenReturn(null);
- when(this.urlFetchService.sendRequest(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(httpResponse);
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
should_return404_getValidStorageRecordsTest();
}
@@ -100,7 +105,7 @@ public class StorageServiceTest {
HttpResponse httpResponse = mock(HttpResponse.class);
Mockito.when(httpResponse.getBody()).thenReturn(emptyDataFromStorage);
- when(this.urlFetchService.sendRequest(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(httpResponse);
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
should_return404_getValidStorageRecordsTest();
}
@@ -113,12 +118,41 @@ public class StorageServiceTest {
HttpResponse httpResponse = mock(HttpResponse.class);
Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
- when(this.urlFetchService.sendRequest(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(httpResponse);
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
Records storageRecords = this.sut.getStorageRecords(ids);
assertEquals(1, storageRecords.getRecords().size());
}
+ @Test
+ public void should_logMissingRecord_given_storageMissedRecords() throws URISyntaxException {
+
+ String validDataFromStorage = "{\"records\":[{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[]}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(any())).thenReturn(httpResponse);
+ Records storageRecords = this.sut.getStorageRecords(ids);
+
+ assertEquals(1, storageRecords.getRecords().size());
+ verify(this.jobStatus).addOrUpdateRecordStatus(singletonList(RECORDS_ID2), IndexingStatus.FAIL, org.apache.http.HttpStatus.SC_NOT_FOUND, "Partial response received from Storage service - missing records", "Partial response received from Storage service: tenant1:doc:15e790a69beb4d789b1f979e2af2e813");
+ }
+
+ @Test
+ public void should_returnValidJobStatus_givenFailedUnitsConversion_processRecordChangedMessageTest() throws URISyntaxException {
+ String validDataFromStorage = "{\"records\":[{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[],\"conversionStatuses\":[{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\",\"status\":\"ERROR\",\"errors\":[\"crs conversion failed\"]}]}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(any())).thenReturn(httpResponse);
+ Records storageRecords = this.sut.getStorageRecords(singletonList("tenant1:doc:15e790a69beb4d789b1f979e2af2e813"));
+
+ assertEquals(1, storageRecords.getRecords().size());
+ verify(this.jobStatus).addOrUpdateRecordStatus("tenant1:doc:15e790a69beb4d789b1f979e2af2e813", IndexingStatus.WARN, org.apache.http.HttpStatus.SC_BAD_REQUEST, "crs conversion failed", String.format("record-id: %s | %s", "tenant1:doc:15e790a69beb4d789b1f979e2af2e813", "crs conversion failed"));
+ }
+
@Test
public void should_returnValidResponse_givenValidRecordQueryRequest_getRecordListByKind() throws Exception {
@@ -127,7 +161,7 @@ public class StorageServiceTest {
HttpResponse httpResponse = new HttpResponse();
httpResponse.setBody(new Gson().toJson(recordReindexRequest, RecordReindexRequest.class));
- when(this.urlFetchService.sendRequest(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(httpResponse);
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
RecordQueryResponse recordQueryResponse = this.sut.getRecordsByKind(recordReindexRequest);
@@ -158,7 +192,7 @@ public class StorageServiceTest {
httpResponse.setResponseCode(HttpStatus.OK.value());
httpResponse.setBody(validSchemaFromStorage);
- when(this.urlFetchService.sendRequest(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(httpResponse);
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
String recordSchemaResponse = this.sut.getStorageSchema(kind);
@@ -173,7 +207,7 @@ public class StorageServiceTest {
HttpResponse httpResponse = new HttpResponse();
httpResponse.setResponseCode(HttpStatus.NOT_FOUND.value());
- when(this.urlFetchService.sendRequest(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(httpResponse);
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
String recordSchemaResponse = this.sut.getStorageSchema(kind);
@@ -183,19 +217,19 @@ public class StorageServiceTest {
@Test
public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsWithInvalidConversionTest() throws URISyntaxException {
- String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"],\"conversionStatuses\": [{\"id\":\"testid\",\"status\":\"ERROR\",\"errors\":[\"conversion error occured\"] } ]}";
+ String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"],\"conversionStatuses\": [{\"id\":\"testid\",\"status\":\"ERROR\",\"errors\":[\"conversion error occurred\"] } ]}";
HttpResponse httpResponse = mock(HttpResponse.class);
Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
- when(this.urlFetchService.sendRequest(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(httpResponse);
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
Records storageRecords = this.sut.getStorageRecords(ids);
assertEquals(1, storageRecords.getRecords().size());
assertEquals(1, storageRecords.getConversionStatuses().get(0).getErrors().size());
- assertEquals("conversion error occured", storageRecords.getConversionStatuses().get(0).getErrors().get(0));
+ assertEquals("conversion error occurred", storageRecords.getConversionStatuses().get(0).getErrors().get(0));
}
private void should_return404_getValidStorageRecordsTest() {
@@ -203,7 +237,7 @@ public class StorageServiceTest {
this.sut.getStorageRecords(ids);
fail("Should throw exception");
} catch (AppException e) {
- assertEquals(HttpStatus.NOT_FOUND, e.getError().getCode());
+ assertEquals(HttpStatus.NOT_FOUND.value(), e.getError().getCode());
} catch (Exception e) {
fail("Should not throw this exception" + e.getMessage());
}
diff --git a/provider/indexer-gcp/pom.xml b/provider/indexer-gcp/pom.xml
index 795053370729a4d60b19488f45f28c4b24116f82..cadaf5037d5ff34b8aa2c1118e9684e456f88eee 100644
--- a/provider/indexer-gcp/pom.xml
+++ b/provider/indexer-gcp/pom.xml
@@ -49,7 +49,6 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
- <version>0.0.13</version>
</dependency>
<dependency>
@@ -137,7 +136,7 @@
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
- <version>3.0.0</version>
+ <version>2.26.0</version>
<scope>test</scope>
</dependency>
<dependency>
diff --git a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..2b637c62629c28c5edfdce57f8821303fb6eb800
--- /dev/null
+++ b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java
@@ -0,0 +1,53 @@
+package org.opengroup.osdu.indexer.middleware;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.MockitoJUnitRunner;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+
+import javax.servlet.FilterChain;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import java.io.IOException;
+import java.util.Collections;
+
+@RunWith(MockitoJUnitRunner.class)
+public class IndexFilterTest {
+
+ @InjectMocks
+ private IndexFilter indexFilter;
+
+ @Mock
+ private DpsHeaders dpsHeaders;
+
+ @Test
+ public void shouldSetCorrectResponseHeaders() throws IOException, ServletException {
+ HttpServletRequest httpServletRequest = Mockito.mock(HttpServletRequest.class);
+ HttpServletResponse httpServletResponse = Mockito.mock(HttpServletResponse.class);
+ FilterChain filterChain = Mockito.mock(FilterChain.class);
+ Mockito.when(httpServletRequest.getRequestURI()).thenReturn("https://test.com");
+ Mockito.when(httpServletRequest.getMethod()).thenReturn("POST");
+ Mockito.when(dpsHeaders.getCorrelationId()).thenReturn("correlation-id-value");
+
+ indexFilter.doFilter(httpServletRequest, httpServletResponse, filterChain);
+
+ Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Origin", Collections.singletonList("*").toString());
+ Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Headers", Collections.singletonList("origin, content-type, accept, authorization, data-partition-id, correlation-id, appkey").toString());
+ Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Methods", Collections.singletonList("GET, POST, PUT, DELETE, OPTIONS, HEAD").toString());
+ Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Credentials", Collections.singletonList("true").toString());
+ Mockito.verify(httpServletResponse).addHeader("X-Frame-Options", Collections.singletonList("DENY").toString());
+ Mockito.verify(httpServletResponse).addHeader("X-XSS-Protection", Collections.singletonList("1; mode=block").toString());
+ Mockito.verify(httpServletResponse).addHeader("X-Content-Type-Options", Collections.singletonList("nosniff").toString());
+ Mockito.verify(httpServletResponse).addHeader("Cache-Control", Collections.singletonList("no-cache, no-store, must-revalidate").toString());
+ Mockito.verify(httpServletResponse).addHeader("Content-Security-Policy", Collections.singletonList("default-src 'self'").toString());
+ Mockito.verify(httpServletResponse).addHeader("Strict-Transport-Security", Collections.singletonList("max-age=31536000; includeSubDomains").toString());
+ Mockito.verify(httpServletResponse).addHeader("Expires", Collections.singletonList("0").toString());
+ Mockito.verify(httpServletResponse).addHeader("correlation-id", "correlation-id-value");
+ Mockito.verify(filterChain).doFilter(httpServletRequest, httpServletResponse);
+ }
+}
diff --git a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java
index 43f24cc6cdbe134714cdb2fd6ca4ce42c1ec8446..d28e1c2746ee215e10c1c460c799f7dda39e2fa8 100644
--- a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java
+++ b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java
@@ -1,314 +1,338 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.service;
-
-import org.apache.http.StatusLine;
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
-import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData;
-import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
-import org.elasticsearch.action.support.master.AcknowledgedResponse;
-import org.elasticsearch.client.*;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.index.reindex.BulkByScrollResponse;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
-import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
-import org.opengroup.osdu.indexer.util.ElasticClientHandler;
-import org.powermock.api.mockito.PowerMockito;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.springframework.test.context.junit4.SpringRunner;
-
-import java.io.IOException;
-import java.util.*;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.powermock.api.mockito.PowerMockito.when;
-
-@Ignore
-@RunWith(SpringRunner.class)
-@PrepareForTest({ RestHighLevelClient.class, IndicesClient.class })
-public class IndexerMappingServiceTest {
-
- private final String kind = "tenant:test:test:1.0.0";
- private final String index = "tenant-test-test-1.0.0";
- private final String type = "test";
- private final String mappingValid = "{\"dynamic\":false,\"properties\":{\"data\":{\"properties\":{\"Location\":{\"type\":\"geo_point\"}}},\"id\":{\"type\":\"keyword\"}}}";
-
- @Mock
- private RestClient restClient;
- @Mock
- private Response response;
- @Mock
- private StatusLine statusLine;
-
- @InjectMocks
- private IndexerMappingServiceImpl sut;
-
- @Mock
- private ElasticClientHandler elasticClientHandler;
-
- @InjectMocks
- private RestHighLevelClient restHighLevelClient;
-
- @InjectMocks
- private IndexSchema indexSchema;
- @InjectMocks
- private IndicesClient indicesClient;
-
- @InjectMocks
- private AcknowledgedResponse mappingResponse;
-
- @Before
- public void setup() throws IOException {
- Map<String, String> dataMapping = new HashMap<>();
- dataMapping.put("Location", "geo_point");
- Map<String, Object> metaMapping = new HashMap<>();
- metaMapping.put(RecordMetaAttribute.ID.getValue(), "keyword");
- this.indexSchema = IndexSchema.builder().kind(kind).type(type).dataSchema(dataMapping).metaSchema(metaMapping)
- .build();
-
- this.indicesClient = PowerMockito.mock(IndicesClient.class);
- this.restHighLevelClient = PowerMockito.mock(RestHighLevelClient.class);
-
- when(this.restHighLevelClient.getLowLevelClient()).thenReturn(restClient);
- when(this.restClient.performRequest(any())).thenReturn(response);
- when(this.response.getStatusLine()).thenReturn(statusLine);
- when(this.statusLine.getStatusCode()).thenReturn(200);
- }
-
- @Test
- public void should_returnValidMapping_givenFalseMerge_createMappingTest() {
- try {
- String mapping = this.sut.createMapping(restHighLevelClient, indexSchema, index, false);
- assertEquals(mappingValid, mapping);
- } catch (Exception e) {
- fail("Should not throw this exception" + e.getMessage());
- }
- }
-
- @Test
- public void should_returnValidMapping_givenTrueMerge_createMappingTest() {
- try {
- doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
-
- String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true);
- assertEquals(this.mappingValid, mapping);
- } catch (Exception e) {
- fail("Should not throw this exception" + e.getMessage());
- }
- }
-
- @Test
- public void should_returnValidMapping_givenExistType_createMappingTest() {
- try {
- doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
-
- IndexerMappingServiceImpl indexerMappingServiceLocal = PowerMockito.spy(new IndexerMappingServiceImpl());
- doReturn(false).when(indexerMappingServiceLocal).isTypeExist(any(), any(), any());
- String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true);
- assertEquals(this.mappingValid, mapping);
- } catch (Exception e) {
- fail("Should not throw this exception" + e.getMessage());
- }
- }
-
- @Test
- public void should_update_indices_field_with_keyword_when_valid_indices() throws Exception {
- try {
- Set<String> indices = new HashSet<String>();
- indices.add("indices 1");
- GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
- doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- when(this.indicesClient.getFieldMapping(any(), any())).thenReturn(getFieldMappingsResponse);
- XContentBuilder builder = XContentFactory.jsonBuilder();
- builder.startObject();
- builder.field("any field", new HashMap());
- builder.endObject();
- BytesReference bytesReference = BytesReference.bytes(builder);
- FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
- Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
- mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
- mappingBuilder.put("any index 1", mapBuilder);
- mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
- mapping.put("indices 1", mappingBuilder);
- when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
- doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
- BulkByScrollResponse response = mock(BulkByScrollResponse.class);
- doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class));
- when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
- when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
-
- this.sut.updateIndexMappingForIndicesOfSameType( indices,"any field");
- } catch (Exception e) {
- fail("Should not throw this exception" + e.getMessage());
- }
- }
-
- @Test(expected = AppException.class)
- public void should_throw_exception_if_someIndex_is_invalid_andWeIndexfield_with_keyword() throws Exception {
- try {
- Set<String> indices = new HashSet<String>();
- indices.add("invalid 1");
- GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
- doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- when(this.indicesClient.getFieldMapping(any(), any())).thenReturn(getFieldMappingsResponse);
- XContentBuilder builder = XContentFactory.jsonBuilder();
- builder.startObject();
- builder.field("any field", new HashMap());
- builder.endObject();
- BytesReference bytesReference = BytesReference.bytes(builder);
- FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
- Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
- mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
- mappingBuilder.put("any index 1", mapBuilder);
- mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
- mapping.put("indices 1", mappingBuilder);
- when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
- doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
- BulkByScrollResponse response = mock(BulkByScrollResponse.class);
- doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class));
- when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
- when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
-
- this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
- } catch (Exception e) {
- throw e;
- }
- }
-
- @Test(expected = AppException.class)
- public void should_throw_exception_if_type_of_index_is_invalid_andWeIndexfield_with_keyword() throws Exception {
- try {
- Set<String> indices = new HashSet<String>();
- indices.add("indices 1");
- GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
- doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- when(this.indicesClient.getFieldMapping(any(), any())).thenReturn(getFieldMappingsResponse);
- XContentBuilder builder = XContentFactory.jsonBuilder();
- builder.startObject();
- builder.field("any field", new HashMap());
- builder.endObject();
- BytesReference bytesReference = BytesReference.bytes(builder);
- FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
- Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
- mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
- mappingBuilder.put("any index 1", mapBuilder);
- mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
- mapping.put("indices 1", mappingBuilder);
- when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
- doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
- BulkByScrollResponse response = mock(BulkByScrollResponse.class);
- doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class));
- when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
- when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
- this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field invalid");
- } catch (Exception e) {
- throw e;
- }
- }
-
- @Test(expected = AppException.class)
- public void should_throw_exception_if_elastic_search_failedToFetch_andWeIndexfield_with_keyword() throws Exception {
- try {
-
- Set<String> indices = new HashSet<String>();
- indices.add("indices 1");
- indices.add("indices Invalid");
- GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
- doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- when(this.indicesClient.getFieldMapping(any(), any())).thenThrow(new ElasticsearchException(""));
- XContentBuilder builder = XContentFactory.jsonBuilder();
- builder.startObject();
- builder.field("any field", new HashMap());
- builder.endObject();
- BytesReference bytesReference = BytesReference.bytes(builder);
- FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
- Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
- mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
- mappingBuilder.put("any index 1", mapBuilder);
- mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
- mapping.put("indices 1", mappingBuilder);
- when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
- doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
- BulkByScrollResponse response = mock(BulkByScrollResponse.class);
- doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class));
- when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
- when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
- this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
- } catch (AppException e) {
- throw e;
- }
- }
-
- @Test(expected = AppException.class)
- public void should_throw_exception_when_elastic_failedToIndex_indices_field_with_keyword() {
- try {
- Set<String> indices = new HashSet<String>();
- indices.add("indices 1");
- indices.add("indices Invalid");
- GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
- doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- when(this.indicesClient.getFieldMapping(any(), any())).thenReturn(getFieldMappingsResponse);
- XContentBuilder builder = XContentFactory.jsonBuilder();
- builder.startObject();
- builder.field("any field", new HashMap());
- builder.endObject();
- BytesReference bytesReference = BytesReference.bytes(builder);
- FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
- Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
- mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
- mappingBuilder.put("any index 1", mapBuilder);
- mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
- mapping.put("indices 1", mappingBuilder);
- when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
- doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
- BulkByScrollResponse response = mock(BulkByScrollResponse.class);
- doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class));
- when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
- when(this.indicesClient.putMapping(any(), any(RequestOptions.class))).thenThrow(new ElasticsearchException(""));
- when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
- this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
- } catch (AppException e) {
- throw e;
- } catch (Exception e) {
- fail("Should not throw this exception" + e.getMessage());
- }
- }
-}
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.service;
+
+import com.google.gson.Gson;
+import org.apache.http.StatusLine;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
+import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData;
+import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.*;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.reindex.BulkByScrollResponse;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
+import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
+import org.opengroup.osdu.core.common.search.Config;
+import org.opengroup.osdu.indexer.util.ElasticClientHandler;
+import org.opengroup.osdu.indexer.util.TypeMapper;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.io.IOException;
+import java.util.*;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.MockitoAnnotations.initMocks;
+import static org.powermock.api.mockito.PowerMockito.mockStatic;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@Ignore
+@RunWith(SpringRunner.class)
+@PrepareForTest({ RestHighLevelClient.class, IndicesClient.class, Config.class })
+public class IndexerMappingServiceTest {
+
+ private final String kind = "tenant:test:test:1.0.0";
+ private final String index = "tenant-test-test-1.0.0";
+ private final String type = "test";
+ private final String mappingValid = "{\"dynamic\":false,\"properties\":{\"data\":{\"properties\":{\"Msg\":{\"type\":\"text\",\"analyzer\":\"de_indexer_analyzer\",\"search_analyzer\":\"de_search_analyzer\"},\"Location\":{\"type\":\"geo_point\"}}},\"id\":{\"type\":\"keyword\"},\"acl\":{\"properties\":{\"viewers\":{\"type\":\"keyword\"},\"owners\":{\"type\":\"keyword\"}}}}}";
+
+ @Mock
+ private RestClient restClient;
+ @Mock
+ private Response response;
+ @Mock
+ private StatusLine statusLine;
+
+ @InjectMocks
+ private IndexerMappingServiceImpl sut;
+
+ @Mock
+ private ElasticClientHandler elasticClientHandler;
+
+ @InjectMocks
+ private RestHighLevelClient restHighLevelClient;
+
+ @InjectMocks
+ private IndexSchema indexSchema;
+ @InjectMocks
+ private IndicesClient indicesClient;
+
+ @InjectMocks
+ private AcknowledgedResponse mappingResponse;
+
+ @Before
+ public void setup() throws IOException {
+ initMocks(this);
+ mockStatic(Config.class);
+ when(Config.isPreDemo()).thenReturn(true);
+ Map<String, String> dataMapping = new HashMap<>();
+ dataMapping.put("Location", "geo_point");
+ dataMapping.put("Msg", "text");
+ Map<String, Object> metaMapping = new HashMap<>();
+ metaMapping.put(RecordMetaAttribute.ID.getValue(), "keyword");
+ metaMapping.put(RecordMetaAttribute.ACL.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.ACL));
+ this.indexSchema = IndexSchema.builder().kind(kind).type(type).dataSchema(dataMapping).metaSchema(metaMapping)
+ .build();
+
+ this.indicesClient = PowerMockito.mock(IndicesClient.class);
+ this.restHighLevelClient = PowerMockito.mock(RestHighLevelClient.class);
+
+ when(this.restHighLevelClient.getLowLevelClient()).thenReturn(restClient);
+ when(this.restClient.performRequest(any())).thenReturn(response);
+ when(this.response.getStatusLine()).thenReturn(statusLine);
+ when(this.statusLine.getStatusCode()).thenReturn(200);
+ }
+
+ @Test
+ public void should_returnValidMapping_givenFalseMerge_createMappingTest() {
+ try {
+ String mapping = this.sut.createMapping(restHighLevelClient, indexSchema, index, false);
+ assertEquals(mappingValid, mapping);
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_returnValidMapping_givenTrueMerge_createMappingTest() {
+ try {
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
+
+ String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true);
+ assertEquals(this.mappingValid, mapping);
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_returnValidMapping_givenExistType_createMappingTest() {
+ try {
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
+
+ IndexerMappingServiceImpl indexerMappingServiceLocal = PowerMockito.spy(new IndexerMappingServiceImpl());
+ doReturn(false).when(indexerMappingServiceLocal).isTypeExist(any(), any(), any());
+ String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true);
+ assertEquals(this.mappingValid, mapping);
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_update_indices_field_with_keyword_when_valid_indices() throws Exception {
+ try {
+ Set<String> indices = new HashSet<String>();
+ indices.add("indices 1");
+ GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ when(this.indicesClient.getFieldMapping(any(), any())).thenReturn(getFieldMappingsResponse);
+ XContentBuilder builder = XContentFactory.jsonBuilder();
+ builder.startObject();
+ builder.field("any field", new HashMap());
+ builder.endObject();
+ BytesReference bytesReference = BytesReference.bytes(builder);
+ FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
+ Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ mapBuilder.put("data.any field", mappingMetaData);
+ Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ mappingBuilder.put("any index 1", mapBuilder);
+ mappingBuilder.put("any index 2", mapBuilder);
+ Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ mapping.put("indices 1", mappingBuilder);
+ when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
+ BulkByScrollResponse response = mock(BulkByScrollResponse.class);
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class));
+ when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+
+ this.sut.updateIndexMappingForIndicesOfSameType( indices,"any field");
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throw_exception_if_someIndex_is_invalid_andWeIndexfield_with_keyword() throws Exception {
+ try {
+ Set<String> indices = new HashSet<String>();
+ indices.add("invalid 1");
+ GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ when(this.indicesClient.getFieldMapping(any(), any())).thenReturn(getFieldMappingsResponse);
+ XContentBuilder builder = XContentFactory.jsonBuilder();
+ builder.startObject();
+ builder.field("any field", new HashMap());
+ builder.endObject();
+ BytesReference bytesReference = BytesReference.bytes(builder);
+ FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
+ Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ mapBuilder.put("data.any field", mappingMetaData);
+ Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ mappingBuilder.put("any index 1", mapBuilder);
+ mappingBuilder.put("any index 2", mapBuilder);
+ Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ mapping.put("indices 1", mappingBuilder);
+ when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
+ BulkByScrollResponse response = mock(BulkByScrollResponse.class);
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class));
+ when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+
+ this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
+ } catch (Exception e) {
+ throw e;
+ }
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throw_exception_if_type_of_index_is_invalid_andWeIndexfield_with_keyword() throws Exception {
+ try {
+ Set<String> indices = new HashSet<String>();
+ indices.add("indices 1");
+ GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ when(this.indicesClient.getFieldMapping(any(), any())).thenReturn(getFieldMappingsResponse);
+ XContentBuilder builder = XContentFactory.jsonBuilder();
+ builder.startObject();
+ builder.field("any field", new HashMap());
+ builder.endObject();
+ BytesReference bytesReference = BytesReference.bytes(builder);
+ FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
+ Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ mapBuilder.put("data.any field", mappingMetaData);
+ Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ mappingBuilder.put("any index 1", mapBuilder);
+ mappingBuilder.put("any index 2", mapBuilder);
+ Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ mapping.put("indices 1", mappingBuilder);
+ when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
+ BulkByScrollResponse response = mock(BulkByScrollResponse.class);
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class));
+ when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field invalid");
+ } catch (Exception e) {
+ throw e;
+ }
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throw_exception_if_elastic_search_failedToFetch_andWeIndexfield_with_keyword() throws Exception {
+ try {
+
+ Set<String> indices = new HashSet<String>();
+ indices.add("indices 1");
+ indices.add("indices Invalid");
+ GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ when(this.indicesClient.getFieldMapping(any(), any())).thenThrow(new ElasticsearchException(""));
+ XContentBuilder builder = XContentFactory.jsonBuilder();
+ builder.startObject();
+ builder.field("any field", new HashMap());
+ builder.endObject();
+ BytesReference bytesReference = BytesReference.bytes(builder);
+ FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
+ Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ mapBuilder.put("data.any field", mappingMetaData);
+ Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ mappingBuilder.put("any index 1", mapBuilder);
+ mappingBuilder.put("any index 2", mapBuilder);
+ Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ mapping.put("indices 1", mappingBuilder);
+ when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
+ BulkByScrollResponse response = mock(BulkByScrollResponse.class);
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class));
+ when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
+ } catch (AppException e) {
+ throw e;
+ }
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throw_exception_when_elastic_failedToIndex_indices_field_with_keyword() {
+ try {
+ Set<String> indices = new HashSet<String>();
+ indices.add("indices 1");
+ indices.add("indices Invalid");
+ GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ when(this.indicesClient.getFieldMapping(any(), any())).thenReturn(getFieldMappingsResponse);
+ XContentBuilder builder = XContentFactory.jsonBuilder();
+ builder.startObject();
+ builder.field("any field", new HashMap());
+ builder.endObject();
+ BytesReference bytesReference = BytesReference.bytes(builder);
+ FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
+ Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ mapBuilder.put("data.any field", mappingMetaData);
+ Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ mappingBuilder.put("any index 1", mapBuilder);
+ mappingBuilder.put("any index 2", mapBuilder);
+ Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ mapping.put("indices 1", mappingBuilder);
+ when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
+ BulkByScrollResponse response = mock(BulkByScrollResponse.class);
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class));
+ when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
+ when(this.indicesClient.putMapping(any(), any(RequestOptions.class))).thenThrow(new ElasticsearchException(""));
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
+ } catch (AppException e) {
+ throw e;
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+
+ @Test
+ public void should_returnDocumentMapping_givenValidIndexSchema() {
+
+ try {
+ Map<String, Object> documentMapping = this.sut.getIndexMappingFromRecordSchema(this.indexSchema);
+ String documentMappingJson = new Gson().toJson(documentMapping);
+ assertEquals(this.mappingValid, documentMappingJson);
+
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+}
diff --git a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java
index a1b47530ee22f7f3a5ed5f51cd4fbc3cc8c8006a..434e9bc803383e3034c65b4d8ea66fccd60fb25c 100644
--- a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java
+++ b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java
@@ -1,323 +1,400 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.service;
-
-import org.apache.http.HttpStatus;
-import org.elasticsearch.client.RestHighLevelClient;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.indexer.OperationType;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache;
-import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
-import org.opengroup.osdu.core.common.model.http.RequestStatus;
-import org.opengroup.osdu.core.common.search.IndicesService;
-import org.opengroup.osdu.indexer.util.ElasticClientHandler;
-import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.springframework.test.context.junit4.SpringRunner;
-
-import java.io.IOException;
-import java.net.URISyntaxException;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.junit.Assert.*;
-import static org.mockito.Mockito.*;
-import static org.mockito.MockitoAnnotations.initMocks;
-import static org.powermock.api.mockito.PowerMockito.mock;
-import static org.powermock.api.mockito.PowerMockito.when;
-
-@RunWith(SpringRunner.class)
-@PrepareForTest({RestHighLevelClient.class})
-public class IndexerSchemaServiceTest {
-
- private final String kind = "tenant:test:test:1.0.0";
- private final String emptySchema = null;
- private final String someSchema = "{\"kind\":\"tenant:test:test:1.0.0\", \"schema\":[{\"path\":\"test-path\", \"kind\":\"tenant:test:test:1.0.0\"}]}";
-
- @Mock
- private JaxRsDpsLog log;
- @Mock
- private StorageService storageService;
- @Mock
- private ElasticClientHandler elasticClientHandler;
- @Mock
- private ElasticIndexNameResolver elasticIndexNameResolver;
- @Mock
- private IndexerMappingService mappingService;
- @Mock
- private IndicesService indicesService;
- @Mock
- private ISchemaCache schemaCache;
- @InjectMocks
- private IndexSchemaServiceImpl sut;
-
- @Before
- public void setup() {
- initMocks(this);
- RestHighLevelClient restHighLevelClient = mock(RestHighLevelClient.class);
- when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
- }
-
- @Test
- public void should_returnNull_givenEmptySchema_getIndexerInputSchemaSchemaTest() throws Exception {
- when(storageService.getStorageSchema(any())).thenReturn(emptySchema);
-
- IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind);
-
- Assert.assertNotNull(indexSchema);
- }
-
- @Test
- public void should_returnValidResponse_givenValidSchema_getIndexerInputSchemaTest() throws Exception {
- when(storageService.getStorageSchema(any())).thenReturn(someSchema);
-
- IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind);
-
- Assert.assertEquals(kind, indexSchema.getKind());
- }
-
- @Test
- public void should_returnValidResponse_givenValidSchemaWithCacheHit_getIndexerInputSchemaTest() throws Exception {
- when(storageService.getStorageSchema(any())).thenReturn(someSchema);
- when(this.schemaCache.get(kind + "_flattened")).thenReturn(someSchema);
-
- IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind);
-
- Assert.assertEquals(kind, indexSchema.getKind());
- }
-
- @Test
- public void should_throw500_givenInvalidSchemaCacheHit_getIndexerInputSchemaTest() {
- try {
- String invalidSchema = "{}}";
- when(storageService.getStorageSchema(any())).thenReturn(invalidSchema);
-
- this.sut.getIndexerInputSchema(kind);
- fail("Should throw exception");
- } catch (AppException e) {
- Assert.assertEquals(HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getError().getCode());
- Assert.assertEquals("An error has occurred while normalizing the schema.", e.getError().getMessage());
- } catch (Exception e) {
- fail("Should not throw exception" + e.getMessage());
- }
- }
-
- @Test
- public void should_return_basic_schema_when_storage_returns_no_schema() {
- IndexSchema returnedSchema = this.sut.getIndexerInputSchema(kind);
-
- assertNotNull(returnedSchema.getDataSchema());
- assertNotNull(returnedSchema);
- assertEquals(kind, returnedSchema.getKind());
- }
-
- @Test
- public void should_create_schema_when_storage_returns_valid_schema() throws IOException, URISyntaxException {
- String kind = "tenant1:avocet:completion:1.0.0";
- String storageSchema = "{" +
- " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
- " \"schema\": [" +
- " {" +
- " \"path\": \"status\"," +
- " \"kind\": \"string\"" +
- " }," +
- " {" +
- " \"path\": \"startDate\"," +
- " \"kind\": \"string\"" +
- " }," +
- " {" +
- " \"path\": \"endDate\"," +
- " \"kind\": \"string\"" +
- " }," +
- " {" +
- " \"path\": \"type \"," +
- " \"kind\": \"string\"" +
- " }," +
- " {" +
- " \"path\": \"itemguid\"," +
- " \"kind\": \"string\"" +
- " }" +
- " ]" +
- "}";
- Map<String, OperationType> schemaMessages = new HashMap<>();
- schemaMessages.put(kind, OperationType.create_schema);
-
- when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
- when(this.schemaCache.get(kind)).thenReturn(null);
- when(this.indicesService.isIndexExist(any(), any())).thenReturn(false);
- when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
-
- this.sut.processSchemaMessages(schemaMessages);
-
- verify(this.mappingService, times(1)).getIndexMappingFromRecordSchema(any());
- verify(this.indicesService, times(1)).createIndex(any(), any(), any(), any(), any());
- verifyNoMoreInteractions(this.mappingService);
- }
-
- @Test
- public void should_merge_mapping_when_storage_returns_valid_schema() throws IOException, URISyntaxException {
- String kind = "tenant1:avocet:completion:1.0.0";
- String storageSchema = "{" +
- " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
- " \"schema\": [" +
- " {" +
- " \"path\": \"status\"," +
- " \"kind\": \"string\"" +
- " }," +
- " {" +
- " \"path\": \"startDate\"," +
- " \"kind\": \"string\"" +
- " }" +
- " ]" +
- "}";
- Map<String, OperationType> schemaMessages = new HashMap<>();
- schemaMessages.put(kind, OperationType.create_schema);
-
- when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
- when(this.schemaCache.get(kind)).thenReturn(null);
- when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
- when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
-
- this.sut.processSchemaMessages(schemaMessages);
-
- verify(this.indicesService, times(0)).createIndex(any(), any(), any(), any(), any());
- verify(this.mappingService, times(1)).createMapping(any(), any(), any(), anyBoolean());
- verifyNoMoreInteractions(this.mappingService);
- }
-
- @Test
- public void should_throw_mapping_conflict_when_elastic_backend_cannot_process_schema_changes() throws IOException, URISyntaxException {
- String kind = "tenant1:avocet:completion:1.0.0";
- String reason = String.format("Could not create type mapping %s/completion.", kind.replace(":", "-"));
- String storageSchema = "{" +
- " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
- " \"schema\": [" +
- " {" +
- " \"path\": \"status\"," +
- " \"kind\": \"string\"" +
- " }" +
- " ]" +
- "}";
- Map<String, OperationType> schemaMessages = new HashMap<>();
- schemaMessages.put(kind, OperationType.create_schema);
-
- when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
- when(this.schemaCache.get(kind)).thenReturn(null);
- when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
- when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
- when(this.mappingService.createMapping(any(), any(), any(), anyBoolean())).thenThrow(new AppException(HttpStatus.SC_BAD_REQUEST, reason, ""));
-
- try {
- this.sut.processSchemaMessages(schemaMessages);
- } catch (AppException e){
- assertEquals(e.getError().getCode(), RequestStatus.SCHEMA_CONFLICT);
- assertEquals(e.getError().getMessage(), "error creating or merging index mapping");
- assertEquals(e.getError().getReason(), reason);
- } catch (Exception e) {
- fail("Should not throw this exception " + e.getMessage());
- }
- }
-
- @Test
- public void should_throw_genericAppException_when_elastic_backend_cannot_process_schema_changes() throws IOException, URISyntaxException {
- String kind = "tenant1:avocet:completion:1.0.0";
- String reason = String.format("Could not create type mapping %s/completion.", kind.replace(":", "-"));
- String storageSchema = "{" +
- " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
- " \"schema\": [" +
- " {" +
- " \"path\": \"status\"," +
- " \"kind\": \"string\"" +
- " }" +
- " ]" +
- "}";
- Map<String, OperationType> schemaMessages = new HashMap<>();
- schemaMessages.put(kind, OperationType.create_schema);
-
- when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
- when(this.schemaCache.get(kind)).thenReturn(null);
- when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
- when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
- when(this.mappingService.createMapping(any(), any(), any(), anyBoolean())).thenThrow(new AppException(HttpStatus.SC_FORBIDDEN, reason, "blah"));
-
- try {
- this.sut.processSchemaMessages(schemaMessages);
- } catch (AppException e){
- assertEquals(e.getError().getCode(), HttpStatus.SC_FORBIDDEN);
- assertEquals(e.getError().getMessage(), "blah");
- assertEquals(e.getError().getReason(), reason);
- } catch (Exception e) {
- fail("Should not throw this exception " + e.getMessage());
- }
- }
-
-
- @Test
- public void should_log_and_do_nothing_when_storage_returns_invalid_schema() throws IOException, URISyntaxException {
- String kind = "tenant1:avocet:completion:1.0.0";
- String storageSchema = "{" +
- " \"kind\": \"tenant1:avocet:completion:1.0.0\"" +
- "}";
- Map<String, OperationType> schemaMessages = new HashMap<>();
- schemaMessages.put(kind, OperationType.create_schema);
-
- when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
- when(this.schemaCache.get(kind)).thenReturn(null);
- when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
- when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
-
- this.sut.processSchemaMessages(schemaMessages);
-
- verify(this.log).warning(eq("schema not found for kind: tenant1:avocet:completion:1.0.0"));
- }
-
- @Test
- public void should_invalidateCache_when_purge_schema_and_schema_found_in_cache() throws IOException {
- String kind = "tenant1:avocet:completion:1.0.0";
- Map<String, OperationType> schemaMessages = new HashMap<>();
- schemaMessages.put(kind, OperationType.purge_schema);
-
- when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
- when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
- when(this.schemaCache.get(kind)).thenReturn("schema");
- when(this.schemaCache.get(kind + "_flattened")).thenReturn("flattened schema");
-
- this.sut.processSchemaMessages(schemaMessages);
-
- verify(this.schemaCache, times(2)).get(anyString());
- verify(this.schemaCache, times(2)).delete(anyString());
- }
-
- @Test
- public void should_log_warning_when_purge_schema_and_schema_not_found_in_cache() throws IOException {
- String kind = "tenant1:avocet:completion:1.0.0";
- Map<String, OperationType> schemaMessages = new HashMap<>();
- schemaMessages.put(kind, OperationType.purge_schema);
-
- when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
- when(this.indicesService.isIndexExist(any(), any())).thenReturn(false);
-
- this.sut.processSchemaMessages(schemaMessages);
-
- verify(this.log).warning(eq(String.format("Kind: %s not found", kind)));
- }
-}
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.service;
+
+import org.apache.http.HttpStatus;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.indexer.OperationType;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache;
+import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
+import org.opengroup.osdu.core.common.model.http.RequestStatus;
+import org.opengroup.osdu.core.common.search.IndicesService;
+import org.opengroup.osdu.indexer.util.ElasticClientHandler;
+import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.*;
+import static org.mockito.MockitoAnnotations.initMocks;
+import static org.powermock.api.mockito.PowerMockito.mock;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@RunWith(SpringRunner.class)
+@PrepareForTest({RestHighLevelClient.class})
+public class IndexerSchemaServiceTest {
+
+ private final String kind = "tenant:test:test:1.0.0";
+ private final String emptySchema = null;
+ private final String someSchema = "{\"kind\":\"tenant:test:test:1.0.0\", \"schema\":[{\"path\":\"test-path\", \"kind\":\"tenant:test:test:1.0.0\"}]}";
+
+ @Mock
+ private JaxRsDpsLog log;
+ @Mock
+ private StorageService storageService;
+ @Mock
+ private ElasticClientHandler elasticClientHandler;
+ @Mock
+ private ElasticIndexNameResolver elasticIndexNameResolver;
+ @Mock
+ private IndexerMappingService mappingService;
+ @Mock
+ private IndicesService indicesService;
+ @Mock
+ private ISchemaCache schemaCache;
+ @InjectMocks
+ private IndexSchemaServiceImpl sut;
+
+ @Before
+ public void setup() {
+ initMocks(this);
+ RestHighLevelClient restHighLevelClient = mock(RestHighLevelClient.class);
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ }
+
+ @Test
+ public void should_returnNull_givenEmptySchema_getIndexerInputSchemaSchemaTest() throws Exception {
+ when(storageService.getStorageSchema(any())).thenReturn(emptySchema);
+
+ IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false);
+
+ Assert.assertNotNull(indexSchema);
+ }
+
+ @Test
+ public void should_returnValidResponse_givenValidSchema_getIndexerInputSchemaTest() throws Exception {
+ when(storageService.getStorageSchema(any())).thenReturn(someSchema);
+
+ IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false);
+
+ Assert.assertEquals(kind, indexSchema.getKind());
+ }
+
+ @Test
+ public void should_returnValidResponse_givenValidSchemaWithCacheHit_getIndexerInputSchemaTest() throws Exception {
+ when(storageService.getStorageSchema(any())).thenReturn(someSchema);
+ when(this.schemaCache.get(kind + "_flattened")).thenReturn(someSchema);
+
+ IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false);
+
+ Assert.assertEquals(kind, indexSchema.getKind());
+ }
+
+ @Test
+ public void should_throw500_givenInvalidSchemaCacheHit_getIndexerInputSchemaTest() {
+ try {
+ String invalidSchema = "{}}";
+ when(storageService.getStorageSchema(any())).thenReturn(invalidSchema);
+
+ this.sut.getIndexerInputSchema(kind, false);
+ fail("Should throw exception");
+ } catch (AppException e) {
+ Assert.assertEquals(HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getError().getCode());
+ Assert.assertEquals("An error has occurred while normalizing the schema.", e.getError().getMessage());
+ } catch (Exception e) {
+ fail("Should not throw exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_return_basic_schema_when_storage_returns_no_schema() {
+ IndexSchema returnedSchema = this.sut.getIndexerInputSchema(kind, false);
+
+ assertNotNull(returnedSchema.getDataSchema());
+ assertNotNull(returnedSchema);
+ assertEquals(kind, returnedSchema.getKind());
+ }
+
+ @Test
+ public void should_create_schema_when_storage_returns_valid_schema() throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"startDate\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"endDate\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"type \"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"itemguid\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.create_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(false);
+ when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
+
+ this.sut.processSchemaMessages(schemaMessages);
+
+ verify(this.mappingService, times(1)).getIndexMappingFromRecordSchema(any());
+ verify(this.indicesService, times(1)).createIndex(any(), any(), any(), any(), any());
+ verifyNoMoreInteractions(this.mappingService);
+ }
+
+ @Test
+ public void should_merge_mapping_when_storage_returns_valid_schema() throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"startDate\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.create_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
+
+ this.sut.processSchemaMessages(schemaMessages);
+
+ verify(this.indicesService, times(0)).createIndex(any(), any(), any(), any(), any());
+ verify(this.mappingService, times(1)).createMapping(any(), any(), any(), anyBoolean());
+ verifyNoMoreInteractions(this.mappingService);
+ }
+
+ @Test
+ public void should_throw_mapping_conflict_when_elastic_backend_cannot_process_schema_changes() throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String reason = String.format("Could not create type mapping %s/completion.", kind.replace(":", "-"));
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.create_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
+ when(this.mappingService.createMapping(any(), any(), any(), anyBoolean())).thenThrow(new AppException(HttpStatus.SC_BAD_REQUEST, reason, ""));
+
+ try {
+ this.sut.processSchemaMessages(schemaMessages);
+ } catch (AppException e) {
+ assertEquals(e.getError().getCode(), RequestStatus.SCHEMA_CONFLICT);
+ assertEquals(e.getError().getMessage(), "error creating or merging index mapping");
+ assertEquals(e.getError().getReason(), reason);
+ } catch (Exception e) {
+ fail("Should not throw this exception " + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_throw_genericAppException_when_elastic_backend_cannot_process_schema_changes() throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String reason = String.format("Could not create type mapping %s/completion.", kind.replace(":", "-"));
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.create_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
+ when(this.mappingService.createMapping(any(), any(), any(), anyBoolean())).thenThrow(new AppException(HttpStatus.SC_FORBIDDEN, reason, "blah"));
+
+ try {
+ this.sut.processSchemaMessages(schemaMessages);
+ } catch (AppException e) {
+ assertEquals(e.getError().getCode(), HttpStatus.SC_FORBIDDEN);
+ assertEquals(e.getError().getMessage(), "blah");
+ assertEquals(e.getError().getReason(), reason);
+ } catch (Exception e) {
+ fail("Should not throw this exception " + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_log_and_do_nothing_when_storage_returns_invalid_schema() throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"" +
+ "}";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.create_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
+
+ this.sut.processSchemaMessages(schemaMessages);
+
+ verify(this.log).warning(eq("schema not found for kind: tenant1:avocet:completion:1.0.0"));
+ }
+
+ @Test
+ public void should_invalidateCache_when_purge_schema_and_schema_found_in_cache() throws IOException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.purge_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.schemaCache.get(kind)).thenReturn("schema");
+ when(this.schemaCache.get(kind + "_flattened")).thenReturn("flattened schema");
+
+ this.sut.processSchemaMessages(schemaMessages);
+
+ verify(this.schemaCache, times(2)).get(anyString());
+ verify(this.schemaCache, times(2)).delete(anyString());
+ }
+
+ @Test
+ public void should_log_warning_when_purge_schema_and_schema_not_found_in_cache() throws IOException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.purge_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(false);
+
+ this.sut.processSchemaMessages(schemaMessages);
+
+ verify(this.log).warning(eq(String.format("Kind: %s not found", kind)));
+ }
+
+ @Test
+ public void should_sync_schema_with_storage() throws Exception {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.indicesService.deleteIndex(any(), any())).thenReturn(true);
+ when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
+
+ this.sut.syncIndexMappingWithStorageSchema(kind);
+
+ verify(this.mappingService, times(1)).getIndexMappingFromRecordSchema(any());
+ verify(this.indicesService, times(1)).isIndexExist(any(), any());
+ verify(this.indicesService, times(1)).deleteIndex(any(), any());
+ verify(this.indicesService, times(1)).createIndex(any(), any(), any(), any(), any());
+ verifyNoMoreInteractions(this.mappingService);
+ }
+
+ @Test
+ public void should_throw_exception_while_snapshot_running_sync_schema_with_storage() throws Exception {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.indicesService.deleteIndex(any(), any())).thenThrow(new AppException(HttpStatus.SC_CONFLICT, "Index deletion error", "blah"));
+
+ try {
+ this.sut.syncIndexMappingWithStorageSchema(kind);
+ } catch (AppException e) {
+ assertEquals(e.getError().getCode(), HttpStatus.SC_CONFLICT);
+ assertEquals(e.getError().getMessage(), "blah");
+ assertEquals(e.getError().getReason(), "Index deletion error");
+ } catch (Exception e) {
+ fail("Should not throw this exception " + e.getMessage());
+ }
+
+ verify(this.indicesService, times(1)).isIndexExist(any(), any());
+ verify(this.indicesService, times(1)).deleteIndex(any(), any());
+ verify(this.mappingService, never()).getIndexMappingFromRecordSchema(any());
+ verify(this.indicesService, never()).createIndex(any(), any(), any(), any(), any());
+ }
+
+ @Test
+ public void should_return_true_while_if_forceClean_requested() throws IOException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+
+ assertTrue(this.sut.isStorageSchemaSyncRequired(kind, true));
+ }
+
+ @Test
+ public void should_return_true_while_if_forceClean_notRequested_and_indexNotFound() throws IOException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(false);
+
+ assertTrue(this.sut.isStorageSchemaSyncRequired(kind, false));
+ }
+
+ @Test
+ public void should_return_false_while_if_forceClean_notRequested_and_indexExist() throws IOException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+
+ assertFalse(this.sut.isStorageSchemaSyncRequired(kind, false));
+ }
+}
diff --git a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java
index cec115db0944b7d8280250904c89344207a5724c..bf2465ed8bc18d41f66d373e4e048e050e3b53ca 100644
--- a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java
+++ b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java
@@ -1,142 +1,151 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.service;
-
-
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
-import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
-import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
-import org.springframework.test.context.junit4.SpringRunner;
-
-import java.util.*;
-
-import static org.junit.Assert.fail;
-import static org.mockito.Matchers.any;
-import static org.mockito.MockitoAnnotations.initMocks;
-import static org.powermock.api.mockito.PowerMockito.mockStatic;
-import static org.powermock.api.mockito.PowerMockito.when;
-
-@RunWith(SpringRunner.class)
-public class ReindexServiceTest {
-
- private final String cursor = "100";
-
- private final String correlationId = UUID.randomUUID().toString();
-
- @Mock
- private StorageService storageService;
-
- @Mock
- private Map<String, String> httpHeaders;
- @Mock
- private IRequestInfo requestInfo;
- @Mock
- private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
- @Mock
- private JaxRsDpsLog log;
- @InjectMocks
- private ReindexServiceImpl sut;
-
- private RecordReindexRequest recordReindexRequest;
- private RecordQueryResponse recordQueryResponse;
-
- @Before
- public void setup() {
- initMocks(this);
-
- mockStatic(UUID.class);
-
- recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor(cursor).build();
- recordQueryResponse = new RecordQueryResponse();
-
- httpHeaders = new HashMap<>();
- httpHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
- httpHeaders.put(DpsHeaders.CORRELATION_ID, correlationId);
- DpsHeaders standardHeaders = DpsHeaders.createFromMap(httpHeaders);
- when(requestInfo.getHeaders()).thenReturn(standardHeaders);
- when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(httpHeaders);
- when(requestInfo.getHeadersWithDwdAuthZ()).thenReturn(standardHeaders);
- }
-
- @Test
- public void should_returnNull_givenNullResponseResult_reIndexRecordsTest() {
- try {
- recordQueryResponse.setResults(null);
- when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
-
- String response = sut.reindexRecords(recordReindexRequest);
-
- Assert.assertNull(response);
- } catch (Exception e) {
- fail("Should not throw this exception" + e.getMessage());
- }
- }
-
- @Test
- public void should_returnNull_givenEmptyResponseResult_reIndexRecordsTest() {
- try {
- recordQueryResponse.setResults(new ArrayList<>());
- when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
-
- String response = sut.reindexRecords(recordReindexRequest);
-
- Assert.assertNull(response);
- } catch (Exception e) {
- fail("Should not throw this exception" + e.getMessage());
- }
- }
-
- @Test
- public void should_returnRecordQueryRequestPayload_givenValidResponseResult_reIndexRecordsTest() {
- try {
- recordQueryResponse.setCursor(cursor);
- List<String> results = new ArrayList<>();
- results.add("test1");
- recordQueryResponse.setResults(results);
- when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
-
- String taskQueuePayload = sut.reindexRecords(recordReindexRequest);
-
- Assert.assertEquals("{\"kind\":\"tenant:test:test:1.0.0\",\"cursor\":\"100\"}", taskQueuePayload);
- } catch (Exception e) {
- fail("Should not throw exception" + e.getMessage());
- }
- }
-
- @Test
- public void should_returnRecordChangedMessage_givenValidResponseResult_reIndexRecordsTest() {
- try {
- List<String> results = new ArrayList<>();
- results.add("test1");
- recordQueryResponse.setResults(results);
- when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
-
- String taskQueuePayload = sut.reindexRecords(recordReindexRequest);
-
- Assert.assertEquals(String.format("{\"data\":\"[{\\\"id\\\":\\\"test1\\\",\\\"kind\\\":\\\"tenant:test:test:1.0.0\\\",\\\"op\\\":\\\"create\\\"}]\",\"attributes\":{\"correlation-id\":\"%s\"}}", correlationId), taskQueuePayload);
- } catch (Exception e) {
- fail("Should not throw exception" + e.getMessage());
- }
- }
-}
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.service;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
+import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.opengroup.osdu.core.common.search.Config;
+import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+import org.powermock.modules.junit4.PowerMockRunnerDelegate;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.util.*;
+
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.MockitoAnnotations.initMocks;
+import static org.powermock.api.mockito.PowerMockito.mockStatic;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@RunWith(PowerMockRunner.class)
+@PowerMockRunnerDelegate(SpringRunner.class)
+@PrepareForTest({Config.class})
+public class ReindexServiceTest {
+
+ private final String cursor = "100";
+
+ private final String correlationId = UUID.randomUUID().toString();
+
+ @Mock
+ private StorageService storageService;
+ @Mock
+ private IRequestInfo requestInfo;
+ @Mock
+ private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
+ @Mock
+ private JaxRsDpsLog log;
+ @InjectMocks
+ private ReindexServiceImpl sut;
+
+ private RecordReindexRequest recordReindexRequest;
+ private RecordQueryResponse recordQueryResponse;
+
+ private Map<String, String> httpHeaders;
+
+ @Before
+ public void setup() {
+ initMocks(this);
+
+ mockStatic(UUID.class);
+
+ recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor(cursor).build();
+ recordQueryResponse = new RecordQueryResponse();
+
+ httpHeaders = new HashMap<>();
+ httpHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
+ httpHeaders.put(DpsHeaders.CORRELATION_ID, correlationId);
+ DpsHeaders standardHeaders = DpsHeaders.createFromMap(httpHeaders);
+ when(requestInfo.getHeaders()).thenReturn(standardHeaders);
+ when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(httpHeaders);
+ when(requestInfo.getHeadersWithDwdAuthZ()).thenReturn(standardHeaders);
+ }
+
+ @Test
+ public void should_returnNull_givenNullResponseResult_reIndexRecordsTest() {
+ try {
+ recordQueryResponse.setResults(null);
+ when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
+
+ String response = sut.reindexRecords(recordReindexRequest, false);
+
+ Assert.assertNull(response);
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_returnNull_givenEmptyResponseResult_reIndexRecordsTest() {
+ try {
+ recordQueryResponse.setResults(new ArrayList<>());
+ when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
+
+ String response = sut.reindexRecords(recordReindexRequest, false);
+
+ Assert.assertNull(response);
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+ @Ignore
+ @Test
+ public void should_returnRecordQueryRequestPayload_givenValidResponseResult_reIndexRecordsTest() {
+ try {
+ recordQueryResponse.setCursor(cursor);
+ List<String> results = new ArrayList<>();
+ results.add("test1");
+ recordQueryResponse.setResults(results);
+
+ mockStatic(Config.class);
+ when(Config.getStorageRecordsBatchSize()).thenReturn(1);
+
+ when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
+
+ String taskQueuePayload = sut.reindexRecords(recordReindexRequest, false);
+
+ Assert.assertEquals("{\"kind\":\"tenant:test:test:1.0.0\",\"cursor\":\"100\"}", taskQueuePayload);
+ } catch (Exception e) {
+ fail("Should not throw exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_returnRecordChangedMessage_givenValidResponseResult_reIndexRecordsTest() {
+ try {
+ List<String> results = new ArrayList<>();
+ results.add("test1");
+ recordQueryResponse.setResults(results);
+ when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
+
+ String taskQueuePayload = sut.reindexRecords(recordReindexRequest, false);
+
+ Assert.assertEquals(String.format("{\"data\":\"[{\\\"id\\\":\\\"test1\\\",\\\"kind\\\":\\\"tenant:test:test:1.0.0\\\",\\\"op\\\":\\\"create\\\"}]\",\"attributes\":{\"correlation-id\":\"%s\"}}", correlationId), taskQueuePayload);
+ } catch (Exception e) {
+ fail("Should not throw exception" + e.getMessage());
+ }
+ }
+}
diff --git a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceTest.java b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceTest.java
index fb5461bb31f90ec9ed85ee23fa305399adac66d4..e868e50c842262a4eba35f2fa49cad1c2aae3cf6 100644
--- a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceTest.java
+++ b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceTest.java
@@ -1,213 +1,248 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.service;
-
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
-
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.mockito.Mockito;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.indexer.JobStatus;
-import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
-import org.opengroup.osdu.core.common.model.http.HttpResponse;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
-import org.opengroup.osdu.core.common.http.IUrlFetchService;
-import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
-import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
-import org.opengroup.osdu.core.common.model.indexer.Records;
-import org.springframework.http.HttpStatus;
-import org.springframework.test.context.junit4.SpringRunner;
-import org.springframework.test.util.ReflectionTestUtils;
-
-import java.lang.reflect.Type;
-import java.net.URISyntaxException;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-
-import static org.junit.Assert.*;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.mock;
-import static org.powermock.api.mockito.PowerMockito.when;
-
-@RunWith(SpringRunner.class)
-public class StorageServiceTest {
-
- @Mock
- private IUrlFetchService urlFetchService;
- @Mock
- private JobStatus jobStatus;
- @Mock
- private JaxRsDpsLog log;
- @Mock
- private IRequestInfo requestInfo;
- @InjectMocks
- private StorageServiceImpl sut;
-
- private List<String> ids;
-
- @Before
- public void setup() {
-
- String recordChangedMessages = "[{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"purge\"}," +
- "{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"create\"}]";
-
- when(this.requestInfo.getHeadersMap()).thenReturn(new HashMap<>());
- when(this.requestInfo.getHeaders()).thenReturn(new DpsHeaders());
-
- Type listType = new TypeToken<List<RecordInfo>>() {}.getType();
-
- List<RecordInfo> msgs = (new Gson()).fromJson(recordChangedMessages, listType);
- jobStatus.initialize(msgs);
- ids = Arrays.asList("tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465", "tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465");
-
- ReflectionTestUtils.setField(this.sut, "STORAGE_RECORDS_BATCH_SIZE", "20");
- }
-
- @Test
- public void should_return404_givenNullData_getValidStorageRecordsTest() throws URISyntaxException {
-
- HttpResponse httpResponse = mock(HttpResponse.class);
- Mockito.when(httpResponse.getBody()).thenReturn(null);
-
- when(this.urlFetchService.sendRequest(any(), any(), any(), any(), any())).thenReturn(httpResponse);
-
- should_return404_getValidStorageRecordsTest();
- }
-
- @Test
- public void should_return404_givenEmptyData_getValidStorageRecordsTest() throws URISyntaxException {
-
- String emptyDataFromStorage = "{\"records\":[],\"notFound\":[]}";
-
- HttpResponse httpResponse = mock(HttpResponse.class);
- Mockito.when(httpResponse.getBody()).thenReturn(emptyDataFromStorage);
-
- when(this.urlFetchService.sendRequest(any(), any(), any(), any(), any())).thenReturn(httpResponse);
-
- should_return404_getValidStorageRecordsTest();
- }
-
- @Test
- public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsTest() throws URISyntaxException {
-
- String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"], \"conversionStatuses\": []}";
-
- HttpResponse httpResponse = mock(HttpResponse.class);
- Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
-
- when(this.urlFetchService.sendRequest(any(), any(), any(), any(), any())).thenReturn(httpResponse);
- Records storageRecords = this.sut.getStorageRecords(ids);
-
- assertEquals(1, storageRecords.getRecords().size());
- }
-
- @Test
- public void should_returnValidResponse_givenValidRecordQueryRequest_getRecordListByKind() throws Exception {
-
- RecordReindexRequest recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor("100").build();
-
- HttpResponse httpResponse = new HttpResponse();
- httpResponse.setBody(new Gson().toJson(recordReindexRequest, RecordReindexRequest.class));
-
- when(this.urlFetchService.sendRequest(any(), any(), any(), any(), any())).thenReturn(httpResponse);
-
- RecordQueryResponse recordQueryResponse = this.sut.getRecordsByKind(recordReindexRequest);
-
- assertEquals("100", recordQueryResponse.getCursor());
- assertNull(recordQueryResponse.getResults());
- }
-
- @Test
- public void should_returnValidResponse_givenValidKind_getSchemaByKind() throws Exception {
-
- String validSchemaFromStorage = "{" +
- " \"kind\": \"tenant:test:test:1.0.0\"," +
- " \"schema\": [" +
- " {" +
- " \"path\": \"msg\"," +
- " \"kind\": \"string\"" +
- " }," +
- " {" +
- " \"path\": \"references.entity\"," +
- " \"kind\": \"string\"" +
- " }" +
- " ]," +
- " \"ext\": null" +
- "}";
- String kind = "tenant:test:test:1.0.0";
-
- HttpResponse httpResponse = new HttpResponse();
- httpResponse.setResponseCode(HttpStatus.OK.value());
- httpResponse.setBody(validSchemaFromStorage);
-
- when(this.urlFetchService.sendRequest(any(), any(), any(), any(), any())).thenReturn(httpResponse);
-
- String recordSchemaResponse = this.sut.getStorageSchema(kind);
-
- assertNotNull(recordSchemaResponse);
- }
-
- @Test
- public void should_returnNullResponse_givenAbsentKind_getSchemaByKind() throws Exception {
-
- String kind = "tenant:test:test:1.0.0";
-
- HttpResponse httpResponse = new HttpResponse();
- httpResponse.setResponseCode(HttpStatus.NOT_FOUND.value());
-
- when(this.urlFetchService.sendRequest(any(), any(), any(), any(), any())).thenReturn(httpResponse);
-
- String recordSchemaResponse = this.sut.getStorageSchema(kind);
-
- assertNull(recordSchemaResponse);
- }
-
- @Test
- public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsWithInvalidConversionTest() throws URISyntaxException {
-
- String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"],\"conversionStatuses\": [{\"id\":\"testid\",\"status\":\"ERROR\",\"errors\":[\"conversion error occured\"] } ]}";
-
- HttpResponse httpResponse = mock(HttpResponse.class);
- Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
-
- when(this.urlFetchService.sendRequest(any(), any(), any(), any(), any())).thenReturn(httpResponse);
- Records storageRecords = this.sut.getStorageRecords(ids);
-
- assertEquals(1, storageRecords.getRecords().size());
-
- assertEquals(1, storageRecords.getConversionStatuses().get(0).getErrors().size());
-
- assertEquals("conversion error occured", storageRecords.getConversionStatuses().get(0).getErrors().get(0));
- }
-
- private void should_return404_getValidStorageRecordsTest() {
- try {
- this.sut.getStorageRecords(ids);
- fail("Should throw exception");
- } catch (AppException e) {
- assertEquals(HttpStatus.NOT_FOUND.value(), e.getError().getCode());
- } catch (Exception e) {
- fail("Should not throw this exception" + e.getMessage());
- }
- }
-}
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.service;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentMatchers;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.indexer.IndexingStatus;
+import org.opengroup.osdu.core.common.model.indexer.JobStatus;
+import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
+import org.opengroup.osdu.core.common.model.http.HttpResponse;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.opengroup.osdu.core.common.http.IUrlFetchService;
+import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
+import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
+import org.opengroup.osdu.core.common.model.indexer.Records;
+import org.springframework.http.HttpStatus;
+import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import java.lang.reflect.Type;
+import java.net.URISyntaxException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+
+import static java.util.Collections.singletonList;
+import static org.junit.Assert.*;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@RunWith(SpringRunner.class)
+public class StorageServiceTest {
+
+ @Mock
+ private IUrlFetchService urlFetchService;
+ @Mock
+ private JobStatus jobStatus;
+ @Mock
+ private JaxRsDpsLog log;
+ @Mock
+ private IRequestInfo requestInfo;
+ @InjectMocks
+ private StorageServiceImpl sut;
+
+ private List<String> ids;
+ private static final String RECORD_ID1 = "tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465";
+ private static final String RECORDS_ID2 = "tenant1:doc:15e790a69beb4d789b1f979e2af2e813";
+
+ @Before
+ public void setup() {
+
+ String recordChangedMessages = "[{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"purge\"}," +
+ "{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"create\"}]";
+
+ when(this.requestInfo.getHeadersMap()).thenReturn(new HashMap<>());
+ when(this.requestInfo.getHeaders()).thenReturn(new DpsHeaders());
+
+ Type listType = new TypeToken<List<RecordInfo>>() {}.getType();
+
+ List<RecordInfo> msgs = (new Gson()).fromJson(recordChangedMessages, listType);
+ jobStatus.initialize(msgs);
+ ids = Arrays.asList(RECORD_ID1, RECORDS_ID2);
+
+ ReflectionTestUtils.setField(this.sut, "STORAGE_RECORDS_BATCH_SIZE", "20");
+ }
+
+ @Test
+ public void should_return404_givenNullData_getValidStorageRecordsTest() throws URISyntaxException {
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(null);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ should_return404_getValidStorageRecordsTest();
+ }
+
+ @Test
+ public void should_return404_givenEmptyData_getValidStorageRecordsTest() throws URISyntaxException {
+
+ String emptyDataFromStorage = "{\"records\":[],\"notFound\":[]}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(emptyDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ should_return404_getValidStorageRecordsTest();
+ }
+
+ @Test
+ public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsTest() throws URISyntaxException {
+
+ String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"], \"conversionStatuses\": []}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+ Records storageRecords = this.sut.getStorageRecords(ids);
+
+ assertEquals(1, storageRecords.getRecords().size());
+ }
+
+ @Test
+ public void should_logMissingRecord_given_storageMissedRecords() throws URISyntaxException {
+
+ String validDataFromStorage = "{\"records\":[{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[]}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(any())).thenReturn(httpResponse);
+ Records storageRecords = this.sut.getStorageRecords(ids);
+
+ assertEquals(1, storageRecords.getRecords().size());
+ verify(this.jobStatus).addOrUpdateRecordStatus(singletonList(RECORDS_ID2), IndexingStatus.FAIL, HttpStatus.NOT_FOUND.value(), "Partial response received from Storage service - missing records", "Partial response received from Storage service: tenant1:doc:15e790a69beb4d789b1f979e2af2e813");
+ }
+
+ @Test
+ public void should_returnValidJobStatus_givenFailedUnitsConversion_processRecordChangedMessageTest() throws URISyntaxException {
+ String validDataFromStorage = "{\"records\":[{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[],\"conversionStatuses\":[{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\",\"status\":\"ERROR\",\"errors\":[\"crs conversion failed\"]}]}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(any())).thenReturn(httpResponse);
+ Records storageRecords = this.sut.getStorageRecords(singletonList(RECORDS_ID2));
+
+ assertEquals(1, storageRecords.getRecords().size());
+ verify(this.jobStatus).addOrUpdateRecordStatus(RECORDS_ID2, IndexingStatus.WARN, HttpStatus.BAD_REQUEST.value(), "crs conversion failed", String.format("record-id: %s | %s", "tenant1:doc:15e790a69beb4d789b1f979e2af2e813", "crs conversion failed"));
+ }
+
+ @Test
+ public void should_returnValidResponse_givenValidRecordQueryRequest_getRecordListByKind() throws Exception {
+
+ RecordReindexRequest recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor("100").build();
+
+ HttpResponse httpResponse = new HttpResponse();
+ httpResponse.setBody(new Gson().toJson(recordReindexRequest, RecordReindexRequest.class));
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ RecordQueryResponse recordQueryResponse = this.sut.getRecordsByKind(recordReindexRequest);
+
+ assertEquals("100", recordQueryResponse.getCursor());
+ assertNull(recordQueryResponse.getResults());
+ }
+
+ @Test
+ public void should_returnValidResponse_givenValidKind_getSchemaByKind() throws Exception {
+
+ String validSchemaFromStorage = "{" +
+ " \"kind\": \"tenant:test:test:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"msg\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"references.entity\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]," +
+ " \"ext\": null" +
+ "}";
+ String kind = "tenant:test:test:1.0.0";
+
+ HttpResponse httpResponse = new HttpResponse();
+ httpResponse.setResponseCode(HttpStatus.OK.value());
+ httpResponse.setBody(validSchemaFromStorage);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ String recordSchemaResponse = this.sut.getStorageSchema(kind);
+
+ assertNotNull(recordSchemaResponse);
+ }
+
+ @Test
+ public void should_returnNullResponse_givenAbsentKind_getSchemaByKind() throws Exception {
+
+ String kind = "tenant:test:test:1.0.0";
+
+ HttpResponse httpResponse = new HttpResponse();
+ httpResponse.setResponseCode(HttpStatus.NOT_FOUND.value());
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ String recordSchemaResponse = this.sut.getStorageSchema(kind);
+
+ assertNull(recordSchemaResponse);
+ }
+
+ @Test
+ public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsWithInvalidConversionTest() throws URISyntaxException {
+
+ String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"],\"conversionStatuses\": [{\"id\":\"testid\",\"status\":\"ERROR\",\"errors\":[\"conversion error occurred\"] } ]}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+ Records storageRecords = this.sut.getStorageRecords(ids);
+
+ assertEquals(1, storageRecords.getRecords().size());
+
+ assertEquals(1, storageRecords.getConversionStatuses().get(0).getErrors().size());
+
+ assertEquals("conversion error occurred", storageRecords.getConversionStatuses().get(0).getErrors().get(0));
+ }
+
+ private void should_return404_getValidStorageRecordsTest() {
+ try {
+ this.sut.getStorageRecords(ids);
+ fail("Should throw exception");
+ } catch (AppException e) {
+ assertEquals(HttpStatus.NOT_FOUND.value(), e.getError().getCode());
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+}
diff --git a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerSchemaServiceTest.java b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerSchemaServiceTest.java
index 25dd2ed446ebc1cb107a0149c139db46705569f2..db2eb01d13ab616671c3a6e855d13c67a4e7ed94 100644
--- a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerSchemaServiceTest.java
+++ b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerSchemaServiceTest.java
@@ -1,329 +1,406 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.ibm.service;
-
-import org.apache.http.HttpStatus;
-import org.elasticsearch.client.RestHighLevelClient;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
-import org.opengroup.osdu.core.common.model.indexer.OperationType;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache;
-import org.opengroup.osdu.indexer.service.IndexSchemaServiceImpl;
-import org.opengroup.osdu.indexer.service.IndexerMappingService;
-import org.opengroup.osdu.indexer.service.StorageService;
-import org.opengroup.osdu.core.common.model.http.RequestStatus;
-import org.opengroup.osdu.core.common.search.IndicesService;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.indexer.util.ElasticClientHandler;
-import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.springframework.test.context.junit4.SpringRunner;
-
-import java.io.IOException;
-import java.net.URISyntaxException;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.junit.Assert.*;
-import static org.mockito.ArgumentMatchers.*;
-import static org.mockito.Mockito.*;
-import static org.mockito.MockitoAnnotations.initMocks;
-import static org.powermock.api.mockito.PowerMockito.mock;
-import static org.powermock.api.mockito.PowerMockito.when;
-
-@Ignore
-@RunWith(SpringRunner.class)
-@PrepareForTest({RestHighLevelClient.class})
-public class IndexerSchemaServiceTest {
-
- private final String kind = "tenant:test:test:1.0.0";
- private final String emptySchema = null;
- private final String someSchema = "{\"kind\":\"tenant:test:test:1.0.0\", \"schema\":[{\"path\":\"test-path\", \"kind\":\"tenant:test:test:1.0.0\"}]}";
-
- @Mock
- private JaxRsDpsLog log;
- @Mock
- private StorageService storageService;
- @Mock
- private ElasticClientHandler elasticClientHandler;
- @Mock
- private ElasticIndexNameResolver elasticIndexNameResolver;
- @Mock
- private IndexerMappingService mappingService;
- @Mock
- private IndicesService indicesService;
- @Mock
- private ISchemaCache schemaCache;
- @InjectMocks
- private IndexSchemaServiceImpl sut;
-
- @Before
- public void setup() {
- initMocks(this);
- RestHighLevelClient restHighLevelClient = mock(RestHighLevelClient.class);
- when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
- }
-
- @Test
- public void should_returnNull_givenEmptySchema_getIndexerInputSchemaSchemaTest() throws Exception {
- when(storageService.getStorageSchema(any())).thenReturn(emptySchema);
-
- IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind);
-
- Assert.assertNotNull(indexSchema);
- }
-
- @Test
- public void should_returnValidResponse_givenValidSchema_getIndexerInputSchemaTest() throws Exception {
- when(storageService.getStorageSchema(any())).thenReturn(someSchema);
-
- IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind);
-
- Assert.assertEquals(kind, indexSchema.getKind());
- }
-
- @Test
- public void should_returnValidResponse_givenValidSchemaWithCacheHit_getIndexerInputSchemaTest() throws Exception {
- when(storageService.getStorageSchema(any())).thenReturn(someSchema);
- when(this.schemaCache.get(kind + "_flattened")).thenReturn(someSchema);
-
- IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind);
-
- Assert.assertEquals(kind, indexSchema.getKind());
- }
-
- @Test
- public void should_throw500_givenInvalidSchemaCacheHit_getIndexerInputSchemaTest() {
- try {
- String invalidSchema = "{}}";
- when(storageService.getStorageSchema(any())).thenReturn(invalidSchema);
-
- this.sut.getIndexerInputSchema(kind);
- fail("Should throw exception");
- } catch (AppException e) {
- Assert.assertEquals(HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getError().getCode());
- Assert.assertEquals("An error has occurred while normalizing the schema.", e.getError().getMessage());
- } catch (Exception e) {
- fail("Should not throw exception" + e.getMessage());
- }
- }
-
- @Test
- public void should_return_basic_schema_when_storage_returns_no_schema() {
- IndexSchema returnedSchema = this.sut.getIndexerInputSchema(kind);
-
- assertNotNull(returnedSchema.getDataSchema());
- assertNotNull(returnedSchema);
- assertEquals(kind, returnedSchema.getKind());
- }
-
- @Test
- public void should_create_schema_when_storage_returns_valid_schema() throws IOException, URISyntaxException {
- String kind = "tenant1:avocet:completion:1.0.0";
- String storageSchema = "{" +
- " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
- " \"schema\": [" +
- " {" +
- " \"path\": \"status\"," +
- " \"kind\": \"string\"" +
- " }," +
- " {" +
- " \"path\": \"startDate\"," +
- " \"kind\": \"string\"" +
- " }," +
- " {" +
- " \"path\": \"endDate\"," +
- " \"kind\": \"string\"" +
- " }," +
- " {" +
- " \"path\": \"type \"," +
- " \"kind\": \"string\"" +
- " }," +
- " {" +
- " \"path\": \"itemguid\"," +
- " \"kind\": \"string\"" +
- " }" +
- " ]" +
- "}";
- Map<String, OperationType> schemaMessages = new HashMap<>();
- schemaMessages.put(kind, OperationType.create_schema);
-
- when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
- when(this.schemaCache.get(kind)).thenReturn(null);
- when(this.indicesService.isIndexExist(any(), any())).thenReturn(false);
- when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
-
- this.sut.processSchemaMessages(schemaMessages);
-
- verify(this.mappingService, times(1)).getIndexMappingFromRecordSchema(any());
- verify(this.indicesService, times(1)).createIndex(any(), any(), any(), any(), any());
- verifyNoMoreInteractions(this.mappingService);
- }
-
- @Test
- public void should_merge_mapping_when_storage_returns_valid_schema() throws IOException, URISyntaxException {
- String kind = "tenant1:avocet:completion:1.0.0";
- String storageSchema = "{" +
- " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
- " \"schema\": [" +
- " {" +
- " \"path\": \"status\"," +
- " \"kind\": \"string\"" +
- " }," +
- " {" +
- " \"path\": \"startDate\"," +
- " \"kind\": \"string\"" +
- " }" +
- " ]" +
- "}";
- Map<String, OperationType> schemaMessages = new HashMap<>();
- schemaMessages.put(kind, OperationType.create_schema);
-
- when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
- when(this.schemaCache.get(kind)).thenReturn(null);
- when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
- when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
-
- this.sut.processSchemaMessages(schemaMessages);
-
- verify(this.indicesService, times(0)).createIndex(any(), any(), any(), any(), any());
- verify(this.mappingService, times(1)).createMapping(any(), any(), any(), anyBoolean());
- verifyNoMoreInteractions(this.mappingService);
- }
-
- @Test
- public void should_throw_mapping_conflict_when_elastic_backend_cannot_process_schema_changes() throws IOException, URISyntaxException {
- String kind = "tenant1:avocet:completion:1.0.0";
- String reason = String.format("Could not create type mapping %s/completion.", kind.replace(":", "-"));
- String storageSchema = "{" +
- " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
- " \"schema\": [" +
- " {" +
- " \"path\": \"status\"," +
- " \"kind\": \"string\"" +
- " }" +
- " ]" +
- "}";
- Map<String, OperationType> schemaMessages = new HashMap<>();
- schemaMessages.put(kind, OperationType.create_schema);
-
- when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
- when(this.schemaCache.get(kind)).thenReturn(null);
- when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
- when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
- when(this.mappingService.createMapping(any(), any(), any(), anyBoolean())).thenThrow(new AppException(HttpStatus.SC_BAD_REQUEST, reason, ""));
-
- try {
- this.sut.processSchemaMessages(schemaMessages);
- } catch (AppException e){
- assertEquals(e.getError().getCode(), RequestStatus.SCHEMA_CONFLICT);
- assertEquals(e.getError().getMessage(), "error creating or merging index mapping");
- assertEquals(e.getError().getReason(), reason);
- } catch (Exception e) {
- fail("Should not throw this exception " + e.getMessage());
- }
- }
-
- @Test
- public void should_throw_genericAppException_when_elastic_backend_cannot_process_schema_changes() throws IOException, URISyntaxException {
- String kind = "tenant1:avocet:completion:1.0.0";
- String reason = String.format("Could not create type mapping %s/completion.", kind.replace(":", "-"));
- String storageSchema = "{" +
- " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
- " \"schema\": [" +
- " {" +
- " \"path\": \"status\"," +
- " \"kind\": \"string\"" +
- " }" +
- " ]" +
- "}";
- Map<String, OperationType> schemaMessages = new HashMap<>();
- schemaMessages.put(kind, OperationType.create_schema);
-
- when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
- when(this.schemaCache.get(kind)).thenReturn(null);
- when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
- when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
- when(this.mappingService.createMapping(any(), any(), any(), anyBoolean())).thenThrow(new AppException(HttpStatus.SC_FORBIDDEN, reason, "blah"));
-
- try {
- this.sut.processSchemaMessages(schemaMessages);
- } catch (AppException e){
- assertEquals(e.getError().getCode(), HttpStatus.SC_FORBIDDEN);
- assertEquals(e.getError().getMessage(), "blah");
- assertEquals(e.getError().getReason(), reason);
- } catch (Exception e) {
- fail("Should not throw this exception " + e.getMessage());
- }
- }
-
-
- @Test
- public void should_log_and_do_nothing_when_storage_returns_invalid_schema() throws IOException, URISyntaxException {
- String kind = "tenant1:avocet:completion:1.0.0";
- String storageSchema = "{" +
- " \"kind\": \"tenant1:avocet:completion:1.0.0\"" +
- "}";
- Map<String, OperationType> schemaMessages = new HashMap<>();
- schemaMessages.put(kind, OperationType.create_schema);
-
- when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
- when(this.schemaCache.get(kind)).thenReturn(null);
- when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
- when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
-
- this.sut.processSchemaMessages(schemaMessages);
-
- verify(this.log).warning(eq("schema not found for kind: tenant1:avocet:completion:1.0.0"));
- }
-
- @Test
- public void should_invalidateCache_when_purge_schema_and_schema_found_in_cache() throws IOException {
- String kind = "tenant1:avocet:completion:1.0.0";
- Map<String, OperationType> schemaMessages = new HashMap<>();
- schemaMessages.put(kind, OperationType.purge_schema);
-
- when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
- when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
- when(this.schemaCache.get(kind)).thenReturn("schema");
- when(this.schemaCache.get(kind + "_flattened")).thenReturn("flattened schema");
-
- this.sut.processSchemaMessages(schemaMessages);
-
- verify(this.schemaCache, times(2)).get(anyString());
- verify(this.schemaCache, times(2)).delete(anyString());
- }
-
- @Test
- public void should_log_warning_when_purge_schema_and_schema_not_found_in_cache() throws IOException {
- String kind = "tenant1:avocet:completion:1.0.0";
- Map<String, OperationType> schemaMessages = new HashMap<>();
- schemaMessages.put(kind, OperationType.purge_schema);
-
- when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
- when(this.indicesService.isIndexExist(any(), any())).thenReturn(false);
-
- this.sut.processSchemaMessages(schemaMessages);
-
- verify(this.log).warning(eq(String.format("Kind: %s not found", kind)));
- }
-}
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.service;
+
+import org.apache.http.HttpStatus;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
+import org.opengroup.osdu.core.common.model.indexer.OperationType;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache;
+import org.opengroup.osdu.indexer.service.IndexSchemaServiceImpl;
+import org.opengroup.osdu.indexer.service.IndexerMappingService;
+import org.opengroup.osdu.indexer.service.StorageService;
+import org.opengroup.osdu.core.common.model.http.RequestStatus;
+import org.opengroup.osdu.core.common.search.IndicesService;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.indexer.util.ElasticClientHandler;
+import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.*;
+import static org.mockito.ArgumentMatchers.*;
+import static org.mockito.Mockito.*;
+import static org.mockito.MockitoAnnotations.initMocks;
+import static org.powermock.api.mockito.PowerMockito.mock;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@Ignore
+@RunWith(SpringRunner.class)
+@PrepareForTest({RestHighLevelClient.class})
+public class IndexerSchemaServiceTest {
+
+ private final String kind = "tenant:test:test:1.0.0";
+ private final String emptySchema = null;
+ private final String someSchema = "{\"kind\":\"tenant:test:test:1.0.0\", \"schema\":[{\"path\":\"test-path\", \"kind\":\"tenant:test:test:1.0.0\"}]}";
+
+ @Mock
+ private JaxRsDpsLog log;
+ @Mock
+ private StorageService storageService;
+ @Mock
+ private ElasticClientHandler elasticClientHandler;
+ @Mock
+ private ElasticIndexNameResolver elasticIndexNameResolver;
+ @Mock
+ private IndexerMappingService mappingService;
+ @Mock
+ private IndicesService indicesService;
+ @Mock
+ private ISchemaCache schemaCache;
+ @InjectMocks
+ private IndexSchemaServiceImpl sut;
+
+ @Before
+ public void setup() {
+ initMocks(this);
+ RestHighLevelClient restHighLevelClient = mock(RestHighLevelClient.class);
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ }
+
+ @Test
+ public void should_returnNull_givenEmptySchema_getIndexerInputSchemaSchemaTest() throws Exception {
+ when(storageService.getStorageSchema(any())).thenReturn(emptySchema);
+
+ IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false);
+
+ Assert.assertNotNull(indexSchema);
+ }
+
+ @Test
+ public void should_returnValidResponse_givenValidSchema_getIndexerInputSchemaTest() throws Exception {
+ when(storageService.getStorageSchema(any())).thenReturn(someSchema);
+
+ IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false);
+
+ Assert.assertEquals(kind, indexSchema.getKind());
+ }
+
+ @Test
+ public void should_returnValidResponse_givenValidSchemaWithCacheHit_getIndexerInputSchemaTest() throws Exception {
+ when(storageService.getStorageSchema(any())).thenReturn(someSchema);
+ when(this.schemaCache.get(kind + "_flattened")).thenReturn(someSchema);
+
+ IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false);
+
+ Assert.assertEquals(kind, indexSchema.getKind());
+ }
+
+ @Test
+ public void should_throw500_givenInvalidSchemaCacheHit_getIndexerInputSchemaTest() {
+ try {
+ String invalidSchema = "{}}";
+ when(storageService.getStorageSchema(any())).thenReturn(invalidSchema);
+
+ this.sut.getIndexerInputSchema(kind, false);
+ fail("Should throw exception");
+ } catch (AppException e) {
+ Assert.assertEquals(HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getError().getCode());
+ Assert.assertEquals("An error has occurred while normalizing the schema.", e.getError().getMessage());
+ } catch (Exception e) {
+ fail("Should not throw exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_return_basic_schema_when_storage_returns_no_schema() {
+ IndexSchema returnedSchema = this.sut.getIndexerInputSchema(kind, false);
+
+ assertNotNull(returnedSchema.getDataSchema());
+ assertNotNull(returnedSchema);
+ assertEquals(kind, returnedSchema.getKind());
+ }
+
+ @Test
+ public void should_create_schema_when_storage_returns_valid_schema() throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"startDate\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"endDate\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"type \"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"itemguid\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.create_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(false);
+ when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
+
+ this.sut.processSchemaMessages(schemaMessages);
+
+ verify(this.mappingService, times(1)).getIndexMappingFromRecordSchema(any());
+ verify(this.indicesService, times(1)).createIndex(any(), any(), any(), any(), any());
+ verifyNoMoreInteractions(this.mappingService);
+ }
+
+ @Test
+ public void should_merge_mapping_when_storage_returns_valid_schema() throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"startDate\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.create_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
+
+ this.sut.processSchemaMessages(schemaMessages);
+
+ verify(this.indicesService, times(0)).createIndex(any(), any(), any(), any(), any());
+ verify(this.mappingService, times(1)).createMapping(any(), any(), any(), anyBoolean());
+ verifyNoMoreInteractions(this.mappingService);
+ }
+
+ @Test
+ public void should_throw_mapping_conflict_when_elastic_backend_cannot_process_schema_changes() throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String reason = String.format("Could not create type mapping %s/completion.", kind.replace(":", "-"));
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.create_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
+ when(this.mappingService.createMapping(any(), any(), any(), anyBoolean())).thenThrow(new AppException(HttpStatus.SC_BAD_REQUEST, reason, ""));
+
+ try {
+ this.sut.processSchemaMessages(schemaMessages);
+ } catch (AppException e) {
+ assertEquals(e.getError().getCode(), RequestStatus.SCHEMA_CONFLICT);
+ assertEquals(e.getError().getMessage(), "error creating or merging index mapping");
+ assertEquals(e.getError().getReason(), reason);
+ } catch (Exception e) {
+ fail("Should not throw this exception " + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_throw_genericAppException_when_elastic_backend_cannot_process_schema_changes() throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String reason = String.format("Could not create type mapping %s/completion.", kind.replace(":", "-"));
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.create_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
+ when(this.mappingService.createMapping(any(), any(), any(), anyBoolean())).thenThrow(new AppException(HttpStatus.SC_FORBIDDEN, reason, "blah"));
+
+ try {
+ this.sut.processSchemaMessages(schemaMessages);
+ } catch (AppException e) {
+ assertEquals(e.getError().getCode(), HttpStatus.SC_FORBIDDEN);
+ assertEquals(e.getError().getMessage(), "blah");
+ assertEquals(e.getError().getReason(), reason);
+ } catch (Exception e) {
+ fail("Should not throw this exception " + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_log_and_do_nothing_when_storage_returns_invalid_schema() throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"" +
+ "}";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.create_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
+
+ this.sut.processSchemaMessages(schemaMessages);
+
+ verify(this.log).warning(eq("schema not found for kind: tenant1:avocet:completion:1.0.0"));
+ }
+
+ @Test
+ public void should_invalidateCache_when_purge_schema_and_schema_found_in_cache() throws IOException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.purge_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.schemaCache.get(kind)).thenReturn("schema");
+ when(this.schemaCache.get(kind + "_flattened")).thenReturn("flattened schema");
+
+ this.sut.processSchemaMessages(schemaMessages);
+
+ verify(this.schemaCache, times(2)).get(anyString());
+ verify(this.schemaCache, times(2)).delete(anyString());
+ }
+
+ @Test
+ public void should_log_warning_when_purge_schema_and_schema_not_found_in_cache() throws IOException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.purge_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(false);
+
+ this.sut.processSchemaMessages(schemaMessages);
+
+ verify(this.log).warning(eq(String.format("Kind: %s not found", kind)));
+ }
+
+ @Test
+ public void should_sync_schema_with_storage() throws Exception {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.indicesService.deleteIndex(any(), any())).thenReturn(true);
+ when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
+
+ this.sut.syncIndexMappingWithStorageSchema(kind);
+
+ verify(this.mappingService, times(1)).getIndexMappingFromRecordSchema(any());
+ verify(this.indicesService, times(1)).isIndexExist(any(), any());
+ verify(this.indicesService, times(1)).deleteIndex(any(), any());
+ verify(this.indicesService, times(1)).createIndex(any(), any(), any(), any(), any());
+ verifyNoMoreInteractions(this.mappingService);
+ }
+
+ @Test
+ public void should_throw_exception_while_snapshot_running_sync_schema_with_storage() throws Exception {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.indicesService.deleteIndex(any(), any())).thenThrow(new AppException(HttpStatus.SC_CONFLICT, "Index deletion error", "blah"));
+
+ try {
+ this.sut.syncIndexMappingWithStorageSchema(kind);
+ } catch (AppException e) {
+ assertEquals(e.getError().getCode(), HttpStatus.SC_CONFLICT);
+ assertEquals(e.getError().getMessage(), "blah");
+ assertEquals(e.getError().getReason(), "Index deletion error");
+ } catch (Exception e) {
+ fail("Should not throw this exception " + e.getMessage());
+ }
+
+ verify(this.indicesService, times(1)).isIndexExist(any(), any());
+ verify(this.indicesService, times(1)).deleteIndex(any(), any());
+ verify(this.mappingService, never()).getIndexMappingFromRecordSchema(any());
+ verify(this.indicesService, never()).createIndex(any(), any(), any(), any(), any());
+ }
+
+ @Test
+ public void should_return_true_while_if_forceClean_requested() throws IOException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+
+ assertTrue(this.sut.isStorageSchemaSyncRequired(kind, true));
+ }
+
+ @Test
+ public void should_return_true_while_if_forceClean_notRequested_and_indexNotFound() throws IOException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(false);
+
+ assertTrue(this.sut.isStorageSchemaSyncRequired(kind, false));
+ }
+
+ @Test
+ public void should_return_false_while_if_forceClean_notRequested_and_indexExist() throws IOException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+
+ assertFalse(this.sut.isStorageSchemaSyncRequired(kind, false));
+ }
+}
diff --git a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/ReindexServiceTest.java b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/ReindexServiceTest.java
index e6a102b7821fa8a58bf38b0a992ce2203d8a2a47..1db13bb3827d6b691075b4ab089b5457e53b6b37 100644
--- a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/ReindexServiceTest.java
+++ b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/ReindexServiceTest.java
@@ -1,145 +1,154 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.ibm.service;
-
-
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.ArgumentMatchers;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
-import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.indexer.service.ReindexServiceImpl;
-import org.opengroup.osdu.indexer.service.StorageService;
-import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
-import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
-import org.springframework.test.context.junit4.SpringRunner;
-
-import java.util.*;
-
-import static org.junit.Assert.fail;
-import static org.mockito.MockitoAnnotations.initMocks;
-import static org.powermock.api.mockito.PowerMockito.mockStatic;
-import static org.powermock.api.mockito.PowerMockito.when;
-
-@Ignore
-@RunWith(SpringRunner.class)
-public class ReindexServiceTest {
-
- private final String cursor = "100";
-
- private final String correlationId = UUID.randomUUID().toString();
-
- @Mock
- private StorageService storageService;
-
- @Mock
- private Map<String, String> httpHeaders;
- @Mock
- private IRequestInfo requestInfo;
- @Mock
- private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
- @Mock
- private JaxRsDpsLog log;
- @InjectMocks
- private ReindexServiceImpl sut;
-
- private RecordReindexRequest recordReindexRequest;
- private RecordQueryResponse recordQueryResponse;
-
- @Before
- public void setup() {
- initMocks(this);
-
- mockStatic(UUID.class);
-
- recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor(cursor).build();
- recordQueryResponse = new RecordQueryResponse();
-
- httpHeaders = new HashMap<>();
- httpHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
- httpHeaders.put(DpsHeaders.CORRELATION_ID, correlationId);
- DpsHeaders standardHeaders = DpsHeaders.createFromMap(httpHeaders);
- when(requestInfo.getHeaders()).thenReturn(standardHeaders);
- when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(httpHeaders);
- }
-
- @Test
- public void should_returnNull_givenNullResponseResult_reIndexRecordsTest() {
- try {
- recordQueryResponse.setResults(null);
- when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
-
- String response = sut.reindexRecords(recordReindexRequest);
-
- Assert.assertNull(response);
- } catch (Exception e) {
- fail("Should not throw this exception" + e.getMessage());
- }
- }
-
- @Test
- public void should_returnNull_givenEmptyResponseResult_reIndexRecordsTest() {
- try {
- recordQueryResponse.setResults(new ArrayList<>());
- when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
-
- String response = sut.reindexRecords(recordReindexRequest);
-
- Assert.assertNull(response);
- } catch (Exception e) {
- fail("Should not throw this exception" + e.getMessage());
- }
- }
-
- @Test
- public void should_returnRecordQueryRequestPayload_givenValidResponseResult_reIndexRecordsTest() {
- try {
- recordQueryResponse.setCursor(cursor);
- List<String> results = new ArrayList<>();
- results.add("test1");
- recordQueryResponse.setResults(results);
- when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
-
- String taskQueuePayload = sut.reindexRecords(recordReindexRequest);
-
- Assert.assertEquals("{\"kind\":\"tenant:test:test:1.0.0\",\"cursor\":\"100\"}", taskQueuePayload);
- } catch (Exception e) {
- fail("Should not throw exception" + e.getMessage());
- }
- }
-
- @Test
- public void should_returnRecordChangedMessage_givenValidResponseResult_reIndexRecordsTest() {
- try {
- List<String> results = new ArrayList<>();
- results.add("test1");
- recordQueryResponse.setResults(results);
- when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
-
- String taskQueuePayload = sut.reindexRecords(recordReindexRequest);
-
- Assert.assertEquals(String.format("{\"data\":\"[{\\\"id\\\":\\\"test1\\\",\\\"kind\\\":\\\"tenant:test:test:1.0.0\\\",\\\"op\\\":\\\"create\\\"}]\",\"attributes\":{\"slb-correlation-id\":\"%s\"}}", correlationId), taskQueuePayload);
- } catch (Exception e) {
- fail("Should not throw exception" + e.getMessage());
- }
- }
-}
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.service;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentMatchers;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
+import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.search.Config;
+import org.opengroup.osdu.indexer.service.ReindexServiceImpl;
+import org.opengroup.osdu.indexer.service.StorageService;
+import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+import org.powermock.modules.junit4.PowerMockRunnerDelegate;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.util.*;
+
+import static org.junit.Assert.fail;
+import static org.mockito.MockitoAnnotations.initMocks;
+import static org.powermock.api.mockito.PowerMockito.mockStatic;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@Ignore
+@RunWith(PowerMockRunner.class)
+@PowerMockRunnerDelegate(SpringRunner.class)
+@PrepareForTest({Config.class})
+public class ReindexServiceTest {
+
+ private final String cursor = "100";
+
+ private final String correlationId = UUID.randomUUID().toString();
+
+ @Mock
+ private StorageService storageService;
+
+ @Mock
+ private Map<String, String> httpHeaders;
+ @Mock
+ private IRequestInfo requestInfo;
+ @Mock
+ private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
+ @Mock
+ private JaxRsDpsLog log;
+ @InjectMocks
+ private ReindexServiceImpl sut;
+
+ private RecordReindexRequest recordReindexRequest;
+ private RecordQueryResponse recordQueryResponse;
+
+ @Before
+ public void setup() {
+ initMocks(this);
+
+ mockStatic(UUID.class);
+
+ recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor(cursor).build();
+ recordQueryResponse = new RecordQueryResponse();
+
+ httpHeaders = new HashMap<>();
+ httpHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
+ httpHeaders.put(DpsHeaders.CORRELATION_ID, correlationId);
+ DpsHeaders standardHeaders = DpsHeaders.createFromMap(httpHeaders);
+ when(requestInfo.getHeaders()).thenReturn(standardHeaders);
+ when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(httpHeaders);
+ }
+
+ @Test
+ public void should_returnNull_givenNullResponseResult_reIndexRecordsTest() {
+ try {
+ recordQueryResponse.setResults(null);
+ when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
+
+ String response = sut.reindexRecords(recordReindexRequest, false);
+
+ Assert.assertNull(response);
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_returnNull_givenEmptyResponseResult_reIndexRecordsTest() {
+ try {
+ recordQueryResponse.setResults(new ArrayList<>());
+ when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
+
+ String response = sut.reindexRecords(recordReindexRequest, false);
+
+ Assert.assertNull(response);
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_returnRecordQueryRequestPayload_givenValidResponseResult_reIndexRecordsTest() {
+ try {
+ recordQueryResponse.setCursor(cursor);
+ List<String> results = new ArrayList<>();
+ results.add("test1");
+ recordQueryResponse.setResults(results);
+
+ mockStatic(Config.class);
+ when(Config.getStorageRecordsBatchSize()).thenReturn(1);
+
+ when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
+
+ String taskQueuePayload = sut.reindexRecords(recordReindexRequest, false);
+
+ Assert.assertEquals("{\"kind\":\"tenant:test:test:1.0.0\",\"cursor\":\"100\"}", taskQueuePayload);
+ } catch (Exception e) {
+ fail("Should not throw exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_returnRecordChangedMessage_givenValidResponseResult_reIndexRecordsTest() {
+ try {
+ List<String> results = new ArrayList<>();
+ results.add("test1");
+ recordQueryResponse.setResults(results);
+ when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
+
+ String taskQueuePayload = sut.reindexRecords(recordReindexRequest, false);
+
+ Assert.assertEquals(String.format("{\"data\":\"[{\\\"id\\\":\\\"test1\\\",\\\"kind\\\":\\\"tenant:test:test:1.0.0\\\",\\\"op\\\":\\\"create\\\"}]\",\"attributes\":{\"slb-correlation-id\":\"%s\"}}", correlationId), taskQueuePayload);
+ } catch (Exception e) {
+ fail("Should not throw exception" + e.getMessage());
+ }
+ }
+}
diff --git a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/StorageServiceTest.java b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/StorageServiceTest.java
index 93e90b30e58f53034af05e8580b945c759f85fe9..013a5a7210806127947185924268cda94e7166cc 100644
--- a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/StorageServiceTest.java
+++ b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/StorageServiceTest.java
@@ -1,211 +1,230 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.ibm.service;
-
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.ArgumentMatchers;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.mockito.Mockito;
-import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
-import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
-import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
-import org.opengroup.osdu.core.common.model.indexer.Records;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.indexer.service.StorageServiceImpl;
-import org.opengroup.osdu.core.common.model.indexer.JobStatus;
-import org.opengroup.osdu.core.common.model.http.HttpResponse;
-import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
-import org.opengroup.osdu.core.common.http.IUrlFetchService;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.springframework.http.HttpStatus;
-import org.springframework.test.context.junit4.SpringRunner;
-
-import java.lang.reflect.Type;
-import java.net.URISyntaxException;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-
-import static org.junit.Assert.*;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.mock;
-import static org.powermock.api.mockito.PowerMockito.when;
-
-@Ignore
-@RunWith(SpringRunner.class)
-public class StorageServiceTest {
-
- @Mock
- private IUrlFetchService urlFetchService;
- @Mock
- private JobStatus jobStatus;
- @Mock
- private JaxRsDpsLog log;
- @Mock
- private IRequestInfo requestInfo;
- @InjectMocks
- private StorageServiceImpl sut;
-
- private List<String> ids;
-
- @Before
- public void setup() {
-
- String recordChangedMessages = "[{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"purge\"}," +
- "{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"create\"}]";
-
- when(this.requestInfo.getHeadersMap()).thenReturn(new HashMap<>());
-
- Type listType = new TypeToken<List<RecordInfo>>() {}.getType();
-
- List<RecordInfo> msgs = (new Gson()).fromJson(recordChangedMessages, listType);
- jobStatus.initialize(msgs);
- ids = Arrays.asList("tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465", "tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465");
- }
-
- @Test
- public void should_return404_givenNullData_getValidStorageRecordsTest() throws URISyntaxException {
-
- HttpResponse httpResponse = mock(HttpResponse.class);
- Mockito.when(httpResponse.getBody()).thenReturn(null);
-
- when(this.urlFetchService.sendRequest(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(httpResponse);
-
- should_return404_getValidStorageRecordsTest();
- }
-
- @Test
- public void should_return404_givenEmptyData_getValidStorageRecordsTest() throws URISyntaxException {
-
- String emptyDataFromStorage = "{\"records\":[],\"notFound\":[]}";
-
- HttpResponse httpResponse = mock(HttpResponse.class);
- Mockito.when(httpResponse.getBody()).thenReturn(emptyDataFromStorage);
-
- when(this.urlFetchService.sendRequest(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(httpResponse);
-
- should_return404_getValidStorageRecordsTest();
- }
-
- @Test
- public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsTest() throws URISyntaxException {
-
- String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"]}";
-
- HttpResponse httpResponse = mock(HttpResponse.class);
- Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
-
- when(this.urlFetchService.sendRequest(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(httpResponse);
- Records storageRecords = this.sut.getStorageRecords(ids);
-
- assertEquals(1, storageRecords.getRecords().size());
- }
-
- @Test
- public void should_returnValidResponse_givenValidRecordQueryRequest_getRecordListByKind() throws Exception {
-
- RecordReindexRequest recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor("100").build();
-
- HttpResponse httpResponse = new HttpResponse();
- httpResponse.setBody(new Gson().toJson(recordReindexRequest, RecordReindexRequest.class));
-
- when(this.urlFetchService.sendRequest(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(httpResponse);
-
- RecordQueryResponse recordQueryResponse = this.sut.getRecordsByKind(recordReindexRequest);
-
- assertEquals("100", recordQueryResponse.getCursor());
- assertNull(recordQueryResponse.getResults());
- }
-
- @Test
- public void should_returnValidResponse_givenValidKind_getSchemaByKind() throws Exception {
-
- String validSchemaFromStorage = "{" +
- " \"kind\": \"tenant:test:test:1.0.0\"," +
- " \"schema\": [" +
- " {" +
- " \"path\": \"msg\"," +
- " \"kind\": \"string\"" +
- " }," +
- " {" +
- " \"path\": \"references.entity\"," +
- " \"kind\": \"string\"" +
- " }" +
- " ]," +
- " \"ext\": null" +
- "}";
- String kind = "tenant:test:test:1.0.0";
-
- HttpResponse httpResponse = new HttpResponse();
- httpResponse.setResponseCode(HttpStatus.OK.value());
- httpResponse.setBody(validSchemaFromStorage);
-
- when(this.urlFetchService.sendRequest(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(httpResponse);
-
- String recordSchemaResponse = this.sut.getStorageSchema(kind);
-
- assertNotNull(recordSchemaResponse);
- }
-
- @Test
- public void should_returnNullResponse_givenAbsentKind_getSchemaByKind() throws Exception {
-
- String kind = "tenant:test:test:1.0.0";
-
- HttpResponse httpResponse = new HttpResponse();
- httpResponse.setResponseCode(HttpStatus.NOT_FOUND.value());
-
- when(this.urlFetchService.sendRequest(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(httpResponse);
-
- String recordSchemaResponse = this.sut.getStorageSchema(kind);
-
- assertNull(recordSchemaResponse);
- }
-
- @Test
- public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsWithInvalidConversionTest() throws URISyntaxException {
-
- String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"],\"conversionStatuses\": [{\"id\":\"testid\",\"status\":\"ERROR\",\"errors\":[\"conversion error occured\"] } ]}";
-
- HttpResponse httpResponse = mock(HttpResponse.class);
- Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
-
- when(this.urlFetchService.sendRequest(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(httpResponse);
- Records storageRecords = this.sut.getStorageRecords(ids);
-
- assertEquals(1, storageRecords.getRecords().size());
-
- assertEquals(1, storageRecords.getConversionStatuses().get(0).getErrors().size());
-
- assertEquals("conversion error occured", storageRecords.getConversionStatuses().get(0).getErrors().get(0));
- }
-
- private void should_return404_getValidStorageRecordsTest() {
- try {
- this.sut.getStorageRecords(ids);
- fail("Should throw exception");
- } catch (AppException e) {
- assertEquals(HttpStatus.NOT_FOUND, e.getError().getCode());
- } catch (Exception e) {
- fail("Should not throw this exception" + e.getMessage());
- }
- }
-}
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.service;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentMatchers;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.opengroup.osdu.core.common.model.indexer.IndexingStatus;
+import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
+import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
+import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
+import org.opengroup.osdu.core.common.model.indexer.Records;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.indexer.service.StorageServiceImpl;
+import org.opengroup.osdu.core.common.model.indexer.JobStatus;
+import org.opengroup.osdu.core.common.model.http.HttpResponse;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.opengroup.osdu.core.common.http.IUrlFetchService;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.springframework.http.HttpStatus;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.lang.reflect.Type;
+import java.net.URISyntaxException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+
+import static java.util.Collections.singletonList;
+import static org.junit.Assert.*;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@Ignore
+@RunWith(SpringRunner.class)
+public class StorageServiceTest {
+
+ @Mock
+ private IUrlFetchService urlFetchService;
+ @Mock
+ private JobStatus jobStatus;
+ @Mock
+ private JaxRsDpsLog log;
+ @Mock
+ private IRequestInfo requestInfo;
+ @InjectMocks
+ private StorageServiceImpl sut;
+
+ private List<String> ids;
+ private static final String RECORD_ID1 = "tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465";
+ private static final String RECORDS_ID2 = "tenant1:doc:15e790a69beb4d789b1f979e2af2e813";
+
+ @Before
+ public void setup() {
+
+ String recordChangedMessages = "[{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"purge\"}," +
+ "{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"create\"}]";
+
+ when(this.requestInfo.getHeadersMap()).thenReturn(new HashMap<>());
+
+ Type listType = new TypeToken<List<RecordInfo>>() {}.getType();
+
+ List<RecordInfo> msgs = (new Gson()).fromJson(recordChangedMessages, listType);
+ jobStatus.initialize(msgs);
+ ids = Arrays.asList(RECORD_ID1, RECORDS_ID2);
+ }
+
+ @Test
+ public void should_return404_givenNullData_getValidStorageRecordsTest() throws URISyntaxException {
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(null);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ should_return404_getValidStorageRecordsTest();
+ }
+
+ @Test
+ public void should_return404_givenEmptyData_getValidStorageRecordsTest() throws URISyntaxException {
+
+ String emptyDataFromStorage = "{\"records\":[],\"notFound\":[]}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(emptyDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ should_return404_getValidStorageRecordsTest();
+ }
+
+ @Test
+ public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsTest() throws URISyntaxException {
+
+ String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"]}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+ Records storageRecords = this.sut.getStorageRecords(ids);
+
+ assertEquals(1, storageRecords.getRecords().size());
+ }
+
+ @Test
+ public void should_returnValidResponse_givenValidRecordQueryRequest_getRecordListByKind() throws Exception {
+
+ RecordReindexRequest recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor("100").build();
+
+ HttpResponse httpResponse = new HttpResponse();
+ httpResponse.setBody(new Gson().toJson(recordReindexRequest, RecordReindexRequest.class));
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ RecordQueryResponse recordQueryResponse = this.sut.getRecordsByKind(recordReindexRequest);
+
+ assertEquals("100", recordQueryResponse.getCursor());
+ assertNull(recordQueryResponse.getResults());
+ }
+
+ @Test
+ public void should_returnValidJobStatus_givenFailedUnitsConversion_processRecordChangedMessageTest() throws URISyntaxException {
+ String validDataFromStorage = "{\"records\":[{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[],\"conversionStatuses\":[{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\",\"status\":\"ERROR\",\"errors\":[\"crs conversion failed\"]}]}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(any())).thenReturn(httpResponse);
+ Records storageRecords = this.sut.getStorageRecords(singletonList("tenant1:doc:15e790a69beb4d789b1f979e2af2e813"));
+
+ assertEquals(1, storageRecords.getRecords().size());
+ verify(this.jobStatus).addOrUpdateRecordStatus(RECORDS_ID2, IndexingStatus.WARN, HttpStatus.BAD_REQUEST.value(), "crs conversion failed", String.format("record-id: %s | %s", "tenant1:doc:15e790a69beb4d789b1f979e2af2e813", "crs conversion failed"));
+ }
+
+ @Test
+ public void should_returnValidResponse_givenValidKind_getSchemaByKind() throws Exception {
+
+ String validSchemaFromStorage = "{" +
+ " \"kind\": \"tenant:test:test:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"msg\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"references.entity\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]," +
+ " \"ext\": null" +
+ "}";
+ String kind = "tenant:test:test:1.0.0";
+
+ HttpResponse httpResponse = new HttpResponse();
+ httpResponse.setResponseCode(HttpStatus.OK.value());
+ httpResponse.setBody(validSchemaFromStorage);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ String recordSchemaResponse = this.sut.getStorageSchema(kind);
+
+ assertNotNull(recordSchemaResponse);
+ }
+
+ @Test
+ public void should_returnNullResponse_givenAbsentKind_getSchemaByKind() throws Exception {
+
+ String kind = "tenant:test:test:1.0.0";
+
+ HttpResponse httpResponse = new HttpResponse();
+ httpResponse.setResponseCode(HttpStatus.NOT_FOUND.value());
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ String recordSchemaResponse = this.sut.getStorageSchema(kind);
+
+ assertNull(recordSchemaResponse);
+ }
+
+ @Test
+ public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsWithInvalidConversionTest() throws URISyntaxException {
+
+ String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"],\"conversionStatuses\": [{\"id\":\"testid\",\"status\":\"ERROR\",\"errors\":[\"conversion error occurred\"] } ]}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+ Records storageRecords = this.sut.getStorageRecords(ids);
+
+ assertEquals(1, storageRecords.getRecords().size());
+
+ assertEquals(1, storageRecords.getConversionStatuses().get(0).getErrors().size());
+
+ assertEquals("conversion error occurred", storageRecords.getConversionStatuses().get(0).getErrors().get(0));
+ }
+
+ private void should_return404_getValidStorageRecordsTest() {
+ try {
+ this.sut.getStorageRecords(ids);
+ fail("Should throw exception");
+ } catch (AppException e) {
+ assertEquals(HttpStatus.NOT_FOUND, e.getError().getCode());
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+}
diff --git a/testing/indexer-test-azure/pom.xml b/testing/indexer-test-azure/pom.xml
index 0dc9d6c176adb45c71059fc4df35c7a60fe04f30..7d96ec79c242458691258ce53923882c3556a2f6 100644
--- a/testing/indexer-test-azure/pom.xml
+++ b/testing/indexer-test-azure/pom.xml
@@ -100,7 +100,7 @@
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-junit-jupiter</artifactId>
- <version>2.23.0</version>
+ <version>2.26.0</version>
<scope>test</scope>
</dependency>
diff --git a/testing/indexer-test-core/pom.xml b/testing/indexer-test-core/pom.xml
index 74ab140f67da111aaa517cc068567979a593b034..eb98c8fcb5ef9c6b195e4c069a561cd7923f02c0 100644
--- a/testing/indexer-test-core/pom.xml
+++ b/testing/indexer-test-core/pom.xml
@@ -12,6 +12,7 @@
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
<cucumber.version>1.2.5</cucumber.version>
+ <os-core-common.version>0.0.18</os-core-common.version>
</properties>
<!-- testing core depends on core libraries in OSDU, so the repository needs to be configured -->
@@ -21,22 +22,11 @@
<url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</url>
</repository>
</repositories>
-
- <distributionManagement>
- <repository>
- <id>${gitlab-server}</id>
- <url>https://community.opengroup.org/api/v4/projects/25/packages/maven</url>
- </repository>
- <snapshotRepository>
- <id>${gitlab-server}</id>
- <url>https://community.opengroup.org/api/v4/projects/25/packages/maven</url>
- </snapshotRepository>
- </distributionManagement>
<dependencies>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
- <version>0.0.13</version>
+ <version>${os-core-common.version}</version>
</dependency>
<dependency>
diff --git a/testing/indexer-test-core/src/main/resources/features/indexrecord/IndexRecord.feature b/testing/indexer-test-core/src/main/resources/features/indexrecord/IndexRecord.feature
index 3d7ccae8b7f836504007de9229096fa91583270d..30ee32eaa576dc0a7369c0a29a9110a20c566775 100644
--- a/testing/indexer-test-core/src/main/resources/features/indexrecord/IndexRecord.feature
+++ b/testing/indexer-test-core/src/main/resources/features/indexrecord/IndexRecord.feature
@@ -15,7 +15,8 @@ Feature: Indexing of the documents
Examples:
| kind | recordFile | number | index | type | acl | mapping |
- | "tenant1:testindex<timestamp>:well:1.0.0" | "index_records_1" | 5 | "tenant1-testindex<timestamp>-well-1.0.0" | "well" | "data.default.viewers@opendes" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
+ | "tenant1:testindex<timestamp>:well:1.0.0" | "index_records_1" | 5 | "tenant1-testindex<timestamp>-well-1.0.0" | "well" | "data.default.viewers@opendes" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
+ | "tenant1:testindex<timestamp>:well:3.0.0" | "index_records_1" | 5 | "tenant1-testindex<timestamp>-well-3.0.0" | "well" | "data.default.viewers@opendes" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
Scenario Outline: Ingest the record and Index in the Elastic Search with bad attribute
When I ingest records with the <recordFile> with <acl> for a given <kind>
diff --git a/testing/indexer-test-core/src/main/resources/testData/index_records_1.json b/testing/indexer-test-core/src/main/resources/testData/index_records_1.json
index 145b5a78bab6867d7635f360383fbedac272a64c..0d9f3c17ef24167affb538fc7db995317a77c348 100644
--- a/testing/indexer-test-core/src/main/resources/testData/index_records_1.json
+++ b/testing/indexer-test-core/src/main/resources/testData/index_records_1.json
@@ -1,112 +1,114 @@
-[
- {
- "id": "tenant1:ihs:testIngest2<timestamp>",
- "data": {
- "Field": "OSDU OFFICE - 2",
- "Location": {
- "latitude":32.406402588,
- "longitude":-86.565592762
- },
- "Basin": "Houston",
- "County": "Harris",
- "State": "TX",
- "Country": "USA",
- "WellStatus": "Under development",
- "OriginalOperator": "OFFICE - 2",
- "WellName": "Data Platform Services",
- "WellType": "Data Lake Cloud",
- "EmptyAttribute": "",
- "Rank": 1,
- "Score" : 10,
- "Established": "2000-03-27T23:38:48Z"
- }
- },
- {
- "id": "tenant1:ihs:testIngest3<timestamp>",
- "data": {
- "Field": "OSDU OFFICE - 2",
- "Location": {
- "latitude":32.406402588,
- "longitude":-86.565592762
- },
- "Basin": "Houston",
- "County": "Harris",
- "State": "TX",
- "Country": "USA",
- "WellStatus": "Under development",
- "OriginalOperator": "OFFICE2",
- "WellName": "Data Platform Services",
- "WellType": "Data Lake Cloud",
- "EmptyAttribute": "",
- "Rank": 1,
- "Score" : 10,
- "Established": "2000-03-27T23:38:48Z"
- }
- },
- {
- "id": "tenant1:ihs:testIngest4<timestamp>",
- "data": {
- "Field": "OSDU OFFICE - 2",
- "Location": {
- "latitude":32.406402588,
- "longitude":-86.565592762
- },
- "Basin": "Houston",
- "County": "Harris",
- "State": "TX",
- "Country": "USA",
- "WellStatus": "Under development",
- "OriginalOperator": "OFFICE2",
- "WellName": "Data Platform Services",
- "WellType": "Data Lake Cloud",
- "EmptyAttribute": "",
- "Rank": 1,
- "Score" : 10,
- "Established": "2000-03-27T23:38:48Z"
- }
- },
- {
- "id": "tenant1:ihs:testIngest5<timestamp>",
- "data": {
- "Field": "OSDU OFFICE - 2",
- "Location": {
- "latitude":32.406402588,
- "longitude":-86.565592762
- },
- "Basin": "Houston",
- "County": "Harris",
- "State": "TX",
- "Country": "USA",
- "WellStatus": "Under development",
- "OriginalOperator": "OFFICE2",
- "WellName": "Data Platform Services",
- "WellType": "Data Lake Cloud",
- "EmptyAttribute": "",
- "Rank": 1,
- "Score" : 10,
- "Established": "2000-03-27T23:38:48Z"
- }
- },
- {
- "id": "tenant1:ihs:testIngest6<timestamp>",
- "data": {
- "Field": "OSDU OFFICE - 2",
- "Location": {
- "latitude":32.406402588,
- "longitude":-86.565592762
- },
- "Basin": "Houston",
- "County": "Harris",
- "State": "TX",
- "Country": "USA",
- "WellStatus": "Under development",
- "OriginalOperator": "OFFICE2",
- "WellName": "Data Platform Services",
- "WellType": "Data Lake Cloud",
- "EmptyAttribute": "",
- "Rank": 1,
- "Score" : 10,
- "Established": "2000-03-27T23:38:48Z"
- }
- }
+[
+ {
+ "id": "tenant1:ihs:testIngest2<timestamp>",
+ "data": {
+ "Field": "OSDU OFFICE - 2",
+ "Location": {
+ "latitude":32.406402588,
+ "longitude":-86.565592762
+ },
+ "Basin": "Houston",
+ "County": "Harris",
+ "State": "TX",
+ "Country": "USA",
+ "WellStatus": "Under development",
+ "OriginalOperator": "OFFICE - 2",
+ "WellName": "Data Platform Services",
+ "WellType": "Data Lake Cloud",
+ "EmptyAttribute": "",
+ "Rank": 1,
+ "Score" : 10,
+ "Established": "2000-03-27T23:38:48Z",
+ "DblArray": [32.40, 36.45, 40.0]
+ }
+ },
+ {
+ "id": "tenant1:ihs:testIngest3<timestamp>",
+ "data": {
+ "Field": "OSDU OFFICE - 2",
+ "Location": {
+ "latitude":32.406402588,
+ "longitude":-86.565592762
+ },
+ "Basin": "Houston",
+ "County": "Harris",
+ "State": "TX",
+ "Country": "USA",
+ "WellStatus": "Under development",
+ "OriginalOperator": "OFFICE2",
+ "WellName": "Data Platform Services",
+ "WellType": "Data Lake Cloud",
+ "EmptyAttribute": "",
+ "Rank": 1,
+ "Score" : 10,
+ "Established": "2000-03-27T23:38:48Z",
+ "DblArray": [62.40, 60.0]
+ }
+ },
+ {
+ "id": "tenant1:ihs:testIngest4<timestamp>",
+ "data": {
+ "Field": "OSDU OFFICE - 2",
+ "Location": {
+ "latitude":32.406402588,
+ "longitude":-86.565592762
+ },
+ "Basin": "Houston",
+ "County": "Harris",
+ "State": "TX",
+ "Country": "USA",
+ "WellStatus": "Under development",
+ "OriginalOperator": "OFFICE2",
+ "WellName": "Data Platform Services",
+ "WellType": "Data Lake Cloud",
+ "EmptyAttribute": "",
+ "Rank": 1,
+ "Score" : 10,
+ "Established": "2000-03-27T23:38:48Z"
+ }
+ },
+ {
+ "id": "tenant1:ihs:testIngest5<timestamp>",
+ "data": {
+ "Field": "OSDU OFFICE - 2",
+ "Location": {
+ "latitude":32.406402588,
+ "longitude":-86.565592762
+ },
+ "Basin": "Houston",
+ "County": "Harris",
+ "State": "TX",
+ "Country": "USA",
+ "WellStatus": "Under development",
+ "OriginalOperator": "OFFICE2",
+ "WellName": "Data Platform Services",
+ "WellType": "Data Lake Cloud",
+ "EmptyAttribute": "",
+ "Rank": 1,
+ "Score" : 10,
+ "Established": "2000-03-27T23:38:48Z"
+ }
+ },
+ {
+ "id": "tenant1:ihs:testIngest6<timestamp>",
+ "data": {
+ "Field": "OSDU OFFICE - 2",
+ "Location": {
+ "latitude":32.406402588,
+ "longitude":-86.565592762
+ },
+ "Basin": "Houston",
+ "County": "Harris",
+ "State": "TX",
+ "Country": "USA",
+ "WellStatus": "Under development",
+ "OriginalOperator": "OFFICE2",
+ "WellName": "Data Platform Services",
+ "WellType": "Data Lake Cloud",
+ "EmptyAttribute": "",
+ "Rank": 1,
+ "Score" : 10,
+ "Established": "2000-03-27T23:38:48Z"
+ }
+ }
]
\ No newline at end of file
diff --git a/testing/indexer-test-core/src/main/resources/testData/index_records_1.schema b/testing/indexer-test-core/src/main/resources/testData/index_records_1.schema
index bfdc78abd4664a98824562eab09d32ffc283395d..a67be0f5383d708571bcbc03b69cb7c818d860d4 100644
--- a/testing/indexer-test-core/src/main/resources/testData/index_records_1.schema
+++ b/testing/indexer-test-core/src/main/resources/testData/index_records_1.schema
@@ -56,6 +56,10 @@
{
"path": "Established",
"kind": "datetime"
+ },
+ {
+ "path": "DblArray",
+ "kind": "[]double"
}
]
}
\ No newline at end of file