diff --git a/.gitignore b/.gitignore
index 9e269f1b78050908088da343128dea99d75bacdc..e8c005ae687055f7d4bcf5ada49f03d435dc6f64 100644
--- a/.gitignore
+++ b/.gitignore
@@ -41,3 +41,5 @@ provider/indexer-gcp/bin/*
# Environment configuration
*.env
.envrc
+/.gradle/checksums/checksums.lock
+/.gradle/6.7/fileHashes/fileHashes.lock
diff --git a/devops/azure/chart/templates/deployment.yaml b/devops/azure/chart/templates/deployment.yaml
index 6f8dabb677ca1a52f3f95c310c9b2c18acf2f6a4..1f8e620484e28c856b2ddad65a3ef0d1bf202156 100644
--- a/devops/azure/chart/templates/deployment.yaml
+++ b/devops/azure/chart/templates/deployment.yaml
@@ -56,7 +56,7 @@ spec:
readOnly: true
env:
- name: spring_application_name
- value: indexer
+ value: {{ .Chart.Name }}
- name: server.servlet.contextPath
value: /api/indexer/v2/
- name: server_port
diff --git a/devops/azure/chart/templates/hpa.yaml b/devops/azure/chart/templates/hpa.yaml
index d9d00bfc65e3df192ca241337772c937dbc1ff9e..68a00c86912a249472eec7cf49411f38b72322d4 100644
--- a/devops/azure/chart/templates/hpa.yaml
+++ b/devops/azure/chart/templates/hpa.yaml
@@ -24,7 +24,7 @@ spec:
kind: Deployment
name: {{ .Chart.Name }}
minReplicas: {{ .Values.global.replicaCount }}
- maxReplicas: 5
+ maxReplicas: 10
metrics:
- type: Resource
resource:
diff --git a/indexer-core/pom.xml b/indexer-core/pom.xml
index 919ad21103a071122b569c0ca33c2215a6ecaff6..e982bd0809d2b38751dfb5b346e893a7ce2ce183 100644
--- a/indexer-core/pom.xml
+++ b/indexer-core/pom.xml
@@ -16,6 +16,7 @@
<properties>
<commons-beanutils.version>1.9.4</commons-beanutils.version>
+ <osdu.oscorecommon.version>0.6.5</osdu.oscorecommon.version>
</properties>
<dependencies>
@@ -33,6 +34,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
+ <version>${osdu.oscorecommon.version}</version>
</dependency>
<!-- spring boot dependencies -->
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/error/GlobalExceptionMapperCore.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/error/GlobalExceptionMapperCore.java
new file mode 100644
index 0000000000000000000000000000000000000000..6e546e14cf7197a480bf84de084a3cf628373ad0
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/error/GlobalExceptionMapperCore.java
@@ -0,0 +1,88 @@
+// Copyright © Microsoft Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.error;
+
+import javax.validation.ValidationException;
+
+import javassist.NotFoundException;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.Ordered;
+import org.springframework.core.annotation.Order;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.security.access.AccessDeniedException;
+import org.springframework.web.bind.annotation.ControllerAdvice;
+import org.springframework.web.bind.annotation.ExceptionHandler;
+import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler;
+import org.opengroup.osdu.core.common.model.http.AppException;
+
+@Order(Ordered.HIGHEST_PRECEDENCE - 1)
+@ControllerAdvice
+public class GlobalExceptionMapperCore extends ResponseEntityExceptionHandler {
+
+ @Autowired
+ private JaxRsDpsLog logger;
+
+ @ExceptionHandler(AppException.class)
+ protected ResponseEntity<Object> handleAppException(AppException e) {
+ return this.getErrorResponse(e);
+ }
+
+ @ExceptionHandler(ValidationException.class)
+ protected ResponseEntity<Object> handleValidationException(ValidationException e) {
+ return this.getErrorResponse(
+ new AppException(HttpStatus.BAD_REQUEST.value(), "Validation error.", e.getMessage(), e));
+ }
+
+ @ExceptionHandler(NotFoundException.class)
+ protected ResponseEntity<Object> handleNotFoundException(NotFoundException e) {
+ return this.getErrorResponse(
+ new AppException(HttpStatus.NOT_FOUND.value(), "Resource not found.", e.getMessage(), e));
+ }
+
+ @ExceptionHandler(AccessDeniedException.class)
+ protected ResponseEntity<Object> handleAccessDeniedException(AccessDeniedException e) {
+ return this.getErrorResponse(
+ new AppException(HttpStatus.FORBIDDEN.value(), "Access denied", e.getMessage(), e));
+ }
+
+ @ExceptionHandler(Exception.class)
+ protected ResponseEntity<Object> handleGeneralException(Exception e) {
+ return this.getErrorResponse(
+ new AppException(HttpStatus.INTERNAL_SERVER_ERROR.value(), "Server error.",
+ "An unknown error has occurred.", e));
+ }
+
+ private ResponseEntity<Object> getErrorResponse(AppException e) {
+
+ String exceptionMsg = e.getOriginalException() != null
+ ? e.getOriginalException().getMessage()
+ : e.getError().getMessage();
+
+ if( e.getCause() instanceof Exception) {
+ Exception original = (Exception) e.getCause();
+ this.logger.error(original.getMessage(), original);
+ }
+
+ if (e.getError().getCode() > 499) {
+ this.logger.error(exceptionMsg, e);
+ } else {
+ this.logger.warning(exceptionMsg, e);
+ }
+
+ return new ResponseEntity<Object>(e.getError(), HttpStatus.resolve(e.getError().getCode()));
+ }
+}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java
index 3506c2ca092db2688a6f15a2eca790082ab9b05b..d98b1f77281138a9c3de3304350da461cda9ac08 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java
@@ -56,8 +56,10 @@ public class PropertiesProcessor {
public Stream<Map<String, Object>> processItem(AllOfItem allOfItem) {
Preconditions.checkNotNull(allOfItem, "allOfItem cannot be null");
- if (Objects.nonNull(allOfItem.getAllOf())) {
- return allOfItem.getAllOf().stream().flatMap(this::processItem);
+ Stream<Map<String, Object>> ofItems = processOfItems(allOfItem.getAllOf(), allOfItem.getAnyOf(), allOfItem.getOneOf());
+
+ if (Objects.nonNull(ofItems)) {
+ return ofItems;
}
String ref = allOfItem.getRef();
@@ -89,13 +91,34 @@ public class PropertiesProcessor {
new AppException(HttpStatus.SC_NOT_FOUND, "Failed to find definition:" + definitionSubRef,
"Unknown definition:" + definitionSubRef));
- if (Objects.nonNull(definition.getAllOf())) {
- return definition.getAllOf().stream().flatMap(this::processItem);
+ Stream<Map<String, Object>> ofItems =
+ processOfItems(definition.getAllOf(), definition.getAnyOf(), definition.getOneOf());
+
+ if (Objects.nonNull(ofItems)) {
+ return ofItems;
}
return processProperties(definition.getProperties());
}
+ private Stream<Map<String, Object>> processOfItems(List<AllOfItem> allOf, List<AllOfItem> anyOf, List<AllOfItem> oneOf) {
+ Stream<Map<String, Object>> ofItems = null;
+
+ if (Objects.nonNull(allOf)) {
+ ofItems = allOf.stream().flatMap(this::processItem);
+ }
+
+ if (Objects.nonNull(anyOf)) {
+ ofItems = Stream.concat(Optional.ofNullable(ofItems).orElseGet(Stream::empty), anyOf.stream().flatMap(this::processItem));
+ }
+
+ if (Objects.nonNull(oneOf)) {
+ ofItems = Stream.concat(Optional.ofNullable(ofItems).orElseGet(Stream::empty), oneOf.stream().flatMap(this::processItem));
+ }
+
+ return ofItems;
+ }
+
public Stream<Map<String, Object>> processProperties(Map<String, TypeProperty> properties){
return properties.entrySet().stream().flatMap(this::processPropertyEntry);
}
@@ -118,11 +141,10 @@ public class PropertiesProcessor {
return Stream.empty();
}
- if (Objects.nonNull(entry.getValue().getAllOf())) {
- PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey()
- , log, new SchemaConverterPropertiesConfig());
+ Stream<Map<String, Object>> ofItems = processOfItems(entry);
- return entry.getValue().getAllOf().stream().flatMap(propertiesProcessor::processItem);
+ if (Objects.nonNull(ofItems)) {
+ return ofItems;
}
if (Objects.nonNull(entry.getValue().getProperties())) {
@@ -139,6 +161,35 @@ public class PropertiesProcessor {
return storageSchemaEntry(getTypeByDefinitionProperty(entry.getValue()), pathPrefixWithDot + entry.getKey());
}
+ private Stream<Map<String, Object>> processOfItems(Map.Entry<String, TypeProperty> entry) {
+ Stream<Map<String, Object>> ofItems = null;
+
+ if (Objects.nonNull(entry.getValue().getAllOf())) {
+ PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey()
+ , log, new SchemaConverterPropertiesConfig());
+
+ ofItems = entry.getValue().getAllOf().stream().flatMap(propertiesProcessor::processItem);
+ }
+
+ if (Objects.nonNull(entry.getValue().getAnyOf())) {
+ PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey()
+ , log, new SchemaConverterPropertiesConfig());
+
+ ofItems = Stream.concat(Optional.ofNullable(ofItems).orElseGet(Stream::empty),
+ entry.getValue().getAnyOf().stream().flatMap(propertiesProcessor::processItem));
+ }
+
+ if (Objects.nonNull(entry.getValue().getOneOf())) {
+ PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey()
+ , log, new SchemaConverterPropertiesConfig());
+
+ ofItems = Stream.concat(Optional.ofNullable(ofItems).orElseGet(Stream::empty),
+ entry.getValue().getOneOf().stream().flatMap(propertiesProcessor::processItem));
+ }
+
+ return ofItems;
+ }
+
private Stream<Map<String, Object>> storageSchemaEntry(String kind, String path) {
Preconditions.checkNotNullOrEmpty(kind, "kind cannot be null or empty");
Preconditions.checkNotNullOrEmpty(path, "path cannot be null or empty");
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java
index 1dc63193b3db8c761f492165f9363390a709bdca..f613d9f9c77e433f62a333b4688ad47b8e67a245 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java
@@ -97,6 +97,18 @@ public class SchemaToStorageFormatImpl implements SchemaToStorageFormat {
.collect(Collectors.toList()));
}
+ if (schemaData.getAnyOf() != null) {
+ storageSchemaItems.addAll(schemaServiceSchema.getProperties().getData().getAnyOf().stream()
+ .flatMap(propertiesProcessor::processItem)
+ .collect(Collectors.toList()));
+ }
+
+ if (schemaData.getOneOf() != null) {
+ storageSchemaItems.addAll(schemaServiceSchema.getProperties().getData().getOneOf().stream()
+ .flatMap(propertiesProcessor::processItem)
+ .collect(Collectors.toList()));
+ }
+
if (schemaData.getRef() != null) {
storageSchemaItems.addAll(propertiesProcessor.processRef(schemaData.getRef())
.collect(Collectors.toList()));
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/readme.md b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/readme.md
index 845325962f475bd6c02e3999d02a32b43b7fafba..3b5e146c5760cd3084651a612fae228d5a80c035 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/readme.md
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/readme.md
@@ -299,4 +299,67 @@ Ignored for now (array of references)
```
-\"kind\": \"long\"
\ No newline at end of file
+\"kind\": \"long\"
+
+Processing specifics
+----------------------------------------------------------------------------
+
+allOf, anyOf and oneOf tags are processed at the same way. All internal data(properties) are included into converted schema.
+
+For instance
+```json
+{
+ "definitions": {
+ "wellboreData1": {
+ "properties": {
+ "prop1": {
+ "type": "string"
+ }
+ }
+ },
+ "wellboreData2": {
+ "properties": {
+ "prop2": {
+ "type": "string"
+ }
+ }
+ }
+ },
+ "properties": {
+ "data": {
+ "allOf": [
+ {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/wellboreData1"
+ } ],
+ "oneOf": [
+ {
+ "$ref": "#/definitions/wellboreData2"
+ }
+ ]
+ }
+ ]
+ }
+ }
+}
+
+```
+
+is converted to
+
+```json
+{
+ "kind": "KIND_VAL",
+ "schema": [
+ {
+ "kind": "string",
+ "path": "prop1"
+ },
+ {
+ "kind": "string",
+ "path": "prop2"
+ }
+ ]
+}
+```
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/AllOfItem.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/AllOfItem.java
index bef9a14bc622a148ddb9d2603914015d1d1bad73..1354c3bd1b1dde40a19214ae1f3d5e56298f5b2a 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/AllOfItem.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/AllOfItem.java
@@ -27,4 +27,6 @@ public class AllOfItem {
private String type;
private Map<String, TypeProperty> properties;
private List<AllOfItem> allOf;
+ private List<AllOfItem> oneOf;
+ private List<AllOfItem> anyOf;
}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/Definition.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/Definition.java
index be8292a287a15885b64f7da6b4770451b5b4d088..4665c6a82c796dc96ec209f30e9257788b70b8f0 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/Definition.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/Definition.java
@@ -23,4 +23,6 @@ import java.util.Map;
public class Definition {
private Map<String, TypeProperty> properties;
private List<AllOfItem> allOf;
+ private List<AllOfItem> oneOf;
+ private List<AllOfItem> anyOf;
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/PropertiesData.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/PropertiesData.java
index 6c09c0116b6fbe269d93c13931a46493950fbc9a..61f1edb98ab5aa76438ece4eee8c0d15cac85fc1 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/PropertiesData.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/PropertiesData.java
@@ -23,6 +23,8 @@ import java.util.Map;
@Data
public class PropertiesData {
private List<AllOfItem> allOf;
+ private List<AllOfItem> oneOf;
+ private List<AllOfItem> anyOf;
@JsonProperty("$ref")
private String ref;
private Map<String, TypeProperty> properties;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java
index 3053e5d20a85ee5165d5436147b9e5c9a1bafc37..b929bad43f401e97fbc4f5b3ce8fd22344e14a76 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java
@@ -30,4 +30,6 @@ public class TypeProperty {
private Items items;
private Map<String, TypeProperty> properties;
private List<AllOfItem> allOf;
+ private List<AllOfItem> oneOf;
+ private List<AllOfItem> anyOf;
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java
index 9e9026c815055600e3b2a58da8e2c82317e812cd..fae0b4e6f31cd18ad960b841b6d9723e0162cf29 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java
@@ -208,6 +208,16 @@ public class AttributeParsingServiceImpl implements IAttributeParsingService {
}
}
+ @Override
+ public void tryParseNested(String recordId, String name, Object value, Map<String, Object> dataMap) {
+ dataMap.put(name,value);
+ }
+
+ @Override
+ public void tryParseObject(String recordId, String name, Object value, Map<String, Object> dataMap) {
+ dataMap.put(name,value);
+ }
+
private List<String> isArrayType(Object attributeVal) {
try {
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java
index 4050444a40b1bef8f184d8b09c80a078d9e73729..c4fb8ecf2966c3b354eb15e9fb32be8608554feb 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java
@@ -1,7 +1,5 @@
package org.opengroup.osdu.indexer.service;
-import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
-
import java.util.Map;
public interface IAttributeParsingService {
@@ -26,4 +24,8 @@ public interface IAttributeParsingService {
void tryParseGeopoint(String recordId, String attributeName, Map<String, Object> storageRecordData, Map<String, Object> dataMap);
void tryParseGeojson(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap);
+
+ void tryParseNested(String recordId, String name, Object value, Map<String, Object> dataMap);
+
+ void tryParseObject(String recordId, String name, Object value, Map<String, Object> dataMap);
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
index 113dc54bde6c9e4bf11e2fc36ab1725d0be77b72..a63848140947c2a425c69458671733c3beaae684 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
@@ -23,6 +23,7 @@ import org.elasticsearch.client.RestHighLevelClient;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.RequestStatus;
+import org.opengroup.osdu.core.common.model.indexer.ElasticType;
import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
import org.opengroup.osdu.core.common.model.indexer.OperationType;
import org.opengroup.osdu.core.common.model.indexer.StorageType;
@@ -47,6 +48,10 @@ import java.util.Map;
public class IndexSchemaServiceImpl implements IndexSchemaService {
private static final String FLATTENED_SCHEMA = "_flattened";
+ private static final String WELLBORE_MARKER_SET = "WellboreMarkerSet";
+ private static final String MARKERS = "Markers";
+ private static final String WELL_LOG = "WellLog";
+ private static final String CURVES = "Curves";
private final Gson gson = new Gson();
@@ -221,6 +226,7 @@ public class IndexSchemaServiceImpl implements IndexSchemaService {
meta.put(RecordMetaAttribute.TYPE.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.TYPE));
meta.put(RecordMetaAttribute.ACL.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.ACL));
meta.put(RecordMetaAttribute.X_ACL.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.X_ACL));
+ meta.put(RecordMetaAttribute.TAGS.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.TAGS));
meta.put(RecordMetaAttribute.LEGAL.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.LEGAL));
meta.put(RecordMetaAttribute.ANCESTRY.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.ANCESTRY));
meta.put(RecordMetaAttribute.INDEX_STATUS.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.INDEX_STATUS));
@@ -228,6 +234,14 @@ public class IndexSchemaServiceImpl implements IndexSchemaService {
String kind = schemaObj.getKind();
String type = kind.split(":")[2];
+ //TODO temporary fix for https://community.opengroup.org/osdu/platform/system/indexer-service/-/issues/1
+ if(data.get(MARKERS) != null){
+ data.put(MARKERS, ElasticType.NESTED.getValue());
+ }
+ if(data.get(CURVES) != null){
+ data.put(CURVES, ElasticType.NESTED.getValue());
+ }
+
return IndexSchema.builder().dataSchema(data).metaSchema(meta).kind(kind).type(type).build();
} catch (Exception e) {
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
index 95f1c504f9eca6e03027fd4e14e8389fb8cb61bc..acd87b30e4a909d57832758b5260aec100421d41 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
@@ -15,12 +15,6 @@
package org.opengroup.osdu.indexer.service;
import com.google.gson.Gson;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-import javax.inject.Inject;
import org.apache.http.HttpStatus;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
@@ -38,28 +32,29 @@ import org.elasticsearch.index.reindex.UpdateByQueryRequest;
import org.opengroup.osdu.core.common.Constants;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.indexer.DEAnalyzerType;
-import org.opengroup.osdu.core.common.model.indexer.ElasticType;
import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
-import org.opengroup.osdu.core.common.model.indexer.Records;
-import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
import org.opengroup.osdu.core.common.search.Preconditions;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
import org.opengroup.osdu.indexer.util.ElasticClientHandler;
+import org.opengroup.osdu.indexer.util.TypeMapper;
import org.springframework.stereotype.Service;
+import javax.inject.Inject;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
@Service
public class IndexerMappingServiceImpl extends MappingServiceImpl implements IndexerMappingService {
- @Inject
- private IndexerConfigurationProperties configurationProperties;
@Inject
private JaxRsDpsLog log;
@Inject
private ElasticClientHandler elasticClientHandler;
private TimeValue REQUEST_TIMEOUT = TimeValue.timeValueMinutes(1);
-
+
/**
* Create a new type in Elasticsearch
*
@@ -81,7 +76,7 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
*
* @param schema Index schema
* @param type Mapping type
- * @return String JSON represnetation of type and elastic type
+ * @return String JSON representation of type and elastic type
*
* sample index mapping:
* "properties": {
@@ -111,25 +106,14 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
// meta attribute
Map<String, Object> metaMapping = new HashMap<>();
for (Map.Entry<String, Object> entry : schema.getMetaSchema().entrySet()) {
- String key = entry.getKey();
- if (key.equals(RecordMetaAttribute.ACL.getValue()) || key.equals(RecordMetaAttribute.LEGAL.getValue()) || key.equals(RecordMetaAttribute.ANCESTRY.getValue()) || key.equals(RecordMetaAttribute.INDEX_STATUS.getValue())) {
- metaMapping.put(key, entry.getValue());
- } else {
- metaMapping.put(key, Records.Type.builder().type(entry.getValue().toString()).build());
- }
+ metaMapping.put(entry.getKey(), TypeMapper.getMetaAttributeIndexerMapping(entry.getKey()));
}
// data-source attributes
Map<String, Object> dataMapping = new HashMap<>();
if (schema.getDataSchema() != null) {
for (Map.Entry<String, String> entry : schema.getDataSchema().entrySet()) {
- // Apply de_indexer_analyzer and de_search_analyzer to TEXT field
- if (configurationProperties.isPreDemo() && ElasticType.TEXT.getValue().equalsIgnoreCase(entry.getValue())) {
- log.info(String.format("indexing %s with custom analyzer", entry.getKey()));
- dataMapping.put(entry.getKey(), Records.Analyzer.builder().type(entry.getValue()).analyzer(DEAnalyzerType.INDEXER_ANALYZER.getValue()).search_analyzer(DEAnalyzerType.SEARCH_ANALYZER.getValue()).build());
- } else {
- dataMapping.put(entry.getKey(), Records.Type.builder().type(entry.getValue()).build());
- }
+ dataMapping.put(entry.getKey(), TypeMapper.getDataAttributeIndexerMapping(entry.getValue()));
}
// inner properties.data.properties block
@@ -159,7 +143,7 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
}
}
}
-
+
private boolean updateMappingToEnableKeywordIndexingForField(RestHighLevelClient client, Set<String> indicesSet, String fieldName) throws IOException {
String[] indices = indicesSet.toArray(new String[indicesSet.size()]);
Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> indexMappingMap = getIndexFieldMap(new String[]{"data."+fieldName}, client, indices);
@@ -190,7 +174,7 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
}
}
}
-
+
return indexMappingMap;
} catch (ElasticsearchException e) {
log.error(String.format("Failed to get indices: %s. | Error: %s", Arrays.toString(indices), e));
@@ -198,7 +182,9 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
}
}
- private boolean updateMappingForAllIndicesOfSameTypeToEnableKeywordIndexingForField(RestHighLevelClient client, String index, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> indexMapping, String fieldName) throws IOException {
+ private boolean updateMappingForAllIndicesOfSameTypeToEnableKeywordIndexingForField(
+ RestHighLevelClient client, String index, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> indexMapping, String fieldName) throws IOException {
+
PutMappingRequest request = new PutMappingRequest(index);
String type = indexMapping.keySet().iterator().next();
if(type.isEmpty()) {
@@ -219,7 +205,7 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
log.error(String.format("Could not find field: %s in the mapping of index: %s.", fieldName, index));
return false;
}
-
+
//Index the field with additional keyword type
Map<String, Object> keywordMap = new HashMap<>();
keywordMap.put(Constants.TYPE, "keyword");
@@ -235,22 +221,22 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
data.put(Constants.DATA,mapping);
Map<String, Object> properties = new HashMap<>();
properties.put(Constants.PROPERTIES, data);
-
+
request.source(new Gson().toJson(properties), XContentType.JSON);
try {
AcknowledgedResponse response = client.indices().putMapping(request, RequestOptions.DEFAULT);
boolean isIndicesUpdated = updateIndices(client, index);
return response.isAcknowledged() && isIndicesUpdated;
-
+
} catch (Exception e) {
log.error(String.format("Could not update mapping of index: %s. | Error: %s", index, e));
return false;
}
}
-
+
private boolean updateIndices(RestHighLevelClient client, String index) throws IOException {
- UpdateByQueryRequest request = new UpdateByQueryRequest(index);
+ UpdateByQueryRequest request = new UpdateByQueryRequest(index);
request.setConflicts("proceed");
BulkByScrollResponse response = client.updateByQuery(request, RequestOptions.DEFAULT);
if(!response.getBulkFailures().isEmpty()) {
@@ -259,7 +245,7 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
}
return true;
}
-
+
/**
* Create a new type in Elasticsearch
*
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
index 6bd98690beaf036862e0f8fdfe5aea2d35b29a3e..af1e5aea27f0b3846c68ba76a0514fa2ae714c22 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
@@ -16,6 +16,7 @@ package org.opengroup.osdu.indexer.service;
import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
import org.apache.http.HttpStatus;
import org.elasticsearch.ElasticsearchStatusException;
import org.elasticsearch.action.bulk.BulkItemResponse;
@@ -47,6 +48,7 @@ import org.opengroup.osdu.indexer.util.ElasticClientHandler;
import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
import org.apache.commons.beanutils.PropertyUtils;
import org.apache.commons.beanutils.NestedNullException;
+import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Service;
import javax.inject.Inject;
@@ -58,13 +60,14 @@ import java.util.logging.Level;
import java.util.stream.Collectors;
@Service
+@Primary
public class IndexerServiceImpl implements IndexerService {
private static final TimeValue BULK_REQUEST_TIMEOUT = TimeValue.timeValueMinutes(1);
private static final List<RestStatus> RETRY_ELASTIC_EXCEPTION = new ArrayList<>(Arrays.asList(RestStatus.TOO_MANY_REQUESTS, RestStatus.BAD_GATEWAY, RestStatus.SERVICE_UNAVAILABLE));
- private final Gson gson = new Gson();
+ private final Gson gson = new GsonBuilder().serializeNulls().create();
@Inject
private JaxRsDpsLog jaxRsDpsLog;
@@ -308,6 +311,9 @@ public class IndexerServiceImpl implements IndexerService {
document.setVersion(storageRecord.getVersion());
document.setAcl(storageRecord.getAcl());
document.setLegal(storageRecord.getLegal());
+ if (storageRecord.getTags() != null) {
+ document.setTags(storageRecord.getTags());
+ }
RecordStatus recordStatus = this.jobStatus.getJobStatusByRecordId(storageRecord.getId());
if (recordStatus.getIndexProgress().getStatusCode() == 0) {
recordStatus.getIndexProgress().setStatusCode(HttpStatus.SC_OK);
@@ -462,6 +468,7 @@ public class IndexerServiceImpl implements IndexerService {
indexerPayload.put(RecordMetaAttribute.TYPE.getValue(), record.getType());
indexerPayload.put(RecordMetaAttribute.VERSION.getValue(), record.getVersion());
indexerPayload.put(RecordMetaAttribute.ACL.getValue(), record.getAcl());
+ indexerPayload.put(RecordMetaAttribute.TAGS.getValue(), record.getTags());
indexerPayload.put(RecordMetaAttribute.X_ACL.getValue(), Acl.flattenAcl(record.getAcl()));
indexerPayload.put(RecordMetaAttribute.LEGAL.getValue(), record.getLegal());
indexerPayload.put(RecordMetaAttribute.INDEX_STATUS.getValue(), record.getIndexProgress());
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java
index 0f3d5c4f18db650813ca096878925dc49a153921..70fa630e6ca33621db37c7b4f1a15344925fb3e1 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java
@@ -16,7 +16,10 @@ package org.opengroup.osdu.indexer.service;
import com.google.common.base.Strings;
import com.google.gson.Gson;
+
import java.util.Objects;
+
+import lombok.SneakyThrows;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.http.AppException;
@@ -53,74 +56,70 @@ public class ReindexServiceImpl implements ReindexService {
@Inject
private JaxRsDpsLog jaxRsDpsLog;
+ @SneakyThrows
@Override
public String reindexRecords(RecordReindexRequest recordReindexRequest, boolean forceClean) {
Long initialDelayMillis = 0l;
- try {
- DpsHeaders headers = this.requestInfo.getHeadersWithDwdAuthZ();
- if (forceClean) {
- this.indexSchemaService.syncIndexMappingWithStorageSchema(recordReindexRequest.getKind());
- initialDelayMillis = 30000l;
- }
+ DpsHeaders headers = this.requestInfo.getHeadersWithDwdAuthZ();
- RecordQueryResponse recordQueryResponse = this.storageService.getRecordsByKind(recordReindexRequest);
+ if (forceClean) {
+ this.indexSchemaService.syncIndexMappingWithStorageSchema(recordReindexRequest.getKind());
+ initialDelayMillis = 30000l;
+ }
- if (recordQueryResponse.getResults() != null && recordQueryResponse.getResults().size() != 0) {
+ RecordQueryResponse recordQueryResponse = this.storageService.getRecordsByKind(recordReindexRequest);
- List<RecordInfo> msgs = recordQueryResponse.getResults().stream()
- .map(record -> RecordInfo.builder().id(record).kind(recordReindexRequest.getKind()).op(OperationType.create.name()).build()).collect(Collectors.toList());
+ if (recordQueryResponse.getResults() != null && recordQueryResponse.getResults().size() != 0) {
- Map<String, String> attributes = new HashMap<>();
- attributes.put(DpsHeaders.ACCOUNT_ID, headers.getAccountId());
- attributes.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
- attributes.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
+ List<RecordInfo> msgs = recordQueryResponse.getResults().stream()
+ .map(record -> RecordInfo.builder().id(record).kind(recordReindexRequest.getKind()).op(OperationType.create.name()).build()).collect(Collectors.toList());
- Gson gson = new Gson();
- RecordChangedMessages recordChangedMessages = RecordChangedMessages.builder().data(gson.toJson(msgs)).attributes(attributes).build();
- String recordChangedMessagePayload = gson.toJson(recordChangedMessages);
- this.indexerQueueTaskBuilder.createWorkerTask(recordChangedMessagePayload, initialDelayMillis, headers);
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(DpsHeaders.ACCOUNT_ID, headers.getAccountId());
+ attributes.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
+ attributes.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
- // don't call reindex-worker endpoint if it's the last batch
- // previous storage query result size will be less then requested (limit param)
- if (!Strings.isNullOrEmpty(recordQueryResponse.getCursor()) && recordQueryResponse.getResults().size() == configurationProperties.getStorageRecordsBatchSize()) {
- String newPayLoad = gson.toJson(RecordReindexRequest.builder().cursor(recordQueryResponse.getCursor()).kind(recordReindexRequest.getKind()).build());
- this.indexerQueueTaskBuilder.createReIndexTask(newPayLoad, initialDelayMillis, headers);
- return newPayLoad;
- }
+ Gson gson = new Gson();
+ RecordChangedMessages recordChangedMessages = RecordChangedMessages.builder().data(gson.toJson(msgs)).attributes(attributes).build();
+ String recordChangedMessagePayload = gson.toJson(recordChangedMessages);
+ this.indexerQueueTaskBuilder.createWorkerTask(recordChangedMessagePayload, initialDelayMillis, headers);
- return recordChangedMessagePayload;
- } else {
- jaxRsDpsLog.info(String.format("kind: %s cannot be re-indexed, storage service cannot locate valid records", recordReindexRequest.getKind()));
+ // don't call reindex-worker endpoint if it's the last batch
+ // previous storage query result size will be less then requested (limit param)
+ if (!Strings.isNullOrEmpty(recordQueryResponse.getCursor()) && recordQueryResponse.getResults().size() == configurationProperties.getStorageRecordsBatchSize()) {
+ String newPayLoad = gson.toJson(RecordReindexRequest.builder().cursor(recordQueryResponse.getCursor()).kind(recordReindexRequest.getKind()).build());
+ this.indexerQueueTaskBuilder.createReIndexTask(newPayLoad, initialDelayMillis, headers);
+ return newPayLoad;
}
- return null;
- } catch (AppException e) {
- throw e;
- } catch (Exception e) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Unknown error", "An unknown error has occurred.", e);
+
+ return recordChangedMessagePayload;
+ } else {
+ jaxRsDpsLog.info(String.format("kind: %s cannot be re-indexed, storage service cannot locate valid records", recordReindexRequest.getKind()));
}
+ return null;
}
- @Override
- public void fullReindex(boolean forceClean) {
- List<String> allKinds = null;
- try {
- allKinds = storageService.getAllKinds();
- } catch (Exception e) {
- jaxRsDpsLog.error("storage service all kinds request failed",e);
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "storage service cannot respond with all kinds", "an unknown error has occurred.", e);
- }
- if (Objects.isNull(allKinds)){
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "storage service cannot respond with all kinds", "full reindex failed");
- }
- for (String kind : allKinds) {
- try {
- reindexRecords(new RecordReindexRequest(kind, ""), forceClean);
- } catch (Exception e) {
- jaxRsDpsLog.warning(String.format("kind: %s cannot be re-indexed", kind));
- continue;
- }
- }
- }
+ @Override
+ public void fullReindex(boolean forceClean) {
+ List<String> allKinds = null;
+ try {
+ allKinds = storageService.getAllKinds();
+ } catch (Exception e) {
+ jaxRsDpsLog.error("storage service all kinds request failed", e);
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "storage service cannot respond with all kinds", "an unknown error has occurred.", e);
+ }
+ if (Objects.isNull(allKinds)) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "storage service cannot respond with all kinds", "full reindex failed");
+ }
+ for (String kind : allKinds) {
+ try {
+ reindexRecords(new RecordReindexRequest(kind, ""), forceClean);
+ } catch (Exception e) {
+ jaxRsDpsLog.warning(String.format("kind: %s cannot be re-indexed", kind));
+ continue;
+ }
+ }
+ }
}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java
index fba62bb8ac5cd08549da9e7814720df38e526651..6d9d8b8cde2bf7664b1a84e2bc7a14c7551e0db3 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java
@@ -37,10 +37,10 @@ public class StorageIndexerPayloadMapper {
Object value = getPropertyValue(recordId, storageRecordData, name);
- if (value == null) continue;
-
ElasticType elasticType = ElasticType.forValue(entry.getValue());
+ if (value == null && !nullIndexedValueSupported(elasticType)) continue;
+
switch (elasticType) {
case KEYWORD:
case KEYWORD_ARRAY:
@@ -88,10 +88,14 @@ public class StorageIndexerPayloadMapper {
this.attributeParsingService.tryParseGeopoint(recordId, name, storageRecordData, dataMap);
break;
case GEO_SHAPE:
- this.attributeParsingService.tryParseGeojson(recordId, name, value, dataMap);
+ this.attributeParsingService.tryParseGeojson(recordId, name, value, dataMap);
break;
case NESTED:
+ this.attributeParsingService.tryParseNested(recordId, name, value, dataMap);
+ break;
case OBJECT:
+ this.attributeParsingService.tryParseObject(recordId, name, value, dataMap);
+ break;
case UNDEFINED:
// don't do anything for now
break;
@@ -123,4 +127,8 @@ public class StorageIndexerPayloadMapper {
}
return null;
}
+
+ private boolean nullIndexedValueSupported(ElasticType type) {
+ return type == ElasticType.TEXT;
+ }
}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/impl/SchemaProviderImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/impl/SchemaProviderImpl.java
index 70ef41dd6cef5e0c6687ad38f2fb1de8b6a08ffe..2323aeecdf9218aa1fa6eef5610d96803100d17c 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/impl/SchemaProviderImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/impl/SchemaProviderImpl.java
@@ -32,6 +32,7 @@ import java.io.UnsupportedEncodingException;
import java.net.URISyntaxException;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
+import java.util.Objects;
/**
* Provides implementation of the client service that retrieves schemas from the Schema Service
@@ -58,13 +59,9 @@ public class SchemaProviderImpl implements SchemaService {
@Override
public String getSchema(String kind) throws URISyntaxException, UnsupportedEncodingException {
- String schemaFromStorageService = getFromStorageService(kind);
+ String schemaServiceSchema = getFromSchemaService(kind);
- if (schemaFromStorageService != null) {
- return schemaFromStorageService;
- }
-
- return getFromSchemaService(kind);
+ return Objects.nonNull(schemaServiceSchema) ? schemaServiceSchema : getFromStorageService(kind);
}
protected String getFromSchemaService(String kind) throws UnsupportedEncodingException, URISyntaxException {
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java
index 2f7c9b7c0e9b3051d6a428fec359be7418f35f93..9e3150abc4006549f51f8ed4b2375ff522c4184a 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java
@@ -22,6 +22,7 @@ import org.opengroup.osdu.core.common.model.indexer.StorageType;
import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
import org.apache.commons.lang3.StringUtils;
+
import java.util.HashMap;
import java.util.Map;
@@ -31,6 +32,8 @@ public class TypeMapper {
private static final Map<String, Object> metaAttributeIndexerType = new HashMap<>();
+ private static final String STORAGE_TYPE_OBJECTS = "[]object";
+
static {
metaAttributeIndexerType.put(RecordMetaAttribute.KIND.getValue(), ElasticType.KEYWORD.getValue());
@@ -40,6 +43,7 @@ public class TypeMapper {
metaAttributeIndexerType.put(RecordMetaAttribute.VERSION.getValue(), ElasticType.LONG.getValue());
metaAttributeIndexerType.put(RecordMetaAttribute.X_ACL.getValue(), ElasticType.KEYWORD.getValue());
metaAttributeIndexerType.put(RecordMetaAttribute.ACL.getValue(), getAclIndexerMapping());
+ metaAttributeIndexerType.put(RecordMetaAttribute.TAGS.getValue(), ElasticType.FLATTENED.getValue());
metaAttributeIndexerType.put(RecordMetaAttribute.LEGAL.getValue(), getLegalIndexerMapping());
metaAttributeIndexerType.put(RecordMetaAttribute.ANCESTRY.getValue(), getAncestryIndexerMapping());
metaAttributeIndexerType.put(RecordMetaAttribute.INDEX_STATUS.getValue(), getIndexStatusMapping());
@@ -62,24 +66,50 @@ public class TypeMapper {
storageToIndexerType.put(StorageType.DATETIME_ARRAY.getValue(), ElasticType.DATE_ARRAY.getValue());
storageToIndexerType.put(StorageType.GEO_POINT.getValue(), ElasticType.GEO_POINT.getValue());
storageToIndexerType.put(StorageType.GEO_SHAPE.getValue(), ElasticType.GEO_SHAPE.getValue());
- }
+ //TODO temporary fix for https://community.opengroup.org/osdu/platform/system/indexer-service/-/issues/1
+ storageToIndexerType.put(STORAGE_TYPE_OBJECTS, ElasticType.OBJECT.getValue());
+ }
public static String getIndexerType(String storageType) {
- String indexedType = storageToIndexerType.getOrDefault(storageType, null);
- if (indexedType != null && indexedType.endsWith("_array")) {
- return StringUtils.substringBefore(indexedType, "_");
- }
- return indexedType;
+ return storageToIndexerType.getOrDefault(storageType, null);
}
public static Object getIndexerType(RecordMetaAttribute attribute) {
return metaAttributeIndexerType.getOrDefault(attribute.getValue(), null);
}
+ public static Object getMetaAttributeIndexerMapping(String key) {
+ if (key.equals(RecordMetaAttribute.ACL.getValue())
+ || key.equals(RecordMetaAttribute.LEGAL.getValue()) || key.equals(RecordMetaAttribute.ANCESTRY.getValue()) || key.equals(RecordMetaAttribute.INDEX_STATUS.getValue())) {
+ return metaAttributeIndexerType.get(key);
+ }
+ return Records.Type.builder().type(metaAttributeIndexerType.get(key).toString()).build();
+ }
+
+ public static Object getDataAttributeIndexerMapping(String indexerType) {
+ if (ElasticType.TEXT.getValue().equalsIgnoreCase(indexerType)) {
+ return getTextIndexerMapping();
+ }
+
+ if (isArray(indexerType)) {
+ return Records.Type.builder().type(getArrayMemberType(indexerType)).build();
+ }
+
+ return Records.Type.builder().type(indexerType).build();
+ }
+
+ private static boolean isArray(String indexerType) {
+ return indexerType != null && indexerType.endsWith("_array");
+ }
+
+ private static String getArrayMemberType(String indexerType) {
+ return StringUtils.substringBefore(indexerType, "_");
+ }
+
private static Object getAclIndexerMapping() {
Map<String, Object> aclRoleMapping = new HashMap<>();
- aclRoleMapping.put(AclRole.VIEWERS.getValue() , Records.Type.builder().type(ElasticType.KEYWORD.getValue()).build());
+ aclRoleMapping.put(AclRole.VIEWERS.getValue(), Records.Type.builder().type(ElasticType.KEYWORD.getValue()).build());
aclRoleMapping.put(AclRole.OWNERS.getValue(), Records.Type.builder().type(ElasticType.KEYWORD.getValue()).build());
Map<String, Object> aclProperties = new HashMap<>();
@@ -121,4 +151,22 @@ public class TypeMapper {
return indexStatusProperties;
}
+
+ private static Object getTextIndexerMapping() {
+ Map<String, Object> fieldIndexTypeMap = getKeywordMap();
+ Map<String, Object> textMap = new HashMap<>();
+ textMap.put("type", "text");
+ textMap.put("fields", fieldIndexTypeMap);
+ return textMap;
+ }
+
+ private static Map<String, Object> getKeywordMap() {
+ Map<String, Object> keywordMap = new HashMap<>();
+ keywordMap.put("type", "keyword");
+ keywordMap.put("ignore_above", 256);
+ keywordMap.put("null_value", "null");
+ Map<String, Object> fieldIndexTypeMap = new HashMap<>();
+ fieldIndexTypeMap.put("keyword", keywordMap);
+ return fieldIndexTypeMap;
+ }
}
\ No newline at end of file
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImplTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImplTest.java
index c90092879cf6047a76bb03a0f876abb59e40d358..07642442dc178ce37cf934fdfbdbf7a0d044171e 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImplTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImplTest.java
@@ -94,6 +94,21 @@ public class SchemaToStorageFormatImplTest {
testSingleFile("/converter/tags/allOf/indefinitions.json", KIND);
}
+ @Test
+ public void oneOfInDefinitions() {
+ testSingleFile("/converter/tags/oneOf/indefinitions.json", KIND);
+ }
+
+ @Test
+ public void anyOfInDefinitions() {
+ testSingleFile("/converter/tags/anyOf/indefinitions.json", KIND);
+ }
+
+ @Test
+ public void mixAllAnyOneOf() {
+ testSingleFile("/converter/tags/mixAllAnyOneOf/mix.json", KIND);
+ }
+
@Test
public void folderPassed() throws URISyntaxException, IOException {
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/impl/SchemaProviderImplTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/impl/SchemaProviderImplTest.java
index a3c5b46e73f938c8b5824fe484a7aa993919d5a0..870f2b76c7178ff6430521f3b90ebf2d39c16d55 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/impl/SchemaProviderImplTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/impl/SchemaProviderImplTest.java
@@ -77,7 +77,7 @@ public class SchemaProviderImplTest {
Assert.assertEquals("{\n" +
" \"kind\" : \"fake\",\n" +
" \"schema\" : [ ]\n" +
- "}", schema);
+ "}", schema.replaceAll("\r", ""));
}
@@ -136,7 +136,7 @@ public class SchemaProviderImplTest {
}
@Test
- public void should_call_Storage_then_Schema() throws Exception {
+ public void should_call_Schema_then_Storage() throws Exception {
String kind = "tenant:test:test:1.0.0";
SchemaProviderImpl schemaService = Mockito.mock(SchemaProviderImpl.class);
@@ -148,20 +148,20 @@ public class SchemaProviderImplTest {
assertNull(recordSchemaResponse);
inOrder.verify(schemaService).getSchema(any());
- inOrder.verify(schemaService).getFromStorageService(any());
inOrder.verify(schemaService).getFromSchemaService(any());
+ inOrder.verify(schemaService).getFromStorageService(any());
verify(schemaService, times(1)).getFromStorageService(any());
verify(schemaService, times(1)).getFromSchemaService(any());
}
@Test
- public void should_call_only_Storage_if_it_returns_result() throws Exception {
+ public void should_call_only_SchemaService_if_it_returns_result() throws Exception {
String kind = "tenant:test:test:1.0.0";
SchemaProviderImpl schemaService = Mockito.mock(SchemaProviderImpl.class);
when(schemaService.getSchema(any())).thenCallRealMethod();
String someSchema = "some schema";
- when(schemaService.getFromStorageService(any())).thenReturn(someSchema);
+ when(schemaService.getFromSchemaService(any())).thenReturn(someSchema);
InOrder inOrder = inOrder(schemaService);
@@ -169,9 +169,9 @@ public class SchemaProviderImplTest {
assertEquals(recordSchemaResponse, someSchema);
inOrder.verify(schemaService).getSchema(any());
- inOrder.verify(schemaService).getFromStorageService(any());
- verify(schemaService, times(0)).getFromSchemaService(any());
- verify(schemaService, times(1)).getFromStorageService(any());
+ inOrder.verify(schemaService).getFromSchemaService(any());
+ verify(schemaService, times(1)).getFromSchemaService(any());
+ verify(schemaService, times(0)).getFromStorageService(any());
}
}
\ No newline at end of file
diff --git a/indexer-core/src/test/resources/converter/integration-tests/index_records_1.schema b/indexer-core/src/test/resources/converter/integration-tests/index_records_1.schema
index 672c3d965d50194ac5c80adecfddbeb24d51a856..779d2d20921f97aa86bed06943358d76c3d5f71f 100644
--- a/indexer-core/src/test/resources/converter/integration-tests/index_records_1.schema
+++ b/indexer-core/src/test/resources/converter/integration-tests/index_records_1.schema
@@ -20,7 +20,11 @@
},
"County": {
"type": "string"
- },
+ }}}],
+ "anyOf": [
+ {
+ "type": "object",
+ "properties": {
"State": {
"type": "string"
},
@@ -41,7 +45,11 @@
},
"EmptyAttribute": {
"type": "string"
- },
+ }}}],
+ "oneOf": [
+ {
+ "type": "object",
+ "properties": {
"Rank": {
"type": "integer"
},
diff --git a/indexer-core/src/test/resources/converter/tags/anyOf/indefinitions.json b/indexer-core/src/test/resources/converter/tags/anyOf/indefinitions.json
new file mode 100644
index 0000000000000000000000000000000000000000..4733286c10bd2d081ae85fa44ee22d499a86908d
--- /dev/null
+++ b/indexer-core/src/test/resources/converter/tags/anyOf/indefinitions.json
@@ -0,0 +1,29 @@
+{
+ "definitions": {
+ "wellboreData1": {
+ "properties": {
+ "prop1": {
+ "type": "string"
+ }
+ }
+ },
+ "wellboreData2": {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/wellboreData1"
+ }
+ ]
+ }
+ },
+ "properties": {
+ "data": {
+ "type": "object",
+ "properties": {
+ "Field": {
+ "$ref": "#/definitions/wellboreData2"
+ }
+ }
+ }
+ }
+}
+
diff --git a/indexer-core/src/test/resources/converter/tags/anyOf/indefinitions.json.res b/indexer-core/src/test/resources/converter/tags/anyOf/indefinitions.json.res
new file mode 100644
index 0000000000000000000000000000000000000000..e17ce90d066d023859f6b91d7756717b494a7a45
--- /dev/null
+++ b/indexer-core/src/test/resources/converter/tags/anyOf/indefinitions.json.res
@@ -0,0 +1,9 @@
+{
+ "kind": "KIND_VAL",
+ "schema": [
+ {
+ "kind": "string",
+ "path": "Field.prop1"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/indexer-core/src/test/resources/converter/tags/mixAllAnyOneOf/mix.json b/indexer-core/src/test/resources/converter/tags/mixAllAnyOneOf/mix.json
new file mode 100644
index 0000000000000000000000000000000000000000..2eb9de93a9911ad6407366a362fa0604219f7361
--- /dev/null
+++ b/indexer-core/src/test/resources/converter/tags/mixAllAnyOneOf/mix.json
@@ -0,0 +1,57 @@
+{
+ "definitions": {
+ "wellboreData1": {
+ "properties": {
+ "prop1": {
+ "type": "string"
+ }
+ }
+ },
+ "wellboreData2": {
+ "properties": {
+ "prop2": {
+ "type": "string"
+ }
+ }
+ },
+ "wellboreData3": {
+ "properties": {
+ "prop3": {
+ "type": "string"
+ }
+ }
+ },
+ "wellboreData4": {
+ "properties": {
+ "prop4": {
+ "type": "string"
+ }
+ }
+ }
+ },
+ "properties": {
+ "data": {
+ "allOf": [
+ {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/wellboreData1"
+ } ],
+ "oneOf": [
+ {
+ "$ref": "#/definitions/wellboreData2"
+ }
+ ]
+ },
+ {
+ "$ref": "#/definitions/wellboreData3"
+ },
+ {
+ "$ref": "#/definitions/wellboreData4"
+ }
+ ]
+ }
+ }
+}
+
+
diff --git a/indexer-core/src/test/resources/converter/tags/mixAllAnyOneOf/mix.json.res b/indexer-core/src/test/resources/converter/tags/mixAllAnyOneOf/mix.json.res
new file mode 100644
index 0000000000000000000000000000000000000000..22a8e6a806ba17e51adbd690f0ef56ac4f9f7bbe
--- /dev/null
+++ b/indexer-core/src/test/resources/converter/tags/mixAllAnyOneOf/mix.json.res
@@ -0,0 +1,21 @@
+{
+ "kind": "KIND_VAL",
+ "schema": [
+ {
+ "kind": "string",
+ "path": "prop1"
+ },
+ {
+ "kind": "string",
+ "path": "prop2"
+ },
+ {
+ "kind": "string",
+ "path": "prop3"
+ },
+ {
+ "kind": "string",
+ "path": "prop4"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/indexer-core/src/test/resources/converter/tags/oneOf/indefinitions.json b/indexer-core/src/test/resources/converter/tags/oneOf/indefinitions.json
new file mode 100644
index 0000000000000000000000000000000000000000..4733286c10bd2d081ae85fa44ee22d499a86908d
--- /dev/null
+++ b/indexer-core/src/test/resources/converter/tags/oneOf/indefinitions.json
@@ -0,0 +1,29 @@
+{
+ "definitions": {
+ "wellboreData1": {
+ "properties": {
+ "prop1": {
+ "type": "string"
+ }
+ }
+ },
+ "wellboreData2": {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/wellboreData1"
+ }
+ ]
+ }
+ },
+ "properties": {
+ "data": {
+ "type": "object",
+ "properties": {
+ "Field": {
+ "$ref": "#/definitions/wellboreData2"
+ }
+ }
+ }
+ }
+}
+
diff --git a/indexer-core/src/test/resources/converter/tags/oneOf/indefinitions.json.res b/indexer-core/src/test/resources/converter/tags/oneOf/indefinitions.json.res
new file mode 100644
index 0000000000000000000000000000000000000000..e17ce90d066d023859f6b91d7756717b494a7a45
--- /dev/null
+++ b/indexer-core/src/test/resources/converter/tags/oneOf/indefinitions.json.res
@@ -0,0 +1,9 @@
+{
+ "kind": "KIND_VAL",
+ "schema": [
+ {
+ "kind": "string",
+ "path": "Field.prop1"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index d24aad221589fd26fd867d4c1b9d8bd03f4d2c24..e3fc3a633fbaa1562563d65295ebf0172dd3c535 100644
--- a/pom.xml
+++ b/pom.xml
@@ -19,7 +19,7 @@
<java.version>1.8</java.version>
<springfox-version>2.7.0</springfox-version>
<spring-cloud.version>Greenwich.SR2</spring-cloud.version>
- <os-core-common.version>0.3.19</os-core-common.version>
+ <os-core-common.version>0.6.5</os-core-common.version>
<snakeyaml.version>1.26</snakeyaml.version>
<hibernate-validator.version>6.1.5.Final</hibernate-validator.version>
<jackson.version>2.11.2</jackson.version>
diff --git a/provider/indexer-azure/pom.xml b/provider/indexer-azure/pom.xml
index f67a7fcbb3ab51a12eafb5120b8a56ffb4e64a16..9be2778ddc36ecfa9daf5725570c360a02f2657d 100644
--- a/provider/indexer-azure/pom.xml
+++ b/provider/indexer-azure/pom.xml
@@ -42,7 +42,7 @@
<indexer-core.version>0.6.0-SNAPSHOT</indexer-core.version>
<spring-security-jwt.version>1.1.1.RELEASE</spring-security-jwt.version>
<osdu.corelibazure.version>0.0.42</osdu.corelibazure.version>
- <osdu.oscorecommon.version>0.3.12</osdu.oscorecommon.version>
+ <osdu.oscorecommon.version>0.6.5</osdu.oscorecommon.version>
<reactor-netty.version>0.9.12.RELEASE</reactor-netty.version>
<java-jwt.version>3.8.1</java-jwt.version>
<powermock.version>2.0.2</powermock.version>
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/config/AzureBootstrapConfig.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/config/AzureBootstrapConfig.java
index 570b932a29f49f8ba849488ed55d9e2f61d440ce..7be75c3be0c99a9f076f36f422b303e948aa035a 100644
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/config/AzureBootstrapConfig.java
+++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/config/AzureBootstrapConfig.java
@@ -19,6 +19,8 @@ import org.opengroup.osdu.azure.KeyVaultFacade;
import org.opengroup.osdu.core.common.entitlements.EntitlementsAPIConfig;
import org.opengroup.osdu.core.common.entitlements.EntitlementsFactory;
import org.opengroup.osdu.core.common.entitlements.IEntitlementsFactory;
+import org.opengroup.osdu.core.common.http.json.HttpResponseBodyMapper;
+import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Component;
@@ -90,12 +92,15 @@ public class AzureBootstrapConfig {
return String.format(urlFormat, tenant);
}
+ @Autowired
+ private HttpResponseBodyMapper httpResponseBodyMapper;
+
@Bean
public IEntitlementsFactory entitlementsFactory() {
EntitlementsAPIConfig apiConfig = EntitlementsAPIConfig.builder()
.apiKey(entitlementsAPIKey)
.rootUrl(entitlementsAPIEndpoint)
.build();
- return new EntitlementsFactory(apiConfig);
+ return new EntitlementsFactory(apiConfig, httpResponseBodyMapper);
}
}
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java
index 555dbc3964e0fe7e8308b329e98656310d13d78b..7210b4aaa06bb72b607ae31353835ca4353a19d1 100644
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java
+++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java
@@ -14,15 +14,23 @@
package org.opengroup.osdu.indexer.azure.util;
+import com.google.common.base.Strings;
import com.google.common.reflect.TypeToken;
import com.google.gson.*;
import com.microsoft.azure.servicebus.Message;
import lombok.extern.java.Log;
+import org.apache.http.HttpStatus;
import org.opengroup.osdu.azure.servicebus.ITopicClientFactory;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.indexer.OperationType;
import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
+import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
+import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
+import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.opengroup.osdu.indexer.service.StorageService;
import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Primary;
@@ -31,11 +39,14 @@ import org.springframework.web.context.annotation.RequestScope;
import javax.inject.Inject;
import javax.inject.Named;
+import java.lang.reflect.Type;
import java.nio.charset.StandardCharsets;
+import java.util.stream.Collectors;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.lang.reflect.Type;
+
@Log
@Component
@@ -46,6 +57,9 @@ public class IndexerQueueTaskBuilderAzure extends IndexerQueueTaskBuilder {
@Autowired
private ITopicClientFactory topicClientFactory;
+ @Inject
+ private IndexerConfigurationProperties configurationProperties;
+
@Inject
private JaxRsDpsLog logger;
@@ -53,14 +67,59 @@ public class IndexerQueueTaskBuilderAzure extends IndexerQueueTaskBuilder {
@Named("SERVICE_BUS_TOPIC")
private String serviceBusTopic;
+ @Inject
+ private StorageService storageService;
+
+
@Override
public void createWorkerTask(String payload, Long countdownMillis, DpsHeaders headers) {
+ headers.addCorrelationIdIfMissing();
createTask(payload, headers);
}
@Override
public void createReIndexTask(String payload, Long countdownMillis, DpsHeaders headers) {
- createTask(payload, headers);
+ headers.addCorrelationIdIfMissing();
+ publishAllRecordsToServiceBus(payload, headers);
+ }
+
+ private void publishAllRecordsToServiceBus(String payload, DpsHeaders headers) {
+ // fetch all the remaining records
+ // This logic is temporary and would be updated to call the storage service async.
+ // Currently the storage client can't be called out of request scope hence making the
+ // storage calls sync here
+ Gson gson = new Gson();
+ RecordReindexRequest recordReindexRequest = gson.fromJson(payload, RecordReindexRequest.class);
+ final String recordKind = recordReindexRequest.getKind();
+ RecordQueryResponse recordQueryResponse = null;
+
+ try {
+ do {
+ if (recordQueryResponse != null) {
+ recordReindexRequest = RecordReindexRequest.builder().cursor(recordQueryResponse.getCursor()).kind(recordKind).build();
+ }
+ recordQueryResponse = this.storageService.getRecordsByKind(recordReindexRequest);
+ if (recordQueryResponse.getResults() != null && recordQueryResponse.getResults().size() != 0) {
+
+ List<RecordInfo> records = recordQueryResponse.getResults().stream()
+ .map(record -> RecordInfo.builder().id(record).kind(recordKind).op(OperationType.create.name()).build()).collect(Collectors.toList());
+
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(DpsHeaders.ACCOUNT_ID, headers.getPartitionIdWithFallbackToAccountId());
+ attributes.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
+ attributes.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
+
+ RecordChangedMessages recordChangedMessages = RecordChangedMessages.builder().data(gson.toJson(records)).attributes(attributes).build();
+ String recordChangedMessagePayload = gson.toJson(recordChangedMessages);
+ createTask(recordChangedMessagePayload, headers);
+ }
+ } while (!Strings.isNullOrEmpty(recordQueryResponse.getCursor()) && recordQueryResponse.getResults().size() == configurationProperties.getStorageRecordsBatchSize());
+
+ } catch (AppException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Unknown error", "An unknown error has occurred.", e);
+ }
}
private void createTask(String payload, DpsHeaders headers) {
diff --git a/provider/indexer-gcp/src/main/resources/application.properties b/provider/indexer-gcp/src/main/resources/application.properties
index c2c74189250e40afae31e9f8c0174370a877c569..08ee586c371bc8fecc4e21227ef2b4f1c8b76c5e 100644
--- a/provider/indexer-gcp/src/main/resources/application.properties
+++ b/provider/indexer-gcp/src/main/resources/application.properties
@@ -43,3 +43,4 @@ indexer.que.service.mail=default@iam.gserviceaccount.com
SCHEMA_HOST=${HOST}/api/schema-service/v1/schema
storage-query-kinds-host=https://${STORAGE_HOSTNAME}/api/storage/v2/query/kinds
+schema.converter.supported-array-types=boolean,integer,number,string,object
diff --git a/provider/indexer-reference/pom.xml b/provider/indexer-reference/pom.xml
index 64212d38f47c19bfb2de3205a4b1fbf5617d84d5..92013da703d4dd0dba9b70f22d87437055328002 100644
--- a/provider/indexer-reference/pom.xml
+++ b/provider/indexer-reference/pom.xml
@@ -48,7 +48,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
- <version>0.0.18</version>
+ <version>0.6.5</version>
</dependency>
<dependency>
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/EntitlementsClientFactory.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/EntitlementsClientFactory.java
index 45131176cd1ff35eb2dc411b0073b245c0d3b8e8..2872c5c53405e12550c901064da117764208dc4f 100644
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/EntitlementsClientFactory.java
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/EntitlementsClientFactory.java
@@ -20,6 +20,7 @@ package org.opengroup.osdu.indexer.di;
import org.opengroup.osdu.core.common.entitlements.EntitlementsAPIConfig;
import org.opengroup.osdu.core.common.entitlements.EntitlementsFactory;
import org.opengroup.osdu.core.common.entitlements.IEntitlementsFactory;
+import org.opengroup.osdu.core.common.http.json.HttpResponseBodyMapper;
import org.opengroup.osdu.indexer.config.EntitlementsConfigProperties;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.config.AbstractFactoryBean;
@@ -34,12 +35,15 @@ public class EntitlementsClientFactory extends AbstractFactoryBean<IEntitlements
private EntitlementsConfigProperties entitlementsConfigProperties;
+ @Autowired
+ private HttpResponseBodyMapper mapper;
+
@Override
protected IEntitlementsFactory createInstance() throws Exception {
return new EntitlementsFactory(EntitlementsAPIConfig
.builder()
.rootUrl(entitlementsConfigProperties.getAuthorizeApi())
- .build());
+ .build(), mapper);
}
@Override
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
index 558e30bcc6d7c6653f51f1d023a56b6f015a5c18..1be2ba81868c6f3ed56cd4b56a25edfdbd576f99 100644
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
@@ -68,7 +68,6 @@ import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Service;
@Service
-@Primary
public class IndexerServiceImpl implements IndexerService {
private static final TimeValue BULK_REQUEST_TIMEOUT = TimeValue.timeValueMinutes(1);
diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java
index 9ec5fb7a30044f7f105c9a97229cb37322f218b9..bfc4f3a3330570ea73c59d2721954dd75aded7a5 100644
--- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java
+++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java
@@ -51,30 +51,17 @@ public class IndexFilterTest {
indexFilter.doFilter(httpServletRequest, httpServletResponse, filterChain);
- Mockito.verify(httpServletResponse)
- .addHeader("Access-Control-Allow-Origin", Collections.singletonList("*").toString());
- Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Headers", Collections
- .singletonList(
- "origin, content-type, accept, authorization, data-partition-id, correlation-id, appkey")
- .toString());
- Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Methods",
- Collections.singletonList("GET, POST, PUT, DELETE, OPTIONS, HEAD").toString());
- Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Credentials",
- Collections.singletonList("true").toString());
- Mockito.verify(httpServletResponse)
- .addHeader("X-Frame-Options", Collections.singletonList("DENY").toString());
- Mockito.verify(httpServletResponse)
- .addHeader("X-XSS-Protection", Collections.singletonList("1; mode=block").toString());
- Mockito.verify(httpServletResponse)
- .addHeader("X-Content-Type-Options", Collections.singletonList("nosniff").toString());
- Mockito.verify(httpServletResponse).addHeader("Cache-Control",
- Collections.singletonList("no-cache, no-store, must-revalidate").toString());
- Mockito.verify(httpServletResponse).addHeader("Content-Security-Policy",
- Collections.singletonList("default-src 'self'").toString());
- Mockito.verify(httpServletResponse).addHeader("Strict-Transport-Security",
- Collections.singletonList("max-age=31536000; includeSubDomains").toString());
- Mockito.verify(httpServletResponse)
- .addHeader("Expires", Collections.singletonList("0").toString());
+ Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Origin", Collections.singletonList("*").toString());
+ Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Headers", Collections.singletonList("origin, content-type, accept, authorization, data-partition-id, correlation-id, appkey").toString());
+ Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Methods", Collections.singletonList("GET, POST, PUT, DELETE, OPTIONS, HEAD, PATCH").toString());
+ Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Credentials", Collections.singletonList("true").toString());
+ Mockito.verify(httpServletResponse).addHeader("X-Frame-Options", Collections.singletonList("DENY").toString());
+ Mockito.verify(httpServletResponse).addHeader("X-XSS-Protection", Collections.singletonList("1; mode=block").toString());
+ Mockito.verify(httpServletResponse).addHeader("X-Content-Type-Options", Collections.singletonList("nosniff").toString());
+ Mockito.verify(httpServletResponse).addHeader("Cache-Control", Collections.singletonList("no-cache, no-store, must-revalidate").toString());
+ Mockito.verify(httpServletResponse).addHeader("Content-Security-Policy", Collections.singletonList("default-src 'self'").toString());
+ Mockito.verify(httpServletResponse).addHeader("Strict-Transport-Security", Collections.singletonList("max-age=31536000; includeSubDomains").toString());
+ Mockito.verify(httpServletResponse).addHeader("Expires", Collections.singletonList("0").toString());
Mockito.verify(httpServletResponse).addHeader("correlation-id", "correlation-id-value");
Mockito.verify(filterChain).doFilter(httpServletRequest, httpServletResponse);
}
diff --git a/testing/indexer-test-aws/pom.xml b/testing/indexer-test-aws/pom.xml
index 9335cf4e8d5323f6ad973d3114af68a272a27220..8bcbc9c60ebf6dbc60127d0f9ff3a23aa0861b39 100644
--- a/testing/indexer-test-aws/pom.xml
+++ b/testing/indexer-test-aws/pom.xml
@@ -42,7 +42,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.6.1-SNAPSHOT</version>
</dependency>
<!-- AWS specific packages -->
diff --git a/testing/indexer-test-azure/pom.xml b/testing/indexer-test-azure/pom.xml
index 2a6703142879b538f3f01319953cb9aa18833633..ca33be462b91e0915c092ac963730b5ff2780c44 100644
--- a/testing/indexer-test-azure/pom.xml
+++ b/testing/indexer-test-azure/pom.xml
@@ -45,14 +45,32 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.6.1-SNAPSHOT</version>
</dependency>
<!-- Azure dependencies -->
+ <!--Added dependency for utilizing azure service principle class-->
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-azure</artifactId>
<version>0.0.33</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>com.azure</groupId>
+ <artifactId>azure-core-http-netty</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ <version>4.1.50.Final</version>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport</artifactId>
+ <version>4.1.15.Final</version>
</dependency>
<dependency>
@@ -166,28 +184,5 @@
<version>3.0.0</version>
<scope>test</scope>
</dependency>
-<!--Added dependency for utilizing azure service principle class-->
- <dependency>
- <groupId>org.opengroup.osdu</groupId>
- <artifactId>core-lib-azure</artifactId>
- <version>0.0.33</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>com.azure</groupId>
- <artifactId>azure-core-http-netty</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-all</artifactId>
- <version>4.1.50.Final</version>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-transport</artifactId>
- <version>4.1.15.Final</version>
- </dependency>
</dependencies>
</project>
\ No newline at end of file
diff --git a/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
index 5cc775b83d14dabb626710768d74ac837783822e..8721cee890fb340474355453f55b71b266b74215 100644
--- a/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
+++ b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -64,4 +64,8 @@ public class Steps extends SchemaServiceRecordSteps {
super.iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(expectedCount, index, skippedAttributes);
}
+ @Then("^I should be able to search (\\d+) record with index \"([^\"]*)\" by tag \"([^\"]*)\" and value \"([^\"]*)\"$")
+ public void iShouldBeAbleToSearchRecordByTagKeyAndTagValue(int expectedNumber, String index, String tagKey, String tagValue) throws Throwable {
+ super.iShouldBeAbleToSearchRecordByTagKeyAndTagValue(index, tagKey, tagValue, expectedNumber);
+ }
}
\ No newline at end of file
diff --git a/testing/indexer-test-core/pom.xml b/testing/indexer-test-core/pom.xml
index 37659e5ba08a4f640dc0d678e5e453128833ba1c..952d7852a09876684b17d7041474ff20277195e1 100644
--- a/testing/indexer-test-core/pom.xml
+++ b/testing/indexer-test-core/pom.xml
@@ -11,7 +11,7 @@
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.6.1-SNAPSHOT</version>
<properties>
<maven.compiler.target>1.8</maven.compiler.target>
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
index c6ebab631a5e46f0d47ae2814cd36e1d412d4b7c..cb3dc70a9da5a688f90033627d507e9af631761a 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
@@ -22,6 +22,7 @@ import java.lang.reflect.Type;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import static org.junit.Assert.*;
@@ -140,6 +141,13 @@ public class RecordSteps extends TestsBase {
assertEquals(expectedCount, documentCountByQuery);
}
+ public void iShouldBeAbleToSearchRecordByTagKeyAndTagValue(String index, String tagKey, String tagValue, int expectedNumber) throws Throwable {
+ TimeUnit.SECONDS.sleep(40);
+ index = generateActualName(index, timeStamp);
+ long actualNumberOfRecords = elasticUtils.fetchRecordsByTags(index, tagKey, tagValue);
+ assertEquals(expectedNumber, actualNumberOfRecords);
+ }
+
private long createIndex(String index) throws InterruptedException, IOException {
long numOfIndexedDocuments = 0;
int iterator;
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
index a63b339ce9421eadddd7d5f5515d33833aef5ff3..df99970ee7ebf1ba09fb3c9ba7b7fdd97fb3bd25 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
@@ -62,8 +62,12 @@ import java.util.ArrayList;
import java.util.Base64;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
+import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
+import static org.elasticsearch.index.query.QueryBuilders.termsQuery;
+
/**
* All util methods to use elastic apis for tests
@@ -217,8 +221,25 @@ public class ElasticUtils {
}
}
- public long fetchRecordsByExistQuery(String index, String attributeName) throws IOException {
+ public long fetchRecordsByTags(String index, String tagKey, String tagValue) throws IOException {
+ try {
+ try (RestHighLevelClient client = this.createClient(username, password, host)) {
+ SearchRequest request = new SearchRequest(index);
+ SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
+ sourceBuilder.query(boolQuery().must(termsQuery(String.format("tags.%s", tagKey), tagValue)));
+ request.source(sourceBuilder);
+ SearchResponse searchResponse = client.search(request, RequestOptions.DEFAULT);
+ return searchResponse.getHits().getTotalHits().value;
+ }
+ } catch (ElasticsearchStatusException e) {
+ log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage()));
+ return -1;
+ }
+ }
+
+ public long fetchRecordsByExistQuery(String index, String attributeName) throws Exception {
try {
+ TimeUnit.SECONDS.sleep(40);
try (RestHighLevelClient client = this.createClient(username, password, host)) {
SearchRequest searchRequest = new SearchRequest(index);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
diff --git a/testing/indexer-test-core/src/main/resources/features/indexrecord/IndexRecord.feature b/testing/indexer-test-core/src/main/resources/features/indexrecord/IndexRecord.feature
index c281ec4a0b174ec98eb781f8e603eba3b3968ca1..a06fa2efcbcb9107b184934c0304bf105afbb85e 100644
--- a/testing/indexer-test-core/src/main/resources/features/indexrecord/IndexRecord.feature
+++ b/testing/indexer-test-core/src/main/resources/features/indexrecord/IndexRecord.feature
@@ -15,8 +15,8 @@ Feature: Indexing of the documents
Examples:
| kind | recordFile | number | index | type | acl | mapping |
- | "tenant1:testindex<timestamp>:well:1.0.0" | "index_records_1" | 5 | "tenant1-testindex<timestamp>-well-1.0.0" | "well" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
- | "tenant1:testindex<timestamp>:well:3.0.0" | "index_records_1" | 5 | "tenant1-testindex<timestamp>-well-3.0.0" | "well" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
+ | "tenant1:testindex<timestamp>:well:1.0.0" | "index_records_1" | 5 | "tenant1-testindex<timestamp>-well-1.0.0" | "well" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Country":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"County":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"EmptyAttribute":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Established":{"type":"date"},"Field":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellName":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellStatus":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellType":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
+ | "tenant1:testindex<timestamp>:well:3.0.0" | "index_records_1" | 5 | "tenant1-testindex<timestamp>-well-3.0.0" | "well" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Country":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"County":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"EmptyAttribute":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Established":{"type":"date"},"Field":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellName":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellStatus":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellType":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
Scenario Outline: Ingest the record and Index in the Elastic Search with bad attribute
When I ingest records with the <recordFile> with <acl> for a given <kind>
diff --git a/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature b/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
index 4152a53252458bd809ae9ffe5da447cf302060d7..11d3ec6443448db6bd6723bed233212b91b89061 100644
--- a/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
+++ b/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
@@ -15,8 +15,8 @@ Feature: Indexing of the documents
Examples:
| kind | recordFile | number | index | type | acl | mapping |
- | "tenant1:indexer-int-test:sample-schema-1:1.0.4" | "index_records_1" | 5 | "tenant1-indexer-int-test-sample-schema-1-1.0.4" | "sample-schema-1" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
- | "tenant1:indexer-int-test:sample-schema-3:1.0.4" | "index_records_1" | 5 | "tenant1-indexer-int-test-sample-schema-3-1.0.4" | "sample-schema-3" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
+ | "tenant1:indexer-int-test:sample-schema-1:1.0.4" | "index_records_schema_1" | 5 | "tenant1-indexer-int-test-sample-schema-1-1.0.4" | "sample-schema-1" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Country":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"County":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"EmptyAttribute":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Established":{"type":"date"},"Field":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellName":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellStatus":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellType":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
+ | "tenant1:indexer-int-test:sample-schema-3:1.0.4" | "index_records_schema_1" | 5 | "tenant1-indexer-int-test-sample-schema-3-1.0.4" | "sample-schema-3" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Country":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"County":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"EmptyAttribute":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Established":{"type":"date"},"Field":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellName":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellStatus":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellType":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
Scenario Outline: Ingest the record and Index in the Elastic Search with bad attribute
When I ingest records with the <recordFile> with <acl> for a given <kind>
@@ -26,3 +26,11 @@ Feature: Indexing of the documents
| kind | recordFile | number | index | skippedAttribute | acl |
| "tenant1:indexer-int-test:sample-schema-2:1.0.4" | "index_records_2" | 4 | "tenant1-indexer-int-test-sample-schema-2-1.0.4" | "data.Location" | "data.default.viewers@tenant1" |
| "tenant1:indexer-int-test:sample-schema-3:1.0.4" | "index_records_3" | 7 | "tenant1-indexer-int-test-sample-schema-3-1.0.4" | "data.GeoShape" | "data.default.viewers@tenant1" |
+
+ Scenario Outline: Ingest the record and Index in the Elastic Search with tags
+ When I ingest records with the <recordFile> with <acl> for a given <kind>
+ Then I should be able to search <number> record with index <index> by tag <tagKey> and value <tagValue>
+
+ Examples:
+ | kind | recordFile | index | acl | tagKey | tagValue | number |
+ | "tenant1:indexer-int-test:sample-schema-1:1.0.4" | "index_records_1" | "tenant1-indexer-int-test-sample-schema-1-1.0.4" | "data.default.viewers@tenant1" | "testtag" | "testvalue" | 5 |
diff --git a/testing/indexer-test-core/src/main/resources/testData/index_records_1.json b/testing/indexer-test-core/src/main/resources/testData/index_records_1.json
index dc83212c545f7478280566be828726f4b0558234..80ac1c787f7cec412a9bd8d9a49985e448aac997 100644
--- a/testing/indexer-test-core/src/main/resources/testData/index_records_1.json
+++ b/testing/indexer-test-core/src/main/resources/testData/index_records_1.json
@@ -1,6 +1,9 @@
[
{
"id": "tenant1:<kindSubType>:testIngest2<timestamp>",
+ "tags": {
+ "testtag": "testvalue"
+ },
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
@@ -24,6 +27,9 @@
},
{
"id": "tenant1:<kindSubType>:testIngest3<timestamp>",
+ "tags": {
+ "testtag": "testvalue"
+ },
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
@@ -47,6 +53,9 @@
},
{
"id": "tenant1:<kindSubType>:testIngest4<timestamp>",
+ "tags": {
+ "testtag": "testvalue"
+ },
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
@@ -69,6 +78,9 @@
},
{
"id": "tenant1:<kindSubType>:testIngest5<timestamp>",
+ "tags": {
+ "testtag": "testvalue"
+ },
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
@@ -91,6 +103,9 @@
},
{
"id": "tenant1:<kindSubType>:testIngest6<timestamp>",
+ "tags": {
+ "testtag": "testvalue"
+ },
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
diff --git a/testing/indexer-test-core/src/main/resources/testData/index_records_1.schema.json b/testing/indexer-test-core/src/main/resources/testData/index_records_1.schema.json
index 2a6e6301a41b6b4f8d82839c2faca994336a46a3..9e54b33862a2f0b8d81277f79e2959b3b141e7bf 100644
--- a/testing/indexer-test-core/src/main/resources/testData/index_records_1.schema.json
+++ b/testing/indexer-test-core/src/main/resources/testData/index_records_1.schema.json
@@ -4,7 +4,7 @@
"authority": "tenant1",
"source": "indexer-int-test",
"entityType": "sample-schema-1",
- "schemaVersionMajor": "1",
+ "schemaVersionMajor": "2",
"schemaVersionMinor": "0",
"schemaVersionPatch": "4"
},
@@ -32,7 +32,14 @@
},
"County": {
"type": "string"
- },
+ }
+ }
+ }
+ ],
+ "anyOf": [
+ {
+ "type": "object",
+ "properties": {
"State": {
"type": "string"
},
@@ -53,7 +60,14 @@
},
"EmptyAttribute": {
"type": "string"
- },
+ }
+ }
+ }
+ ],
+ "oneOf": [
+ {
+ "type": "object",
+ "properties": {
"Rank": {
"type": "integer"
},
diff --git a/testing/indexer-test-gcp/pom.xml b/testing/indexer-test-gcp/pom.xml
index 74f56c7ace3750aa7165602b6e19c394229ed3a3..8c0c97e0735515c96320b3f5e0049cad728c8564 100644
--- a/testing/indexer-test-gcp/pom.xml
+++ b/testing/indexer-test-gcp/pom.xml
@@ -37,7 +37,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.6.1-SNAPSHOT</version>
</dependency>
<!-- Cucumber -->
diff --git a/testing/indexer-test-ibm/pom.xml b/testing/indexer-test-ibm/pom.xml
index f85b8b8ad1a2d3216ac246e5fe580596d3cf6e35..f515de2aea26efda6e0fedd394d478a17643ba30 100644
--- a/testing/indexer-test-ibm/pom.xml
+++ b/testing/indexer-test-ibm/pom.xml
@@ -38,7 +38,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.6.1-SNAPSHOT</version>
</dependency>
<dependency>