diff --git a/indexer-core/pom.xml b/indexer-core/pom.xml index 08b4a00a5edbe2f94fac86ab2c2b253dae7ad9f6..ba48b3cc7976a73c3b69b4433ca9110e91f03562 100644 --- a/indexer-core/pom.xml +++ b/indexer-core/pom.xml @@ -16,7 +16,7 @@ 1.9.4 - 0.6.9 + 0.9.0-SNAPSHOT diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java index a0a71a3370fad6b291ca35b42497d241d50ecb0b..04a3211fda1a6a91a71fc7cea639951b41b90642 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java @@ -14,7 +14,18 @@ package org.opengroup.osdu.indexer.schema.converter; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.http.HttpStatus; +import org.opengroup.osdu.core.common.Constants; import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.search.Preconditions; import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterConfig; @@ -23,16 +34,14 @@ import org.opengroup.osdu.indexer.schema.converter.exeption.SchemaProcessingExce import org.opengroup.osdu.indexer.schema.converter.tags.AllOfItem; import org.opengroup.osdu.indexer.schema.converter.tags.Definition; import org.opengroup.osdu.indexer.schema.converter.tags.Definitions; +import org.opengroup.osdu.indexer.schema.converter.tags.Items; import org.opengroup.osdu.indexer.schema.converter.tags.TypeProperty; -import java.util.*; -import java.util.function.Supplier; -import java.util.stream.Stream; - public class PropertiesProcessor { private SchemaConverterConfig schemaConverterConfig; + private static final String TYPE_KEY = "type"; private static final String DEF_PREFIX = "#/definitions/"; private static final String LINK_PREFIX = "^srn"; private static final String LINK_TYPE = "link"; @@ -176,7 +185,13 @@ public class PropertiesProcessor { } if ("array".equals(entry.getValue().getType())) { - if (schemaConverterConfig.getSupportedArrayTypes().contains(entry.getValue().getItems().getType())) { + + Items items = entry.getValue().getItems(); + if(Objects.nonNull(items) && items.isComplexTypeItems()){ + return processComplexTypeItems(entry, items); + } + + if (schemaConverterConfig.getSupportedArrayTypes().contains(entry.getValue().getItems().getType()) && !items.isComplexTypeItems()) { return storageSchemaEntry("[]" + getTypeByDefinitionProperty(entry.getValue()), pathPrefixWithDot + entry.getKey()); } @@ -213,6 +228,36 @@ public class PropertiesProcessor { } + private Stream> processComplexTypeItems(Entry entry, Items items) { + Map indexHint = entry.getValue().getIndexHint(); + + String indexingType = Objects.isNull(indexHint) ? + schemaConverterConfig.getDefaultObjectArraysType() : + indexHint.getOrDefault(TYPE_KEY,schemaConverterConfig.getDefaultObjectArraysType()); + + if(schemaConverterConfig.getProcessedArraysTypes().contains(indexingType)){ + PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, new SchemaConverterPropertiesConfig()); + + Stream> propertiesStream = Stream.empty(); + + if(Objects.nonNull(items.getProperties())){ + propertiesStream = items.getProperties().entrySet().stream().flatMap(propertiesProcessor::processPropertyEntry); + } + if (Objects.nonNull(items.getRef())){ + propertiesStream = Stream.concat(propertiesStream, propertiesProcessor.processRef(items.getRef())); + } + if(Objects.nonNull(items.getAllOf())){ + propertiesStream = Stream.concat(propertiesStream, items.getAllOf().stream().flatMap(propertiesProcessor::processItem)); + } + return storageSchemaObjectArrayEntry( + indexingType, + entry.getKey(), + propertiesStream); + }else { + return storageSchemaEntry(indexingType, pathPrefixWithDot + entry.getKey()); + } + } + private Stream> processOfItems(Map.Entry entry) { Stream> ofItems = null; @@ -255,6 +300,17 @@ public class PropertiesProcessor { return Stream.of(map); } + private Stream> storageSchemaObjectArrayEntry(String kind, String path,Stream> mapStream) { + Preconditions.checkNotNullOrEmpty(kind, "kind cannot be null or empty"); + Preconditions.checkNotNullOrEmpty(path, "path cannot be null or empty"); + + Map map = new HashMap<>(); + map.put("kind", kind); + map.put("path", path); + map.put(Constants.PROPERTIES,mapStream.collect(Collectors.toList())); + return Stream.of(map); + } + private String getTypeByDefinitionProperty(TypeProperty definitionProperty) { Preconditions.checkNotNull(definitionProperty, "definitionProperty cannot be null"); diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterConfig.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterConfig.java index 539b2bdfd16b67422e680d7b3b58ac64948937e4..f31c28b663b314295389df432156b5dc1af29548 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterConfig.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterConfig.java @@ -11,4 +11,6 @@ public interface SchemaConverterConfig { Set getSupportedArrayTypes(); Map getSpecialDefinitionsMap(); Map getPrimitiveTypesMap(); + Set getProcessedArraysTypes(); + String getDefaultObjectArraysType(); } diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterPropertiesConfig.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterPropertiesConfig.java index 9e5eaff440b905d9bce2db41eff1f69f555e4469..e969f8e31970a105e8ca7becf06b8831baaa5b9d 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterPropertiesConfig.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterPropertiesConfig.java @@ -1,12 +1,15 @@ package org.opengroup.osdu.indexer.schema.converter.config; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; import lombok.Getter; import lombok.Setter; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; -import java.util.*; - @Configuration @ConfigurationProperties(prefix = "schema.converter") @Getter @@ -17,10 +20,13 @@ public class SchemaConverterPropertiesConfig implements SchemaConverterConfig { private Set supportedArrayTypes = getDefaultSupportedArrayTypes(); private Map specialDefinitionsMap = getDefaultSpecialDefinitionsMap(); private Map primitiveTypesMap = getDefaultPrimitiveTypesMap(); + private Set processedArraysTypes = getDefaultArraysTypesForProcessing(); + private String defaultObjectArraysType = getObjectArraysDefaultType(); + private Set getDefaultSkippedDefinitions() { return new HashSet<>(Arrays.asList("AbstractAnyCrsFeatureCollection", - "anyCrsGeoJsonFeatureCollection")); + "anyCrsGeoJsonFeatureCollection")); } private Set getDefaultSupportedArrayTypes() { @@ -51,4 +57,12 @@ public class SchemaConverterPropertiesConfig implements SchemaConverterConfig { return defaultPrimitiveTypesMap; } + + private Set getDefaultArraysTypesForProcessing() { + return new HashSet<>(Arrays.asList("nested")); + } + + private String getObjectArraysDefaultType() { + return "[]object"; + } } diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/readme.md b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/readme.md index 4c39a276469a7a3b43d7c4caf2d78d940fb70a42..17a4ca99a040170dd656acdd655a55d3b6cd3a06 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/readme.md +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/readme.md @@ -203,8 +203,68 @@ Is converted to 2. Arrays -Arrays of complex types are ignored, only following arrays of primitive -types are supported +Arrays of complex types by default will be consumed as object type +```json +"Markers": { +"type": "array", + "items": { + "type": "object", + "properties": { + "NegativeVerticalDelta"{ + "description": "The distance vertically below the Marker position that marks the limit of the high confidence range for the Marker pick.", + "x-osdu-frame-of-reference": "UOM:length", + "type": "number" + }, + ..... + } + } +} + +``` +Without inner objects processing +```json +{ + path = Markers, + kind = []object +} +``` + +Processing can be specified with optional "x-osdu-indexing" property +```json +"properties": { + "Markers": { + "x-osdu-indexing": { + "type": "nested" + }, + "type": "array", + "items": { + "type": "object", + "properties": { + "NegativeVerticalDelta"{ + "description": "The distance vertically below the Marker position that marks the limit of the high confidence range for the Marker pick.", + "x-osdu-frame-of-reference": "UOM:length", + "type": "number" + }, + ..... +``` +"x-osdu-indexing" property values +```json +"nested" , "flattened" +``` +By default, only "nested" type will lead to inner objects processing +```json +{ +path = Markers, + kind = nested, + properties = [{ + path = NegativeVerticalDelta, + kind = double + }, + ..... +} +``` + +Arrays of primitive types are supported ```json "number", "string", "integer", "boolean" @@ -370,4 +430,4 @@ is converted to } ] } -``` +``` \ No newline at end of file diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/Items.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/Items.java index dc3ef69e9ef19c7cdfbfebb48ddedc393067f98f..ed1678cf0238ccb89557bcb8fefc01429b6698ab 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/Items.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/Items.java @@ -14,10 +14,23 @@ package org.opengroup.osdu.indexer.schema.converter.tags; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; +import java.util.Map; +import java.util.Objects; import lombok.Data; @Data public class Items { + @JsonProperty("$ref") + private String ref; + private List allOf; private String type; private String pattern; + private Map properties; + + public boolean isComplexTypeItems(){ + return Objects.nonNull(ref) || Objects.nonNull(allOf) || Objects.nonNull(properties); + } + } diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java index b929bad43f401e97fbc4f5b3ce8fd22344e14a76..d06a47c265e268b2a95322574dd6fcdfecc019ee 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java @@ -15,13 +15,14 @@ package org.opengroup.osdu.indexer.schema.converter.tags; import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - import java.util.List; import java.util.Map; +import lombok.Data; @Data public class TypeProperty { + @JsonProperty("x-osdu-indexing") + private Map indexHint; private String type; private String pattern; private String format; diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java index 733408ab3257219f75899e1e57b6a491abad08d2..66d47186931a086c5cf952da892612da185a6fd5 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java @@ -19,13 +19,18 @@ import com.google.gson.Gson; import com.google.gson.JsonSyntaxException; import com.google.gson.internal.LinkedTreeMap; import com.google.gson.reflect.TypeToken; +import java.lang.reflect.Array; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.function.BiFunction; +import javax.inject.Inject; import org.apache.http.HttpStatus; import org.opengroup.osdu.core.common.model.indexer.ElasticType; -import org.opengroup.osdu.core.common.model.indexer.IndexSchema; import org.opengroup.osdu.core.common.model.indexer.IndexingStatus; import org.opengroup.osdu.core.common.model.indexer.JobStatus; -import org.opengroup.osdu.core.common.Constants; -import org.opengroup.osdu.indexer.model.geojson.FeatureCollection; import org.opengroup.osdu.indexer.util.parser.BooleanParser; import org.opengroup.osdu.indexer.util.parser.DateTimeParser; import org.opengroup.osdu.indexer.util.parser.GeoShapeParser; @@ -33,12 +38,6 @@ import org.opengroup.osdu.indexer.util.parser.NumberParser; import org.springframework.stereotype.Service; import org.springframework.web.context.annotation.RequestScope; -import javax.inject.Inject; -import java.lang.reflect.Array; -import java.lang.reflect.Type; -import java.util.*; -import java.util.function.BiFunction; - @Service @RequestScope public class AttributeParsingServiceImpl implements IAttributeParsingService { @@ -219,6 +218,11 @@ public class AttributeParsingServiceImpl implements IAttributeParsingService { dataMap.put(name,value); } + @Override + public void tryParseFlattened(String recordId, String name, Object value, Map dataMap) { + dataMap.put(name,value); + } + private List isArrayType(Object attributeVal) { try { diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java index c4fb8ecf2966c3b354eb15e9fb32be8608554feb..a658cc6a1f289d5140cabb39d94864d4f173ef00 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java @@ -28,4 +28,6 @@ public interface IAttributeParsingService { void tryParseNested(String recordId, String name, Object value, Map dataMap); void tryParseObject(String recordId, String name, Object value, Map dataMap); + + void tryParseFlattened(String recordId, String name, Object value, Map dataMap); } diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java index a63848140947c2a425c69458671733c3beaae684..095b66a84fd8b3174140d0294012e0aa0c025b5e 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java @@ -16,6 +16,12 @@ package org.opengroup.osdu.indexer.service; import com.google.common.base.Strings; import com.google.gson.Gson; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URISyntaxException; +import java.util.HashMap; +import java.util.Map; +import javax.inject.Inject; import org.apache.http.HttpStatus; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; @@ -26,7 +32,6 @@ import org.opengroup.osdu.core.common.model.http.RequestStatus; import org.opengroup.osdu.core.common.model.indexer.ElasticType; import org.opengroup.osdu.core.common.model.indexer.IndexSchema; import org.opengroup.osdu.core.common.model.indexer.OperationType; -import org.opengroup.osdu.core.common.model.indexer.StorageType; import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute; import org.opengroup.osdu.core.common.model.storage.Schema; import org.opengroup.osdu.core.common.model.storage.SchemaItem; @@ -37,13 +42,6 @@ import org.opengroup.osdu.indexer.util.ElasticClientHandler; import org.opengroup.osdu.indexer.util.TypeMapper; import org.springframework.stereotype.Service; -import javax.inject.Inject; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.net.URISyntaxException; -import java.util.HashMap; -import java.util.Map; - @Service public class IndexSchemaServiceImpl implements IndexSchemaService { @@ -204,15 +202,16 @@ public class IndexSchemaServiceImpl implements IndexSchemaService { if (schemaObj == null) return null; - Map data = new HashMap<>(); + Map data = new HashMap<>(); Map meta = new HashMap<>(); if (schemaObj.getSchema() != null && schemaObj.getSchema().length > 0) { for (SchemaItem schemaItem : schemaObj.getSchema()) { String dataType = schemaItem.getKind(); - String elasticDataType = TypeMapper.getIndexerType(dataType); - if (elasticDataType == null) { - elasticDataType = TypeMapper.getIndexerType(StorageType.STRING.getValue()); + Object elasticDataType = TypeMapper.getIndexerType(dataType, ElasticType.TEXT.getValue()); + if(schemaItem.getProperties() != null){ + HashMap propertiesMap = normalizeInnerProperties(schemaItem); + elasticDataType = TypeMapper.getObjectsArrayMapping(dataType, propertiesMap); } data.put(schemaItem.getPath(), elasticDataType); } @@ -234,14 +233,6 @@ public class IndexSchemaServiceImpl implements IndexSchemaService { String kind = schemaObj.getKind(); String type = kind.split(":")[2]; - //TODO temporary fix for https://community.opengroup.org/osdu/platform/system/indexer-service/-/issues/1 - if(data.get(MARKERS) != null){ - data.put(MARKERS, ElasticType.NESTED.getValue()); - } - if(data.get(CURVES) != null){ - data.put(CURVES, ElasticType.NESTED.getValue()); - } - return IndexSchema.builder().dataSchema(data).metaSchema(meta).kind(kind).type(type).build(); } catch (Exception e) { @@ -249,4 +240,14 @@ public class IndexSchemaServiceImpl implements IndexSchemaService { } } + private HashMap normalizeInnerProperties(SchemaItem schemaItem) { + HashMap propertiesMap = new HashMap<>(); + for (SchemaItem propertiesItem : schemaItem.getProperties()) { + String propertiesItemKind = propertiesItem.getKind(); + String propertiesElasticType = TypeMapper.getIndexerType(propertiesItemKind,ElasticType.TEXT.getValue()); + propertiesMap.put(propertiesItem.getPath(),propertiesElasticType); + } + return propertiesMap; + } + } \ No newline at end of file diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java index acd87b30e4a909d57832758b5260aec100421d41..b70bb211f3f722d048d930e8c5dae437d1de3072 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java @@ -15,6 +15,12 @@ package org.opengroup.osdu.indexer.service; import com.google.gson.Gson; +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import javax.inject.Inject; import org.apache.http.HttpStatus; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.support.master.AcknowledgedResponse; @@ -38,13 +44,6 @@ import org.opengroup.osdu.indexer.util.ElasticClientHandler; import org.opengroup.osdu.indexer.util.TypeMapper; import org.springframework.stereotype.Service; -import javax.inject.Inject; -import java.io.IOException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - @Service public class IndexerMappingServiceImpl extends MappingServiceImpl implements IndexerMappingService { @@ -112,7 +111,7 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind // data-source attributes Map dataMapping = new HashMap<>(); if (schema.getDataSchema() != null) { - for (Map.Entry entry : schema.getDataSchema().entrySet()) { + for (Map.Entry entry : schema.getDataSchema().entrySet()) { dataMapping.put(entry.getKey(), TypeMapper.getDataAttributeIndexerMapping(entry.getValue())); } @@ -131,7 +130,6 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind // don't add dynamic mapping documentMapping.put("dynamic", false); - return documentMapping; } diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java index 6d9d8b8cde2bf7664b1a84e2bc7a14c7551e0db3..e550d4e0949cf2afb6458a341713d3b713106558 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java @@ -1,115 +1,171 @@ package org.opengroup.osdu.indexer.service; +import static org.opengroup.osdu.indexer.service.IAttributeParsingService.DATA_GEOJSON_TAG; +import static org.opengroup.osdu.indexer.service.IAttributeParsingService.RECORD_GEOJSON_TAG; + +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import javax.inject.Inject; import org.apache.commons.beanutils.NestedNullException; import org.apache.commons.beanutils.PropertyUtils; +import org.apache.http.HttpStatus; +import org.opengroup.osdu.core.common.Constants; import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.core.common.model.indexer.ElasticType; import org.opengroup.osdu.core.common.model.indexer.IndexSchema; +import org.opengroup.osdu.core.common.model.indexer.IndexingStatus; +import org.opengroup.osdu.core.common.model.indexer.JobStatus; +import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterConfig; import org.springframework.stereotype.Component; -import javax.inject.Inject; -import java.lang.reflect.InvocationTargetException; -import java.util.Date; -import java.util.HashMap; -import java.util.Map; - -import static org.opengroup.osdu.indexer.service.IAttributeParsingService.DATA_GEOJSON_TAG; -import static org.opengroup.osdu.indexer.service.IAttributeParsingService.RECORD_GEOJSON_TAG; - @Component public class StorageIndexerPayloadMapper { - @Inject - private JaxRsDpsLog log; - @Inject - private IAttributeParsingService attributeParsingService; - - public Map mapDataPayload(IndexSchema storageSchema, Map storageRecordData, String recordId) { - - Map dataMap = new HashMap<>(); - - if (storageSchema.isDataSchemaMissing()) return dataMap; - - // get the key and get the corresponding object from the storageRecord object - for (Map.Entry entry : storageSchema.getDataSchema().entrySet()) { - - String name = entry.getKey(); - - Object value = getPropertyValue(recordId, storageRecordData, name); - - ElasticType elasticType = ElasticType.forValue(entry.getValue()); - - if (value == null && !nullIndexedValueSupported(elasticType)) continue; - - switch (elasticType) { - case KEYWORD: - case KEYWORD_ARRAY: - case TEXT: - case TEXT_ARRAY: - dataMap.put(name, value); - break; - case INTEGER_ARRAY: - this.attributeParsingService.tryParseValueArray(Integer.class, recordId, name, value, dataMap); - break; - case INTEGER: - this.attributeParsingService.tryParseInteger(recordId, name, value, dataMap); - break; - case LONG_ARRAY: - this.attributeParsingService.tryParseValueArray(Long.class, recordId, name, value, dataMap); - break; - case LONG: - this.attributeParsingService.tryParseLong(recordId, name, value, dataMap); - break; - case FLOAT_ARRAY: - this.attributeParsingService.tryParseValueArray(Float.class, recordId, name, value, dataMap); - break; - case FLOAT: - this.attributeParsingService.tryParseFloat(recordId, name, value, dataMap); - break; - case DOUBLE_ARRAY: - this.attributeParsingService.tryParseValueArray(Double.class, recordId, name, value, dataMap); - break; - case DOUBLE: - this.attributeParsingService.tryParseDouble(recordId, name, value, dataMap); - break; - case BOOLEAN_ARRAY: - this.attributeParsingService.tryParseValueArray(Boolean.class, recordId, name, value, dataMap); - break; - case BOOLEAN: - this.attributeParsingService.tryParseBoolean(recordId, name, value, dataMap); - break; - case DATE_ARRAY: - this.attributeParsingService.tryParseValueArray(Date.class, recordId, name, value, dataMap); - break; - case DATE: - this.attributeParsingService.tryParseDate(recordId, name, value, dataMap); - break; - case GEO_POINT: - this.attributeParsingService.tryParseGeopoint(recordId, name, storageRecordData, dataMap); - break; - case GEO_SHAPE: - this.attributeParsingService.tryParseGeojson(recordId, name, value, dataMap); - break; - case NESTED: - this.attributeParsingService.tryParseNested(recordId, name, value, dataMap); - break; - case OBJECT: - this.attributeParsingService.tryParseObject(recordId, name, value, dataMap); - break; - case UNDEFINED: - // don't do anything for now - break; - } - } - - // add these once iterated over the list - storageSchema.getDataSchema().put(DATA_GEOJSON_TAG, ElasticType.GEO_SHAPE.getValue()); - storageSchema.getDataSchema().remove(RECORD_GEOJSON_TAG); - - return dataMap; - } - - private Object getPropertyValue(String recordId, Map storageRecordData, String propertyKey) { + @Inject + private JaxRsDpsLog log; + @Inject + private IAttributeParsingService attributeParsingService; + @Inject + private JobStatus jobStatus; + @Inject + private SchemaConverterConfig schemaConfig; + + public Map mapDataPayload(IndexSchema storageSchema, Map storageRecordData, + String recordId) { + + Map dataCollectorMap = new HashMap<>(); + + if (storageSchema.isDataSchemaMissing()) { + this.log.warning(String.format("record-id: %s | schema mismatching: %s ", recordId, storageSchema.getKind())); + return dataCollectorMap; + } + + mapDataPayload(storageSchema.getDataSchema(), storageRecordData, recordId, dataCollectorMap); + + // add these once iterated over the list + storageSchema.getDataSchema().put(DATA_GEOJSON_TAG, ElasticType.GEO_SHAPE.getValue()); + storageSchema.getDataSchema().remove(RECORD_GEOJSON_TAG); + + return dataCollectorMap; + } + + private Map mapDataPayload(Map dataSchema, Map storageRecordData, + String recordId, Map dataCollectorMap) { + + // get the key and get the corresponding object from the storageRecord object + for (Map.Entry entry : dataSchema.entrySet()) { + String schemaPropertyName = entry.getKey(); + Object storageRecordValue = getPropertyValue(recordId, storageRecordData, schemaPropertyName); + ElasticType elasticType = defineElasticType(entry.getValue()); + + if (Objects.isNull(elasticType)) { + this.jobStatus + .addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, + String.format("record-id: %s | %s for entry %s", recordId, "Not resolvable elastic type", schemaPropertyName)); + continue; + } + + if (schemaConfig.getProcessedArraysTypes().contains(elasticType.getValue().toLowerCase()) && Objects.nonNull(storageRecordValue)) { + processInnerProperties(recordId, dataCollectorMap, entry.getValue(), schemaPropertyName, (List) storageRecordValue); + } + + if (storageRecordValue == null && !nullIndexedValueSupported(elasticType)) { + continue; + } + + switch (elasticType) { + case KEYWORD: + case KEYWORD_ARRAY: + case TEXT: + case TEXT_ARRAY: + dataCollectorMap.put(schemaPropertyName, storageRecordValue); + break; + case INTEGER_ARRAY: + this.attributeParsingService.tryParseValueArray(Integer.class, recordId, schemaPropertyName, storageRecordValue, dataCollectorMap); + break; + case INTEGER: + this.attributeParsingService.tryParseInteger(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap); + break; + case LONG_ARRAY: + this.attributeParsingService.tryParseValueArray(Long.class, recordId, schemaPropertyName, storageRecordValue, dataCollectorMap); + break; + case LONG: + this.attributeParsingService.tryParseLong(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap); + break; + case FLOAT_ARRAY: + this.attributeParsingService.tryParseValueArray(Float.class, recordId, schemaPropertyName, storageRecordValue, dataCollectorMap); + break; + case FLOAT: + this.attributeParsingService.tryParseFloat(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap); + break; + case DOUBLE_ARRAY: + this.attributeParsingService.tryParseValueArray(Double.class, recordId, schemaPropertyName, storageRecordValue, dataCollectorMap); + break; + case DOUBLE: + this.attributeParsingService.tryParseDouble(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap); + break; + case BOOLEAN_ARRAY: + this.attributeParsingService.tryParseValueArray(Boolean.class, recordId, schemaPropertyName, storageRecordValue, dataCollectorMap); + break; + case BOOLEAN: + this.attributeParsingService.tryParseBoolean(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap); + break; + case DATE_ARRAY: + this.attributeParsingService.tryParseValueArray(Date.class, recordId, schemaPropertyName, storageRecordValue, dataCollectorMap); + break; + case DATE: + this.attributeParsingService.tryParseDate(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap); + break; + case GEO_POINT: + this.attributeParsingService.tryParseGeopoint(recordId, schemaPropertyName, storageRecordData, dataCollectorMap); + break; + case GEO_SHAPE: + this.attributeParsingService.tryParseGeojson(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap); + break; + case FLATTENED: + // flattened type inner properties will be added "as is" without parsing as they types not present in schema + this.attributeParsingService.tryParseFlattened(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap); + break; + case OBJECT: + // object type inner properties will be added "as is" without parsing as they types not present in schema + this.attributeParsingService.tryParseObject(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap); + break; + case UNDEFINED: + // don't do anything for now + break; + } + } + + return dataCollectorMap; + } + + private void processInnerProperties(String recordId, Map dataCollectorMap, Object schemaPropertyWithInnerProperties, + String name, List storageRecordValue) { + Map schemaPropertyMap = (Map) schemaPropertyWithInnerProperties; + Map innerProperties = (Map) schemaPropertyMap.get(Constants.PROPERTIES); + ArrayList innerPropertiesMappingCollector = new ArrayList<>(); + storageRecordValue.forEach(recordData -> innerPropertiesMappingCollector.add(mapDataPayload(innerProperties, recordData, recordId, new HashMap<>()))); + dataCollectorMap.put(name, innerPropertiesMappingCollector); + } + + private ElasticType defineElasticType(Object entryValue) { + ElasticType elasticType = null; + if (entryValue instanceof String) { + elasticType = ElasticType.forValue(entryValue.toString()); + } else if (entryValue instanceof Map) { + Map map = (Map) entryValue; + elasticType = ElasticType.forValue(map.get(Constants.TYPE).toString()); + } + return elasticType; + } + + private Object getPropertyValue(String recordId, Map storageRecordData, String propertyKey) { try { // try getting first level property using optimized collection diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java index 416728dc3cb6689e4486e41798f37d74b7e7e627..7141317fa91b646dd22ed35a3746fa5ffefa275b 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java @@ -14,6 +14,9 @@ package org.opengroup.osdu.indexer.util; +import java.util.HashMap; +import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.opengroup.osdu.core.common.Constants; import org.opengroup.osdu.core.common.model.entitlements.AclRole; import org.opengroup.osdu.core.common.model.indexer.ElasticType; @@ -21,11 +24,6 @@ import org.opengroup.osdu.core.common.model.indexer.Records; import org.opengroup.osdu.core.common.model.indexer.StorageType; import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute; -import org.apache.commons.lang3.StringUtils; - -import java.util.HashMap; -import java.util.Map; - public class TypeMapper { private static final Map storageToIndexerType = new HashMap<>(); @@ -34,6 +32,10 @@ public class TypeMapper { private static final String STORAGE_TYPE_OBJECTS = "[]object"; + private static final String STORAGE_TYPE_NESTED = "nested"; + + private static final String STORAGE_TYPE_FLATTENED = "flattened"; + static { metaAttributeIndexerType.put(RecordMetaAttribute.KIND.getValue(), ElasticType.KEYWORD.getValue()); @@ -69,10 +71,12 @@ public class TypeMapper { //TODO temporary fix for https://community.opengroup.org/osdu/platform/system/indexer-service/-/issues/1 storageToIndexerType.put(STORAGE_TYPE_OBJECTS, ElasticType.OBJECT.getValue()); + storageToIndexerType.put(STORAGE_TYPE_NESTED, ElasticType.NESTED.getValue()); + storageToIndexerType.put(STORAGE_TYPE_FLATTENED,ElasticType.FLATTENED.getValue()); } - public static String getIndexerType(String storageType) { - return storageToIndexerType.getOrDefault(storageType, null); + public static String getIndexerType(String storageType, String defaultType) { + return storageToIndexerType.getOrDefault(storageType, defaultType); } public static Object getIndexerType(RecordMetaAttribute attribute) { @@ -87,16 +91,29 @@ public class TypeMapper { return Records.Type.builder().type(metaAttributeIndexerType.get(key).toString()).build(); } - public static Object getDataAttributeIndexerMapping(String indexerType) { - if (ElasticType.TEXT.getValue().equalsIgnoreCase(indexerType)) { + public static Object getDataAttributeIndexerMapping(Object indexerType) { + if (ElasticType.TEXT.getValue().equalsIgnoreCase(indexerType.toString())) { return getTextIndexerMapping(); } - if (isArray(indexerType)) { - return Records.Type.builder().type(getArrayMemberType(indexerType)).build(); + if (isArray(indexerType.toString())) { + return Records.Type.builder().type(getArrayMemberType(indexerType.toString())).build(); + } + + if(isMap(indexerType)){ + Map type = (Map) indexerType; + Map propertiesMap = (Map) type.get(Constants.PROPERTIES); + for (Map.Entry entry : propertiesMap.entrySet()){ + entry.setValue(Records.Type.builder().type(entry.getValue().toString()).build()); + } + return indexerType; } - return Records.Type.builder().type(indexerType).build(); + return Records.Type.builder().type(indexerType.toString()).build(); + } + + private static boolean isMap(Object indexerType) { + return indexerType instanceof Map; } private static boolean isArray(String indexerType) { @@ -140,6 +157,13 @@ public class TypeMapper { return ancestryProperties; } + public static Object getObjectsArrayMapping(String dataType, Object properties) { + Map nestedMapping = new HashMap<>(); + nestedMapping.put(Constants.TYPE,storageToIndexerType.getOrDefault(dataType, dataType)); + nestedMapping.put(Constants.PROPERTIES,properties); + return nestedMapping; + } + private static Object getIndexStatusMapping() { Map indexStatusMapping = new HashMap<>(); indexStatusMapping.put("statusCode", Records.Type.builder().type(ElasticType.INTEGER.getValue()).build()); diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessorTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessorTest.java index f130fa139304cee3dbc12ebd8e7cafc985390ad8..fdcfeba5de48702c853597f8dcd5dbc5d642c821 100644 --- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessorTest.java +++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessorTest.java @@ -14,6 +14,7 @@ package org.opengroup.osdu.indexer.schema.converter; +import com.google.common.collect.ImmutableMap; import org.junit.Test; import org.mockito.Mockito; import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; @@ -21,6 +22,7 @@ import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterPropert import org.opengroup.osdu.indexer.schema.converter.tags.AllOfItem; import org.opengroup.osdu.indexer.schema.converter.tags.Definition; import org.opengroup.osdu.indexer.schema.converter.tags.Definitions; +import org.opengroup.osdu.indexer.schema.converter.tags.Items; import org.opengroup.osdu.indexer.schema.converter.tags.TypeProperty; import java.util.LinkedHashMap; @@ -102,4 +104,109 @@ public class PropertiesProcessorTest { .processItem(allOfItem).map(Object::toString).reduce("", String::concat); assertEquals("{path=" + PATH + ", kind=int}", res); } + + @Test + public void should_return_processed_nested_array_items(){ + JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class); + + Map itemsProperties = new LinkedHashMap<>(); + TypeProperty intProperty = new TypeProperty(); + intProperty.setType("integer"); + itemsProperties.put(PATH, intProperty); + + Items items = new Items(); + items.setProperties(itemsProperties); + + TypeProperty arrayProperty = new TypeProperty(); + arrayProperty.setIndexHint(ImmutableMap.of("type","nested")); + arrayProperty.setType("array"); + arrayProperty.setItems(items); + + Map allOfItemProperties = new LinkedHashMap<>(); + allOfItemProperties.put(PATH,arrayProperty); + + AllOfItem allOfItem = new AllOfItem(); + allOfItem.setProperties(allOfItemProperties); + + String res = new PropertiesProcessor(Mockito.mock(Definitions.class), new SchemaConverterPropertiesConfig()) + .processItem(allOfItem).map(Object::toString).reduce("", String::concat); + assertEquals("{path="+ PATH + ", kind=nested, properties=[{path="+ PATH + ", kind=int}]}",res); + } + + @Test + public void should_return_flattened_array_without_processing(){ + JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class); + + Map itemsProperties = new LinkedHashMap<>(); + TypeProperty intProperty = new TypeProperty(); + intProperty.setType("integer"); + itemsProperties.put(PATH, intProperty); + + Items items = new Items(); + items.setProperties(itemsProperties); + + TypeProperty arrayProperty = new TypeProperty(); + arrayProperty.setIndexHint(ImmutableMap.of("type","flattened")); + arrayProperty.setType("array"); + arrayProperty.setItems(items); + + Map allOfItemProperties = new LinkedHashMap<>(); + allOfItemProperties.put(PATH,arrayProperty); + + AllOfItem allOfItem = new AllOfItem(); + allOfItem.setProperties(allOfItemProperties); + + String res = new PropertiesProcessor(Mockito.mock(Definitions.class), new SchemaConverterPropertiesConfig()) + .processItem(allOfItem).map(Object::toString).reduce("", String::concat); + assertEquals("{path="+ PATH + ", kind=flattened}",res); + } + + @Test + public void should_return_object_array_without_hints_in_schema_without_processing(){ + JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class); + + Map itemsProperties = new LinkedHashMap<>(); + TypeProperty intProperty = new TypeProperty(); + intProperty.setType("integer"); + itemsProperties.put(PATH, intProperty); + + Items items = new Items(); + items.setProperties(itemsProperties); + + TypeProperty arrayProperty = new TypeProperty(); + arrayProperty.setType("array"); + arrayProperty.setItems(items); + + Map allOfItemProperties = new LinkedHashMap<>(); + allOfItemProperties.put(PATH,arrayProperty); + + AllOfItem allOfItem = new AllOfItem(); + allOfItem.setProperties(allOfItemProperties); + + String res = new PropertiesProcessor(Mockito.mock(Definitions.class), new SchemaConverterPropertiesConfig()) + .processItem(allOfItem).map(Object::toString).reduce("", String::concat); + assertEquals("{path="+ PATH + ", kind=[]object}",res); + } + + @Test + public void should_process_not_object_array_type(){ + JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class); + + Items items = new Items(); + items.setType("integer"); + + TypeProperty arrayProperty = new TypeProperty(); + arrayProperty.setType("array"); + arrayProperty.setItems(items); + + Map allOfItemProperties = new LinkedHashMap<>(); + allOfItemProperties.put(PATH,arrayProperty); + + AllOfItem allOfItem = new AllOfItem(); + allOfItem.setProperties(allOfItemProperties); + + String res = new PropertiesProcessor(Mockito.mock(Definitions.class), new SchemaConverterPropertiesConfig()) + .processItem(allOfItem).map(Object::toString).reduce("", String::concat); + assertEquals("{path="+ PATH + ", kind=[]int}",res); + } } \ No newline at end of file diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapperTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapperTest.java new file mode 100644 index 0000000000000000000000000000000000000000..f3a1ac0bf85b49eeee27380bb90010005a0ebf9a --- /dev/null +++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapperTest.java @@ -0,0 +1,140 @@ +package org.opengroup.osdu.indexer.service; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import com.google.common.collect.ImmutableMap; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.opengroup.osdu.core.common.Constants; +import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; +import org.opengroup.osdu.core.common.model.indexer.IndexSchema; +import org.opengroup.osdu.core.common.model.indexer.JobStatus; +import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterPropertiesConfig; +import org.opengroup.osdu.indexer.util.parser.BooleanParser; +import org.opengroup.osdu.indexer.util.parser.DateTimeParser; +import org.opengroup.osdu.indexer.util.parser.GeoShapeParser; +import org.opengroup.osdu.indexer.util.parser.NumberParser; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; + +@RunWith(SpringRunner.class) +@SpringBootTest(classes = {StorageIndexerPayloadMapper.class, AttributeParsingServiceImpl.class, NumberParser.class, + BooleanParser.class, DateTimeParser.class, + GeoShapeParser.class, GeometryConversionService.class, JobStatus.class, SchemaConverterPropertiesConfig.class,JaxRsDpsLog.class}) +public class StorageIndexerPayloadMapperTest { + + public static final String FIRST_OBJECT_INNER_PROPERTY = "FirstObjectInnerProperty"; + public static final String SECOND_OBJECT_INNER_PROPERTY = "SecondObjectInnerProperty"; + public static final String FIRST_OBJECT_TEST_VALUE = "first-object-test-value"; + public static final String SECOND_OBJECT_TEST_VALUE = "second-object-test-value"; + public static final String OBJECT_PROPERTY = "ObjectProperty"; + public static final String NESTED_PROPERTY = "NestedProperty"; + public static final String FIRST_NESTED_INNER_PROPERTY = "FirstNestedInnerProperty"; + public static final String SECOND_NESTED_INNER_PROPERTY = "SecondNestedInnerProperty"; + public static final String FIRST_NESTED_VALUE = "first-nested-value"; + public static final String SECOND_NESTED_VALUE = "second-nested-value"; + public static final String FLATTENED_PROPERTY = "FlattenedProperty"; + public static final String FIRST_FLATTENED_INNER_PROPERTY = "FirstFlattenedInnerProperty"; + public static final String SECOND_FLATTENED_INNER_PROPERTY = "SecondFlattenedInnerProperty"; + public static final String FIRST_FLATTENED_TEST_VALUE = "first-flattened-test-value"; + public static final String SECOND_FLATTENED_TEST_VALUE = "second-flattened-test-value"; + public static final String RECORD_TEST_ID = "test-id"; + + private static IndexSchema indexSchema; + private static Map storageRecordData; + + @Autowired + private StorageIndexerPayloadMapper payloadMapper; + + @BeforeClass + public static void setUp() { + HashMap dataMap = new HashMap<>(); + dataMap.put("TextProperty", "text"); + dataMap.put("TextArrayProperty", "text_array"); + dataMap.put("DoubleProperty", "double"); + dataMap.put(OBJECT_PROPERTY, "object"); + dataMap.put(FLATTENED_PROPERTY, "flattened"); + dataMap.put(NESTED_PROPERTY, ImmutableMap.of( + Constants.TYPE, "nested", + Constants.PROPERTIES, ImmutableMap.of( + FIRST_NESTED_INNER_PROPERTY, "text", + SECOND_NESTED_INNER_PROPERTY, "double") + )); + dataMap.put("DateProperty", "date"); + indexSchema = IndexSchema.builder().kind("kind").type(Constants.TYPE).dataSchema(dataMap).build(); + + storageRecordData = new HashMap<>(); + storageRecordData.put("TextProperty", "Testing"); + storageRecordData.put("TextArrayProperty", Arrays.asList("test", "test-value")); + storageRecordData.put("DoubleProperty", "0.1"); + + storageRecordData.put(OBJECT_PROPERTY, Arrays.asList( + ImmutableMap.of(FIRST_OBJECT_INNER_PROPERTY, FIRST_OBJECT_TEST_VALUE), + ImmutableMap.of(SECOND_OBJECT_INNER_PROPERTY, SECOND_OBJECT_TEST_VALUE) + )); + + storageRecordData.put(FLATTENED_PROPERTY, Arrays.asList( + ImmutableMap.of(FIRST_FLATTENED_INNER_PROPERTY, FIRST_FLATTENED_TEST_VALUE), + ImmutableMap.of(SECOND_FLATTENED_INNER_PROPERTY, SECOND_FLATTENED_TEST_VALUE) + )); + + storageRecordData.put(NESTED_PROPERTY, Arrays.asList( + ImmutableMap.of(FIRST_NESTED_INNER_PROPERTY, FIRST_NESTED_VALUE, SECOND_NESTED_INNER_PROPERTY,"0.1"), + ImmutableMap.of(FIRST_NESTED_INNER_PROPERTY, SECOND_NESTED_VALUE, SECOND_NESTED_INNER_PROPERTY,"0.2") + )); + storageRecordData.put("DateProperty", "2021-03-02T00:17:20.640Z"); + } + + @Test + public void mapDataPayloadTestNested() { + Map stringObjectMap = payloadMapper.mapDataPayload(indexSchema, storageRecordData, + RECORD_TEST_ID); + Object nestedProperty = stringObjectMap.get(NESTED_PROPERTY); + + assertTrue(nestedProperty instanceof List); + List> nestedProperty1 = (List>) nestedProperty; + Object firstNestedInnerProperty = nestedProperty1.get(0).get(FIRST_NESTED_INNER_PROPERTY); + assertEquals(FIRST_NESTED_VALUE,firstNestedInnerProperty); + Object secondNestedInnerProperty = nestedProperty1.get(0).get(SECOND_NESTED_INNER_PROPERTY); + assertEquals(0.1,secondNestedInnerProperty); + Object firstNestedInnerProperty1 = nestedProperty1.get(1).get(FIRST_NESTED_INNER_PROPERTY); + assertEquals(SECOND_NESTED_VALUE,firstNestedInnerProperty1); + Object secondNestedInnerProperty1 = nestedProperty1.get(1).get(SECOND_NESTED_INNER_PROPERTY); + assertEquals(0.2,secondNestedInnerProperty1); + } + + @Test + public void mapDataPayloadTestFlattened() { + Map stringObjectMap = payloadMapper.mapDataPayload(indexSchema, storageRecordData, + RECORD_TEST_ID); + Object objectProperty = stringObjectMap.get(FLATTENED_PROPERTY); + + assertTrue(objectProperty instanceof List); + List> objectProperties = (List>) objectProperty; + Object firstInnerProperty = objectProperties.get(0).get(FIRST_FLATTENED_INNER_PROPERTY); + assertEquals(FIRST_FLATTENED_TEST_VALUE,firstInnerProperty); + Object secondInnerProperty = objectProperties.get(1).get(SECOND_FLATTENED_INNER_PROPERTY); + assertEquals(SECOND_FLATTENED_TEST_VALUE,secondInnerProperty); + } + + @Test + public void mapDataPayloadTestObject() { + Map stringObjectMap = payloadMapper.mapDataPayload(indexSchema, storageRecordData, + RECORD_TEST_ID); + Object objectProperty = stringObjectMap.get(OBJECT_PROPERTY); + + assertTrue(objectProperty instanceof List); + List> objectProperties = (List>) objectProperty; + Object firstInnerProperty = objectProperties.get(0).get(FIRST_OBJECT_INNER_PROPERTY); + assertEquals(FIRST_OBJECT_TEST_VALUE,firstInnerProperty); + Object secondInnerProperty = objectProperties.get(1).get(SECOND_OBJECT_INNER_PROPERTY); + assertEquals(SECOND_OBJECT_TEST_VALUE,secondInnerProperty); + } +} \ No newline at end of file diff --git a/indexer-core/src/test/resources/converter/basic/schema.json.res b/indexer-core/src/test/resources/converter/basic/schema.json.res index 416ac54e5d60a96858525c138d92186109a1f97e..e3b4fd5f35f322cba5814ffdfbee18eaac72fb0b 100644 --- a/indexer-core/src/test/resources/converter/basic/schema.json.res +++ b/indexer-core/src/test/resources/converter/basic/schema.json.res @@ -132,6 +132,42 @@ { "kind": "link", "path": "GeographicBottomHoleLocation.SpatialGeometryTypeID" + }, + { + "path": "VerticalMeasurements", + "kind": "[]object" + }, + { + "path": "DrillingReason", + "kind": "[]object" + }, + { + "path": "FacilityNameAlias", + "kind": "[]object" + }, + { + "path": "FacilityState", + "kind": "[]object" + }, + { + "path": "FacilityEvent", + "kind": "[]object" + }, + { + "path": "FacilitySpecification", + "kind": "[]object" + }, + { + "path": "FacilityOperator", + "kind": "[]object" + }, + { + "path": "SpatialLocation", + "kind": "[]object" + }, + { + "path": "GeoContexts", + "kind": "[]object" } ] } \ No newline at end of file diff --git a/indexer-core/src/test/resources/converter/new-definitions-format/colons-sample.json.res b/indexer-core/src/test/resources/converter/new-definitions-format/colons-sample.json.res index 8f3ca4d7266d4f92177459314efe2445f6a25949..4c099bb3224947924cc16d44f3a11037e3ea2724 100644 --- a/indexer-core/src/test/resources/converter/new-definitions-format/colons-sample.json.res +++ b/indexer-core/src/test/resources/converter/new-definitions-format/colons-sample.json.res @@ -72,6 +72,14 @@ { "path": "VersionCreationReason", "kind": "string" + }, + { + "path": "GeoContexts", + "kind": "[]object" + }, + { + "path": "NameAliases", + "kind": "[]object" } ] } \ No newline at end of file diff --git a/pom.xml b/pom.xml index 61d185ec97ef7ce70c7c55a573f5fc4342d082a8..7546228a31d394b89b303783770b268ef74ded5a 100644 --- a/pom.xml +++ b/pom.xml @@ -19,7 +19,7 @@ 1.8 2.7.0 Greenwich.SR2 - 0.6.9 + 0.9.0-SNAPSHOT 1.26 6.1.5.Final 2.11.2 diff --git a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexerMappingServiceTest.java b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexerMappingServiceTest.java index 3fc9f04f711391c860528a1af7c9b34a01eb1cf9..01b50be0be12488791a03f54b8c46dce65c4bbd6 100644 --- a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexerMappingServiceTest.java +++ b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexerMappingServiceTest.java @@ -88,7 +88,7 @@ public class IndexerMappingServiceTest { @Before public void setup() throws IOException { - Map dataMapping = new HashMap<>(); + Map dataMapping = new HashMap<>(); dataMapping.put("Location", "geo_point"); Map metaMapping = new HashMap<>(); metaMapping.put(RecordMetaAttribute.ID.getValue(), "keyword"); diff --git a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java index 019bae350d179f8d038f31883284af4e2d6e439b..f8debf207f2366c68fe36941e8a0c58eefdf54c6 100644 --- a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java +++ b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java @@ -74,7 +74,7 @@ public class IndexerMappingServiceTest { @Before public void setup() throws IOException { - Map dataMapping = new HashMap<>(); + Map dataMapping = new HashMap<>(); dataMapping.put("Location", "geo_point"); Map metaMapping = new HashMap<>(); metaMapping.put(RecordMetaAttribute.ID.getValue(), "keyword"); diff --git a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerMappingServiceTest.java b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerMappingServiceTest.java index d6982c113fc059e87626d2f931e4f2288f7fd3fd..9e22a80fce0c02ff55d9bd940c07279dfcd39472 100644 --- a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerMappingServiceTest.java +++ b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerMappingServiceTest.java @@ -76,7 +76,7 @@ public class IndexerMappingServiceTest { @Before public void setup() throws IOException { - Map dataMapping = new HashMap<>(); + Map dataMapping = new HashMap<>(); dataMapping.put("Location", "geo_point"); Map metaMapping = new HashMap<>(); metaMapping.put(RecordMetaAttribute.ID.getValue(), "keyword"); diff --git a/provider/indexer-reference/pom.xml b/provider/indexer-reference/pom.xml index 3e5a5328dd666658d15098f5cb003613da6b0ee9..745a8c4fd3531ea7eb2bc5b64e71582ac57667f9 100644 --- a/provider/indexer-reference/pom.xml +++ b/provider/indexer-reference/pom.xml @@ -48,7 +48,6 @@ org.opengroup.osdu os-core-common - 0.6.5 diff --git a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java index 03912e841850bb4b0eb985ff9ea5f52bc1bed098..c565d38835769e8e5d5100583fd61202fbc26cd9 100644 --- a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java +++ b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java @@ -105,4 +105,25 @@ public class Steps extends SchemaServiceRecordSteps { int expectedCount, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field) throws Throwable { super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery(expectedCount, index, topLatitude, topLongitude, bottomLatitude, bottomLongitude, field); } + + @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by nested \"([^\"]*)\" and properties \\(\"([^\"]*)\", (\\d+)\\) and \\(\"([^\"]*)\", \"([^\"]*)\"\\)$") + public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery( + int expectedCount, String index, String path, String firstNestedProperty, String firstNestedValue, String secondNestedProperty, + String secondNestedValue) throws Throwable { + super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery(expectedCount, index, path, firstNestedProperty, firstNestedValue, + secondNestedProperty, secondNestedValue); + } + + @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by flattened inner properties \\(\"([^\"]*)\", \"([^\"]*)\"\\)$") + public void i_should_be_able_search_documents_for_the_by_flattened_inner_properties(int expectedCount, String index, String flattenedField, + String flattenedFieldValue) throws Throwable { + super.i_should_be_able_search_documents_for_the_by_flattened_inner_properties(expectedCount, index, flattenedField, flattenedFieldValue); + + } + + @Then("^I should get \"([^\"]*)\" in response, without hints in schema for the \"([^\"]*)\" that present in the \"([^\"]*)\" with \"([^\"]*)\" for a given \"([^\"]*)\"$") + public void i_should_get_object_in_search_response_without_hints_in_schema(String objectInnerField, String index, String recordFile, String acl, String kind) + throws Throwable { + super.i_should_get_object_in_search_response_without_hints_in_schema(objectInnerField ,index, recordFile, acl, kind); + } } \ No newline at end of file diff --git a/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java index 97779ee8b1f9f0cea9d6c23e0a88229225deb9fb..b1dc2343728b339278710726c0d37d4a1735dd0f 100644 --- a/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java +++ b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java @@ -73,4 +73,25 @@ public class Steps extends SchemaServiceRecordSteps { int expectedCount, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field) throws Throwable { super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery(expectedCount, index, topLatitude, topLongitude, bottomLatitude, bottomLongitude, field); } + + @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by nested \"([^\"]*)\" and properties \\(\"([^\"]*)\", (\\d+)\\) and \\(\"([^\"]*)\", \"([^\"]*)\"\\)$") + public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery( + int expectedCount, String index, String path, String firstNestedProperty, String firstNestedValue, String secondNestedProperty, + String secondNestedValue) throws Throwable { + super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery(expectedCount, index, path, firstNestedProperty, firstNestedValue, + secondNestedProperty, secondNestedValue); + } + + @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by flattened inner properties \\(\"([^\"]*)\", \"([^\"]*)\"\\)$") + public void i_should_be_able_search_documents_for_the_by_flattened_inner_properties(int expectedCount, String index, String flattenedField, + String flattenedFieldValue) throws Throwable { + super.i_should_be_able_search_documents_for_the_by_flattened_inner_properties(expectedCount, index, flattenedField, flattenedFieldValue); + + } + + @Then("^I should get \"([^\"]*)\" in response, without hints in schema for the \"([^\"]*)\" that present in the \"([^\"]*)\" with \"([^\"]*)\" for a given \"([^\"]*)\"$") + public void i_should_get_object_in_search_response_without_hints_in_schema(String objectInnerField, String index, String recordFile, String acl, String kind) + throws Throwable { + super.i_should_get_object_in_search_response_without_hints_in_schema(objectInnerField ,index, recordFile, acl, kind); + } } \ No newline at end of file diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java index 4f3b563a3502abe88be51e3742c40c2bfe6a9f99..cd7f2eb02ee34ec824ca51b88a47c57460ef211e 100644 --- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java +++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java @@ -1,5 +1,6 @@ package org.opengroup.osdu.common; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.MapDifference; import com.google.common.collect.Maps; import com.google.gson.Gson; @@ -11,6 +12,7 @@ import org.elasticsearch.cluster.metadata.MappingMetadata; import org.opengroup.osdu.core.common.model.entitlements.Acl; import org.opengroup.osdu.models.Setup; import org.opengroup.osdu.models.TestIndex; +import org.opengroup.osdu.models.record.RecordData; import org.opengroup.osdu.util.ElasticUtils; import org.opengroup.osdu.util.FileHandler; import org.opengroup.osdu.util.HTTPClient; @@ -32,6 +34,7 @@ import static org.opengroup.osdu.util.Config.getStorageBaseURL; @Log public class RecordSteps extends TestsBase { private Map inputIndexMap = new HashMap<>(); + private ObjectMapper mapper = new ObjectMapper(); private boolean shutDownHookAdded = false; private String timeStamp = String.valueOf(System.currentTimeMillis()); @@ -156,6 +159,38 @@ public class RecordSteps extends TestsBase { assertEquals(expectedNumber, actualNumberOfRecords); } + public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery( + int expectedNumber, String index, String path, String firstNestedField, String firstNestedValue, String secondNestedField, String secondNestedValue) + throws Throwable { + index = generateActualName(index, timeStamp); + long numOfIndexedDocuments = createIndex(index); + long actualNumberOfRecords = elasticUtils.fetchRecordsByNestedQuery(index, path, firstNestedField, firstNestedValue, secondNestedField, secondNestedValue); + assertEquals(expectedNumber, actualNumberOfRecords); + } + + public void i_should_be_able_search_documents_for_the_by_flattened_inner_properties(int expectedCount, String index, String flattenedField, + String flattenedFieldValue) throws Throwable { + index = generateActualName(index, timeStamp); + long numOfIndexedDocuments = createIndex(index); + long actualNumberOfRecords = elasticUtils.fetchRecordsWithFlattenedFieldsQuery(index, flattenedField, flattenedFieldValue); + assertEquals(expectedCount, actualNumberOfRecords); + } + + public void i_should_get_object_in_search_response_without_hints_in_schema(String objectField, String index, String recordFile, String acl, String kind) + throws Throwable { + index = generateActualName(index, timeStamp); + long numOfIndexedDocuments = createIndex(index); + String expectedRecord = FileHandler.readFile(String.format("%s.%s", recordFile, "json")); + + RecordData[] fileRecordData = mapper.readValue(expectedRecord, RecordData[].class); + RecordData expectedRecordData = fileRecordData[0]; + + String elasticRecordData = elasticUtils.fetchDataFromObjectsArrayRecords(index); + RecordData actualRecordData = mapper.readValue(elasticRecordData, RecordData.class); + + assertEquals(expectedRecordData.getData().get(objectField),actualRecordData.getData().get(objectField)); + } + private long createIndex(String index) throws InterruptedException, IOException { long numOfIndexedDocuments = 0; int iterator; diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/models/record/RecordData.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/models/record/RecordData.java new file mode 100644 index 0000000000000000000000000000000000000000..ec0a6a221efe767579871b4877c7f6d61d0c7127 --- /dev/null +++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/models/record/RecordData.java @@ -0,0 +1,18 @@ +package org.opengroup.osdu.models.record; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; +import java.util.Map; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class RecordData { + + @JsonProperty("data") + private Map>> data; + + public Map>> getData() { + return data; + } + +} diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java index 30a6104ec6de56ad349000e509e8a307372c4231..3d029aa6ecbd34fb9ec08c2e9909cf933e33843a 100644 --- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java +++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java @@ -32,6 +32,7 @@ import org.apache.http.conn.ssl.TrustSelfSignedStrategy; import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; import org.apache.http.message.BasicHeader; import org.apache.http.ssl.SSLContextBuilder; +import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.client.indices.CloseIndexRequest; @@ -51,6 +52,7 @@ import org.elasticsearch.client.indices.GetIndexRequest; import org.elasticsearch.client.indices.GetMappingsRequest; import org.elasticsearch.client.indices.GetMappingsResponse; import org.elasticsearch.cluster.metadata.MappingMetadata; +import org.elasticsearch.search.SearchHits; import org.locationtech.jts.geom.Coordinate; import org.elasticsearch.common.geo.builders.EnvelopeBuilder; import org.elasticsearch.common.settings.Settings; @@ -274,6 +276,58 @@ public class ElasticUtils { } } + public long fetchRecordsByNestedQuery(String index, String path, String firstNestedField, String firstNestedValue, String secondNestedField, String secondNestedValue) throws Exception{ + try (RestHighLevelClient client = this.createClient(username, password, host)) { + SearchRequest searchRequest = new SearchRequest(index); + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(nestedQuery(path,boolQuery().must(matchQuery(firstNestedField,firstNestedValue)).must(matchQuery(secondNestedField,secondNestedValue)), ScoreMode.Avg)); + + searchRequest.source(searchSourceBuilder); + + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + return searchResponse.getHits().getTotalHits().value; + } catch (ElasticsearchStatusException e) { + log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage())); + return -1; + } + } + + + public long fetchRecordsWithFlattenedFieldsQuery(String index, String flattenedField, String flattenedFieldValue) throws IOException { + try (RestHighLevelClient client = this.createClient(username, password, host)) { + SearchRequest searchRequest = new SearchRequest(index); + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(boolQuery().must(matchQuery(flattenedField,flattenedFieldValue))); + searchRequest.source(searchSourceBuilder); + + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + return searchResponse.getHits().getTotalHits().value; + } catch (ElasticsearchStatusException e) { + log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage())); + return -1; + } + } + + public String fetchDataFromObjectsArrayRecords(String index) throws IOException { + try { + try (RestHighLevelClient client = this.createClient(username, password, host)) { + SearchRequest request = new SearchRequest(index); + SearchResponse searchResponse = client.search(request, RequestOptions.DEFAULT); + + SearchHits searchHits = searchResponse.getHits(); + if (searchHits.getHits().length != 0) { + return searchHits.getHits()[0].getSourceAsString(); + } + return null; + } + } catch (ElasticsearchStatusException e) { + log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage())); + return null; + } + } + public Map getMapping(String index) throws IOException { try (RestHighLevelClient client = this.createClient(username, password, host)) { GetMappingsRequest request = new GetMappingsRequest(); @@ -415,4 +469,5 @@ public class ElasticUtils { } return false; } + } \ No newline at end of file diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/IndexerClientUtil.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/IndexerClientUtil.java new file mode 100644 index 0000000000000000000000000000000000000000..33771a92bcf0bf93852027e1a63e722d9cfd5752 --- /dev/null +++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/IndexerClientUtil.java @@ -0,0 +1,38 @@ +package org.opengroup.osdu.util; + +import static org.opengroup.osdu.util.Config.getDataPartitionIdTenant1; +import static org.opengroup.osdu.util.Config.getIndexerBaseURL; + +import com.google.gson.Gson; +import com.sun.jersey.api.client.ClientResponse; +import java.util.Map; +import javax.ws.rs.HttpMethod; +import lombok.extern.java.Log; +import org.opengroup.osdu.core.common.model.search.RecordChangedMessages; + +@Log +public class IndexerClientUtil { + + private final String purgeMessage = "{\"data\":\"[{\\\"kind\\\":\\\"%s\\\",\\\"op\\\":\\\"purge_schema\\\"}]\",\"attributes\":{\"account-id\":\"%s\"}}"; + + private final HTTPClient httpClient; + private Map headers; + + public IndexerClientUtil(HTTPClient httpClient) { + this.httpClient = httpClient; + headers = httpClient.getCommonHeader(); + } + + public void deleteIndex(String kind) { + String url = getIndexerBaseURL() + "index-cleanup"; + log.info("URL: " + url); + ClientResponse response = httpClient.send(HttpMethod.POST, url, convertMessageIntoJson(kind), headers, httpClient.getAccessToken()); + log.info(response.toString()); + } + + private String convertMessageIntoJson(String kind) { + RecordChangedMessages + recordChangedMessages = (new Gson()).fromJson(String.format(purgeMessage, kind, getDataPartitionIdTenant1()), RecordChangedMessages.class); + return new Gson().toJson(recordChangedMessages); + } +} diff --git a/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature b/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature index 711ce2de1bcad49d6f46e7578c26526b28590cf9..50051edcf49854757c25ce510b0125945f38aa7b 100644 --- a/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature +++ b/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature @@ -3,11 +3,12 @@ Feature: Indexing of the documents Background: Given the schema is created with the following kind - | kind | index | schemaFile | - | tenant1:indexer:test-data--Integration:1.0.0 | tenant1-indexer-test-data--integration-1.0.0 | index_records_1 | - | tenant1:indexer:test-data--Integration:2.0.0 | tenant1-indexer-test-data--integration-2.0.0 | index_records_2 | - | tenant1:indexer:test-data--Integration:3.0.0 | tenant1-indexer-test-data--integration-3.0.0 | index_records_3 | - | tenant1:wks:master-data--Wellbore:2.0.3 | tenant1-wks-master-data--wellbore-2.0.3 | r3-index_record_wks_master | + | kind | index | schemaFile | + | tenant1:indexer:test-data--Integration:1.0.0 | tenant1-indexer-test-data--integration-1.0.0 | index_records_1 | + | tenant1:indexer:test-data--Integration:2.0.0 | tenant1-indexer-test-data--integration-2.0.0 | index_records_2 | + | tenant1:indexer:test-data--Integration:3.0.0 | tenant1-indexer-test-data--integration-3.0.0 | index_records_3 | + | tenant1:wks:master-data--Wellbore:2.0.3 | tenant1-wks-master-data--wellbore-2.0.3 | r3-index_record_wks_master | + | tenant1:wks:ArraysOfObjectsTestCollection:4.0.0 | tenant1-wks-arraysofobjectstestcollection-4.0.0 | r3-index_record_arrayofobjects | Scenario Outline: Ingest the record and Index in the Elastic Search When I ingest records with the with for a given @@ -42,3 +43,13 @@ Feature: Indexing of the documents Examples: | kind | recordFile | number | index | acl | field | top_left_latitude | top_left_longitude | bottom_right_latitude | bottom_right_longitude | | "tenant1:wks:master-data--Wellbore:2.0.3" | "r3-index_record_wks_master" | 1 | "tenant1-wks-master-data--wellbore-2.0.3" | "data.default.viewers@tenant1" | "data.SpatialLocation.Wgs84Coordinates" | 52 | -100 | 0 | 100 | + + Scenario Outline: Ingest the r3-record with arrays of objects and hints in schema and Index in the Elastic Search + When I ingest records with the with for a given + Then I should be able search documents for the by nested and properties (, ) and (, ) + Then I should be able search documents for the by flattened inner properties (, ) + Then I should get in response, without hints in schema for the that present in the with for a given + + Examples: + | kind | recordFile | number | index | acl | path | first_nested_field | first_nested_value | second_nested_field | second_nested_value | flattened_inner_field | flattened_inner_value | object_inner_field | + | "tenant1:wks:ArraysOfObjectsTestCollection:4.0.0" | "r3-index_record_arrayofobjects" | 1 | "tenant1-wks-arraysofobjectstestcollection-4.0.0" | "data.default.viewers@tenant1" | "data.NestedTest" | "data.NestedTest.NumberTest" | 12345 | "data.NestedTest.StringTest" | "test string" | "data.FlattenedTest.StringTest" | "test string" | "ObjectTest" | \ No newline at end of file diff --git a/testing/indexer-test-core/src/main/resources/testData/r3-index_record_arrayofobjects.json b/testing/indexer-test-core/src/main/resources/testData/r3-index_record_arrayofobjects.json new file mode 100644 index 0000000000000000000000000000000000000000..f37563c946a8b75e530dd1cd96017233ab40db8f --- /dev/null +++ b/testing/indexer-test-core/src/main/resources/testData/r3-index_record_arrayofobjects.json @@ -0,0 +1,43 @@ +[ + { + "id": "tenant1::testIngest70", + "data": { + "NestedTest": [ + { + "DateTimeTest": "2020-02-13T09:13:15.55Z", + "NumberTest": "12345", + "StringTest": "test string" + }, + { + "DateTimeTest": "2020-02-13T09:13:15.55Z", + "NumberTest": "567890", + "StringTest": "test string" + } + ], + "FlattenedTest": [ + { + "DateTimeTest": "2020-02-13T09:13:15.55Z", + "NumberTest": "12345", + "StringTest": "test string" + }, + { + "DateTimeTest": "2020-02-13T09:13:15.55Z", + "NumberTest": "567890", + "StringTest": "test string" + } + ], + "ObjectTest": [ + { + "DateTimeTest": "2020-02-13T09:13:15.55Z", + "NumberTest": "12345", + "StringTest": "test string" + }, + { + "DateTimeTest": "2020-02-13T09:13:15.55Z", + "NumberTest": "567890", + "StringTest": "test string" + } + ] + } + } +] \ No newline at end of file diff --git a/testing/indexer-test-core/src/main/resources/testData/r3-index_record_arrayofobjects.schema.json b/testing/indexer-test-core/src/main/resources/testData/r3-index_record_arrayofobjects.schema.json new file mode 100644 index 0000000000000000000000000000000000000000..c68efaceb42e6b11cb8ea857a85459e1dedc89a1 --- /dev/null +++ b/testing/indexer-test-core/src/main/resources/testData/r3-index_record_arrayofobjects.schema.json @@ -0,0 +1,112 @@ +{ + "schemaInfo": { + "schemaIdentity": { + "authority": "tenant1", + "source": "wks", + "entityType": "ArraysOfObjectsTestCollection", + "schemaVersionMajor": 4, + "schemaVersionMinor": 0, + "schemaVersionPatch": 0 + }, + "status": "DEVELOPMENT" + }, + "schema": { + "x-osdu-license": "Copyright 2021, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.", + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "Array of objects testing schema", + "title": "Test", + "type": "object", + "required": [ + "kind", + "acl", + "legal" + ], + "properties": { + "data": { + "allOf": [ + { + "type": "object", + "properties": { + "NestedTest": { + "description": "nested type test", + "type": "array", + "x-osdu-indexing": { + "type": "nested" + }, + "items": { + "type": "object", + "properties": { + "DateTimeTest": { + "description": "date and time test", + "type": "string", + "format": "date-time", + "x-osdu-frame-of-reference": "DateTime" + }, + "NumberTest": { + "description": "number test", + "type": "number" + }, + "StringTest": { + "description": "string test", + "type": "string" + } + } + } + }, + "FlattenedTest": { + "description": "flattened type test", + "type": "array", + "x-osdu-indexing": { + "type": "flattened" + }, + "items": { + "type": "object", + "properties": { + "DateTimeTest": { + "description": "date and time test", + "type": "string", + "format": "date-time", + "x-osdu-frame-of-reference": "DateTime" + }, + "NumberTest": { + "description": "number test", + "type": "number" + }, + "StringTest": { + "description": "string test", + "type": "string" + } + } + } + }, + "ObjectTest": { + "description": "default object type test", + "type": "array", + "items": { + "type": "object", + "properties": { + "DateTimeTest": { + "description": "date and time test", + "type": "string", + "format": "date-time", + "x-osdu-frame-of-reference": "DateTime" + }, + "NumberTest": { + "description": "number test", + "type": "number" + }, + "StringTest": { + "description": "string test", + "type": "string" + } + } + } + } + } + } + ] + } + }, + "x-osdu-inheriting-from-kind": [] + } +} \ No newline at end of file diff --git a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java index 4978ddfccad197628432d4ccb8b5985b4af5017e..4d9708e9794d942fd4a287a4bebfde5fce204695 100644 --- a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java +++ b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java @@ -6,7 +6,7 @@ import org.junit.runner.RunWith; @RunWith(Cucumber.class) @CucumberOptions( - features = "classpath:features/indexrecord/IndexRecord.feature", + features = "classpath:features/indexrecord/indexRecord-schema-service.feature", glue = {"classpath:org.opengroup.osdu.step_definitions/index/record"}, plugin = {"pretty", "junit:target/cucumber-reports/TEST-indexrecord.xml"}) public class RunTest { diff --git a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java index 348474e7769bb54d2362d079ad9aa093760d6fa2..26060454cb859e06fb80bbfa8a8c0b2d4d8c45a2 100644 --- a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java +++ b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java @@ -1,27 +1,29 @@ package org.opengroup.osdu.step_definitions.index.record; -import lombok.extern.java.Log; -import org.opengroup.osdu.common.RecordSteps; -import org.opengroup.osdu.util.GCPHTTPClient; - +import cucumber.api.DataTable; import cucumber.api.Scenario; import cucumber.api.java.Before; -import cucumber.api.DataTable; import cucumber.api.java.en.Given; import cucumber.api.java.en.Then; import cucumber.api.java.en.When; +import lombok.extern.java.Log; +import org.opengroup.osdu.common.SchemaServiceRecordSteps; +import org.opengroup.osdu.util.ElasticUtils; +import org.opengroup.osdu.util.GCPHTTPClient; +import org.opengroup.osdu.util.GcpElasticUtils; @Log -public class Steps extends RecordSteps { +public class Steps extends SchemaServiceRecordSteps { public Steps() { - super(new GCPHTTPClient()); + super(new GCPHTTPClient(), new ElasticUtils()); } @Before public void before(Scenario scenario) { this.scenario = scenario; this.httpClient = new GCPHTTPClient(); + this.elasticUtils = new GcpElasticUtils(this.httpClient); } @Given("^the schema is created with the following kind$") @@ -40,13 +42,47 @@ public class Steps extends RecordSteps { } @Then("^I should get the elastic \"(.*?)\" for the \"([^\"]*)\" and \"([^\"]*)\" in the Elastic Search$") - public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String type, String index) throws Throwable { + public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String type, String index) + throws Throwable { super.i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(expectedMapping, type, index); } @Then("^I should get the (\\d+) documents for the \"([^\"]*)\" in the Elastic Search with out \"(.*?)\"$") - public void iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(int expectedCount, String index, String skippedAttributes) throws Throwable { + public void iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(int expectedCount, String index, String skippedAttributes) + throws Throwable { super.iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(expectedCount, index, skippedAttributes); } + @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by bounding box query with points \\((-?\\d+), (-?\\d+)\\) and \\((-?\\d+), (-?\\d+)\\) on field \"([^\"]*)\"$") + public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery( + int expectedCount, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field) + throws Throwable { + String actualName = generateActualName(index, null); + super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery(expectedCount, actualName, topLatitude, topLongitude, bottomLatitude, + bottomLongitude, field); + } + + @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by nested \"([^\"]*)\" and properties \\(\"([^\"]*)\", (\\d+)\\) and \\(\"([^\"]*)\", \"([^\"]*)\"\\)$") + public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery( + int expectedCount, String index, String path, String firstNestedProperty, String firstNestedValue, String secondNestedProperty, + String secondNestedValue) throws Throwable { + String actualName = generateActualName(index, null); + super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery(expectedCount, actualName, path, firstNestedProperty, firstNestedValue, + secondNestedProperty, secondNestedValue); + } + + @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by flattened inner properties \\(\"([^\"]*)\", \"([^\"]*)\"\\)$") + public void i_should_be_able_search_documents_for_the_by_flattened_inner_properties(int expectedCount, String index, String flattenedField, + String flattenedFieldValue) throws Throwable { + String actualName = generateActualName(index, null); + super.i_should_be_able_search_documents_for_the_by_flattened_inner_properties(expectedCount, actualName, flattenedField, flattenedFieldValue); + + } + + @Then("^I should get \"([^\"]*)\" in response, without hints in schema for the \"([^\"]*)\" that present in the \"([^\"]*)\" with \"([^\"]*)\" for a given \"([^\"]*)\"$") + public void i_should_get_object_in_search_response_without_hints_in_schema(String objectInnerField, String index, String recordFile, String acl, String kind) + throws Throwable { + String actualName = generateActualName(index, null); + super.i_should_get_object_in_search_response_without_hints_in_schema(objectInnerField ,actualName, recordFile, acl, kind); + } } \ No newline at end of file diff --git a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/util/GcpElasticUtils.java b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/util/GcpElasticUtils.java new file mode 100644 index 0000000000000000000000000000000000000000..ce781c6e8cafb3cd208c52e6ca86e5c670ce2a2e --- /dev/null +++ b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/util/GcpElasticUtils.java @@ -0,0 +1,20 @@ +package org.opengroup.osdu.util; + +public class GcpElasticUtils extends ElasticUtils { + + private IndexerClientUtil indexerClientUtil; + + public GcpElasticUtils(HTTPClient httpClient) { + super(); + this.indexerClientUtil = new IndexerClientUtil(httpClient); + } + + @Override + public void deleteIndex(String index) { + indexerClientUtil.deleteIndex(convertIndexToKindName(index)); + } + + private String convertIndexToKindName(String index) { + return index.replaceAll("-",":"); + } +} diff --git a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java index 897f02c15372583ef985dfc77e2884c4035640fd..cecc3b7ddf23d559c03a65f755d01e2c9184da1a 100644 --- a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java +++ b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java @@ -6,7 +6,7 @@ import org.junit.runner.RunWith; @RunWith(Cucumber.class) @CucumberOptions( - features = {"classpath:features/indexrecord/IndexRecord.feature","classpath:apikey.feature"}, + features = {"classpath:features/indexrecord/indexRecord-schema-service.feature","classpath:apikey.feature"}, glue = {"classpath:org.opengroup.osdu.step_definitions/index/record"}, plugin = {"pretty", "junit:target/cucumber-reports/TEST-indexrecord.xml"}) public class RunTest { diff --git a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java index 3c885b11ca68f15dc3cad6ba15fae1ae7395b514..8caf85cacd30b7f146c13d0cb58cd156cbf9fc9a 100644 --- a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java +++ b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java @@ -8,6 +8,7 @@ import java.util.Map; import org.apache.commons.lang3.StringUtils; import org.opengroup.osdu.common.RecordSteps; +import org.opengroup.osdu.common.SchemaServiceRecordSteps; import org.opengroup.osdu.core.common.Constants; import org.opengroup.osdu.core.common.http.HttpClient; import org.opengroup.osdu.core.common.http.HttpRequest; @@ -16,6 +17,7 @@ import org.opengroup.osdu.core.common.model.http.AppError; import org.opengroup.osdu.core.common.model.http.DpsHeaders; import org.opengroup.osdu.core.common.model.search.RecordChangedMessages; import org.opengroup.osdu.core.ibm.util.Config; +import org.opengroup.osdu.util.ElasticUtils; import org.opengroup.osdu.util.IBMHTTPClient; import com.google.gson.Gson; @@ -29,10 +31,10 @@ import cucumber.api.java.en.When; import lombok.extern.java.Log; @Log -public class Steps extends RecordSteps { +public class Steps extends SchemaServiceRecordSteps { public Steps() { - super(new IBMHTTPClient()); + super(new IBMHTTPClient(), new ElasticUtils()); } @Before @@ -66,6 +68,38 @@ public class Steps extends RecordSteps { super.iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(expectedCount, index, skippedAttributes); } + @Then("^I should be able to search (\\d+) record with index \"([^\"]*)\" by tag \"([^\"]*)\" and value \"([^\"]*)\"$") + public void iShouldBeAbleToSearchRecordByTagKeyAndTagValue(int expectedNumber, String index, String tagKey, String tagValue) throws Throwable { + super.iShouldBeAbleToSearchRecordByTagKeyAndTagValue(index, tagKey, tagValue, expectedNumber); + } + + @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by bounding box query with points \\((-?\\d+), (-?\\d+)\\) and \\((-?\\d+), (-?\\d+)\\) on field \"(.*?)\"$") + public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery ( + int expectedCount, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field) throws Throwable { + super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery(expectedCount, index, topLatitude, topLongitude, bottomLatitude, bottomLongitude, field); + } + + @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by nested \"([^\"]*)\" and properties \\(\"([^\"]*)\", (\\d+)\\) and \\(\"([^\"]*)\", \"([^\"]*)\"\\)$") + public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery( + int expectedCount, String index, String path, String firstNestedProperty, String firstNestedValue, String secondNestedProperty, + String secondNestedValue) throws Throwable { + super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery(expectedCount, index, path, firstNestedProperty, firstNestedValue, + secondNestedProperty, secondNestedValue); + } + + @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by flattened inner properties \\(\"([^\"]*)\", \"([^\"]*)\"\\)$") + public void i_should_be_able_search_documents_for_the_by_flattened_inner_properties(int expectedCount, String index, String flattenedField, + String flattenedFieldValue) throws Throwable { + super.i_should_be_able_search_documents_for_the_by_flattened_inner_properties(expectedCount, index, flattenedField, flattenedFieldValue); + + } + + @Then("^I should get \"([^\"]*)\" in response, without hints in schema for the \"([^\"]*)\" that present in the \"([^\"]*)\" with \"([^\"]*)\" for a given \"([^\"]*)\"$") + public void i_should_get_object_in_search_response_without_hints_in_schema(String objectInnerField, String index, String recordFile, String acl, String kind) + throws Throwable { + super.i_should_get_object_in_search_response_without_hints_in_schema(objectInnerField ,index, recordFile, acl, kind); + } + @When("^I pass api key$") public void i_pass_the_api_key() { }