From deea6ca4df0018d31cacc2dcbfb635a596337d29 Mon Sep 17 00:00:00 2001 From: Rustam_Lotsmanenko Date: Fri, 26 Mar 2021 14:55:16 +0400 Subject: [PATCH 01/15] GONRG-2028 Objects array Changes: Added inner properties processing for Arrays of objects --- indexer-core/pom.xml | 2 +- .../schema/converter/PropertiesProcessor.java | 71 ++++- .../config/SchemaConverterConfig.java | 2 + .../SchemaConverterPropertiesConfig.java | 23 +- .../indexer/schema/converter/tags/Items.java | 2 + .../schema/converter/tags/TypeProperty.java | 5 +- .../service/AttributeParsingServiceImpl.java | 22 +- .../service/IAttributeParsingService.java | 2 + .../service/IndexSchemaServiceImpl.java | 43 +-- .../service/IndexerMappingServiceImpl.java | 16 +- .../service/StorageIndexerPayloadMapper.java | 258 +++++++++++------- .../osdu/indexer/util/TypeMapper.java | 44 ++- .../StorageIndexerPayloadMapperTest.java | 140 ++++++++++ pom.xml | 2 +- .../service/IndexerMappingServiceTest.java | 2 +- .../service/IndexerMappingServiceTest.java | 2 +- .../service/IndexerMappingServiceTest.java | 2 +- 17 files changed, 478 insertions(+), 160 deletions(-) create mode 100644 indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapperTest.java diff --git a/indexer-core/pom.xml b/indexer-core/pom.xml index 49ce44c6..587bd561 100644 --- a/indexer-core/pom.xml +++ b/indexer-core/pom.xml @@ -16,7 +16,7 @@ 1.9.4 - 0.6.9 + 0.8.1-SNAPSHOT diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java index 57c2cae2..52e171df 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java @@ -14,7 +14,16 @@ package org.opengroup.osdu.indexer.schema.converter; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.http.HttpStatus; +import org.opengroup.osdu.core.common.Constants; import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.search.Preconditions; @@ -23,12 +32,9 @@ import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterPropert import org.opengroup.osdu.indexer.schema.converter.tags.AllOfItem; import org.opengroup.osdu.indexer.schema.converter.tags.Definition; import org.opengroup.osdu.indexer.schema.converter.tags.Definitions; +import org.opengroup.osdu.indexer.schema.converter.tags.Items; import org.opengroup.osdu.indexer.schema.converter.tags.TypeProperty; -import java.util.*; -import java.util.function.Supplier; -import java.util.stream.Stream; - public class PropertiesProcessor { private JaxRsDpsLog log; @@ -151,6 +157,7 @@ public class PropertiesProcessor { private Stream> processPropertyEntry(Map.Entry entry) { Preconditions.checkNotNull(entry, "entry cannot be null"); + if ("object".equals(entry.getValue().getType()) && Objects.isNull(entry.getValue().getItems()) && Objects.isNull(entry.getValue().getRef()) @@ -159,7 +166,46 @@ public class PropertiesProcessor { } if ("array".equals(entry.getValue().getType())) { - if (schemaConverterConfig.getSupportedArrayTypes().contains(entry.getValue().getItems().getType())) { + + Items items = entry.getValue().getItems(); + + if(Objects.nonNull(items.getProperties()) && !items.getProperties().isEmpty()){ + String indexingType = getFromIndexingType(entry.getValue().getIndexingType()); + /*Schema item inner properties will be processed if they are present & indexingType in schema configured for processing + result ex: + { + path = ArrayItem, + kind = nested, + properties = [{ + path = InnerProperty, + kind = double + }, { + path = OtherInnerProperty, + kind = string + } + ] + } + */ + if(schemaConverterConfig.getProcessedArraysTypes().contains(indexingType)){ + PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, log, new SchemaConverterPropertiesConfig()); + return storageSchemaObjectArrayEntry( + indexingType, + entry.getKey(), + items.getProperties().entrySet().stream().flatMap(propertiesProcessor::processPropertyEntry)); + + /*Otherwise inner properties won't be processed + result ex: + { + path = ArrayItem, + kind = []object + } + */ + }else { + return storageSchemaEntry(indexingType, pathPrefixWithDot + entry.getKey()); + } + } + + if (schemaConverterConfig.getSupportedArrayTypes().contains(items.getType()) && Objects.isNull(items.getProperties())) { return storageSchemaEntry("[]" + getTypeByDefinitionProperty(entry.getValue()), pathPrefixWithDot + entry.getKey()); } @@ -225,6 +271,17 @@ public class PropertiesProcessor { return Stream.of(map); } + private Stream> storageSchemaObjectArrayEntry(String kind, String path,Stream> mapStream) { + Preconditions.checkNotNullOrEmpty(kind, "kind cannot be null or empty"); + Preconditions.checkNotNullOrEmpty(path, "path cannot be null or empty"); + + Map map = new HashMap<>(); + map.put("kind", kind); + map.put("path", path); + map.put(Constants.PROPERTIES,mapStream.collect(Collectors.toList())); + return Stream.of(map); + } + private String getTypeByDefinitionProperty(TypeProperty definitionProperty) { Preconditions.checkNotNull(definitionProperty, "definitionProperty cannot be null"); @@ -270,4 +327,8 @@ public class PropertiesProcessor { }; } + private String getFromIndexingType(String indexingType) { + return schemaConverterConfig.getArraysTypesMap().getOrDefault(indexingType, "[]object"); + } + } diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterConfig.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterConfig.java index 539b2bdf..5e945aa5 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterConfig.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterConfig.java @@ -11,4 +11,6 @@ public interface SchemaConverterConfig { Set getSupportedArrayTypes(); Map getSpecialDefinitionsMap(); Map getPrimitiveTypesMap(); + Map getArraysTypesMap(); + Set getProcessedArraysTypes(); } diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterPropertiesConfig.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterPropertiesConfig.java index 9e5eaff4..33511852 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterPropertiesConfig.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterPropertiesConfig.java @@ -1,12 +1,15 @@ package org.opengroup.osdu.indexer.schema.converter.config; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; import lombok.Getter; import lombok.Setter; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; -import java.util.*; - @Configuration @ConfigurationProperties(prefix = "schema.converter") @Getter @@ -17,6 +20,10 @@ public class SchemaConverterPropertiesConfig implements SchemaConverterConfig { private Set supportedArrayTypes = getDefaultSupportedArrayTypes(); private Map specialDefinitionsMap = getDefaultSpecialDefinitionsMap(); private Map primitiveTypesMap = getDefaultPrimitiveTypesMap(); + private Map arraysTypesMap = getDefaultArraysTypesMap(); + private Set processedArraysTypes = getDefaultArraysTypesForProcessing(); + + private Set getDefaultSkippedDefinitions() { return new HashSet<>(Arrays.asList("AbstractAnyCrsFeatureCollection", @@ -51,4 +58,16 @@ public class SchemaConverterPropertiesConfig implements SchemaConverterConfig { return defaultPrimitiveTypesMap; } + + private Map getDefaultArraysTypesMap() { + Map defaultArrayTypesMap = new HashMap<>(); + defaultArrayTypesMap.put("x-type-object","[]object"); + defaultArrayTypesMap.put("x-type-flattened","flattened"); + defaultArrayTypesMap.put("x-type-nested","nested"); + return defaultArrayTypesMap; + } + + private Set getDefaultArraysTypesForProcessing(){ + return new HashSet<>(Arrays.asList("nested")); + } } diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/Items.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/Items.java index dc3ef69e..21721cca 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/Items.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/Items.java @@ -14,10 +14,12 @@ package org.opengroup.osdu.indexer.schema.converter.tags; +import java.util.Map; import lombok.Data; @Data public class Items { private String type; private String pattern; + private Map properties; } diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java index b929bad4..af9ec6bd 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java @@ -15,13 +15,14 @@ package org.opengroup.osdu.indexer.schema.converter.tags; import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - import java.util.List; import java.util.Map; +import lombok.Data; @Data public class TypeProperty { + @JsonProperty("x-osdu-indexing") + private String indexingType; private String type; private String pattern; private String format; diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java index 733408ab..66d47186 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java @@ -19,13 +19,18 @@ import com.google.gson.Gson; import com.google.gson.JsonSyntaxException; import com.google.gson.internal.LinkedTreeMap; import com.google.gson.reflect.TypeToken; +import java.lang.reflect.Array; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.function.BiFunction; +import javax.inject.Inject; import org.apache.http.HttpStatus; import org.opengroup.osdu.core.common.model.indexer.ElasticType; -import org.opengroup.osdu.core.common.model.indexer.IndexSchema; import org.opengroup.osdu.core.common.model.indexer.IndexingStatus; import org.opengroup.osdu.core.common.model.indexer.JobStatus; -import org.opengroup.osdu.core.common.Constants; -import org.opengroup.osdu.indexer.model.geojson.FeatureCollection; import org.opengroup.osdu.indexer.util.parser.BooleanParser; import org.opengroup.osdu.indexer.util.parser.DateTimeParser; import org.opengroup.osdu.indexer.util.parser.GeoShapeParser; @@ -33,12 +38,6 @@ import org.opengroup.osdu.indexer.util.parser.NumberParser; import org.springframework.stereotype.Service; import org.springframework.web.context.annotation.RequestScope; -import javax.inject.Inject; -import java.lang.reflect.Array; -import java.lang.reflect.Type; -import java.util.*; -import java.util.function.BiFunction; - @Service @RequestScope public class AttributeParsingServiceImpl implements IAttributeParsingService { @@ -219,6 +218,11 @@ public class AttributeParsingServiceImpl implements IAttributeParsingService { dataMap.put(name,value); } + @Override + public void tryParseFlattened(String recordId, String name, Object value, Map dataMap) { + dataMap.put(name,value); + } + private List isArrayType(Object attributeVal) { try { diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java index c4fb8ecf..a658cc6a 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java @@ -28,4 +28,6 @@ public interface IAttributeParsingService { void tryParseNested(String recordId, String name, Object value, Map dataMap); void tryParseObject(String recordId, String name, Object value, Map dataMap); + + void tryParseFlattened(String recordId, String name, Object value, Map dataMap); } diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java index a6384814..093c5d40 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java @@ -16,6 +16,12 @@ package org.opengroup.osdu.indexer.service; import com.google.common.base.Strings; import com.google.gson.Gson; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URISyntaxException; +import java.util.HashMap; +import java.util.Map; +import javax.inject.Inject; import org.apache.http.HttpStatus; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; @@ -23,7 +29,6 @@ import org.elasticsearch.client.RestHighLevelClient; import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.model.http.RequestStatus; -import org.opengroup.osdu.core.common.model.indexer.ElasticType; import org.opengroup.osdu.core.common.model.indexer.IndexSchema; import org.opengroup.osdu.core.common.model.indexer.OperationType; import org.opengroup.osdu.core.common.model.indexer.StorageType; @@ -37,13 +42,6 @@ import org.opengroup.osdu.indexer.util.ElasticClientHandler; import org.opengroup.osdu.indexer.util.TypeMapper; import org.springframework.stereotype.Service; -import javax.inject.Inject; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.net.URISyntaxException; -import java.util.HashMap; -import java.util.Map; - @Service public class IndexSchemaServiceImpl implements IndexSchemaService { @@ -204,16 +202,20 @@ public class IndexSchemaServiceImpl implements IndexSchemaService { if (schemaObj == null) return null; - Map data = new HashMap<>(); + Map data = new HashMap<>(); Map meta = new HashMap<>(); if (schemaObj.getSchema() != null && schemaObj.getSchema().length > 0) { for (SchemaItem schemaItem : schemaObj.getSchema()) { String dataType = schemaItem.getKind(); - String elasticDataType = TypeMapper.getIndexerType(dataType); + Object elasticDataType = TypeMapper.getIndexerType(dataType); if (elasticDataType == null) { elasticDataType = TypeMapper.getIndexerType(StorageType.STRING.getValue()); } + if(schemaItem.getProperties() != null){ + HashMap propertiesMap = normalizeInnerProperties(schemaItem); + elasticDataType = TypeMapper.getObjectsArrayMapping(dataType, propertiesMap); + } data.put(schemaItem.getPath(), elasticDataType); } } @@ -234,14 +236,6 @@ public class IndexSchemaServiceImpl implements IndexSchemaService { String kind = schemaObj.getKind(); String type = kind.split(":")[2]; - //TODO temporary fix for https://community.opengroup.org/osdu/platform/system/indexer-service/-/issues/1 - if(data.get(MARKERS) != null){ - data.put(MARKERS, ElasticType.NESTED.getValue()); - } - if(data.get(CURVES) != null){ - data.put(CURVES, ElasticType.NESTED.getValue()); - } - return IndexSchema.builder().dataSchema(data).metaSchema(meta).kind(kind).type(type).build(); } catch (Exception e) { @@ -249,4 +243,17 @@ public class IndexSchemaServiceImpl implements IndexSchemaService { } } + private HashMap normalizeInnerProperties(SchemaItem schemaItem) { + HashMap propertiesMap = new HashMap<>(); + for (SchemaItem propertiesItem : schemaItem.getProperties()) { + String propertiesItemKind = propertiesItem.getKind(); + String propertiesElasticType = TypeMapper.getIndexerType(propertiesItemKind); + if (propertiesElasticType == null) { + propertiesElasticType = TypeMapper.getIndexerType(StorageType.STRING.getValue()); + } + propertiesMap.put(propertiesItem.getPath(),propertiesElasticType); + } + return propertiesMap; + } + } \ No newline at end of file diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java index acd87b30..b70bb211 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java @@ -15,6 +15,12 @@ package org.opengroup.osdu.indexer.service; import com.google.gson.Gson; +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import javax.inject.Inject; import org.apache.http.HttpStatus; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.support.master.AcknowledgedResponse; @@ -38,13 +44,6 @@ import org.opengroup.osdu.indexer.util.ElasticClientHandler; import org.opengroup.osdu.indexer.util.TypeMapper; import org.springframework.stereotype.Service; -import javax.inject.Inject; -import java.io.IOException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - @Service public class IndexerMappingServiceImpl extends MappingServiceImpl implements IndexerMappingService { @@ -112,7 +111,7 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind // data-source attributes Map dataMapping = new HashMap<>(); if (schema.getDataSchema() != null) { - for (Map.Entry entry : schema.getDataSchema().entrySet()) { + for (Map.Entry entry : schema.getDataSchema().entrySet()) { dataMapping.put(entry.getKey(), TypeMapper.getDataAttributeIndexerMapping(entry.getValue())); } @@ -131,7 +130,6 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind // don't add dynamic mapping documentMapping.put("dynamic", false); - return documentMapping; } diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java index 6d9d8b8c..59fb3ecd 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java @@ -1,115 +1,173 @@ package org.opengroup.osdu.indexer.service; +import static org.opengroup.osdu.indexer.service.IAttributeParsingService.DATA_GEOJSON_TAG; +import static org.opengroup.osdu.indexer.service.IAttributeParsingService.RECORD_GEOJSON_TAG; + +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import javax.inject.Inject; import org.apache.commons.beanutils.NestedNullException; import org.apache.commons.beanutils.PropertyUtils; +import org.apache.http.HttpStatus; +import org.opengroup.osdu.core.common.Constants; import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.core.common.model.indexer.ElasticType; import org.opengroup.osdu.core.common.model.indexer.IndexSchema; +import org.opengroup.osdu.core.common.model.indexer.IndexingStatus; +import org.opengroup.osdu.core.common.model.indexer.JobStatus; +import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterConfig; import org.springframework.stereotype.Component; -import javax.inject.Inject; -import java.lang.reflect.InvocationTargetException; -import java.util.Date; -import java.util.HashMap; -import java.util.Map; - -import static org.opengroup.osdu.indexer.service.IAttributeParsingService.DATA_GEOJSON_TAG; -import static org.opengroup.osdu.indexer.service.IAttributeParsingService.RECORD_GEOJSON_TAG; - @Component public class StorageIndexerPayloadMapper { - @Inject - private JaxRsDpsLog log; - @Inject - private IAttributeParsingService attributeParsingService; - - public Map mapDataPayload(IndexSchema storageSchema, Map storageRecordData, String recordId) { - - Map dataMap = new HashMap<>(); - - if (storageSchema.isDataSchemaMissing()) return dataMap; - - // get the key and get the corresponding object from the storageRecord object - for (Map.Entry entry : storageSchema.getDataSchema().entrySet()) { - - String name = entry.getKey(); - - Object value = getPropertyValue(recordId, storageRecordData, name); - - ElasticType elasticType = ElasticType.forValue(entry.getValue()); - - if (value == null && !nullIndexedValueSupported(elasticType)) continue; - - switch (elasticType) { - case KEYWORD: - case KEYWORD_ARRAY: - case TEXT: - case TEXT_ARRAY: - dataMap.put(name, value); - break; - case INTEGER_ARRAY: - this.attributeParsingService.tryParseValueArray(Integer.class, recordId, name, value, dataMap); - break; - case INTEGER: - this.attributeParsingService.tryParseInteger(recordId, name, value, dataMap); - break; - case LONG_ARRAY: - this.attributeParsingService.tryParseValueArray(Long.class, recordId, name, value, dataMap); - break; - case LONG: - this.attributeParsingService.tryParseLong(recordId, name, value, dataMap); - break; - case FLOAT_ARRAY: - this.attributeParsingService.tryParseValueArray(Float.class, recordId, name, value, dataMap); - break; - case FLOAT: - this.attributeParsingService.tryParseFloat(recordId, name, value, dataMap); - break; - case DOUBLE_ARRAY: - this.attributeParsingService.tryParseValueArray(Double.class, recordId, name, value, dataMap); - break; - case DOUBLE: - this.attributeParsingService.tryParseDouble(recordId, name, value, dataMap); - break; - case BOOLEAN_ARRAY: - this.attributeParsingService.tryParseValueArray(Boolean.class, recordId, name, value, dataMap); - break; - case BOOLEAN: - this.attributeParsingService.tryParseBoolean(recordId, name, value, dataMap); - break; - case DATE_ARRAY: - this.attributeParsingService.tryParseValueArray(Date.class, recordId, name, value, dataMap); - break; - case DATE: - this.attributeParsingService.tryParseDate(recordId, name, value, dataMap); - break; - case GEO_POINT: - this.attributeParsingService.tryParseGeopoint(recordId, name, storageRecordData, dataMap); - break; - case GEO_SHAPE: - this.attributeParsingService.tryParseGeojson(recordId, name, value, dataMap); - break; - case NESTED: - this.attributeParsingService.tryParseNested(recordId, name, value, dataMap); - break; - case OBJECT: - this.attributeParsingService.tryParseObject(recordId, name, value, dataMap); - break; - case UNDEFINED: - // don't do anything for now - break; - } - } - - // add these once iterated over the list - storageSchema.getDataSchema().put(DATA_GEOJSON_TAG, ElasticType.GEO_SHAPE.getValue()); - storageSchema.getDataSchema().remove(RECORD_GEOJSON_TAG); - - return dataMap; - } - - private Object getPropertyValue(String recordId, Map storageRecordData, String propertyKey) { + @Inject + private JaxRsDpsLog log; + @Inject + private IAttributeParsingService attributeParsingService; + @Inject + private JobStatus jobStatus; + @Inject + private SchemaConverterConfig schemaConfig; + + public Map mapDataPayload(IndexSchema storageSchema, Map storageRecordData, + String recordId) { + + Map dataMap = new HashMap<>(); + + if (storageSchema.isDataSchemaMissing()) { + return dataMap; + } + + mapDataPayload(storageSchema.getDataSchema(), storageRecordData, recordId, dataMap); + + // add these once iterated over the list + storageSchema.getDataSchema().put(DATA_GEOJSON_TAG, ElasticType.GEO_SHAPE.getValue()); + storageSchema.getDataSchema().remove(RECORD_GEOJSON_TAG); + + return dataMap; + } + + private Map mapDataPayload(Map dataSchema, Map storageRecordData, + String recordId, Map dataMap) { + + // get the key and get the corresponding object from the storageRecord object + for (Map.Entry entry : dataSchema.entrySet()) { + String name = entry.getKey(); + Object value = getPropertyValue(recordId, storageRecordData, name); + ElasticType elasticType = defineElasticType(entry); + + if (Objects.isNull(elasticType)) { + this.jobStatus + .addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, "e.getMessage()", + String.format("record-id: %s | %s", recordId, "e.getMessage()")); + continue; + } + + if (schemaConfig.getProcessedArraysTypes().contains(elasticType.getValue().toLowerCase())) { + processInnerProperties(recordId, dataMap, entry, name, (List) value); + } + + if (value == null && !nullIndexedValueSupported(elasticType)) { + continue; + } + + switch (elasticType) { + case KEYWORD: + case KEYWORD_ARRAY: + case TEXT: + case TEXT_ARRAY: + dataMap.put(name, value); + break; + case INTEGER_ARRAY: + this.attributeParsingService.tryParseValueArray(Integer.class, recordId, name, value, dataMap); + break; + case INTEGER: + this.attributeParsingService.tryParseInteger(recordId, name, value, dataMap); + break; + case LONG_ARRAY: + this.attributeParsingService.tryParseValueArray(Long.class, recordId, name, value, dataMap); + break; + case LONG: + this.attributeParsingService.tryParseLong(recordId, name, value, dataMap); + break; + case FLOAT_ARRAY: + this.attributeParsingService.tryParseValueArray(Float.class, recordId, name, value, dataMap); + break; + case FLOAT: + this.attributeParsingService.tryParseFloat(recordId, name, value, dataMap); + break; + case DOUBLE_ARRAY: + this.attributeParsingService.tryParseValueArray(Double.class, recordId, name, value, dataMap); + break; + case DOUBLE: + this.attributeParsingService.tryParseDouble(recordId, name, value, dataMap); + break; + case BOOLEAN_ARRAY: + this.attributeParsingService.tryParseValueArray(Boolean.class, recordId, name, value, dataMap); + break; + case BOOLEAN: + this.attributeParsingService.tryParseBoolean(recordId, name, value, dataMap); + break; + case DATE_ARRAY: + this.attributeParsingService.tryParseValueArray(Date.class, recordId, name, value, dataMap); + break; + case DATE: + this.attributeParsingService.tryParseDate(recordId, name, value, dataMap); + break; + case GEO_POINT: + this.attributeParsingService.tryParseGeopoint(recordId, name, storageRecordData, dataMap); + break; + case GEO_SHAPE: + this.attributeParsingService.tryParseGeojson(recordId, name, value, dataMap); + break; + case NESTED: + // don't do anything , each nested property will be mapped separately + break; + case FLATTENED: + // flattened type inner properties won't be mapped as they types not present in schema + this.attributeParsingService.tryParseFlattened(recordId, name, value, dataMap); + break; + case OBJECT: + // object type inner properties won't be mapped as they types not present in schema + this.attributeParsingService.tryParseObject(recordId, name, value, dataMap); + break; + case UNDEFINED: + // don't do anything for now + break; + } + } + + return dataMap; + } + + private void processInnerProperties(String recordId, Map dataMap, Entry entry, + String name, List value) { + Map map = (Map) entry.getValue(); + Map innerProperties = (Map) map.get(Constants.PROPERTIES); + ArrayList maps = new ArrayList<>(); + value.forEach(recordData -> maps.add(mapDataPayload(innerProperties, recordData, recordId, new HashMap<>()))); + dataMap.put(name, maps); + } + + private ElasticType defineElasticType(Map.Entry entry) { + ElasticType elasticType = null; + if (entry.getValue() instanceof String) { + elasticType = ElasticType.forValue(entry.getValue().toString()); + } else if (entry.getValue() instanceof Map) { + Map map = (Map) entry.getValue(); + elasticType = ElasticType.forValue(map.get(Constants.TYPE).toString()); + } + return elasticType; + } + + private Object getPropertyValue(String recordId, Map storageRecordData, String propertyKey) { try { // try getting first level property using optimized collection diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java index 416728dc..5887e7d9 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java @@ -14,6 +14,9 @@ package org.opengroup.osdu.indexer.util; +import java.util.HashMap; +import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.opengroup.osdu.core.common.Constants; import org.opengroup.osdu.core.common.model.entitlements.AclRole; import org.opengroup.osdu.core.common.model.indexer.ElasticType; @@ -21,11 +24,6 @@ import org.opengroup.osdu.core.common.model.indexer.Records; import org.opengroup.osdu.core.common.model.indexer.StorageType; import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute; -import org.apache.commons.lang3.StringUtils; - -import java.util.HashMap; -import java.util.Map; - public class TypeMapper { private static final Map storageToIndexerType = new HashMap<>(); @@ -34,6 +32,10 @@ public class TypeMapper { private static final String STORAGE_TYPE_OBJECTS = "[]object"; + private static final String STORAGE_TYPE_NESTED = "nested"; + + private static final String STORAGE_TYPE_FLATTENED = "flattened"; + static { metaAttributeIndexerType.put(RecordMetaAttribute.KIND.getValue(), ElasticType.KEYWORD.getValue()); @@ -69,6 +71,8 @@ public class TypeMapper { //TODO temporary fix for https://community.opengroup.org/osdu/platform/system/indexer-service/-/issues/1 storageToIndexerType.put(STORAGE_TYPE_OBJECTS, ElasticType.OBJECT.getValue()); + storageToIndexerType.put(STORAGE_TYPE_NESTED, ElasticType.NESTED.getValue()); + storageToIndexerType.put(STORAGE_TYPE_FLATTENED,ElasticType.FLATTENED.getValue()); } public static String getIndexerType(String storageType) { @@ -87,16 +91,29 @@ public class TypeMapper { return Records.Type.builder().type(metaAttributeIndexerType.get(key).toString()).build(); } - public static Object getDataAttributeIndexerMapping(String indexerType) { - if (ElasticType.TEXT.getValue().equalsIgnoreCase(indexerType)) { + public static Object getDataAttributeIndexerMapping(Object indexerType) { + if (ElasticType.TEXT.getValue().equalsIgnoreCase(indexerType.toString())) { return getTextIndexerMapping(); } - if (isArray(indexerType)) { - return Records.Type.builder().type(getArrayMemberType(indexerType)).build(); + if (isArray(indexerType.toString())) { + return Records.Type.builder().type(getArrayMemberType(indexerType.toString())).build(); + } + + if(isMap(indexerType)){ + Map type = (Map) indexerType; + HashMap propertiesMap = (HashMap) type.get(Constants.PROPERTIES); + for (Map.Entry entry : propertiesMap.entrySet()){ + entry.setValue(Records.Type.builder().type(entry.getValue().toString()).build()); + } + return indexerType; } - return Records.Type.builder().type(indexerType).build(); + return Records.Type.builder().type(indexerType.toString()).build(); + } + + private static boolean isMap(Object indexerType) { + return indexerType instanceof Map; } private static boolean isArray(String indexerType) { @@ -140,6 +157,13 @@ public class TypeMapper { return ancestryProperties; } + public static Object getObjectsArrayMapping(String dataType, Object properties) { + Map nestedMapping = new HashMap<>(); + nestedMapping.put(Constants.TYPE,storageToIndexerType.getOrDefault(dataType, dataType)); + nestedMapping.put(Constants.PROPERTIES,properties); + return nestedMapping; + } + private static Object getIndexStatusMapping() { Map indexStatusMapping = new HashMap<>(); indexStatusMapping.put("statusCode", Records.Type.builder().type(ElasticType.INTEGER.getValue()).build()); diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapperTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapperTest.java new file mode 100644 index 00000000..f3a1ac0b --- /dev/null +++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapperTest.java @@ -0,0 +1,140 @@ +package org.opengroup.osdu.indexer.service; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import com.google.common.collect.ImmutableMap; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.opengroup.osdu.core.common.Constants; +import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; +import org.opengroup.osdu.core.common.model.indexer.IndexSchema; +import org.opengroup.osdu.core.common.model.indexer.JobStatus; +import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterPropertiesConfig; +import org.opengroup.osdu.indexer.util.parser.BooleanParser; +import org.opengroup.osdu.indexer.util.parser.DateTimeParser; +import org.opengroup.osdu.indexer.util.parser.GeoShapeParser; +import org.opengroup.osdu.indexer.util.parser.NumberParser; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; + +@RunWith(SpringRunner.class) +@SpringBootTest(classes = {StorageIndexerPayloadMapper.class, AttributeParsingServiceImpl.class, NumberParser.class, + BooleanParser.class, DateTimeParser.class, + GeoShapeParser.class, GeometryConversionService.class, JobStatus.class, SchemaConverterPropertiesConfig.class,JaxRsDpsLog.class}) +public class StorageIndexerPayloadMapperTest { + + public static final String FIRST_OBJECT_INNER_PROPERTY = "FirstObjectInnerProperty"; + public static final String SECOND_OBJECT_INNER_PROPERTY = "SecondObjectInnerProperty"; + public static final String FIRST_OBJECT_TEST_VALUE = "first-object-test-value"; + public static final String SECOND_OBJECT_TEST_VALUE = "second-object-test-value"; + public static final String OBJECT_PROPERTY = "ObjectProperty"; + public static final String NESTED_PROPERTY = "NestedProperty"; + public static final String FIRST_NESTED_INNER_PROPERTY = "FirstNestedInnerProperty"; + public static final String SECOND_NESTED_INNER_PROPERTY = "SecondNestedInnerProperty"; + public static final String FIRST_NESTED_VALUE = "first-nested-value"; + public static final String SECOND_NESTED_VALUE = "second-nested-value"; + public static final String FLATTENED_PROPERTY = "FlattenedProperty"; + public static final String FIRST_FLATTENED_INNER_PROPERTY = "FirstFlattenedInnerProperty"; + public static final String SECOND_FLATTENED_INNER_PROPERTY = "SecondFlattenedInnerProperty"; + public static final String FIRST_FLATTENED_TEST_VALUE = "first-flattened-test-value"; + public static final String SECOND_FLATTENED_TEST_VALUE = "second-flattened-test-value"; + public static final String RECORD_TEST_ID = "test-id"; + + private static IndexSchema indexSchema; + private static Map storageRecordData; + + @Autowired + private StorageIndexerPayloadMapper payloadMapper; + + @BeforeClass + public static void setUp() { + HashMap dataMap = new HashMap<>(); + dataMap.put("TextProperty", "text"); + dataMap.put("TextArrayProperty", "text_array"); + dataMap.put("DoubleProperty", "double"); + dataMap.put(OBJECT_PROPERTY, "object"); + dataMap.put(FLATTENED_PROPERTY, "flattened"); + dataMap.put(NESTED_PROPERTY, ImmutableMap.of( + Constants.TYPE, "nested", + Constants.PROPERTIES, ImmutableMap.of( + FIRST_NESTED_INNER_PROPERTY, "text", + SECOND_NESTED_INNER_PROPERTY, "double") + )); + dataMap.put("DateProperty", "date"); + indexSchema = IndexSchema.builder().kind("kind").type(Constants.TYPE).dataSchema(dataMap).build(); + + storageRecordData = new HashMap<>(); + storageRecordData.put("TextProperty", "Testing"); + storageRecordData.put("TextArrayProperty", Arrays.asList("test", "test-value")); + storageRecordData.put("DoubleProperty", "0.1"); + + storageRecordData.put(OBJECT_PROPERTY, Arrays.asList( + ImmutableMap.of(FIRST_OBJECT_INNER_PROPERTY, FIRST_OBJECT_TEST_VALUE), + ImmutableMap.of(SECOND_OBJECT_INNER_PROPERTY, SECOND_OBJECT_TEST_VALUE) + )); + + storageRecordData.put(FLATTENED_PROPERTY, Arrays.asList( + ImmutableMap.of(FIRST_FLATTENED_INNER_PROPERTY, FIRST_FLATTENED_TEST_VALUE), + ImmutableMap.of(SECOND_FLATTENED_INNER_PROPERTY, SECOND_FLATTENED_TEST_VALUE) + )); + + storageRecordData.put(NESTED_PROPERTY, Arrays.asList( + ImmutableMap.of(FIRST_NESTED_INNER_PROPERTY, FIRST_NESTED_VALUE, SECOND_NESTED_INNER_PROPERTY,"0.1"), + ImmutableMap.of(FIRST_NESTED_INNER_PROPERTY, SECOND_NESTED_VALUE, SECOND_NESTED_INNER_PROPERTY,"0.2") + )); + storageRecordData.put("DateProperty", "2021-03-02T00:17:20.640Z"); + } + + @Test + public void mapDataPayloadTestNested() { + Map stringObjectMap = payloadMapper.mapDataPayload(indexSchema, storageRecordData, + RECORD_TEST_ID); + Object nestedProperty = stringObjectMap.get(NESTED_PROPERTY); + + assertTrue(nestedProperty instanceof List); + List> nestedProperty1 = (List>) nestedProperty; + Object firstNestedInnerProperty = nestedProperty1.get(0).get(FIRST_NESTED_INNER_PROPERTY); + assertEquals(FIRST_NESTED_VALUE,firstNestedInnerProperty); + Object secondNestedInnerProperty = nestedProperty1.get(0).get(SECOND_NESTED_INNER_PROPERTY); + assertEquals(0.1,secondNestedInnerProperty); + Object firstNestedInnerProperty1 = nestedProperty1.get(1).get(FIRST_NESTED_INNER_PROPERTY); + assertEquals(SECOND_NESTED_VALUE,firstNestedInnerProperty1); + Object secondNestedInnerProperty1 = nestedProperty1.get(1).get(SECOND_NESTED_INNER_PROPERTY); + assertEquals(0.2,secondNestedInnerProperty1); + } + + @Test + public void mapDataPayloadTestFlattened() { + Map stringObjectMap = payloadMapper.mapDataPayload(indexSchema, storageRecordData, + RECORD_TEST_ID); + Object objectProperty = stringObjectMap.get(FLATTENED_PROPERTY); + + assertTrue(objectProperty instanceof List); + List> objectProperties = (List>) objectProperty; + Object firstInnerProperty = objectProperties.get(0).get(FIRST_FLATTENED_INNER_PROPERTY); + assertEquals(FIRST_FLATTENED_TEST_VALUE,firstInnerProperty); + Object secondInnerProperty = objectProperties.get(1).get(SECOND_FLATTENED_INNER_PROPERTY); + assertEquals(SECOND_FLATTENED_TEST_VALUE,secondInnerProperty); + } + + @Test + public void mapDataPayloadTestObject() { + Map stringObjectMap = payloadMapper.mapDataPayload(indexSchema, storageRecordData, + RECORD_TEST_ID); + Object objectProperty = stringObjectMap.get(OBJECT_PROPERTY); + + assertTrue(objectProperty instanceof List); + List> objectProperties = (List>) objectProperty; + Object firstInnerProperty = objectProperties.get(0).get(FIRST_OBJECT_INNER_PROPERTY); + assertEquals(FIRST_OBJECT_TEST_VALUE,firstInnerProperty); + Object secondInnerProperty = objectProperties.get(1).get(SECOND_OBJECT_INNER_PROPERTY); + assertEquals(SECOND_OBJECT_TEST_VALUE,secondInnerProperty); + } +} \ No newline at end of file diff --git a/pom.xml b/pom.xml index 51f7584b..5ec2aaf8 100644 --- a/pom.xml +++ b/pom.xml @@ -19,7 +19,7 @@ 1.8 2.7.0 Greenwich.SR2 - 0.6.9 + 0.8.1-SNAPSHOT 1.26 6.1.5.Final 2.11.2 diff --git a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexerMappingServiceTest.java b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexerMappingServiceTest.java index 3fc9f04f..01b50be0 100644 --- a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexerMappingServiceTest.java +++ b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexerMappingServiceTest.java @@ -88,7 +88,7 @@ public class IndexerMappingServiceTest { @Before public void setup() throws IOException { - Map dataMapping = new HashMap<>(); + Map dataMapping = new HashMap<>(); dataMapping.put("Location", "geo_point"); Map metaMapping = new HashMap<>(); metaMapping.put(RecordMetaAttribute.ID.getValue(), "keyword"); diff --git a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java index 019bae35..f8debf20 100644 --- a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java +++ b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java @@ -74,7 +74,7 @@ public class IndexerMappingServiceTest { @Before public void setup() throws IOException { - Map dataMapping = new HashMap<>(); + Map dataMapping = new HashMap<>(); dataMapping.put("Location", "geo_point"); Map metaMapping = new HashMap<>(); metaMapping.put(RecordMetaAttribute.ID.getValue(), "keyword"); diff --git a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerMappingServiceTest.java b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerMappingServiceTest.java index d6982c11..9e22a80f 100644 --- a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerMappingServiceTest.java +++ b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerMappingServiceTest.java @@ -76,7 +76,7 @@ public class IndexerMappingServiceTest { @Before public void setup() throws IOException { - Map dataMapping = new HashMap<>(); + Map dataMapping = new HashMap<>(); dataMapping.put("Location", "geo_point"); Map metaMapping = new HashMap<>(); metaMapping.put(RecordMetaAttribute.ID.getValue(), "keyword"); -- GitLab From 430445b703cd2b2e0242a82bd523f6a9b7fc0440 Mon Sep 17 00:00:00 2001 From: Rustam_Lotsmanenko Date: Wed, 14 Apr 2021 12:25:04 +0400 Subject: [PATCH 02/15] object type is implicit, update core-common --- indexer-core/pom.xml | 2 +- .../converter/config/SchemaConverterPropertiesConfig.java | 1 - pom.xml | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/indexer-core/pom.xml b/indexer-core/pom.xml index 587bd561..bdcfd691 100644 --- a/indexer-core/pom.xml +++ b/indexer-core/pom.xml @@ -16,7 +16,7 @@ 1.9.4 - 0.8.1-SNAPSHOT + 0.9.0-SNAPSHOT diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterPropertiesConfig.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterPropertiesConfig.java index 33511852..bfcbec64 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterPropertiesConfig.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterPropertiesConfig.java @@ -61,7 +61,6 @@ public class SchemaConverterPropertiesConfig implements SchemaConverterConfig { private Map getDefaultArraysTypesMap() { Map defaultArrayTypesMap = new HashMap<>(); - defaultArrayTypesMap.put("x-type-object","[]object"); defaultArrayTypesMap.put("x-type-flattened","flattened"); defaultArrayTypesMap.put("x-type-nested","nested"); return defaultArrayTypesMap; diff --git a/pom.xml b/pom.xml index 5ec2aaf8..4f446ace 100644 --- a/pom.xml +++ b/pom.xml @@ -19,7 +19,7 @@ 1.8 2.7.0 Greenwich.SR2 - 0.8.1-SNAPSHOT + 0.9.0-SNAPSHOT 1.26 6.1.5.Final 2.11.2 -- GitLab From bad80ef7b185e169acd099c8f4f3fa90388d013e Mon Sep 17 00:00:00 2001 From: Rustam_Lotsmanenko Date: Wed, 14 Apr 2021 12:58:35 +0400 Subject: [PATCH 03/15] update core-common for reference --- provider/indexer-reference/pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/provider/indexer-reference/pom.xml b/provider/indexer-reference/pom.xml index 3e5a5328..745a8c4f 100644 --- a/provider/indexer-reference/pom.xml +++ b/provider/indexer-reference/pom.xml @@ -48,7 +48,6 @@ org.opengroup.osdu os-core-common - 0.6.5 -- GitLab From 335802427f7eebbe6ac5c4b0414617c34e48bb5f Mon Sep 17 00:00:00 2001 From: Rustam_Lotsmanenko Date: Thu, 22 Apr 2021 17:16:08 +0400 Subject: [PATCH 04/15] Change hint structure --- .../schema/converter/PropertiesProcessor.java | 10 +++++----- .../converter/config/SchemaConverterConfig.java | 2 +- .../config/SchemaConverterPropertiesConfig.java | 16 ++++++---------- .../schema/converter/tags/TypeProperty.java | 2 +- 4 files changed, 13 insertions(+), 17 deletions(-) diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java index d0dd334c..2731083d 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java @@ -40,6 +40,7 @@ public class PropertiesProcessor { private JaxRsDpsLog log; private SchemaConverterConfig schemaConverterConfig; + private static final String TYPE_KEY = "type"; private static final String DEF_PREFIX = "#/definitions/"; private static final String LINK_PREFIX = "^srn"; private static final String LINK_TYPE = "link"; @@ -174,7 +175,10 @@ public class PropertiesProcessor { Items items = entry.getValue().getItems(); if(Objects.nonNull(items.getProperties()) && !items.getProperties().isEmpty()){ - String indexingType = getFromIndexingType(entry.getValue().getIndexingType()); + Map type = entry.getValue().getIndexingType(); + String indexingType = Objects.isNull(type) ? + schemaConverterConfig.getDefaultObjectArraysType() : + type.getOrDefault(TYPE_KEY,schemaConverterConfig.getDefaultObjectArraysType()); /*Schema item inner properties will be processed if they are present & indexingType in schema configured for processing result ex: { @@ -331,8 +335,4 @@ public class PropertiesProcessor { }; } - private String getFromIndexingType(String indexingType) { - return schemaConverterConfig.getArraysTypesMap().getOrDefault(indexingType, "[]object"); - } - } diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterConfig.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterConfig.java index 5e945aa5..f31c28b6 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterConfig.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterConfig.java @@ -11,6 +11,6 @@ public interface SchemaConverterConfig { Set getSupportedArrayTypes(); Map getSpecialDefinitionsMap(); Map getPrimitiveTypesMap(); - Map getArraysTypesMap(); Set getProcessedArraysTypes(); + String getDefaultObjectArraysType(); } diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterPropertiesConfig.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterPropertiesConfig.java index bfcbec64..e969f8e3 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterPropertiesConfig.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterPropertiesConfig.java @@ -20,14 +20,13 @@ public class SchemaConverterPropertiesConfig implements SchemaConverterConfig { private Set supportedArrayTypes = getDefaultSupportedArrayTypes(); private Map specialDefinitionsMap = getDefaultSpecialDefinitionsMap(); private Map primitiveTypesMap = getDefaultPrimitiveTypesMap(); - private Map arraysTypesMap = getDefaultArraysTypesMap(); private Set processedArraysTypes = getDefaultArraysTypesForProcessing(); - + private String defaultObjectArraysType = getObjectArraysDefaultType(); private Set getDefaultSkippedDefinitions() { return new HashSet<>(Arrays.asList("AbstractAnyCrsFeatureCollection", - "anyCrsGeoJsonFeatureCollection")); + "anyCrsGeoJsonFeatureCollection")); } private Set getDefaultSupportedArrayTypes() { @@ -59,14 +58,11 @@ public class SchemaConverterPropertiesConfig implements SchemaConverterConfig { return defaultPrimitiveTypesMap; } - private Map getDefaultArraysTypesMap() { - Map defaultArrayTypesMap = new HashMap<>(); - defaultArrayTypesMap.put("x-type-flattened","flattened"); - defaultArrayTypesMap.put("x-type-nested","nested"); - return defaultArrayTypesMap; + private Set getDefaultArraysTypesForProcessing() { + return new HashSet<>(Arrays.asList("nested")); } - private Set getDefaultArraysTypesForProcessing(){ - return new HashSet<>(Arrays.asList("nested")); + private String getObjectArraysDefaultType() { + return "[]object"; } } diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java index af9ec6bd..91d10c55 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java @@ -22,7 +22,7 @@ import lombok.Data; @Data public class TypeProperty { @JsonProperty("x-osdu-indexing") - private String indexingType; + private Map indexingType; private String type; private String pattern; private String format; -- GitLab From 3372a074da63857a56d3ee42f90d51eb15bfe181 Mon Sep 17 00:00:00 2001 From: Rustam_Lotsmanenko Date: Thu, 22 Apr 2021 18:39:19 +0400 Subject: [PATCH 05/15] draft migrate int tests to v3 schema --- .../opengroup/osdu/common/RecordSteps.java | 9 ++ .../org/opengroup/osdu/util/ElasticUtils.java | 18 +++ .../indexRecord-schema-service.feature | 19 ++- .../r3-index_record_arrayofobjects.json | 43 +++++++ ...r3-index_record_arrayofobjects.schema.json | 112 ++++++++++++++++++ .../index/record/RunTest.java | 2 +- .../step_definitions/index/record/Steps.java | 33 ++++-- 7 files changed, 223 insertions(+), 13 deletions(-) create mode 100644 testing/indexer-test-core/src/main/resources/testData/r3-index_record_arrayofobjects.json create mode 100644 testing/indexer-test-core/src/main/resources/testData/r3-index_record_arrayofobjects.schema.json diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java index 4f3b563a..322faff8 100644 --- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java +++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java @@ -156,6 +156,15 @@ public class RecordSteps extends TestsBase { assertEquals(expectedNumber, actualNumberOfRecords); } + public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery( + int expectedNumber, String index, String path, String firstNestedField, String firstNestedValue, String secondNestedField, String secondNestedValue) + throws Exception { + + long numOfIndexedDocuments = createIndex(index); + long actualNumberOfRecords = elasticUtils.fetchRecordsByNestedQuery(index, path, firstNestedField, firstNestedValue, secondNestedField, secondNestedValue); + assertEquals(expectedNumber, actualNumberOfRecords); + } + private long createIndex(String index) throws InterruptedException, IOException { long numOfIndexedDocuments = 0; int iterator; diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java index 30a6104e..d42a4fd0 100644 --- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java +++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java @@ -32,6 +32,7 @@ import org.apache.http.conn.ssl.TrustSelfSignedStrategy; import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; import org.apache.http.message.BasicHeader; import org.apache.http.ssl.SSLContextBuilder; +import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.client.indices.CloseIndexRequest; @@ -274,6 +275,23 @@ public class ElasticUtils { } } + public long fetchRecordsByNestedQuery(String index, String path, String firstNestedField, String firstNestedValue, String secondNestedField, String secondNestedValue) throws Exception{ + try (RestHighLevelClient client = this.createClient(username, password, host)) { + SearchRequest searchRequest = new SearchRequest(index); + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(nestedQuery(path,boolQuery().must(matchQuery(firstNestedField,firstNestedValue)).must(matchQuery(secondNestedField,secondNestedValue)), ScoreMode.Avg)); + + searchRequest.source(searchSourceBuilder); + + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + return searchResponse.getHits().getTotalHits().value; + } catch (ElasticsearchStatusException e) { + log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage())); + return -1; + } + } + public Map getMapping(String index) throws IOException { try (RestHighLevelClient client = this.createClient(username, password, host)) { GetMappingsRequest request = new GetMappingsRequest(); diff --git a/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature b/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature index 711ce2de..62530314 100644 --- a/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature +++ b/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature @@ -3,11 +3,12 @@ Feature: Indexing of the documents Background: Given the schema is created with the following kind - | kind | index | schemaFile | - | tenant1:indexer:test-data--Integration:1.0.0 | tenant1-indexer-test-data--integration-1.0.0 | index_records_1 | - | tenant1:indexer:test-data--Integration:2.0.0 | tenant1-indexer-test-data--integration-2.0.0 | index_records_2 | - | tenant1:indexer:test-data--Integration:3.0.0 | tenant1-indexer-test-data--integration-3.0.0 | index_records_3 | - | tenant1:wks:master-data--Wellbore:2.0.3 | tenant1-wks-master-data--wellbore-2.0.3 | r3-index_record_wks_master | + | kind | index | schemaFile | + | tenant1:indexer:test-data--Integration:1.0.0 | tenant1-indexer-test-data--integration-1.0.0 | index_records_1 | + | tenant1:indexer:test-data--Integration:2.0.0 | tenant1-indexer-test-data--integration-2.0.0 | index_records_2 | + | tenant1:indexer:test-data--Integration:3.0.0 | tenant1-indexer-test-data--integration-3.0.0 | index_records_3 | + | tenant1:wks:master-data--Wellbore:2.0.3 | tenant1-wks-master-data--wellbore-2.0.3 | r3-index_record_wks_master | + | tenant1:wks:ArraysOfObjectsTestCollection:1.0.0 | tenant1-wks-arraysofobjectstestcollection-1.0.0 | r3-index_record_arrayofobjects | Scenario Outline: Ingest the record and Index in the Elastic Search When I ingest records with the with for a given @@ -42,3 +43,11 @@ Feature: Indexing of the documents Examples: | kind | recordFile | number | index | acl | field | top_left_latitude | top_left_longitude | bottom_right_latitude | bottom_right_longitude | | "tenant1:wks:master-data--Wellbore:2.0.3" | "r3-index_record_wks_master" | 1 | "tenant1-wks-master-data--wellbore-2.0.3" | "data.default.viewers@tenant1" | "data.SpatialLocation.Wgs84Coordinates" | 52 | -100 | 0 | 100 | + + Scenario Outline: Ingest the r3-record with arrays of objects and hints in schema and Index in the Elastic Search + When I ingest records with the with for a given + Then I should be able search documents for the by nested and properties (, ) and (, ) + + Examples: + | kind | recordFile | number | index | acl | path | first_nested_field | first_nested_value | second_nested_field | second_nested_value | + | "tenant1:wks:ArraysOfObjectsTestCollection:1.0.0" | "r3-index_record_arrayofobjects" | 1 | "tenant1-wks-arraysofobjectstestcollection-1.0.0" | "data.default.viewers@tenant1" | "data.NestedTest" | "data.NestedTest.NumberTest" | 12345 | "data.NestedTest.StringTest" | "test string" | \ No newline at end of file diff --git a/testing/indexer-test-core/src/main/resources/testData/r3-index_record_arrayofobjects.json b/testing/indexer-test-core/src/main/resources/testData/r3-index_record_arrayofobjects.json new file mode 100644 index 00000000..4a820133 --- /dev/null +++ b/testing/indexer-test-core/src/main/resources/testData/r3-index_record_arrayofobjects.json @@ -0,0 +1,43 @@ +[ + { + "id": "tenant1::testIngest70", + "data": { + "NestedTest": [ + { + "DateTimeTest": "2020-02-13T09:13:15.55Z", + "NumberTest": "12345", + "StringTest": "test string" + }, + { + "DateTimeTest": "2020-02-13T09:13:15.55Z", + "NumberTest": "567890", + "StringTest": "test string" + } + ], + "FlattenedTest": [ + { + "DateTimeTest": "2020-02-13T09:13:15.55Z", + "NumberTest": "12345", + "StringTest": "test string" + }, + { + "DateTimeTest": "2020-02-13T09:13:15.55Z", + "NumberTest": "567890", + "StringTest": "test string" + } + ], + "ObjectTest": [ + { + "DateTimeTest": "2020-02-13T09:13:15.55Z", + "NumberTest": "12345", + "StringTest": "test string" + }, + { + "DateTimeTest": "2020-02-13T09:13:15.55Z", + "NumberTest": "567890", + "StringTest": "test string" + } + ] + } + } +] \ No newline at end of file diff --git a/testing/indexer-test-core/src/main/resources/testData/r3-index_record_arrayofobjects.schema.json b/testing/indexer-test-core/src/main/resources/testData/r3-index_record_arrayofobjects.schema.json new file mode 100644 index 00000000..780a82cd --- /dev/null +++ b/testing/indexer-test-core/src/main/resources/testData/r3-index_record_arrayofobjects.schema.json @@ -0,0 +1,112 @@ +{ + "schemaInfo": { + "schemaIdentity": { + "authority": "tenant1", + "source": "wks", + "entityType": "ArraysOfObjectsTestCollection", + "schemaVersionMajor": 1, + "schemaVersionMinor": 0, + "schemaVersionPatch": 0 + }, + "status": "DEVELOPMENT" + }, + "schema": { + "x-osdu-license": "Copyright 2021, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.", + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "Array of objects testing schema", + "title": "Test", + "type": "object", + "required": [ + "kind", + "acl", + "legal" + ], + "properties": { + "data": { + "allOf": [ + { + "type": "object", + "properties": { + "NestedTest": { + "description": "nested type test", + "type": "array", + "x-osdu-indexing": { + "type": "nested" + }, + "items": { + "type": "object", + "properties": { + "DateTimeTest": { + "description": "date and time test", + "type": "string", + "format": "date-time", + "x-osdu-frame-of-reference": "DateTime" + }, + "NumberTest": { + "description": "number test", + "type": "number" + }, + "StringTest": { + "description": "string test", + "type": "string" + } + } + } + }, + "FlattenedTest": { + "description": "flattened type test", + "type": "array", + "x-osdu-indexing": { + "type": "flattened" + }, + "items": { + "type": "object", + "properties": { + "DateTimeTest": { + "description": "date and time test", + "type": "string", + "format": "date-time", + "x-osdu-frame-of-reference": "DateTime" + }, + "NumberTest": { + "description": "number test", + "type": "number" + }, + "StringTest": { + "description": "string test", + "type": "string" + } + } + } + }, + "ObjectTest": { + "description": "default object type test", + "type": "array", + "items": { + "type": "object", + "properties": { + "DateTimeTest": { + "description": "date and time test", + "type": "string", + "format": "date-time", + "x-osdu-frame-of-reference": "DateTime" + }, + "NumberTest": { + "description": "number test", + "type": "number" + }, + "StringTest": { + "description": "string test", + "type": "string" + } + } + } + } + } + } + ] + } + }, + "x-osdu-inheriting-from-kind": [] + } +} \ No newline at end of file diff --git a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java index 4978ddfc..4d9708e9 100644 --- a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java +++ b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java @@ -6,7 +6,7 @@ import org.junit.runner.RunWith; @RunWith(Cucumber.class) @CucumberOptions( - features = "classpath:features/indexrecord/IndexRecord.feature", + features = "classpath:features/indexrecord/indexRecord-schema-service.feature", glue = {"classpath:org.opengroup.osdu.step_definitions/index/record"}, plugin = {"pretty", "junit:target/cucumber-reports/TEST-indexrecord.xml"}) public class RunTest { diff --git a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java index 348474e7..1d435fd8 100644 --- a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java +++ b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java @@ -1,21 +1,21 @@ package org.opengroup.osdu.step_definitions.index.record; -import lombok.extern.java.Log; -import org.opengroup.osdu.common.RecordSteps; -import org.opengroup.osdu.util.GCPHTTPClient; - +import cucumber.api.DataTable; import cucumber.api.Scenario; import cucumber.api.java.Before; -import cucumber.api.DataTable; import cucumber.api.java.en.Given; import cucumber.api.java.en.Then; import cucumber.api.java.en.When; +import lombok.extern.java.Log; +import org.opengroup.osdu.common.SchemaServiceRecordSteps; +import org.opengroup.osdu.util.ElasticUtils; +import org.opengroup.osdu.util.GCPHTTPClient; @Log -public class Steps extends RecordSteps { +public class Steps extends SchemaServiceRecordSteps { public Steps() { - super(new GCPHTTPClient()); + super(new GCPHTTPClient(), new ElasticUtils()); } @Before @@ -49,4 +49,23 @@ public class Steps extends RecordSteps { super.iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(expectedCount, index, skippedAttributes); } + @Then("^I should be able to search (\\d+) record with index \"([^\"]*)\" by tag \"([^\"]*)\" and value \"([^\"]*)\"$") + public void iShouldBeAbleToSearchRecordByTagKeyAndTagValue(int expectedNumber, String index, String tagKey, String tagValue) throws Throwable { + super.iShouldBeAbleToSearchRecordByTagKeyAndTagValue(index, tagKey, tagValue, expectedNumber); + } + + @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by bounding box query with points \\((-?\\d+), (-?\\d+)\\) and \\((-?\\d+), (-?\\d+)\\) on field \"([^\"]*)\"$") + public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery( + int expectedCount, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field) throws Throwable { + super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery(expectedCount, index, topLatitude, topLongitude, bottomLatitude, bottomLongitude, field); + } + + @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by nested \"([^\"]*)\" and properties \\(\"([^\"]*)\", (\\d+)\\) and \\(\"([^\"]*)\", \"([^\"]*)\"\\)$") + public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery( + int expectedCount, String index, String path, String firstNestedProperty, String firstNestedValue, String secondNestedProperty, + String secondNestedValue) throws Exception { + super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery(expectedCount, index, path, firstNestedProperty, firstNestedValue, + secondNestedProperty, secondNestedValue); + } + } \ No newline at end of file -- GitLab From 2df0e08e3caa070b16f022fa15f1fa9027b2b92a Mon Sep 17 00:00:00 2001 From: Rustam_Lotsmanenko Date: Fri, 23 Apr 2021 16:26:46 +0400 Subject: [PATCH 06/15] class cast bugfix --- .../main/java/org/opengroup/osdu/indexer/util/TypeMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java index 5887e7d9..2e7a41d0 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java @@ -102,7 +102,7 @@ public class TypeMapper { if(isMap(indexerType)){ Map type = (Map) indexerType; - HashMap propertiesMap = (HashMap) type.get(Constants.PROPERTIES); + Map propertiesMap = (Map) type.get(Constants.PROPERTIES); for (Map.Entry entry : propertiesMap.entrySet()){ entry.setValue(Records.Type.builder().type(entry.getValue().toString()).build()); } -- GitLab From 88df491a6c0f0493f6a508a98ac27c5aaffce49a Mon Sep 17 00:00:00 2001 From: Rustam_Lotsmanenko Date: Fri, 23 Apr 2021 17:34:17 +0400 Subject: [PATCH 07/15] migrate int tests to v3 schema, added cases for hints in schema --- .../opengroup/osdu/common/RecordSteps.java | 19 +++++++--- .../osdu/common/SchemaServiceRecordSteps.java | 17 ++++++--- .../org/opengroup/osdu/models/TestIndex.java | 5 ++- .../org/opengroup/osdu/util/ElasticUtils.java | 18 +++++++++ .../osdu/util/IndexerClientUtil.java | 38 +++++++++++++++++++ .../indexRecord-schema-service.feature | 7 ++-- .../r3-index_record_arrayofobjects.json | 2 +- ...r3-index_record_arrayofobjects.schema.json | 2 +- .../step_definitions/index/record/Steps.java | 18 ++++++--- 9 files changed, 104 insertions(+), 22 deletions(-) create mode 100644 testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/IndexerClientUtil.java diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java index 322faff8..da05637c 100644 --- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java +++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java @@ -151,20 +151,29 @@ public class RecordSteps extends TestsBase { public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery ( int expectedNumber, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field) throws Throwable { index = generateActualName(index, timeStamp); - long numOfIndexedDocuments = createIndex(index); - long actualNumberOfRecords = elasticUtils.fetchRecordsByBoundingBoxQuery(index, field, topLatitude, topLongitude, bottomLatitude, bottomLongitude); + String actualName = generateActualName(index, timeStamp); + long numOfIndexedDocuments = createIndex(actualName); + long actualNumberOfRecords = elasticUtils.fetchRecordsByBoundingBoxQuery(actualName, field, topLatitude, topLongitude, bottomLatitude, bottomLongitude); assertEquals(expectedNumber, actualNumberOfRecords); } public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery( int expectedNumber, String index, String path, String firstNestedField, String firstNestedValue, String secondNestedField, String secondNestedValue) throws Exception { - - long numOfIndexedDocuments = createIndex(index); - long actualNumberOfRecords = elasticUtils.fetchRecordsByNestedQuery(index, path, firstNestedField, firstNestedValue, secondNestedField, secondNestedValue); + String actualName = generateActualName(index, timeStamp); + long numOfIndexedDocuments = createIndex(actualName); + long actualNumberOfRecords = elasticUtils.fetchRecordsByNestedQuery(actualName, path, firstNestedField, firstNestedValue, secondNestedField, secondNestedValue); assertEquals(expectedNumber, actualNumberOfRecords); } + public void i_should_be_able_search_documents_for_the_by_flattened_inner_properties(int expectedCount, String index, String flattenedField, + String flattenedFieldValue) throws IOException, InterruptedException { + String actualName = generateActualName(index, timeStamp); + long numOfIndexedDocuments = createIndex(actualName); + long actualNumberOfRecords = elasticUtils.fetchRecordsWithFlattenedFieldsQuery(actualName, flattenedField, flattenedFieldValue); + assertEquals(expectedCount, actualNumberOfRecords); + } + private long createIndex(String index) throws InterruptedException, IOException { long numOfIndexedDocuments = 0; int iterator; diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/SchemaServiceRecordSteps.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/SchemaServiceRecordSteps.java index 993c26b2..05acd914 100644 --- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/SchemaServiceRecordSteps.java +++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/SchemaServiceRecordSteps.java @@ -1,24 +1,29 @@ package org.opengroup.osdu.common; import cucumber.api.DataTable; +import java.util.List; +import java.util.Map; +import lombok.extern.java.Log; import org.opengroup.osdu.models.Setup; import org.opengroup.osdu.models.schema.PersistentSchemaTestIndex; import org.opengroup.osdu.util.ElasticUtils; import org.opengroup.osdu.util.HTTPClient; +import org.opengroup.osdu.util.IndexerClientUtil; -import java.util.List; -import java.util.Map; - +@Log public class SchemaServiceRecordSteps extends RecordSteps { + private IndexerClientUtil indexerClient; + public SchemaServiceRecordSteps(HTTPClient httpClient, ElasticUtils elasticUtils) { super(httpClient, elasticUtils); + indexerClient = new IndexerClientUtil(this.httpClient); } public void the_schema_is_created_with_the_following_kind(DataTable dataTable) { List inputList = dataTable.asList(Setup.class); inputList.forEach(this::createSchema); - inputList.forEach(s -> deleteIndex(generateActualNameWithoutTs(s.getIndex()))); + inputList.forEach(s -> deleteIndex(generateActualNameWithoutTs(s.getKind()))); super.addShutDownHook(); } @@ -33,8 +38,8 @@ public class SchemaServiceRecordSteps extends RecordSteps { super.getInputIndexMap().put(testIndex.getKind(), testIndex); } - private void deleteIndex(String index) { - this.elasticUtils.deleteIndex(index); + private void deleteIndex(String kind) { + indexerClient.deleteIndex(kind); } @Override diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/models/TestIndex.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/models/TestIndex.java index d1071270..0a5725af 100644 --- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/models/TestIndex.java +++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/models/TestIndex.java @@ -19,6 +19,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.logging.Logger; +import org.opengroup.osdu.util.IndexerClientUtil; import static org.junit.Assert.assertEquals; import static org.opengroup.osdu.util.Config.*; @@ -38,6 +39,7 @@ public class TestIndex { private HTTPClient httpClient; private Map headers; private ElasticUtils elasticUtils; + protected IndexerClientUtil indexerClient; private Gson gson = new Gson(); public TestIndex(ElasticUtils elasticUtils){ @@ -47,6 +49,7 @@ public class TestIndex { public void setHttpClient(HTTPClient httpClient) { this.httpClient = httpClient; headers = httpClient.getCommonHeader(); + indexerClient = new IndexerClientUtil(this.httpClient); } public void setupIndex() { @@ -73,7 +76,7 @@ public class TestIndex { } public void cleanupIndex() { - this.elasticUtils.deleteIndex(index); + this.indexerClient.deleteIndex(kind); } private String getRecordFile() { diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java index d42a4fd0..668d1888 100644 --- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java +++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java @@ -292,6 +292,23 @@ public class ElasticUtils { } } + + public long fetchRecordsWithFlattenedFieldsQuery(String index, String flattenedField, String flattenedFieldValue) throws IOException { + try (RestHighLevelClient client = this.createClient(username, password, host)) { + SearchRequest searchRequest = new SearchRequest(index); + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(boolQuery().must(matchQuery(flattenedField,flattenedFieldValue))); + searchRequest.source(searchSourceBuilder); + + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + return searchResponse.getHits().getTotalHits().value; + } catch (ElasticsearchStatusException e) { + log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage())); + return -1; + } + } + public Map getMapping(String index) throws IOException { try (RestHighLevelClient client = this.createClient(username, password, host)) { GetMappingsRequest request = new GetMappingsRequest(); @@ -433,4 +450,5 @@ public class ElasticUtils { } return false; } + } \ No newline at end of file diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/IndexerClientUtil.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/IndexerClientUtil.java new file mode 100644 index 00000000..33771a92 --- /dev/null +++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/IndexerClientUtil.java @@ -0,0 +1,38 @@ +package org.opengroup.osdu.util; + +import static org.opengroup.osdu.util.Config.getDataPartitionIdTenant1; +import static org.opengroup.osdu.util.Config.getIndexerBaseURL; + +import com.google.gson.Gson; +import com.sun.jersey.api.client.ClientResponse; +import java.util.Map; +import javax.ws.rs.HttpMethod; +import lombok.extern.java.Log; +import org.opengroup.osdu.core.common.model.search.RecordChangedMessages; + +@Log +public class IndexerClientUtil { + + private final String purgeMessage = "{\"data\":\"[{\\\"kind\\\":\\\"%s\\\",\\\"op\\\":\\\"purge_schema\\\"}]\",\"attributes\":{\"account-id\":\"%s\"}}"; + + private final HTTPClient httpClient; + private Map headers; + + public IndexerClientUtil(HTTPClient httpClient) { + this.httpClient = httpClient; + headers = httpClient.getCommonHeader(); + } + + public void deleteIndex(String kind) { + String url = getIndexerBaseURL() + "index-cleanup"; + log.info("URL: " + url); + ClientResponse response = httpClient.send(HttpMethod.POST, url, convertMessageIntoJson(kind), headers, httpClient.getAccessToken()); + log.info(response.toString()); + } + + private String convertMessageIntoJson(String kind) { + RecordChangedMessages + recordChangedMessages = (new Gson()).fromJson(String.format(purgeMessage, kind, getDataPartitionIdTenant1()), RecordChangedMessages.class); + return new Gson().toJson(recordChangedMessages); + } +} diff --git a/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature b/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature index 62530314..2991c969 100644 --- a/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature +++ b/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature @@ -8,7 +8,7 @@ Feature: Indexing of the documents | tenant1:indexer:test-data--Integration:2.0.0 | tenant1-indexer-test-data--integration-2.0.0 | index_records_2 | | tenant1:indexer:test-data--Integration:3.0.0 | tenant1-indexer-test-data--integration-3.0.0 | index_records_3 | | tenant1:wks:master-data--Wellbore:2.0.3 | tenant1-wks-master-data--wellbore-2.0.3 | r3-index_record_wks_master | - | tenant1:wks:ArraysOfObjectsTestCollection:1.0.0 | tenant1-wks-arraysofobjectstestcollection-1.0.0 | r3-index_record_arrayofobjects | + | tenant1:wks:ArraysOfObjectsTestCollection:4.0.0 | tenant1-wks-arraysofobjectstestcollection-4.0.0 | r3-index_record_arrayofobjects | Scenario Outline: Ingest the record and Index in the Elastic Search When I ingest records with the with for a given @@ -47,7 +47,8 @@ Feature: Indexing of the documents Scenario Outline: Ingest the r3-record with arrays of objects and hints in schema and Index in the Elastic Search When I ingest records with the with for a given Then I should be able search documents for the by nested and properties (, ) and (, ) + Then I should be able search documents for the by flattened inner properties (, ) Examples: - | kind | recordFile | number | index | acl | path | first_nested_field | first_nested_value | second_nested_field | second_nested_value | - | "tenant1:wks:ArraysOfObjectsTestCollection:1.0.0" | "r3-index_record_arrayofobjects" | 1 | "tenant1-wks-arraysofobjectstestcollection-1.0.0" | "data.default.viewers@tenant1" | "data.NestedTest" | "data.NestedTest.NumberTest" | 12345 | "data.NestedTest.StringTest" | "test string" | \ No newline at end of file + | kind | recordFile | number | index | acl | path | first_nested_field | first_nested_value | second_nested_field | second_nested_value | flattened_inner_field | flattened_inner_value | + | "tenant1:wks:ArraysOfObjectsTestCollection:4.0.0" | "r3-index_record_arrayofobjects" | 1 | "tenant1-wks-arraysofobjectstestcollection-4.0.0" | "data.default.viewers@tenant1" | "data.NestedTest" | "data.NestedTest.NumberTest" | 12345 | "data.NestedTest.StringTest" | "test string" | "data.FlattenedTest.StringTest" | "test string" | \ No newline at end of file diff --git a/testing/indexer-test-core/src/main/resources/testData/r3-index_record_arrayofobjects.json b/testing/indexer-test-core/src/main/resources/testData/r3-index_record_arrayofobjects.json index 4a820133..f37563c9 100644 --- a/testing/indexer-test-core/src/main/resources/testData/r3-index_record_arrayofobjects.json +++ b/testing/indexer-test-core/src/main/resources/testData/r3-index_record_arrayofobjects.json @@ -1,6 +1,6 @@ [ { - "id": "tenant1::testIngest70", + "id": "tenant1::testIngest70", "data": { "NestedTest": [ { diff --git a/testing/indexer-test-core/src/main/resources/testData/r3-index_record_arrayofobjects.schema.json b/testing/indexer-test-core/src/main/resources/testData/r3-index_record_arrayofobjects.schema.json index 780a82cd..c68eface 100644 --- a/testing/indexer-test-core/src/main/resources/testData/r3-index_record_arrayofobjects.schema.json +++ b/testing/indexer-test-core/src/main/resources/testData/r3-index_record_arrayofobjects.schema.json @@ -4,7 +4,7 @@ "authority": "tenant1", "source": "wks", "entityType": "ArraysOfObjectsTestCollection", - "schemaVersionMajor": 1, + "schemaVersionMajor": 4, "schemaVersionMinor": 0, "schemaVersionPatch": 0 }, diff --git a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java index 1d435fd8..186a9fc4 100644 --- a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java +++ b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java @@ -6,6 +6,7 @@ import cucumber.api.java.Before; import cucumber.api.java.en.Given; import cucumber.api.java.en.Then; import cucumber.api.java.en.When; +import java.io.IOException; import lombok.extern.java.Log; import org.opengroup.osdu.common.SchemaServiceRecordSteps; import org.opengroup.osdu.util.ElasticUtils; @@ -48,12 +49,13 @@ public class Steps extends SchemaServiceRecordSteps { public void iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(int expectedCount, String index, String skippedAttributes) throws Throwable { super.iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(expectedCount, index, skippedAttributes); } +//TODO fix tags step - @Then("^I should be able to search (\\d+) record with index \"([^\"]*)\" by tag \"([^\"]*)\" and value \"([^\"]*)\"$") - public void iShouldBeAbleToSearchRecordByTagKeyAndTagValue(int expectedNumber, String index, String tagKey, String tagValue) throws Throwable { - super.iShouldBeAbleToSearchRecordByTagKeyAndTagValue(index, tagKey, tagValue, expectedNumber); - } - +// @Then("^I should be able to search (\\d+) record with index \"([^\"]*)\" by tag \"([^\"]*)\" and value \"([^\"]*)\"$") +// public void iShouldBeAbleToSearchRecordByTagKeyAndTagValue(int expectedNumber, String index, String tagKey, String tagValue) throws Throwable { +// super.iShouldBeAbleToSearchRecordByTagKeyAndTagValue(index, tagKey, tagValue, expectedNumber); +// } +// @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by bounding box query with points \\((-?\\d+), (-?\\d+)\\) and \\((-?\\d+), (-?\\d+)\\) on field \"([^\"]*)\"$") public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery( int expectedCount, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field) throws Throwable { @@ -68,4 +70,10 @@ public class Steps extends SchemaServiceRecordSteps { secondNestedProperty, secondNestedValue); } + @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by flattened inner properties \\(\"([^\"]*)\", \"([^\"]*)\"\\)$") + public void i_should_be_able_search_documents_for_the_by_flattened_inner_properties(int expectedCount, String index, String flattenedField, String flattenedFieldValue) + throws IOException, InterruptedException { + super.i_should_be_able_search_documents_for_the_by_flattened_inner_properties(expectedCount,index,flattenedField,flattenedFieldValue); + + } } \ No newline at end of file -- GitLab From 4920f6eafc3ec52563d3169bf604730306111c89 Mon Sep 17 00:00:00 2001 From: Rustam_Lotsmanenko Date: Fri, 23 Apr 2021 18:07:40 +0400 Subject: [PATCH 08/15] review fix --- .../indexer/schema/converter/PropertiesProcessor.java | 3 +++ .../indexer/service/StorageIndexerPayloadMapper.java | 11 ++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java index 2731083d..0c57b560 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java @@ -173,6 +173,9 @@ public class PropertiesProcessor { if ("array".equals(entry.getValue().getType())) { Items items = entry.getValue().getItems(); + if(Objects.isNull(items)){ + return Stream.empty(); + } if(Objects.nonNull(items.getProperties()) && !items.getProperties().isEmpty()){ Map type = entry.getValue().getIndexingType(); diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java index 59fb3ecd..1af15db2 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java @@ -42,6 +42,7 @@ public class StorageIndexerPayloadMapper { Map dataMap = new HashMap<>(); if (storageSchema.isDataSchemaMissing()) { + this.log.warning(String.format("record-id: %s | schema mismatching: %s ", recordId, storageSchema.getKind())); return dataMap; } @@ -65,8 +66,8 @@ public class StorageIndexerPayloadMapper { if (Objects.isNull(elasticType)) { this.jobStatus - .addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, "e.getMessage()", - String.format("record-id: %s | %s", recordId, "e.getMessage()")); + .addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, + String.format("record-id: %s | %s for entry %s", recordId, "Not resolvable elastic type", name)); continue; } @@ -128,14 +129,14 @@ public class StorageIndexerPayloadMapper { this.attributeParsingService.tryParseGeojson(recordId, name, value, dataMap); break; case NESTED: - // don't do anything , each nested property will be mapped separately + // don't do anything , each nested property will be parsed separately break; case FLATTENED: - // flattened type inner properties won't be mapped as they types not present in schema + // flattened type inner properties will be added "as is" without parsing as they types not present in schema this.attributeParsingService.tryParseFlattened(recordId, name, value, dataMap); break; case OBJECT: - // object type inner properties won't be mapped as they types not present in schema + // object type inner properties will be added "as is" without parsing as they types not present in schema this.attributeParsingService.tryParseObject(recordId, name, value, dataMap); break; case UNDEFINED: -- GitLab From 914a0049ab18bf57f45e966ac06fb2a5bf9e2efc Mon Sep 17 00:00:00 2001 From: Rustam_Lotsmanenko Date: Sat, 24 Apr 2021 20:38:08 +0400 Subject: [PATCH 09/15] undo test-core index deletion changes & records steps --- .../opengroup/osdu/common/RecordSteps.java | 15 ++++------ .../osdu/common/SchemaServiceRecordSteps.java | 17 ++++------- .../org/opengroup/osdu/models/TestIndex.java | 5 +--- .../step_definitions/index/record/Steps.java | 30 +++++++++++-------- .../opengroup/osdu/util/GcpElasticUtils.java | 20 +++++++++++++ 5 files changed, 50 insertions(+), 37 deletions(-) create mode 100644 testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/util/GcpElasticUtils.java diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java index da05637c..fbf12a6f 100644 --- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java +++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java @@ -151,26 +151,23 @@ public class RecordSteps extends TestsBase { public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery ( int expectedNumber, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field) throws Throwable { index = generateActualName(index, timeStamp); - String actualName = generateActualName(index, timeStamp); - long numOfIndexedDocuments = createIndex(actualName); - long actualNumberOfRecords = elasticUtils.fetchRecordsByBoundingBoxQuery(actualName, field, topLatitude, topLongitude, bottomLatitude, bottomLongitude); + long numOfIndexedDocuments = createIndex(index); + long actualNumberOfRecords = elasticUtils.fetchRecordsByBoundingBoxQuery(index, field, topLatitude, topLongitude, bottomLatitude, bottomLongitude); assertEquals(expectedNumber, actualNumberOfRecords); } public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery( int expectedNumber, String index, String path, String firstNestedField, String firstNestedValue, String secondNestedField, String secondNestedValue) throws Exception { - String actualName = generateActualName(index, timeStamp); - long numOfIndexedDocuments = createIndex(actualName); - long actualNumberOfRecords = elasticUtils.fetchRecordsByNestedQuery(actualName, path, firstNestedField, firstNestedValue, secondNestedField, secondNestedValue); + long numOfIndexedDocuments = createIndex(index); + long actualNumberOfRecords = elasticUtils.fetchRecordsByNestedQuery(index, path, firstNestedField, firstNestedValue, secondNestedField, secondNestedValue); assertEquals(expectedNumber, actualNumberOfRecords); } public void i_should_be_able_search_documents_for_the_by_flattened_inner_properties(int expectedCount, String index, String flattenedField, String flattenedFieldValue) throws IOException, InterruptedException { - String actualName = generateActualName(index, timeStamp); - long numOfIndexedDocuments = createIndex(actualName); - long actualNumberOfRecords = elasticUtils.fetchRecordsWithFlattenedFieldsQuery(actualName, flattenedField, flattenedFieldValue); + long numOfIndexedDocuments = createIndex(index); + long actualNumberOfRecords = elasticUtils.fetchRecordsWithFlattenedFieldsQuery(index, flattenedField, flattenedFieldValue); assertEquals(expectedCount, actualNumberOfRecords); } diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/SchemaServiceRecordSteps.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/SchemaServiceRecordSteps.java index 05acd914..993c26b2 100644 --- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/SchemaServiceRecordSteps.java +++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/SchemaServiceRecordSteps.java @@ -1,29 +1,24 @@ package org.opengroup.osdu.common; import cucumber.api.DataTable; -import java.util.List; -import java.util.Map; -import lombok.extern.java.Log; import org.opengroup.osdu.models.Setup; import org.opengroup.osdu.models.schema.PersistentSchemaTestIndex; import org.opengroup.osdu.util.ElasticUtils; import org.opengroup.osdu.util.HTTPClient; -import org.opengroup.osdu.util.IndexerClientUtil; -@Log -public class SchemaServiceRecordSteps extends RecordSteps { +import java.util.List; +import java.util.Map; - private IndexerClientUtil indexerClient; +public class SchemaServiceRecordSteps extends RecordSteps { public SchemaServiceRecordSteps(HTTPClient httpClient, ElasticUtils elasticUtils) { super(httpClient, elasticUtils); - indexerClient = new IndexerClientUtil(this.httpClient); } public void the_schema_is_created_with_the_following_kind(DataTable dataTable) { List inputList = dataTable.asList(Setup.class); inputList.forEach(this::createSchema); - inputList.forEach(s -> deleteIndex(generateActualNameWithoutTs(s.getKind()))); + inputList.forEach(s -> deleteIndex(generateActualNameWithoutTs(s.getIndex()))); super.addShutDownHook(); } @@ -38,8 +33,8 @@ public class SchemaServiceRecordSteps extends RecordSteps { super.getInputIndexMap().put(testIndex.getKind(), testIndex); } - private void deleteIndex(String kind) { - indexerClient.deleteIndex(kind); + private void deleteIndex(String index) { + this.elasticUtils.deleteIndex(index); } @Override diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/models/TestIndex.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/models/TestIndex.java index 0a5725af..d1071270 100644 --- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/models/TestIndex.java +++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/models/TestIndex.java @@ -19,7 +19,6 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.logging.Logger; -import org.opengroup.osdu.util.IndexerClientUtil; import static org.junit.Assert.assertEquals; import static org.opengroup.osdu.util.Config.*; @@ -39,7 +38,6 @@ public class TestIndex { private HTTPClient httpClient; private Map headers; private ElasticUtils elasticUtils; - protected IndexerClientUtil indexerClient; private Gson gson = new Gson(); public TestIndex(ElasticUtils elasticUtils){ @@ -49,7 +47,6 @@ public class TestIndex { public void setHttpClient(HTTPClient httpClient) { this.httpClient = httpClient; headers = httpClient.getCommonHeader(); - indexerClient = new IndexerClientUtil(this.httpClient); } public void setupIndex() { @@ -76,7 +73,7 @@ public class TestIndex { } public void cleanupIndex() { - this.indexerClient.deleteIndex(kind); + this.elasticUtils.deleteIndex(index); } private String getRecordFile() { diff --git a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java index 186a9fc4..bc1ee68b 100644 --- a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java +++ b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java @@ -11,6 +11,7 @@ import lombok.extern.java.Log; import org.opengroup.osdu.common.SchemaServiceRecordSteps; import org.opengroup.osdu.util.ElasticUtils; import org.opengroup.osdu.util.GCPHTTPClient; +import org.opengroup.osdu.util.GcpElasticUtils; @Log public class Steps extends SchemaServiceRecordSteps { @@ -23,6 +24,7 @@ public class Steps extends SchemaServiceRecordSteps { public void before(Scenario scenario) { this.scenario = scenario; this.httpClient = new GCPHTTPClient(); + this.elasticUtils = new GcpElasticUtils(this.httpClient); } @Given("^the schema is created with the following kind$") @@ -41,39 +43,41 @@ public class Steps extends SchemaServiceRecordSteps { } @Then("^I should get the elastic \"(.*?)\" for the \"([^\"]*)\" and \"([^\"]*)\" in the Elastic Search$") - public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String type, String index) throws Throwable { + public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String type, String index) + throws Throwable { super.i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(expectedMapping, type, index); } @Then("^I should get the (\\d+) documents for the \"([^\"]*)\" in the Elastic Search with out \"(.*?)\"$") - public void iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(int expectedCount, String index, String skippedAttributes) throws Throwable { + public void iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(int expectedCount, String index, String skippedAttributes) + throws Throwable { super.iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(expectedCount, index, skippedAttributes); } -//TODO fix tags step -// @Then("^I should be able to search (\\d+) record with index \"([^\"]*)\" by tag \"([^\"]*)\" and value \"([^\"]*)\"$") -// public void iShouldBeAbleToSearchRecordByTagKeyAndTagValue(int expectedNumber, String index, String tagKey, String tagValue) throws Throwable { -// super.iShouldBeAbleToSearchRecordByTagKeyAndTagValue(index, tagKey, tagValue, expectedNumber); -// } -// @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by bounding box query with points \\((-?\\d+), (-?\\d+)\\) and \\((-?\\d+), (-?\\d+)\\) on field \"([^\"]*)\"$") public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery( - int expectedCount, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field) throws Throwable { - super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery(expectedCount, index, topLatitude, topLongitude, bottomLatitude, bottomLongitude, field); + int expectedCount, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field) + throws Throwable { + String actualName = generateActualName(index, null); + super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery(expectedCount, actualName, topLatitude, topLongitude, bottomLatitude, + bottomLongitude, field); } @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by nested \"([^\"]*)\" and properties \\(\"([^\"]*)\", (\\d+)\\) and \\(\"([^\"]*)\", \"([^\"]*)\"\\)$") public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery( int expectedCount, String index, String path, String firstNestedProperty, String firstNestedValue, String secondNestedProperty, String secondNestedValue) throws Exception { - super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery(expectedCount, index, path, firstNestedProperty, firstNestedValue, + String actualName = generateActualName(index, null); + super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery(expectedCount, actualName, path, firstNestedProperty, firstNestedValue, secondNestedProperty, secondNestedValue); } @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by flattened inner properties \\(\"([^\"]*)\", \"([^\"]*)\"\\)$") - public void i_should_be_able_search_documents_for_the_by_flattened_inner_properties(int expectedCount, String index, String flattenedField, String flattenedFieldValue) + public void i_should_be_able_search_documents_for_the_by_flattened_inner_properties(int expectedCount, String index, String flattenedField, + String flattenedFieldValue) throws IOException, InterruptedException { - super.i_should_be_able_search_documents_for_the_by_flattened_inner_properties(expectedCount,index,flattenedField,flattenedFieldValue); + String actualName = generateActualName(index, null); + super.i_should_be_able_search_documents_for_the_by_flattened_inner_properties(expectedCount, actualName, flattenedField, flattenedFieldValue); } } \ No newline at end of file diff --git a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/util/GcpElasticUtils.java b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/util/GcpElasticUtils.java new file mode 100644 index 00000000..ce781c6e --- /dev/null +++ b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/util/GcpElasticUtils.java @@ -0,0 +1,20 @@ +package org.opengroup.osdu.util; + +public class GcpElasticUtils extends ElasticUtils { + + private IndexerClientUtil indexerClientUtil; + + public GcpElasticUtils(HTTPClient httpClient) { + super(); + this.indexerClientUtil = new IndexerClientUtil(httpClient); + } + + @Override + public void deleteIndex(String index) { + indexerClientUtil.deleteIndex(convertIndexToKindName(index)); + } + + private String convertIndexToKindName(String index) { + return index.replaceAll("-",":"); + } +} -- GitLab From b803c74ac767890c616f1ccf13d4fe314e4ce923 Mon Sep 17 00:00:00 2001 From: Rustam_Lotsmanenko Date: Tue, 27 Apr 2021 16:37:46 +0400 Subject: [PATCH 10/15] review fix --- .../osdu/indexer/schema/converter/readme.md | 66 ++++++++++- .../service/StorageIndexerPayloadMapper.java | 3 - .../converter/PropertiesProcessorTest.java | 107 ++++++++++++++++++ .../opengroup/osdu/common/RecordSteps.java | 21 +++- .../osdu/models/record/RecordData.java | 18 +++ .../org/opengroup/osdu/util/ElasticUtils.java | 19 ++++ .../indexRecord-schema-service.feature | 5 +- .../step_definitions/index/record/Steps.java | 13 ++- 8 files changed, 238 insertions(+), 14 deletions(-) create mode 100644 testing/indexer-test-core/src/main/java/org/opengroup/osdu/models/record/RecordData.java diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/readme.md b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/readme.md index 4c39a276..17a4ca99 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/readme.md +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/readme.md @@ -203,8 +203,68 @@ Is converted to 2. Arrays -Arrays of complex types are ignored, only following arrays of primitive -types are supported +Arrays of complex types by default will be consumed as object type +```json +"Markers": { +"type": "array", + "items": { + "type": "object", + "properties": { + "NegativeVerticalDelta"{ + "description": "The distance vertically below the Marker position that marks the limit of the high confidence range for the Marker pick.", + "x-osdu-frame-of-reference": "UOM:length", + "type": "number" + }, + ..... + } + } +} + +``` +Without inner objects processing +```json +{ + path = Markers, + kind = []object +} +``` + +Processing can be specified with optional "x-osdu-indexing" property +```json +"properties": { + "Markers": { + "x-osdu-indexing": { + "type": "nested" + }, + "type": "array", + "items": { + "type": "object", + "properties": { + "NegativeVerticalDelta"{ + "description": "The distance vertically below the Marker position that marks the limit of the high confidence range for the Marker pick.", + "x-osdu-frame-of-reference": "UOM:length", + "type": "number" + }, + ..... +``` +"x-osdu-indexing" property values +```json +"nested" , "flattened" +``` +By default, only "nested" type will lead to inner objects processing +```json +{ +path = Markers, + kind = nested, + properties = [{ + path = NegativeVerticalDelta, + kind = double + }, + ..... +} +``` + +Arrays of primitive types are supported ```json "number", "string", "integer", "boolean" @@ -370,4 +430,4 @@ is converted to } ] } -``` +``` \ No newline at end of file diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java index 1af15db2..f8d197dd 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java @@ -128,9 +128,6 @@ public class StorageIndexerPayloadMapper { case GEO_SHAPE: this.attributeParsingService.tryParseGeojson(recordId, name, value, dataMap); break; - case NESTED: - // don't do anything , each nested property will be parsed separately - break; case FLATTENED: // flattened type inner properties will be added "as is" without parsing as they types not present in schema this.attributeParsingService.tryParseFlattened(recordId, name, value, dataMap); diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessorTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessorTest.java index 182bc58d..837e203f 100644 --- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessorTest.java +++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessorTest.java @@ -14,6 +14,7 @@ package org.opengroup.osdu.indexer.schema.converter; +import com.google.common.collect.ImmutableMap; import org.junit.Test; import org.mockito.Mockito; import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; @@ -22,6 +23,7 @@ import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterPropert import org.opengroup.osdu.indexer.schema.converter.tags.AllOfItem; import org.opengroup.osdu.indexer.schema.converter.tags.Definition; import org.opengroup.osdu.indexer.schema.converter.tags.Definitions; +import org.opengroup.osdu.indexer.schema.converter.tags.Items; import org.opengroup.osdu.indexer.schema.converter.tags.TypeProperty; import java.util.LinkedHashMap; @@ -101,4 +103,109 @@ public class PropertiesProcessorTest { .processItem(allOfItem).map(Object::toString).reduce("", String::concat); assertEquals("{path=" + PATH + ", kind=int}", res); } + + @Test + public void should_return_processed_nested_array_items(){ + JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class); + + Map itemsProperties = new LinkedHashMap<>(); + TypeProperty intProperty = new TypeProperty(); + intProperty.setType("integer"); + itemsProperties.put(PATH, intProperty); + + Items items = new Items(); + items.setProperties(itemsProperties); + + TypeProperty arrayProperty = new TypeProperty(); + arrayProperty.setIndexingType(ImmutableMap.of("type","nested")); + arrayProperty.setType("array"); + arrayProperty.setItems(items); + + Map allOfItemProperties = new LinkedHashMap<>(); + allOfItemProperties.put(PATH,arrayProperty); + + AllOfItem allOfItem = new AllOfItem(); + allOfItem.setProperties(allOfItemProperties); + + String res = new PropertiesProcessor(Mockito.mock(Definitions.class), log, new SchemaConverterPropertiesConfig()) + .processItem(allOfItem).map(Object::toString).reduce("", String::concat); + assertEquals("{path="+ PATH + ", kind=nested, properties=[{path="+ PATH + ", kind=int}]}",res); + } + + @Test + public void should_return_flattened_array_without_processing(){ + JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class); + + Map itemsProperties = new LinkedHashMap<>(); + TypeProperty intProperty = new TypeProperty(); + intProperty.setType("integer"); + itemsProperties.put(PATH, intProperty); + + Items items = new Items(); + items.setProperties(itemsProperties); + + TypeProperty arrayProperty = new TypeProperty(); + arrayProperty.setIndexingType(ImmutableMap.of("type","flattened")); + arrayProperty.setType("array"); + arrayProperty.setItems(items); + + Map allOfItemProperties = new LinkedHashMap<>(); + allOfItemProperties.put(PATH,arrayProperty); + + AllOfItem allOfItem = new AllOfItem(); + allOfItem.setProperties(allOfItemProperties); + + String res = new PropertiesProcessor(Mockito.mock(Definitions.class), log, new SchemaConverterPropertiesConfig()) + .processItem(allOfItem).map(Object::toString).reduce("", String::concat); + assertEquals("{path="+ PATH + ", kind=flattened}",res); + } + + @Test + public void should_return_object_array_without_hints_in_schema_without_processing(){ + JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class); + + Map itemsProperties = new LinkedHashMap<>(); + TypeProperty intProperty = new TypeProperty(); + intProperty.setType("integer"); + itemsProperties.put(PATH, intProperty); + + Items items = new Items(); + items.setProperties(itemsProperties); + + TypeProperty arrayProperty = new TypeProperty(); + arrayProperty.setType("array"); + arrayProperty.setItems(items); + + Map allOfItemProperties = new LinkedHashMap<>(); + allOfItemProperties.put(PATH,arrayProperty); + + AllOfItem allOfItem = new AllOfItem(); + allOfItem.setProperties(allOfItemProperties); + + String res = new PropertiesProcessor(Mockito.mock(Definitions.class), log, new SchemaConverterPropertiesConfig()) + .processItem(allOfItem).map(Object::toString).reduce("", String::concat); + assertEquals("{path="+ PATH + ", kind=[]object}",res); + } + + @Test + public void should_process_not_object_array_type(){ + JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class); + + Items items = new Items(); + items.setType("integer"); + + TypeProperty arrayProperty = new TypeProperty(); + arrayProperty.setType("array"); + arrayProperty.setItems(items); + + Map allOfItemProperties = new LinkedHashMap<>(); + allOfItemProperties.put(PATH,arrayProperty); + + AllOfItem allOfItem = new AllOfItem(); + allOfItem.setProperties(allOfItemProperties); + + String res = new PropertiesProcessor(Mockito.mock(Definitions.class), log, new SchemaConverterPropertiesConfig()) + .processItem(allOfItem).map(Object::toString).reduce("", String::concat); + assertEquals("{path="+ PATH + ", kind=[]int}",res); + } } \ No newline at end of file diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java index fbf12a6f..4da17d87 100644 --- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java +++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java @@ -1,5 +1,6 @@ package org.opengroup.osdu.common; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.MapDifference; import com.google.common.collect.Maps; import com.google.gson.Gson; @@ -11,6 +12,7 @@ import org.elasticsearch.cluster.metadata.MappingMetadata; import org.opengroup.osdu.core.common.model.entitlements.Acl; import org.opengroup.osdu.models.Setup; import org.opengroup.osdu.models.TestIndex; +import org.opengroup.osdu.models.record.RecordData; import org.opengroup.osdu.util.ElasticUtils; import org.opengroup.osdu.util.FileHandler; import org.opengroup.osdu.util.HTTPClient; @@ -32,6 +34,7 @@ import static org.opengroup.osdu.util.Config.getStorageBaseURL; @Log public class RecordSteps extends TestsBase { private Map inputIndexMap = new HashMap<>(); + private ObjectMapper mapper = new ObjectMapper(); private boolean shutDownHookAdded = false; private String timeStamp = String.valueOf(System.currentTimeMillis()); @@ -158,19 +161,33 @@ public class RecordSteps extends TestsBase { public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery( int expectedNumber, String index, String path, String firstNestedField, String firstNestedValue, String secondNestedField, String secondNestedValue) - throws Exception { + throws Throwable { long numOfIndexedDocuments = createIndex(index); long actualNumberOfRecords = elasticUtils.fetchRecordsByNestedQuery(index, path, firstNestedField, firstNestedValue, secondNestedField, secondNestedValue); assertEquals(expectedNumber, actualNumberOfRecords); } public void i_should_be_able_search_documents_for_the_by_flattened_inner_properties(int expectedCount, String index, String flattenedField, - String flattenedFieldValue) throws IOException, InterruptedException { + String flattenedFieldValue) throws Throwable { long numOfIndexedDocuments = createIndex(index); long actualNumberOfRecords = elasticUtils.fetchRecordsWithFlattenedFieldsQuery(index, flattenedField, flattenedFieldValue); assertEquals(expectedCount, actualNumberOfRecords); } + public void i_should_get_object_in_search_response_without_hints_in_schema(String objectField, String index, String recordFile, String acl, String kind) + throws Throwable { + long numOfIndexedDocuments = createIndex(index); + String expectedRecord = FileHandler.readFile(String.format("%s.%s", recordFile, "json")); + + RecordData[] fileRecordData = mapper.readValue(expectedRecord, RecordData[].class); + RecordData expectedRecordData = fileRecordData[0]; + + String elasticRecordData = elasticUtils.fetchDataFromObjectsArrayRecords(index); + RecordData actualRecordData = mapper.readValue(elasticRecordData, RecordData.class); + + assertEquals(expectedRecordData.getData().get(objectField),actualRecordData.getData().get(objectField)); + } + private long createIndex(String index) throws InterruptedException, IOException { long numOfIndexedDocuments = 0; int iterator; diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/models/record/RecordData.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/models/record/RecordData.java new file mode 100644 index 00000000..ec0a6a22 --- /dev/null +++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/models/record/RecordData.java @@ -0,0 +1,18 @@ +package org.opengroup.osdu.models.record; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; +import java.util.Map; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class RecordData { + + @JsonProperty("data") + private Map>> data; + + public Map>> getData() { + return data; + } + +} diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java index 668d1888..3d029aa6 100644 --- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java +++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java @@ -52,6 +52,7 @@ import org.elasticsearch.client.indices.GetIndexRequest; import org.elasticsearch.client.indices.GetMappingsRequest; import org.elasticsearch.client.indices.GetMappingsResponse; import org.elasticsearch.cluster.metadata.MappingMetadata; +import org.elasticsearch.search.SearchHits; import org.locationtech.jts.geom.Coordinate; import org.elasticsearch.common.geo.builders.EnvelopeBuilder; import org.elasticsearch.common.settings.Settings; @@ -309,6 +310,24 @@ public class ElasticUtils { } } + public String fetchDataFromObjectsArrayRecords(String index) throws IOException { + try { + try (RestHighLevelClient client = this.createClient(username, password, host)) { + SearchRequest request = new SearchRequest(index); + SearchResponse searchResponse = client.search(request, RequestOptions.DEFAULT); + + SearchHits searchHits = searchResponse.getHits(); + if (searchHits.getHits().length != 0) { + return searchHits.getHits()[0].getSourceAsString(); + } + return null; + } + } catch (ElasticsearchStatusException e) { + log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage())); + return null; + } + } + public Map getMapping(String index) throws IOException { try (RestHighLevelClient client = this.createClient(username, password, host)) { GetMappingsRequest request = new GetMappingsRequest(); diff --git a/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature b/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature index 2991c969..50051edc 100644 --- a/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature +++ b/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature @@ -48,7 +48,8 @@ Feature: Indexing of the documents When I ingest records with the with for a given Then I should be able search documents for the by nested and properties (, ) and (, ) Then I should be able search documents for the by flattened inner properties (, ) + Then I should get in response, without hints in schema for the that present in the with for a given Examples: - | kind | recordFile | number | index | acl | path | first_nested_field | first_nested_value | second_nested_field | second_nested_value | flattened_inner_field | flattened_inner_value | - | "tenant1:wks:ArraysOfObjectsTestCollection:4.0.0" | "r3-index_record_arrayofobjects" | 1 | "tenant1-wks-arraysofobjectstestcollection-4.0.0" | "data.default.viewers@tenant1" | "data.NestedTest" | "data.NestedTest.NumberTest" | 12345 | "data.NestedTest.StringTest" | "test string" | "data.FlattenedTest.StringTest" | "test string" | \ No newline at end of file + | kind | recordFile | number | index | acl | path | first_nested_field | first_nested_value | second_nested_field | second_nested_value | flattened_inner_field | flattened_inner_value | object_inner_field | + | "tenant1:wks:ArraysOfObjectsTestCollection:4.0.0" | "r3-index_record_arrayofobjects" | 1 | "tenant1-wks-arraysofobjectstestcollection-4.0.0" | "data.default.viewers@tenant1" | "data.NestedTest" | "data.NestedTest.NumberTest" | 12345 | "data.NestedTest.StringTest" | "test string" | "data.FlattenedTest.StringTest" | "test string" | "ObjectTest" | \ No newline at end of file diff --git a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java index bc1ee68b..26060454 100644 --- a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java +++ b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java @@ -6,7 +6,6 @@ import cucumber.api.java.Before; import cucumber.api.java.en.Given; import cucumber.api.java.en.Then; import cucumber.api.java.en.When; -import java.io.IOException; import lombok.extern.java.Log; import org.opengroup.osdu.common.SchemaServiceRecordSteps; import org.opengroup.osdu.util.ElasticUtils; @@ -66,7 +65,7 @@ public class Steps extends SchemaServiceRecordSteps { @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by nested \"([^\"]*)\" and properties \\(\"([^\"]*)\", (\\d+)\\) and \\(\"([^\"]*)\", \"([^\"]*)\"\\)$") public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery( int expectedCount, String index, String path, String firstNestedProperty, String firstNestedValue, String secondNestedProperty, - String secondNestedValue) throws Exception { + String secondNestedValue) throws Throwable { String actualName = generateActualName(index, null); super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery(expectedCount, actualName, path, firstNestedProperty, firstNestedValue, secondNestedProperty, secondNestedValue); @@ -74,10 +73,16 @@ public class Steps extends SchemaServiceRecordSteps { @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by flattened inner properties \\(\"([^\"]*)\", \"([^\"]*)\"\\)$") public void i_should_be_able_search_documents_for_the_by_flattened_inner_properties(int expectedCount, String index, String flattenedField, - String flattenedFieldValue) - throws IOException, InterruptedException { + String flattenedFieldValue) throws Throwable { String actualName = generateActualName(index, null); super.i_should_be_able_search_documents_for_the_by_flattened_inner_properties(expectedCount, actualName, flattenedField, flattenedFieldValue); } + + @Then("^I should get \"([^\"]*)\" in response, without hints in schema for the \"([^\"]*)\" that present in the \"([^\"]*)\" with \"([^\"]*)\" for a given \"([^\"]*)\"$") + public void i_should_get_object_in_search_response_without_hints_in_schema(String objectInnerField, String index, String recordFile, String acl, String kind) + throws Throwable { + String actualName = generateActualName(index, null); + super.i_should_get_object_in_search_response_without_hints_in_schema(objectInnerField ,actualName, recordFile, acl, kind); + } } \ No newline at end of file -- GitLab From e529821a233faf3b1ec3a6ebb5b35f08a01e4b5c Mon Sep 17 00:00:00 2001 From: Rustam_Lotsmanenko Date: Wed, 28 Apr 2021 12:25:43 +0400 Subject: [PATCH 11/15] int test for aws & azure --- .../step_definitions/index/record/Steps.java | 21 +++++++++++++++++++ .../step_definitions/index/record/Steps.java | 21 +++++++++++++++++++ .../opengroup/osdu/common/RecordSteps.java | 3 +++ 3 files changed, 45 insertions(+) diff --git a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java index 03912e84..c565d388 100644 --- a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java +++ b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java @@ -105,4 +105,25 @@ public class Steps extends SchemaServiceRecordSteps { int expectedCount, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field) throws Throwable { super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery(expectedCount, index, topLatitude, topLongitude, bottomLatitude, bottomLongitude, field); } + + @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by nested \"([^\"]*)\" and properties \\(\"([^\"]*)\", (\\d+)\\) and \\(\"([^\"]*)\", \"([^\"]*)\"\\)$") + public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery( + int expectedCount, String index, String path, String firstNestedProperty, String firstNestedValue, String secondNestedProperty, + String secondNestedValue) throws Throwable { + super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery(expectedCount, index, path, firstNestedProperty, firstNestedValue, + secondNestedProperty, secondNestedValue); + } + + @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by flattened inner properties \\(\"([^\"]*)\", \"([^\"]*)\"\\)$") + public void i_should_be_able_search_documents_for_the_by_flattened_inner_properties(int expectedCount, String index, String flattenedField, + String flattenedFieldValue) throws Throwable { + super.i_should_be_able_search_documents_for_the_by_flattened_inner_properties(expectedCount, index, flattenedField, flattenedFieldValue); + + } + + @Then("^I should get \"([^\"]*)\" in response, without hints in schema for the \"([^\"]*)\" that present in the \"([^\"]*)\" with \"([^\"]*)\" for a given \"([^\"]*)\"$") + public void i_should_get_object_in_search_response_without_hints_in_schema(String objectInnerField, String index, String recordFile, String acl, String kind) + throws Throwable { + super.i_should_get_object_in_search_response_without_hints_in_schema(objectInnerField ,index, recordFile, acl, kind); + } } \ No newline at end of file diff --git a/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java index 97779ee8..b1dc2343 100644 --- a/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java +++ b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java @@ -73,4 +73,25 @@ public class Steps extends SchemaServiceRecordSteps { int expectedCount, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field) throws Throwable { super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery(expectedCount, index, topLatitude, topLongitude, bottomLatitude, bottomLongitude, field); } + + @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by nested \"([^\"]*)\" and properties \\(\"([^\"]*)\", (\\d+)\\) and \\(\"([^\"]*)\", \"([^\"]*)\"\\)$") + public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery( + int expectedCount, String index, String path, String firstNestedProperty, String firstNestedValue, String secondNestedProperty, + String secondNestedValue) throws Throwable { + super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery(expectedCount, index, path, firstNestedProperty, firstNestedValue, + secondNestedProperty, secondNestedValue); + } + + @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by flattened inner properties \\(\"([^\"]*)\", \"([^\"]*)\"\\)$") + public void i_should_be_able_search_documents_for_the_by_flattened_inner_properties(int expectedCount, String index, String flattenedField, + String flattenedFieldValue) throws Throwable { + super.i_should_be_able_search_documents_for_the_by_flattened_inner_properties(expectedCount, index, flattenedField, flattenedFieldValue); + + } + + @Then("^I should get \"([^\"]*)\" in response, without hints in schema for the \"([^\"]*)\" that present in the \"([^\"]*)\" with \"([^\"]*)\" for a given \"([^\"]*)\"$") + public void i_should_get_object_in_search_response_without_hints_in_schema(String objectInnerField, String index, String recordFile, String acl, String kind) + throws Throwable { + super.i_should_get_object_in_search_response_without_hints_in_schema(objectInnerField ,index, recordFile, acl, kind); + } } \ No newline at end of file diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java index 4da17d87..cd7f2eb0 100644 --- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java +++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java @@ -162,6 +162,7 @@ public class RecordSteps extends TestsBase { public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery( int expectedNumber, String index, String path, String firstNestedField, String firstNestedValue, String secondNestedField, String secondNestedValue) throws Throwable { + index = generateActualName(index, timeStamp); long numOfIndexedDocuments = createIndex(index); long actualNumberOfRecords = elasticUtils.fetchRecordsByNestedQuery(index, path, firstNestedField, firstNestedValue, secondNestedField, secondNestedValue); assertEquals(expectedNumber, actualNumberOfRecords); @@ -169,6 +170,7 @@ public class RecordSteps extends TestsBase { public void i_should_be_able_search_documents_for_the_by_flattened_inner_properties(int expectedCount, String index, String flattenedField, String flattenedFieldValue) throws Throwable { + index = generateActualName(index, timeStamp); long numOfIndexedDocuments = createIndex(index); long actualNumberOfRecords = elasticUtils.fetchRecordsWithFlattenedFieldsQuery(index, flattenedField, flattenedFieldValue); assertEquals(expectedCount, actualNumberOfRecords); @@ -176,6 +178,7 @@ public class RecordSteps extends TestsBase { public void i_should_get_object_in_search_response_without_hints_in_schema(String objectField, String index, String recordFile, String acl, String kind) throws Throwable { + index = generateActualName(index, timeStamp); long numOfIndexedDocuments = createIndex(index); String expectedRecord = FileHandler.readFile(String.format("%s.%s", recordFile, "json")); -- GitLab From 2a62637c01658a57b706139b07d8158dcfbd2e89 Mon Sep 17 00:00:00 2001 From: Rustam_Lotsmanenko Date: Wed, 28 Apr 2021 15:30:26 +0400 Subject: [PATCH 12/15] review fix --- .../service/IndexSchemaServiceImpl.java | 12 +-- .../service/StorageIndexerPayloadMapper.java | 84 +++++++++---------- .../osdu/indexer/util/TypeMapper.java | 4 +- 3 files changed, 47 insertions(+), 53 deletions(-) diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java index 093c5d40..095b66a8 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java @@ -29,9 +29,9 @@ import org.elasticsearch.client.RestHighLevelClient; import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.model.http.RequestStatus; +import org.opengroup.osdu.core.common.model.indexer.ElasticType; import org.opengroup.osdu.core.common.model.indexer.IndexSchema; import org.opengroup.osdu.core.common.model.indexer.OperationType; -import org.opengroup.osdu.core.common.model.indexer.StorageType; import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute; import org.opengroup.osdu.core.common.model.storage.Schema; import org.opengroup.osdu.core.common.model.storage.SchemaItem; @@ -208,10 +208,7 @@ public class IndexSchemaServiceImpl implements IndexSchemaService { if (schemaObj.getSchema() != null && schemaObj.getSchema().length > 0) { for (SchemaItem schemaItem : schemaObj.getSchema()) { String dataType = schemaItem.getKind(); - Object elasticDataType = TypeMapper.getIndexerType(dataType); - if (elasticDataType == null) { - elasticDataType = TypeMapper.getIndexerType(StorageType.STRING.getValue()); - } + Object elasticDataType = TypeMapper.getIndexerType(dataType, ElasticType.TEXT.getValue()); if(schemaItem.getProperties() != null){ HashMap propertiesMap = normalizeInnerProperties(schemaItem); elasticDataType = TypeMapper.getObjectsArrayMapping(dataType, propertiesMap); @@ -247,10 +244,7 @@ public class IndexSchemaServiceImpl implements IndexSchemaService { HashMap propertiesMap = new HashMap<>(); for (SchemaItem propertiesItem : schemaItem.getProperties()) { String propertiesItemKind = propertiesItem.getKind(); - String propertiesElasticType = TypeMapper.getIndexerType(propertiesItemKind); - if (propertiesElasticType == null) { - propertiesElasticType = TypeMapper.getIndexerType(StorageType.STRING.getValue()); - } + String propertiesElasticType = TypeMapper.getIndexerType(propertiesItemKind,ElasticType.TEXT.getValue()); propertiesMap.put(propertiesItem.getPath(),propertiesElasticType); } return propertiesMap; diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java index f8d197dd..e550d4e0 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java @@ -39,43 +39,43 @@ public class StorageIndexerPayloadMapper { public Map mapDataPayload(IndexSchema storageSchema, Map storageRecordData, String recordId) { - Map dataMap = new HashMap<>(); + Map dataCollectorMap = new HashMap<>(); if (storageSchema.isDataSchemaMissing()) { this.log.warning(String.format("record-id: %s | schema mismatching: %s ", recordId, storageSchema.getKind())); - return dataMap; + return dataCollectorMap; } - mapDataPayload(storageSchema.getDataSchema(), storageRecordData, recordId, dataMap); + mapDataPayload(storageSchema.getDataSchema(), storageRecordData, recordId, dataCollectorMap); // add these once iterated over the list storageSchema.getDataSchema().put(DATA_GEOJSON_TAG, ElasticType.GEO_SHAPE.getValue()); storageSchema.getDataSchema().remove(RECORD_GEOJSON_TAG); - return dataMap; + return dataCollectorMap; } private Map mapDataPayload(Map dataSchema, Map storageRecordData, - String recordId, Map dataMap) { + String recordId, Map dataCollectorMap) { // get the key and get the corresponding object from the storageRecord object for (Map.Entry entry : dataSchema.entrySet()) { - String name = entry.getKey(); - Object value = getPropertyValue(recordId, storageRecordData, name); - ElasticType elasticType = defineElasticType(entry); + String schemaPropertyName = entry.getKey(); + Object storageRecordValue = getPropertyValue(recordId, storageRecordData, schemaPropertyName); + ElasticType elasticType = defineElasticType(entry.getValue()); if (Objects.isNull(elasticType)) { this.jobStatus .addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, - String.format("record-id: %s | %s for entry %s", recordId, "Not resolvable elastic type", name)); + String.format("record-id: %s | %s for entry %s", recordId, "Not resolvable elastic type", schemaPropertyName)); continue; } - if (schemaConfig.getProcessedArraysTypes().contains(elasticType.getValue().toLowerCase())) { - processInnerProperties(recordId, dataMap, entry, name, (List) value); + if (schemaConfig.getProcessedArraysTypes().contains(elasticType.getValue().toLowerCase()) && Objects.nonNull(storageRecordValue)) { + processInnerProperties(recordId, dataCollectorMap, entry.getValue(), schemaPropertyName, (List) storageRecordValue); } - if (value == null && !nullIndexedValueSupported(elasticType)) { + if (storageRecordValue == null && !nullIndexedValueSupported(elasticType)) { continue; } @@ -84,57 +84,57 @@ public class StorageIndexerPayloadMapper { case KEYWORD_ARRAY: case TEXT: case TEXT_ARRAY: - dataMap.put(name, value); + dataCollectorMap.put(schemaPropertyName, storageRecordValue); break; case INTEGER_ARRAY: - this.attributeParsingService.tryParseValueArray(Integer.class, recordId, name, value, dataMap); + this.attributeParsingService.tryParseValueArray(Integer.class, recordId, schemaPropertyName, storageRecordValue, dataCollectorMap); break; case INTEGER: - this.attributeParsingService.tryParseInteger(recordId, name, value, dataMap); + this.attributeParsingService.tryParseInteger(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap); break; case LONG_ARRAY: - this.attributeParsingService.tryParseValueArray(Long.class, recordId, name, value, dataMap); + this.attributeParsingService.tryParseValueArray(Long.class, recordId, schemaPropertyName, storageRecordValue, dataCollectorMap); break; case LONG: - this.attributeParsingService.tryParseLong(recordId, name, value, dataMap); + this.attributeParsingService.tryParseLong(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap); break; case FLOAT_ARRAY: - this.attributeParsingService.tryParseValueArray(Float.class, recordId, name, value, dataMap); + this.attributeParsingService.tryParseValueArray(Float.class, recordId, schemaPropertyName, storageRecordValue, dataCollectorMap); break; case FLOAT: - this.attributeParsingService.tryParseFloat(recordId, name, value, dataMap); + this.attributeParsingService.tryParseFloat(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap); break; case DOUBLE_ARRAY: - this.attributeParsingService.tryParseValueArray(Double.class, recordId, name, value, dataMap); + this.attributeParsingService.tryParseValueArray(Double.class, recordId, schemaPropertyName, storageRecordValue, dataCollectorMap); break; case DOUBLE: - this.attributeParsingService.tryParseDouble(recordId, name, value, dataMap); + this.attributeParsingService.tryParseDouble(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap); break; case BOOLEAN_ARRAY: - this.attributeParsingService.tryParseValueArray(Boolean.class, recordId, name, value, dataMap); + this.attributeParsingService.tryParseValueArray(Boolean.class, recordId, schemaPropertyName, storageRecordValue, dataCollectorMap); break; case BOOLEAN: - this.attributeParsingService.tryParseBoolean(recordId, name, value, dataMap); + this.attributeParsingService.tryParseBoolean(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap); break; case DATE_ARRAY: - this.attributeParsingService.tryParseValueArray(Date.class, recordId, name, value, dataMap); + this.attributeParsingService.tryParseValueArray(Date.class, recordId, schemaPropertyName, storageRecordValue, dataCollectorMap); break; case DATE: - this.attributeParsingService.tryParseDate(recordId, name, value, dataMap); + this.attributeParsingService.tryParseDate(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap); break; case GEO_POINT: - this.attributeParsingService.tryParseGeopoint(recordId, name, storageRecordData, dataMap); + this.attributeParsingService.tryParseGeopoint(recordId, schemaPropertyName, storageRecordData, dataCollectorMap); break; case GEO_SHAPE: - this.attributeParsingService.tryParseGeojson(recordId, name, value, dataMap); + this.attributeParsingService.tryParseGeojson(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap); break; case FLATTENED: // flattened type inner properties will be added "as is" without parsing as they types not present in schema - this.attributeParsingService.tryParseFlattened(recordId, name, value, dataMap); + this.attributeParsingService.tryParseFlattened(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap); break; case OBJECT: // object type inner properties will be added "as is" without parsing as they types not present in schema - this.attributeParsingService.tryParseObject(recordId, name, value, dataMap); + this.attributeParsingService.tryParseObject(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap); break; case UNDEFINED: // don't do anything for now @@ -142,24 +142,24 @@ public class StorageIndexerPayloadMapper { } } - return dataMap; + return dataCollectorMap; } - private void processInnerProperties(String recordId, Map dataMap, Entry entry, - String name, List value) { - Map map = (Map) entry.getValue(); - Map innerProperties = (Map) map.get(Constants.PROPERTIES); - ArrayList maps = new ArrayList<>(); - value.forEach(recordData -> maps.add(mapDataPayload(innerProperties, recordData, recordId, new HashMap<>()))); - dataMap.put(name, maps); + private void processInnerProperties(String recordId, Map dataCollectorMap, Object schemaPropertyWithInnerProperties, + String name, List storageRecordValue) { + Map schemaPropertyMap = (Map) schemaPropertyWithInnerProperties; + Map innerProperties = (Map) schemaPropertyMap.get(Constants.PROPERTIES); + ArrayList innerPropertiesMappingCollector = new ArrayList<>(); + storageRecordValue.forEach(recordData -> innerPropertiesMappingCollector.add(mapDataPayload(innerProperties, recordData, recordId, new HashMap<>()))); + dataCollectorMap.put(name, innerPropertiesMappingCollector); } - private ElasticType defineElasticType(Map.Entry entry) { + private ElasticType defineElasticType(Object entryValue) { ElasticType elasticType = null; - if (entry.getValue() instanceof String) { - elasticType = ElasticType.forValue(entry.getValue().toString()); - } else if (entry.getValue() instanceof Map) { - Map map = (Map) entry.getValue(); + if (entryValue instanceof String) { + elasticType = ElasticType.forValue(entryValue.toString()); + } else if (entryValue instanceof Map) { + Map map = (Map) entryValue; elasticType = ElasticType.forValue(map.get(Constants.TYPE).toString()); } return elasticType; diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java index 2e7a41d0..7141317f 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java @@ -75,8 +75,8 @@ public class TypeMapper { storageToIndexerType.put(STORAGE_TYPE_FLATTENED,ElasticType.FLATTENED.getValue()); } - public static String getIndexerType(String storageType) { - return storageToIndexerType.getOrDefault(storageType, null); + public static String getIndexerType(String storageType, String defaultType) { + return storageToIndexerType.getOrDefault(storageType, defaultType); } public static Object getIndexerType(RecordMetaAttribute attribute) { -- GitLab From 823bd7465325a6eda5c424f89fc6d677eb8c9fdd Mon Sep 17 00:00:00 2001 From: Gokul Nagare Date: Thu, 29 Apr 2021 15:04:53 +0000 Subject: [PATCH 13/15] v3 feature file changes --- .../index/record/RunTest.java | 2 +- .../step_definitions/index/record/Steps.java | 38 ++++++++++++++++++- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java index 897f02c1..cecc3b7d 100644 --- a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java +++ b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java @@ -6,7 +6,7 @@ import org.junit.runner.RunWith; @RunWith(Cucumber.class) @CucumberOptions( - features = {"classpath:features/indexrecord/IndexRecord.feature","classpath:apikey.feature"}, + features = {"classpath:features/indexrecord/indexRecord-schema-service.feature","classpath:apikey.feature"}, glue = {"classpath:org.opengroup.osdu.step_definitions/index/record"}, plugin = {"pretty", "junit:target/cucumber-reports/TEST-indexrecord.xml"}) public class RunTest { diff --git a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java index 3c885b11..8caf85ca 100644 --- a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java +++ b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java @@ -8,6 +8,7 @@ import java.util.Map; import org.apache.commons.lang3.StringUtils; import org.opengroup.osdu.common.RecordSteps; +import org.opengroup.osdu.common.SchemaServiceRecordSteps; import org.opengroup.osdu.core.common.Constants; import org.opengroup.osdu.core.common.http.HttpClient; import org.opengroup.osdu.core.common.http.HttpRequest; @@ -16,6 +17,7 @@ import org.opengroup.osdu.core.common.model.http.AppError; import org.opengroup.osdu.core.common.model.http.DpsHeaders; import org.opengroup.osdu.core.common.model.search.RecordChangedMessages; import org.opengroup.osdu.core.ibm.util.Config; +import org.opengroup.osdu.util.ElasticUtils; import org.opengroup.osdu.util.IBMHTTPClient; import com.google.gson.Gson; @@ -29,10 +31,10 @@ import cucumber.api.java.en.When; import lombok.extern.java.Log; @Log -public class Steps extends RecordSteps { +public class Steps extends SchemaServiceRecordSteps { public Steps() { - super(new IBMHTTPClient()); + super(new IBMHTTPClient(), new ElasticUtils()); } @Before @@ -66,6 +68,38 @@ public class Steps extends RecordSteps { super.iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(expectedCount, index, skippedAttributes); } + @Then("^I should be able to search (\\d+) record with index \"([^\"]*)\" by tag \"([^\"]*)\" and value \"([^\"]*)\"$") + public void iShouldBeAbleToSearchRecordByTagKeyAndTagValue(int expectedNumber, String index, String tagKey, String tagValue) throws Throwable { + super.iShouldBeAbleToSearchRecordByTagKeyAndTagValue(index, tagKey, tagValue, expectedNumber); + } + + @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by bounding box query with points \\((-?\\d+), (-?\\d+)\\) and \\((-?\\d+), (-?\\d+)\\) on field \"(.*?)\"$") + public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery ( + int expectedCount, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field) throws Throwable { + super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery(expectedCount, index, topLatitude, topLongitude, bottomLatitude, bottomLongitude, field); + } + + @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by nested \"([^\"]*)\" and properties \\(\"([^\"]*)\", (\\d+)\\) and \\(\"([^\"]*)\", \"([^\"]*)\"\\)$") + public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery( + int expectedCount, String index, String path, String firstNestedProperty, String firstNestedValue, String secondNestedProperty, + String secondNestedValue) throws Throwable { + super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery(expectedCount, index, path, firstNestedProperty, firstNestedValue, + secondNestedProperty, secondNestedValue); + } + + @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by flattened inner properties \\(\"([^\"]*)\", \"([^\"]*)\"\\)$") + public void i_should_be_able_search_documents_for_the_by_flattened_inner_properties(int expectedCount, String index, String flattenedField, + String flattenedFieldValue) throws Throwable { + super.i_should_be_able_search_documents_for_the_by_flattened_inner_properties(expectedCount, index, flattenedField, flattenedFieldValue); + + } + + @Then("^I should get \"([^\"]*)\" in response, without hints in schema for the \"([^\"]*)\" that present in the \"([^\"]*)\" with \"([^\"]*)\" for a given \"([^\"]*)\"$") + public void i_should_get_object_in_search_response_without_hints_in_schema(String objectInnerField, String index, String recordFile, String acl, String kind) + throws Throwable { + super.i_should_get_object_in_search_response_without_hints_in_schema(objectInnerField ,index, recordFile, acl, kind); + } + @When("^I pass api key$") public void i_pass_the_api_key() { } -- GitLab From 31f562f315b31b638fc857cf254a27db51c78c53 Mon Sep 17 00:00:00 2001 From: Rustam_Lotsmanenko Date: Sat, 1 May 2021 18:08:09 +0400 Subject: [PATCH 14/15] support array of complex types processing for Items with ref or allOf --- .../schema/converter/PropertiesProcessor.java | 76 +++++++++---------- .../indexer/schema/converter/tags/Items.java | 11 +++ .../schema/converter/tags/TypeProperty.java | 2 +- .../converter/PropertiesProcessorTest.java | 4 +- .../resources/converter/basic/schema.json.res | 36 +++++++++ .../colons-sample.json.res | 8 ++ 6 files changed, 92 insertions(+), 45 deletions(-) diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java index 0c57b560..1cf63e8d 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java @@ -17,6 +17,7 @@ package org.opengroup.osdu.indexer.schema.converter; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Objects; import java.util.Optional; import java.util.function.Supplier; @@ -173,50 +174,11 @@ public class PropertiesProcessor { if ("array".equals(entry.getValue().getType())) { Items items = entry.getValue().getItems(); - if(Objects.isNull(items)){ - return Stream.empty(); + if(Objects.nonNull(items) && items.isComplexTypeItems()){ + return processComplexTypeItems(entry, items); } - if(Objects.nonNull(items.getProperties()) && !items.getProperties().isEmpty()){ - Map type = entry.getValue().getIndexingType(); - String indexingType = Objects.isNull(type) ? - schemaConverterConfig.getDefaultObjectArraysType() : - type.getOrDefault(TYPE_KEY,schemaConverterConfig.getDefaultObjectArraysType()); - /*Schema item inner properties will be processed if they are present & indexingType in schema configured for processing - result ex: - { - path = ArrayItem, - kind = nested, - properties = [{ - path = InnerProperty, - kind = double - }, { - path = OtherInnerProperty, - kind = string - } - ] - } - */ - if(schemaConverterConfig.getProcessedArraysTypes().contains(indexingType)){ - PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, log, new SchemaConverterPropertiesConfig()); - return storageSchemaObjectArrayEntry( - indexingType, - entry.getKey(), - items.getProperties().entrySet().stream().flatMap(propertiesProcessor::processPropertyEntry)); - - /*Otherwise inner properties won't be processed - result ex: - { - path = ArrayItem, - kind = []object - } - */ - }else { - return storageSchemaEntry(indexingType, pathPrefixWithDot + entry.getKey()); - } - } - - if (schemaConverterConfig.getSupportedArrayTypes().contains(items.getType()) && Objects.isNull(items.getProperties())) { + if (schemaConverterConfig.getSupportedArrayTypes().contains(items.getType()) && !items.isComplexTypeItems()) { return storageSchemaEntry("[]" + getTypeByDefinitionProperty(entry.getValue()), pathPrefixWithDot + entry.getKey()); } @@ -243,6 +205,36 @@ public class PropertiesProcessor { return storageSchemaEntry(getTypeByDefinitionProperty(entry.getValue()), pathPrefixWithDot + entry.getKey()); } + private Stream> processComplexTypeItems(Entry entry, Items items) { + Map indexHint = entry.getValue().getIndexHint(); + + String indexingType = Objects.isNull(indexHint) ? + schemaConverterConfig.getDefaultObjectArraysType() : + indexHint.getOrDefault(TYPE_KEY,schemaConverterConfig.getDefaultObjectArraysType()); + + if(schemaConverterConfig.getProcessedArraysTypes().contains(indexingType)){ + PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, log, new SchemaConverterPropertiesConfig()); + + Stream> propertiesStream = Stream.empty(); + + if(Objects.nonNull(items.getProperties())){ + propertiesStream = items.getProperties().entrySet().stream().flatMap(propertiesProcessor::processPropertyEntry); + } + if (Objects.nonNull(items.getRef())){ + propertiesStream = Stream.concat(propertiesStream, propertiesProcessor.processRef(items.getRef())); + } + if(Objects.nonNull(items.getAllOf())){ + propertiesStream = Stream.concat(propertiesStream, items.getAllOf().stream().flatMap(propertiesProcessor::processItem)); + } + return storageSchemaObjectArrayEntry( + indexingType, + entry.getKey(), + propertiesStream); + }else { + return storageSchemaEntry(indexingType, pathPrefixWithDot + entry.getKey()); + } + } + private Stream> processOfItems(Map.Entry entry) { Stream> ofItems = null; diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/Items.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/Items.java index 21721cca..ed1678cf 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/Items.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/Items.java @@ -14,12 +14,23 @@ package org.opengroup.osdu.indexer.schema.converter.tags; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; import java.util.Map; +import java.util.Objects; import lombok.Data; @Data public class Items { + @JsonProperty("$ref") + private String ref; + private List allOf; private String type; private String pattern; private Map properties; + + public boolean isComplexTypeItems(){ + return Objects.nonNull(ref) || Objects.nonNull(allOf) || Objects.nonNull(properties); + } + } diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java index 91d10c55..d06a47c2 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java @@ -22,7 +22,7 @@ import lombok.Data; @Data public class TypeProperty { @JsonProperty("x-osdu-indexing") - private Map indexingType; + private Map indexHint; private String type; private String pattern; private String format; diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessorTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessorTest.java index 837e203f..3789a250 100644 --- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessorTest.java +++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessorTest.java @@ -117,7 +117,7 @@ public class PropertiesProcessorTest { items.setProperties(itemsProperties); TypeProperty arrayProperty = new TypeProperty(); - arrayProperty.setIndexingType(ImmutableMap.of("type","nested")); + arrayProperty.setIndexHint(ImmutableMap.of("type","nested")); arrayProperty.setType("array"); arrayProperty.setItems(items); @@ -145,7 +145,7 @@ public class PropertiesProcessorTest { items.setProperties(itemsProperties); TypeProperty arrayProperty = new TypeProperty(); - arrayProperty.setIndexingType(ImmutableMap.of("type","flattened")); + arrayProperty.setIndexHint(ImmutableMap.of("type","flattened")); arrayProperty.setType("array"); arrayProperty.setItems(items); diff --git a/indexer-core/src/test/resources/converter/basic/schema.json.res b/indexer-core/src/test/resources/converter/basic/schema.json.res index 416ac54e..e3b4fd5f 100644 --- a/indexer-core/src/test/resources/converter/basic/schema.json.res +++ b/indexer-core/src/test/resources/converter/basic/schema.json.res @@ -132,6 +132,42 @@ { "kind": "link", "path": "GeographicBottomHoleLocation.SpatialGeometryTypeID" + }, + { + "path": "VerticalMeasurements", + "kind": "[]object" + }, + { + "path": "DrillingReason", + "kind": "[]object" + }, + { + "path": "FacilityNameAlias", + "kind": "[]object" + }, + { + "path": "FacilityState", + "kind": "[]object" + }, + { + "path": "FacilityEvent", + "kind": "[]object" + }, + { + "path": "FacilitySpecification", + "kind": "[]object" + }, + { + "path": "FacilityOperator", + "kind": "[]object" + }, + { + "path": "SpatialLocation", + "kind": "[]object" + }, + { + "path": "GeoContexts", + "kind": "[]object" } ] } \ No newline at end of file diff --git a/indexer-core/src/test/resources/converter/new-definitions-format/colons-sample.json.res b/indexer-core/src/test/resources/converter/new-definitions-format/colons-sample.json.res index 8f3ca4d7..4c099bb3 100644 --- a/indexer-core/src/test/resources/converter/new-definitions-format/colons-sample.json.res +++ b/indexer-core/src/test/resources/converter/new-definitions-format/colons-sample.json.res @@ -72,6 +72,14 @@ { "path": "VersionCreationReason", "kind": "string" + }, + { + "path": "GeoContexts", + "kind": "[]object" + }, + { + "path": "NameAliases", + "kind": "[]object" } ] } \ No newline at end of file -- GitLab From a0597d437caeeea7e06c79777ecf053c7aee8518 Mon Sep 17 00:00:00 2001 From: Rustam_Lotsmanenko Date: Tue, 4 May 2021 16:00:38 +0400 Subject: [PATCH 15/15] sync with master --- .../schema/converter/PropertiesProcessor.java | 22 ++++++++----------- .../converter/PropertiesProcessorTest.java | 8 +++---- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java index 830ba9d3..04a3211f 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java @@ -15,6 +15,7 @@ package org.opengroup.osdu.indexer.schema.converter; import java.util.HashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -25,7 +26,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.http.HttpStatus; import org.opengroup.osdu.core.common.Constants; -import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.search.Preconditions; import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterConfig; @@ -185,19 +185,15 @@ public class PropertiesProcessor { } if ("array".equals(entry.getValue().getType())) { - if (schemaConverterConfig.getSupportedArrayTypes().contains(entry.getValue().getItems().getType())) { - return storageSchemaEntry("[]" + getTypeByDefinitionProperty(entry.getValue()), pathPrefixWithDot + entry.getKey()); - } - if ("array".equals(entry.getValue().getType())) { - Items items = entry.getValue().getItems(); - if(Objects.nonNull(items) && items.isComplexTypeItems()){ - return processComplexTypeItems(entry, items); - } + Items items = entry.getValue().getItems(); + if(Objects.nonNull(items) && items.isComplexTypeItems()){ + return processComplexTypeItems(entry, items); + } - if (schemaConverterConfig.getSupportedArrayTypes().contains(items.getType()) && !items.isComplexTypeItems()) { - return storageSchemaEntry("[]" + getTypeByDefinitionProperty(entry.getValue()), pathPrefixWithDot + entry.getKey()); - } + if (schemaConverterConfig.getSupportedArrayTypes().contains(entry.getValue().getItems().getType()) && !items.isComplexTypeItems()) { + return storageSchemaEntry("[]" + getTypeByDefinitionProperty(entry.getValue()), pathPrefixWithDot + entry.getKey()); + } return Stream.empty(); } @@ -240,7 +236,7 @@ public class PropertiesProcessor { indexHint.getOrDefault(TYPE_KEY,schemaConverterConfig.getDefaultObjectArraysType()); if(schemaConverterConfig.getProcessedArraysTypes().contains(indexingType)){ - PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, log, new SchemaConverterPropertiesConfig()); + PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, new SchemaConverterPropertiesConfig()); Stream> propertiesStream = Stream.empty(); diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessorTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessorTest.java index 0075a77c..fdcfeba5 100644 --- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessorTest.java +++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessorTest.java @@ -128,7 +128,7 @@ public class PropertiesProcessorTest { AllOfItem allOfItem = new AllOfItem(); allOfItem.setProperties(allOfItemProperties); - String res = new PropertiesProcessor(Mockito.mock(Definitions.class), log, new SchemaConverterPropertiesConfig()) + String res = new PropertiesProcessor(Mockito.mock(Definitions.class), new SchemaConverterPropertiesConfig()) .processItem(allOfItem).map(Object::toString).reduce("", String::concat); assertEquals("{path="+ PATH + ", kind=nested, properties=[{path="+ PATH + ", kind=int}]}",res); } @@ -156,7 +156,7 @@ public class PropertiesProcessorTest { AllOfItem allOfItem = new AllOfItem(); allOfItem.setProperties(allOfItemProperties); - String res = new PropertiesProcessor(Mockito.mock(Definitions.class), log, new SchemaConverterPropertiesConfig()) + String res = new PropertiesProcessor(Mockito.mock(Definitions.class), new SchemaConverterPropertiesConfig()) .processItem(allOfItem).map(Object::toString).reduce("", String::concat); assertEquals("{path="+ PATH + ", kind=flattened}",res); } @@ -183,7 +183,7 @@ public class PropertiesProcessorTest { AllOfItem allOfItem = new AllOfItem(); allOfItem.setProperties(allOfItemProperties); - String res = new PropertiesProcessor(Mockito.mock(Definitions.class), log, new SchemaConverterPropertiesConfig()) + String res = new PropertiesProcessor(Mockito.mock(Definitions.class), new SchemaConverterPropertiesConfig()) .processItem(allOfItem).map(Object::toString).reduce("", String::concat); assertEquals("{path="+ PATH + ", kind=[]object}",res); } @@ -205,7 +205,7 @@ public class PropertiesProcessorTest { AllOfItem allOfItem = new AllOfItem(); allOfItem.setProperties(allOfItemProperties); - String res = new PropertiesProcessor(Mockito.mock(Definitions.class), log, new SchemaConverterPropertiesConfig()) + String res = new PropertiesProcessor(Mockito.mock(Definitions.class), new SchemaConverterPropertiesConfig()) .processItem(allOfItem).map(Object::toString).reduce("", String::concat); assertEquals("{path="+ PATH + ", kind=[]int}",res); } -- GitLab