From b803c74ac767890c616f1ccf13d4fe314e4ce923 Mon Sep 17 00:00:00 2001
From: Rustam_Lotsmanenko <Rustam_Lotsmanenko@epam.com>
Date: Tue, 27 Apr 2021 16:37:46 +0400
Subject: [PATCH] review fix
---
.../osdu/indexer/schema/converter/readme.md | 66 ++++++++++-
.../service/StorageIndexerPayloadMapper.java | 3 -
.../converter/PropertiesProcessorTest.java | 107 ++++++++++++++++++
.../opengroup/osdu/common/RecordSteps.java | 21 +++-
.../osdu/models/record/RecordData.java | 18 +++
.../org/opengroup/osdu/util/ElasticUtils.java | 19 ++++
.../indexRecord-schema-service.feature | 5 +-
.../step_definitions/index/record/Steps.java | 13 ++-
8 files changed, 238 insertions(+), 14 deletions(-)
create mode 100644 testing/indexer-test-core/src/main/java/org/opengroup/osdu/models/record/RecordData.java
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/readme.md b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/readme.md
index 4c39a2764..17a4ca99a 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/readme.md
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/readme.md
@@ -203,8 +203,68 @@ Is converted to
2. Arrays
-Arrays of complex types are ignored, only following arrays of primitive
-types are supported
+Arrays of complex types by default will be consumed as object type
+```json
+"Markers": {
+"type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "NegativeVerticalDelta"{
+ "description": "The distance vertically below the Marker position that marks the limit of the high confidence range for the Marker pick.",
+ "x-osdu-frame-of-reference": "UOM:length",
+ "type": "number"
+ },
+ .....
+ }
+ }
+}
+
+```
+Without inner objects processing
+```json
+{
+ path = Markers,
+ kind = []object
+}
+```
+
+Processing can be specified with optional "x-osdu-indexing" property
+```json
+"properties": {
+ "Markers": {
+ "x-osdu-indexing": {
+ "type": "nested"
+ },
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "NegativeVerticalDelta"{
+ "description": "The distance vertically below the Marker position that marks the limit of the high confidence range for the Marker pick.",
+ "x-osdu-frame-of-reference": "UOM:length",
+ "type": "number"
+ },
+ .....
+```
+"x-osdu-indexing" property values
+```json
+"nested" , "flattened"
+```
+By default, only "nested" type will lead to inner objects processing
+```json
+{
+path = Markers,
+ kind = nested,
+ properties = [{
+ path = NegativeVerticalDelta,
+ kind = double
+ },
+ .....
+}
+```
+
+Arrays of primitive types are supported
```json
"number", "string", "integer", "boolean"
@@ -370,4 +430,4 @@ is converted to
}
]
}
-```
+```
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java
index 1af15db21..f8d197dd9 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java
@@ -128,9 +128,6 @@ public class StorageIndexerPayloadMapper {
case GEO_SHAPE:
this.attributeParsingService.tryParseGeojson(recordId, name, value, dataMap);
break;
- case NESTED:
- // don't do anything , each nested property will be parsed separately
- break;
case FLATTENED:
// flattened type inner properties will be added "as is" without parsing as they types not present in schema
this.attributeParsingService.tryParseFlattened(recordId, name, value, dataMap);
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessorTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessorTest.java
index 182bc58d3..837e203f4 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessorTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessorTest.java
@@ -14,6 +14,7 @@
package org.opengroup.osdu.indexer.schema.converter;
+import com.google.common.collect.ImmutableMap;
import org.junit.Test;
import org.mockito.Mockito;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
@@ -22,6 +23,7 @@ import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterPropert
import org.opengroup.osdu.indexer.schema.converter.tags.AllOfItem;
import org.opengroup.osdu.indexer.schema.converter.tags.Definition;
import org.opengroup.osdu.indexer.schema.converter.tags.Definitions;
+import org.opengroup.osdu.indexer.schema.converter.tags.Items;
import org.opengroup.osdu.indexer.schema.converter.tags.TypeProperty;
import java.util.LinkedHashMap;
@@ -101,4 +103,109 @@ public class PropertiesProcessorTest {
.processItem(allOfItem).map(Object::toString).reduce("", String::concat);
assertEquals("{path=" + PATH + ", kind=int}", res);
}
+
+ @Test
+ public void should_return_processed_nested_array_items(){
+ JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class);
+
+ Map<String, TypeProperty> itemsProperties = new LinkedHashMap<>();
+ TypeProperty intProperty = new TypeProperty();
+ intProperty.setType("integer");
+ itemsProperties.put(PATH, intProperty);
+
+ Items items = new Items();
+ items.setProperties(itemsProperties);
+
+ TypeProperty arrayProperty = new TypeProperty();
+ arrayProperty.setIndexingType(ImmutableMap.of("type","nested"));
+ arrayProperty.setType("array");
+ arrayProperty.setItems(items);
+
+ Map<String, TypeProperty> allOfItemProperties = new LinkedHashMap<>();
+ allOfItemProperties.put(PATH,arrayProperty);
+
+ AllOfItem allOfItem = new AllOfItem();
+ allOfItem.setProperties(allOfItemProperties);
+
+ String res = new PropertiesProcessor(Mockito.mock(Definitions.class), log, new SchemaConverterPropertiesConfig())
+ .processItem(allOfItem).map(Object::toString).reduce("", String::concat);
+ assertEquals("{path="+ PATH + ", kind=nested, properties=[{path="+ PATH + ", kind=int}]}",res);
+ }
+
+ @Test
+ public void should_return_flattened_array_without_processing(){
+ JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class);
+
+ Map<String, TypeProperty> itemsProperties = new LinkedHashMap<>();
+ TypeProperty intProperty = new TypeProperty();
+ intProperty.setType("integer");
+ itemsProperties.put(PATH, intProperty);
+
+ Items items = new Items();
+ items.setProperties(itemsProperties);
+
+ TypeProperty arrayProperty = new TypeProperty();
+ arrayProperty.setIndexingType(ImmutableMap.of("type","flattened"));
+ arrayProperty.setType("array");
+ arrayProperty.setItems(items);
+
+ Map<String, TypeProperty> allOfItemProperties = new LinkedHashMap<>();
+ allOfItemProperties.put(PATH,arrayProperty);
+
+ AllOfItem allOfItem = new AllOfItem();
+ allOfItem.setProperties(allOfItemProperties);
+
+ String res = new PropertiesProcessor(Mockito.mock(Definitions.class), log, new SchemaConverterPropertiesConfig())
+ .processItem(allOfItem).map(Object::toString).reduce("", String::concat);
+ assertEquals("{path="+ PATH + ", kind=flattened}",res);
+ }
+
+ @Test
+ public void should_return_object_array_without_hints_in_schema_without_processing(){
+ JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class);
+
+ Map<String, TypeProperty> itemsProperties = new LinkedHashMap<>();
+ TypeProperty intProperty = new TypeProperty();
+ intProperty.setType("integer");
+ itemsProperties.put(PATH, intProperty);
+
+ Items items = new Items();
+ items.setProperties(itemsProperties);
+
+ TypeProperty arrayProperty = new TypeProperty();
+ arrayProperty.setType("array");
+ arrayProperty.setItems(items);
+
+ Map<String, TypeProperty> allOfItemProperties = new LinkedHashMap<>();
+ allOfItemProperties.put(PATH,arrayProperty);
+
+ AllOfItem allOfItem = new AllOfItem();
+ allOfItem.setProperties(allOfItemProperties);
+
+ String res = new PropertiesProcessor(Mockito.mock(Definitions.class), log, new SchemaConverterPropertiesConfig())
+ .processItem(allOfItem).map(Object::toString).reduce("", String::concat);
+ assertEquals("{path="+ PATH + ", kind=[]object}",res);
+ }
+
+ @Test
+ public void should_process_not_object_array_type(){
+ JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class);
+
+ Items items = new Items();
+ items.setType("integer");
+
+ TypeProperty arrayProperty = new TypeProperty();
+ arrayProperty.setType("array");
+ arrayProperty.setItems(items);
+
+ Map<String, TypeProperty> allOfItemProperties = new LinkedHashMap<>();
+ allOfItemProperties.put(PATH,arrayProperty);
+
+ AllOfItem allOfItem = new AllOfItem();
+ allOfItem.setProperties(allOfItemProperties);
+
+ String res = new PropertiesProcessor(Mockito.mock(Definitions.class), log, new SchemaConverterPropertiesConfig())
+ .processItem(allOfItem).map(Object::toString).reduce("", String::concat);
+ assertEquals("{path="+ PATH + ", kind=[]int}",res);
+ }
}
\ No newline at end of file
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
index fbf12a6f9..4da17d875 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
@@ -1,5 +1,6 @@
package org.opengroup.osdu.common;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.MapDifference;
import com.google.common.collect.Maps;
import com.google.gson.Gson;
@@ -11,6 +12,7 @@ import org.elasticsearch.cluster.metadata.MappingMetadata;
import org.opengroup.osdu.core.common.model.entitlements.Acl;
import org.opengroup.osdu.models.Setup;
import org.opengroup.osdu.models.TestIndex;
+import org.opengroup.osdu.models.record.RecordData;
import org.opengroup.osdu.util.ElasticUtils;
import org.opengroup.osdu.util.FileHandler;
import org.opengroup.osdu.util.HTTPClient;
@@ -32,6 +34,7 @@ import static org.opengroup.osdu.util.Config.getStorageBaseURL;
@Log
public class RecordSteps extends TestsBase {
private Map<String, TestIndex> inputIndexMap = new HashMap<>();
+ private ObjectMapper mapper = new ObjectMapper();
private boolean shutDownHookAdded = false;
private String timeStamp = String.valueOf(System.currentTimeMillis());
@@ -158,19 +161,33 @@ public class RecordSteps extends TestsBase {
public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery(
int expectedNumber, String index, String path, String firstNestedField, String firstNestedValue, String secondNestedField, String secondNestedValue)
- throws Exception {
+ throws Throwable {
long numOfIndexedDocuments = createIndex(index);
long actualNumberOfRecords = elasticUtils.fetchRecordsByNestedQuery(index, path, firstNestedField, firstNestedValue, secondNestedField, secondNestedValue);
assertEquals(expectedNumber, actualNumberOfRecords);
}
public void i_should_be_able_search_documents_for_the_by_flattened_inner_properties(int expectedCount, String index, String flattenedField,
- String flattenedFieldValue) throws IOException, InterruptedException {
+ String flattenedFieldValue) throws Throwable {
long numOfIndexedDocuments = createIndex(index);
long actualNumberOfRecords = elasticUtils.fetchRecordsWithFlattenedFieldsQuery(index, flattenedField, flattenedFieldValue);
assertEquals(expectedCount, actualNumberOfRecords);
}
+ public void i_should_get_object_in_search_response_without_hints_in_schema(String objectField, String index, String recordFile, String acl, String kind)
+ throws Throwable {
+ long numOfIndexedDocuments = createIndex(index);
+ String expectedRecord = FileHandler.readFile(String.format("%s.%s", recordFile, "json"));
+
+ RecordData[] fileRecordData = mapper.readValue(expectedRecord, RecordData[].class);
+ RecordData expectedRecordData = fileRecordData[0];
+
+ String elasticRecordData = elasticUtils.fetchDataFromObjectsArrayRecords(index);
+ RecordData actualRecordData = mapper.readValue(elasticRecordData, RecordData.class);
+
+ assertEquals(expectedRecordData.getData().get(objectField),actualRecordData.getData().get(objectField));
+ }
+
private long createIndex(String index) throws InterruptedException, IOException {
long numOfIndexedDocuments = 0;
int iterator;
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/models/record/RecordData.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/models/record/RecordData.java
new file mode 100644
index 000000000..ec0a6a221
--- /dev/null
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/models/record/RecordData.java
@@ -0,0 +1,18 @@
+package org.opengroup.osdu.models.record;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.List;
+import java.util.Map;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class RecordData {
+
+ @JsonProperty("data")
+ private Map<String, List<Map<String,Object>>> data;
+
+ public Map<String, List<Map<String, Object>>> getData() {
+ return data;
+ }
+
+}
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
index 668d1888c..3d029aa6e 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
@@ -52,6 +52,7 @@ import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.client.indices.GetMappingsRequest;
import org.elasticsearch.client.indices.GetMappingsResponse;
import org.elasticsearch.cluster.metadata.MappingMetadata;
+import org.elasticsearch.search.SearchHits;
import org.locationtech.jts.geom.Coordinate;
import org.elasticsearch.common.geo.builders.EnvelopeBuilder;
import org.elasticsearch.common.settings.Settings;
@@ -309,6 +310,24 @@ public class ElasticUtils {
}
}
+ public String fetchDataFromObjectsArrayRecords(String index) throws IOException {
+ try {
+ try (RestHighLevelClient client = this.createClient(username, password, host)) {
+ SearchRequest request = new SearchRequest(index);
+ SearchResponse searchResponse = client.search(request, RequestOptions.DEFAULT);
+
+ SearchHits searchHits = searchResponse.getHits();
+ if (searchHits.getHits().length != 0) {
+ return searchHits.getHits()[0].getSourceAsString();
+ }
+ return null;
+ }
+ } catch (ElasticsearchStatusException e) {
+ log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage()));
+ return null;
+ }
+ }
+
public Map<String, MappingMetadata> getMapping(String index) throws IOException {
try (RestHighLevelClient client = this.createClient(username, password, host)) {
GetMappingsRequest request = new GetMappingsRequest();
diff --git a/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature b/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
index 2991c9691..50051edcf 100644
--- a/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
+++ b/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
@@ -48,7 +48,8 @@ Feature: Indexing of the documents
When I ingest records with the <recordFile> with <acl> for a given <kind>
Then I should be able search <number> documents for the <index> by nested <path> and properties (<first_nested_field>, <first_nested_value>) and (<second_nested_field>, <second_nested_value>)
Then I should be able search <number> documents for the <index> by flattened inner properties (<flattened_inner_field>, <flattened_inner_value>)
+ Then I should get <object_inner_field> in response, without hints in schema for the <index> that present in the <recordFile> with <acl> for a given <kind>
Examples:
- | kind | recordFile | number | index | acl | path | first_nested_field | first_nested_value | second_nested_field | second_nested_value | flattened_inner_field | flattened_inner_value |
- | "tenant1:wks:ArraysOfObjectsTestCollection:4.0.0" | "r3-index_record_arrayofobjects" | 1 | "tenant1-wks-arraysofobjectstestcollection-4.0.0" | "data.default.viewers@tenant1" | "data.NestedTest" | "data.NestedTest.NumberTest" | 12345 | "data.NestedTest.StringTest" | "test string" | "data.FlattenedTest.StringTest" | "test string" |
\ No newline at end of file
+ | kind | recordFile | number | index | acl | path | first_nested_field | first_nested_value | second_nested_field | second_nested_value | flattened_inner_field | flattened_inner_value | object_inner_field |
+ | "tenant1:wks:ArraysOfObjectsTestCollection:4.0.0" | "r3-index_record_arrayofobjects" | 1 | "tenant1-wks-arraysofobjectstestcollection-4.0.0" | "data.default.viewers@tenant1" | "data.NestedTest" | "data.NestedTest.NumberTest" | 12345 | "data.NestedTest.StringTest" | "test string" | "data.FlattenedTest.StringTest" | "test string" | "ObjectTest" |
\ No newline at end of file
diff --git a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
index bc1ee68b4..26060454c 100644
--- a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
+++ b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -6,7 +6,6 @@ import cucumber.api.java.Before;
import cucumber.api.java.en.Given;
import cucumber.api.java.en.Then;
import cucumber.api.java.en.When;
-import java.io.IOException;
import lombok.extern.java.Log;
import org.opengroup.osdu.common.SchemaServiceRecordSteps;
import org.opengroup.osdu.util.ElasticUtils;
@@ -66,7 +65,7 @@ public class Steps extends SchemaServiceRecordSteps {
@Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by nested \"([^\"]*)\" and properties \\(\"([^\"]*)\", (\\d+)\\) and \\(\"([^\"]*)\", \"([^\"]*)\"\\)$")
public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery(
int expectedCount, String index, String path, String firstNestedProperty, String firstNestedValue, String secondNestedProperty,
- String secondNestedValue) throws Exception {
+ String secondNestedValue) throws Throwable {
String actualName = generateActualName(index, null);
super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery(expectedCount, actualName, path, firstNestedProperty, firstNestedValue,
secondNestedProperty, secondNestedValue);
@@ -74,10 +73,16 @@ public class Steps extends SchemaServiceRecordSteps {
@Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by flattened inner properties \\(\"([^\"]*)\", \"([^\"]*)\"\\)$")
public void i_should_be_able_search_documents_for_the_by_flattened_inner_properties(int expectedCount, String index, String flattenedField,
- String flattenedFieldValue)
- throws IOException, InterruptedException {
+ String flattenedFieldValue) throws Throwable {
String actualName = generateActualName(index, null);
super.i_should_be_able_search_documents_for_the_by_flattened_inner_properties(expectedCount, actualName, flattenedField, flattenedFieldValue);
}
+
+ @Then("^I should get \"([^\"]*)\" in response, without hints in schema for the \"([^\"]*)\" that present in the \"([^\"]*)\" with \"([^\"]*)\" for a given \"([^\"]*)\"$")
+ public void i_should_get_object_in_search_response_without_hints_in_schema(String objectInnerField, String index, String recordFile, String acl, String kind)
+ throws Throwable {
+ String actualName = generateActualName(index, null);
+ super.i_should_get_object_in_search_response_without_hints_in_schema(objectInnerField ,actualName, recordFile, acl, kind);
+ }
}
\ No newline at end of file
--
GitLab