review fix

indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/readme.md

@@ -203,8 +203,68 @@ Is converted to
 
 2.  Arrays
 
-Arrays of complex types are ignored, only following arrays of primitive
-types are supported
+Arrays of complex types by default will be consumed as object type
+```json
+"Markers": {
+"type": "array",
+    "items": {
+        "type": "object", 
+        "properties":  {
+            "NegativeVerticalDelta"{
+                "description": "The distance vertically below the Marker position that marks the limit of the high confidence range for the Marker pick.",
+                "x-osdu-frame-of-reference": "UOM:length",
+                "type": "number"
+            },
+          .....
+        }
+    }
+}        
+
+```
+Without inner objects processing
+```json
+{
+        path = Markers,
+        kind = []object
+}
+```
+
+Processing can be specified with optional "x-osdu-indexing" property 
+```json
+"properties": {
+    "Markers": {
+        "x-osdu-indexing": {
+        "type": "nested"
+         },
+        "type": "array",
+        "items": {
+            "type": "object",
+        "properties":  {
+            "NegativeVerticalDelta"{
+                "description": "The distance vertically below the Marker position that marks the limit of the high confidence range for the Marker pick.",
+                "x-osdu-frame-of-reference": "UOM:length",
+                "type": "number"
+            },
+          .....
+```
+"x-osdu-indexing" property values
+```json
+"nested" , "flattened"
+```
+By default, only "nested" type will lead to inner objects processing
+```json
+{ 
+path = Markers,
+        kind = nested,
+        properties = [{
+                path = NegativeVerticalDelta,
+                kind = double
+            },
+        .....
+}
+```
+
+Arrays of primitive types are supported
 
 ```json
 "number", "string", "integer", "boolean"
@@ -370,4 +430,4 @@ is converted to
     }
     ]
 }
-```
+```
\ No newline at end of file

indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java

@@ -128,9 +128,6 @@ public class StorageIndexerPayloadMapper {
 				case GEO_SHAPE:
 					this.attributeParsingService.tryParseGeojson(recordId, name, value, dataMap);
 					break;
-				case NESTED:
-					// don't do anything , each nested property will be parsed separately
-					break;
 				case FLATTENED:
 					// flattened type inner properties will be added "as is" without parsing as they types not present in schema
 					this.attributeParsingService.tryParseFlattened(recordId, name, value, dataMap);

indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessorTest.java

@@ -14,6 +14,7 @@
 
 package org.opengroup.osdu.indexer.schema.converter;
 
+import com.google.common.collect.ImmutableMap;
 import org.junit.Test;
 import org.mockito.Mockito;
 import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
@@ -22,6 +23,7 @@ import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterPropert
 import org.opengroup.osdu.indexer.schema.converter.tags.AllOfItem;
 import org.opengroup.osdu.indexer.schema.converter.tags.Definition;
 import org.opengroup.osdu.indexer.schema.converter.tags.Definitions;
+import org.opengroup.osdu.indexer.schema.converter.tags.Items;
 import org.opengroup.osdu.indexer.schema.converter.tags.TypeProperty;
 
 import java.util.LinkedHashMap;
@@ -101,4 +103,109 @@ public class PropertiesProcessorTest {
                 .processItem(allOfItem).map(Object::toString).reduce("", String::concat);
         assertEquals("{path=" + PATH + ", kind=int}", res);
     }
+
+    @Test
+    public void should_return_processed_nested_array_items(){
+        JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class);
+
+        Map<String, TypeProperty> itemsProperties = new LinkedHashMap<>();
+        TypeProperty intProperty = new TypeProperty();
+        intProperty.setType("integer");
+        itemsProperties.put(PATH, intProperty);
+
+        Items items = new Items();
+        items.setProperties(itemsProperties);
+
+        TypeProperty arrayProperty = new TypeProperty();
+        arrayProperty.setIndexingType(ImmutableMap.of("type","nested"));
+        arrayProperty.setType("array");
+        arrayProperty.setItems(items);
+
+        Map<String, TypeProperty> allOfItemProperties = new LinkedHashMap<>();
+        allOfItemProperties.put(PATH,arrayProperty);
+
+        AllOfItem allOfItem = new AllOfItem();
+        allOfItem.setProperties(allOfItemProperties);
+
+        String res = new PropertiesProcessor(Mockito.mock(Definitions.class), log, new SchemaConverterPropertiesConfig())
+            .processItem(allOfItem).map(Object::toString).reduce("", String::concat);
+        assertEquals("{path="+ PATH + ", kind=nested, properties=[{path="+ PATH + ", kind=int}]}",res);
+    }
+
+    @Test
+    public void should_return_flattened_array_without_processing(){
+        JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class);
+
+        Map<String, TypeProperty> itemsProperties = new LinkedHashMap<>();
+        TypeProperty intProperty = new TypeProperty();
+        intProperty.setType("integer");
+        itemsProperties.put(PATH, intProperty);
+
+        Items items = new Items();
+        items.setProperties(itemsProperties);
+
+        TypeProperty arrayProperty = new TypeProperty();
+        arrayProperty.setIndexingType(ImmutableMap.of("type","flattened"));
+        arrayProperty.setType("array");
+        arrayProperty.setItems(items);
+
+        Map<String, TypeProperty> allOfItemProperties = new LinkedHashMap<>();
+        allOfItemProperties.put(PATH,arrayProperty);
+
+        AllOfItem allOfItem = new AllOfItem();
+        allOfItem.setProperties(allOfItemProperties);
+
+        String res = new PropertiesProcessor(Mockito.mock(Definitions.class), log, new SchemaConverterPropertiesConfig())
+            .processItem(allOfItem).map(Object::toString).reduce("", String::concat);
+        assertEquals("{path="+ PATH + ", kind=flattened}",res);
+    }
+
+    @Test
+    public void should_return_object_array_without_hints_in_schema_without_processing(){
+        JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class);
+
+        Map<String, TypeProperty> itemsProperties = new LinkedHashMap<>();
+        TypeProperty intProperty = new TypeProperty();
+        intProperty.setType("integer");
+        itemsProperties.put(PATH, intProperty);
+
+        Items items = new Items();
+        items.setProperties(itemsProperties);
+
+        TypeProperty arrayProperty = new TypeProperty();
+        arrayProperty.setType("array");
+        arrayProperty.setItems(items);
+
+        Map<String, TypeProperty> allOfItemProperties = new LinkedHashMap<>();
+        allOfItemProperties.put(PATH,arrayProperty);
+
+        AllOfItem allOfItem = new AllOfItem();
+        allOfItem.setProperties(allOfItemProperties);
+
+        String res = new PropertiesProcessor(Mockito.mock(Definitions.class), log, new SchemaConverterPropertiesConfig())
+            .processItem(allOfItem).map(Object::toString).reduce("", String::concat);
+        assertEquals("{path="+ PATH + ", kind=[]object}",res);
+    }
+
+    @Test
+    public void should_process_not_object_array_type(){
+        JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class);
+
+        Items items = new Items();
+        items.setType("integer");
+
+        TypeProperty arrayProperty = new TypeProperty();
+        arrayProperty.setType("array");
+        arrayProperty.setItems(items);
+
+        Map<String, TypeProperty> allOfItemProperties = new LinkedHashMap<>();
+        allOfItemProperties.put(PATH,arrayProperty);
+
+        AllOfItem allOfItem = new AllOfItem();
+        allOfItem.setProperties(allOfItemProperties);
+
+        String res = new PropertiesProcessor(Mockito.mock(Definitions.class), log, new SchemaConverterPropertiesConfig())
+            .processItem(allOfItem).map(Object::toString).reduce("", String::concat);
+        assertEquals("{path="+ PATH + ", kind=[]int}",res);
+    }
 }
\ No newline at end of file

testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java

 package org.opengroup.osdu.common;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.MapDifference;
 import com.google.common.collect.Maps;
 import com.google.gson.Gson;
@@ -11,6 +12,7 @@ import org.elasticsearch.cluster.metadata.MappingMetadata;
 import org.opengroup.osdu.core.common.model.entitlements.Acl;
 import org.opengroup.osdu.models.Setup;
 import org.opengroup.osdu.models.TestIndex;
+import org.opengroup.osdu.models.record.RecordData;
 import org.opengroup.osdu.util.ElasticUtils;
 import org.opengroup.osdu.util.FileHandler;
 import org.opengroup.osdu.util.HTTPClient;
@@ -32,6 +34,7 @@ import static org.opengroup.osdu.util.Config.getStorageBaseURL;
 @Log
 public class RecordSteps extends TestsBase {
     private Map<String, TestIndex> inputIndexMap = new HashMap<>();
+    private ObjectMapper mapper = new ObjectMapper();
     private boolean shutDownHookAdded = false;
 
     private String timeStamp = String.valueOf(System.currentTimeMillis());
@@ -158,19 +161,33 @@ public class RecordSteps extends TestsBase {
 
     public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery(
         int expectedNumber, String index, String path, String firstNestedField, String firstNestedValue, String secondNestedField, String secondNestedValue)
-        throws Exception {
+        throws Throwable {
         long numOfIndexedDocuments = createIndex(index);
         long actualNumberOfRecords = elasticUtils.fetchRecordsByNestedQuery(index, path, firstNestedField, firstNestedValue, secondNestedField, secondNestedValue);
         assertEquals(expectedNumber, actualNumberOfRecords);
     }
 
     public void i_should_be_able_search_documents_for_the_by_flattened_inner_properties(int expectedCount, String index, String flattenedField,
-        String flattenedFieldValue) throws IOException, InterruptedException {
+        String flattenedFieldValue) throws Throwable {
         long numOfIndexedDocuments = createIndex(index);
         long actualNumberOfRecords = elasticUtils.fetchRecordsWithFlattenedFieldsQuery(index, flattenedField, flattenedFieldValue);
         assertEquals(expectedCount, actualNumberOfRecords);
     }
 
+    public void i_should_get_object_in_search_response_without_hints_in_schema(String objectField, String index, String recordFile, String acl, String kind)
+        throws Throwable {
+        long numOfIndexedDocuments = createIndex(index);
+        String expectedRecord = FileHandler.readFile(String.format("%s.%s", recordFile, "json"));
+
+        RecordData[] fileRecordData = mapper.readValue(expectedRecord, RecordData[].class);
+        RecordData expectedRecordData = fileRecordData[0];
+
+        String elasticRecordData = elasticUtils.fetchDataFromObjectsArrayRecords(index);
+        RecordData actualRecordData = mapper.readValue(elasticRecordData, RecordData.class);
+
+        assertEquals(expectedRecordData.getData().get(objectField),actualRecordData.getData().get(objectField));
+    }
+
     private long createIndex(String index) throws InterruptedException, IOException {
         long numOfIndexedDocuments = 0;
         int iterator;

testing/indexer-test-core/src/main/java/org/opengroup/osdu/models/record/RecordData.java

+package org.opengroup.osdu.models.record;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.List;
+import java.util.Map;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class RecordData {
+
+    @JsonProperty("data")
+    private Map<String, List<Map<String,Object>>> data;
+
+    public Map<String, List<Map<String, Object>>> getData() {
+        return data;
+    }
+
+}

testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java

@@ -52,6 +52,7 @@ import org.elasticsearch.client.indices.GetIndexRequest;
 import org.elasticsearch.client.indices.GetMappingsRequest;
 import org.elasticsearch.client.indices.GetMappingsResponse;
 import org.elasticsearch.cluster.metadata.MappingMetadata;
+import org.elasticsearch.search.SearchHits;
 import org.locationtech.jts.geom.Coordinate;
 import org.elasticsearch.common.geo.builders.EnvelopeBuilder;
 import org.elasticsearch.common.settings.Settings;
@@ -309,6 +310,24 @@ public class ElasticUtils {
         }
     }
 
+    public String fetchDataFromObjectsArrayRecords(String index) throws IOException {
+        try {
+            try (RestHighLevelClient client = this.createClient(username, password, host)) {
+                SearchRequest request = new SearchRequest(index);
+                SearchResponse searchResponse = client.search(request, RequestOptions.DEFAULT);
+
+                SearchHits searchHits = searchResponse.getHits();
+                if (searchHits.getHits().length != 0) {
+                    return searchHits.getHits()[0].getSourceAsString();
+                }
+                return null;
+            }
+        } catch (ElasticsearchStatusException e) {
+            log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage()));
+            return null;
+        }
+    }
+
     public Map<String, MappingMetadata> getMapping(String index) throws IOException {
         try (RestHighLevelClient client = this.createClient(username, password, host)) {
             GetMappingsRequest request = new GetMappingsRequest();

testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature

@@ -48,7 +48,8 @@ Feature: Indexing of the documents
     When I ingest records with the <recordFile> with <acl> for a given <kind>
     Then I should be able search <number> documents for the <index> by nested <path> and properties (<first_nested_field>, <first_nested_value>) and  (<second_nested_field>, <second_nested_value>)
     Then I should be able search <number> documents for the <index> by flattened inner properties (<flattened_inner_field>, <flattened_inner_value>)
+    Then I should get <object_inner_field> in response, without hints in schema for the <index> that present in the <recordFile> with <acl> for a given <kind>
 
     Examples:
-      | kind                                              | recordFile                       | number | index                                             | acl                            | path              | first_nested_field           | first_nested_value | second_nested_field          | second_nested_value | flattened_inner_field           | flattened_inner_value |
-      | "tenant1:wks:ArraysOfObjectsTestCollection:4.0.0" | "r3-index_record_arrayofobjects" | 1      | "tenant1-wks-arraysofobjectstestcollection-4.0.0" | "data.default.viewers@tenant1" | "data.NestedTest" | "data.NestedTest.NumberTest" | 12345              | "data.NestedTest.StringTest" | "test string"       | "data.FlattenedTest.StringTest" | "test string"         |
+      | kind                                              | recordFile                       | number | index                                             | acl                            | path              | first_nested_field           | first_nested_value | second_nested_field          | second_nested_value | flattened_inner_field           | flattened_inner_value | object_inner_field |
+      | "tenant1:wks:ArraysOfObjectsTestCollection:4.0.0" | "r3-index_record_arrayofobjects" | 1      | "tenant1-wks-arraysofobjectstestcollection-4.0.0" | "data.default.viewers@tenant1" | "data.NestedTest" | "data.NestedTest.NumberTest" | 12345              | "data.NestedTest.StringTest" | "test string"       | "data.FlattenedTest.StringTest" | "test string"         | "ObjectTest"       |
\ No newline at end of file

testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java

@@ -6,7 +6,6 @@ import cucumber.api.java.Before;
 import cucumber.api.java.en.Given;
 import cucumber.api.java.en.Then;
 import cucumber.api.java.en.When;
-import java.io.IOException;
 import lombok.extern.java.Log;
 import org.opengroup.osdu.common.SchemaServiceRecordSteps;
 import org.opengroup.osdu.util.ElasticUtils;
@@ -66,7 +65,7 @@ public class Steps extends SchemaServiceRecordSteps {
     @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by nested \"([^\"]*)\" and properties \\(\"([^\"]*)\", (\\d+)\\) and  \\(\"([^\"]*)\", \"([^\"]*)\"\\)$")
     public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery(
         int expectedCount, String index, String path, String firstNestedProperty, String firstNestedValue, String secondNestedProperty,
-        String secondNestedValue) throws Exception {
+        String secondNestedValue) throws Throwable {
         String actualName = generateActualName(index, null);
         super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery(expectedCount, actualName, path, firstNestedProperty, firstNestedValue,
             secondNestedProperty, secondNestedValue);
@@ -74,10 +73,16 @@ public class Steps extends SchemaServiceRecordSteps {
 
     @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by flattened inner properties \\(\"([^\"]*)\", \"([^\"]*)\"\\)$")
     public void i_should_be_able_search_documents_for_the_by_flattened_inner_properties(int expectedCount, String index, String flattenedField,
-        String flattenedFieldValue)
-        throws IOException, InterruptedException {
+        String flattenedFieldValue) throws Throwable {
         String actualName = generateActualName(index, null);
         super.i_should_be_able_search_documents_for_the_by_flattened_inner_properties(expectedCount, actualName, flattenedField, flattenedFieldValue);
 
     }
+
+    @Then("^I should get \"([^\"]*)\" in response, without hints in schema for the \"([^\"]*)\" that present in the \"([^\"]*)\" with \"([^\"]*)\" for a given \"([^\"]*)\"$")
+    public void i_should_get_object_in_search_response_without_hints_in_schema(String objectInnerField, String index, String recordFile, String acl, String kind)
+        throws Throwable {
+        String actualName = generateActualName(index, null);
+        super.i_should_get_object_in_search_response_without_hints_in_schema(objectInnerField ,actualName, recordFile, acl, kind);
+    }
 }
\ No newline at end of file