Merge branch 'feature/GONRG-2028-objects-array-support' into array-of-objects

indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java

@@ -102,7 +102,7 @@ public class TypeMapper {
 
         if(isMap(indexerType)){
             Map<String,Object> type = (Map<String, Object>) indexerType;
-            HashMap<String, Object> propertiesMap = (HashMap<String, Object>) type.get(Constants.PROPERTIES);
+            Map<String, Object> propertiesMap = (Map<String, Object>) type.get(Constants.PROPERTIES);
             for (Map.Entry<String,Object> entry : propertiesMap.entrySet()){
                 entry.setValue(Records.Type.builder().type(entry.getValue().toString()).build());
             }

testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java

@@ -151,11 +151,29 @@ public class RecordSteps extends TestsBase {
     public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery (
             int expectedNumber, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field) throws Throwable {
         index = generateActualName(index, timeStamp);
-        long numOfIndexedDocuments = createIndex(index);
-        long actualNumberOfRecords = elasticUtils.fetchRecordsByBoundingBoxQuery(index, field, topLatitude, topLongitude, bottomLatitude, bottomLongitude);
+        String actualName = generateActualName(index, timeStamp);
+        long numOfIndexedDocuments = createIndex(actualName);
+        long actualNumberOfRecords = elasticUtils.fetchRecordsByBoundingBoxQuery(actualName, field, topLatitude, topLongitude, bottomLatitude, bottomLongitude);
+        assertEquals(expectedNumber, actualNumberOfRecords);
+    }
+
+    public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery(
+        int expectedNumber, String index, String path, String firstNestedField, String firstNestedValue, String secondNestedField, String secondNestedValue)
+        throws Exception {
+        String actualName = generateActualName(index, timeStamp);
+        long numOfIndexedDocuments = createIndex(actualName);
+        long actualNumberOfRecords = elasticUtils.fetchRecordsByNestedQuery(actualName, path, firstNestedField, firstNestedValue, secondNestedField, secondNestedValue);
         assertEquals(expectedNumber, actualNumberOfRecords);
     }
 
+    public void i_should_be_able_search_documents_for_the_by_flattened_inner_properties(int expectedCount, String index, String flattenedField,
+        String flattenedFieldValue) throws IOException, InterruptedException {
+        String actualName = generateActualName(index, timeStamp);
+        long numOfIndexedDocuments = createIndex(actualName);
+        long actualNumberOfRecords = elasticUtils.fetchRecordsWithFlattenedFieldsQuery(actualName, flattenedField, flattenedFieldValue);
+        assertEquals(expectedCount, actualNumberOfRecords);
+    }
+
     private long createIndex(String index) throws InterruptedException, IOException {
         long numOfIndexedDocuments = 0;
         int iterator;

testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/SchemaServiceRecordSteps.java

 package org.opengroup.osdu.common;
 
 import cucumber.api.DataTable;
+import java.util.List;
+import java.util.Map;
+import lombok.extern.java.Log;
 import org.opengroup.osdu.models.Setup;
 import org.opengroup.osdu.models.schema.PersistentSchemaTestIndex;
 import org.opengroup.osdu.util.ElasticUtils;
 import org.opengroup.osdu.util.HTTPClient;
+import org.opengroup.osdu.util.IndexerClientUtil;
 
-import java.util.List;
-import java.util.Map;
-
+@Log
 public class SchemaServiceRecordSteps extends RecordSteps {
 
+    private IndexerClientUtil indexerClient;
+
     public SchemaServiceRecordSteps(HTTPClient httpClient, ElasticUtils elasticUtils) {
         super(httpClient, elasticUtils);
+        indexerClient = new IndexerClientUtil(this.httpClient);
     }
 
     public void the_schema_is_created_with_the_following_kind(DataTable dataTable) {
         List<Setup> inputList = dataTable.asList(Setup.class);
         inputList.forEach(this::createSchema);
-        inputList.forEach(s -> deleteIndex(generateActualNameWithoutTs(s.getIndex())));
+        inputList.forEach(s -> deleteIndex(generateActualNameWithoutTs(s.getKind())));
         super.addShutDownHook();
     }
 
@@ -33,8 +38,8 @@ public class SchemaServiceRecordSteps extends RecordSteps {
         super.getInputIndexMap().put(testIndex.getKind(), testIndex);
     }
 
-    private void deleteIndex(String index) {
-        this.elasticUtils.deleteIndex(index);
+    private void deleteIndex(String kind) {
+        indexerClient.deleteIndex(kind);
     }
 
     @Override

testing/indexer-test-core/src/main/java/org/opengroup/osdu/models/TestIndex.java

@@ -19,6 +19,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.logging.Logger;
+import org.opengroup.osdu.util.IndexerClientUtil;
 
 import static org.junit.Assert.assertEquals;
 import static org.opengroup.osdu.util.Config.*;
@@ -38,6 +39,7 @@ public class TestIndex {
     private HTTPClient httpClient;
     private Map<String, String> headers;
     private ElasticUtils elasticUtils;
+    protected IndexerClientUtil indexerClient;
     private Gson gson = new Gson();
 
     public TestIndex(ElasticUtils elasticUtils){
@@ -47,6 +49,7 @@ public class TestIndex {
     public void setHttpClient(HTTPClient httpClient) {
         this.httpClient = httpClient;
         headers = httpClient.getCommonHeader();
+        indexerClient = new IndexerClientUtil(this.httpClient);
     }
 
     public void setupIndex() {
@@ -73,7 +76,7 @@ public class TestIndex {
     }
 
     public void cleanupIndex() {
-        this.elasticUtils.deleteIndex(index);
+        this.indexerClient.deleteIndex(kind);
     }
 
     private String getRecordFile() {

testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java

@@ -32,6 +32,7 @@ import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
 import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
 import org.apache.http.message.BasicHeader;
 import org.apache.http.ssl.SSLContextBuilder;
+import org.apache.lucene.search.join.ScoreMode;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.client.indices.CloseIndexRequest;
@@ -274,6 +275,40 @@ public class ElasticUtils {
         }
     }
 
+    public long fetchRecordsByNestedQuery(String index, String path, String firstNestedField, String firstNestedValue, String secondNestedField, String secondNestedValue) throws Exception{
+        try (RestHighLevelClient client = this.createClient(username, password, host)) {
+            SearchRequest searchRequest = new SearchRequest(index);
+
+            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+            searchSourceBuilder.query(nestedQuery(path,boolQuery().must(matchQuery(firstNestedField,firstNestedValue)).must(matchQuery(secondNestedField,secondNestedValue)), ScoreMode.Avg));
+
+            searchRequest.source(searchSourceBuilder);
+
+            SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
+            return searchResponse.getHits().getTotalHits().value;
+        } catch (ElasticsearchStatusException e) {
+            log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage()));
+            return -1;
+        }
+    }
+
+
+    public long fetchRecordsWithFlattenedFieldsQuery(String index, String flattenedField, String flattenedFieldValue) throws IOException {
+        try (RestHighLevelClient client = this.createClient(username, password, host)) {
+            SearchRequest searchRequest = new SearchRequest(index);
+
+            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+            searchSourceBuilder.query(boolQuery().must(matchQuery(flattenedField,flattenedFieldValue)));
+            searchRequest.source(searchSourceBuilder);
+
+            SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
+            return searchResponse.getHits().getTotalHits().value;
+        } catch (ElasticsearchStatusException e) {
+            log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage()));
+            return -1;
+        }
+    }
+
     public Map<String, MappingMetadata> getMapping(String index) throws IOException {
         try (RestHighLevelClient client = this.createClient(username, password, host)) {
             GetMappingsRequest request = new GetMappingsRequest();
@@ -415,4 +450,5 @@ public class ElasticUtils {
         }
         return false;
     }
+
 }
\ No newline at end of file

testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/IndexerClientUtil.java

+package org.opengroup.osdu.util;
+
+import static org.opengroup.osdu.util.Config.getDataPartitionIdTenant1;
+import static org.opengroup.osdu.util.Config.getIndexerBaseURL;
+
+import com.google.gson.Gson;
+import com.sun.jersey.api.client.ClientResponse;
+import java.util.Map;
+import javax.ws.rs.HttpMethod;
+import lombok.extern.java.Log;
+import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
+
+@Log
+public class IndexerClientUtil {
+
+    private final String purgeMessage = "{\"data\":\"[{\\\"kind\\\":\\\"%s\\\",\\\"op\\\":\\\"purge_schema\\\"}]\",\"attributes\":{\"account-id\":\"%s\"}}";
+
+    private final HTTPClient httpClient;
+    private Map<String, String> headers;
+
+    public IndexerClientUtil(HTTPClient httpClient) {
+        this.httpClient = httpClient;
+        headers = httpClient.getCommonHeader();
+    }
+
+    public void deleteIndex(String kind) {
+        String url = getIndexerBaseURL() + "index-cleanup";
+        log.info("URL: " + url);
+        ClientResponse response = httpClient.send(HttpMethod.POST, url, convertMessageIntoJson(kind), headers, httpClient.getAccessToken());
+        log.info(response.toString());
+    }
+
+    private String convertMessageIntoJson(String kind) {
+        RecordChangedMessages
+            recordChangedMessages = (new Gson()).fromJson(String.format(purgeMessage, kind, getDataPartitionIdTenant1()), RecordChangedMessages.class);
+        return new Gson().toJson(recordChangedMessages);
+    }
+}

testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature

@@ -3,11 +3,12 @@ Feature: Indexing of the documents
 
   Background:
     Given the schema is created with the following kind
-      | kind                                         | index                                        | schemaFile                 |
-      | tenant1:indexer:test-data--Integration:1.0.0 | tenant1-indexer-test-data--integration-1.0.0 | index_records_1            |
-      | tenant1:indexer:test-data--Integration:2.0.0 | tenant1-indexer-test-data--integration-2.0.0 | index_records_2            |
-      | tenant1:indexer:test-data--Integration:3.0.0 | tenant1-indexer-test-data--integration-3.0.0 | index_records_3            |
-      | tenant1:wks:master-data--Wellbore:2.0.3      | tenant1-wks-master-data--wellbore-2.0.3      | r3-index_record_wks_master |
+      | kind                                            | index                                           | schemaFile                     |
+      | tenant1:indexer:test-data--Integration:1.0.0    | tenant1-indexer-test-data--integration-1.0.0    | index_records_1                |
+      | tenant1:indexer:test-data--Integration:2.0.0    | tenant1-indexer-test-data--integration-2.0.0    | index_records_2                |
+      | tenant1:indexer:test-data--Integration:3.0.0    | tenant1-indexer-test-data--integration-3.0.0    | index_records_3                |
+      | tenant1:wks:master-data--Wellbore:2.0.3         | tenant1-wks-master-data--wellbore-2.0.3         | r3-index_record_wks_master     |
+      | tenant1:wks:ArraysOfObjectsTestCollection:4.0.0 | tenant1-wks-arraysofobjectstestcollection-4.0.0 | r3-index_record_arrayofobjects |
 
   Scenario Outline: Ingest the record and Index in the Elastic Search
     When I ingest records with the <recordFile> with <acl> for a given <kind>
@@ -42,3 +43,12 @@ Feature: Indexing of the documents
     Examples:
       | kind                                      | recordFile                   | number | index                                     | acl                            | field                                   | top_left_latitude | top_left_longitude | bottom_right_latitude | bottom_right_longitude |
       | "tenant1:wks:master-data--Wellbore:2.0.3" | "r3-index_record_wks_master" | 1      | "tenant1-wks-master-data--wellbore-2.0.3" | "data.default.viewers@tenant1" | "data.SpatialLocation.Wgs84Coordinates" | 52                | -100               | 0                     | 100                    |
+
+  Scenario Outline: Ingest the r3-record with arrays of objects and hints in schema and Index in the Elastic Search
+    When I ingest records with the <recordFile> with <acl> for a given <kind>
+    Then I should be able search <number> documents for the <index> by nested <path> and properties (<first_nested_field>, <first_nested_value>) and  (<second_nested_field>, <second_nested_value>)
+    Then I should be able search <number> documents for the <index> by flattened inner properties (<flattened_inner_field>, <flattened_inner_value>)
+
+    Examples:
+      | kind                                              | recordFile                       | number | index                                             | acl                            | path              | first_nested_field           | first_nested_value | second_nested_field          | second_nested_value | flattened_inner_field           | flattened_inner_value |
+      | "tenant1:wks:ArraysOfObjectsTestCollection:4.0.0" | "r3-index_record_arrayofobjects" | 1      | "tenant1-wks-arraysofobjectstestcollection-4.0.0" | "data.default.viewers@tenant1" | "data.NestedTest" | "data.NestedTest.NumberTest" | 12345              | "data.NestedTest.StringTest" | "test string"       | "data.FlattenedTest.StringTest" | "test string"         |
\ No newline at end of file

testing/indexer-test-core/src/main/resources/testData/r3-index_record_arrayofobjects.json

+[
+    {
+        "id": "tenant1:<kindSubType>:testIngest70",
+        "data": {
+            "NestedTest": [
+                {
+                    "DateTimeTest": "2020-02-13T09:13:15.55Z",
+                    "NumberTest": "12345",
+                    "StringTest": "test string"
+                },
+                {
+                    "DateTimeTest": "2020-02-13T09:13:15.55Z",
+                    "NumberTest": "567890",
+                    "StringTest": "test string"
+                }
+            ],
+            "FlattenedTest": [
+                {
+                    "DateTimeTest": "2020-02-13T09:13:15.55Z",
+                    "NumberTest": "12345",
+                    "StringTest": "test string"
+                },
+                {
+                    "DateTimeTest": "2020-02-13T09:13:15.55Z",
+                    "NumberTest": "567890",
+                    "StringTest": "test string"
+                }
+            ],
+            "ObjectTest": [
+                {
+                    "DateTimeTest": "2020-02-13T09:13:15.55Z",
+                    "NumberTest": "12345",
+                    "StringTest": "test string"
+                },
+                {
+                    "DateTimeTest": "2020-02-13T09:13:15.55Z",
+                    "NumberTest": "567890",
+                    "StringTest": "test string"
+                }
+            ]
+        }
+    }
+]
\ No newline at end of file

testing/indexer-test-core/src/main/resources/testData/r3-index_record_arrayofobjects.schema.json

+{
+    "schemaInfo": {
+        "schemaIdentity": {
+            "authority": "tenant1",
+            "source": "wks",
+            "entityType": "ArraysOfObjectsTestCollection",
+            "schemaVersionMajor": 4,
+            "schemaVersionMinor": 0,
+            "schemaVersionPatch": 0
+        },
+        "status": "DEVELOPMENT"
+    },
+    "schema": {
+        "x-osdu-license": "Copyright 2021, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+        "$schema": "http://json-schema.org/draft-07/schema#",
+        "description": "Array of objects testing schema",
+        "title": "Test",
+        "type": "object",
+        "required": [
+            "kind",
+            "acl",
+            "legal"
+        ],
+        "properties": {
+            "data": {
+                "allOf": [
+                    {
+                        "type": "object",
+                        "properties": {
+                            "NestedTest": {
+                                "description": "nested type test",
+                                "type": "array",
+                                "x-osdu-indexing": {
+                                    "type": "nested"
+                                },
+                                "items": {
+                                    "type": "object",
+                                    "properties": {
+                                        "DateTimeTest": {
+                                            "description": "date and time test",
+                                            "type": "string",
+                                            "format": "date-time",
+                                            "x-osdu-frame-of-reference": "DateTime"
+                                        },
+                                        "NumberTest": {
+                                            "description": "number test",
+                                            "type": "number"
+                                        },
+                                        "StringTest": {
+                                            "description": "string test",
+                                            "type": "string"
+                                        }
+                                    }
+                                }
+                            },
+                            "FlattenedTest": {
+                                "description": "flattened type test",
+                                "type": "array",
+                                "x-osdu-indexing": {
+                                    "type": "flattened"
+                                },
+                                "items": {
+                                    "type": "object",
+                                    "properties": {
+                                        "DateTimeTest": {
+                                            "description": "date and time test",
+                                            "type": "string",
+                                            "format": "date-time",
+                                            "x-osdu-frame-of-reference": "DateTime"
+                                        },
+                                        "NumberTest": {
+                                            "description": "number test",
+                                            "type": "number"
+                                        },
+                                        "StringTest": {
+                                            "description": "string test",
+                                            "type": "string"
+                                        }
+                                    }
+                                }
+                            },
+                            "ObjectTest": {
+                                "description": "default object type test",
+                                "type": "array",
+                                "items": {
+                                    "type": "object",
+                                    "properties": {
+                                        "DateTimeTest": {
+                                            "description": "date and time test",
+                                            "type": "string",
+                                            "format": "date-time",
+                                            "x-osdu-frame-of-reference": "DateTime"
+                                        },
+                                        "NumberTest": {
+                                            "description": "number test",
+                                            "type": "number"
+                                        },
+                                        "StringTest": {
+                                            "description": "string test",
+                                            "type": "string"
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                ]
+            }
+        },
+        "x-osdu-inheriting-from-kind": []
+    }
+}
\ No newline at end of file

testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java

@@ -6,7 +6,7 @@ import org.junit.runner.RunWith;
 
 @RunWith(Cucumber.class)
 @CucumberOptions(
-        features = "classpath:features/indexrecord/IndexRecord.feature",
+        features = "classpath:features/indexrecord/indexRecord-schema-service.feature",
         glue = {"classpath:org.opengroup.osdu.step_definitions/index/record"},
         plugin = {"pretty", "junit:target/cucumber-reports/TEST-indexrecord.xml"})
 public class RunTest {

testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java

 package org.opengroup.osdu.step_definitions.index.record;
 
-import lombok.extern.java.Log;
-import org.opengroup.osdu.common.RecordSteps;
-import org.opengroup.osdu.util.GCPHTTPClient;
-
+import cucumber.api.DataTable;
 import cucumber.api.Scenario;
 import cucumber.api.java.Before;
-import cucumber.api.DataTable;
 import cucumber.api.java.en.Given;
 import cucumber.api.java.en.Then;
 import cucumber.api.java.en.When;
+import java.io.IOException;
+import lombok.extern.java.Log;
+import org.opengroup.osdu.common.SchemaServiceRecordSteps;
+import org.opengroup.osdu.util.ElasticUtils;
+import org.opengroup.osdu.util.GCPHTTPClient;
 
 @Log
-public class Steps extends RecordSteps {
+public class Steps extends SchemaServiceRecordSteps {
 
     public Steps() {
-        super(new GCPHTTPClient());
+        super(new GCPHTTPClient(), new ElasticUtils());
     }
 
     @Before
@@ -48,5 +49,31 @@ public class Steps extends RecordSteps {
     public void iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(int expectedCount, String index, String skippedAttributes) throws Throwable {
         super.iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(expectedCount, index, skippedAttributes);
     }
+//TODO fix tags step
 
+//    @Then("^I should be able to search (\\d+) record with index \"([^\"]*)\" by tag \"([^\"]*)\" and value \"([^\"]*)\"$")
+//    public void iShouldBeAbleToSearchRecordByTagKeyAndTagValue(int expectedNumber, String index, String tagKey, String tagValue) throws Throwable {
+//        super.iShouldBeAbleToSearchRecordByTagKeyAndTagValue(index, tagKey, tagValue, expectedNumber);
+//    }
+//
+    @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by bounding box query with points \\((-?\\d+), (-?\\d+)\\) and  \\((-?\\d+), (-?\\d+)\\) on field \"([^\"]*)\"$")
+    public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery(
+        int expectedCount, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field) throws Throwable {
+        super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery(expectedCount, index, topLatitude, topLongitude, bottomLatitude, bottomLongitude, field);
+    }
+
+    @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by nested \"([^\"]*)\" and properties \\(\"([^\"]*)\", (\\d+)\\) and  \\(\"([^\"]*)\", \"([^\"]*)\"\\)$")
+    public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery(
+        int expectedCount, String index, String path, String firstNestedProperty, String firstNestedValue, String secondNestedProperty,
+        String secondNestedValue) throws Exception {
+        super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery(expectedCount, index, path, firstNestedProperty, firstNestedValue,
+            secondNestedProperty, secondNestedValue);
+    }
+
+    @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by flattened inner properties \\(\"([^\"]*)\", \"([^\"]*)\"\\)$")
+    public void i_should_be_able_search_documents_for_the_by_flattened_inner_properties(int expectedCount, String index, String flattenedField, String flattenedFieldValue)
+        throws IOException, InterruptedException {
+        super.i_should_be_able_search_documents_for_the_by_flattened_inner_properties(expectedCount,index,flattenedField,flattenedFieldValue);
+
+    }
 }
\ No newline at end of file