diff --git a/testing/indexer-test-anthos/src/test/java/org/opengroup/osdu/step_definitions/record/Steps.java b/testing/indexer-test-anthos/src/test/java/org/opengroup/osdu/step_definitions/record/Steps.java
index 40620fbf4a626a1eb279f28fae937a55ffdc8469..96121b12f01b0f418efad0104f84189335619fcc 100644
--- a/testing/indexer-test-anthos/src/test/java/org/opengroup/osdu/step_definitions/record/Steps.java
+++ b/testing/indexer-test-anthos/src/test/java/org/opengroup/osdu/step_definitions/record/Steps.java
@@ -88,6 +88,16 @@ public class Steps extends SchemaServiceRecordSteps {
super.iShouldBeAbleToSearchRecordByTagKeyAndTagValue(index, tagKey, tagValue, expectedNumber);
}
+ @Then("^I clean up the index of the extended kinds \"([^\"]*)\" in the Elastic Search$")
+ public void iShouldCleanupIndicesOfExtendedKinds(String extendedKinds) throws Throwable {
+ super.iShouldCleanupIndicesOfExtendedKinds(extendedKinds);
+ }
+
+ @Then("^I should be able to search (\\d+) record with index \"([^\"]*)\" by extended data field \"([^\"]*)\" and value \"([^\"]*)\"$")
+ public void iShouldBeAbleToSearchRecordByFieldAndFieldValue(int expectedNumber, String index, String fieldKey, String fieldValue) throws Throwable {
+ super.iShouldBeAbleToSearchRecordByFieldAndFieldValue(index, fieldKey, fieldValue, expectedNumber);
+ }
+
@Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by bounding box query with points \\((-?\\d+), (-?\\d+)\\) and \\((-?\\d+), (-?\\d+)\\) on field \"([^\"]*)\"$")
public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery(
int expectedCount, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field)
@@ -121,4 +131,4 @@ public class Steps extends SchemaServiceRecordSteps {
String actualName = generateActualName(index, null);
super.i_should_get_object_in_search_response_without_hints_in_schema(objectInnerField, actualName, recordFile, acl, kind);
}
-}
\ No newline at end of file
+}
diff --git a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
index 37dd8f26ea74440a4c9affde1844a74d4168ec0b..0499ea073979803f4b78549c934e56d2ade13a32 100644
--- a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
+++ b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -115,6 +115,16 @@ public class Steps extends SchemaServiceRecordSteps {
super.iShouldBeAbleToSearchRecordByTagKeyAndTagValue(index, tagKey, tagValue, expectedNumber);
}
+ @Then("^I clean up the index of the extended kinds \"([^\"]*)\" in the Elastic Search$")
+ public void iShouldCleanupIndicesOfExtendedKinds(String extendedKinds) throws Throwable {
+ super.iShouldCleanupIndicesOfExtendedKinds(extendedKinds);
+ }
+
+ @Then("^I should be able to search (\\d+) record with index \"([^\"]*)\" by extended data field \"([^\"]*)\" and value \"([^\"]*)\"$")
+ public void iShouldBeAbleToSearchRecordByFieldAndFieldValue(int expectedNumber, String index, String fieldKey, String fieldValue) throws Throwable {
+ super.iShouldBeAbleToSearchRecordByFieldAndFieldValue(index, fieldKey, fieldValue, expectedNumber);
+ }
+
@Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by bounding box query with points \\((-?\\d+), (-?\\d+)\\) and \\((-?\\d+), (-?\\d+)\\) on field \"(.*?)\"$")
public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery (
int expectedCount, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field) throws Throwable {
@@ -141,4 +151,4 @@ public class Steps extends SchemaServiceRecordSteps {
throws Throwable {
super.i_should_get_object_in_search_response_without_hints_in_schema(objectInnerField ,index, recordFile, acl, kind);
}
-}
\ No newline at end of file
+}
diff --git a/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
index ae5e82f842d377236fccbedf449f30d25167f811..1742398ba09a8cd1f2f978f1df37e7246b6026ed 100644
--- a/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
+++ b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -83,6 +83,16 @@ public class Steps extends SchemaServiceRecordSteps {
super.iShouldBeAbleToSearchRecordByTagKeyAndTagValue(index, tagKey, tagValue, expectedNumber);
}
+ @Then("^I clean up the index of the extended kinds \"([^\"]*)\" in the Elastic Search$")
+ public void iShouldCleanupIndicesOfExtendedKinds(String extendedKinds) throws Throwable {
+ super.iShouldCleanupIndicesOfExtendedKinds(extendedKinds);
+ }
+
+ @Then("^I should be able to search (\\d+) record with index \"([^\"]*)\" by extended data field \"([^\"]*)\" and value \"([^\"]*)\"$")
+ public void iShouldBeAbleToSearchRecordByFieldAndFieldValue(int expectedNumber, String index, String fieldKey, String fieldValue) throws Throwable {
+ super.iShouldBeAbleToSearchRecordByFieldAndFieldValue(index, fieldKey, fieldValue, expectedNumber);
+ }
+
@Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by bounding box query with points \\((-?\\d+), (-?\\d+)\\) and \\((-?\\d+), (-?\\d+)\\) on field \"([^\"]*)\"$")
public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery(
int expectedCount, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field) throws Throwable {
@@ -109,4 +119,4 @@ public class Steps extends SchemaServiceRecordSteps {
throws Throwable {
super.i_should_get_object_in_search_response_without_hints_in_schema(objectInnerField ,index, recordFile, acl, kind);
}
-}
\ No newline at end of file
+}
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
index ebfaba6b5ef99eaf3a5429799057d459172c5063..da0676cf610850a43b80e0f31cc6e465525b2de1 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
@@ -21,6 +21,7 @@ import org.springframework.util.CollectionUtils;
import javax.ws.rs.HttpMethod;
import java.io.IOException;
import java.lang.reflect.Type;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -101,6 +102,14 @@ public class RecordSteps extends TestsBase {
String createTime = java.time.Instant.now().toString();
for (Map<String, Object> testRecord : records) {
+ if(testRecord.containsKey("data")) {
+ Map<String, Object> data = (Map<String, Object>)testRecord.get("data");
+ if(data != null && data.size() > 0) {
+ data = replaceValues(data, timeStamp);
+ testRecord.put("data", data);
+ }
+ }
+
testRecord.put("kind", actualKind);
testRecord.put("id", generateRecordId(testRecord));
testRecord.put("legal", generateLegalTag());
@@ -184,6 +193,22 @@ public class RecordSteps extends TestsBase {
assertEquals(expectedNumber, actualNumberOfRecords);
}
+ public void iShouldCleanupIndicesOfExtendedKinds(String extendedKinds) throws Throwable {
+ String[] kinds = extendedKinds.split(",");
+ for(String kind : kinds) {
+ String actualKind = this.generateActualName(kind.trim(), timeStamp);
+ TestIndex testIndex = this.getInputIndexMap().get(actualKind);
+ testIndex.cleanupIndex(actualKind);
+ }
+ }
+
+ public void iShouldBeAbleToSearchRecordByFieldAndFieldValue(String index, String fieldKey, String fieldValue, int expectedNumber) throws Throwable {
+ TimeUnit.SECONDS.sleep(60);
+ index = generateActualName(index, timeStamp);
+ long actualNumberOfRecords = elasticUtils.fetchRecordsByFieldAndFieldValue(index, fieldKey, fieldValue);
+ assertEquals(expectedNumber, actualNumberOfRecords);
+ }
+
public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery (
int expectedNumber, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field) throws Throwable {
index = generateActualName(index, timeStamp);
@@ -232,6 +257,46 @@ public class RecordSteps extends TestsBase {
testIndex.addIndex();
}
+ private Map<String, Object> replaceValues(Map<String, Object> data, String timeStamp) {
+ for(String key : data.keySet()) {
+ Object value = data.get(key);
+ Object replacedValue = replaceValue(value, timeStamp);
+ data.put(key, replacedValue);
+ }
+ return data;
+ }
+
+ private List<Object> replaceValues(List<Object> values, String timeStamp) {
+ List<Object> replacedValues = new ArrayList<>();
+ for(Object value : values) {
+ Object replacedValue = replaceValue(value, timeStamp);
+ replacedValues.add(replacedValue);
+ }
+
+ return replacedValues;
+ }
+
+ private Object replaceValue(Object value, String timeStamp) {
+ Object replacedValue = value;
+
+ if(value instanceof String) {
+ String rawValue = (String) value;
+ for (Map.Entry<String, String> tenant : tenantMap.entrySet()) {
+ rawValue = rawValue.replaceAll(tenant.getKey() + ":", tenant.getValue() + ":");
+ }
+ replacedValue = rawValue.replaceAll("<timestamp>", timeStamp);
+ }
+ else if(value instanceof List) {
+ replacedValue = replaceValues((List)value, timeStamp);
+ }
+ else if(value instanceof Map) {
+ replacedValue = replaceValues((Map<String, Object>) value, timeStamp);
+ }
+
+ return replacedValue;
+ }
+
+
private long createIndex(String index) throws InterruptedException, IOException {
long numOfIndexedDocuments = 0;
int iterator;
@@ -320,4 +385,4 @@ public class RecordSteps extends TestsBase {
shutDownHookAdded = true;
}
}
-}
\ No newline at end of file
+}
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/SchemaServiceRecordSteps.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/SchemaServiceRecordSteps.java
index af48584f1b8e315f0435c65d78442913b8cbfd6f..78e445a7234d291975ab9e84b4437c77e0be2712 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/SchemaServiceRecordSteps.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/SchemaServiceRecordSteps.java
@@ -11,14 +11,19 @@ import java.util.Map;
public class SchemaServiceRecordSteps extends RecordSteps {
+ private static boolean runBackgroundOnce = false;
+
public SchemaServiceRecordSteps(HTTPClient httpClient, ElasticUtils elasticUtils) {
super(httpClient, elasticUtils);
}
public void the_schema_is_created_with_the_following_kind(DataTable dataTable) {
- List<Setup> inputList = dataTable.asList(Setup.class);
- inputList.forEach(this::setup);
- super.addShutDownHook();
+ if(!runBackgroundOnce) {
+ List<Setup> inputList = dataTable.asList(Setup.class);
+ inputList.forEach(this::setup);
+ super.addShutDownHook();
+ runBackgroundOnce = true;
+ }
}
private void setup(Setup input) {
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
index b37e7daa324d2f73d3c85598dcb64a5317abaf5b..de2b99f64cf6857596062d614281dfcd345321a0 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
@@ -244,6 +244,24 @@ public class ElasticUtils {
}
}
+ public long fetchRecordsByFieldAndFieldValue(String index, String fieldKey, String fieldValue) throws IOException {
+ try {
+ try (RestHighLevelClient client = this.createClient(username, password, host)) {
+ SearchRequest request = new SearchRequest(index);
+ if(!Strings.isNullOrEmpty(fieldKey)) {
+ SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
+ sourceBuilder.query(boolQuery().must(matchQuery(fieldKey, fieldValue)));
+ request.source(sourceBuilder);
+ }
+ SearchResponse searchResponse = client.search(request, RequestOptions.DEFAULT);
+ return searchResponse.getHits().getTotalHits().value;
+ }
+ } catch (ElasticsearchStatusException e) {
+ log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage()));
+ return -1;
+ }
+ }
+
public List<Map<String, Object>> fetchRecordsByAttribute(String index, String attributeKey, String attributeValue) throws IOException {
List<Map<String, Object>> out = new ArrayList<>();
try {
@@ -499,4 +517,4 @@ public class ElasticUtils {
return false;
}
-}
\ No newline at end of file
+}
diff --git a/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature b/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
index d11cd33d5bf4076f0881a0966f8e25d573394644..61eb8ac8682dbfb6665bec81e9590b4b3f8164af 100644
--- a/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
+++ b/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
@@ -13,7 +13,39 @@ Feature: Indexing of the documents
| tenant1:indexer:test-update-data--Integration:1.0.1 | tenant1-indexer-test-update-data--integration-1.0.1 | index_update_records_kind_v1 |
| tenant1:indexer:test-update-data--Integration:2.0.1 | tenant1-indexer-test-update-data--integration-2.0.1 | index_update_records_kind_v2 |
| tenant1:indexer:virtual-properties-Integration:1.0.0 | tenant1-indexer-virtual-properties-integration-1.0.0 | index_record_virtual_properties |
- | tenant1:indexer:decimation-Integration:1.0.0 | tenant1-indexer-decimation-integration-1.0.0 | index_record_seismic_survey |
+ | tenant1:indexer:decimation-Integration:1.0.0 | tenant1-indexer-decimation-integration-1.0.0 | index_record_seismic_survey |
+ | osdu:wks:reference-data--IndexPropertyPathConfiguration:1.0.0 | osdu-wks-reference-data--indexpropertypathconfiguration-1.0.0 | osdu_wks_IndexPropertyPathConfiguration_v1 |
+ | test:indexer:index-property--Wellbore:1.0.0 | test-indexer-index-property--wellbore-1.0.0 | index-property-wellbore_v1 |
+ | test:indexer:index-property--WellLog:1.0.0 | test-indexer-index-property--welllog-1.0.0 | index-property-welllog_v1 |
+
+
+ Scenario Outline: Prepare the index property configuration records and clean up index of the extended kinds in the Elastic Search
+ When I ingest records with the <recordFile> with <acl> for a given <kind>
+ Then I should get the <number> documents for the <index> in the Elastic Search
+ Then I clean up the index of the extended kinds <extendedKinds> in the Elastic Search
+
+ Examples:
+ | kind | recordFile | number | index | acl | extendedKinds |
+ | "osdu:wks:reference-data--IndexPropertyPathConfiguration:1.0.0" | "osdu_wks_IndexPropertyPathConfiguration_v1" | 2 | "osdu-wks-reference-data--indexpropertypathconfiguration-1.0.0" | "data.default.viewers@tenant1" | "test:indexer:index-property--Wellbore:1.0.0,test:indexer:index-property--WellLog:1.0.0" |
+
+ Scenario Outline: Ingest the records of the extended kinds, Index in the Elastic Search and Search string field
+ When I ingest records with the <recordFile> with <acl> for a given <kind>
+ Then I should be able to search <number> record with index <index> by extended data field <field> and value <value>
+
+ Examples:
+ | kind | recordFile | number | index | acl | field | value |
+ | "test:indexer:index-property--Wellbore:1.0.0" | "index-property-wellbore_v1" | 1 | "test-indexer-index-property--wellbore-1.0.0" | "data.default.viewers@tenant1" | "data.WellUWI" | "123454321" |
+ | "test:indexer:index-property--WellLog:1.0.0" | "index-property-welllog_v1" | 1 | "test-indexer-index-property--welllog-1.0.0" | "data.default.viewers@tenant1" | "data.WellboreName" | "Facility_123" |
+
+ Scenario Outline: Ingest the records of the extended kinds, Index in the Elastic Search and Search spatial field
+ When I ingest records with the <recordFile> with <acl> for a given <kind>
+ Then I should be able search <number> documents for the <index> by bounding box query with points (<top_left_latitude>, <top_left_longitude>) and (<bottom_right_latitude>, <bottom_right_longitude>) on field <field>
+
+ Examples:
+ | kind | recordFile | number | index | acl | field | top_left_latitude | top_left_longitude | bottom_right_latitude | bottom_right_longitude |
+ | "test:indexer:index-property--Wellbore:1.0.0" | "index-property-wellbore_v1" | 1 | "test-indexer-index-property--wellbore-1.0.0" | "data.default.viewers@tenant1" | "data.Location" | 30 | -96 | 29 | -95 |
+ | "test:indexer:index-property--WellLog:1.0.0" | "index-property-welllog_v1" | 1 | "test-indexer-index-property--welllog-1.0.0" | "data.default.viewers@tenant1" | "data.SpatialLocation" | 30 | -96 | 29 | -95 |
+
Scenario Outline: Ingest the record and Index in the Elastic Search
When I ingest records with the <recordFile> with <acl> for a given <kind>
diff --git a/testing/indexer-test-core/src/main/resources/testData/index-property-wellbore_v1.json b/testing/indexer-test-core/src/main/resources/testData/index-property-wellbore_v1.json
new file mode 100644
index 0000000000000000000000000000000000000000..469a0d22c7637a8fd867bbf2ed9aad733b87b839
--- /dev/null
+++ b/testing/indexer-test-core/src/main/resources/testData/index-property-wellbore_v1.json
@@ -0,0 +1,16 @@
+[{
+ "id": "tenant1:index-property--Wellbore:testIngest1",
+ "data": {
+ "Location": {
+ "latitude": 29.170640,
+ "longitude": -95.002875
+ },
+ "FacilityName": "Facility_123",
+ "NameAliases": [{
+ "AliasName": "123454321",
+ "AliasNameTypeID": "tenant1:reference-data--AliasNameType:UniqueIdentifier:"
+ }
+ ]
+ }
+ }
+]
diff --git a/testing/indexer-test-core/src/main/resources/testData/index-property-wellbore_v1.schema.json b/testing/indexer-test-core/src/main/resources/testData/index-property-wellbore_v1.schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..93d616f44e393c27e176724c2a97c80028f613bb
--- /dev/null
+++ b/testing/indexer-test-core/src/main/resources/testData/index-property-wellbore_v1.schema.json
@@ -0,0 +1,118 @@
+{
+ "schemaInfo": {
+ "schemaIdentity": {
+ "authority": "test",
+ "source": "indexer",
+ "entityType": "index-property--Wellbore",
+ "schemaVersionMajor": 1,
+ "schemaVersionMinor": 0,
+ "schemaVersionPatch": 0,
+ "id": "test:indexer:index-property--Wellbore:1.0.0"
+ },
+ "status": "DEVELOPMENT"
+ },
+ "schema": {
+ "definitions": {
+ "opendes:wks:core_dl_geopoint:1.0.0": {
+ "description": "A 2D point location in latitude and longitude referenced to WGS 84 if not specified otherwise.",
+ "properties": {
+ "latitude": {
+ "description": "The latitude value in degrees of arc (dega). Value range [-90, 90].",
+ "maximum": 90,
+ "minimum": -90,
+ "title": "Latitude",
+ "type": "number"
+ },
+ "longitude": {
+ "description": "The longitude value in degrees of arc (dega). Value range [-180, 180]",
+ "maximum": 180,
+ "minimum": -180,
+ "title": "Longitude",
+ "type": "number"
+ }
+ },
+ "required": [
+ "latitude",
+ "longitude"
+ ],
+ "title": "2D Map Location",
+ "type": "object"
+ },
+ "opendes:wks:AbstractAliasNames:1.0.0": {
+ "title": "AbstractAliasNames",
+ "type": "object",
+ "properties": {
+ "AliasNameTypeID": {
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-AliasNameType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "A classification of alias names such as by role played or type of source, such as regulatory name, regulatory code, company code, international standard name, etc.",
+ "x-osdu-relationship": [{
+ "EntityType": "AliasNameType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "type": "string"
+ },
+ "EffectiveDateTime": {
+ "format": "date-time",
+ "type": "string",
+ "description": "The date and time when an alias name becomes effective."
+ },
+ "AliasName": {
+ "type": "string",
+ "description": "Alternative Name value of defined name type for an object."
+ },
+ "TerminationDateTime": {
+ "format": "date-time",
+ "type": "string",
+ "description": "The data and time when an alias name is no longer in effect."
+ },
+ "DefinitionOrganisationID": {
+ "pattern": "^[\\w\\-\\.]+:(reference-data\\-\\-StandardsOrganisation|master-data\\-\\-Organisation):[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The StandardsOrganisation (reference-data) or Organisation (master-data) that provided the name (the source).",
+ "x-osdu-relationship": [{
+ "EntityType": "StandardsOrganisation",
+ "GroupType": "reference-data"
+ }, {
+ "EntityType": "Organisation",
+ "GroupType": "master-data"
+ }
+ ],
+ "type": "string"
+ }
+ }
+ }
+ },
+ "properties": {
+ "data": {
+ "allOf": [
+ {
+ "type": "object",
+ "properties": {
+ "Location": {
+ "$ref": "#/definitions/opendes:wks:core_dl_geopoint:1.0.0",
+ "description": "The wellbore's position .",
+ "format": "core:dl:geopoint:1.0.0",
+ "title": "WGS 84 Position",
+ "type": "object"
+ },
+ "NameAliases": {
+ "x-osdu-indexing": {
+ "type": "nested"
+ },
+ "description": "Alternative names, including historical, by which this master data is/has been known (it should include all the identifiers).",
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/opendes:wks:AbstractAliasNames:1.0.0"
+ }
+ },
+ "FacilityName": {
+ "description": "Name of the Facility.",
+ "type": "string"
+ }
+ }
+ }
+ ]
+ }
+ }
+ }
+}
diff --git a/testing/indexer-test-core/src/main/resources/testData/index-property-welllog_v1.json b/testing/indexer-test-core/src/main/resources/testData/index-property-welllog_v1.json
new file mode 100644
index 0000000000000000000000000000000000000000..b22a36722b78de757f1631809f9f4ea51fa02f25
--- /dev/null
+++ b/testing/indexer-test-core/src/main/resources/testData/index-property-welllog_v1.json
@@ -0,0 +1,8 @@
+[{
+ "id": "tenant1:index-property--WellLog:testIngest2",
+ "data": {
+ "Name": "Log_ABC",
+ "WellboreID": "tenant1:index-property--Wellbore:testIngest1"
+ }
+ }
+]
diff --git a/testing/indexer-test-core/src/main/resources/testData/index-property-welllog_v1.schema.json b/testing/indexer-test-core/src/main/resources/testData/index-property-welllog_v1.schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..1b9549b616e5cc889ed200f08decc0b08d4a481b
--- /dev/null
+++ b/testing/indexer-test-core/src/main/resources/testData/index-property-welllog_v1.schema.json
@@ -0,0 +1,36 @@
+{
+ "schemaInfo": {
+ "schemaIdentity": {
+ "authority": "test",
+ "source": "indexer",
+ "entityType": "index-property--WellLog",
+ "schemaVersionMajor": 1,
+ "schemaVersionMinor": 0,
+ "schemaVersionPatch": 0,
+ "id": "test:indexer:index-property--WellLog:1.0.0"
+ },
+ "status": "DEVELOPMENT"
+ },
+ "schema": {
+ "properties": {
+ "data": {
+ "allOf": [
+ {
+ "type": "object",
+ "properties": {
+ "WellboreID": {
+ "description": "Id of parent wellbore.",
+ "type": "string"
+ },
+ "Name": {
+ "description": "Name of welllog.",
+ "type": "string"
+ }
+ }
+ }
+ ]
+ }
+ }
+ }
+
+}
diff --git a/testing/indexer-test-core/src/main/resources/testData/osdu_wks_IndexPropertyPathConfiguration_v1.json b/testing/indexer-test-core/src/main/resources/testData/osdu_wks_IndexPropertyPathConfiguration_v1.json
new file mode 100644
index 0000000000000000000000000000000000000000..f8ba28eb63ab5e68676610c029c12defbd962e2e
--- /dev/null
+++ b/testing/indexer-test-core/src/main/resources/testData/osdu_wks_IndexPropertyPathConfiguration_v1.json
@@ -0,0 +1,70 @@
+[{
+ "id": "tenant1:reference-data--IndexPropertyPathConfiguration:index-property--WellLog:1.",
+ "data": {
+ "Name": "WellLogIndex-PropertyPathConfiguration",
+ "Description": "The index property list for WellLog:1., valid for all WellLog kinds for major version 1.",
+ "Code": "test:indexer:index-property--WellLog:1.",
+ "AttributionAuthority": "OSDU",
+ "Configurations": [{
+ "Name": "WellboreName",
+ "Policy": "ExtractFirstMatch",
+ "Paths": [{
+ "RelatedObjectsSpec": {
+ "RelationshipDirection": "ChildToParent",
+ "RelatedObjectKind": "test:indexer:index-property--Wellbore:1.",
+ "RelatedObjectID": "data.WellboreID"
+ },
+ "ValueExtraction": {
+ "ValuePath": "data.FacilityName"
+ }
+ }
+ ],
+ "UseCase": "As a user I want to discover WellLog instances by the wellbore's name value."
+ }, {
+ "Name": "SpatialLocation",
+ "Policy": "ExtractFirstMatch",
+ "Paths": [{
+ "RelatedObjectsSpec": {
+ "RelationshipDirection": "ChildToParent",
+ "RelatedObjectKind": "test:indexer:index-property--Wellbore:1.",
+ "RelatedObjectID": "data.WellboreID"
+ },
+ "ValueExtraction": {
+ "ValuePath": "data.Location"
+ }
+ }
+ ],
+ "UseCase": "As a user I want to discover WellLog instances by spatial location."
+ }
+ ]
+ }
+ }, {
+ "id": "tenant1:reference-data--IndexPropertyPathConfiguration:index-property--Wellbore:1.",
+ "data": {
+ "Name": "Wellbore-IndexPropertyPathConfiguration",
+ "Description": "The index property list for index-property--Wellbore:1., valid for all index-property--Wellbore kinds for major version 1.",
+ "Code": "test:indexer:index-property--Wellbore:1.",
+ "AttributionAuthority": "OSDU",
+ "Configurations": [{
+ "Name": "WellUWI",
+ "Policy": "ExtractFirstMatch",
+ "Paths": [{
+ "ValueExtraction": {
+ "RelatedConditionMatches": [
+ "tenant1:reference-data--AliasNameType:UniqueIdentifier:",
+ "tenant1:reference-data--AliasNameType:RegulatoryName:",
+ "tenant1:reference-data--AliasNameType:PreferredName:",
+ "tenant1:reference-data--AliasNameType:CommonName:",
+ "tenant1:reference-data--AliasNameType:ShortName:"
+ ],
+ "RelatedConditionProperty": "data.NameAliases[].AliasNameTypeID",
+ "ValuePath": "data.NameAliases[].AliasName"
+ }
+ }
+ ],
+ "UseCase": "As a user I want to discover and match Wells by their UWI. I am aware that this is not globally reliable, however, I am able to specify a prioritized AliasNameType list to look up value in the NameAliases array."
+ }
+ ]
+ }
+ }
+]
diff --git a/testing/indexer-test-core/src/main/resources/testData/osdu_wks_IndexPropertyPathConfiguration_v1.schema.json b/testing/indexer-test-core/src/main/resources/testData/osdu_wks_IndexPropertyPathConfiguration_v1.schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..7cf15888121f227c11303d934c9af6340f384798
--- /dev/null
+++ b/testing/indexer-test-core/src/main/resources/testData/osdu_wks_IndexPropertyPathConfiguration_v1.schema.json
@@ -0,0 +1,284 @@
+{
+ "schemaInfo": {
+ "schemaIdentity": {
+ "authority": "osdu",
+ "source": "wks",
+ "entityType": "reference-data--IndexPropertyPathConfiguration",
+ "schemaVersionMajor": 1,
+ "schemaVersionMinor": 0,
+ "schemaVersionPatch": 0,
+ "id": "osdu:wks:reference-data--IndexPropertyPathConfiguration:1.0.0"
+ },
+ "status": "DEVELOPMENT"
+ },
+ "schema": {
+ "x-osdu-license": "Copyright 2023, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$id": "https://schema.osdu.opengroup.org/json/reference-data/IndexPropertyPathConfiguration.1.0.0.json",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:reference-data--IndexPropertyPathConfiguration:1.0.0",
+ "title": "IndexPropertyPathConfiguration",
+ "description": "IndexPropertyPathConfiguration contains the de-normalization configuration settings for the Search index. The record id contains the kind so that there is a unique relationship between the kind and the kind's index extensions.",
+ "type": "object",
+ "properties": {
+ "id": {
+ "description": "Previously called ResourceID or SRN which identifies this OSDU resource object without version.",
+ "title": "Entity ID",
+ "type": "string",
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-IndexPropertyPathConfiguration:[\\w\\-\\.\\:\\%]+$",
+ "example": "namespace:reference-data--IndexPropertyPathConfiguration:c9d84708-2b1b-5e0b-954e-9621132f7154"
+ },
+ "kind": {
+ "description": "The schema identification for the OSDU resource object following the pattern {Namespace}:{Source}:{Type}:{VersionMajor}.{VersionMinor}.{VersionPatch}. The versioning scheme follows the semantic versioning, https://semver.org/.",
+ "title": "Entity Kind",
+ "type": "string",
+ "pattern": "^[\\w\\-\\.]+:[\\w\\-\\.]+:[\\w\\-\\.]+:[0-9]+.[0-9]+.[0-9]+$",
+ "example": "osdu:wks:reference-data--IndexPropertyPathConfiguration:1.0.0"
+ },
+ "version": {
+ "description": "The version number of this OSDU resource; set by the framework.",
+ "title": "Version Number",
+ "type": "integer",
+ "format": "int64",
+ "example": 1562066009929332
+ },
+ "acl": {
+ "description": "The access control tags associated with this entity.",
+ "title": "Access Control List",
+ "$ref": "osdu:wks:AbstractAccessControlList:1.0.0"
+ },
+ "legal": {
+ "description": "The entity's legal tags and compliance status. The actual contents associated with the legal tags is managed by the Compliance Service.",
+ "title": "Legal Tags",
+ "$ref": "osdu:wks:AbstractLegalTags:1.0.0"
+ },
+ "tags": {
+ "title": "Tag Dictionary",
+ "description": "A generic dictionary of string keys mapping to string value. Only strings are permitted as keys and values.",
+ "type": "object",
+ "additionalProperties": {
+ "type": "string"
+ },
+ "example": {
+ "NameOfKey": "String value"
+ }
+ },
+ "createTime": {
+ "description": "Timestamp of the time at which initial version of this OSDU resource object was created. Set by the System. The value is a combined date-time string in ISO-8601 given in UTC.",
+ "title": "Resource Object Creation DateTime",
+ "type": "string",
+ "format": "date-time",
+ "example": "2020-12-16T11:46:20.163Z"
+ },
+ "createUser": {
+ "title": "Resource Object Creation User Reference",
+ "description": "The user reference, which created the first version of this resource object. Set by the System.",
+ "type": "string",
+ "example": "some-user@some-company-cloud.com"
+ },
+ "modifyTime": {
+ "description": "Timestamp of the time at which this version of the OSDU resource object was created. Set by the System. The value is a combined date-time string in ISO-8601 given in UTC.",
+ "title": "Resource Object Version Creation DateTime",
+ "type": "string",
+ "format": "date-time",
+ "example": "2020-12-16T11:52:24.477Z"
+ },
+ "modifyUser": {
+ "title": "Resource Object Version Creation User Reference",
+ "description": "The user reference, which created this version of this resource object. Set by the System.",
+ "type": "string",
+ "example": "some-user@some-company-cloud.com"
+ },
+ "ancestry": {
+ "description": "The links to data, which constitute the inputs, from which this record instance is derived.",
+ "title": "Ancestry",
+ "$ref": "osdu:wks:AbstractLegalParentList:1.0.0"
+ },
+ "meta": {
+ "description": "The Frame of Reference meta data section linking the named properties to self-contained definitions.",
+ "title": "Frame of Reference Meta Data",
+ "type": "array",
+ "items": {
+ "$ref": "osdu:wks:AbstractMetaItem:1.0.0"
+ }
+ },
+ "data": {
+ "allOf": [
+ {
+ "$ref": "osdu:wks:AbstractCommonResources:1.0.0"
+ },
+ {
+ "$ref": "osdu:wks:AbstractReferenceType:1.0.0"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "Configurations": {
+ "type": "array",
+ "title": "Configurations",
+ "description": "The list of index property configurations for the specific kind.",
+ "x-osdu-indexing": {
+ "type": "nested"
+ },
+ "items": {
+ "type": "object",
+ "title": "Configuration",
+ "description": "One single configuration to derive an Search index property value and assign it to the index 'column' with Name.",
+ "properties": {
+ "Name": {
+ "type": "string",
+ "title": "Name",
+ "description": "The name of the indexed property, i.e., this is the property name used in Search.",
+ "example": "CountryNames"
+ },
+ "UseCase": {
+ "type": "string",
+ "title": "Use Case",
+ "description": "The use case description this configuration satisfies.",
+ "example": "As a user I want to find objects by a country name, with the understanding that an object may extend over country boundaries."
+ },
+ "Policy": {
+ "type": "string",
+ "title": "Extraction Policy",
+ "description": "Current supported policies are 'ExtractAllMatches' resulting in an array of values or 'ExtractFirstMatch' single value. The policy applies only to the Paths[].ValueExtraction.",
+ "example": "ExtractAllMatches",
+ "pattern": "^(ExtractFirstMatch|ExtractAllMatches)$"
+ },
+ "Paths": {
+ "type": "array",
+ "title": "Paths",
+ "description": "The list of path definitions to derive the property value from.",
+ "x-osdu-indexing": {
+ "type": "nested"
+ },
+ "items": {
+ "type": "object",
+ "title": "Path",
+ "description": "A single path definition to derive a property value from.",
+ "properties": {
+ "RelatedObjectsSpec": {
+ "type": "object",
+ "title": "Related Objects Specification",
+ "description": "The specification to extract related objects, from which to derive the ValueExtraction. If this property is empty or absent, the ValueExtraction is done on the current object to be indexed.",
+ "properties": {
+ "RelatedObjectID": {
+ "type": "string",
+ "title": "Related Object ID",
+ "description": "The path to the property containing the ID of the target record to chase. This property is only populated if the property is extracted from a related object, which must be chased. If the property is derived from 'within' the same record, which triggered the indexing, the RelatedObjectID is left absent.",
+ "example": "GeoContexts[].GeoPoliticalEntityID"
+ },
+ "RelatedObjectKind": {
+ "type": "string",
+ "title": "Related Object Kind",
+ "description": "The kind or schema id expected as the target object type. This property is only populated if the property is extracted from a related object, which must be chased. If the property is derived from 'within' the same record, which triggered the indexing, the RelatedObjectKind is left absent.",
+ "example": "osdu:wks:master-data--GeoPoliticalEntity:1.",
+ "pattern": "^[\\w\\-\\.]+:[\\w\\-\\.]+:[\\w\\-\\.]+:[0-9]+.$"
+ },
+ "RelationshipDirection": {
+ "type": "string",
+ "title": "Relationship Direction",
+ "description": "The direction of the relationship definition seen from the object being indexed. 'ChildToParent' assumes an outgoing relationship with the target record defined in the object being indexed. 'ParentToChildren' assumes that the related objects have a relationship by RelatedObjectID to the id of the record being indexed.",
+ "example": "ChildToParent",
+ "pattern": "^(ChildToParent|ParentToChildren)$"
+ },
+ "RelatedConditionProperty": {
+ "type": "string",
+ "title": "Related Condition Property",
+ "description": "The property path of the target record data block, which needs subjected to the conditional matching. The data prefix is not required.",
+ "example": "GeoContexts[].GeoTypeID"
+ },
+ "RelatedConditionMatches": {
+ "type": "array",
+ "title": "Related Condition Matches",
+ "description": "The RelatedConditionProperty values, which need to match in order to be accepted as de-normalized value(s). If the Policy is ExtractFirstMatch, the list is prioritized and the first match is accepted as final value. Policy ExtractAllMatches collects all matching values as array.",
+ "example": [
+ "namespace:reference-data--GeoPoliticalEntityType:Country:"
+ ],
+ "items": {
+ "type": "string"
+ }
+ }
+ }
+ },
+ "ValueExtraction": {
+ "type": "object",
+ "title": "Value Extraction",
+ "description": "The instructions from where to derive the value.",
+ "properties": {
+ "RelatedConditionProperty": {
+ "type": "string",
+ "title": "Related Condition Property",
+ "description": "The property path of the target record data block, which needs to be subjected to the conditional matching. The data prefix is not required in the path."
+ },
+ "RelatedConditionMatches": {
+ "type": "array",
+ "title": "Related Condition Matches",
+ "description": "The RelatedConditionProperty values, which need to match in order to be accepted as de-normalized value(s). If the Policy is ExtractFirstMatch, the list is prioritized and the first match is accepted as final value. Policy ExtractAllMatches collects all matching values as array.",
+ "items": {
+ "type": "string"
+ }
+ },
+ "ValuePath": {
+ "type": "string",
+ "title": "Value Path",
+ "description": "The path to the property from where to extract the de-normalized value. The data prefix is not required in the path.",
+ "example": "GeoPoliticalEntityName"
+ }
+ },
+ "required": [
+ "ValuePath"
+ ]
+ }
+ },
+ "required": [
+ "ValueExtraction"
+ ]
+ }
+ }
+ }
+ }
+ }
+ },
+ "title": "IndividualProperties"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "ExtensionProperties": {
+ "type": "object"
+ }
+ },
+ "title": "ExtensionProperties"
+ }
+ ]
+ }
+ },
+ "required": [
+ "kind",
+ "acl",
+ "legal"
+ ],
+ "additionalProperties": false,
+ "x-osdu-review-status": "Accepted",
+ "x-osdu-governance-model": "OPEN",
+ "x-osdu-governance-authorities": [
+ "Energistics",
+ "OSDU"
+ ],
+ "x-osdu-virtual-properties": {
+ "data.VirtualProperties.DefaultName": {
+ "type": "string",
+ "priority": [
+ {
+ "path": "data.Name"
+ }
+ ]
+ }
+ },
+ "x-osdu-inheriting-from-kind": [
+ {
+ "name": "ReferenceType",
+ "kind": "osdu:wks:AbstractReferenceType:1.0.0"
+ }
+ ]
+ }
+}
diff --git a/testing/indexer-test-gc/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-gc/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
index a01ca063d369d546a19d52e69470784459f5b70c..fcbbe30a1d5b50a67ed45faa6331ba3c444ba015 100644
--- a/testing/indexer-test-gc/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
+++ b/testing/indexer-test-gc/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -70,6 +70,16 @@ public class Steps extends SchemaServiceRecordSteps {
super.iShouldBeAbleToSearchRecordByTagKeyAndTagValue(index, tagKey, tagValue, expectedNumber);
}
+ @Then("^I clean up the index of the extended kinds \"([^\"]*)\" in the Elastic Search$")
+ public void iShouldCleanupIndicesOfExtendedKinds(String extendedKinds) throws Throwable {
+ super.iShouldCleanupIndicesOfExtendedKinds(extendedKinds);
+ }
+
+ @Then("^I should be able to search (\\d+) record with index \"([^\"]*)\" by extended data field \"([^\"]*)\" and value \"([^\"]*)\"$")
+ public void iShouldBeAbleToSearchRecordByFieldAndFieldValue(int expectedNumber, String index, String fieldKey, String fieldValue) throws Throwable {
+ super.iShouldBeAbleToSearchRecordByFieldAndFieldValue(index, fieldKey, fieldValue, expectedNumber);
+ }
+
@Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by bounding box query with points \\((-?\\d+), (-?\\d+)\\) and \\((-?\\d+), (-?\\d+)\\) on field \"([^\"]*)\"$")
public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery(
int expectedCount, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field)
@@ -102,4 +112,4 @@ public class Steps extends SchemaServiceRecordSteps {
String actualName = generateActualName(index, null);
super.i_should_get_object_in_search_response_without_hints_in_schema(objectInnerField ,actualName, recordFile, acl, kind);
}
-}
\ No newline at end of file
+}
diff --git a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
index 624f994deeeded56f15e7acd660e8873e7ed0518..4e2adfde8928f6eb8bdc11ba3b2dc3279c5eff86 100644
--- a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
+++ b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -88,6 +88,16 @@ public class Steps extends SchemaServiceRecordSteps {
super.iShouldBeAbleToSearchRecordByTagKeyAndTagValue(index, tagKey, tagValue, expectedNumber);
}
+ @Then("^I clean up the index of the extended kinds \"([^\"]*)\" in the Elastic Search$")
+ public void iShouldCleanupIndicesOfExtendedKinds(String extendedKinds) throws Throwable {
+ super.iShouldCleanupIndicesOfExtendedKinds(extendedKinds);
+ }
+
+ @Then("^I should be able to search (\\d+) record with index \"([^\"]*)\" by extended data field \"([^\"]*)\" and value \"([^\"]*)\"$")
+ public void iShouldBeAbleToSearchRecordByFieldAndFieldValue(int expectedNumber, String index, String fieldKey, String fieldValue) throws Throwable {
+ super.iShouldBeAbleToSearchRecordByFieldAndFieldValue(index, fieldKey, fieldValue, expectedNumber);
+ }
+
@Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by bounding box query with points \\((-?\\d+), (-?\\d+)\\) and \\((-?\\d+), (-?\\d+)\\) on field \"(.*?)\"$")
public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery (
int expectedCount, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field) throws Throwable {
@@ -156,4 +166,4 @@ public class Steps extends SchemaServiceRecordSteps {
}
}
-}
\ No newline at end of file
+}