Skip to content
Snippets Groups Projects
Commit e01ba240 authored by Neelesh Thakur's avatar Neelesh Thakur
Browse files

enable sort and aggregation on text array

parent 3327bc6b
Branches
Tags
2 merge requests!346Merge branch 'aws-integration' into 'master',!315enable sort and aggregation on primitive text array
Pipeline #108442 failed
Stage: review
Stage: build
Stage: coverage
Stage: containerize
Stage: scan
Stage: deploy
Stage: integration
Hide whitespace changes
Inline Side-by-side
indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java
+ 5
− 1
View file @ e01ba240
......@@ -117,7 +117,11 @@ public class TypeMapper {
}
if (isArray(indexerType.toString())) {
return Records.Type.builder().type(getArrayMemberType(indexerType.toString())).build();
String memberType = getArrayMemberType(indexerType.toString());
if (ElasticType.TEXT.getValue().equalsIgnoreCase(memberType)) {
return getTextIndexerMapping();
}
return Records.Type.builder().type(memberType).build();
}
if (isMap(indexerType)) {
......
testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
+ 15
− 15
View file @ e01ba240
......@@ -3,15 +3,15 @@ Feature: Indexing of the documents
Background:
Given the schema is created with the following kind
| kind | index | schemaFile |
| tenant1:indexer:test-data--Integration:1.0.1 | tenant1-indexer-test-data--integration-1.0.1 | index_records_1 |
| tenant1:indexer:test-data--Integration:2.0.1 | tenant1-indexer-test-data--integration-2.0.1 | index_records_2 |
| tenant1:indexer:test-data--Integration:3.0.1 | tenant1-indexer-test-data--integration-3.0.1 | index_records_3 |
| tenant1:wks:master-data--Wellbore:2.0.3 | tenant1-wks-master-data--wellbore-2.0.3 | r3-index_record_wks_master |
| tenant1:wks:ArraysOfObjectsTestCollection:4.0.0 | tenant1-wks-arraysofobjectstestcollection-4.0.0 | r3-index_record_arrayofobjects |
| tenant1:indexer:test-mapping--Sync:2.0.0 | tenant1-indexer-test-mapping--sync-2.0.0 | index_record_sync_mapping |
| tenant1:indexer:test-update-data--Integration:1.0.1 | tenant1-indexer-test-update-data--integration-1.0.1 | index_update_records_kind_v1 |
| tenant1:indexer:test-update-data--Integration:2.0.1 | tenant1-indexer-test-update-data--integration-2.0.1 | index_update_records_kind_v2 |
| kind | index | schemaFile |
| tenant1:indexer:test-data--Integration:1.0.2 | tenant1-indexer-test-data--integration-1.0.2 | index_records_1 |
| tenant1:indexer:test-data--Integration:2.0.1 | tenant1-indexer-test-data--integration-2.0.1 | index_records_2 |
| tenant1:indexer:test-data--Integration:3.0.1 | tenant1-indexer-test-data--integration-3.0.1 | index_records_3 |
| tenant1:wks:master-data--Wellbore:2.0.3 | tenant1-wks-master-data--wellbore-2.0.3 | r3-index_record_wks_master |
| tenant1:wks:ArraysOfObjectsTestCollection:4.0.0 | tenant1-wks-arraysofobjectstestcollection-4.0.0 | r3-index_record_arrayofobjects |
| tenant1:indexer:test-mapping--Sync:2.0.0 | tenant1-indexer-test-mapping--sync-2.0.0 | index_record_sync_mapping |
| tenant1:indexer:test-update-data--Integration:1.0.1 | tenant1-indexer-test-update-data--integration-1.0.1 | index_update_records_kind_v1 |
| tenant1:indexer:test-update-data--Integration:2.0.1 | tenant1-indexer-test-update-data--integration-2.0.1 | index_update_records_kind_v2 |
Scenario Outline: Ingest the record and Index in the Elastic Search
When I ingest records with the <recordFile> with <acl> for a given <kind>
......@@ -20,9 +20,9 @@ Feature: Indexing of the documents
Then I can validate indexed meta attributes for the <index> and given <kind>
Examples:
| kind | recordFile | number | index | acl | mapping |
| "tenant1:indexer:test-data--Integration:1.0.1" | "index_records_schema_1" | 5 | "tenant1-indexer-test-data--integration-1.0.1" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Country":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"County":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"EmptyAttribute":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Established":{"type":"date"},"Field":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellName":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellStatus":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellType":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"authority":{"type":"constant_keyword","value":"<authority-id>"},"createTime":{"type":"date"},"createUser":{"type":"keyword"},"modifyTime":{"type":"date"},"modifyUser":{"type":"keyword"},"source":{"type":"constant_keyword","value":"<source-id>"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
| "tenant1:indexer:test-data--Integration:3.0.1" | "index_records_schema_3" | 7 | "tenant1-indexer-test-data--integration-3.0.1" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Country":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"County":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"EmptyAttribute":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Established":{"type":"date"},"Field":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellName":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellStatus":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellType":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"authority":{"type":"constant_keyword","value":"<authority-id>"},"createTime":{"type":"date"},"createUser":{"type":"keyword"},"modifyTime":{"type":"date"},"modifyUser":{"type":"keyword"},"source":{"type":"constant_keyword","value":"<source-id>"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
| kind | recordFile | number | index | acl | mapping |
| "tenant1:indexer:test-data--Integration:1.0.2" | "index_records_schema_1" | 5 | "tenant1-indexer-test-data--integration-1.0.2" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Country":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"County":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"EmptyAttribute":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Established":{"type":"date"},"Field":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellName":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellStatus":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellType":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"DblArray":{"type":"double"},"TextArray":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"authority":{"type":"constant_keyword","value":"<authority-id>"},"createTime":{"type":"date"},"createUser":{"type":"keyword"},"modifyTime":{"type":"date"},"modifyUser":{"type":"keyword"},"source":{"type":"constant_keyword","value":"<source-id>"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
| "tenant1:indexer:test-data--Integration:3.0.1" | "index_records_schema_3" | 7 | "tenant1-indexer-test-data--integration-3.0.1" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Country":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"County":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"EmptyAttribute":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Established":{"type":"date"},"Field":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellName":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellStatus":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellType":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"authority":{"type":"constant_keyword","value":"<authority-id>"},"createTime":{"type":"date"},"createUser":{"type":"keyword"},"modifyTime":{"type":"date"},"modifyUser":{"type":"keyword"},"source":{"type":"constant_keyword","value":"<source-id>"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
Scenario Outline: Ingest the record and Index in the Elastic Search with bad attribute
When I ingest records with the <recordFile> with <acl> for a given <kind>
......@@ -39,7 +39,7 @@ Feature: Indexing of the documents
Examples:
| kind | recordFile | index | acl | tagKey | tagValue | number |
| "tenant1:indexer:test-data--Integration:1.0.1" | "index_records_schema_1" | "tenant1-indexer-test-data--integration-1.0.1" | "data.default.viewers@tenant1" | "testtag" | "testvalue" | 5 |
| "tenant1:indexer:test-data--Integration:1.0.2" | "index_records_schema_1" | "tenant1-indexer-test-data--integration-1.0.2" | "data.default.viewers@tenant1" | "testtag" | "testvalue" | 5 |
Scenario Outline: Ingest the r3-record with geo-shape and Index in the Elastic Search
When I ingest records with the <recordFile> with <acl> for a given <kind>
......@@ -76,5 +76,5 @@ Feature: Indexing of the documents
Then I should not get any documents for the <index_v1> in the Elastic Search
Examples:
| kind_v1 | index_v1 | recordFile | acl | kind_v2 | index_v2 |
| "tenant1:indexer:test-update-data--Integration:1.0.1" | "tenant1-indexer-test-update-data--integration-1.0.1" | "index_update_records_kind_v1" | "data.default.viewers@tenant1" | "tenant1:indexer:test-update-data--Integration:2.0.1" | "tenant1-indexer-test-update-data--integration-2.0.1" |
\ No newline at end of file
| kind_v1 | index_v1 | recordFile | acl | kind_v2 | index_v2 |
| "tenant1:indexer:test-update-data--Integration:1.0.1" | "tenant1-indexer-test-update-data--integration-1.0.1" | "index_update_records_kind_v1" | "data.default.viewers@tenant1" | "tenant1:indexer:test-update-data--Integration:2.0.1" | "tenant1-indexer-test-update-data--integration-2.0.1" |
\ No newline at end of file
testing/indexer-test-core/src/main/resources/testData/index_records_1.json
+ 40
− 18
View file @ e01ba240
......@@ -7,8 +7,8 @@
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
"lat":32.406402588,
"lon":-86.565592762
"lat": 32.406402588,
"lon": -86.565592762
},
"Basin": "Houston",
"County": "Harris",
......@@ -20,9 +20,18 @@
"WellType": "Data Lake Cloud",
"EmptyAttribute": "",
"Rank": 1,
"Score" : 10,
"Score": 10,
"Established": "2000-03-27T23:38:48Z",
"DblArray": [32.40, 36.45, 40.0]
"DblArray": [
32.40,
36.45,
40.0
],
"TextArray": [
"Elevation (m)",
"SpatialLocation",
"Well Type"
]
}
},
{
......@@ -33,8 +42,8 @@
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
"lat":32.406402588,
"lon":-86.565592762
"lat": 32.406402588,
"lon": -86.565592762
},
"Basin": "Houston",
"County": "Harris",
......@@ -46,9 +55,17 @@
"WellType": "Data Lake Cloud",
"EmptyAttribute": "",
"Rank": 1,
"Score" : 10,
"Score": 10,
"Established": "2000-03-27T23:38:48Z",
"DblArray": [62.40, 60.0]
"DblArray": [
62.40,
60.0
],
"TextArray": [
"WCR",
"License",
"Completion Date"
]
}
},
{
......@@ -59,8 +76,8 @@
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
"lat":32.406402588,
"lon":-86.565592762
"lat": 32.406402588,
"lon": -86.565592762
},
"Basin": "Houston",
"County": "Harris",
......@@ -72,8 +89,13 @@
"WellType": "Data Lake Cloud",
"EmptyAttribute": "",
"Rank": 1,
"Score" : 10,
"Established": "2000-03-27T23:38:48Z"
"Score": 10,
"Established": "2000-03-27T23:38:48Z",
"TextArray": [
"Wgs84Coordinates",
"Petroleum Field"
]
}
},
{
......@@ -84,8 +106,8 @@
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
"lat":32.406402588,
"lon":-86.565592762
"lat": 32.406402588,
"lon": -86.565592762
},
"Basin": "Houston",
"County": "Harris",
......@@ -97,7 +119,7 @@
"WellType": "Data Lake Cloud",
"EmptyAttribute": "",
"Rank": 1,
"Score" : 10,
"Score": 10,
"Established": "2000-03-27T23:38:48Z"
}
},
......@@ -109,8 +131,8 @@
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
"lat":32.406402588,
"lon":-86.565592762
"lat": 32.406402588,
"lon": -86.565592762
},
"Basin": "Houston",
"County": "Harris",
......@@ -122,7 +144,7 @@
"WellType": "Data Lake Cloud",
"EmptyAttribute": "",
"Rank": 1,
"Score" : 10,
"Score": 10,
"Established": "2000-03-27T23:38:48Z"
}
}
......
testing/indexer-test-core/src/main/resources/testData/index_records_1.schema.json
+ 7
− 1
View file @ e01ba240
......@@ -6,7 +6,7 @@
"entityType": "test-data--Integration",
"schemaVersionMajor": "1",
"schemaVersionMinor": "0",
"schemaVersionPatch": "1"
"schemaVersionPatch": "2"
},
"status": "DEVELOPMENT"
},
......@@ -84,6 +84,12 @@
"items": {
"type": "number"
}
},
"TextArray": {
"type": "array",
"items": {
"type": "string"
}
}
}
}
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment