Skip to content
Snippets Groups Projects
Commit d4567b00 authored by Matt Wise's avatar Matt Wise Committed by Yauheni Lesnikau
Browse files

Fix Int tests failing due to new storage record ID validation rules

parent 08bdcaeb
No related branches found
No related tags found
1 merge request!92Add support for indexing tags meta-attribute
Pipeline #27221 passed
Stage: build
Stage: scan
Hide whitespace changes
Inline Side-by-side
Showing
with 52 additions and 33 deletions
testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/CleanupIndiciesSteps.java
+ 1
− 1
View file @ d4567b00
......@@ -87,8 +87,8 @@ public class CleanupIndiciesSteps extends TestsBase {
records = new Gson().fromJson(fileContent, new TypeToken<List<Map<String, Object>>>() {}.getType());
for (Map<String, Object> testRecord : records) {
testRecord.put("id", generateActualName(testRecord.get("id").toString(), timeStamp));
testRecord.put("kind", actualKind);
testRecord.put("id", generateActualId(testRecord.get("id").toString(), timeStamp, actualKind));
testRecord.put("legal", generateLegalTag());
String[] x_acl = {generateActualName(dataGroup,timeStamp)+"."+getEntitlementsDomain()};
Acl acl = Acl.builder().viewers(x_acl).owners(x_acl).build();
......
testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
+ 2
− 2
View file @ d4567b00
......@@ -96,8 +96,8 @@ public class RecordSteps extends TestsBase {
records = new Gson().fromJson(fileContent, new TypeToken<List<Map<String, Object>>>() {}.getType());
for (Map<String, Object> testRecord : records) {
testRecord.put("id", generateRecordId(testRecord));
testRecord.put("kind", actualKind);
testRecord.put("id", generateRecordId(testRecord));
testRecord.put("legal", generateLegalTag());
String[] x_acl = {generateActualName(dataGroup,timeStamp)+"."+getEntitlementsDomain()};
Acl acl = Acl.builder().viewers(x_acl).owners(x_acl).build();
......@@ -113,7 +113,7 @@ public class RecordSteps extends TestsBase {
}
protected String generateRecordId(Map<String, Object> testRecord) {
return generateActualName(testRecord.get("id").toString(), timeStamp);
return generateActualId(testRecord.get("id").toString(), timeStamp, testRecord.get("kind").toString());
}
public void i_should_get_the_documents_for_the_in_the_Elastic_Search(int expectedCount, String index) throws Throwable {
......
testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/SchemaServiceRecordSteps.java
+ 1
− 1
View file @ d4567b00
......@@ -39,7 +39,7 @@ public class SchemaServiceRecordSteps extends RecordSteps {
@Override
protected String generateRecordId(Map<String, Object> testRecord) {
return generateActualNameWithoutTs(testRecord.get("id").toString());
return generateActualIdWithoutTs(testRecord.get("id").toString(), testRecord.get("kind").toString());
}
@Override
......
testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/TestsBase.java
+ 19
− 0
View file @ d4567b00
......@@ -130,6 +130,17 @@ public abstract class TestsBase {
return rawName.replaceAll("<timestamp>", timeStamp);
}
protected String generateActualId(String rawName, String timeStamp, String kind) {
rawName = generateActualName(rawName, timeStamp);
String kindSubType = kind.split(":")[2];
return rawName.replaceAll("<kindSubType>", kindSubType);
}
public String generateActualNameWithoutTs(String rawName) {
for (Map.Entry<String, String> tenant : tenantMap.entrySet()) {
rawName = rawName.replaceAll(tenant.getKey(), tenant.getValue());
......@@ -137,6 +148,14 @@ public abstract class TestsBase {
return rawName.replaceAll("<timestamp>", "");
}
public String generateActualIdWithoutTs(String rawName, String kind) {
rawName = generateActualNameWithoutTs(rawName);
String kindSubType = kind.split(":")[2];
return rawName.replaceAll("<kindSubType>", kindSubType);
}
protected Legal generateLegalTag() {
Legal legal = new Legal();
Set<String> legalTags = new HashSet<>();
......
testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
+ 3
− 3
View file @ d4567b00
......@@ -14,9 +14,9 @@ Feature: Indexing of the documents
Then I should get the elastic <mapping> for the <type> and <index> in the Elastic Search
Examples:
| kind | recordFile | number | index | type | acl | mapping |
| "tenant1:indexer-int-test:sample-schema-1:1.0.4" | "index_records_schema_1" | 5 | "tenant1-indexer-int-test-sample-schema-1-1.0.4" | "sample-schema-1" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
| "tenant1:indexer-int-test:sample-schema-3:1.0.4" | "index_records_schema_1" | 5 | "tenant1-indexer-int-test-sample-schema-3-1.0.4" | "sample-schema-3" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
| kind | recordFile | number | index | type | acl | mapping |
| "tenant1:indexer-int-test:sample-schema-1:1.0.4" | "index_records_1" | 5 | "tenant1-indexer-int-test-sample-schema-1-1.0.4" | "sample-schema-1" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
| "tenant1:indexer-int-test:sample-schema-3:1.0.4" | "index_records_1" | 5 | "tenant1-indexer-int-test-sample-schema-3-1.0.4" | "sample-schema-3" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
Scenario Outline: Ingest the record and Index in the Elastic Search with bad attribute
When I ingest records with the <recordFile> with <acl> for a given <kind>
......
testing/indexer-test-core/src/main/resources/testData/index_records_1.json
+ 5
− 5
View file @ d4567b00
[
{
"id": "tenant1:ihs:testIngest2<timestamp>",
"id": "tenant1:<kindSubType>:testIngest2<timestamp>",
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
......@@ -23,7 +23,7 @@
}
},
{
"id": "tenant1:ihs:testIngest3<timestamp>",
"id": "tenant1:<kindSubType>:testIngest3<timestamp>",
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
......@@ -46,7 +46,7 @@
}
},
{
"id": "tenant1:ihs:testIngest4<timestamp>",
"id": "tenant1:<kindSubType>:testIngest4<timestamp>",
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
......@@ -68,7 +68,7 @@
}
},
{
"id": "tenant1:ihs:testIngest5<timestamp>",
"id": "tenant1:<kindSubType>:testIngest5<timestamp>",
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
......@@ -90,7 +90,7 @@
}
},
{
"id": "tenant1:ihs:testIngest6<timestamp>",
"id": "tenant1:<kindSubType>:testIngest6<timestamp>",
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
......
testing/indexer-test-core/src/main/resources/testData/index_records_2.json
+ 5
− 5
View file @ d4567b00
[
{
"id": "tenant1:ihs:testIngest7<timestamp>",
"id": "tenant1:<kindSubType>:testIngest7<timestamp>",
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
......@@ -22,7 +22,7 @@
}
},
{
"id": "tenant1:ihs:testIngest8<timestamp>",
"id": "tenant1:<kindSubType>:testIngest8<timestamp>",
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
......@@ -44,7 +44,7 @@
}
},
{
"id": "tenant1:ihs:testIngest9<timestamp>",
"id": "tenant1:<kindSubType>:testIngest9<timestamp>",
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
......@@ -66,7 +66,7 @@
}
},
{
"id": "tenant1:ihs:testIngest10<timestamp>",
"id": "tenant1:<kindSubType>:testIngest10<timestamp>",
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
......@@ -88,7 +88,7 @@
}
},
{
"id": "tenant1:ihs:testIngest11<timestamp>",
"id": "tenant1:<kindSubType>:testIngest11<timestamp>",
"data": {
"Field": 1234,
"Location": {
......
testing/indexer-test-core/src/main/resources/testData/index_records_3.json
+ 7
− 7
View file @ d4567b00
[
{
"id": "tenant1:ihs:testIngest51<timestamp>",
"id": "tenant1:<kindSubType>:testIngest51<timestamp>",
"data": {
"WellName": "Data Platform Services - 51",
"GeoShape": {
......@@ -13,7 +13,7 @@
}
},
{
"id": "tenant1:ihs:testIngest52<timestamp>",
"id": "tenant1:<kindSubType>:testIngest52<timestamp>",
"data": {
"WellName": "Data Platform Services - 52",
"GeoShape": {
......@@ -32,7 +32,7 @@
}
},
{
"id": "tenant1:ihs:testIngest53<timestamp>",
"id": "tenant1:<kindSubType>:testIngest53<timestamp>",
"data": {
"WellName": "Data Platform Services - 53",
"GeoShape": {
......@@ -147,7 +147,7 @@
}
},
{
"id": "tenant1:ihs:testIngest54<timestamp>",
"id": "tenant1:<kindSubType>:testIngest54<timestamp>",
"data": {
"WellName": "Data Platform Services - 54",
"GeoShape": {
......@@ -238,7 +238,7 @@
}
},
{
"id": "tenant1:ihs:testIngest55<timestamp>",
"id": "tenant1:<kindSubType>:testIngest55<timestamp>",
"data": {
"WellName": "Data Platform Services - 55",
"GeoShape": {
......@@ -271,7 +271,7 @@
}
},
{
"id": "tenant1:ihs:testIngest56<timestamp>",
"id": "tenant1:<kindSubType>:testIngest56<timestamp>",
"data": {
"WellName": "Data Platform Services - 56",
"GeoShape": {
......@@ -330,7 +330,7 @@
}
},
{
"id": "tenant1:ihs:testIngest57<timestamp>",
"id": "tenant1:<kindSubType>:testIngest57<timestamp>",
"data": {
"WellName": "Data Platform Services - 57",
"GeoShape": {
......
testing/indexer-test-core/src/main/resources/testData/records_1.json
+ 3
− 3
View file @ d4567b00
[
{
"id": "test:well:1.0.0:1",
"id": "tenant1:well:1.0.0:1",
"kind": "<Kind>",
"namespace": "tenant1:testdatasource",
"type": "well",
......@@ -38,7 +38,7 @@
}
},
{
"id": "test:well:1.0.0:2",
"id": "tenant1:well:1.0.0:2",
"kind": "<Kind>",
"namespace": "tenant1:testdatasource",
"type": "well",
......@@ -77,7 +77,7 @@
}
},
{
"id": "test:well:1.0.0:3",
"id": "tenant1:well:1.0.0:3",
"kind": "<Kind>",
"namespace": "tenant1:testdatasource",
"type": "well",
......
testing/indexer-test-core/src/main/resources/testData/records_2.json
+ 3
− 3
View file @ d4567b00
[
{
"id": "test:well:2.0.0:1",
"id": "tenant1:well:2.0.0:1",
"kind": "<kind>",
"acl": "{viewers=[<Data-Group>], owners=[<Data-Group>]}",
"x-acl": [
......@@ -37,7 +37,7 @@
}
},
{
"id": "test:well:2.0.0:2",
"id": "tenant1:well:2.0.0:2",
"kind": "<kind>",
"acl": "{viewers=[<Data-Group>], owners=[<Data-Group>]}",
"x-acl": [
......@@ -74,7 +74,7 @@
}
},
{
"id": "test:well:2.0.0:3",
"id": "tenant1:well:2.0.0:3",
"kind": "<kind>",
"acl": "{viewers=[<Data-Group>], owners=[<Data-Group>]}",
"x-acl": [
......
testing/indexer-test-core/src/main/resources/testData/records_3.json
+ 3
− 3
View file @ d4567b00
[
{
"id": "test:well:1.0.0:1",
"id": "tenant1:well:1.0.0:1",
"kind": "<Kind>",
"namespace": "tenant1:testdatasource",
"type": "well",
......@@ -39,7 +39,7 @@
}
},
{
"id": "test:well:1.0.0:2",
"id": "tenant1:well:1.0.0:2",
"kind": "<Kind>",
"namespace": "tenant1:testdatasource",
"type": "well",
......@@ -79,7 +79,7 @@
}
},
{
"id": "test:well:1.0.0:3",
"id": "tenant1:well:1.0.0:3",
"kind": "<Kind>",
"namespace": "tenant1:testdatasource",
"type": "well",
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment