Skip to content
Snippets Groups Projects
Commit 7c23ed3f authored by ethiraj krishnamanaidu's avatar ethiraj krishnamanaidu
Browse files

Merge branch 'aws-fix-int-tests' into 'master' 

Fix Int tests failing due to new storage record ID validation rules

See merge request !83
parents af0d36ee d3463832
No related branches found
No related tags found
1 merge request!83Fix Int tests failing due to new storage record ID validation rules
Pipeline #25368 passed
Stage: build
Stage: containerize
Stage: scan
Stage: deploy
Stage: integration
Stage: attribution
Stage: publish
Hide whitespace changes
Inline Side-by-side
Showing
with 52 additions and 33 deletions
testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/CleanupIndiciesSteps.java
+ 1
− 1
View file @ 7c23ed3f
......@@ -87,8 +87,8 @@ public class CleanupIndiciesSteps extends TestsBase {
records = new Gson().fromJson(fileContent, new TypeToken<List<Map<String, Object>>>() {}.getType());
for (Map<String, Object> testRecord : records) {
testRecord.put("id", generateActualName(testRecord.get("id").toString(), timeStamp));
testRecord.put("kind", actualKind);
testRecord.put("id", generateActualId(testRecord.get("id").toString(), timeStamp, actualKind));
testRecord.put("legal", generateLegalTag());
String[] x_acl = {generateActualName(dataGroup,timeStamp)+"."+getEntitlementsDomain()};
Acl acl = Acl.builder().viewers(x_acl).owners(x_acl).build();
......
testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
+ 2
− 2
View file @ 7c23ed3f
......@@ -97,8 +97,8 @@ public class RecordSteps extends TestsBase {
records = new Gson().fromJson(fileContent, new TypeToken<List<Map<String, Object>>>() {}.getType());
for (Map<String, Object> testRecord : records) {
testRecord.put("id", generateRecordId(testRecord));
testRecord.put("kind", actualKind);
testRecord.put("id", generateRecordId(testRecord));
testRecord.put("legal", generateLegalTag());
String[] x_acl = {generateActualName(dataGroup,timeStamp)+"."+getEntitlementsDomain()};
Acl acl = Acl.builder().viewers(x_acl).owners(x_acl).build();
......@@ -114,7 +114,7 @@ public class RecordSteps extends TestsBase {
}
protected String generateRecordId(Map<String, Object> testRecord) {
return generateActualName(testRecord.get("id").toString(), timeStamp);
return generateActualId(testRecord.get("id").toString(), timeStamp, testRecord.get("kind").toString());
}
public void i_should_get_the_documents_for_the_in_the_Elastic_Search(int expectedCount, String index) throws Throwable {
......
testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/SchemaServiceRecordSteps.java
+ 1
− 1
View file @ 7c23ed3f
......@@ -39,7 +39,7 @@ public class SchemaServiceRecordSteps extends RecordSteps {
@Override
protected String generateRecordId(Map<String, Object> testRecord) {
return generateActualNameWithoutTs(testRecord.get("id").toString());
return generateActualIdWithoutTs(testRecord.get("id").toString(), testRecord.get("kind").toString());
}
@Override
......
testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/TestsBase.java
+ 19
− 0
View file @ 7c23ed3f
......@@ -130,6 +130,17 @@ public abstract class TestsBase {
return rawName.replaceAll("<timestamp>", timeStamp);
}
protected String generateActualId(String rawName, String timeStamp, String kind) {
rawName = generateActualName(rawName, timeStamp);
String kindSubType = kind.split(":")[2];
return rawName.replaceAll("<kindSubType>", kindSubType);
}
public String generateActualNameWithoutTs(String rawName) {
for (Map.Entry<String, String> tenant : tenantMap.entrySet()) {
rawName = rawName.replaceAll(tenant.getKey(), tenant.getValue());
......@@ -137,6 +148,14 @@ public abstract class TestsBase {
return rawName.replaceAll("<timestamp>", "");
}
public String generateActualIdWithoutTs(String rawName, String kind) {
rawName = generateActualNameWithoutTs(rawName);
String kindSubType = kind.split(":")[2];
return rawName.replaceAll("<kindSubType>", kindSubType);
}
protected Legal generateLegalTag() {
Legal legal = new Legal();
Set<String> legalTags = new HashSet<>();
......
testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
+ 3
− 3
View file @ 7c23ed3f
......@@ -14,9 +14,9 @@ Feature: Indexing of the documents
Then I should get the elastic <mapping> for the <type> and <index> in the Elastic Search
Examples:
| kind | recordFile | number | index | type | acl | mapping |
| "tenant1:indexer-int-test:sample-schema-1:1.0.4" | "index_records_schema_1" | 5 | "tenant1-indexer-int-test-sample-schema-1-1.0.4" | "sample-schema-1" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
| "tenant1:indexer-int-test:sample-schema-3:1.0.4" | "index_records_schema_1" | 5 | "tenant1-indexer-int-test-sample-schema-3-1.0.4" | "sample-schema-3" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
| kind | recordFile | number | index | type | acl | mapping |
| "tenant1:indexer-int-test:sample-schema-1:1.0.4" | "index_records_1" | 5 | "tenant1-indexer-int-test-sample-schema-1-1.0.4" | "sample-schema-1" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
| "tenant1:indexer-int-test:sample-schema-3:1.0.4" | "index_records_1" | 5 | "tenant1-indexer-int-test-sample-schema-3-1.0.4" | "sample-schema-3" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
Scenario Outline: Ingest the record and Index in the Elastic Search with bad attribute
When I ingest records with the <recordFile> with <acl> for a given <kind>
......
testing/indexer-test-core/src/main/resources/testData/index_records_1.json
+ 5
− 5
View file @ 7c23ed3f
[
{
"id": "tenant1:ihs:testIngest2<timestamp>",
"id": "tenant1:<kindSubType>:testIngest2<timestamp>",
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
......@@ -23,7 +23,7 @@
}
},
{
"id": "tenant1:ihs:testIngest3<timestamp>",
"id": "tenant1:<kindSubType>:testIngest3<timestamp>",
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
......@@ -46,7 +46,7 @@
}
},
{
"id": "tenant1:ihs:testIngest4<timestamp>",
"id": "tenant1:<kindSubType>:testIngest4<timestamp>",
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
......@@ -68,7 +68,7 @@
}
},
{
"id": "tenant1:ihs:testIngest5<timestamp>",
"id": "tenant1:<kindSubType>:testIngest5<timestamp>",
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
......@@ -90,7 +90,7 @@
}
},
{
"id": "tenant1:ihs:testIngest6<timestamp>",
"id": "tenant1:<kindSubType>:testIngest6<timestamp>",
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
......
testing/indexer-test-core/src/main/resources/testData/index_records_2.json
+ 5
− 5
View file @ 7c23ed3f
[
{
"id": "tenant1:ihs:testIngest7<timestamp>",
"id": "tenant1:<kindSubType>:testIngest7<timestamp>",
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
......@@ -22,7 +22,7 @@
}
},
{
"id": "tenant1:ihs:testIngest8<timestamp>",
"id": "tenant1:<kindSubType>:testIngest8<timestamp>",
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
......@@ -44,7 +44,7 @@
}
},
{
"id": "tenant1:ihs:testIngest9<timestamp>",
"id": "tenant1:<kindSubType>:testIngest9<timestamp>",
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
......@@ -66,7 +66,7 @@
}
},
{
"id": "tenant1:ihs:testIngest10<timestamp>",
"id": "tenant1:<kindSubType>:testIngest10<timestamp>",
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
......@@ -88,7 +88,7 @@
}
},
{
"id": "tenant1:ihs:testIngest11<timestamp>",
"id": "tenant1:<kindSubType>:testIngest11<timestamp>",
"data": {
"Field": 1234,
"Location": {
......
testing/indexer-test-core/src/main/resources/testData/index_records_3.json
+ 7
− 7
View file @ 7c23ed3f
[
{
"id": "tenant1:ihs:testIngest51<timestamp>",
"id": "tenant1:<kindSubType>:testIngest51<timestamp>",
"data": {
"WellName": "Data Platform Services - 51",
"GeoShape": {
......@@ -13,7 +13,7 @@
}
},
{
"id": "tenant1:ihs:testIngest52<timestamp>",
"id": "tenant1:<kindSubType>:testIngest52<timestamp>",
"data": {
"WellName": "Data Platform Services - 52",
"GeoShape": {
......@@ -32,7 +32,7 @@
}
},
{
"id": "tenant1:ihs:testIngest53<timestamp>",
"id": "tenant1:<kindSubType>:testIngest53<timestamp>",
"data": {
"WellName": "Data Platform Services - 53",
"GeoShape": {
......@@ -147,7 +147,7 @@
}
},
{
"id": "tenant1:ihs:testIngest54<timestamp>",
"id": "tenant1:<kindSubType>:testIngest54<timestamp>",
"data": {
"WellName": "Data Platform Services - 54",
"GeoShape": {
......@@ -238,7 +238,7 @@
}
},
{
"id": "tenant1:ihs:testIngest55<timestamp>",
"id": "tenant1:<kindSubType>:testIngest55<timestamp>",
"data": {
"WellName": "Data Platform Services - 55",
"GeoShape": {
......@@ -271,7 +271,7 @@
}
},
{
"id": "tenant1:ihs:testIngest56<timestamp>",
"id": "tenant1:<kindSubType>:testIngest56<timestamp>",
"data": {
"WellName": "Data Platform Services - 56",
"GeoShape": {
......@@ -330,7 +330,7 @@
}
},
{
"id": "tenant1:ihs:testIngest57<timestamp>",
"id": "tenant1:<kindSubType>:testIngest57<timestamp>",
"data": {
"WellName": "Data Platform Services - 57",
"GeoShape": {
......
testing/indexer-test-core/src/main/resources/testData/records_1.json
+ 3
− 3
View file @ 7c23ed3f
[
{
"id": "test:well:1.0.0:1",
"id": "tenant1:well:1.0.0:1",
"kind": "<Kind>",
"namespace": "tenant1:testdatasource",
"type": "well",
......@@ -38,7 +38,7 @@
}
},
{
"id": "test:well:1.0.0:2",
"id": "tenant1:well:1.0.0:2",
"kind": "<Kind>",
"namespace": "tenant1:testdatasource",
"type": "well",
......@@ -77,7 +77,7 @@
}
},
{
"id": "test:well:1.0.0:3",
"id": "tenant1:well:1.0.0:3",
"kind": "<Kind>",
"namespace": "tenant1:testdatasource",
"type": "well",
......
testing/indexer-test-core/src/main/resources/testData/records_2.json
+ 3
− 3
View file @ 7c23ed3f
[
{
"id": "test:well:2.0.0:1",
"id": "tenant1:well:2.0.0:1",
"kind": "<kind>",
"acl": "{viewers=[<Data-Group>], owners=[<Data-Group>]}",
"x-acl": [
......@@ -37,7 +37,7 @@
}
},
{
"id": "test:well:2.0.0:2",
"id": "tenant1:well:2.0.0:2",
"kind": "<kind>",
"acl": "{viewers=[<Data-Group>], owners=[<Data-Group>]}",
"x-acl": [
......@@ -74,7 +74,7 @@
}
},
{
"id": "test:well:2.0.0:3",
"id": "tenant1:well:2.0.0:3",
"kind": "<kind>",
"acl": "{viewers=[<Data-Group>], owners=[<Data-Group>]}",
"x-acl": [
......
testing/indexer-test-core/src/main/resources/testData/records_3.json
+ 3
− 3
View file @ 7c23ed3f
[
{
"id": "test:well:1.0.0:1",
"id": "tenant1:well:1.0.0:1",
"kind": "<Kind>",
"namespace": "tenant1:testdatasource",
"type": "well",
......@@ -39,7 +39,7 @@
}
},
{
"id": "test:well:1.0.0:2",
"id": "tenant1:well:1.0.0:2",
"kind": "<Kind>",
"namespace": "tenant1:testdatasource",
"type": "well",
......@@ -79,7 +79,7 @@
}
},
{
"id": "test:well:1.0.0:3",
"id": "tenant1:well:1.0.0:3",
"kind": "<Kind>",
"namespace": "tenant1:testdatasource",
"type": "well",
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment