diff --git a/indexer-core/pom.xml b/indexer-core/pom.xml
index 3c75f434ba4f5f4af8de3a28044b74dc2e53bae1..650b0702f9d6140364f4b8635753a1e0d125ffac 100644
--- a/indexer-core/pom.xml
+++ b/indexer-core/pom.xml
@@ -16,7 +16,7 @@
<properties>
<commons-beanutils.version>1.9.4</commons-beanutils.version>
- <os-core-common.version>0.11.0-SNAPSHOT</os-core-common.version>
+ <osdu.oscorecommon.version>0.11.0-rc4</osdu.oscorecommon.version>
</properties>
<dependencies>
@@ -34,7 +34,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
- <version>${os-core-common.version}</version>
+ <version>${osdu.oscorecommon.version}</version>
</dependency>
<!-- spring boot dependencies -->
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
index a979c0868f074adc77c7bcaec6d51ed2ab1eac86..009247ba48e40da753c5c10a38b8a3d265e325e9 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
@@ -236,6 +236,12 @@ public class IndexSchemaServiceImpl implements IndexSchemaService {
meta.put(RecordMetaAttribute.LEGAL.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.LEGAL));
meta.put(RecordMetaAttribute.ANCESTRY.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.ANCESTRY));
meta.put(RecordMetaAttribute.INDEX_STATUS.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.INDEX_STATUS));
+ meta.put(RecordMetaAttribute.AUTHORITY.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.AUTHORITY));
+ meta.put(RecordMetaAttribute.SOURCE.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.SOURCE));
+ meta.put(RecordMetaAttribute.CREATE_USER.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.CREATE_USER));
+ meta.put(RecordMetaAttribute.CREATE_TIME.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.CREATE_TIME));
+ meta.put(RecordMetaAttribute.MODIFY_USER.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.MODIFY_USER));
+ meta.put(RecordMetaAttribute.MODIFY_TIME.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.MODIFY_TIME));
String kind = schemaObj.getKind();
String type = kind.split(":")[2];
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
index 6f796030f9d6ec73e8e6675c6c24882c86646a14..fed2822af6d993b407e4d7502f0685d35e42a184 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
@@ -14,6 +14,7 @@
package org.opengroup.osdu.indexer.service;
+import com.google.common.base.Strings;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import org.apache.http.HttpStatus;
@@ -129,8 +130,8 @@ public class IndexerServiceImpl implements IndexerService {
}
auditLogger.indexStarted(recordInfos.stream()
- .map(RecordInfo::getKind)
- .collect(Collectors.toList()));
+ .map(RecordInfo::getKind)
+ .collect(Collectors.toList()));
// process schema change messages
Map<String, OperationType> schemaMsgs = RecordInfo.getSchemaMsgs(recordInfos);
@@ -306,10 +307,14 @@ public class IndexerServiceImpl implements IndexerService {
try {
// index individual parts of kind
String[] kindParts = storageRecord.getKind().split(":");
-
+ String authority = kindParts[0];
+ String source = kindParts[1];
+ String type = kindParts[2];
document.setKind(storageRecord.getKind());
- document.setNamespace(kindParts[0] + ":" + kindParts[1]);
- document.setType(kindParts[2]);
+ document.setNamespace(authority + ":" + source);
+ document.setAuthority(authority);
+ document.setSource(source);
+ document.setType(type);
document.setId(storageRecord.getId());
document.setVersion(storageRecord.getVersion());
document.setAcl(storageRecord.getAcl());
@@ -317,6 +322,14 @@ public class IndexerServiceImpl implements IndexerService {
if (storageRecord.getTags() != null) {
document.setTags(storageRecord.getTags());
}
+ document.setCreateUser(storageRecord.getCreateUser());
+ document.setCreateTime(storageRecord.getCreateTime());
+ if (!Strings.isNullOrEmpty(storageRecord.getModifyUser())) {
+ document.setModifyUser(storageRecord.getModifyUser());
+ }
+ if (!Strings.isNullOrEmpty(storageRecord.getModifyTime())) {
+ document.setModifyTime(storageRecord.getModifyTime());
+ }
RecordStatus recordStatus = this.jobStatus.getJobStatusByRecordId(storageRecord.getId());
if (recordStatus.getIndexProgress().getStatusCode() == 0) {
recordStatus.getIndexProgress().setStatusCode(HttpStatus.SC_OK);
@@ -467,6 +480,8 @@ public class IndexerServiceImpl implements IndexerService {
indexerPayload.put(RecordMetaAttribute.ID.getValue(), record.getId());
indexerPayload.put(RecordMetaAttribute.KIND.getValue(), record.getKind());
+ indexerPayload.put(RecordMetaAttribute.AUTHORITY.getValue(), record.getAuthority());
+ indexerPayload.put(RecordMetaAttribute.SOURCE.getValue(), record.getSource());
indexerPayload.put(RecordMetaAttribute.NAMESPACE.getValue(), record.getNamespace());
indexerPayload.put(RecordMetaAttribute.TYPE.getValue(), record.getType());
indexerPayload.put(RecordMetaAttribute.VERSION.getValue(), record.getVersion());
@@ -478,6 +493,14 @@ public class IndexerServiceImpl implements IndexerService {
if (record.getAncestry() != null) {
indexerPayload.put(RecordMetaAttribute.ANCESTRY.getValue(), record.getAncestry());
}
+ indexerPayload.put(RecordMetaAttribute.CREATE_USER.getValue(), record.getCreateUser());
+ indexerPayload.put(RecordMetaAttribute.CREATE_TIME.getValue(), record.getCreateTime());
+ if (!Strings.isNullOrEmpty(record.getModifyUser())) {
+ indexerPayload.put(RecordMetaAttribute.MODIFY_USER.getValue(), record.getModifyUser());
+ }
+ if (!Strings.isNullOrEmpty(record.getModifyTime())) {
+ indexerPayload.put(RecordMetaAttribute.MODIFY_TIME.getValue(), record.getModifyTime());
+ }
return indexerPayload;
}
@@ -512,13 +535,13 @@ public class IndexerServiceImpl implements IndexerService {
private void logAuditEvents(OperationType operationType, Consumer<List<String>> successEvent, Consumer<List<String>> failedEvent) {
List<RecordStatus> succeededRecords = this.jobStatus.getRecordStatuses(IndexingStatus.SUCCESS, operationType);
- if(!succeededRecords.isEmpty()) {
+ if (!succeededRecords.isEmpty()) {
successEvent.accept(succeededRecords.stream().map(RecordStatus::succeededAuditLogMessage).collect(Collectors.toList()));
}
List<RecordStatus> skippedRecords = this.jobStatus.getRecordStatuses(IndexingStatus.SKIP, operationType);
List<RecordStatus> failedRecords = this.jobStatus.getRecordStatuses(IndexingStatus.FAIL, operationType);
failedRecords.addAll(skippedRecords);
- if(!failedRecords.isEmpty()) {
+ if (!failedRecords.isEmpty()) {
failedEvent.accept(failedRecords.stream().map(RecordStatus::failedAuditLogMessage).collect(Collectors.toList()));
}
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java
index 6a9a9dc4c30900a76fa811ac63315bce6358efb0..122711c9a2226d292fda4141a9c949dcc286fbf7 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java
@@ -49,6 +49,12 @@ public class TypeMapper {
metaAttributeIndexerType.put(RecordMetaAttribute.LEGAL.getValue(), getLegalIndexerMapping());
metaAttributeIndexerType.put(RecordMetaAttribute.ANCESTRY.getValue(), getAncestryIndexerMapping());
metaAttributeIndexerType.put(RecordMetaAttribute.INDEX_STATUS.getValue(), getIndexStatusMapping());
+ metaAttributeIndexerType.put(RecordMetaAttribute.AUTHORITY.getValue(), ElasticType.CONSTANT_KEYWORD.getValue());
+ metaAttributeIndexerType.put(RecordMetaAttribute.SOURCE.getValue(), ElasticType.CONSTANT_KEYWORD.getValue());
+ metaAttributeIndexerType.put(RecordMetaAttribute.CREATE_USER.getValue(), ElasticType.KEYWORD.getValue());
+ metaAttributeIndexerType.put(RecordMetaAttribute.MODIFY_USER.getValue(), ElasticType.KEYWORD.getValue());
+ metaAttributeIndexerType.put(RecordMetaAttribute.CREATE_TIME.getValue(), ElasticType.DATE.getValue());
+ metaAttributeIndexerType.put(RecordMetaAttribute.MODIFY_TIME.getValue(), ElasticType.DATE.getValue());
storageToIndexerType.put(StorageType.LINK.getValue(), ElasticType.KEYWORD.getValue());
storageToIndexerType.put(StorageType.LINK_ARRAY.getValue(), ElasticType.KEYWORD_ARRAY.getValue());
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/TypeMapperTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/TypeMapperTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..6880026132541a16c108b2b90efa14c6a0c13395
--- /dev/null
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/TypeMapperTest.java
@@ -0,0 +1,41 @@
+package org.opengroup.osdu.indexer.util;
+
+import org.junit.Test;
+import org.opengroup.osdu.core.common.model.indexer.ElasticType;
+import org.opengroup.osdu.core.common.model.indexer.StorageType;
+import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
+
+import static org.junit.Assert.assertEquals;
+
+public class TypeMapperTest {
+
+ @Test
+ public void getIndexerTypeTest() {
+ assertEquals(ElasticType.KEYWORD.getValue(), TypeMapper.getIndexerType(StorageType.LINK.getValue(), ElasticType.TEXT.getValue()));
+ assertEquals(ElasticType.KEYWORD_ARRAY.getValue(), TypeMapper.getIndexerType(StorageType.LINK_ARRAY.getValue(), ElasticType.TEXT.getValue()));
+ assertEquals(ElasticType.BOOLEAN.getValue(), TypeMapper.getIndexerType(StorageType.BOOLEAN.getValue(), ElasticType.TEXT.getValue()));
+ assertEquals(ElasticType.TEXT.getValue(), TypeMapper.getIndexerType(StorageType.STRING.getValue(), ElasticType.TEXT.getValue()));
+ assertEquals(ElasticType.INTEGER.getValue(), TypeMapper.getIndexerType(StorageType.INT.getValue(), ElasticType.TEXT.getValue()));
+ assertEquals(ElasticType.FLOAT.getValue(), TypeMapper.getIndexerType(StorageType.FLOAT.getValue(), ElasticType.TEXT.getValue()));
+ assertEquals(ElasticType.DOUBLE.getValue(), TypeMapper.getIndexerType(StorageType.DOUBLE.getValue(), ElasticType.TEXT.getValue()));
+ assertEquals(ElasticType.DOUBLE_ARRAY.getValue(), TypeMapper.getIndexerType(StorageType.DOUBLE_ARRAY.getValue(), ElasticType.TEXT.getValue()));
+ assertEquals(ElasticType.LONG.getValue(), TypeMapper.getIndexerType(StorageType.LONG.getValue(), ElasticType.TEXT.getValue()));
+ assertEquals(ElasticType.DATE.getValue(), TypeMapper.getIndexerType(StorageType.DATETIME.getValue(), ElasticType.TEXT.getValue()));
+ assertEquals(ElasticType.GEO_POINT.getValue(), TypeMapper.getIndexerType(StorageType.GEO_POINT.getValue(), ElasticType.TEXT.getValue()));
+ assertEquals(ElasticType.GEO_SHAPE.getValue(), TypeMapper.getIndexerType(StorageType.GEO_SHAPE.getValue(), ElasticType.TEXT.getValue()));
+
+ assertEquals(ElasticType.KEYWORD.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.KIND));
+ assertEquals(ElasticType.KEYWORD.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.TYPE));
+ assertEquals(ElasticType.KEYWORD.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.ID));
+ assertEquals(ElasticType.KEYWORD.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.NAMESPACE));
+ assertEquals(ElasticType.LONG.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.VERSION));
+ assertEquals(ElasticType.KEYWORD.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.X_ACL));
+
+ assertEquals(ElasticType.CONSTANT_KEYWORD.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.AUTHORITY));
+ assertEquals(ElasticType.CONSTANT_KEYWORD.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.SOURCE));
+ assertEquals(ElasticType.KEYWORD.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.CREATE_USER));
+ assertEquals(ElasticType.KEYWORD.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.MODIFY_USER));
+ assertEquals(ElasticType.DATE.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.CREATE_TIME));
+ assertEquals(ElasticType.DATE.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.MODIFY_TIME));
+ }
+}
diff --git a/pom.xml b/pom.xml
index 06c96a405e8dbb32b2d91bf8fd6214bc66809d43..a59ee5d608b24071b98192b249c169d14a3aa4fa 100644
--- a/pom.xml
+++ b/pom.xml
@@ -19,7 +19,7 @@
<java.version>1.8</java.version>
<springfox-version>2.7.0</springfox-version>
<spring-cloud.version>Greenwich.SR2</spring-cloud.version>
- <os-core-common.version>0.11.0-SNAPSHOT</os-core-common.version>
+ <os-core-common.version>0.11.0-rc4</os-core-common.version>
<snakeyaml.version>1.26</snakeyaml.version>
<hibernate-validator.version>6.1.5.Final</hibernate-validator.version>
<jackson.version>2.11.4</jackson.version>
diff --git a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
index c565d38835769e8e5d5100583fd61202fbc26cd9..01691b4f8b53f1d0db827844231553a969da8529 100644
--- a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
+++ b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -86,8 +86,13 @@ public class Steps extends SchemaServiceRecordSteps {
}
@Then("^I should get the elastic \"(.*?)\" for the \"([^\"]*)\" and \"([^\"]*)\" in the Elastic Search$")
- public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String type, String index) throws Throwable {
- super.i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(expectedMapping, type, index);
+ public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String kind, String index) throws Throwable {
+ super.i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(expectedMapping, kind, index);
+ }
+
+ @Then("^I can validate indexed meta attributes for the \"([^\"]*)\" and given \"([^\"]*)\"$")
+ public void i_can_validate_indexed_meta_attributes(String index, String kind) throws Throwable {
+ super.i_can_validate_indexed_attributes(index, kind);
}
@Then("^I should get the (\\d+) documents for the \"([^\"]*)\" in the Elastic Search with out \"(.*?)\"$")
diff --git a/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
index b1dc2343728b339278710726c0d37d4a1735dd0f..5c0f44c22eff61886aba89ecb1693b7a73856999 100644
--- a/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
+++ b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -54,8 +54,13 @@ public class Steps extends SchemaServiceRecordSteps {
}
@Then("^I should get the elastic \"(.*?)\" for the \"([^\"]*)\" and \"([^\"]*)\" in the Elastic Search$")
- public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String type, String index) throws Throwable {
- super.i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(expectedMapping, type, index);
+ public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String kind, String index) throws Throwable {
+ super.i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(expectedMapping, kind, index);
+ }
+
+ @Then("^I can validate indexed meta attributes for the \"([^\"]*)\" and given \"([^\"]*)\"$")
+ public void i_can_validate_indexed_meta_attributes(String index, String kind) throws Throwable {
+ super.i_can_validate_indexed_attributes(index, kind);
}
@Then("^I should get the (\\d+) documents for the \"([^\"]*)\" in the Elastic Search with out \"(.*?)\"$")
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
index cd7f2eb02ee34ec824ca51b88a47c57460ef211e..d78985d0d2992c41bb3707cf4ee92414045d58bd 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
@@ -48,7 +48,7 @@ public class RecordSteps extends TestsBase {
public RecordSteps(HTTPClient httpClient, ElasticUtils elasticUtils) {
super(httpClient, elasticUtils);
}
-
+
/******************One time cleanup for whole feature**************/
public void tearDown() {
for (String kind : inputIndexMap.keySet()) {
@@ -98,14 +98,22 @@ public class RecordSteps extends TestsBase {
try {
String fileContent = FileHandler.readFile(String.format("%s.%s", record, "json"));
records = new Gson().fromJson(fileContent, new TypeToken<List<Map<String, Object>>>() {}.getType());
+ String createTime = java.time.Instant.now().toString();
for (Map<String, Object> testRecord : records) {
testRecord.put("kind", actualKind);
- testRecord.put("id", generateRecordId(testRecord));
+ testRecord.put("id", generateRecordId(testRecord));
testRecord.put("legal", generateLegalTag());
String[] x_acl = {generateActualName(dataGroup,timeStamp)+"."+getEntitlementsDomain()};
Acl acl = Acl.builder().viewers(x_acl).owners(x_acl).build();
testRecord.put("acl", acl);
+ String[] kindParts = kind.split(":");
+ String authority = tenantMap.get(kindParts[0]);
+ String source = kindParts[1];
+ testRecord.put("authority", authority);
+ testRecord.put("source", source);
+ testRecord.put("createUser", "TestUser");
+ testRecord.put("createTime", createTime);
}
String payLoad = new Gson().toJson(records);
log.log(Level.INFO, "Start ingesting records={0}", payLoad);
@@ -126,17 +134,36 @@ public class RecordSteps extends TestsBase {
assertEquals(expectedCount, numOfIndexedDocuments);
}
- public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String type, String index) throws Throwable {
+ public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String kind, String index) throws Throwable {
index = generateActualName(index, timeStamp);
Map<String, MappingMetadata> elasticMapping = elasticUtils.getMapping(index);
assertNotNull(elasticMapping);
+ String[] kindParts = kind.split(":");
+ String authority = tenantMap.get(kindParts[0]);
+ String source = kindParts[1];
+ expectedMapping = expectedMapping.replaceAll("<authority-id>", authority).replaceAll("<source-id>", source);
MappingMetadata typeMapping = elasticMapping.get(index);
Map<String, Object> mapping = typeMapping.sourceAsMap();
assertNotNull(mapping);
assertTrue(areJsonEqual(expectedMapping, mapping.toString()));
}
+ public void i_can_validate_indexed_attributes(String index, String kind) throws Throwable {
+ String authority = tenantMap.get(kind.substring(0, kind.indexOf(":")));
+ index = generateActualName(index, timeStamp);
+ List<Map<String, Object>> hits = elasticUtils.fetchRecordsByAttribute(index, "authority", authority);
+
+ assertTrue(hits.size() > 0);
+ for (Map<String, Object> result : hits) {
+ assertTrue(result.containsKey("authority"));
+ assertEquals(authority, result.get("authority"));
+ assertTrue(result.containsKey("source"));
+ assertTrue(result.containsKey("createUser"));
+ assertTrue(result.containsKey("createTime"));
+ }
+ }
+
public void iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(int expectedCount, String index, String skippedAttributes) throws Throwable {
index = generateActualName(index, timeStamp);
long numOfIndexedDocuments = createIndex(index);
@@ -228,7 +255,7 @@ public class RecordSteps extends TestsBase {
log.info(String.format("difference: %s", result.entriesDiffering()));
return false;
}
-
+
@Override
protected String getApi() {
return null;
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
index 3d029aa6ecbd34fb9ec08c2e9909cf933e33843a..23419f3fd798d59eec1a771c366a5acff8b577b4 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
@@ -52,6 +52,7 @@ import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.client.indices.GetMappingsRequest;
import org.elasticsearch.client.indices.GetMappingsResponse;
import org.elasticsearch.cluster.metadata.MappingMetadata;
+import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.locationtech.jts.geom.Coordinate;
import org.elasticsearch.common.geo.builders.EnvelopeBuilder;
@@ -241,6 +242,24 @@ public class ElasticUtils {
}
}
+ public List<Map<String, Object>> fetchRecordsByAttribute(String index, String attributeKey, String attributeValue) throws IOException {
+ List<Map<String, Object>> out = new ArrayList<>();
+ try {
+ try (RestHighLevelClient client = this.createClient(username, password, host)) {
+ SearchRequest request = new SearchRequest(index);
+ SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
+ sourceBuilder.query(boolQuery().must(termsQuery(attributeKey, attributeValue)));
+ request.source(sourceBuilder);
+ SearchResponse searchResponse = client.search(request, RequestOptions.DEFAULT);
+ for(SearchHit searchHit : searchResponse.getHits()) out.add(searchHit.getSourceAsMap());
+ return out;
+ }
+ } catch (ElasticsearchStatusException e) {
+ log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage()));
+ return out;
+ }
+ }
+
public long fetchRecordsByExistQuery(String index, String attributeName) throws Exception {
try {
TimeUnit.SECONDS.sleep(40);
diff --git a/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature b/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
index 05c25961282d3ebec5af580932fd7d73f5917f7b..0888f9528b60901a73d69862039479726a0648a7 100644
--- a/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
+++ b/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
@@ -4,21 +4,22 @@ Feature: Indexing of the documents
Background:
Given the schema is created with the following kind
| kind | index | schemaFile |
- | tenant1:indexer:test-data--Integration:1.0.0 | tenant1-indexer-test-data--integration-1.0.0 | index_records_1 |
- | tenant1:indexer:test-data--Integration:2.0.0 | tenant1-indexer-test-data--integration-2.0.0 | index_records_2 |
- | tenant1:indexer:test-data--Integration:3.0.0 | tenant1-indexer-test-data--integration-3.0.0 | index_records_3 |
+ | tenant1:indexer:test-data--Integration:1.0.1 | tenant1-indexer-test-data--integration-1.0.1 | index_records_1 |
+ | tenant1:indexer:test-data--Integration:2.0.1 | tenant1-indexer-test-data--integration-2.0.1 | index_records_2 |
+ | tenant1:indexer:test-data--Integration:3.0.1 | tenant1-indexer-test-data--integration-3.0.1 | index_records_3 |
| tenant1:wks:master-data--Wellbore:2.0.3 | tenant1-wks-master-data--wellbore-2.0.3 | r3-index_record_wks_master |
| tenant1:wks:ArraysOfObjectsTestCollection:4.0.0 | tenant1-wks-arraysofobjectstestcollection-4.0.0 | r3-index_record_arrayofobjects |
Scenario Outline: Ingest the record and Index in the Elastic Search
When I ingest records with the <recordFile> with <acl> for a given <kind>
Then I should get the <number> documents for the <index> in the Elastic Search
- Then I should get the elastic <mapping> for the <type> and <index> in the Elastic Search
+ Then I should get the elastic <mapping> for the <kind> and <index> in the Elastic Search
+ Then I can validate indexed meta attributes for the <index> and given <kind>
Examples:
- | kind | recordFile | number | index | type | acl | mapping |
- | "tenant1:indexer:test-data--Integration:1.0.0" | "index_records_schema_1" | 5 | "tenant1-indexer-test-data--integration-1.0.0" | "test-data--Integration" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Country":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"County":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"EmptyAttribute":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Established":{"type":"date"},"Field":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellName":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellStatus":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellType":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
- | "tenant1:indexer:test-data--Integration:3.0.0" | "index_records_schema_3" | 7 | "tenant1-indexer-test-data--integration-3.0.0" | "test-data--Integration" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Country":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"County":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"EmptyAttribute":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Established":{"type":"date"},"Field":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellName":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellStatus":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellType":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
+ | kind | recordFile | number | index | acl | mapping |
+ | "tenant1:indexer:test-data--Integration:1.0.1" | "index_records_schema_1" | 5 | "tenant1-indexer-test-data--integration-1.0.1" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Country":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"County":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"EmptyAttribute":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Established":{"type":"date"},"Field":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellName":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellStatus":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellType":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"authority":{"type":"constant_keyword","value":"<authority-id>"},"createTime":{"type":"date"},"createUser":{"type":"keyword"},"modifyTime":{"type":"date"},"modifyUser":{"type":"keyword"},"source":{"type":"constant_keyword","value":"<source-id>"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
+ | "tenant1:indexer:test-data--Integration:3.0.1" | "index_records_schema_3" | 7 | "tenant1-indexer-test-data--integration-3.0.1" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Country":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"County":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"EmptyAttribute":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Established":{"type":"date"},"Field":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellName":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellStatus":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellType":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"authority":{"type":"constant_keyword","value":"<authority-id>"},"createTime":{"type":"date"},"createUser":{"type":"keyword"},"modifyTime":{"type":"date"},"modifyUser":{"type":"keyword"},"source":{"type":"constant_keyword","value":"<source-id>"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
Scenario Outline: Ingest the record and Index in the Elastic Search with bad attribute
When I ingest records with the <recordFile> with <acl> for a given <kind>
@@ -26,8 +27,8 @@ Feature: Indexing of the documents
Examples:
| kind | recordFile | number | index | skippedAttribute | acl |
- | "tenant1:indexer:test-data--Integration:2.0.0" | "index_records_2" | 4 | "tenant1-indexer-test-data--integration-2.0.0" | "data.Location" | "data.default.viewers@tenant1" |
- | "tenant1:indexer:test-data--Integration:2.0.0" | "index_records_2" | 1 | "tenant1-indexer-test-data--integration-2.0.0" | "data.InvalidInteger" | "data.default.viewers@tenant1" |
+ | "tenant1:indexer:test-data--Integration:2.0.1" | "index_records_2" | 4 | "tenant1-indexer-test-data--integration-2.0.1" | "data.Location" | "data.default.viewers@tenant1" |
+ | "tenant1:indexer:test-data--Integration:2.0.1" | "index_records_2" | 1 | "tenant1-indexer-test-data--integration-2.0.1" | "data.InvalidInteger" | "data.default.viewers@tenant1" |
Scenario Outline: Ingest the record and Index in the Elastic Search with tags
When I ingest records with the <recordFile> with <acl> for a given <kind>
@@ -35,7 +36,7 @@ Feature: Indexing of the documents
Examples:
| kind | recordFile | index | acl | tagKey | tagValue | number |
- | "tenant1:indexer:test-data--Integration:1.0.0" | "index_records_schema_1" | "tenant1-indexer-test-data--integration-1.0.0" | "data.default.viewers@tenant1" | "testtag" | "testvalue" | 5 |
+ | "tenant1:indexer:test-data--Integration:1.0.1" | "index_records_schema_1" | "tenant1-indexer-test-data--integration-1.0.1" | "data.default.viewers@tenant1" | "testtag" | "testvalue" | 5 |
Scenario Outline: Ingest the r3-record with geo-shape and Index in the Elastic Search
When I ingest records with the <recordFile> with <acl> for a given <kind>
diff --git a/testing/indexer-test-core/src/main/resources/testData/index_records_1.schema.json b/testing/indexer-test-core/src/main/resources/testData/index_records_1.schema.json
index b7ecdbc656441b9db8a04ec27484493d0b40a73a..b2a02ea146ec5896262020ddf8c7a014bfdb9056 100644
--- a/testing/indexer-test-core/src/main/resources/testData/index_records_1.schema.json
+++ b/testing/indexer-test-core/src/main/resources/testData/index_records_1.schema.json
@@ -6,7 +6,7 @@
"entityType": "test-data--Integration",
"schemaVersionMajor": "1",
"schemaVersionMinor": "0",
- "schemaVersionPatch": "0"
+ "schemaVersionPatch": "1"
},
"status": "DEVELOPMENT"
},
diff --git a/testing/indexer-test-core/src/main/resources/testData/index_records_2.schema.json b/testing/indexer-test-core/src/main/resources/testData/index_records_2.schema.json
index e67e19502e0283bdf977b03ff5a59dd27883cfa0..e5ee1cb04eedeafed586fa3baa9664ba1d57d2cb 100644
--- a/testing/indexer-test-core/src/main/resources/testData/index_records_2.schema.json
+++ b/testing/indexer-test-core/src/main/resources/testData/index_records_2.schema.json
@@ -6,7 +6,7 @@
"entityType": "test-data--Integration",
"schemaVersionMajor": "2",
"schemaVersionMinor": "0",
- "schemaVersionPatch": "0"
+ "schemaVersionPatch": "1"
},
"status": "DEVELOPMENT"
},
diff --git a/testing/indexer-test-core/src/main/resources/testData/index_records_3.schema.json b/testing/indexer-test-core/src/main/resources/testData/index_records_3.schema.json
index 87bc4b647523c6ac0ab810853a94c3f0dba54976..cb41544608a800f73b61de0ea449cd5a1b12e537 100644
--- a/testing/indexer-test-core/src/main/resources/testData/index_records_3.schema.json
+++ b/testing/indexer-test-core/src/main/resources/testData/index_records_3.schema.json
@@ -6,7 +6,7 @@
"entityType": "test-data--Integration",
"schemaVersionMajor": "3",
"schemaVersionMinor": "0",
- "schemaVersionPatch": "0"
+ "schemaVersionPatch": "1"
},
"status": "DEVELOPMENT"
},
diff --git a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
index ab965b04ed67538ee2e97d80537afd2d87b81839..0409e9a21c866f5e3e81aad5af72ae01e56b88c0 100644
--- a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
+++ b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -42,9 +42,13 @@ public class Steps extends SchemaServiceRecordSteps {
}
@Then("^I should get the elastic \"(.*?)\" for the \"([^\"]*)\" and \"([^\"]*)\" in the Elastic Search$")
- public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String type, String index)
- throws Throwable {
- super.i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(expectedMapping, type, index);
+ public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String kind, String index) throws Throwable {
+ super.i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(expectedMapping, kind, index);
+ }
+
+ @Then("^I can validate indexed meta attributes for the \"([^\"]*)\" and given \"([^\"]*)\"$")
+ public void i_can_validate_indexed_meta_attributes(String index, String kind) throws Throwable {
+ super.i_can_validate_indexed_attributes(index, kind);
}
@Then("^I should get the (\\d+) documents for the \"([^\"]*)\" in the Elastic Search with out \"(.*?)\"$")
diff --git a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
index 8caf85cacd30b7f146c13d0cb58cd156cbf9fc9a..3ec2f0033a5cabb1eba90983c1fc3c4692094e53 100644
--- a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
+++ b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -59,8 +59,13 @@ public class Steps extends SchemaServiceRecordSteps {
}
@Then("^I should get the elastic \"(.*?)\" for the \"([^\"]*)\" and \"([^\"]*)\" in the Elastic Search$")
- public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String type, String index) throws Throwable {
- super.i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(expectedMapping, type, index);
+ public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String kind, String index) throws Throwable {
+ super.i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(expectedMapping, kind, index);
+ }
+
+ @Then("^I can validate indexed meta attributes for the \"([^\"]*)\" and given \"([^\"]*)\"$")
+ public void i_can_validate_indexed_meta_attributes(String index, String kind) throws Throwable {
+ super.i_can_validate_indexed_attributes(index, kind);
}
@Then("^I should get the (\\d+) documents for the \"([^\"]*)\" in the Elastic Search with out \"(.*?)\"$")