Commit 503a4372 authored by Alok Joshi's avatar Alok Joshi
Browse files

Merge branch 'update_event' into 'master'

Update kind event support

See merge request !284
parents b379024b b14ee5ea
Pipeline #99576 passed with stages
in 49 minutes and 22 seconds
This diff is collapsed.
......@@ -16,7 +16,7 @@
<properties>
<commons-beanutils.version>1.9.4</commons-beanutils.version>
<osdu.oscorecommon.version>0.13.0</osdu.oscorecommon.version>
<osdu.oscorecommon.version>0.14.0-rc4</osdu.oscorecommon.version>
<spring-security-web.version>5.4.9</spring-security-web.version>
<gson.version>2.8.9</gson.version>
<netty.version>4.1.70.Final</netty.version>
......
......@@ -400,7 +400,7 @@ public class IndexerServiceImpl implements IndexerService {
IndexRequest indexRequest = new IndexRequest(index).id(record.getId()).source(this.gson.toJson(sourceMap), XContentType.JSON);
bulkRequest.add(indexRequest);
} else if (operation == OperationType.update) {
UpdateRequest updateRequest = new UpdateRequest(index, record.getId()).upsert(this.gson.toJson(sourceMap), XContentType.JSON);
UpdateRequest updateRequest = new UpdateRequest(index, "_doc", record.getId()).doc(this.gson.toJson(sourceMap), XContentType.JSON).docAsUpsert(true);
bulkRequest.add(updateRequest);
}
}
......
......@@ -13,7 +13,7 @@
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
<spring-cloud.version>Greenwich.SR2</spring-cloud.version>
<os-core-common.version>0.13.0</os-core-common.version>
<os-core-common.version>0.14.0-rc4</os-core-common.version>
<snakeyaml.version>1.26</snakeyaml.version>
<hibernate-validator.version>6.1.5.Final</hibernate-validator.version>
<jackson.version>2.11.4</jackson.version>
......
......@@ -90,6 +90,11 @@ public class Steps extends SchemaServiceRecordSteps {
super.i_should_get_the_documents_for_the_in_the_Elastic_Search(expectedCount, index);
}
@Then("^I should not get any documents for the \"([^\"]*)\" in the Elastic Search$")
public void i_should_not_get_any_documents_for_the_index_in_the_Elastic_Search(String index) throws Throwable {
super.i_should_not_get_any_documents_for_the_index_in_the_Elastic_Search(index);
}
@Then("^I should get the elastic \"(.*?)\" for the \"([^\"]*)\" and \"([^\"]*)\" in the Elastic Search$")
public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String kind, String index) throws Throwable {
super.i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(expectedMapping, kind, index);
......
......@@ -36,7 +36,7 @@
<cucumber.version>1.2.5</cucumber.version>
<junit.jupiter.version>5.6.0</junit.jupiter.version>
<elasticsearch.version>7.8.1</elasticsearch.version>
<os-core-common.version>0.12.0-rc3</os-core-common.version>
<os-core-common.version>0.14.0-rc4</os-core-common.version>
</properties>
<dependencies>
......
......@@ -58,6 +58,11 @@ public class Steps extends SchemaServiceRecordSteps {
super.i_should_get_the_documents_for_the_in_the_Elastic_Search(expectedCount, index);
}
@Then("^I should not get any documents for the \"([^\"]*)\" in the Elastic Search$")
public void i_should_not_get_any_documents_for_the_index_in_the_Elastic_Search(String index) throws Throwable {
super.i_should_not_get_any_documents_for_the_index_in_the_Elastic_Search(index);
}
@Then("^I should get the elastic \"(.*?)\" for the \"([^\"]*)\" and \"([^\"]*)\" in the Elastic Search$")
public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String kind, String index) throws Throwable {
super.i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(expectedMapping, kind, index);
......
......@@ -17,7 +17,7 @@
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
<cucumber.version>1.2.5</cucumber.version>
<os-core-common.version>0.12.0-rc1</os-core-common.version>
<os-core-common.version>0.14.0-rc4</os-core-common.version>
</properties>
<dependencies>
......
......@@ -134,6 +134,11 @@ public class RecordSteps extends TestsBase {
assertEquals(expectedCount, numOfIndexedDocuments);
}
public void i_should_not_get_any_documents_for_the_index_in_the_Elastic_Search(String index) throws Throwable {
index = generateActualName(index, timeStamp);
getRecordsInIndex(index, 0);
}
public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String kind, String index) throws Throwable {
index = generateActualName(index, timeStamp);
Map<String, MappingMetadata> elasticMapping = elasticUtils.getMapping(index);
......@@ -253,6 +258,32 @@ public class RecordSteps extends TestsBase {
return numOfIndexedDocuments;
}
private long getRecordsInIndex(String index, int expectedCount) throws InterruptedException, IOException {
long numOfIndexedDocuments = 0;
int iterator;
// index.refresh_interval is set to default 30s, wait for 40s initially
Thread.sleep(40000);
for (iterator = 0; iterator < 20; iterator++) {
numOfIndexedDocuments = elasticUtils.fetchRecords(index);
if (expectedCount == numOfIndexedDocuments) {
log.info(String.format("index: %s | attempts: %s | documents acknowledged by elastic: %s", index, iterator, numOfIndexedDocuments));
break;
} else {
log.info(String.format("index: %s | documents acknowledged by elastic: %s", index, numOfIndexedDocuments));
Thread.sleep(5000);
}
if ((iterator + 1) % 5 == 0) elasticUtils.refreshIndex(index);
}
if (iterator >= 20) {
fail(String.format("index not created after waiting for %s seconds", ((40000 + iterator * 5000) / 1000)));
}
return numOfIndexedDocuments;
}
private Boolean areJsonEqual(String firstJson, String secondJson) {
Gson gson = new Gson();
Type mapType = new TypeToken<Map<String, Object>>() {}.getType();
......
......@@ -3,13 +3,15 @@ Feature: Indexing of the documents
Background:
Given the schema is created with the following kind
| kind | index | schemaFile |
| tenant1:indexer:test-data--Integration:1.0.1 | tenant1-indexer-test-data--integration-1.0.1 | index_records_1 |
| tenant1:indexer:test-data--Integration:2.0.1 | tenant1-indexer-test-data--integration-2.0.1 | index_records_2 |
| tenant1:indexer:test-data--Integration:3.0.1 | tenant1-indexer-test-data--integration-3.0.1 | index_records_3 |
| tenant1:wks:master-data--Wellbore:2.0.3 | tenant1-wks-master-data--wellbore-2.0.3 | r3-index_record_wks_master |
| tenant1:wks:ArraysOfObjectsTestCollection:4.0.0 | tenant1-wks-arraysofobjectstestcollection-4.0.0 | r3-index_record_arrayofobjects |
| tenant1:indexer:test-mapping--Sync:2.0.0 | tenant1-indexer-test-mapping--sync-2.0.0 | index_record_sync_mapping |
| kind | index | schemaFile |
| tenant1:indexer:test-data--Integration:1.0.1 | tenant1-indexer-test-data--integration-1.0.1 | index_records_1 |
| tenant1:indexer:test-data--Integration:2.0.1 | tenant1-indexer-test-data--integration-2.0.1 | index_records_2 |
| tenant1:indexer:test-data--Integration:3.0.1 | tenant1-indexer-test-data--integration-3.0.1 | index_records_3 |
| tenant1:wks:master-data--Wellbore:2.0.3 | tenant1-wks-master-data--wellbore-2.0.3 | r3-index_record_wks_master |
| tenant1:wks:ArraysOfObjectsTestCollection:4.0.0 | tenant1-wks-arraysofobjectstestcollection-4.0.0 | r3-index_record_arrayofobjects |
| tenant1:indexer:test-mapping--Sync:2.0.0 | tenant1-indexer-test-mapping--sync-2.0.0 | index_record_sync_mapping |
| tenant1:indexer:test-update-data--Integration:1.0.1 | tenant1-indexer-test-update-data--integration-1.0.1 | index_update_records_kind_v1 |
| tenant1:indexer:test-update-data--Integration:2.0.1 | tenant1-indexer-test-update-data--integration-2.0.1 | index_update_records_kind_v2 |
Scenario Outline: Ingest the record and Index in the Elastic Search
When I ingest records with the <recordFile> with <acl> for a given <kind>
......@@ -64,4 +66,15 @@ Feature: Indexing of the documents
Examples:
| kind | index | recordFile | mappingFile | acl |
| "tenant1:indexer:test-mapping--Sync:2.0.0" | "tenant1-indexer-test-mapping--sync-2.0.0" | "index_record_sync_mapping" | "index_record_sync_mapping" | "data.default.viewers@tenant1" |
\ No newline at end of file
| "tenant1:indexer:test-mapping--Sync:2.0.0" | "tenant1-indexer-test-mapping--sync-2.0.0" | "index_record_sync_mapping" | "index_record_sync_mapping" | "data.default.viewers@tenant1" |
Scenario Outline: Ingest the record and Index in the Elastic Search
When I ingest records with the <recordFile> with <acl> for a given <kind_v1>
Then I should get the 1 documents for the <index_v1> in the Elastic Search
Then I ingest records with the <recordFile> with <acl> for a given <kind_v2>
Then I should get the 1 documents for the <index_v2> in the Elastic Search
Then I should not get any documents for the <index_v1> in the Elastic Search
Examples:
| kind_v1 | index_v1 | recordFile | acl | kind_v2 | index_v2 |
| "tenant1:indexer:test-update-data--Integration:1.0.1" | "tenant1-indexer-test-update-data--integration-1.0.1" | "index_update_records_kind_v1" | "data.default.viewers@tenant1" | "tenant1:indexer:test-update-data--Integration:2.0.1" | "tenant1-indexer-test-update-data--integration-2.0.1" |
\ No newline at end of file
[
{
"id": "tenant1:test:testSchemaUpdate<timestamp>",
"data": {
"message": "test record1"
}
}
]
\ No newline at end of file
{
"schemaInfo": {
"schemaIdentity": {
"authority": "tenant1",
"source": "indexer",
"entityType": "test-update-data--Integration",
"schemaVersionMajor": "1",
"schemaVersionMinor": "0",
"schemaVersionPatch": "1"
},
"status": "DEVELOPMENT"
},
"schema": {
"properties": {
"data": {
"allOf": [
{
"type": "object",
"properties": {
"message": {
"type": "string"
}
}
}
]
}
}
}
}
\ No newline at end of file
[
{
"id": "tenant1:test:testSchemaUpdate<timestamp>",
"data": {
"message": "test record2"
}
}
]
\ No newline at end of file
{
"schemaInfo": {
"schemaIdentity": {
"authority": "tenant1",
"source": "indexer",
"entityType": "test-update-data--Integration",
"schemaVersionMajor": "2",
"schemaVersionMinor": "0",
"schemaVersionPatch": "1"
},
"status": "DEVELOPMENT"
},
"schema": {
"properties": {
"data": {
"allOf": [
{
"type": "object",
"properties": {
"message": {
"type": "string"
}
}
}
]
}
}
}
}
\ No newline at end of file
......@@ -46,6 +46,11 @@ public class Steps extends SchemaServiceRecordSteps {
super.i_should_get_the_documents_for_the_in_the_Elastic_Search(expectedCount, index);
}
@Then("^I should not get any documents for the \"([^\"]*)\" in the Elastic Search$")
public void i_should_not_get_any_documents_for_the_index_in_the_Elastic_Search(String index) throws Throwable {
super.i_should_not_get_any_documents_for_the_index_in_the_Elastic_Search(index);
}
@Then("^I should get the elastic \"(.*?)\" for the \"([^\"]*)\" and \"([^\"]*)\" in the Elastic Search$")
public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String kind, String index) throws Throwable {
super.i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(expectedMapping, kind, index);
......
......@@ -44,7 +44,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
<version>0.12.0-rc1</version>
<version>0.14.0-rc4</version>
</dependency>
<dependency>
......
......@@ -63,6 +63,11 @@ public class Steps extends SchemaServiceRecordSteps {
super.i_should_get_the_documents_for_the_in_the_Elastic_Search(expectedCount, index);
}
@Then("^I should not get any documents for the \"([^\"]*)\" in the Elastic Search$")
public void i_should_not_get_any_documents_for_the_index_in_the_Elastic_Search(String index) throws Throwable {
super.i_should_not_get_any_documents_for_the_index_in_the_Elastic_Search(index);
}
@Then("^I should get the elastic \"(.*?)\" for the \"([^\"]*)\" and \"([^\"]*)\" in the Elastic Search$")
public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String kind, String index) throws Throwable {
super.i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(expectedMapping, kind, index);
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment