diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/CleanupIndiciesSteps.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/CleanupIndiciesSteps.java
new file mode 100644
index 0000000000000000000000000000000000000000..d6324b425e2a51c858d97f513fee8a58b2f25062
--- /dev/null
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/CleanupIndiciesSteps.java
@@ -0,0 +1,182 @@
+/* Copyright 2017-2019, Schlumberger
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.*/
+
+package org.opengroup.osdu.common;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.opengroup.osdu.util.Config.getEntitlementsDomain;
+import static org.opengroup.osdu.util.Config.getIndexerBaseURL;
+import static org.opengroup.osdu.util.Config.getStorageBaseURL;
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import com.sun.jersey.api.client.ClientResponse;
+import cucumber.api.DataTable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import javax.ws.rs.HttpMethod;
+import lombok.extern.java.Log;
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.core.common.model.entitlements.Acl;
+import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
+import org.opengroup.osdu.models.Setup;
+import org.opengroup.osdu.models.TestIndex;
+import org.opengroup.osdu.util.FileHandler;
+import org.opengroup.osdu.util.HTTPClient;
+
+@Log
+public class CleanupIndiciesSteps extends TestsBase {
+ private final String timeStamp = String.valueOf(System.currentTimeMillis());
+ private Map<String, TestIndex> inputIndexMap = new HashMap<>();
+ private List<Map<String, Object>> records;
+ private boolean shutDownHookAdded = false;
+ private final Map<String, String> headers = httpClient.getCommonHeader();
+
+ public CleanupIndiciesSteps(HTTPClient httpClient) {
+ super(httpClient);
+ }
+
+ public void the_schema_is_created_with_the_following_kind(DataTable dataTable) {
+ List<Setup> inputList = dataTable.asList(Setup.class);
+ for (Setup input : inputList) {
+ TestIndex testIndex = getTextIndex();
+ testIndex.setHttpClient(httpClient);
+ testIndex.setIndex(generateActualName(input.getIndex(), timeStamp));
+ testIndex.setKind(generateActualName(input.getKind(), timeStamp));
+ testIndex.setSchemaFile(input.getSchemaFile());
+ inputIndexMap.put(testIndex.getKind(), testIndex);
+ }
+
+ if (!shutDownHookAdded) {
+ shutDownHookAdded = true;
+ for (Map.Entry<String, TestIndex> kind : inputIndexMap.entrySet()) {
+ kind.getValue().setupSchema();
+ }
+ }
+ }
+
+ public void i_ingest_records_with_the_for_a_given(String record, String dataGroup, String kind) {
+ String actualKind = generateActualName(kind, timeStamp);
+ try {
+ String fileContent = FileHandler.readFile(String.format("%s.%s", record, "json"));
+ records = new Gson().fromJson(fileContent, new TypeToken<List<Map<String, Object>>>() {}.getType());
+
+ for (Map<String, Object> testRecord : records) {
+ testRecord.put("id", generateActualName(testRecord.get("id").toString(), timeStamp));
+ testRecord.put("kind", actualKind);
+ testRecord.put("legal", generateLegalTag());
+ String[] x_acl = {generateActualName(dataGroup,timeStamp)+"."+getEntitlementsDomain()};
+ Acl acl = Acl.builder().viewers(x_acl).owners(x_acl).build();
+ testRecord.put("acl", acl);
+ }
+ String payLoad = new Gson().toJson(records);
+ ClientResponse clientResponse = httpClient.send(HttpMethod.PUT, getStorageBaseURL() + "records", payLoad, headers, httpClient.getAccessToken());
+ assertEquals(201, clientResponse.getStatus());
+ } catch (Exception ex) {
+ throw new AssertionError(ex.getMessage());
+ }
+ }
+
+ public void i_check_that_the_index_for_has_been_created(String kind) throws IOException, InterruptedException {
+ assertTrue(isNewIndexCreated(generateActualName(kind, timeStamp)));
+ }
+
+ public void i_should_delete_the_records_for_i_created_earlier() {
+ List<Map<String, Object>> deletedRecords = new ArrayList<>();
+ if (records != null && !records.isEmpty()) {
+ for (Map<String, Object> testRecord : records) {
+ String id = testRecord.get("id").toString();
+ ClientResponse clientResponse = httpClient.send(HttpMethod.DELETE, getStorageBaseURL()
+ + "records/" + id, null, headers, httpClient.getAccessToken());
+ if (clientResponse.getStatus() == 204) {
+ deletedRecords.add(testRecord);
+ log.info("Deleted the records with id " + id);
+ }
+ }
+ assertEquals(records.size(), deletedRecords.size());
+ }
+ }
+
+ public void i_should_delete_the_schema_for_i_created_earlier(String kind) {
+ ClientResponse response = httpClient.send(HttpMethod.DELETE,
+ String.format("%sschemas%s", getStorageBaseURL(), "/" + generateActualName(kind, timeStamp)),null,
+ headers, httpClient.getAccessToken());
+ assertEquals(HttpStatus.SC_NO_CONTENT, response.getStatus());
+ }
+
+ public void i_should_check_that_the_index_for_has_not_been_deleted(String kind) throws IOException, InterruptedException {
+ assertTrue(isNewIndexExist(generateActualName(kind, timeStamp)));
+ }
+
+ public void i_should_to_run_cleanup_of_indexes_for_and(String kind, String message) {
+
+ ClientResponse response = httpClient.send(HttpMethod.POST, String.format("%sindex-cleanup", getIndexerBaseURL()),
+ convertMessageIntoJson(kind, message), headers, httpClient.getAccessToken());
+ assertEquals(HttpStatus.SC_OK, response.getStatus());
+ }
+
+ public void i_should_check_that_the_index_for_has_been_deleted(String kind) throws IOException, InterruptedException {
+ assertFalse(isNewIndexExist(generateActualName(kind, timeStamp)));
+ }
+
+ private String convertMessageIntoJson(String kind, String message) {
+ String actualKind = generateActualName(kind, timeStamp);
+ RecordChangedMessages recordChangedMessages = (new Gson()).fromJson(String.format(message,
+ actualKind, actualKind, timeStamp), RecordChangedMessages.class);
+ return new Gson().toJson(recordChangedMessages);
+ }
+
+ private boolean isNewIndexExist(String index) throws IOException {
+ return elasticUtils.isIndexExist(index.replace(":", "-"));
+ }
+
+ private boolean isNewIndexCreated(String index) throws IOException, InterruptedException {
+ int iterator;
+ boolean indexExist = false;
+
+ // index.refresh_interval is set to default 30s, wait for 40s initially
+ Thread.sleep(40000);
+
+ for (iterator = 0; iterator < 20; iterator++) {
+ indexExist = elasticUtils.isIndexExist(index.replace(":", "-"));
+ if (indexExist) {
+ break;
+ } else {
+ Thread.sleep(5000);
+ }
+ if ((iterator + 1) % 5 == 0) {
+ elasticUtils.refreshIndex(index.replace(":", "-"));
+ }
+ }
+ if (iterator >= 20) {
+ fail(String.format("index not created after waiting for %s seconds", ((40000 + iterator * 5000) / 1000)));
+ }
+ return indexExist;
+ }
+
+ @Override
+ protected String getApi() {
+ return null;
+ }
+
+ @Override
+ protected String getHttpMethod() {
+ return null;
+ }
+}
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
index 4922e8498e5cebe4d59fcc2ac2ba08d8ed43c217..f6d7ac6ad5596d81e20203997cfcdb542ded04c6 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
@@ -11,6 +11,7 @@ import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
+import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
@@ -297,5 +298,17 @@ public class ElasticUtils {
return builder;
}
-
+ public boolean isIndexExist(String index) throws IOException {
+ boolean exists = false;
+ try {
+ try (RestHighLevelClient client = this.createClient(username, password, host)) {
+ GetIndexRequest request = new GetIndexRequest();
+ request.indices(index);
+ exists = client.indices().exists(request, RequestOptions.DEFAULT);
+ }
+ } catch (ElasticsearchStatusException e) {
+ log.log(Level.INFO, String.format("Error getting index: %s status", index));
+ }
+ return exists;
+ }
}
\ No newline at end of file
diff --git a/testing/indexer-test-core/src/main/resources/features/indexcleanup/IndexCleanup.feature b/testing/indexer-test-core/src/main/resources/features/indexcleanup/IndexCleanup.feature
new file mode 100644
index 0000000000000000000000000000000000000000..610baacea59ae288bf4fa42fe5728f6ec759afcc
--- /dev/null
+++ b/testing/indexer-test-core/src/main/resources/features/indexcleanup/IndexCleanup.feature
@@ -0,0 +1,20 @@
+Feature: Indexing of the documents
+ This feature deals to check for index deletion after schema deletion.
+
+ Background:
+ Given the schema is created with the following kind
+ | kind | index | schemaFile |
+ | tenant1:testindex<timestamp>:well:1.0.0 | tenant1-testindex<timestamp>-well-1.0.0 | index_records_1 |
+
+ Scenario Outline: Index creation and deletion in the Elastic Search
+ When I ingest records with the <recordFile> with <acl> for a given <kind>
+ Then I check that the index for <kind> has been created
+ Then I should delete the records I created earlier
+ Then I should delete the schema for <kind> I created earlier
+ Then I should check that the index for <kind> has not been deleted
+ Then I should to run cleanup of indexes for <kind> and <message>
+ Then I should check that the index for <kind> has been deleted
+
+ Examples:
+ | kind | recordFile | acl | message |
+ | "tenant1:testindex<timestamp>:well:1.0.0" | "index_records_1" | "data.default.viewers@tenant1" | "{"data":"[{\"id\":\"%s-d9033ae1-fb15-496c-9ba0-880fd1d2b2cf\",\"kind\":\"%s\",\"op\":\"purge_schema\"}]","attributes":{"account-id":"opendes","correlation-id":"b5a281bd-f59d-4db2-9939-b2d85036fc7e"},"messageId":"%s","publishTime":"2018-05-08T21:48:56.131Z"}"|
diff --git a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/cleanup/RunTest.java b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/cleanup/RunTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..f01c141fa43122aa39e1632336bcf6530514baff
--- /dev/null
+++ b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/cleanup/RunTest.java
@@ -0,0 +1,13 @@
+package org.opengroup.osdu.step_definitions.index.cleanup;
+
+import cucumber.api.CucumberOptions;
+import cucumber.api.junit.Cucumber;
+import org.junit.runner.RunWith;
+
+@RunWith(Cucumber.class)
+@CucumberOptions(
+ features = "classpath:features/indexcleanup/IndexCleanup.feature",
+ glue = {"classpath:org.opengroup.osdu.step_definitions/index/cleanup"},
+ plugin = {"pretty", "junit:target/cucumber-reports/TEST-indexcleanup.xml"})
+public class RunTest {
+}
\ No newline at end of file
diff --git a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/cleanup/Steps.java b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/cleanup/Steps.java
new file mode 100644
index 0000000000000000000000000000000000000000..017ca4df69ba0bf4229f76bd8d9aca489622ff7c
--- /dev/null
+++ b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/cleanup/Steps.java
@@ -0,0 +1,66 @@
+package org.opengroup.osdu.step_definitions.index.cleanup;
+
+import cucumber.api.DataTable;
+import cucumber.api.Scenario;
+import cucumber.api.java.Before;
+import cucumber.api.java.en.Given;
+import cucumber.api.java.en.Then;
+import cucumber.api.java.en.When;
+import java.io.IOException;
+import lombok.extern.java.Log;
+import org.opengroup.osdu.common.CleanupIndiciesSteps;
+import org.opengroup.osdu.util.GCPHTTPClient;
+
+@Log
+public class Steps extends CleanupIndiciesSteps {
+
+ public Steps() {
+ super(new GCPHTTPClient());
+ }
+
+ @Before
+ public void before(Scenario scenario) {
+ this.scenario = scenario;
+ this.httpClient = new GCPHTTPClient();
+ }
+
+ @Given("^the schema is created with the following kind$")
+ public void the_schema_is_created_with_the_following_kind(DataTable dataTable) {
+ super.the_schema_is_created_with_the_following_kind(dataTable);
+ }
+
+ @When("^I ingest records with the \"(.*?)\" with \"(.*?)\" for a given \"(.*?)\"$")
+ public void i_ingest_records_with_the_for_a_given(String record, String dataGroup, String kind) {
+ super.i_ingest_records_with_the_for_a_given(record, dataGroup, kind);
+ }
+
+ @Then("^I check that the index for \"(.*?)\" has been created$")
+ public void i_check_that_the_index_for_has_been_created(String kind) throws IOException, InterruptedException {
+ super.i_check_that_the_index_for_has_been_created(kind);
+ }
+
+ @Then("^I should delete the records I created earlier$")
+ public void i_should_delete_the_records_for_i_created_earlier() {
+ super.i_should_delete_the_records_for_i_created_earlier();
+ }
+
+ @Then("^I should delete the schema for \"(.*?)\" I created earlier$")
+ public void i_should_delete_the_schema_for_i_created_earlier(String kind) {
+ super.i_should_delete_the_schema_for_i_created_earlier(kind);
+ }
+
+ @Then("^I should check that the index for \"(.*?)\" has not been deleted$")
+ public void i_should_check_that_the_index_for_has_not_been_deleted(String kind) throws IOException, InterruptedException {
+ super.i_should_check_that_the_index_for_has_not_been_deleted(kind);
+ }
+
+ @Then("^I should to run cleanup of indexes for \"(.*?)\" and \"(.*?)\"$")
+ public void i_should_to_run_cleanup_of_indexes_for_and(String kind, String message) {
+ super.i_should_to_run_cleanup_of_indexes_for_and(kind, message);
+ }
+
+ @Then("^I should check that the index for \"(.*?)\" has been deleted$")
+ public void i_should_check_that_the_index_for_has_been_deleted(String kind) throws IOException, InterruptedException {
+ super.i_should_check_that_the_index_for_has_been_deleted(kind);
+ }
+}
\ No newline at end of file