Merge branch 'GCP-add_int_test_for_index_cleanup' into 'master'

GCP add integration test for index cleanup (GONRG-1024)

See merge request !51

testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/CleanupIndiciesSteps.java

+/*
+  Copyright 2020 Google LLC
+  Copyright 2020 EPAM Systems, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+ */
+
+package org.opengroup.osdu.common;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.opengroup.osdu.util.Config.getEntitlementsDomain;
+import static org.opengroup.osdu.util.Config.getIndexerBaseURL;
+import static org.opengroup.osdu.util.Config.getStorageBaseURL;
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import com.sun.jersey.api.client.ClientResponse;
+import cucumber.api.DataTable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import javax.ws.rs.HttpMethod;
+import lombok.extern.java.Log;
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.core.common.model.entitlements.Acl;
+import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
+import org.opengroup.osdu.models.Setup;
+import org.opengroup.osdu.models.TestIndex;
+import org.opengroup.osdu.util.FileHandler;
+import org.opengroup.osdu.util.HTTPClient;
+
+@Log
+public class CleanupIndiciesSteps extends TestsBase {
+  private final String timeStamp = String.valueOf(System.currentTimeMillis());
+  private Map<String, TestIndex> inputIndexMap = new HashMap<>();
+  private List<Map<String, Object>> records;
+  private boolean shutDownHookAdded = false;
+  private final Map<String, String> headers = httpClient.getCommonHeader();
+
+  public CleanupIndiciesSteps(HTTPClient httpClient) {
+    super(httpClient);
+  }
+
+  public void theSchemaIsCreatedWithTheFollowingKind(DataTable dataTable) {
+    List<Setup> inputList = dataTable.asList(Setup.class);
+    for (Setup input : inputList) {
+      TestIndex testIndex = getTextIndex();
+      testIndex.setHttpClient(httpClient);
+      testIndex.setIndex(generateActualName(input.getIndex(), timeStamp));
+      testIndex.setKind(generateActualName(input.getKind(), timeStamp));
+      testIndex.setSchemaFile(input.getSchemaFile());
+      inputIndexMap.put(testIndex.getKind(), testIndex);
+    }
+
+    if (!shutDownHookAdded) {
+      shutDownHookAdded = true;
+      for (Map.Entry<String, TestIndex> kind : inputIndexMap.entrySet()) {
+        kind.getValue().setupSchema();
+      }
+    }
+  }
+
+  public void iIngestRecordsWithTheforAGiven(String record, String dataGroup, String kind) {
+    String actualKind = generateActualName(kind, timeStamp);
+    try {
+      String fileContent = FileHandler.readFile(String.format("%s.%s", record, "json"));
+      records = new Gson().fromJson(fileContent, new TypeToken<List<Map<String, Object>>>() {}.getType());
+
+      for (Map<String, Object> testRecord : records) {
+        testRecord.put("id", generateActualName(testRecord.get("id").toString(), timeStamp));
+        testRecord.put("kind", actualKind);
+        testRecord.put("legal", generateLegalTag());
+        String[] x_acl = {generateActualName(dataGroup,timeStamp)+"."+getEntitlementsDomain()};
+        Acl acl = Acl.builder().viewers(x_acl).owners(x_acl).build();
+        testRecord.put("acl", acl);
+      }
+      String payLoad = new Gson().toJson(records);
+      ClientResponse clientResponse = httpClient.send(HttpMethod.PUT, getStorageBaseURL() + "records", payLoad, headers, httpClient.getAccessToken());
+      assertEquals(201, clientResponse.getStatus());
+    } catch (Exception ex) {
+      throw new AssertionError(ex.getMessage());
+    }
+  }
+
+  public void iCheckThatTheIndexForHasBeenCreated(String kind) throws IOException, InterruptedException {
+    assertTrue(isNewIndexCreated(generateActualName(kind, timeStamp)));
+  }
+
+  public void iShouldDeleteTheRecordsForICreatedEarlier() {
+    List<Map<String, Object>> deletedRecords = new ArrayList<>();
+    if (records != null && !records.isEmpty()) {
+      for (Map<String, Object> testRecord : records) {
+        String id = testRecord.get("id").toString();
+        ClientResponse clientResponse = httpClient.send(HttpMethod.DELETE, getStorageBaseURL()
+            + "records/" + id, null, headers, httpClient.getAccessToken());
+        if (clientResponse.getStatus() == 204) {
+          deletedRecords.add(testRecord);
+          log.info("Deleted the records with id " + id);
+        }
+      }
+      assertEquals(records.size(), deletedRecords.size());
+    }
+  }
+
+  public void iShouldDeleteTheSchemaForICreatedEarlier(String kind) {
+    ClientResponse response = httpClient.send(HttpMethod.DELETE,
+        String.format("%sschemas%s", getStorageBaseURL(), "/" + generateActualName(kind, timeStamp)),null,
+        headers, httpClient.getAccessToken());
+    assertEquals(HttpStatus.SC_NO_CONTENT, response.getStatus());
+  }
+
+  public void iShouldCheckThetTheIndexforHasNotBeenDeleted(String kind) throws IOException, InterruptedException {
+    assertTrue(isNewIndexExist(generateActualName(kind, timeStamp)));
+  }
+
+  public void iShouldToRunCleanupOfIndexesForAnd(String kind, String message) {
+
+    String url = getIndexerBaseURL() + "index-cleanup";
+    log.info("URL: " + url);
+    ClientResponse response = httpClient.send(HttpMethod.POST, url,
+        convertMessageIntoJson(kind, message), headers, httpClient.getAccessToken());
+    assertEquals(HttpStatus.SC_OK, response.getStatus());
+  }
+
+  public void iShouldCheckThatTheIndexForHasBeenDeleted(String kind) throws IOException, InterruptedException {
+    assertFalse(isNewIndexExist(generateActualName(kind, timeStamp)));
+  }
+
+  private String convertMessageIntoJson(String kind, String message) {
+    String actualKind = generateActualName(kind, timeStamp);
+    RecordChangedMessages recordChangedMessages = (new Gson()).fromJson(String.format(message,
+        actualKind, actualKind, timeStamp), RecordChangedMessages.class);
+    return new Gson().toJson(recordChangedMessages);
+  }
+
+  private boolean isNewIndexExist(String index) throws IOException {
+    return elasticUtils.isIndexExist(index.replace(":", "-"));
+  }
+
+  private boolean isNewIndexCreated(String index) throws IOException, InterruptedException {
+    int iterator;
+    boolean indexExist = false;
+
+    // index.refresh_interval is set to default 30s, wait for 40s initially
+    Thread.sleep(40000);
+
+    for (iterator = 0; iterator < 20; iterator++) {
+      indexExist = elasticUtils.isIndexExist(index.replace(":", "-"));
+      if (indexExist) {
+        break;
+      } else {
+        Thread.sleep(5000);
+      }
+      if ((iterator + 1) % 5 == 0) {
+        elasticUtils.refreshIndex(index.replace(":", "-"));
+      }
+    }
+    if (iterator >= 20) {
+      fail(String.format("index not created after waiting for %s seconds", ((40000 + iterator * 5000) / 1000)));
+    }
+    return indexExist;
+  }
+
+  @Override
+  protected String getApi() {
+    return null;
+  }
+
+  @Override
+  protected String getHttpMethod() {
+    return null;
+  }
+}

testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java

+/*
+  Copyright 2020 Google LLC
+  Copyright 2020 EPAM Systems, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+ */
+
 package org.opengroup.osdu.util;
 
 import com.google.gson.Gson;
@@ -11,6 +28,7 @@ import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
+import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
@@ -297,5 +315,24 @@ public class ElasticUtils {
         return builder;
     }
 
+    public boolean isIndexExist(String index) {
+        boolean exists = false;
+        try {
+            exists = createRestClientAndCheckIndexExist(index);
+        } catch (ElasticsearchStatusException e) {
+            log.log(Level.INFO, String.format("Error getting index: %s %s", index, e.getMessage()));
+        }
+        return exists;
+    }
 
+    private boolean createRestClientAndCheckIndexExist(String index) {
+        try (RestHighLevelClient client = this.createClient(username, password, host)) {
+            GetIndexRequest request = new GetIndexRequest();
+            request.indices(index);
+            return client.indices().exists(request, RequestOptions.DEFAULT);
+        } catch (IOException e) {
+            log.log(Level.INFO, String.format("Error getting index: %s %s", index, e.getMessage()));
+        }
+        return false;
+    }
 }
\ No newline at end of file

testing/indexer-test-core/src/main/resources/features/indexcleanup/IndexCleanup.feature

+Feature: Indexing of the documents
+  This feature deals to check for index deletion after schema deletion.
+
+  Background:
+    Given the schema is created with the following kind
+      | kind                                    | index                                   | schemaFile      |
+      | tenant1:testindex<timestamp>:well:1.0.0 | tenant1-testindex<timestamp>-well-1.0.0 | index_records_1 |
+
+  Scenario Outline: Index creation and deletion in the Elastic Search
+    When I ingest records with the <recordFile> with <acl> for a given <kind>
+    Then I check that the index for <kind> has been created
+    Then I should delete the records I created earlier
+    Then I should delete the schema for <kind> I created earlier
+    Then I should check that the index for <kind> has not been deleted
+    Then I should to run cleanup of indexes for <kind> and <message>
+    Then I should check that the index for <kind> has been deleted
+
+    Examples:
+      | kind                                      | recordFile        | acl                            | message                                                                                                                                                                                                                                                            |
+      | "tenant1:testindex<timestamp>:well:1.0.0" | "index_records_1" | "data.default.viewers@tenant1" | "{"data":"[{\"id\":\"%s-d9033ae1-fb15-496c-9ba0-880fd1d2b2cf\",\"kind\":\"%s\",\"op\":\"purge_schema\"}]","attributes":{"account-id":"opendes","correlation-id":"b5a281bd-f59d-4db2-9939-b2d85036fc7e"},"messageId":"%s","publishTime":"2018-05-08T21:48:56.131Z"}"|

testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/cleanup/RunTest.java

+/*
+  Copyright 2020 Google LLC
+  Copyright 2020 EPAM Systems, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+ */
+
+package org.opengroup.osdu.step_definitions.index.cleanup;
+
+import cucumber.api.CucumberOptions;
+import cucumber.api.junit.Cucumber;
+import org.junit.runner.RunWith;
+
+@RunWith(Cucumber.class)
+@CucumberOptions(
+        features = "classpath:features/indexcleanup/IndexCleanup.feature",
+        glue = {"classpath:org.opengroup.osdu.step_definitions/index/cleanup"},
+        plugin = {"pretty", "junit:target/cucumber-reports/TEST-indexcleanup.xml"})
+public class RunTest {
+}
\ No newline at end of file

testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/cleanup/Steps.java

+/*
+  Copyright 2020 Google LLC
+  Copyright 2020 EPAM Systems, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+ */
+
+package org.opengroup.osdu.step_definitions.index.cleanup;
+
+import cucumber.api.DataTable;
+import cucumber.api.Scenario;
+import cucumber.api.java.Before;
+import cucumber.api.java.en.Given;
+import cucumber.api.java.en.Then;
+import cucumber.api.java.en.When;
+import java.io.IOException;
+import lombok.extern.java.Log;
+import org.opengroup.osdu.common.CleanupIndiciesSteps;
+import org.opengroup.osdu.util.GCPHTTPClient;
+
+@Log
+public class Steps extends CleanupIndiciesSteps {
+
+    public Steps() {
+        super(new GCPHTTPClient());
+    }
+
+    @Before
+    public void before(Scenario scenario) {
+        this.scenario = scenario;
+        this.httpClient = new GCPHTTPClient();
+    }
+
+    @Given("^the schema is created with the following kind$")
+    public void theSchemaIsCreatedWithTheFollowingKind(DataTable dataTable) {
+        super.theSchemaIsCreatedWithTheFollowingKind(dataTable);
+    }
+
+    @When("^I ingest records with the \"(.*?)\" with \"(.*?)\" for a given \"(.*?)\"$")
+    public void iIngestRecordsWithTheforAGiven(String record, String dataGroup, String kind) {
+        super.iIngestRecordsWithTheforAGiven(record, dataGroup, kind);
+    }
+
+    @Then("^I check that the index for \"(.*?)\" has been created$")
+    public void iCheckThatTheIndexForHasBeenCreated(String kind) throws IOException, InterruptedException {
+        super.iCheckThatTheIndexForHasBeenCreated(kind);
+    }
+
+    @Then("^I should delete the records I created earlier$")
+    public void iShouldDeleteTheRecordsForICreatedEarlier() {
+        super.iShouldDeleteTheRecordsForICreatedEarlier();
+    }
+
+    @Then("^I should delete the schema for \"(.*?)\" I created earlier$")
+    public void iShouldDeleteTheSchemaForICreatedEarlier(String kind) {
+        super.iShouldDeleteTheSchemaForICreatedEarlier(kind);
+    }
+
+    @Then("^I should check that the index for \"(.*?)\" has not been deleted$")
+    public void iShouldCheckThetTheIndexforHasNotBeenDeleted(String kind) throws IOException, InterruptedException {
+        super.iShouldCheckThetTheIndexforHasNotBeenDeleted(kind);
+    }
+
+    @Then("^I should to run cleanup of indexes for \"(.*?)\" and \"(.*?)\"$")
+    public void iShouldToRunCleanupOfIndexesForAnd(String kind, String message) {
+        super.iShouldToRunCleanupOfIndexesForAnd(kind, message);
+    }
+
+    @Then("^I should check that the index for \"(.*?)\" has been deleted$")
+    public void iShouldCheckThatTheIndexForHasBeenDeleted(String kind) throws IOException, InterruptedException {
+        super.iShouldCheckThatTheIndexForHasBeenDeleted(kind);
+    }
+}
\ No newline at end of file