From f7538fa77b549082fbd4acc8e6a87a3f7c6d8e8b Mon Sep 17 00:00:00 2001
From: Rustam_Lotsmanenko <Rustam_Lotsmanenko@epam.com>
Date: Mon, 28 Dec 2020 18:06:45 +0400
Subject: [PATCH] GONRG-1438 fix int tests
---
.../service/GeometryConversionService.java | 5 +-
.../osdu/common/CleanupIndiciesSteps.java | 193 ++++++++++++++++++
.../opengroup/osdu/common/RecordSteps.java | 6 +-
.../org/opengroup/osdu/util/ElasticUtils.java | 2 +-
.../resources/testData/index_records_1.json | 20 +-
.../resources/testData/index_records_2.json | 20 +-
.../main/resources/testData/records_1.mapping | 3 +-
.../main/resources/testData/records_2.mapping | 3 +-
.../main/resources/testData/records_3.mapping | 3 +-
.../step_definitions/index/record/Steps.java | 4 +-
10 files changed, 225 insertions(+), 34 deletions(-)
create mode 100644 testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/CleanupIndiciesSteps.java
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/GeometryConversionService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/GeometryConversionService.java
index 83aadeadf..91384e267 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/GeometryConversionService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/GeometryConversionService.java
@@ -15,6 +15,7 @@
package org.opengroup.osdu.indexer.service;
import com.google.gson.internal.LinkedTreeMap;
+import java.util.Optional;
import org.opengroup.osdu.core.common.Constants;
import org.springframework.stereotype.Service;
import org.springframework.web.context.annotation.RequestScope;
@@ -76,10 +77,10 @@ public class GeometryConversionService {
try {
Map<String, Double> position = new HashMap<>();
- double lon = new Double(DECIMAL_FORMAT.format(positionMap.get("longitude")));
+ double lon = new Double(DECIMAL_FORMAT.format(Optional.ofNullable(positionMap.get("longitude")).orElse(positionMap.get("lon"))));
if (lon > 180 && lon < -180)
throw new IllegalArgumentException("'longitude' value is out of the range [-180, 180]");
- double lat = new Double(DECIMAL_FORMAT.format(positionMap.get("latitude")));
+ double lat = new Double(DECIMAL_FORMAT.format(Optional.ofNullable(positionMap.get("latitude")).orElse(positionMap.get("lat"))));
if (lat > 90 && lat < -90)
throw new IllegalArgumentException("'latitude' value is out of the range [-90, 90]");
position.put("lon", lon);
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/CleanupIndiciesSteps.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/CleanupIndiciesSteps.java
new file mode 100644
index 000000000..d15a6668d
--- /dev/null
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/CleanupIndiciesSteps.java
@@ -0,0 +1,193 @@
+/*
+ Copyright 2020 Google LLC
+ Copyright 2020 EPAM Systems, Inc
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+package org.opengroup.osdu.common;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.opengroup.osdu.util.Config.getEntitlementsDomain;
+import static org.opengroup.osdu.util.Config.getIndexerBaseURL;
+import static org.opengroup.osdu.util.Config.getStorageBaseURL;
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import com.sun.jersey.api.client.ClientResponse;
+import cucumber.api.DataTable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import javax.ws.rs.HttpMethod;
+import lombok.extern.java.Log;
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.core.common.model.entitlements.Acl;
+import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
+import org.opengroup.osdu.models.Setup;
+import org.opengroup.osdu.models.TestIndex;
+import org.opengroup.osdu.util.FileHandler;
+import org.opengroup.osdu.util.HTTPClient;
+import org.opengroup.osdu.util.ElasticUtils;
+
+@Log
+public class CleanupIndiciesSteps extends TestsBase {
+ private final String timeStamp = String.valueOf(System.currentTimeMillis());
+ private Map<String, TestIndex> inputIndexMap = new HashMap<>();
+ private List<Map<String, Object>> records;
+ private boolean shutDownHookAdded = false;
+ private final Map<String, String> headers = httpClient.getCommonHeader();
+
+ public CleanupIndiciesSteps(HTTPClient httpClient) {
+ super(httpClient);
+ }
+
+ public CleanupIndiciesSteps(HTTPClient httpClient, ElasticUtils elasticUtils) {
+ super(httpClient, elasticUtils);
+ }
+
+
+ public void theSchemaIsCreatedWithTheFollowingKind(DataTable dataTable) {
+ List<Setup> inputList = dataTable.asList(Setup.class);
+ for (Setup input : inputList) {
+ TestIndex testIndex = getTextIndex();
+ testIndex.setHttpClient(httpClient);
+ testIndex.setIndex(generateActualName(input.getIndex(), timeStamp));
+ testIndex.setKind(generateActualName(input.getKind(), timeStamp));
+ testIndex.setSchemaFile(input.getSchemaFile());
+ inputIndexMap.put(testIndex.getKind(), testIndex);
+ }
+
+ if (!shutDownHookAdded) {
+ shutDownHookAdded = true;
+ for (Map.Entry<String, TestIndex> kind : inputIndexMap.entrySet()) {
+ kind.getValue().setupSchema();
+ }
+ }
+ }
+
+ public void iIngestRecordsWithTheforAGiven(String record, String dataGroup, String kind) {
+ String actualKind = generateActualName(kind, timeStamp);
+ try {
+ String fileContent = FileHandler.readFile(String.format("%s.%s", record, "json"));
+ records = new Gson().fromJson(fileContent, new TypeToken<List<Map<String, Object>>>() {}.getType());
+
+ for (Map<String, Object> testRecord : records) {
+ testRecord.put("id", generateActualName(testRecord.get("id").toString(), timeStamp));
+ testRecord.put("kind", actualKind);
+ testRecord.put("legal", generateLegalTag());
+ String[] x_acl = {generateActualName(dataGroup,timeStamp)+"."+getEntitlementsDomain()};
+ Acl acl = Acl.builder().viewers(x_acl).owners(x_acl).build();
+ testRecord.put("acl", acl);
+ }
+ String payLoad = new Gson().toJson(records);
+ ClientResponse clientResponse = httpClient.send(HttpMethod.PUT, getStorageBaseURL() + "records", payLoad, headers, httpClient.getAccessToken());
+ assertEquals(201, clientResponse.getStatus());
+ } catch (Exception ex) {
+ throw new AssertionError(ex.getMessage());
+ }
+ }
+
+ public void iCheckThatTheIndexForHasBeenCreated(String kind) throws IOException, InterruptedException {
+ assertTrue(isNewIndexCreated(generateActualName(kind, timeStamp)));
+ }
+
+ public void iShouldDeleteTheRecordsForICreatedEarlier() {
+ List<Map<String, Object>> deletedRecords = new ArrayList<>();
+ if (records != null && !records.isEmpty()) {
+ for (Map<String, Object> testRecord : records) {
+ String id = testRecord.get("id").toString();
+ ClientResponse clientResponse = httpClient.send(HttpMethod.DELETE, getStorageBaseURL()
+ + "records/" + id, null, headers, httpClient.getAccessToken());
+ if (clientResponse.getStatus() == 204) {
+ deletedRecords.add(testRecord);
+ log.info("Deleted the records with id " + id);
+ }
+ }
+ assertEquals(records.size(), deletedRecords.size());
+ }
+ }
+
+ public void iShouldDeleteTheSchemaForICreatedEarlier(String kind) {
+ ClientResponse response = httpClient.send(HttpMethod.DELETE,
+ String.format("%sschemas%s", getStorageBaseURL(), "/" + generateActualName(kind, timeStamp)),null,
+ headers, httpClient.getAccessToken());
+ assertEquals(HttpStatus.SC_NO_CONTENT, response.getStatus());
+ }
+
+ public void iShouldCheckThetTheIndexforHasNotBeenDeleted(String kind) throws IOException, InterruptedException {
+ assertTrue(isNewIndexExist(generateActualName(kind, timeStamp)));
+ }
+
+ public void iShouldToRunCleanupOfIndexesForAnd(String kind, String message) {
+
+ String url = getIndexerBaseURL() + "index-cleanup";
+ log.info("URL: " + url);
+ ClientResponse response = httpClient.send(HttpMethod.POST, url,
+ convertMessageIntoJson(kind, message), headers, httpClient.getAccessToken());
+ assertEquals(HttpStatus.SC_OK, response.getStatus());
+ }
+
+ public void iShouldCheckThatTheIndexForHasBeenDeleted(String kind) throws IOException, InterruptedException {
+ assertFalse(isNewIndexExist(generateActualName(kind, timeStamp)));
+ }
+
+ private String convertMessageIntoJson(String kind, String message) {
+ String actualKind = generateActualName(kind, timeStamp);
+ RecordChangedMessages recordChangedMessages = (new Gson()).fromJson(String.format(message,
+ actualKind, actualKind, timeStamp), RecordChangedMessages.class);
+ return new Gson().toJson(recordChangedMessages);
+ }
+
+ private boolean isNewIndexExist(String index) throws IOException {
+ return elasticUtils.isIndexExist(index.replace(":", "-"));
+ }
+
+ private boolean isNewIndexCreated(String index) throws IOException, InterruptedException {
+ int iterator;
+ boolean indexExist = false;
+
+ // index.refresh_interval is set to default 30s, wait for 40s initially
+ Thread.sleep(40000);
+
+ for (iterator = 0; iterator < 20; iterator++) {
+ indexExist = elasticUtils.isIndexExist(index.replace(":", "-"));
+ if (indexExist) {
+ break;
+ } else {
+ Thread.sleep(5000);
+ }
+ if ((iterator + 1) % 5 == 0) {
+ elasticUtils.refreshIndex(index.replace(":", "-"));
+ }
+ }
+ if (iterator >= 20) {
+ fail(String.format("index not created after waiting for %s seconds", ((40000 + iterator * 5000) / 1000)));
+ }
+ return indexExist;
+ }
+
+ @Override
+ protected String getApi() {
+ return null;
+ }
+
+ @Override
+ protected String getHttpMethod() {
+ return null;
+ }
+}
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
index c040e0d09..2baffdbc4 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
@@ -103,7 +103,7 @@ public class RecordSteps extends TestsBase {
}
String payLoad = new Gson().toJson(records);
ClientResponse clientResponse = httpClient.send(HttpMethod.PUT, getStorageBaseURL() + "records", payLoad, headers, httpClient.getAccessToken());
- //assertEquals(201, clientResponse.getStatus());
+ assertEquals(201, clientResponse.getStatus());
} catch (Exception ex) {
throw new AssertionError(ex.getMessage());
}
@@ -112,7 +112,7 @@ public class RecordSteps extends TestsBase {
public void i_should_get_the_documents_for_the_in_the_Elastic_Search(int expectedCount, String index) throws Throwable {
index = generateActualName(index, timeStamp);
long numOfIndexedDocuments = createIndex(index);
- //assertEquals(expectedCount, numOfIndexedDocuments);
+ assertEquals(expectedCount, numOfIndexedDocuments);
}
public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String type, String index) throws Throwable {
@@ -120,7 +120,7 @@ public class RecordSteps extends TestsBase {
Map<String, MappingMetadata> elasticMapping = elasticUtils.getMapping(index);
assertNotNull(elasticMapping);
- MappingMetadata typeMapping = elasticMapping.get(type);
+ MappingMetadata typeMapping = elasticMapping.get(index);
Map<String, Object> mapping = typeMapping.sourceAsMap();
assertNotNull(mapping);
assertTrue(areJsonEqual(expectedMapping, mapping.toString()));
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
index 43a8d1b9c..207a95478 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
@@ -319,7 +319,7 @@ public class ElasticUtils {
new BasicHeader("Authorization", String.format("Basic %s", Base64.getEncoder().encodeToString(usernameAndPassword.getBytes()))),
};
- if (true) {
+ if ("https".equals(scheme) && true) {
log.warning("Elastic client connection uses TrustSelfSignedStrategy()");
SSLContext sslContext = createSSLContext();
builder.setHttpClientConfigCallback(httpClientBuilder ->
diff --git a/testing/indexer-test-core/src/main/resources/testData/index_records_1.json b/testing/indexer-test-core/src/main/resources/testData/index_records_1.json
index 0d9f3c17e..dca7c54fe 100644
--- a/testing/indexer-test-core/src/main/resources/testData/index_records_1.json
+++ b/testing/indexer-test-core/src/main/resources/testData/index_records_1.json
@@ -4,8 +4,8 @@
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
- "latitude":32.406402588,
- "longitude":-86.565592762
+ "lat":32.406402588,
+ "lon":-86.565592762
},
"Basin": "Houston",
"County": "Harris",
@@ -27,8 +27,8 @@
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
- "latitude":32.406402588,
- "longitude":-86.565592762
+ "lat":32.406402588,
+ "lon":-86.565592762
},
"Basin": "Houston",
"County": "Harris",
@@ -50,8 +50,8 @@
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
- "latitude":32.406402588,
- "longitude":-86.565592762
+ "lat":32.406402588,
+ "lon":-86.565592762
},
"Basin": "Houston",
"County": "Harris",
@@ -72,8 +72,8 @@
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
- "latitude":32.406402588,
- "longitude":-86.565592762
+ "lat":32.406402588,
+ "lon":-86.565592762
},
"Basin": "Houston",
"County": "Harris",
@@ -94,8 +94,8 @@
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
- "latitude":32.406402588,
- "longitude":-86.565592762
+ "lat":32.406402588,
+ "lon":-86.565592762
},
"Basin": "Houston",
"County": "Harris",
diff --git a/testing/indexer-test-core/src/main/resources/testData/index_records_2.json b/testing/indexer-test-core/src/main/resources/testData/index_records_2.json
index 9c35fcd6e..721bf64f9 100644
--- a/testing/indexer-test-core/src/main/resources/testData/index_records_2.json
+++ b/testing/indexer-test-core/src/main/resources/testData/index_records_2.json
@@ -4,8 +4,8 @@
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
- "latitude":32.406402588,
- "longitude":-86.565592762
+ "lat":32.406402588,
+ "lon":-86.565592762
},
"Basin": "Houston",
"County": "Harris",
@@ -26,8 +26,8 @@
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
- "latitude":32.406402588,
- "longitude":-86.565592762
+ "lat":32.406402588,
+ "lon":-86.565592762
},
"Basin": "Houston",
"County": "Harris",
@@ -48,8 +48,8 @@
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
- "latitude":32.406402588,
- "longitude":-86.565592762
+ "lat":32.406402588,
+ "lon":-86.565592762
},
"Basin": "Houston",
"County": "Harris",
@@ -70,8 +70,8 @@
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
- "latitude":32.406402588,
- "longitude":-86.565592762
+ "lat":32.406402588,
+ "lon":-86.565592762
},
"Basin": "Houston",
"County": "Harris",
@@ -92,8 +92,8 @@
"data": {
"Field": 1234,
"Location": {
- "latitude":"BA1",
- "longitude":-86.565592762
+ "lat":"BA1",
+ "lon":-86.565592762
},
"Basin": 789,
"County": 0.99,
diff --git a/testing/indexer-test-core/src/main/resources/testData/records_1.mapping b/testing/indexer-test-core/src/main/resources/testData/records_1.mapping
index 1d2be2d85..311cb573d 100644
--- a/testing/indexer-test-core/src/main/resources/testData/records_1.mapping
+++ b/testing/indexer-test-core/src/main/resources/testData/records_1.mapping
@@ -1,5 +1,5 @@
{
- "well": {
+"dynamic":false,
"properties": {
"id": {
"type": "keyword"
@@ -89,5 +89,4 @@
}
}
}
- }
}
\ No newline at end of file
diff --git a/testing/indexer-test-core/src/main/resources/testData/records_2.mapping b/testing/indexer-test-core/src/main/resources/testData/records_2.mapping
index 16703041f..362ad7722 100644
--- a/testing/indexer-test-core/src/main/resources/testData/records_2.mapping
+++ b/testing/indexer-test-core/src/main/resources/testData/records_2.mapping
@@ -1,5 +1,5 @@
{
- "well": {
+"dynamic":false,
"properties": {
"id": {
"type": "keyword"
@@ -86,5 +86,4 @@
}
}
}
- }
}
\ No newline at end of file
diff --git a/testing/indexer-test-core/src/main/resources/testData/records_3.mapping b/testing/indexer-test-core/src/main/resources/testData/records_3.mapping
index e13d948c8..3823c22a9 100644
--- a/testing/indexer-test-core/src/main/resources/testData/records_3.mapping
+++ b/testing/indexer-test-core/src/main/resources/testData/records_3.mapping
@@ -1,5 +1,5 @@
{
- "well": {
+"dynamic":false,
"properties": {
"id": {
"type": "keyword"
@@ -97,5 +97,4 @@
}
}
}
- }
}
\ No newline at end of file
diff --git a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
index b759e1ebd..348474e77 100644
--- a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
+++ b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -39,10 +39,10 @@ public class Steps extends RecordSteps {
super.i_should_get_the_documents_for_the_in_the_Elastic_Search(expectedCount, index);
}
- /* @Then("^I should get the elastic \"(.*?)\" for the \"([^\"]*)\" and \"([^\"]*)\" in the Elastic Search$")
+ @Then("^I should get the elastic \"(.*?)\" for the \"([^\"]*)\" and \"([^\"]*)\" in the Elastic Search$")
public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String type, String index) throws Throwable {
super.i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(expectedMapping, type, index);
- }*/
+ }
@Then("^I should get the (\\d+) documents for the \"([^\"]*)\" in the Elastic Search with out \"(.*?)\"$")
public void iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(int expectedCount, String index, String skippedAttributes) throws Throwable {
--
GitLab