diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/GeometryConversionService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/GeometryConversionService.java
index 83aadeadf483f71aca465ed4754713a242588061..91384e26714534864434bb5aa360211febf3fb9d 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/GeometryConversionService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/GeometryConversionService.java
@@ -15,6 +15,7 @@
package org.opengroup.osdu.indexer.service;
import com.google.gson.internal.LinkedTreeMap;
+import java.util.Optional;
import org.opengroup.osdu.core.common.Constants;
import org.springframework.stereotype.Service;
import org.springframework.web.context.annotation.RequestScope;
@@ -76,10 +77,10 @@ public class GeometryConversionService {
try {
Map<String, Double> position = new HashMap<>();
- double lon = new Double(DECIMAL_FORMAT.format(positionMap.get("longitude")));
+ double lon = new Double(DECIMAL_FORMAT.format(Optional.ofNullable(positionMap.get("longitude")).orElse(positionMap.get("lon"))));
if (lon > 180 && lon < -180)
throw new IllegalArgumentException("'longitude' value is out of the range [-180, 180]");
- double lat = new Double(DECIMAL_FORMAT.format(positionMap.get("latitude")));
+ double lat = new Double(DECIMAL_FORMAT.format(Optional.ofNullable(positionMap.get("latitude")).orElse(positionMap.get("lat"))));
if (lat > 90 && lat < -90)
throw new IllegalArgumentException("'latitude' value is out of the range [-90, 90]");
position.put("lon", lon);
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
index 332fcb49608ec808673924efa3add582396223b6..95f1c504f9eca6e03027fd4e14e8389fb8cb61bc 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
@@ -14,42 +14,39 @@
package org.opengroup.osdu.indexer.service;
+import com.google.gson.Gson;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
-
+import javax.inject.Inject;
import org.apache.http.HttpStatus;
import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest;
-import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
-import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData;
-import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.client.indices.GetFieldMappingsRequest;
+import org.elasticsearch.client.indices.GetFieldMappingsResponse;
+import org.elasticsearch.client.indices.PutMappingRequest;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.reindex.BulkByScrollResponse;
import org.elasticsearch.index.reindex.UpdateByQueryRequest;
-
-import com.google.gson.Gson;
-import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.Constants;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.indexer.DEAnalyzerType;
import org.opengroup.osdu.core.common.model.indexer.ElasticType;
-import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
-import org.opengroup.osdu.core.common.search.Preconditions;
import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
import org.opengroup.osdu.core.common.model.indexer.Records;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
+import org.opengroup.osdu.core.common.search.Preconditions;
import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
import org.opengroup.osdu.indexer.util.ElasticClientHandler;
import org.springframework.stereotype.Service;
-import javax.inject.Inject;
@Service
public class IndexerMappingServiceImpl extends MappingServiceImpl implements IndexerMappingService {
@@ -165,7 +162,7 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
private boolean updateMappingToEnableKeywordIndexingForField(RestHighLevelClient client, Set<String> indicesSet, String fieldName) throws IOException {
String[] indices = indicesSet.toArray(new String[indicesSet.size()]);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> indexMappingMap = getIndexFieldMap(new String[]{"data."+fieldName}, client, indices);
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> indexMappingMap = getIndexFieldMap(new String[]{"data."+fieldName}, client, indices);
boolean failure = false;
for (String index : indicesSet) {
if (indexMappingMap.get(index)!=null && updateMappingForAllIndicesOfSameTypeToEnableKeywordIndexingForField(client, index, indexMappingMap.get(index), fieldName)) {
@@ -178,20 +175,18 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
return !failure;
}
- private Map<String, Map<String, Map<String, FieldMappingMetaData>>> getIndexFieldMap(String[] fieldNames, RestHighLevelClient client, String[] indices) throws IOException {
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> indexMappingMap = new HashMap<>();
+ private Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> getIndexFieldMap(String[] fieldNames, RestHighLevelClient client, String[] indices) throws IOException {
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> indexMappingMap = new HashMap<>();
GetFieldMappingsRequest request = new GetFieldMappingsRequest();
request.indices(indices);
request.fields(fieldNames);
try {
GetFieldMappingsResponse response = client.indices().getFieldMapping(request, RequestOptions.DEFAULT);
if (response != null && !response.mappings().isEmpty()) {
- final Map<String, Map<String, Map<String, FieldMappingMetaData>>> mappings = response.mappings();
+ final Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappings = response.mappings();
for (String index : indices) {
- //extract mapping of each index
- final Map<String, Map<String, FieldMappingMetaData>> indexMapping = mappings.get(index);
- if (indexMapping != null && !indexMapping.isEmpty()) {
- indexMappingMap.put(index, indexMapping);
+ if (mappings != null && !mappings.isEmpty()) {
+ indexMappingMap.put(index, mappings);
}
}
}
@@ -203,23 +198,22 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
}
}
- private boolean updateMappingForAllIndicesOfSameTypeToEnableKeywordIndexingForField(RestHighLevelClient client, String index, Map<String, Map<String, FieldMappingMetaData>> indexMapping, String fieldName) throws IOException {
+ private boolean updateMappingForAllIndicesOfSameTypeToEnableKeywordIndexingForField(RestHighLevelClient client, String index, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> indexMapping, String fieldName) throws IOException {
PutMappingRequest request = new PutMappingRequest(index);
String type = indexMapping.keySet().iterator().next();
if(type.isEmpty()) {
log.error(String.format("Could not find type of the mappings for index: %s.", index));
return false;
}
-
- request.type(type);
- request.timeout(REQUEST_TIMEOUT);
- Map<String, FieldMappingMetaData> metaData = indexMapping.get(type);
+
+ request.setTimeout(REQUEST_TIMEOUT);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> metaData = indexMapping.get(type);
if(metaData==null || metaData.get("data." + fieldName)==null) {
log.error(String.format("Could not find field: %s in the mapping of index: %s.", fieldName, index));
return false;
}
- FieldMappingMetaData fieldMetaData = metaData.get("data." + fieldName);
+ GetFieldMappingsResponse.FieldMappingMetadata fieldMetaData = metaData.get("data." + fieldName);
Map<String, Object> source = fieldMetaData.sourceAsMap();
if(!source.containsKey(fieldName)){
log.error(String.format("Could not find field: %s in the mapping of index: %s.", fieldName, index));
@@ -319,9 +313,8 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
try {
if (mapping != null) {
PutMappingRequest request = new PutMappingRequest(index);
- request.type(type);
request.source(mapping, XContentType.JSON);
- request.timeout(REQUEST_TIMEOUT);
+ request.setTimeout(REQUEST_TIMEOUT);
AcknowledgedResponse response = client.indices().putMapping(request, RequestOptions.DEFAULT);
return response.isAcknowledged();
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
index e77ef8f0908bbf9e718562ad4322aa9284756f73..6bd98690beaf036862e0f8fdfe5aea2d35b29a3e 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
@@ -372,10 +372,10 @@ public class IndexerServiceImpl implements IndexerService {
String index = this.elasticIndexNameResolver.getIndexNameFromKind(record.getKind());
if (operation == OperationType.create) {
- IndexRequest indexRequest = new IndexRequest(index, record.getType(), record.getId()).source(this.gson.toJson(sourceMap), XContentType.JSON);
+ IndexRequest indexRequest = new IndexRequest(index).id(record.getId()).source(this.gson.toJson(sourceMap), XContentType.JSON);
bulkRequest.add(indexRequest);
} else if (operation == OperationType.update) {
- UpdateRequest updateRequest = new UpdateRequest(index, record.getType(), record.getId()).upsert(this.gson.toJson(sourceMap), XContentType.JSON);
+ UpdateRequest updateRequest = new UpdateRequest(index, record.getId()).upsert(this.gson.toJson(sourceMap), XContentType.JSON);
bulkRequest.add(updateRequest);
}
}
@@ -389,13 +389,10 @@ public class IndexerServiceImpl implements IndexerService {
for (Map.Entry<String, List<String>> record : deleteRecordMap.entrySet()) {
- String[] kindParts = record.getKey().split(":");
- String type = kindParts[2];
-
String index = this.elasticIndexNameResolver.getIndexNameFromKind(record.getKey());
for (String id : record.getValue()) {
- DeleteRequest deleteRequest = new DeleteRequest(index, type, id);
+ DeleteRequest deleteRequest = new DeleteRequest(index, id);
bulkRequest.add(deleteRequest);
}
}
@@ -410,8 +407,6 @@ public class IndexerServiceImpl implements IndexerService {
List<String> failureRecordIds = new LinkedList<>();
if (bulkRequest.numberOfActions() == 0) return failureRecordIds;
-
-
try {
BulkResponse bulkResponse = restClient.bulk(bulkRequest, RequestOptions.DEFAULT);
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java
index 0f13dde455b6eb97b8cef6de07d4fde14fbd77cb..ec68f6cccbdc88ea0f8a605c90f301fba53e56bc 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java
@@ -22,15 +22,15 @@ import org.apache.http.HttpStatus;
import org.apache.http.util.EntityUtils;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchStatusException;
-import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
-import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
-import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.client.indices.CreateIndexRequest;
+import org.elasticsearch.client.indices.CreateIndexResponse;
+import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentType;
@@ -93,11 +93,10 @@ public class IndicesServiceImpl implements IndicesService {
request.settings(settings != null ? settings : DEFAULT_INDEX_SETTINGS);
if (mapping != null) {
String mappingJsonString = new Gson().toJson(mapping, Map.class);
- request.mapping(type, mappingJsonString, XContentType.JSON);
+ request.mapping(mappingJsonString,XContentType.JSON);
}
- request.timeout(REQUEST_TIMEOUT);
+ request.setTimeout(REQUEST_TIMEOUT);
CreateIndexResponse response = client.indices().create(request, RequestOptions.DEFAULT);
-
// cache the index status
boolean indexStatus = response.isAcknowledged() && response.isShardsAcknowledged();
if (indexStatus) this.indicesExistCache.put(index, true);
@@ -130,8 +129,7 @@ public class IndicesServiceImpl implements IndicesService {
//In case the format of cache changes then clean the cache
this.indicesExistCache.delete(index);
}
- GetIndexRequest request = new GetIndexRequest();
- request.indices(index);
+ GetIndexRequest request = new GetIndexRequest(index);
boolean exists = client.indices().exists(request, RequestOptions.DEFAULT);
if (exists) this.indicesExistCache.put(index, true);
return exists;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/MappingServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/MappingServiceImpl.java
index 6b52cc19b4a8e9f56339ce41bebfc79db0aa0c54..3f067f74ef2e7cf7762f196d428fd7b4f889fed2 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/MappingServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/MappingServiceImpl.java
@@ -15,10 +15,10 @@
package org.opengroup.osdu.indexer.service;
import org.apache.http.HttpStatus;
-import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
-import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.client.indices.GetMappingsRequest;
+import org.elasticsearch.client.indices.GetMappingsResponse;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
import org.opengroup.osdu.core.common.search.IndicesService;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/ElasticClientHandler.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/ElasticClientHandler.java
index 7c40de07302ec2c533f400f5e3b329dabe2060f4..e7737e73c614f87857dabbaa16255c833a352efe 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/ElasticClientHandler.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/ElasticClientHandler.java
@@ -96,7 +96,6 @@ public class ElasticClientHandler {
builder.setRequestConfigCallback(
requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(REST_CLIENT_CONNECT_TIMEOUT)
.setSocketTimeout(REST_CLIENT_SOCKET_TIMEOUT));
- builder.setMaxRetryTimeoutMillis(REST_CLIENT_RETRY_TIMEOUT);
Header[] defaultHeaders = new Header[]{
new BasicHeader("client.transport.nodes_sampler_interval", "30s"),
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/parser/GeoShapeParser.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/parser/GeoShapeParser.java
index 08634523cf91d72e1f7916b9727b67ae69613b68..9532e498b0e1fee048c163f34baa953d702e6ace 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/parser/GeoShapeParser.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/parser/GeoShapeParser.java
@@ -42,7 +42,7 @@ public class GeoShapeParser {
try {
// use elasticsearch's ShapeParser to validate shape
- ShapeBuilder<?, ?> shapeBuilder = getShapeBuilderFromObject(geoShapeObject);
+ ShapeBuilder shapeBuilder = getShapeBuilderFromObject(geoShapeObject);
Shape shape = shapeBuilder.buildS4J();
if (shape == null) {
throw new IllegalArgumentException("unable to parse shape");
@@ -54,7 +54,7 @@ public class GeoShapeParser {
}
}
- private ShapeBuilder<?, ?> getShapeBuilderFromObject(Map<String, Object> object) throws IOException {
+ private ShapeBuilder getShapeBuilderFromObject(Map<String, Object> object) throws IOException {
XContentBuilder contentBuilder = JsonXContent.contentBuilder().value(object);
XContentParser parser = JsonXContent.jsonXContent.createParser(
diff --git a/pom.xml b/pom.xml
index a85a2d468a67dec3503db2bae8cb53a4be71f1aa..af837eb4e68da25419542c76461d32943e68cda3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -27,7 +27,7 @@
<jackson.version>2.11.2</jackson.version>
<tomcat-embed-core.version>9.0.37</tomcat-embed-core.version>
<common-codec.version>1.14</common-codec.version>
- <elasticsearch.version>6.8.1</elasticsearch.version>
+ <elasticsearch.version>7.8.1</elasticsearch.version>
<netty.version>4.1.51.Final</netty.version>
<reactor-netty.version>0.8.20.RELEASE</reactor-netty.version>
<woodstox-core.version>6.2.3</woodstox-core.version>
diff --git a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexerMappingServiceTest.java b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexerMappingServiceTest.java
index a83f2c7dab97d19426ad1e1800cf48dada775d66..3fc9f04f711391c860528a1af7c9b34a01eb1cf9 100644
--- a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexerMappingServiceTest.java
+++ b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexerMappingServiceTest.java
@@ -18,7 +18,6 @@ import org.apache.http.StatusLine;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest;
import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
-import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
@@ -156,13 +155,13 @@ public class IndexerMappingServiceTest {
builder.field("any field", new HashMap());
builder.endObject();
BytesReference bytesReference = BytesReference.bytes(builder);
- FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
- Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
mappingBuilder.put("any index 1", mapBuilder);
mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
mapping.put("indices 1", mappingBuilder);
when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
@@ -190,13 +189,13 @@ public class IndexerMappingServiceTest {
builder.field("any field", new HashMap());
builder.endObject();
BytesReference bytesReference = BytesReference.bytes(builder);
- FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
- Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
mappingBuilder.put("any index 1", mapBuilder);
mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
mapping.put("indices 1", mappingBuilder);
when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
@@ -224,13 +223,13 @@ public class IndexerMappingServiceTest {
builder.field("any field", new HashMap());
builder.endObject();
BytesReference bytesReference = BytesReference.bytes(builder);
- FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
- Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
mappingBuilder.put("any index 1", mapBuilder);
mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
mapping.put("indices 1", mappingBuilder);
when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
@@ -259,13 +258,13 @@ public class IndexerMappingServiceTest {
builder.field("any field", new HashMap());
builder.endObject();
BytesReference bytesReference = BytesReference.bytes(builder);
- FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
- Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
mappingBuilder.put("any index 1", mapBuilder);
mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
mapping.put("indices 1", mappingBuilder);
when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
@@ -293,13 +292,13 @@ public class IndexerMappingServiceTest {
builder.field("any field", new HashMap());
builder.endObject();
BytesReference bytesReference = BytesReference.bytes(builder);
- FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
- Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
mappingBuilder.put("any index 1", mapBuilder);
mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
mapping.put("indices 1", mappingBuilder);
when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
diff --git a/provider/indexer-gcp/README.md b/provider/indexer-gcp/README.md
index 373972ee9e4561bab282bb0e84e80954293724f0..b4b676171db6246e08ab4819d63580600dd7eb01 100644
--- a/provider/indexer-gcp/README.md
+++ b/provider/indexer-gcp/README.md
@@ -147,6 +147,8 @@ You will need to have the following environment variables defined.
| `SEARCH_INTEGRATION_TESTER` | `********` | Service account for API calls. Note: this user must have entitlements configured already | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
| `SEARCH_HOST` | ex `http://localhost:8080/api/search/v2/` | Endpoint of search service | no | - |
| `STORAGE_HOST` | ex `http://os-storage-dot-opendes.appspot.com/api/storage/v2/schemas` | Storage API endpoint | Storage Host | no | output of infrastructure deployment |
+| `SECURITY_HTTPS_CERTIFICATE_TRUST` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment |
+
**Entitlements configuration for integration accounts**
diff --git a/provider/indexer-gcp/pom.xml b/provider/indexer-gcp/pom.xml
index 2b91fa1322e2b54eb010e91ef1af984f70de7ed7..80d1d46e25c23f830a051c867c9c9e8653199ce5 100644
--- a/provider/indexer-gcp/pom.xml
+++ b/provider/indexer-gcp/pom.xml
@@ -76,17 +76,14 @@
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
- <version>6.6.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
- <version>6.6.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
- <version>6.6.2</version>
</dependency>
<!-- Test Dependencies -->
diff --git a/provider/indexer-gcp/src/main/resources/application-dev.properties b/provider/indexer-gcp/src/main/resources/application-dev.properties
index 7ec1744f095c0ee84ad53abafae545ce7ad7bdfc..6c49ae23e07fddb7668a52d645d70268579ce9a9 100644
--- a/provider/indexer-gcp/src/main/resources/application-dev.properties
+++ b/provider/indexer-gcp/src/main/resources/application-dev.properties
@@ -1,21 +1,21 @@
google-cloud-project=opendes
-indexer-host=os-indexer-dot-opendes.appspot.com
-STORAGE_HOSTNAME=os-storage-dot-opendes.appspot.com
+indexer-host=indexer-jvmvia5dea-uc.a.run.app
+STORAGE_HOSTNAME=storage-jvmvia5dea-uc.a.run.app
-storage-schema-host=https://os-storage-dot-opendes.appspot.com/api/storage/v2/schemas
-storage-query-record-host=https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records
-storage-query-record-for-conversion-host=https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records:batch
+storage-schema-host=http://127.0.0.1:8081/api/storage/v2/schemas
+storage-query-record-host=http://127.0.0.1:8081/api/storage/v2/query/records
+storage-query-record-for-conversion-host=https://storage-jvmvia5dea-uc.a.run.app/api/storage/v2/query/records:batch
storage-records-batch-size=20
-indexer-queue-host=https://os-indexer-queue-dot-opendes.appspot.com/_dps/task-handlers/enqueue
+indexer-queue-host=https://indexer-queue-jvmvia5dea-uc.a.run.app/_dps/task-handlers/enqueue
-AUTHORIZE_API=https://entitlements-dot-opendes.appspot.com/entitlements/v1
-LEGALTAG_API=https://os-legal-dot-opendes.appspot.com/api/legal/v1
-CRS_API=https://crs-converter-gae-dot-opendes.appspot.com/api/crs/v1
+AUTHORIZE_API=https://os-entitlements-gcp-jvmvia5dea-uc.a.run.app/entitlements/v1
+LEGALTAG_API=https://os-legal-jvmvia5dea-uc.a.run.app/api/legal/v1
+CRS_API=https://crs-converter-jvmvia5dea-uc.a.run.app/api/crs/v1
## use below values for gcp: opendes
-REDIS_GROUP_HOST=10.0.16.28
-redis-search-host=10.0.16.20
+REDIS_GROUP_HOST=127.0.0.1
+redis-search-host=127.0.0.1
-google-audiences=245464679631-ktfdfpl147m1mjpbutl00b3cmffissgq.apps.googleusercontent.com
\ No newline at end of file
+google-audiences=519000754840-09v7ssbpku7sevhvbtq3bdoi70es16p5.apps.googleusercontent.com
\ No newline at end of file
diff --git a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java
index ac48c1ac516308b045206b2c33595fc10914ce13..019bae350d179f8d038f31883284af4e2d6e439b 100644
--- a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java
+++ b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java
@@ -1,24 +1,10 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
package org.opengroup.osdu.indexer.service;
-import com.google.gson.Gson;
import org.apache.http.StatusLine;
import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest;
import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
-import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.*;
@@ -30,14 +16,14 @@ import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
+import org.mockito.ArgumentMatchers;
import org.mockito.InjectMocks;
import org.mockito.Mock;
-import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
+import org.opengroup.osdu.indexer.service.IndexerMappingServiceImpl;
import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.indexer.util.ElasticClientHandler;
-import org.opengroup.osdu.indexer.util.TypeMapper;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.springframework.test.context.junit4.SpringRunner;
@@ -50,21 +36,18 @@ import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
-import static org.mockito.MockitoAnnotations.initMocks;
import static org.powermock.api.mockito.PowerMockito.when;
@Ignore
@RunWith(SpringRunner.class)
-@PrepareForTest({ RestHighLevelClient.class, IndicesClient.class})
+@PrepareForTest({ RestHighLevelClient.class, IndicesClient.class })
public class IndexerMappingServiceTest {
private final String kind = "tenant:test:test:1.0.0";
private final String index = "tenant-test-test-1.0.0";
private final String type = "test";
- private final String mappingValid = "{\"dynamic\":false,\"properties\":{\"data\":{\"properties\":{\"Msg\":{\"type\":\"text\",\"analyzer\":\"de_indexer_analyzer\",\"search_analyzer\":\"de_search_analyzer\"},\"Location\":{\"type\":\"geo_point\"}}},\"id\":{\"type\":\"keyword\"},\"acl\":{\"properties\":{\"viewers\":{\"type\":\"keyword\"},\"owners\":{\"type\":\"keyword\"}}}}}";
+ private final String mappingValid = "{\"dynamic\":false,\"properties\":{\"data\":{\"properties\":{\"Location\":{\"type\":\"geo_point\"}}},\"id\":{\"type\":\"keyword\"}}}";
- @Mock
- private IndexerConfigurationProperties configurationProperties;
@Mock
private RestClient restClient;
@Mock
@@ -76,7 +59,7 @@ public class IndexerMappingServiceTest {
private IndexerMappingServiceImpl sut;
@Mock
- private ElasticClientHandler elasticClientHandler;
+ private ElasticClientHandler elasticClientHandler;
@InjectMocks
private RestHighLevelClient restHighLevelClient;
@@ -91,14 +74,10 @@ public class IndexerMappingServiceTest {
@Before
public void setup() throws IOException {
- initMocks(this);
- when(configurationProperties.isPreDemo()).thenReturn(true);
Map<String, String> dataMapping = new HashMap<>();
dataMapping.put("Location", "geo_point");
- dataMapping.put("Msg", "text");
Map<String, Object> metaMapping = new HashMap<>();
metaMapping.put(RecordMetaAttribute.ID.getValue(), "keyword");
- metaMapping.put(RecordMetaAttribute.ACL.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.ACL));
this.indexSchema = IndexSchema.builder().kind(kind).type(type).dataSchema(dataMapping).metaSchema(metaMapping)
.build();
@@ -106,7 +85,7 @@ public class IndexerMappingServiceTest {
this.restHighLevelClient = PowerMockito.mock(RestHighLevelClient.class);
when(this.restHighLevelClient.getLowLevelClient()).thenReturn(restClient);
- when(this.restClient.performRequest(any())).thenReturn(response);
+ when(this.restClient.performRequest(ArgumentMatchers.any())).thenReturn(response);
when(this.response.getStatusLine()).thenReturn(statusLine);
when(this.statusLine.getStatusCode()).thenReturn(200);
}
@@ -125,7 +104,7 @@ public class IndexerMappingServiceTest {
public void should_returnValidMapping_givenTrueMerge_createMappingTest() {
try {
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true);
assertEquals(this.mappingValid, mapping);
@@ -138,10 +117,10 @@ public class IndexerMappingServiceTest {
public void should_returnValidMapping_givenExistType_createMappingTest() {
try {
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
IndexerMappingServiceImpl indexerMappingServiceLocal = PowerMockito.spy(new IndexerMappingServiceImpl());
- doReturn(false).when(indexerMappingServiceLocal).isTypeExist(any(), any(), any());
+ doReturn(false).when(indexerMappingServiceLocal).isTypeExist(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any());
String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true);
assertEquals(this.mappingValid, mapping);
} catch (Exception e) {
@@ -156,24 +135,24 @@ public class IndexerMappingServiceTest {
indices.add("indices 1");
GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- when(this.indicesClient.getFieldMapping(any(), any())).thenReturn(getFieldMappingsResponse);
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();
builder.field("any field", new HashMap());
builder.endObject();
BytesReference bytesReference = BytesReference.bytes(builder);
- FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
- Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
mappingBuilder.put("any index 1", mapBuilder);
mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
mapping.put("indices 1", mappingBuilder);
when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
- doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
BulkByScrollResponse response = mock(BulkByScrollResponse.class);
- doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class));
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
@@ -190,24 +169,24 @@ public class IndexerMappingServiceTest {
indices.add("invalid 1");
GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- when(this.indicesClient.getFieldMapping(any(), any())).thenReturn(getFieldMappingsResponse);
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();
builder.field("any field", new HashMap());
builder.endObject();
BytesReference bytesReference = BytesReference.bytes(builder);
- FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
- Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
mappingBuilder.put("any index 1", mapBuilder);
mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
mapping.put("indices 1", mappingBuilder);
when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
- doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
BulkByScrollResponse response = mock(BulkByScrollResponse.class);
- doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class));
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
@@ -224,24 +203,24 @@ public class IndexerMappingServiceTest {
indices.add("indices 1");
GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- when(this.indicesClient.getFieldMapping(any(), any())).thenReturn(getFieldMappingsResponse);
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();
builder.field("any field", new HashMap());
builder.endObject();
BytesReference bytesReference = BytesReference.bytes(builder);
- FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
- Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
mappingBuilder.put("any index 1", mapBuilder);
mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
mapping.put("indices 1", mappingBuilder);
when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
- doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
BulkByScrollResponse response = mock(BulkByScrollResponse.class);
- doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class));
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field invalid");
@@ -259,24 +238,24 @@ public class IndexerMappingServiceTest {
indices.add("indices Invalid");
GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- when(this.indicesClient.getFieldMapping(any(), any())).thenThrow(new ElasticsearchException(""));
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenThrow(new ElasticsearchException(""));
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();
builder.field("any field", new HashMap());
builder.endObject();
BytesReference bytesReference = BytesReference.bytes(builder);
- FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
- Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
mappingBuilder.put("any index 1", mapBuilder);
mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
mapping.put("indices 1", mappingBuilder);
when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
- doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
BulkByScrollResponse response = mock(BulkByScrollResponse.class);
- doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class));
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
@@ -293,26 +272,26 @@ public class IndexerMappingServiceTest {
indices.add("indices Invalid");
GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- when(this.indicesClient.getFieldMapping(any(), any())).thenReturn(getFieldMappingsResponse);
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();
builder.field("any field", new HashMap());
builder.endObject();
BytesReference bytesReference = BytesReference.bytes(builder);
- FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
- Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
mappingBuilder.put("any index 1", mapBuilder);
mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
mapping.put("indices 1", mappingBuilder);
when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
- doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
BulkByScrollResponse response = mock(BulkByScrollResponse.class);
- doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class));
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
- when(this.indicesClient.putMapping(any(), any(RequestOptions.class))).thenThrow(new ElasticsearchException(""));
+ when(this.indicesClient.putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class))).thenThrow(new ElasticsearchException(""));
when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
} catch (AppException e) {
@@ -321,18 +300,4 @@ public class IndexerMappingServiceTest {
fail("Should not throw this exception" + e.getMessage());
}
}
-
-
- @Test
- public void should_returnDocumentMapping_givenValidIndexSchema() {
-
- try {
- Map<String, Object> documentMapping = this.sut.getIndexMappingFromRecordSchema(this.indexSchema);
- String documentMappingJson = new Gson().toJson(documentMapping);
- assertEquals(this.mappingValid, documentMappingJson);
-
- } catch (Exception e) {
- fail("Should not throw this exception" + e.getMessage());
- }
- }
}
diff --git a/testing/indexer-test-aws/pom.xml b/testing/indexer-test-aws/pom.xml
index e31c28157f62c209cf96815321dd459f1d8e7419..813616b929575b8095aecf0276d31f7add422fdd 100644
--- a/testing/indexer-test-aws/pom.xml
+++ b/testing/indexer-test-aws/pom.xml
@@ -113,23 +113,6 @@
<version>27.1-jre</version>
</dependency>
- <!--Elasticsearch-->
- <dependency>
- <groupId>org.elasticsearch</groupId>
- <artifactId>elasticsearch</artifactId>
- <version>6.6.2</version>
- </dependency>
- <dependency>
- <groupId>org.elasticsearch.client</groupId>
- <artifactId>elasticsearch-rest-client</artifactId>
- <version>6.6.2</version>
- </dependency>
- <dependency>
- <groupId>org.elasticsearch.client</groupId>
- <artifactId>elasticsearch-rest-high-level-client</artifactId>
- <version>6.6.2</version>
- </dependency>
-
<!--Logging-->
<dependency>
<groupId>org.apache.logging.log4j</groupId>
diff --git a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/ElasticUtilsAws.java b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/ElasticUtilsAws.java
index 28d3646d655232dc147719945b02282748e7d498..fa172dddc65900906a66e98f6a3fe1b58982bb12 100644
--- a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/ElasticUtilsAws.java
+++ b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/ElasticUtilsAws.java
@@ -32,7 +32,6 @@ public class ElasticUtilsAws extends ElasticUtils {
RestClientBuilder builder = RestClient.builder(new HttpHost(host, port, "https"));
builder.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(REST_CLIENT_CONNECT_TIMEOUT)
.setSocketTimeout(REST_CLIENT_SOCKET_TIMEOUT));
- builder.setMaxRetryTimeoutMillis(REST_CLIENT_RETRY_TIMEOUT);
builder.setHttpClientConfigCallback(httpAsyncClientBuilder -> httpAsyncClientBuilder.setSSLHostnameVerifier((s, sslSession) -> true));
Header[] defaultHeaders = new Header[]{
diff --git a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/JwtTokenUtil.java b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/JwtTokenUtil.java
index 3ff2b4e3166783fb6d44b1fd5ccba19055a237b4..46a3fce10ea09e5a212865453043d4bb33a05d6d 100644
--- a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/JwtTokenUtil.java
+++ b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/JwtTokenUtil.java
@@ -25,4 +25,4 @@ class JwtTokenUtil {
AWSCognitoClient client = new AWSCognitoClient(clientId, authFlow, user, password);
return client.getTokenForUserWithAccess();
}
-}
+}
\ No newline at end of file
diff --git a/testing/indexer-test-core/pom.xml b/testing/indexer-test-core/pom.xml
index d61bc6a5bb57e7d89dbeb5b8a13a4db14719daaf..f9ad2b8c1e37648f97eb09813a4789f914bf75af 100644
--- a/testing/indexer-test-core/pom.xml
+++ b/testing/indexer-test-core/pom.xml
@@ -99,17 +99,17 @@
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
- <version>6.6.2</version>
+ <version>7.8.1</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
- <version>6.6.2</version>
+ <version>7.8.1</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
- <version>6.6.2</version>
+ <version>7.8.1</version>
</dependency>
<!--Logging-->
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
index 3a013e1530b093e9c779890c5a12b230257028b4..55b4413c0f997ef6f675b71385fbb430e7c4dceb 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
@@ -7,8 +7,7 @@ import com.google.gson.reflect.TypeToken;
import com.sun.jersey.api.client.ClientResponse;
import cucumber.api.DataTable;
import lombok.extern.java.Log;
-import org.elasticsearch.cluster.metadata.MappingMetaData;
-import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.cluster.metadata.MappingMetadata;
import org.opengroup.osdu.core.common.model.entitlements.Acl;
import org.opengroup.osdu.models.Setup;
import org.opengroup.osdu.models.TestIndex;
@@ -125,10 +124,10 @@ public class RecordSteps extends TestsBase {
public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String type, String index) throws Throwable {
index = generateActualName(index, timeStamp);
- ImmutableOpenMap<String, MappingMetaData> elasticMapping = elasticUtils.getMapping(index);
+ Map<String, MappingMetadata> elasticMapping = elasticUtils.getMapping(index);
assertNotNull(elasticMapping);
- MappingMetaData typeMapping = elasticMapping.get(type);
+ MappingMetadata typeMapping = elasticMapping.get(index);
Map<String, Object> mapping = typeMapping.sourceAsMap();
assertNotNull(mapping);
assertTrue(areJsonEqual(expectedMapping, mapping.toString()));
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/Config.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/Config.java
index 3d97833f4f9edeaf3fb5d530d3136b9640ab7ad6..1c16fb2c504355dd28c80c6337ee4a0cb954e625 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/Config.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/Config.java
@@ -24,6 +24,8 @@ public class Config {
private static final String DEFAULT_ENTITLEMENTS_DOMAIN = "";
private static final String SCHEMA_PATH = "/api/schema-service/v1";
+ private static final String DEFAULT_SECURITY_HTTPS_CERTIFICATE_TRUST = "false";
+
public static int getPort() {
return Integer.parseInt(getEnvironmentVariableOrDefaultValue("ELASTIC_PORT", String.valueOf(PORT)));
@@ -105,6 +107,12 @@ public class Config {
return getEnvironmentVariableOrDefaultValue("AWS_COGNITO_AUTH_PARAMS_PASSWORD", "");
}
+ public static boolean isSecurityHttpsCertificateTrust() {
+ return Boolean.parseBoolean(
+ getEnvironmentVariableOrDefaultValue("SECURITY_HTTPS_CERTIFICATE_TRUST",
+ DEFAULT_SECURITY_HTTPS_CERTIFICATE_TRUST));
+ }
+
private static String getEnvironmentVariableOrDefaultValue(String key, String defaultValue) {
String environmentVariable = getEnvironmentVariable(key);
if (environmentVariable == null) {
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
index 4612f07cf3efd111c178ef607c6d1af2789d60dd..a63b339ce9421eadddd7d5f5515d33833aef5ff3 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
@@ -18,19 +18,22 @@
package org.opengroup.osdu.util;
import com.google.gson.Gson;
+import java.security.KeyManagementException;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import javax.net.ssl.SSLContext;
import lombok.extern.java.Log;
import org.apache.http.Header;
import org.apache.http.HttpHost;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
+import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
+import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
import org.apache.http.message.BasicHeader;
+import org.apache.http.ssl.SSLContextBuilder;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchStatusException;
-import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
-import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
-import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
+import org.elasticsearch.client.indices.CloseIndexRequest;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
-import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
-import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
-import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
import org.elasticsearch.action.bulk.BulkRequest;
@@ -40,7 +43,12 @@ import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.*;
-import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.client.indices.CreateIndexRequest;
+import org.elasticsearch.client.indices.CreateIndexResponse;
+import org.elasticsearch.client.indices.GetIndexRequest;
+import org.elasticsearch.client.indices.GetMappingsRequest;
+import org.elasticsearch.client.indices.GetMappingsResponse;
+import org.elasticsearch.cluster.metadata.MappingMetadata;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
@@ -91,9 +99,9 @@ public class ElasticUtils {
// creating index + add mapping to the index
log.info("Creating index with name: " + index);
- CreateIndexRequest request = new CreateIndexRequest(index, settings);
+ CreateIndexRequest request = new CreateIndexRequest(index).settings(settings);
request.source("{\"mappings\":" + mapping + "}", XContentType.JSON);
- request.timeout(REQUEST_TIMEOUT);
+ request.setTimeout(REQUEST_TIMEOUT);
CreateIndexResponse response = client.indices().create(request, RequestOptions.DEFAULT);
//wait for ack
@@ -201,7 +209,7 @@ public class ElasticUtils {
try (RestHighLevelClient client = this.createClient(username, password, host)) {
SearchRequest request = new SearchRequest(index);
SearchResponse searchResponse = client.search(request, RequestOptions.DEFAULT);
- return searchResponse.getHits().totalHits;
+ return searchResponse.getHits().getTotalHits().value;
}
} catch (ElasticsearchStatusException e) {
log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage()));
@@ -218,7 +226,7 @@ public class ElasticUtils {
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
- return searchResponse.getHits().totalHits;
+ return searchResponse.getHits().getTotalHits().value;
}
} catch (ElasticsearchStatusException e) {
log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage()));
@@ -226,13 +234,13 @@ public class ElasticUtils {
}
}
- public ImmutableOpenMap<String, MappingMetaData> getMapping(String index) throws IOException {
+ public Map<String, MappingMetadata> getMapping(String index) throws IOException {
try (RestHighLevelClient client = this.createClient(username, password, host)) {
GetMappingsRequest request = new GetMappingsRequest();
request.indices(index);
GetMappingsResponse response = client.indices().getMapping(request, RequestOptions.DEFAULT);
- ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> allMappings = response.mappings();
- return allMappings.get(index);
+ Map<String, MappingMetadata> mappings = response.mappings();
+ return mappings;
}
}
@@ -250,7 +258,8 @@ public class ElasticUtils {
private boolean closeIndex(RestHighLevelClient client, String index) {
try {
CloseIndexRequest request = new CloseIndexRequest(index);
- request.timeout(TimeValue.timeValueMinutes(1));
+ request.setTimeout(TimeValue.timeValueMinutes(1));
+ request.timeout();
AcknowledgedResponse closeIndexResponse = client.indices().close(request, RequestOptions.DEFAULT);
return closeIndexResponse.isAcknowledged();
} catch (ElasticsearchException | IOException exception) {
@@ -267,7 +276,7 @@ public class ElasticUtils {
for (Map<String, Object> record : testRecords) {
String id = (String) record.get("id");
Map<String, Object> mapData = gson.fromJson(gson.toJson(record), Map.class);
- IndexRequest indexRequest = new IndexRequest(index, kind.split(":")[2], id).source(mapData);
+ IndexRequest indexRequest = new IndexRequest(index).id(id).source(mapData);
dataList.add(indexRequest);
}
} catch (Exception e) {
@@ -299,7 +308,6 @@ public class ElasticUtils {
RestClientBuilder builder = RestClient.builder(new HttpHost(host, port, scheme));
builder.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(REST_CLIENT_CONNECT_TIMEOUT)
.setSocketTimeout(REST_CLIENT_SOCKET_TIMEOUT));
- builder.setMaxRetryTimeoutMillis(REST_CLIENT_RETRY_TIMEOUT);
Header[] defaultHeaders = new Header[]{
new BasicHeader("client.transport.nodes_sampler_interval", "30s"),
@@ -311,10 +319,43 @@ public class ElasticUtils {
new BasicHeader("Authorization", String.format("Basic %s", Base64.getEncoder().encodeToString(usernameAndPassword.getBytes()))),
};
+ boolean isSecurityHttpsCertificateTrust = Config.isSecurityHttpsCertificateTrust();
+ log.info(String.format(
+ "Elastic client connection uses protocolScheme = %s with a flag "
+ + "'security.https.certificate.trust' = %s",
+ scheme, isSecurityHttpsCertificateTrust));
+
+ if ("https".equals(scheme) && isSecurityHttpsCertificateTrust) {
+ log.warning("Elastic client connection uses TrustSelfSignedStrategy()");
+ SSLContext sslContext = createSSLContext();
+ builder.setHttpClientConfigCallback(httpClientBuilder ->
+ {
+ HttpAsyncClientBuilder httpAsyncClientBuilder = httpClientBuilder.setSSLContext(sslContext)
+ .setSSLHostnameVerifier(
+ NoopHostnameVerifier.INSTANCE);
+ return httpAsyncClientBuilder;
+ });
+ }
+
builder.setDefaultHeaders(defaultHeaders);
return builder;
}
+ private SSLContext createSSLContext() {
+ SSLContextBuilder sslContextBuilder = new SSLContextBuilder();
+ try {
+ sslContextBuilder.loadTrustMaterial(null, new TrustSelfSignedStrategy());
+ return sslContextBuilder.build();
+ } catch (NoSuchAlgorithmException e) {
+ log.severe(e.getMessage());
+ } catch (KeyStoreException e) {
+ log.severe(e.getMessage());
+ } catch (KeyManagementException e) {
+ log.severe(e.getMessage());
+ }
+ return null;
+ }
+
public boolean isIndexExist(String index) {
boolean exists = false;
try {
@@ -327,8 +368,7 @@ public class ElasticUtils {
private boolean createRestClientAndCheckIndexExist(String index) {
try (RestHighLevelClient client = this.createClient(username, password, host)) {
- GetIndexRequest request = new GetIndexRequest();
- request.indices(index);
+ GetIndexRequest request = new GetIndexRequest(index);
return client.indices().exists(request, RequestOptions.DEFAULT);
} catch (IOException e) {
log.log(Level.INFO, String.format("Error getting index: %s %s", index, e.getMessage()));
diff --git a/testing/indexer-test-core/src/main/resources/features/indexcleanup/IndexCleanup.feature b/testing/indexer-test-core/src/main/resources/features/indexcleanup/IndexCleanup.feature
deleted file mode 100644
index 610baacea59ae288bf4fa42fe5728f6ec759afcc..0000000000000000000000000000000000000000
--- a/testing/indexer-test-core/src/main/resources/features/indexcleanup/IndexCleanup.feature
+++ /dev/null
@@ -1,20 +0,0 @@
-Feature: Indexing of the documents
- This feature deals to check for index deletion after schema deletion.
-
- Background:
- Given the schema is created with the following kind
- | kind | index | schemaFile |
- | tenant1:testindex<timestamp>:well:1.0.0 | tenant1-testindex<timestamp>-well-1.0.0 | index_records_1 |
-
- Scenario Outline: Index creation and deletion in the Elastic Search
- When I ingest records with the <recordFile> with <acl> for a given <kind>
- Then I check that the index for <kind> has been created
- Then I should delete the records I created earlier
- Then I should delete the schema for <kind> I created earlier
- Then I should check that the index for <kind> has not been deleted
- Then I should to run cleanup of indexes for <kind> and <message>
- Then I should check that the index for <kind> has been deleted
-
- Examples:
- | kind | recordFile | acl | message |
- | "tenant1:testindex<timestamp>:well:1.0.0" | "index_records_1" | "data.default.viewers@tenant1" | "{"data":"[{\"id\":\"%s-d9033ae1-fb15-496c-9ba0-880fd1d2b2cf\",\"kind\":\"%s\",\"op\":\"purge_schema\"}]","attributes":{"account-id":"opendes","correlation-id":"b5a281bd-f59d-4db2-9939-b2d85036fc7e"},"messageId":"%s","publishTime":"2018-05-08T21:48:56.131Z"}"|
diff --git a/testing/indexer-test-core/src/main/resources/testData/index_records_1.json b/testing/indexer-test-core/src/main/resources/testData/index_records_1.json
index 0d9f3c17ef24167affb538fc7db995317a77c348..dca7c54fe5b38b566791744bb1f34e7bdec73937 100644
--- a/testing/indexer-test-core/src/main/resources/testData/index_records_1.json
+++ b/testing/indexer-test-core/src/main/resources/testData/index_records_1.json
@@ -4,8 +4,8 @@
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
- "latitude":32.406402588,
- "longitude":-86.565592762
+ "lat":32.406402588,
+ "lon":-86.565592762
},
"Basin": "Houston",
"County": "Harris",
@@ -27,8 +27,8 @@
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
- "latitude":32.406402588,
- "longitude":-86.565592762
+ "lat":32.406402588,
+ "lon":-86.565592762
},
"Basin": "Houston",
"County": "Harris",
@@ -50,8 +50,8 @@
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
- "latitude":32.406402588,
- "longitude":-86.565592762
+ "lat":32.406402588,
+ "lon":-86.565592762
},
"Basin": "Houston",
"County": "Harris",
@@ -72,8 +72,8 @@
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
- "latitude":32.406402588,
- "longitude":-86.565592762
+ "lat":32.406402588,
+ "lon":-86.565592762
},
"Basin": "Houston",
"County": "Harris",
@@ -94,8 +94,8 @@
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
- "latitude":32.406402588,
- "longitude":-86.565592762
+ "lat":32.406402588,
+ "lon":-86.565592762
},
"Basin": "Houston",
"County": "Harris",
diff --git a/testing/indexer-test-core/src/main/resources/testData/index_records_2.json b/testing/indexer-test-core/src/main/resources/testData/index_records_2.json
index 9c35fcd6e58bcd19e01eef2b858ce19b7efcea71..721bf64f92b951992e6735a097f76d8c0e82a59b 100644
--- a/testing/indexer-test-core/src/main/resources/testData/index_records_2.json
+++ b/testing/indexer-test-core/src/main/resources/testData/index_records_2.json
@@ -4,8 +4,8 @@
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
- "latitude":32.406402588,
- "longitude":-86.565592762
+ "lat":32.406402588,
+ "lon":-86.565592762
},
"Basin": "Houston",
"County": "Harris",
@@ -26,8 +26,8 @@
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
- "latitude":32.406402588,
- "longitude":-86.565592762
+ "lat":32.406402588,
+ "lon":-86.565592762
},
"Basin": "Houston",
"County": "Harris",
@@ -48,8 +48,8 @@
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
- "latitude":32.406402588,
- "longitude":-86.565592762
+ "lat":32.406402588,
+ "lon":-86.565592762
},
"Basin": "Houston",
"County": "Harris",
@@ -70,8 +70,8 @@
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
- "latitude":32.406402588,
- "longitude":-86.565592762
+ "lat":32.406402588,
+ "lon":-86.565592762
},
"Basin": "Houston",
"County": "Harris",
@@ -92,8 +92,8 @@
"data": {
"Field": 1234,
"Location": {
- "latitude":"BA1",
- "longitude":-86.565592762
+ "lat":"BA1",
+ "lon":-86.565592762
},
"Basin": 789,
"County": 0.99,
diff --git a/testing/indexer-test-core/src/main/resources/testData/records_1.mapping b/testing/indexer-test-core/src/main/resources/testData/records_1.mapping
index 1d2be2d85a56b2b1c26fa9c8881351242e44f010..311cb573db3d78dcb9d3e1a67721e7fd14b2a6e7 100644
--- a/testing/indexer-test-core/src/main/resources/testData/records_1.mapping
+++ b/testing/indexer-test-core/src/main/resources/testData/records_1.mapping
@@ -1,5 +1,5 @@
{
- "well": {
+"dynamic":false,
"properties": {
"id": {
"type": "keyword"
@@ -89,5 +89,4 @@
}
}
}
- }
}
\ No newline at end of file
diff --git a/testing/indexer-test-core/src/main/resources/testData/records_2.mapping b/testing/indexer-test-core/src/main/resources/testData/records_2.mapping
index 16703041fde747a34fe2e911674b3c3f4b31bee5..362ad772264ebe956532edee74a283c2a5e112b8 100644
--- a/testing/indexer-test-core/src/main/resources/testData/records_2.mapping
+++ b/testing/indexer-test-core/src/main/resources/testData/records_2.mapping
@@ -1,5 +1,5 @@
{
- "well": {
+"dynamic":false,
"properties": {
"id": {
"type": "keyword"
@@ -86,5 +86,4 @@
}
}
}
- }
}
\ No newline at end of file
diff --git a/testing/indexer-test-core/src/main/resources/testData/records_3.mapping b/testing/indexer-test-core/src/main/resources/testData/records_3.mapping
index e13d948c883920aef7577d59ea01e5eea08a653b..3823c22a974043897938553841de212c1d2b50a1 100644
--- a/testing/indexer-test-core/src/main/resources/testData/records_3.mapping
+++ b/testing/indexer-test-core/src/main/resources/testData/records_3.mapping
@@ -1,5 +1,5 @@
{
- "well": {
+"dynamic":false,
"properties": {
"id": {
"type": "keyword"
@@ -97,5 +97,4 @@
}
}
}
- }
}
\ No newline at end of file
diff --git a/testing/indexer-test-gcp/pom.xml b/testing/indexer-test-gcp/pom.xml
index 15ad0ca19399f6b6bb07b4eb4c9f41ad9abb0cb9..88e66880baec4719aa5c1c498d9c9ecf7b851e43 100644
--- a/testing/indexer-test-gcp/pom.xml
+++ b/testing/indexer-test-gcp/pom.xml
@@ -99,23 +99,6 @@
<version>2.6</version>
</dependency>
- <!--Elasticsearch-->
- <dependency>
- <groupId>org.elasticsearch</groupId>
- <artifactId>elasticsearch</artifactId>
- <version>6.6.2</version>
- </dependency>
- <dependency>
- <groupId>org.elasticsearch.client</groupId>
- <artifactId>elasticsearch-rest-client</artifactId>
- <version>6.6.2</version>
- </dependency>
- <dependency>
- <groupId>org.elasticsearch.client</groupId>
- <artifactId>elasticsearch-rest-high-level-client</artifactId>
- <version>6.6.2</version>
- </dependency>
-
<!--Logging-->
<dependency>
<groupId>org.apache.logging.log4j</groupId>
diff --git a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/cleanup/RunTest.java b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/cleanup/RunTest.java
deleted file mode 100644
index baa367cab29a6bda105b22654ee69d5850033bec..0000000000000000000000000000000000000000
--- a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/cleanup/RunTest.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- Copyright 2020 Google LLC
- Copyright 2020 EPAM Systems, Inc
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-
-package org.opengroup.osdu.step_definitions.index.cleanup;
-
-import cucumber.api.CucumberOptions;
-import cucumber.api.junit.Cucumber;
-import org.junit.runner.RunWith;
-
-@RunWith(Cucumber.class)
-@CucumberOptions(
- features = "classpath:features/indexcleanup/IndexCleanup.feature",
- glue = {"classpath:org.opengroup.osdu.step_definitions/index/cleanup"},
- plugin = {"pretty", "junit:target/cucumber-reports/TEST-indexcleanup.xml"})
-public class RunTest {
-}
\ No newline at end of file
diff --git a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/cleanup/Steps.java b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/cleanup/Steps.java
deleted file mode 100644
index d40b77949c357be505521d5207e176ac4698f5fb..0000000000000000000000000000000000000000
--- a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/cleanup/Steps.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- Copyright 2020 Google LLC
- Copyright 2020 EPAM Systems, Inc
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-
-package org.opengroup.osdu.step_definitions.index.cleanup;
-
-import cucumber.api.DataTable;
-import cucumber.api.Scenario;
-import cucumber.api.java.Before;
-import cucumber.api.java.en.Given;
-import cucumber.api.java.en.Then;
-import cucumber.api.java.en.When;
-import java.io.IOException;
-import lombok.extern.java.Log;
-import org.opengroup.osdu.common.CleanupIndiciesSteps;
-import org.opengroup.osdu.util.GCPHTTPClient;
-
-@Log
-public class Steps extends CleanupIndiciesSteps {
-
- public Steps() {
- super(new GCPHTTPClient());
- }
-
- @Before
- public void before(Scenario scenario) {
- this.scenario = scenario;
- this.httpClient = new GCPHTTPClient();
- }
-
- @Given("^the schema is created with the following kind$")
- public void theSchemaIsCreatedWithTheFollowingKind(DataTable dataTable) {
- super.theSchemaIsCreatedWithTheFollowingKind(dataTable);
- }
-
- @When("^I ingest records with the \"(.*?)\" with \"(.*?)\" for a given \"(.*?)\"$")
- public void iIngestRecordsWithTheforAGiven(String record, String dataGroup, String kind) {
- super.iIngestRecordsWithTheforAGiven(record, dataGroup, kind);
- }
-
- @Then("^I check that the index for \"(.*?)\" has been created$")
- public void iCheckThatTheIndexForHasBeenCreated(String kind) throws IOException, InterruptedException {
- super.iCheckThatTheIndexForHasBeenCreated(kind);
- }
-
- @Then("^I should delete the records I created earlier$")
- public void iShouldDeleteTheRecordsForICreatedEarlier() {
- super.iShouldDeleteTheRecordsForICreatedEarlier();
- }
-
- @Then("^I should delete the schema for \"(.*?)\" I created earlier$")
- public void iShouldDeleteTheSchemaForICreatedEarlier(String kind) {
- super.iShouldDeleteTheSchemaForICreatedEarlier(kind);
- }
-
- @Then("^I should check that the index for \"(.*?)\" has not been deleted$")
- public void iShouldCheckThetTheIndexforHasNotBeenDeleted(String kind) throws IOException, InterruptedException {
- super.iShouldCheckThetTheIndexforHasNotBeenDeleted(kind);
- }
-
- @Then("^I should to run cleanup of indexes for \"(.*?)\" and \"(.*?)\"$")
- public void iShouldToRunCleanupOfIndexesForAnd(String kind, String message) {
- super.iShouldToRunCleanupOfIndexesForAnd(kind, message);
- }
-
- @Then("^I should check that the index for \"(.*?)\" has been deleted$")
- public void iShouldCheckThatTheIndexForHasBeenDeleted(String kind) throws IOException, InterruptedException {
- super.iShouldCheckThatTheIndexForHasBeenDeleted(kind);
- }
-}
\ No newline at end of file
diff --git a/testing/pom.xml b/testing/pom.xml
index 742fcdfc699312d137db006d325881da06ed2768..4edb18ba9f90b10b4a991c06a1629af6c19a8af9 100644
--- a/testing/pom.xml
+++ b/testing/pom.xml
@@ -36,10 +36,7 @@
<modules>
<module>indexer-test-core</module>
- <module>indexer-test-aws</module>
- <module>indexer-test-azure</module>
<module>indexer-test-gcp</module>
- <module>indexer-test-ibm</module>
</modules>
<repositories>