diff --git a/NOTICE b/NOTICE
index 878e5e17c130b55f98a6cc98bc813ced6ffe695a..313c46e4e329833ef5b24cb514cbf17b0456f536 100644
--- a/NOTICE
+++ b/NOTICE
@@ -231,6 +231,7 @@ The following software have components provided under the terms of this license:
- Doxia :: XDoc Module (from )
- Doxia :: XHTML Module (from )
- Elastic JNA Distribution (from https://github.com/java-native-access/jna)
+- Elastic JNA Distribution (from https://github.com/java-native-access/jna)
- Elasticsearch: 5.0.0-alpha5 (from https://github.com/elastic/elasticsearch)
- Elasticsearch: 5.0.0-alpha5 (from https://github.com/elastic/elasticsearch)
- FindBugs-jsr305 (from http://findbugs.sourceforge.net/)
@@ -254,6 +255,7 @@ The following software have components provided under the terms of this license:
- Guava: Google Core Libraries for Java (from https://github.com/google/guava.git)
- Guava: Google Core Libraries for Java (from https://github.com/google/guava.git)
- HPPC Collections (from http://labs.carrotsearch.com)
+- HPPC Collections (from http://labs.carrotsearch.com)
- Hibernate Validator Engine (from )
- IBM COS Java SDK for Amazon S3 (from https://github.com/ibm/ibm-cos-sdk-java)
- IBM COS Java SDK for COS KMS (from https://github.com/ibm/ibm-cos-sdk-java)
@@ -313,6 +315,9 @@ The following software have components provided under the terms of this license:
- KeePassJava2 :: KDB (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2-kdb)
- KeePassJava2 :: KDBX (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2-kdbx)
- KeePassJava2 :: Simple (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2-simple)
+- Logback Contrib :: JSON :: Classic (from )
+- Logback Contrib :: JSON :: Core (from )
+- Logback Contrib :: Jackson (from )
- Lucene Common Analyzers (from )
- Lucene Common Analyzers (from )
- Lucene Core (from )
@@ -336,7 +341,6 @@ The following software have components provided under the terms of this license:
- Lucene Sandbox (from )
- Lucene Sandbox (from )
- Lucene Spatial (from )
-- Lucene Spatial (from )
- Lucene Spatial 3D (from )
- Lucene Spatial 3D (from )
- Lucene Spatial Extras (from )
@@ -367,8 +371,8 @@ The following software have components provided under the terms of this license:
- Microsoft Azure Netty HTTP Client Library (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java)
- Mockito (from http://mockito.org)
-- Mockito (from http://www.mockito.org)
- Mockito (from http://mockito.org)
+- Mockito (from http://www.mockito.org)
- Mockito (from http://mockito.org)
- Mojo's Maven plugin for Cobertura (from http://mojo.codehaus.org/cobertura-maven-plugin/)
- Netty Reactive Streams HTTP support (from )
@@ -385,11 +389,15 @@ The following software have components provided under the terms of this license:
- Netty/TomcatNative [BoringSSL - Static] (from )
- Netty/Transport (from http://netty.io/)
- Netty/Transport/Native/Unix/Common (from )
+- Nimbus Content Type (from https://bitbucket.org/connect2id/nimbus-content-type)
+- Nimbus JOSE+JWT (from https://bitbucket.org/connect2id/nimbus-jose-jwt)
- Nimbus JOSE+JWT (from https://bitbucket.org/connect2id/nimbus-jose-jwt)
- Nimbus JOSE+JWT (from https://bitbucket.org/connect2id/nimbus-jose-jwt)
- Nimbus LangTag (from https://bitbucket.org/connect2id/nimbus-language-tags)
+- Nimbus LangTag (from https://bitbucket.org/connect2id/nimbus-language-tags)
- Non-Blocking Reactive Foundation for the JVM (from https://github.com/reactor/reactor)
- OAuth 2.0 SDK with OpenID Connect extensions (from https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions)
+- OAuth 2.0 SDK with OpenID Connect extensions (from https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions)
- OAuth2 for Spring Security (from )
- Objenesis (from http://objenesis.org)
- OkHttp (from )
@@ -473,9 +481,11 @@ The following software have components provided under the terms of this license:
- com.google.api.grpc:proto-google-iam-v1 (from https://github.com/googleapis/googleapis)
- commons-collections (from )
- compiler (from http://github.com/spullara/mustache.java)
+- compiler (from http://github.com/spullara/mustache.java)
- datastore-v1-proto-client (from )
- elasticsearch-core (from https://github.com/elastic/elasticsearch)
- elasticsearch-core (from https://github.com/elastic/elasticsearch)
+- elasticsearch-geo (from https://github.com/elastic/elasticsearch)
- error-prone annotations (from )
- error-prone annotations (from )
- io.grpc:grpc-alts (from https://github.com/grpc/grpc-java)
@@ -511,6 +521,7 @@ The following software have components provided under the terms of this license:
- lang-mustache (from https://github.com/elastic/elasticsearch)
- lang-mustache (from https://github.com/elastic/elasticsearch)
- lettuce (from http://github.com/mp911de/lettuce/wiki)
+- mapper-extras (from https://github.com/elastic/elasticsearch)
- micrometer-core (from https://github.com/micrometer-metrics/micrometer)
- micrometer-registry-azure-monitor (from https://github.com/micrometer-metrics/micrometer)
- org.xmlunit:xmlunit-core (from http://www.xmlunit.org/)
@@ -571,6 +582,7 @@ The following software have components provided under the terms of this license:
- Hamcrest Core (from http://hamcrest.org/)
- Lucene Common Analyzers (from )
- Lucene Common Analyzers (from )
+- Lucene Core (from )
- Plexus :: Default Container (from )
- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils)
- StAX (from http://stax.codehaus.org/)
@@ -719,6 +731,9 @@ EPL-1.0
The following software have components provided under the terms of this license:
- Logback Classic Module (from )
+- Logback Contrib :: JSON :: Classic (from )
+- Logback Contrib :: JSON :: Core (from )
+- Logback Contrib :: Jackson (from )
- Logback Core Module (from )
- Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java)
@@ -825,11 +840,15 @@ The following software have components provided under the terms of this license:
- Cobertura code coverage (from http://cobertura.sourceforge.net)
- Commons Lang (from http://commons.apache.org/lang/)
- Elastic JNA Distribution (from https://github.com/java-native-access/jna)
+- Elastic JNA Distribution (from https://github.com/java-native-access/jna)
- Java Native Access (from https://github.com/java-native-access/jna)
- Java Native Access Platform (from https://github.com/java-native-access/jna)
- Javassist (from http://www.javassist.org/)
- Javassist (from http://www.javassist.org/)
- Logback Classic Module (from )
+- Logback Contrib :: JSON :: Classic (from )
+- Logback Contrib :: JSON :: Core (from )
+- Logback Contrib :: Jackson (from )
- Logback Core Module (from )
- Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java)
@@ -879,7 +898,6 @@ The following software have components provided under the terms of this license:
- Lucene Core (from )
- Lucene Core (from )
- Lucene Sandbox (from )
-- Lucene Sandbox (from )
- Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java)
@@ -896,8 +914,8 @@ The following software have components provided under the terms of this license:
- Microsoft Azure internal Avro module for Storage (from https://github.com/Azure/azure-sdk-for-java)
- Mockito (from http://mockito.org)
- Mockito (from http://mockito.org)
-- Mockito (from http://www.mockito.org)
- Mockito (from http://mockito.org)
+- Mockito (from http://www.mockito.org)
- Netty/Codec/HTTP (from )
- Netty/Common (from )
- Plexus :: Default Container (from )
diff --git a/devops/azure/chart/templates/deployment.yaml b/devops/azure/chart/templates/deployment.yaml
index 8a7b075e355448152cb8b55795f58460a2a4c899..818f9caceecdbaa7d2492671fa8b8629c1ab6e11 100644
--- a/devops/azure/chart/templates/deployment.yaml
+++ b/devops/azure/chart/templates/deployment.yaml
@@ -86,7 +86,7 @@ spec:
- name: cosmosdb_database
value: osdu-db
- name: servicebus_topic_name
- value: recordstopic
+ value: indexing-progress
- name: entitlements_service_endpoint
value: http://entitlements-azure/entitlements/v1
- name: entitlements_service_api_key
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/GeometryConversionService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/GeometryConversionService.java
index 83aadeadf483f71aca465ed4754713a242588061..91384e26714534864434bb5aa360211febf3fb9d 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/GeometryConversionService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/GeometryConversionService.java
@@ -15,6 +15,7 @@
package org.opengroup.osdu.indexer.service;
import com.google.gson.internal.LinkedTreeMap;
+import java.util.Optional;
import org.opengroup.osdu.core.common.Constants;
import org.springframework.stereotype.Service;
import org.springframework.web.context.annotation.RequestScope;
@@ -76,10 +77,10 @@ public class GeometryConversionService {
try {
Map<String, Double> position = new HashMap<>();
- double lon = new Double(DECIMAL_FORMAT.format(positionMap.get("longitude")));
+ double lon = new Double(DECIMAL_FORMAT.format(Optional.ofNullable(positionMap.get("longitude")).orElse(positionMap.get("lon"))));
if (lon > 180 && lon < -180)
throw new IllegalArgumentException("'longitude' value is out of the range [-180, 180]");
- double lat = new Double(DECIMAL_FORMAT.format(positionMap.get("latitude")));
+ double lat = new Double(DECIMAL_FORMAT.format(Optional.ofNullable(positionMap.get("latitude")).orElse(positionMap.get("lat"))));
if (lat > 90 && lat < -90)
throw new IllegalArgumentException("'latitude' value is out of the range [-90, 90]");
position.put("lon", lon);
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
index 332fcb49608ec808673924efa3add582396223b6..95f1c504f9eca6e03027fd4e14e8389fb8cb61bc 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
@@ -14,42 +14,39 @@
package org.opengroup.osdu.indexer.service;
+import com.google.gson.Gson;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
-
+import javax.inject.Inject;
import org.apache.http.HttpStatus;
import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest;
-import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
-import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData;
-import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.client.indices.GetFieldMappingsRequest;
+import org.elasticsearch.client.indices.GetFieldMappingsResponse;
+import org.elasticsearch.client.indices.PutMappingRequest;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.reindex.BulkByScrollResponse;
import org.elasticsearch.index.reindex.UpdateByQueryRequest;
-
-import com.google.gson.Gson;
-import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.Constants;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.indexer.DEAnalyzerType;
import org.opengroup.osdu.core.common.model.indexer.ElasticType;
-import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
-import org.opengroup.osdu.core.common.search.Preconditions;
import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
import org.opengroup.osdu.core.common.model.indexer.Records;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
+import org.opengroup.osdu.core.common.search.Preconditions;
import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
import org.opengroup.osdu.indexer.util.ElasticClientHandler;
import org.springframework.stereotype.Service;
-import javax.inject.Inject;
@Service
public class IndexerMappingServiceImpl extends MappingServiceImpl implements IndexerMappingService {
@@ -165,7 +162,7 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
private boolean updateMappingToEnableKeywordIndexingForField(RestHighLevelClient client, Set<String> indicesSet, String fieldName) throws IOException {
String[] indices = indicesSet.toArray(new String[indicesSet.size()]);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> indexMappingMap = getIndexFieldMap(new String[]{"data."+fieldName}, client, indices);
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> indexMappingMap = getIndexFieldMap(new String[]{"data."+fieldName}, client, indices);
boolean failure = false;
for (String index : indicesSet) {
if (indexMappingMap.get(index)!=null && updateMappingForAllIndicesOfSameTypeToEnableKeywordIndexingForField(client, index, indexMappingMap.get(index), fieldName)) {
@@ -178,20 +175,18 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
return !failure;
}
- private Map<String, Map<String, Map<String, FieldMappingMetaData>>> getIndexFieldMap(String[] fieldNames, RestHighLevelClient client, String[] indices) throws IOException {
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> indexMappingMap = new HashMap<>();
+ private Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> getIndexFieldMap(String[] fieldNames, RestHighLevelClient client, String[] indices) throws IOException {
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> indexMappingMap = new HashMap<>();
GetFieldMappingsRequest request = new GetFieldMappingsRequest();
request.indices(indices);
request.fields(fieldNames);
try {
GetFieldMappingsResponse response = client.indices().getFieldMapping(request, RequestOptions.DEFAULT);
if (response != null && !response.mappings().isEmpty()) {
- final Map<String, Map<String, Map<String, FieldMappingMetaData>>> mappings = response.mappings();
+ final Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappings = response.mappings();
for (String index : indices) {
- //extract mapping of each index
- final Map<String, Map<String, FieldMappingMetaData>> indexMapping = mappings.get(index);
- if (indexMapping != null && !indexMapping.isEmpty()) {
- indexMappingMap.put(index, indexMapping);
+ if (mappings != null && !mappings.isEmpty()) {
+ indexMappingMap.put(index, mappings);
}
}
}
@@ -203,23 +198,22 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
}
}
- private boolean updateMappingForAllIndicesOfSameTypeToEnableKeywordIndexingForField(RestHighLevelClient client, String index, Map<String, Map<String, FieldMappingMetaData>> indexMapping, String fieldName) throws IOException {
+ private boolean updateMappingForAllIndicesOfSameTypeToEnableKeywordIndexingForField(RestHighLevelClient client, String index, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> indexMapping, String fieldName) throws IOException {
PutMappingRequest request = new PutMappingRequest(index);
String type = indexMapping.keySet().iterator().next();
if(type.isEmpty()) {
log.error(String.format("Could not find type of the mappings for index: %s.", index));
return false;
}
-
- request.type(type);
- request.timeout(REQUEST_TIMEOUT);
- Map<String, FieldMappingMetaData> metaData = indexMapping.get(type);
+
+ request.setTimeout(REQUEST_TIMEOUT);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> metaData = indexMapping.get(type);
if(metaData==null || metaData.get("data." + fieldName)==null) {
log.error(String.format("Could not find field: %s in the mapping of index: %s.", fieldName, index));
return false;
}
- FieldMappingMetaData fieldMetaData = metaData.get("data." + fieldName);
+ GetFieldMappingsResponse.FieldMappingMetadata fieldMetaData = metaData.get("data." + fieldName);
Map<String, Object> source = fieldMetaData.sourceAsMap();
if(!source.containsKey(fieldName)){
log.error(String.format("Could not find field: %s in the mapping of index: %s.", fieldName, index));
@@ -319,9 +313,8 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
try {
if (mapping != null) {
PutMappingRequest request = new PutMappingRequest(index);
- request.type(type);
request.source(mapping, XContentType.JSON);
- request.timeout(REQUEST_TIMEOUT);
+ request.setTimeout(REQUEST_TIMEOUT);
AcknowledgedResponse response = client.indices().putMapping(request, RequestOptions.DEFAULT);
return response.isAcknowledged();
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
index e77ef8f0908bbf9e718562ad4322aa9284756f73..6bd98690beaf036862e0f8fdfe5aea2d35b29a3e 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
@@ -372,10 +372,10 @@ public class IndexerServiceImpl implements IndexerService {
String index = this.elasticIndexNameResolver.getIndexNameFromKind(record.getKind());
if (operation == OperationType.create) {
- IndexRequest indexRequest = new IndexRequest(index, record.getType(), record.getId()).source(this.gson.toJson(sourceMap), XContentType.JSON);
+ IndexRequest indexRequest = new IndexRequest(index).id(record.getId()).source(this.gson.toJson(sourceMap), XContentType.JSON);
bulkRequest.add(indexRequest);
} else if (operation == OperationType.update) {
- UpdateRequest updateRequest = new UpdateRequest(index, record.getType(), record.getId()).upsert(this.gson.toJson(sourceMap), XContentType.JSON);
+ UpdateRequest updateRequest = new UpdateRequest(index, record.getId()).upsert(this.gson.toJson(sourceMap), XContentType.JSON);
bulkRequest.add(updateRequest);
}
}
@@ -389,13 +389,10 @@ public class IndexerServiceImpl implements IndexerService {
for (Map.Entry<String, List<String>> record : deleteRecordMap.entrySet()) {
- String[] kindParts = record.getKey().split(":");
- String type = kindParts[2];
-
String index = this.elasticIndexNameResolver.getIndexNameFromKind(record.getKey());
for (String id : record.getValue()) {
- DeleteRequest deleteRequest = new DeleteRequest(index, type, id);
+ DeleteRequest deleteRequest = new DeleteRequest(index, id);
bulkRequest.add(deleteRequest);
}
}
@@ -410,8 +407,6 @@ public class IndexerServiceImpl implements IndexerService {
List<String> failureRecordIds = new LinkedList<>();
if (bulkRequest.numberOfActions() == 0) return failureRecordIds;
-
-
try {
BulkResponse bulkResponse = restClient.bulk(bulkRequest, RequestOptions.DEFAULT);
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java
index 0f13dde455b6eb97b8cef6de07d4fde14fbd77cb..ec68f6cccbdc88ea0f8a605c90f301fba53e56bc 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java
@@ -22,15 +22,15 @@ import org.apache.http.HttpStatus;
import org.apache.http.util.EntityUtils;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchStatusException;
-import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
-import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
-import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.client.indices.CreateIndexRequest;
+import org.elasticsearch.client.indices.CreateIndexResponse;
+import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentType;
@@ -93,11 +93,10 @@ public class IndicesServiceImpl implements IndicesService {
request.settings(settings != null ? settings : DEFAULT_INDEX_SETTINGS);
if (mapping != null) {
String mappingJsonString = new Gson().toJson(mapping, Map.class);
- request.mapping(type, mappingJsonString, XContentType.JSON);
+ request.mapping(mappingJsonString,XContentType.JSON);
}
- request.timeout(REQUEST_TIMEOUT);
+ request.setTimeout(REQUEST_TIMEOUT);
CreateIndexResponse response = client.indices().create(request, RequestOptions.DEFAULT);
-
// cache the index status
boolean indexStatus = response.isAcknowledged() && response.isShardsAcknowledged();
if (indexStatus) this.indicesExistCache.put(index, true);
@@ -130,8 +129,7 @@ public class IndicesServiceImpl implements IndicesService {
//In case the format of cache changes then clean the cache
this.indicesExistCache.delete(index);
}
- GetIndexRequest request = new GetIndexRequest();
- request.indices(index);
+ GetIndexRequest request = new GetIndexRequest(index);
boolean exists = client.indices().exists(request, RequestOptions.DEFAULT);
if (exists) this.indicesExistCache.put(index, true);
return exists;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/MappingServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/MappingServiceImpl.java
index 6b52cc19b4a8e9f56339ce41bebfc79db0aa0c54..3f067f74ef2e7cf7762f196d428fd7b4f889fed2 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/MappingServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/MappingServiceImpl.java
@@ -15,10 +15,10 @@
package org.opengroup.osdu.indexer.service;
import org.apache.http.HttpStatus;
-import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
-import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.client.indices.GetMappingsRequest;
+import org.elasticsearch.client.indices.GetMappingsResponse;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
import org.opengroup.osdu.core.common.search.IndicesService;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/ElasticClientHandler.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/ElasticClientHandler.java
index 7c40de07302ec2c533f400f5e3b329dabe2060f4..e7737e73c614f87857dabbaa16255c833a352efe 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/ElasticClientHandler.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/ElasticClientHandler.java
@@ -96,7 +96,6 @@ public class ElasticClientHandler {
builder.setRequestConfigCallback(
requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(REST_CLIENT_CONNECT_TIMEOUT)
.setSocketTimeout(REST_CLIENT_SOCKET_TIMEOUT));
- builder.setMaxRetryTimeoutMillis(REST_CLIENT_RETRY_TIMEOUT);
Header[] defaultHeaders = new Header[]{
new BasicHeader("client.transport.nodes_sampler_interval", "30s"),
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/parser/GeoShapeParser.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/parser/GeoShapeParser.java
index 08634523cf91d72e1f7916b9727b67ae69613b68..9532e498b0e1fee048c163f34baa953d702e6ace 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/parser/GeoShapeParser.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/parser/GeoShapeParser.java
@@ -42,7 +42,7 @@ public class GeoShapeParser {
try {
// use elasticsearch's ShapeParser to validate shape
- ShapeBuilder<?, ?> shapeBuilder = getShapeBuilderFromObject(geoShapeObject);
+ ShapeBuilder shapeBuilder = getShapeBuilderFromObject(geoShapeObject);
Shape shape = shapeBuilder.buildS4J();
if (shape == null) {
throw new IllegalArgumentException("unable to parse shape");
@@ -54,7 +54,7 @@ public class GeoShapeParser {
}
}
- private ShapeBuilder<?, ?> getShapeBuilderFromObject(Map<String, Object> object) throws IOException {
+ private ShapeBuilder getShapeBuilderFromObject(Map<String, Object> object) throws IOException {
XContentBuilder contentBuilder = JsonXContent.contentBuilder().value(object);
XContentParser parser = JsonXContent.jsonXContent.createParser(
diff --git a/pom.xml b/pom.xml
index def52c1db676f2e41dad2e018833da0e40dac63b..d24aad221589fd26fd867d4c1b9d8bd03f4d2c24 100644
--- a/pom.xml
+++ b/pom.xml
@@ -25,7 +25,7 @@
<jackson.version>2.11.2</jackson.version>
<tomcat-embed-core.version>9.0.37</tomcat-embed-core.version>
<common-codec.version>1.14</common-codec.version>
- <elasticsearch.version>6.8.1</elasticsearch.version>
+ <elasticsearch.version>7.8.1</elasticsearch.version>
<netty.version>4.1.51.Final</netty.version>
<reactor-netty.version>0.8.20.RELEASE</reactor-netty.version>
<woodstox-core.version>6.2.3</woodstox-core.version>
diff --git a/provider/indexer-aws/build-aws/Dockerfile b/provider/indexer-aws/build-aws/Dockerfile
index 4af0e6ae8f3b38370edf5354972609066316b86b..615dde332e1de01c951a32c0b17677bfe72ea7d2 100644
--- a/provider/indexer-aws/build-aws/Dockerfile
+++ b/provider/indexer-aws/build-aws/Dockerfile
@@ -16,7 +16,14 @@
FROM amazoncorretto:8
ARG JAR_FILE=provider/indexer-aws/target/*spring-boot.jar
+
+#Default to using self signed generated TLS cert
+ENV USE_SELF_SIGNED_SSL_CERT true
+
WORKDIR /
COPY ${JAR_FILE} app.jar
+COPY /provider/indexer-aws/build-aws/ssl.sh /ssl.sh
+COPY /provider/indexer-aws/build-aws/entrypoint.sh /entrypoint.sh
EXPOSE 8080
-ENTRYPOINT java $JAVA_OPTS -jar /app.jar
\ No newline at end of file
+
+ENTRYPOINT ["/bin/sh", "-c", ". /entrypoint.sh"]
\ No newline at end of file
diff --git a/provider/indexer-aws/build-aws/buildspec.yaml b/provider/indexer-aws/build-aws/buildspec.yaml
index 7f263524ddd942206451b15dfd7aea7b09a6442f..71e7e2c50d1f2ad6960513070a2200eb36d9105a 100644
--- a/provider/indexer-aws/build-aws/buildspec.yaml
+++ b/provider/indexer-aws/build-aws/buildspec.yaml
@@ -27,6 +27,8 @@ phases:
runtime-versions:
java: corretto8
commands:
+ # fix error noted here: https://github.com/yarnpkg/yarn/issues/7866
+ - curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
- if [ $(echo $CODEBUILD_SOURCE_VERSION | grep -c ^refs/heads.*) -eq 1 ]; then echo "Branch name found"; else echo "This build only supports branch builds" && exit 1; fi
- apt-get update -y
- apt-get install -y maven
diff --git a/provider/indexer-aws/build-aws/entrypoint.sh b/provider/indexer-aws/build-aws/entrypoint.sh
new file mode 100755
index 0000000000000000000000000000000000000000..9bd3ec69d01fba69f4bece2162e7faba5cc0f0cb
--- /dev/null
+++ b/provider/indexer-aws/build-aws/entrypoint.sh
@@ -0,0 +1,15 @@
+
+
+if [ -n $USE_SELF_SIGNED_SSL_CERT ];
+then
+ export SSL_KEY_PASSWORD=$RANDOM$RANDOM$RANDOM;
+ export SSL_KEY_STORE_PASSWORD=$SSL_KEY_PASSWORD;
+ export SSL_KEY_STORE_DIR=/tmp/certs;
+ export SSL_KEY_STORE_NAME=osduonaws.p12;
+ export SSL_KEY_STORE_PATH=$SSL_KEY_STORE_DIR/$SSL_KEY_STORE_NAME;
+ export SSL_KEY_ALIAS=osduonaws;
+
+ ./ssl.sh;
+fi
+
+java $JAVA_OPTS -jar /app.jar
\ No newline at end of file
diff --git a/provider/indexer-aws/build-aws/ssl.sh b/provider/indexer-aws/build-aws/ssl.sh
new file mode 100755
index 0000000000000000000000000000000000000000..9ede565684bdd46cb09e56fce721ced55206ca07
--- /dev/null
+++ b/provider/indexer-aws/build-aws/ssl.sh
@@ -0,0 +1,34 @@
+# Copyright © 2021 Amazon Web Services
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/usr/bin/env bash
+
+#Future: Support for using Amazon Cert Manager
+# if [ "$1" == "webserver" ] && [ -n $ACM_CERTIFICATE_ARN ];
+# then
+
+# aws acm export-certificate --certificate-arn $ACM_CERTIFICATE_ARN --passphrase $(echo -n 'aws123' | openssl base64 -e) | jq -r '"\(.PrivateKey)"' > ${SSL_KEY_PATH}.enc
+# openssl rsa -in ${SSL_KEY_PATH}.enc -out $SSL_KEY_PATH -passin pass:aws123
+# aws acm get-certificate --certificate-arn $ACM_CERTIFICATE_ARN | jq -r '"\(.CertificateChain)"' > $SSL_CERT_PATH
+# aws acm get-certificate --certificate-arn $ACM_CERTIFICATE_ARN | jq -r '"\(.Certificate)"' >> $SSL_CERT_PATH
+
+# fi
+
+if [ -n $USE_SELF_SIGNED_SSL_CERT ];
+then
+ mkdir -p $SSL_KEY_STORE_DIR
+ pushd $SSL_KEY_STORE_DIR
+ keytool -genkeypair -alias $SSL_KEY_ALIAS -keyalg RSA -keysize 2048 -storetype PKCS12 -keystore $SSL_KEY_STORE_NAME -validity 3650 -keypass $SSL_KEY_PASSWORD -storepass $SSL_KEY_PASSWORD -dname "CN=localhost, OU=AWS, O=Energy, L=Houston, ST=TX, C=US"
+ popd
+fi
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/IndexerAwsApplication.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/IndexerAwsApplication.java
index 8d550c871e75408fa2a9e2fc2c90f32dc8dd4124..04de6340810e6d84e8c551a70d9b7e353b08b265 100644
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/IndexerAwsApplication.java
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/IndexerAwsApplication.java
@@ -14,15 +14,22 @@
package org.opengroup.osdu.indexer.aws;
+import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
import org.springframework.context.annotation.ComponentScan;
-import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.FilterType;
@SpringBootApplication(exclude = { SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class })
-@ComponentScan({"org.opengroup.osdu"})
+@ComponentScan(
+ basePackages = {"org.opengroup.osdu"},
+ excludeFilters={
+ @ComponentScan.Filter(
+ type=FilterType.ASSIGNABLE_TYPE,
+ value=IndexerQueueTaskBuilder.class)
+ })
public class IndexerAwsApplication {
public static void main(String[] args) {
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java
index 8d593794ffbb74b8dd2efccc37fdb44e8d19152d..b93d3c94144d859cdf25cde6ff4e1cfcd7ff9358 100644
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java
@@ -31,7 +31,7 @@ import org.springframework.stereotype.Component;
import javax.inject.Inject;
import java.util.HashMap;
import java.util.Map;
-
+//
@Primary
@Component
public class IndexerQueueTaskBuilderAws extends IndexerQueueTaskBuilder {
@@ -43,8 +43,8 @@ public class IndexerQueueTaskBuilderAws extends IndexerQueueTaskBuilder {
private ParameterStorePropertySource ssm;
- private String amazonSQSQueueUrl;
-
+ private String storageQueue;
+ private String dlq;
private final String retryString = "retry";
private Gson gson;
@@ -54,7 +54,8 @@ public class IndexerQueueTaskBuilderAws extends IndexerQueueTaskBuilder {
@Value("${aws.storage.sqs.queue.url}")
String sqsStorageQueueParameter;
-
+ @Value("${aws.indexer.sqs.dlq.url}")
+ String deadLetterQueueParameter;
@Inject
public void init() {
@@ -63,17 +64,26 @@ public class IndexerQueueTaskBuilderAws extends IndexerQueueTaskBuilder {
gson =new Gson();
SSMConfig ssmConfig = new SSMConfig();
ssm = ssmConfig.amazonSSM();
- amazonSQSQueueUrl = ssm.getProperty(sqsStorageQueueParameter).toString();
+ storageQueue = ssm.getProperty(sqsStorageQueueParameter).toString();
+ dlq = ssm.getProperty(deadLetterQueueParameter).toString();
}
@Override
public void createWorkerTask(String payload, DpsHeaders headers) {
- createTask(payload, headers);
+ this.createTask(payload, headers);
}
+ @Override
+ public void createWorkerTask(String payload, Long countDownMillis, DpsHeaders headers){
+ this.createTask(payload, headers);
+ }
@Override
public void createReIndexTask(String payload,DpsHeaders headers) {
- createTask(payload, headers);
+ this.createTask(payload, headers);
+ }
+ @Override
+ public void createReIndexTask(String payload, Long countDownMillis, DpsHeaders headers){
+ this.createTask(payload, headers);
}
private void createTask(String payload, DpsHeaders headers) {
@@ -119,11 +129,19 @@ public class IndexerQueueTaskBuilderAws extends IndexerQueueTaskBuilder {
);
// Send a message with an attribute and a delay
- final SendMessageRequest sendMessageRequest = new SendMessageRequest()
- .withQueueUrl(amazonSQSQueueUrl)
- .withMessageBody(message.getData())
- .withDelaySeconds(new Integer(retryDelay))
- .withMessageAttributes(messageAttributes);
+ final SendMessageRequest sendMessageRequest ;
+ if (retryCount< 10) {
+
+ sendMessageRequest = new SendMessageRequest()
+ .withQueueUrl(storageQueue)
+ .withMessageBody(message.getData())
+ .withDelaySeconds(new Integer(retryDelay))
+ .withMessageAttributes(messageAttributes);
+ }else{
+ sendMessageRequest = new SendMessageRequest()
+ .withQueueUrl(dlq)
+ .withMessageBody(message.getData());
+ }
sqsClient.sendMessage(sendMessageRequest);
}
diff --git a/provider/indexer-aws/src/main/resources/application.properties b/provider/indexer-aws/src/main/resources/application.properties
index d29cc7a7661161f50377c9577cb7c4f418771c70..d42824c82a6154f8d27c9366053323857f9e847a 100644
--- a/provider/indexer-aws/src/main/resources/application.properties
+++ b/provider/indexer-aws/src/main/resources/application.properties
@@ -59,3 +59,11 @@ aws.indexer.sns.topic.arn=${aws.ssm.prefix}/indexer/indexer-sns-topic-arn
aws.storage.sns.topic.arn=${aws.ssm.prefix}/storage/storage-sns-topic-arn
aws.storage.sqs.queue.url=${aws.ssm.prefix}/storage/storage-sqs-url
+aws.indexer.sqs.dlq.url=${aws.ssm.prefix}/indexer-queue/indexer-deadletter-queue-sqs-url
+
+server.ssl.enabled=${SSL_ENABLED:true}
+server.ssl.key-store-type=PKCS12
+server.ssl.key-store=${SSL_KEY_STORE_PATH:/certs/osduonaws.p12}
+server.ssl.key-alias=${SSL_KEY_ALIAS:osduonaws}
+server.ssl.key-password=${SSL_KEY_PASSWORD:}
+server.ssl.key-store-password=${SSL_KEY_STORE_PASSWORD:}
\ No newline at end of file
diff --git a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexerMappingServiceTest.java b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexerMappingServiceTest.java
index a83f2c7dab97d19426ad1e1800cf48dada775d66..3fc9f04f711391c860528a1af7c9b34a01eb1cf9 100644
--- a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexerMappingServiceTest.java
+++ b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexerMappingServiceTest.java
@@ -18,7 +18,6 @@ import org.apache.http.StatusLine;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest;
import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
-import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
@@ -156,13 +155,13 @@ public class IndexerMappingServiceTest {
builder.field("any field", new HashMap());
builder.endObject();
BytesReference bytesReference = BytesReference.bytes(builder);
- FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
- Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
mappingBuilder.put("any index 1", mapBuilder);
mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
mapping.put("indices 1", mappingBuilder);
when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
@@ -190,13 +189,13 @@ public class IndexerMappingServiceTest {
builder.field("any field", new HashMap());
builder.endObject();
BytesReference bytesReference = BytesReference.bytes(builder);
- FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
- Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
mappingBuilder.put("any index 1", mapBuilder);
mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
mapping.put("indices 1", mappingBuilder);
when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
@@ -224,13 +223,13 @@ public class IndexerMappingServiceTest {
builder.field("any field", new HashMap());
builder.endObject();
BytesReference bytesReference = BytesReference.bytes(builder);
- FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
- Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
mappingBuilder.put("any index 1", mapBuilder);
mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
mapping.put("indices 1", mappingBuilder);
when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
@@ -259,13 +258,13 @@ public class IndexerMappingServiceTest {
builder.field("any field", new HashMap());
builder.endObject();
BytesReference bytesReference = BytesReference.bytes(builder);
- FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
- Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
mappingBuilder.put("any index 1", mapBuilder);
mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
mapping.put("indices 1", mappingBuilder);
when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
@@ -293,13 +292,13 @@ public class IndexerMappingServiceTest {
builder.field("any field", new HashMap());
builder.endObject();
BytesReference bytesReference = BytesReference.bytes(builder);
- FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
- Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
mappingBuilder.put("any index 1", mapBuilder);
mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
mapping.put("indices 1", mappingBuilder);
when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
diff --git a/provider/indexer-gcp/README.md b/provider/indexer-gcp/README.md
index 373972ee9e4561bab282bb0e84e80954293724f0..c40c11f423195ef35f0f093617f1dbcdcf0d3c75 100644
--- a/provider/indexer-gcp/README.md
+++ b/provider/indexer-gcp/README.md
@@ -35,7 +35,7 @@ In order to run the service locally or remotely, you will need to have the follo
| `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials |
| `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
| `security.https.certificate.trust` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment |
-| `indexer.que.service.mail` | ex `default@iam.gserviceaccount.com` | IndexerQue environment service account mail, required if Indexer Que deployed in cloud task mode, to validate token from it | yes | - |
+| `indexer.que.service.mail` | ex `default@iam.gserviceaccount.com` | Indexer Que environment service account mail, required if Indexer Que deployed in cloud task mode, to validate token from it | yes | - |
### Run Locally
Check that maven is installed:
@@ -147,12 +147,14 @@ You will need to have the following environment variables defined.
| `SEARCH_INTEGRATION_TESTER` | `********` | Service account for API calls. Note: this user must have entitlements configured already | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
| `SEARCH_HOST` | ex `http://localhost:8080/api/search/v2/` | Endpoint of search service | no | - |
| `STORAGE_HOST` | ex `http://os-storage-dot-opendes.appspot.com/api/storage/v2/schemas` | Storage API endpoint | Storage Host | no | output of infrastructure deployment |
+| `SECURITY_HTTPS_CERTIFICATE_TRUST` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment |
+
**Entitlements configuration for integration accounts**
| INTEGRATION_TESTER | NO_DATA_ACCESS_TESTER |
| --- | --- |
-| users<br/>service.entitlements.user<br/>service.search.user<br/>data.test1<br/>data.integration.test<br/>users@{tenant1}@{domain}.com |
+| users<br/>service.entitlements.user<br/>service.search.user<br/>service.search.admin<br/>data.test1<br/>data.integration.test<br/>users@{tenant1}@{domain}.com |
Execute following command to build code and run all the integration tests:
@@ -197,9 +199,9 @@ Create king ring and key in the ***master project***
--purpose encryption
```
-Add **Cloud KMS CryptoKey Encrypter/Decrypter** role to the **App Engine default service account** of the master project through IAM - Role tab
+Add **Cloud KMS CryptoKey Encrypter/Decrypter** role to the **App Engine default service account** of the ***master project*** through IAM - Role tab
-Add "Cloud KMS Encrypt/Decrypt" role to the "App Engine default service account" of ***master project***
+Add **Cloud KMS Encrypt/Decrypt** role to the **App Engine default service account** of ***master project***
#### Memory Store (Redis Instance) Setup
diff --git a/provider/indexer-gcp/pom.xml b/provider/indexer-gcp/pom.xml
index 1152119b412b5d75a638f68ef0c7c40766f340b0..a37a4df8947684574ceb67a8389f5b658fef23c7 100644
--- a/provider/indexer-gcp/pom.xml
+++ b/provider/indexer-gcp/pom.xml
@@ -25,7 +25,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-gcp</artifactId>
- <version>0.3.23</version>
+ <version>0.6.1-SNAPSHOT</version>
</dependency>
<dependency>
@@ -74,17 +74,14 @@
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
- <version>6.6.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
- <version>6.6.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
- <version>6.6.2</version>
</dependency>
<!-- Test Dependencies -->
@@ -151,7 +148,16 @@
<artifactId>proto-google-iam-v1</artifactId>
<version>0.12.0</version>
</dependency>
-
+ <dependency>
+ <groupId>ch.qos.logback.contrib</groupId>
+ <artifactId>logback-json-classic</artifactId>
+ <version>0.1.5</version>
+ </dependency>
+ <dependency>
+ <groupId>ch.qos.logback.contrib</groupId>
+ <artifactId>logback-jackson</artifactId>
+ <version>0.1.5</version>
+ </dependency>
</dependencies>
<build>
diff --git a/provider/indexer-gcp/src/main/resources/application-dev.properties b/provider/indexer-gcp/src/main/resources/application-dev.properties
index 7ec1744f095c0ee84ad53abafae545ce7ad7bdfc..6c49ae23e07fddb7668a52d645d70268579ce9a9 100644
--- a/provider/indexer-gcp/src/main/resources/application-dev.properties
+++ b/provider/indexer-gcp/src/main/resources/application-dev.properties
@@ -1,21 +1,21 @@
google-cloud-project=opendes
-indexer-host=os-indexer-dot-opendes.appspot.com
-STORAGE_HOSTNAME=os-storage-dot-opendes.appspot.com
+indexer-host=indexer-jvmvia5dea-uc.a.run.app
+STORAGE_HOSTNAME=storage-jvmvia5dea-uc.a.run.app
-storage-schema-host=https://os-storage-dot-opendes.appspot.com/api/storage/v2/schemas
-storage-query-record-host=https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records
-storage-query-record-for-conversion-host=https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records:batch
+storage-schema-host=http://127.0.0.1:8081/api/storage/v2/schemas
+storage-query-record-host=http://127.0.0.1:8081/api/storage/v2/query/records
+storage-query-record-for-conversion-host=https://storage-jvmvia5dea-uc.a.run.app/api/storage/v2/query/records:batch
storage-records-batch-size=20
-indexer-queue-host=https://os-indexer-queue-dot-opendes.appspot.com/_dps/task-handlers/enqueue
+indexer-queue-host=https://indexer-queue-jvmvia5dea-uc.a.run.app/_dps/task-handlers/enqueue
-AUTHORIZE_API=https://entitlements-dot-opendes.appspot.com/entitlements/v1
-LEGALTAG_API=https://os-legal-dot-opendes.appspot.com/api/legal/v1
-CRS_API=https://crs-converter-gae-dot-opendes.appspot.com/api/crs/v1
+AUTHORIZE_API=https://os-entitlements-gcp-jvmvia5dea-uc.a.run.app/entitlements/v1
+LEGALTAG_API=https://os-legal-jvmvia5dea-uc.a.run.app/api/legal/v1
+CRS_API=https://crs-converter-jvmvia5dea-uc.a.run.app/api/crs/v1
## use below values for gcp: opendes
-REDIS_GROUP_HOST=10.0.16.28
-redis-search-host=10.0.16.20
+REDIS_GROUP_HOST=127.0.0.1
+redis-search-host=127.0.0.1
-google-audiences=245464679631-ktfdfpl147m1mjpbutl00b3cmffissgq.apps.googleusercontent.com
\ No newline at end of file
+google-audiences=519000754840-09v7ssbpku7sevhvbtq3bdoi70es16p5.apps.googleusercontent.com
\ No newline at end of file
diff --git a/provider/indexer-gcp/src/main/resources/application.properties b/provider/indexer-gcp/src/main/resources/application.properties
index b3d58b0c9b895a6ca38e8b29143edd258e7d8c8b..c2c74189250e40afae31e9f8c0174370a877c569 100644
--- a/provider/indexer-gcp/src/main/resources/application.properties
+++ b/provider/indexer-gcp/src/main/resources/application.properties
@@ -1,7 +1,7 @@
LOG_PREFIX=indexer
server.servlet.contextPath=/api/indexer/v2
-logging.level.org.springframework.web=DEBUG
+logging.level.org.springframework.web=${LOG_LEVEL:DEBUG}
server.port=8080
JAVA_OPTS=-Xms3072m -Xmx3072m
JAVA_GC_OPTS=-XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45
diff --git a/provider/indexer-gcp/src/main/resources/logback.xml b/provider/indexer-gcp/src/main/resources/logback.xml
new file mode 100644
index 0000000000000000000000000000000000000000..3d4fbc039ba4367897cba9a478d4ed44a2059b76
--- /dev/null
+++ b/provider/indexer-gcp/src/main/resources/logback.xml
@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<configuration>
+ <include resource="org/springframework/boot/logging/logback/defaults.xml"/>
+ <logger name="org.opengroup.osdu" level="DEBUG"/>
+ <springProfile name="local">
+ <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%yellow([%thread]) %highlight(| %-5level |) %green(%d) %cyan(| %logger{15} |) %highlight(%msg) %n</pattern>
+ <charset>utf8</charset>
+ </encoder>
+ </appender>
+ <root level="info">
+ <appender-ref ref="CONSOLE"/>
+ </root>
+ </springProfile>
+
+ <springProfile name="!local">
+ <appender name="stdout" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder class="ch.qos.logback.core.encoder.LayoutWrappingEncoder">
+ <layout class="ch.qos.logback.contrib.json.classic.JsonLayout">
+ <timestampFormat>yyyy-MM-dd HH:mm:ss.SSS</timestampFormat>
+ <timestampFormatTimezoneId>Etc/UTC</timestampFormatTimezoneId>
+ <appendLineSeparator>true</appendLineSeparator>
+
+ <jsonFormatter class="org.opengroup.osdu.core.gcp.logging.formatter.GoogleJsonFormatter">
+ <prettyPrint>false</prettyPrint>
+ </jsonFormatter>
+ </layout>
+ </encoder>
+ </appender>
+
+ <root level="info">
+ <appender-ref ref="stdout"/>
+ </root>
+ </springProfile>
+
+</configuration>
diff --git a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java
index ac48c1ac516308b045206b2c33595fc10914ce13..019bae350d179f8d038f31883284af4e2d6e439b 100644
--- a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java
+++ b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java
@@ -1,24 +1,10 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
package org.opengroup.osdu.indexer.service;
-import com.google.gson.Gson;
import org.apache.http.StatusLine;
import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest;
import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
-import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.*;
@@ -30,14 +16,14 @@ import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
+import org.mockito.ArgumentMatchers;
import org.mockito.InjectMocks;
import org.mockito.Mock;
-import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
+import org.opengroup.osdu.indexer.service.IndexerMappingServiceImpl;
import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.indexer.util.ElasticClientHandler;
-import org.opengroup.osdu.indexer.util.TypeMapper;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.springframework.test.context.junit4.SpringRunner;
@@ -50,21 +36,18 @@ import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
-import static org.mockito.MockitoAnnotations.initMocks;
import static org.powermock.api.mockito.PowerMockito.when;
@Ignore
@RunWith(SpringRunner.class)
-@PrepareForTest({ RestHighLevelClient.class, IndicesClient.class})
+@PrepareForTest({ RestHighLevelClient.class, IndicesClient.class })
public class IndexerMappingServiceTest {
private final String kind = "tenant:test:test:1.0.0";
private final String index = "tenant-test-test-1.0.0";
private final String type = "test";
- private final String mappingValid = "{\"dynamic\":false,\"properties\":{\"data\":{\"properties\":{\"Msg\":{\"type\":\"text\",\"analyzer\":\"de_indexer_analyzer\",\"search_analyzer\":\"de_search_analyzer\"},\"Location\":{\"type\":\"geo_point\"}}},\"id\":{\"type\":\"keyword\"},\"acl\":{\"properties\":{\"viewers\":{\"type\":\"keyword\"},\"owners\":{\"type\":\"keyword\"}}}}}";
+ private final String mappingValid = "{\"dynamic\":false,\"properties\":{\"data\":{\"properties\":{\"Location\":{\"type\":\"geo_point\"}}},\"id\":{\"type\":\"keyword\"}}}";
- @Mock
- private IndexerConfigurationProperties configurationProperties;
@Mock
private RestClient restClient;
@Mock
@@ -76,7 +59,7 @@ public class IndexerMappingServiceTest {
private IndexerMappingServiceImpl sut;
@Mock
- private ElasticClientHandler elasticClientHandler;
+ private ElasticClientHandler elasticClientHandler;
@InjectMocks
private RestHighLevelClient restHighLevelClient;
@@ -91,14 +74,10 @@ public class IndexerMappingServiceTest {
@Before
public void setup() throws IOException {
- initMocks(this);
- when(configurationProperties.isPreDemo()).thenReturn(true);
Map<String, String> dataMapping = new HashMap<>();
dataMapping.put("Location", "geo_point");
- dataMapping.put("Msg", "text");
Map<String, Object> metaMapping = new HashMap<>();
metaMapping.put(RecordMetaAttribute.ID.getValue(), "keyword");
- metaMapping.put(RecordMetaAttribute.ACL.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.ACL));
this.indexSchema = IndexSchema.builder().kind(kind).type(type).dataSchema(dataMapping).metaSchema(metaMapping)
.build();
@@ -106,7 +85,7 @@ public class IndexerMappingServiceTest {
this.restHighLevelClient = PowerMockito.mock(RestHighLevelClient.class);
when(this.restHighLevelClient.getLowLevelClient()).thenReturn(restClient);
- when(this.restClient.performRequest(any())).thenReturn(response);
+ when(this.restClient.performRequest(ArgumentMatchers.any())).thenReturn(response);
when(this.response.getStatusLine()).thenReturn(statusLine);
when(this.statusLine.getStatusCode()).thenReturn(200);
}
@@ -125,7 +104,7 @@ public class IndexerMappingServiceTest {
public void should_returnValidMapping_givenTrueMerge_createMappingTest() {
try {
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true);
assertEquals(this.mappingValid, mapping);
@@ -138,10 +117,10 @@ public class IndexerMappingServiceTest {
public void should_returnValidMapping_givenExistType_createMappingTest() {
try {
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
IndexerMappingServiceImpl indexerMappingServiceLocal = PowerMockito.spy(new IndexerMappingServiceImpl());
- doReturn(false).when(indexerMappingServiceLocal).isTypeExist(any(), any(), any());
+ doReturn(false).when(indexerMappingServiceLocal).isTypeExist(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any());
String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true);
assertEquals(this.mappingValid, mapping);
} catch (Exception e) {
@@ -156,24 +135,24 @@ public class IndexerMappingServiceTest {
indices.add("indices 1");
GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- when(this.indicesClient.getFieldMapping(any(), any())).thenReturn(getFieldMappingsResponse);
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();
builder.field("any field", new HashMap());
builder.endObject();
BytesReference bytesReference = BytesReference.bytes(builder);
- FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
- Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
mappingBuilder.put("any index 1", mapBuilder);
mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
mapping.put("indices 1", mappingBuilder);
when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
- doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
BulkByScrollResponse response = mock(BulkByScrollResponse.class);
- doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class));
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
@@ -190,24 +169,24 @@ public class IndexerMappingServiceTest {
indices.add("invalid 1");
GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- when(this.indicesClient.getFieldMapping(any(), any())).thenReturn(getFieldMappingsResponse);
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();
builder.field("any field", new HashMap());
builder.endObject();
BytesReference bytesReference = BytesReference.bytes(builder);
- FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
- Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
mappingBuilder.put("any index 1", mapBuilder);
mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
mapping.put("indices 1", mappingBuilder);
when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
- doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
BulkByScrollResponse response = mock(BulkByScrollResponse.class);
- doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class));
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
@@ -224,24 +203,24 @@ public class IndexerMappingServiceTest {
indices.add("indices 1");
GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- when(this.indicesClient.getFieldMapping(any(), any())).thenReturn(getFieldMappingsResponse);
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();
builder.field("any field", new HashMap());
builder.endObject();
BytesReference bytesReference = BytesReference.bytes(builder);
- FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
- Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
mappingBuilder.put("any index 1", mapBuilder);
mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
mapping.put("indices 1", mappingBuilder);
when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
- doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
BulkByScrollResponse response = mock(BulkByScrollResponse.class);
- doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class));
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field invalid");
@@ -259,24 +238,24 @@ public class IndexerMappingServiceTest {
indices.add("indices Invalid");
GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- when(this.indicesClient.getFieldMapping(any(), any())).thenThrow(new ElasticsearchException(""));
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenThrow(new ElasticsearchException(""));
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();
builder.field("any field", new HashMap());
builder.endObject();
BytesReference bytesReference = BytesReference.bytes(builder);
- FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
- Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
mappingBuilder.put("any index 1", mapBuilder);
mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
mapping.put("indices 1", mappingBuilder);
when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
- doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
BulkByScrollResponse response = mock(BulkByScrollResponse.class);
- doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class));
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
@@ -293,26 +272,26 @@ public class IndexerMappingServiceTest {
indices.add("indices Invalid");
GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- when(this.indicesClient.getFieldMapping(any(), any())).thenReturn(getFieldMappingsResponse);
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();
builder.field("any field", new HashMap());
builder.endObject();
BytesReference bytesReference = BytesReference.bytes(builder);
- FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
- Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
mappingBuilder.put("any index 1", mapBuilder);
mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
mapping.put("indices 1", mappingBuilder);
when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
- doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
BulkByScrollResponse response = mock(BulkByScrollResponse.class);
- doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class));
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
- when(this.indicesClient.putMapping(any(), any(RequestOptions.class))).thenThrow(new ElasticsearchException(""));
+ when(this.indicesClient.putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class))).thenThrow(new ElasticsearchException(""));
when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
} catch (AppException e) {
@@ -321,18 +300,4 @@ public class IndexerMappingServiceTest {
fail("Should not throw this exception" + e.getMessage());
}
}
-
-
- @Test
- public void should_returnDocumentMapping_givenValidIndexSchema() {
-
- try {
- Map<String, Object> documentMapping = this.sut.getIndexMappingFromRecordSchema(this.indexSchema);
- String documentMappingJson = new Gson().toJson(documentMapping);
- assertEquals(this.mappingValid, documentMappingJson);
-
- } catch (Exception e) {
- fail("Should not throw this exception" + e.getMessage());
- }
- }
}
diff --git a/testing/indexer-test-aws/pom.xml b/testing/indexer-test-aws/pom.xml
index ca8036cc484ebc53d8c37303078b57ffb3c4b8fc..e8c2f455b1a331a3cff88dbb309e509e59b1cbb5 100644
--- a/testing/indexer-test-aws/pom.xml
+++ b/testing/indexer-test-aws/pom.xml
@@ -111,23 +111,6 @@
<version>27.1-jre</version>
</dependency>
- <!--Elasticsearch-->
- <dependency>
- <groupId>org.elasticsearch</groupId>
- <artifactId>elasticsearch</artifactId>
- <version>6.6.2</version>
- </dependency>
- <dependency>
- <groupId>org.elasticsearch.client</groupId>
- <artifactId>elasticsearch-rest-client</artifactId>
- <version>6.6.2</version>
- </dependency>
- <dependency>
- <groupId>org.elasticsearch.client</groupId>
- <artifactId>elasticsearch-rest-high-level-client</artifactId>
- <version>6.6.2</version>
- </dependency>
-
<!--Logging-->
<dependency>
<groupId>org.apache.logging.log4j</groupId>
diff --git a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/ElasticUtilsAws.java b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/ElasticUtilsAws.java
index 28d3646d655232dc147719945b02282748e7d498..fa172dddc65900906a66e98f6a3fe1b58982bb12 100644
--- a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/ElasticUtilsAws.java
+++ b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/ElasticUtilsAws.java
@@ -32,7 +32,6 @@ public class ElasticUtilsAws extends ElasticUtils {
RestClientBuilder builder = RestClient.builder(new HttpHost(host, port, "https"));
builder.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(REST_CLIENT_CONNECT_TIMEOUT)
.setSocketTimeout(REST_CLIENT_SOCKET_TIMEOUT));
- builder.setMaxRetryTimeoutMillis(REST_CLIENT_RETRY_TIMEOUT);
builder.setHttpClientConfigCallback(httpAsyncClientBuilder -> httpAsyncClientBuilder.setSSLHostnameVerifier((s, sslSession) -> true));
Header[] defaultHeaders = new Header[]{
diff --git a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/JwtTokenUtil.java b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/JwtTokenUtil.java
index 3ff2b4e3166783fb6d44b1fd5ccba19055a237b4..46a3fce10ea09e5a212865453043d4bb33a05d6d 100644
--- a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/JwtTokenUtil.java
+++ b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/JwtTokenUtil.java
@@ -25,4 +25,4 @@ class JwtTokenUtil {
AWSCognitoClient client = new AWSCognitoClient(clientId, authFlow, user, password);
return client.getTokenForUserWithAccess();
}
-}
+}
\ No newline at end of file
diff --git a/testing/indexer-test-azure/pom.xml b/testing/indexer-test-azure/pom.xml
index 69273b80281a7825581b8a87a85aa2b1475ae232..2a6703142879b538f3f01319953cb9aa18833633 100644
--- a/testing/indexer-test-azure/pom.xml
+++ b/testing/indexer-test-azure/pom.xml
@@ -36,6 +36,7 @@
<cucumber.version>1.2.5</cucumber.version>
<azure.keyvault.version>4.1.0</azure.keyvault.version>
<junit.jupiter.version>5.6.0</junit.jupiter.version>
+ <elasticsearch.version>7.8.1</elasticsearch.version>
</properties>
<dependencies>
@@ -135,17 +136,17 @@
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
- <version>6.6.2</version>
+ <version>${elasticsearch.version}</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
- <version>6.6.2</version>
+ <version>${elasticsearch.version}</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
- <version>6.6.2</version>
+ <version>${elasticsearch.version}</version>
</dependency>
<!--Logging-->
diff --git a/testing/indexer-test-core/pom.xml b/testing/indexer-test-core/pom.xml
index 46e5f7eabf80cc978a238595036c366805b66573..37659e5ba08a4f640dc0d678e5e453128833ba1c 100644
--- a/testing/indexer-test-core/pom.xml
+++ b/testing/indexer-test-core/pom.xml
@@ -97,17 +97,17 @@
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
- <version>6.6.2</version>
+ <version>7.8.1</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
- <version>6.6.2</version>
+ <version>7.8.1</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
- <version>6.6.2</version>
+ <version>7.8.1</version>
</dependency>
<!--Logging-->
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
index 948cad79655e474cd4c0639ff4305c3b82d4fd35..c6ebab631a5e46f0d47ae2814cd36e1d412d4b7c 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
@@ -7,8 +7,7 @@ import com.google.gson.reflect.TypeToken;
import com.sun.jersey.api.client.ClientResponse;
import cucumber.api.DataTable;
import lombok.extern.java.Log;
-import org.elasticsearch.cluster.metadata.MappingMetaData;
-import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.cluster.metadata.MappingMetadata;
import org.opengroup.osdu.core.common.model.entitlements.Acl;
import org.opengroup.osdu.models.Setup;
import org.opengroup.osdu.models.TestIndex;
@@ -125,10 +124,10 @@ public class RecordSteps extends TestsBase {
public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String type, String index) throws Throwable {
index = generateActualName(index, timeStamp);
- ImmutableOpenMap<String, MappingMetaData> elasticMapping = elasticUtils.getMapping(index);
+ Map<String, MappingMetadata> elasticMapping = elasticUtils.getMapping(index);
assertNotNull(elasticMapping);
- MappingMetaData typeMapping = elasticMapping.get(type);
+ MappingMetadata typeMapping = elasticMapping.get(index);
Map<String, Object> mapping = typeMapping.sourceAsMap();
assertNotNull(mapping);
assertTrue(areJsonEqual(expectedMapping, mapping.toString()));
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/Config.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/Config.java
index 3d97833f4f9edeaf3fb5d530d3136b9640ab7ad6..1c16fb2c504355dd28c80c6337ee4a0cb954e625 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/Config.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/Config.java
@@ -24,6 +24,8 @@ public class Config {
private static final String DEFAULT_ENTITLEMENTS_DOMAIN = "";
private static final String SCHEMA_PATH = "/api/schema-service/v1";
+ private static final String DEFAULT_SECURITY_HTTPS_CERTIFICATE_TRUST = "false";
+
public static int getPort() {
return Integer.parseInt(getEnvironmentVariableOrDefaultValue("ELASTIC_PORT", String.valueOf(PORT)));
@@ -105,6 +107,12 @@ public class Config {
return getEnvironmentVariableOrDefaultValue("AWS_COGNITO_AUTH_PARAMS_PASSWORD", "");
}
+ public static boolean isSecurityHttpsCertificateTrust() {
+ return Boolean.parseBoolean(
+ getEnvironmentVariableOrDefaultValue("SECURITY_HTTPS_CERTIFICATE_TRUST",
+ DEFAULT_SECURITY_HTTPS_CERTIFICATE_TRUST));
+ }
+
private static String getEnvironmentVariableOrDefaultValue(String key, String defaultValue) {
String environmentVariable = getEnvironmentVariable(key);
if (environmentVariable == null) {
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
index 4612f07cf3efd111c178ef607c6d1af2789d60dd..a63b339ce9421eadddd7d5f5515d33833aef5ff3 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
@@ -18,19 +18,22 @@
package org.opengroup.osdu.util;
import com.google.gson.Gson;
+import java.security.KeyManagementException;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import javax.net.ssl.SSLContext;
import lombok.extern.java.Log;
import org.apache.http.Header;
import org.apache.http.HttpHost;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
+import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
+import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
import org.apache.http.message.BasicHeader;
+import org.apache.http.ssl.SSLContextBuilder;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchStatusException;
-import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
-import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
-import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
+import org.elasticsearch.client.indices.CloseIndexRequest;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
-import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
-import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
-import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
import org.elasticsearch.action.bulk.BulkRequest;
@@ -40,7 +43,12 @@ import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.*;
-import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.client.indices.CreateIndexRequest;
+import org.elasticsearch.client.indices.CreateIndexResponse;
+import org.elasticsearch.client.indices.GetIndexRequest;
+import org.elasticsearch.client.indices.GetMappingsRequest;
+import org.elasticsearch.client.indices.GetMappingsResponse;
+import org.elasticsearch.cluster.metadata.MappingMetadata;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
@@ -91,9 +99,9 @@ public class ElasticUtils {
// creating index + add mapping to the index
log.info("Creating index with name: " + index);
- CreateIndexRequest request = new CreateIndexRequest(index, settings);
+ CreateIndexRequest request = new CreateIndexRequest(index).settings(settings);
request.source("{\"mappings\":" + mapping + "}", XContentType.JSON);
- request.timeout(REQUEST_TIMEOUT);
+ request.setTimeout(REQUEST_TIMEOUT);
CreateIndexResponse response = client.indices().create(request, RequestOptions.DEFAULT);
//wait for ack
@@ -201,7 +209,7 @@ public class ElasticUtils {
try (RestHighLevelClient client = this.createClient(username, password, host)) {
SearchRequest request = new SearchRequest(index);
SearchResponse searchResponse = client.search(request, RequestOptions.DEFAULT);
- return searchResponse.getHits().totalHits;
+ return searchResponse.getHits().getTotalHits().value;
}
} catch (ElasticsearchStatusException e) {
log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage()));
@@ -218,7 +226,7 @@ public class ElasticUtils {
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
- return searchResponse.getHits().totalHits;
+ return searchResponse.getHits().getTotalHits().value;
}
} catch (ElasticsearchStatusException e) {
log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage()));
@@ -226,13 +234,13 @@ public class ElasticUtils {
}
}
- public ImmutableOpenMap<String, MappingMetaData> getMapping(String index) throws IOException {
+ public Map<String, MappingMetadata> getMapping(String index) throws IOException {
try (RestHighLevelClient client = this.createClient(username, password, host)) {
GetMappingsRequest request = new GetMappingsRequest();
request.indices(index);
GetMappingsResponse response = client.indices().getMapping(request, RequestOptions.DEFAULT);
- ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> allMappings = response.mappings();
- return allMappings.get(index);
+ Map<String, MappingMetadata> mappings = response.mappings();
+ return mappings;
}
}
@@ -250,7 +258,8 @@ public class ElasticUtils {
private boolean closeIndex(RestHighLevelClient client, String index) {
try {
CloseIndexRequest request = new CloseIndexRequest(index);
- request.timeout(TimeValue.timeValueMinutes(1));
+ request.setTimeout(TimeValue.timeValueMinutes(1));
+ request.timeout();
AcknowledgedResponse closeIndexResponse = client.indices().close(request, RequestOptions.DEFAULT);
return closeIndexResponse.isAcknowledged();
} catch (ElasticsearchException | IOException exception) {
@@ -267,7 +276,7 @@ public class ElasticUtils {
for (Map<String, Object> record : testRecords) {
String id = (String) record.get("id");
Map<String, Object> mapData = gson.fromJson(gson.toJson(record), Map.class);
- IndexRequest indexRequest = new IndexRequest(index, kind.split(":")[2], id).source(mapData);
+ IndexRequest indexRequest = new IndexRequest(index).id(id).source(mapData);
dataList.add(indexRequest);
}
} catch (Exception e) {
@@ -299,7 +308,6 @@ public class ElasticUtils {
RestClientBuilder builder = RestClient.builder(new HttpHost(host, port, scheme));
builder.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(REST_CLIENT_CONNECT_TIMEOUT)
.setSocketTimeout(REST_CLIENT_SOCKET_TIMEOUT));
- builder.setMaxRetryTimeoutMillis(REST_CLIENT_RETRY_TIMEOUT);
Header[] defaultHeaders = new Header[]{
new BasicHeader("client.transport.nodes_sampler_interval", "30s"),
@@ -311,10 +319,43 @@ public class ElasticUtils {
new BasicHeader("Authorization", String.format("Basic %s", Base64.getEncoder().encodeToString(usernameAndPassword.getBytes()))),
};
+ boolean isSecurityHttpsCertificateTrust = Config.isSecurityHttpsCertificateTrust();
+ log.info(String.format(
+ "Elastic client connection uses protocolScheme = %s with a flag "
+ + "'security.https.certificate.trust' = %s",
+ scheme, isSecurityHttpsCertificateTrust));
+
+ if ("https".equals(scheme) && isSecurityHttpsCertificateTrust) {
+ log.warning("Elastic client connection uses TrustSelfSignedStrategy()");
+ SSLContext sslContext = createSSLContext();
+ builder.setHttpClientConfigCallback(httpClientBuilder ->
+ {
+ HttpAsyncClientBuilder httpAsyncClientBuilder = httpClientBuilder.setSSLContext(sslContext)
+ .setSSLHostnameVerifier(
+ NoopHostnameVerifier.INSTANCE);
+ return httpAsyncClientBuilder;
+ });
+ }
+
builder.setDefaultHeaders(defaultHeaders);
return builder;
}
+ private SSLContext createSSLContext() {
+ SSLContextBuilder sslContextBuilder = new SSLContextBuilder();
+ try {
+ sslContextBuilder.loadTrustMaterial(null, new TrustSelfSignedStrategy());
+ return sslContextBuilder.build();
+ } catch (NoSuchAlgorithmException e) {
+ log.severe(e.getMessage());
+ } catch (KeyStoreException e) {
+ log.severe(e.getMessage());
+ } catch (KeyManagementException e) {
+ log.severe(e.getMessage());
+ }
+ return null;
+ }
+
public boolean isIndexExist(String index) {
boolean exists = false;
try {
@@ -327,8 +368,7 @@ public class ElasticUtils {
private boolean createRestClientAndCheckIndexExist(String index) {
try (RestHighLevelClient client = this.createClient(username, password, host)) {
- GetIndexRequest request = new GetIndexRequest();
- request.indices(index);
+ GetIndexRequest request = new GetIndexRequest(index);
return client.indices().exists(request, RequestOptions.DEFAULT);
} catch (IOException e) {
log.log(Level.INFO, String.format("Error getting index: %s %s", index, e.getMessage()));
diff --git a/testing/indexer-test-core/src/main/resources/features/indexcleanup/IndexCleanup.feature b/testing/indexer-test-core/src/main/resources/features/indexcleanup/IndexCleanup.feature
deleted file mode 100644
index 610baacea59ae288bf4fa42fe5728f6ec759afcc..0000000000000000000000000000000000000000
--- a/testing/indexer-test-core/src/main/resources/features/indexcleanup/IndexCleanup.feature
+++ /dev/null
@@ -1,20 +0,0 @@
-Feature: Indexing of the documents
- This feature deals to check for index deletion after schema deletion.
-
- Background:
- Given the schema is created with the following kind
- | kind | index | schemaFile |
- | tenant1:testindex<timestamp>:well:1.0.0 | tenant1-testindex<timestamp>-well-1.0.0 | index_records_1 |
-
- Scenario Outline: Index creation and deletion in the Elastic Search
- When I ingest records with the <recordFile> with <acl> for a given <kind>
- Then I check that the index for <kind> has been created
- Then I should delete the records I created earlier
- Then I should delete the schema for <kind> I created earlier
- Then I should check that the index for <kind> has not been deleted
- Then I should to run cleanup of indexes for <kind> and <message>
- Then I should check that the index for <kind> has been deleted
-
- Examples:
- | kind | recordFile | acl | message |
- | "tenant1:testindex<timestamp>:well:1.0.0" | "index_records_1" | "data.default.viewers@tenant1" | "{"data":"[{\"id\":\"%s-d9033ae1-fb15-496c-9ba0-880fd1d2b2cf\",\"kind\":\"%s\",\"op\":\"purge_schema\"}]","attributes":{"account-id":"opendes","correlation-id":"b5a281bd-f59d-4db2-9939-b2d85036fc7e"},"messageId":"%s","publishTime":"2018-05-08T21:48:56.131Z"}"|
diff --git a/testing/indexer-test-core/src/main/resources/testData/index_records_1.json b/testing/indexer-test-core/src/main/resources/testData/index_records_1.json
index 81613de69df0524fa6d608ffc8eff55d08cfe1c3..dc83212c545f7478280566be828726f4b0558234 100644
--- a/testing/indexer-test-core/src/main/resources/testData/index_records_1.json
+++ b/testing/indexer-test-core/src/main/resources/testData/index_records_1.json
@@ -4,8 +4,8 @@
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
- "latitude":32.406402588,
- "longitude":-86.565592762
+ "lat":32.406402588,
+ "lon":-86.565592762
},
"Basin": "Houston",
"County": "Harris",
@@ -27,8 +27,8 @@
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
- "latitude":32.406402588,
- "longitude":-86.565592762
+ "lat":32.406402588,
+ "lon":-86.565592762
},
"Basin": "Houston",
"County": "Harris",
@@ -50,8 +50,8 @@
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
- "latitude":32.406402588,
- "longitude":-86.565592762
+ "lat":32.406402588,
+ "lon":-86.565592762
},
"Basin": "Houston",
"County": "Harris",
@@ -72,8 +72,8 @@
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
- "latitude":32.406402588,
- "longitude":-86.565592762
+ "lat":32.406402588,
+ "lon":-86.565592762
},
"Basin": "Houston",
"County": "Harris",
@@ -94,8 +94,8 @@
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
- "latitude":32.406402588,
- "longitude":-86.565592762
+ "lat":32.406402588,
+ "lon":-86.565592762
},
"Basin": "Houston",
"County": "Harris",
diff --git a/testing/indexer-test-core/src/main/resources/testData/index_records_2.json b/testing/indexer-test-core/src/main/resources/testData/index_records_2.json
index ed11325951b0bc1194eb08ca0a9323358917bee2..ef6634d9663cce67428f5009d64464eba62ec42f 100644
--- a/testing/indexer-test-core/src/main/resources/testData/index_records_2.json
+++ b/testing/indexer-test-core/src/main/resources/testData/index_records_2.json
@@ -4,8 +4,8 @@
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
- "latitude":32.406402588,
- "longitude":-86.565592762
+ "lat":32.406402588,
+ "lon":-86.565592762
},
"Basin": "Houston",
"County": "Harris",
@@ -26,8 +26,8 @@
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
- "latitude":32.406402588,
- "longitude":-86.565592762
+ "lat":32.406402588,
+ "lon":-86.565592762
},
"Basin": "Houston",
"County": "Harris",
@@ -48,8 +48,8 @@
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
- "latitude":32.406402588,
- "longitude":-86.565592762
+ "lat":32.406402588,
+ "lon":-86.565592762
},
"Basin": "Houston",
"County": "Harris",
@@ -70,8 +70,8 @@
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
- "latitude":32.406402588,
- "longitude":-86.565592762
+ "lat":32.406402588,
+ "lon":-86.565592762
},
"Basin": "Houston",
"County": "Harris",
@@ -92,8 +92,8 @@
"data": {
"Field": 1234,
"Location": {
- "latitude":"BA1",
- "longitude":-86.565592762
+ "lat":"BA1",
+ "lon":-86.565592762
},
"Basin": 789,
"County": 0.99,
diff --git a/testing/indexer-test-core/src/main/resources/testData/records_1.mapping b/testing/indexer-test-core/src/main/resources/testData/records_1.mapping
index 1d2be2d85a56b2b1c26fa9c8881351242e44f010..311cb573db3d78dcb9d3e1a67721e7fd14b2a6e7 100644
--- a/testing/indexer-test-core/src/main/resources/testData/records_1.mapping
+++ b/testing/indexer-test-core/src/main/resources/testData/records_1.mapping
@@ -1,5 +1,5 @@
{
- "well": {
+"dynamic":false,
"properties": {
"id": {
"type": "keyword"
@@ -89,5 +89,4 @@
}
}
}
- }
}
\ No newline at end of file
diff --git a/testing/indexer-test-core/src/main/resources/testData/records_2.mapping b/testing/indexer-test-core/src/main/resources/testData/records_2.mapping
index 16703041fde747a34fe2e911674b3c3f4b31bee5..362ad772264ebe956532edee74a283c2a5e112b8 100644
--- a/testing/indexer-test-core/src/main/resources/testData/records_2.mapping
+++ b/testing/indexer-test-core/src/main/resources/testData/records_2.mapping
@@ -1,5 +1,5 @@
{
- "well": {
+"dynamic":false,
"properties": {
"id": {
"type": "keyword"
@@ -86,5 +86,4 @@
}
}
}
- }
}
\ No newline at end of file
diff --git a/testing/indexer-test-core/src/main/resources/testData/records_3.mapping b/testing/indexer-test-core/src/main/resources/testData/records_3.mapping
index e13d948c883920aef7577d59ea01e5eea08a653b..3823c22a974043897938553841de212c1d2b50a1 100644
--- a/testing/indexer-test-core/src/main/resources/testData/records_3.mapping
+++ b/testing/indexer-test-core/src/main/resources/testData/records_3.mapping
@@ -1,5 +1,5 @@
{
- "well": {
+"dynamic":false,
"properties": {
"id": {
"type": "keyword"
@@ -97,5 +97,4 @@
}
}
}
- }
}
\ No newline at end of file
diff --git a/testing/indexer-test-gcp/pom.xml b/testing/indexer-test-gcp/pom.xml
index 1ed50ebbbfac1e941fb09e41a8db562cbdcdd899..74f56c7ace3750aa7165602b6e19c394229ed3a3 100644
--- a/testing/indexer-test-gcp/pom.xml
+++ b/testing/indexer-test-gcp/pom.xml
@@ -97,23 +97,6 @@
<version>2.6</version>
</dependency>
- <!--Elasticsearch-->
- <dependency>
- <groupId>org.elasticsearch</groupId>
- <artifactId>elasticsearch</artifactId>
- <version>6.6.2</version>
- </dependency>
- <dependency>
- <groupId>org.elasticsearch.client</groupId>
- <artifactId>elasticsearch-rest-client</artifactId>
- <version>6.6.2</version>
- </dependency>
- <dependency>
- <groupId>org.elasticsearch.client</groupId>
- <artifactId>elasticsearch-rest-high-level-client</artifactId>
- <version>6.6.2</version>
- </dependency>
-
<!--Logging-->
<dependency>
<groupId>org.apache.logging.log4j</groupId>
diff --git a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/cleanup/RunTest.java b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/cleanup/RunTest.java
deleted file mode 100644
index baa367cab29a6bda105b22654ee69d5850033bec..0000000000000000000000000000000000000000
--- a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/cleanup/RunTest.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- Copyright 2020 Google LLC
- Copyright 2020 EPAM Systems, Inc
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-
-package org.opengroup.osdu.step_definitions.index.cleanup;
-
-import cucumber.api.CucumberOptions;
-import cucumber.api.junit.Cucumber;
-import org.junit.runner.RunWith;
-
-@RunWith(Cucumber.class)
-@CucumberOptions(
- features = "classpath:features/indexcleanup/IndexCleanup.feature",
- glue = {"classpath:org.opengroup.osdu.step_definitions/index/cleanup"},
- plugin = {"pretty", "junit:target/cucumber-reports/TEST-indexcleanup.xml"})
-public class RunTest {
-}
\ No newline at end of file
diff --git a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/cleanup/Steps.java b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/cleanup/Steps.java
deleted file mode 100644
index d40b77949c357be505521d5207e176ac4698f5fb..0000000000000000000000000000000000000000
--- a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/cleanup/Steps.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- Copyright 2020 Google LLC
- Copyright 2020 EPAM Systems, Inc
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-
-package org.opengroup.osdu.step_definitions.index.cleanup;
-
-import cucumber.api.DataTable;
-import cucumber.api.Scenario;
-import cucumber.api.java.Before;
-import cucumber.api.java.en.Given;
-import cucumber.api.java.en.Then;
-import cucumber.api.java.en.When;
-import java.io.IOException;
-import lombok.extern.java.Log;
-import org.opengroup.osdu.common.CleanupIndiciesSteps;
-import org.opengroup.osdu.util.GCPHTTPClient;
-
-@Log
-public class Steps extends CleanupIndiciesSteps {
-
- public Steps() {
- super(new GCPHTTPClient());
- }
-
- @Before
- public void before(Scenario scenario) {
- this.scenario = scenario;
- this.httpClient = new GCPHTTPClient();
- }
-
- @Given("^the schema is created with the following kind$")
- public void theSchemaIsCreatedWithTheFollowingKind(DataTable dataTable) {
- super.theSchemaIsCreatedWithTheFollowingKind(dataTable);
- }
-
- @When("^I ingest records with the \"(.*?)\" with \"(.*?)\" for a given \"(.*?)\"$")
- public void iIngestRecordsWithTheforAGiven(String record, String dataGroup, String kind) {
- super.iIngestRecordsWithTheforAGiven(record, dataGroup, kind);
- }
-
- @Then("^I check that the index for \"(.*?)\" has been created$")
- public void iCheckThatTheIndexForHasBeenCreated(String kind) throws IOException, InterruptedException {
- super.iCheckThatTheIndexForHasBeenCreated(kind);
- }
-
- @Then("^I should delete the records I created earlier$")
- public void iShouldDeleteTheRecordsForICreatedEarlier() {
- super.iShouldDeleteTheRecordsForICreatedEarlier();
- }
-
- @Then("^I should delete the schema for \"(.*?)\" I created earlier$")
- public void iShouldDeleteTheSchemaForICreatedEarlier(String kind) {
- super.iShouldDeleteTheSchemaForICreatedEarlier(kind);
- }
-
- @Then("^I should check that the index for \"(.*?)\" has not been deleted$")
- public void iShouldCheckThetTheIndexforHasNotBeenDeleted(String kind) throws IOException, InterruptedException {
- super.iShouldCheckThetTheIndexforHasNotBeenDeleted(kind);
- }
-
- @Then("^I should to run cleanup of indexes for \"(.*?)\" and \"(.*?)\"$")
- public void iShouldToRunCleanupOfIndexesForAnd(String kind, String message) {
- super.iShouldToRunCleanupOfIndexesForAnd(kind, message);
- }
-
- @Then("^I should check that the index for \"(.*?)\" has been deleted$")
- public void iShouldCheckThatTheIndexForHasBeenDeleted(String kind) throws IOException, InterruptedException {
- super.iShouldCheckThatTheIndexForHasBeenDeleted(kind);
- }
-}
\ No newline at end of file
diff --git a/testing/pom.xml b/testing/pom.xml
index bcbc486166b0193225b59372060485413ea33eaa..530619057e5638534ea2c8cb7bc0608cf1b8de1e 100644
--- a/testing/pom.xml
+++ b/testing/pom.xml
@@ -35,10 +35,7 @@
<modules>
<module>indexer-test-core</module>
- <module>indexer-test-aws</module>
- <module>indexer-test-azure</module>
<module>indexer-test-gcp</module>
- <module>indexer-test-ibm</module>
</modules>
<repositories>