diff --git a/.gitignore b/.gitignore
index 9e269f1b78050908088da343128dea99d75bacdc..e8c005ae687055f7d4bcf5ada49f03d435dc6f64 100644
--- a/.gitignore
+++ b/.gitignore
@@ -41,3 +41,5 @@ provider/indexer-gcp/bin/*
# Environment configuration
*.env
.envrc
+/.gradle/checksums/checksums.lock
+/.gradle/6.7/fileHashes/fileHashes.lock
diff --git a/NOTICE b/NOTICE
index 313c46e4e329833ef5b24cb514cbf17b0456f536..cbd4bf4a82a797ceb9a2f1bec0bd42363cd633ec 100644
--- a/NOTICE
+++ b/NOTICE
@@ -231,8 +231,6 @@ The following software have components provided under the terms of this license:
- Doxia :: XDoc Module (from )
- Doxia :: XHTML Module (from )
- Elastic JNA Distribution (from https://github.com/java-native-access/jna)
-- Elastic JNA Distribution (from https://github.com/java-native-access/jna)
-- Elasticsearch: 5.0.0-alpha5 (from https://github.com/elastic/elasticsearch)
- Elasticsearch: 5.0.0-alpha5 (from https://github.com/elastic/elasticsearch)
- FindBugs-jsr305 (from http://findbugs.sourceforge.net/)
- Google APIs Client Library for Java (from )
@@ -255,7 +253,6 @@ The following software have components provided under the terms of this license:
- Guava: Google Core Libraries for Java (from https://github.com/google/guava.git)
- Guava: Google Core Libraries for Java (from https://github.com/google/guava.git)
- HPPC Collections (from http://labs.carrotsearch.com)
-- HPPC Collections (from http://labs.carrotsearch.com)
- Hibernate Validator Engine (from )
- IBM COS Java SDK for Amazon S3 (from https://github.com/ibm/ibm-cos-sdk-java)
- IBM COS Java SDK for COS KMS (from https://github.com/ibm/ibm-cos-sdk-java)
@@ -319,33 +316,18 @@ The following software have components provided under the terms of this license:
- Logback Contrib :: JSON :: Core (from )
- Logback Contrib :: Jackson (from )
- Lucene Common Analyzers (from )
-- Lucene Common Analyzers (from )
-- Lucene Core (from )
- Lucene Core (from )
- Lucene Grouping (from )
-- Lucene Grouping (from )
-- Lucene Highlighter (from )
- Lucene Highlighter (from )
- Lucene Join (from )
-- Lucene Join (from )
-- Lucene Memory (from )
-- Lucene Memory (from )
- Lucene Memory (from )
- Lucene Memory (from )
- Lucene Miscellaneous (from )
-- Lucene Miscellaneous (from )
-- Lucene Queries (from )
- Lucene Queries (from )
- Lucene QueryParsers (from )
-- Lucene QueryParsers (from )
- Lucene Sandbox (from )
-- Lucene Sandbox (from )
-- Lucene Spatial (from )
-- Lucene Spatial 3D (from )
- Lucene Spatial 3D (from )
- Lucene Spatial Extras (from )
-- Lucene Spatial Extras (from )
-- Lucene Suggest (from )
- Lucene Suggest (from )
- MapStruct Core (from )
- Maven Artifact (from )
@@ -370,9 +352,9 @@ The following software have components provided under the terms of this license:
- Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Azure Netty HTTP Client Library (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java)
+- Mockito (from http://www.mockito.org)
- Mockito (from http://mockito.org)
- Mockito (from http://mockito.org)
-- Mockito (from http://www.mockito.org)
- Mockito (from http://mockito.org)
- Mojo's Maven plugin for Cobertura (from http://mojo.codehaus.org/cobertura-maven-plugin/)
- Netty Reactive Streams HTTP support (from )
@@ -470,10 +452,8 @@ The following software have components provided under the terms of this license:
- Xerces2-j (from https://xerces.apache.org/xerces2-j/)
- aalto-xml (from )
- aggs-matrix-stats (from https://github.com/elastic/elasticsearch)
-- aggs-matrix-stats (from https://github.com/elastic/elasticsearch)
- aws-ssm-java-caching-client (from https://github.com/awslabs/aws-ssm-java-caching-client)
- cli (from https://github.com/elastic/elasticsearch)
-- cli (from https://github.com/elastic/elasticsearch)
- com.google.api.grpc:grpc-google-cloud-pubsub-v1 (from https://github.com/googleapis/googleapis)
- com.google.api.grpc:proto-google-cloud-logging-v2 (from https://github.com/googleapis/googleapis)
- com.google.api.grpc:proto-google-cloud-pubsub-v1 (from https://github.com/googleapis/googleapis)
@@ -481,10 +461,8 @@ The following software have components provided under the terms of this license:
- com.google.api.grpc:proto-google-iam-v1 (from https://github.com/googleapis/googleapis)
- commons-collections (from )
- compiler (from http://github.com/spullara/mustache.java)
-- compiler (from http://github.com/spullara/mustache.java)
- datastore-v1-proto-client (from )
- elasticsearch-core (from https://github.com/elastic/elasticsearch)
-- elasticsearch-core (from https://github.com/elastic/elasticsearch)
- elasticsearch-geo (from https://github.com/elastic/elasticsearch)
- error-prone annotations (from )
- error-prone annotations (from )
@@ -519,7 +497,6 @@ The following software have components provided under the terms of this license:
- jersey-spring4 (from )
- jose4j (from https://bitbucket.org/b_c/jose4j/)
- lang-mustache (from https://github.com/elastic/elasticsearch)
-- lang-mustache (from https://github.com/elastic/elasticsearch)
- lettuce (from http://github.com/mp911de/lettuce/wiki)
- mapper-extras (from https://github.com/elastic/elasticsearch)
- micrometer-core (from https://github.com/micrometer-metrics/micrometer)
@@ -527,7 +504,6 @@ The following software have components provided under the terms of this license:
- org.xmlunit:xmlunit-core (from http://www.xmlunit.org/)
- oro (from )
- parent-join (from https://github.com/elastic/elasticsearch)
-- parent-join (from https://github.com/elastic/elasticsearch)
- powermock-api-support (from )
- powermock-core (from http://www.powermock.org)
- powermock-module-junit4 (from http://www.powermock.org)
@@ -536,14 +512,10 @@ The following software have components provided under the terms of this license:
- proto-google-cloud-datastore-v1 (from https://github.com/googleapis/api-client-staging)
- proton-j (from )
- rank-eval (from https://github.com/elastic/elasticsearch)
-- rank-eval (from https://github.com/elastic/elasticsearch)
-- rest (from https://github.com/elastic/elasticsearch)
- rest (from https://github.com/elastic/elasticsearch)
- rest-high-level (from https://github.com/elastic/elasticsearch)
-- rest-high-level (from https://github.com/elastic/elasticsearch)
- rxjava (from https://github.com/ReactiveX/RxJava)
- secure-sm (from https://github.com/elastic/elasticsearch)
-- secure-sm (from https://github.com/elastic/elasticsearch)
- spring-security-config (from http://spring.io/spring-security)
- spring-security-core (from http://spring.io/spring-security)
- spring-security-oauth2-client (from http://spring.io/spring-security)
@@ -567,7 +539,6 @@ The following software have components provided under the terms of this license:
- tomcat-embed-el (from http://tomcat.apache.org/)
- tomcat-embed-websocket (from http://tomcat.apache.org/)
- x-content (from https://github.com/elastic/elasticsearch)
-- x-content (from https://github.com/elastic/elasticsearch)
- xml-apis (from )
========================================================================
@@ -581,7 +552,6 @@ The following software have components provided under the terms of this license:
- GAX (Google Api eXtensions) (from https://github.com/googleapis)
- Hamcrest Core (from http://hamcrest.org/)
- Lucene Common Analyzers (from )
-- Lucene Common Analyzers (from )
- Lucene Core (from )
- Plexus :: Default Container (from )
- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils)
@@ -615,11 +585,8 @@ The following software have components provided under the terms of this license:
- JSch (from http://www.jcraft.com/jsch/)
- JavaBeans Activation Framework API jar (from )
- Lucene Common Analyzers (from )
-- Lucene Common Analyzers (from )
-- Lucene Core (from )
- Lucene Core (from )
- Lucene Suggest (from )
-- Lucene Suggest (from )
- Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java)
@@ -840,7 +807,6 @@ The following software have components provided under the terms of this license:
- Cobertura code coverage (from http://cobertura.sourceforge.net)
- Commons Lang (from http://commons.apache.org/lang/)
- Elastic JNA Distribution (from https://github.com/java-native-access/jna)
-- Elastic JNA Distribution (from https://github.com/java-native-access/jna)
- Java Native Access (from https://github.com/java-native-access/jna)
- Java Native Access Platform (from https://github.com/java-native-access/jna)
- Javassist (from http://www.javassist.org/)
@@ -889,15 +855,12 @@ The following software have components provided under the terms of this license:
- Bouncy Castle Provider (from http://www.bouncycastle.org/java.html)
- Checker Qual (from https://checkerframework.org)
- Checker Qual (from https://checkerframework.org)
-- Elasticsearch: 5.0.0-alpha5 (from https://github.com/elastic/elasticsearch)
- Extensions on Apache Proton-J library (from https://github.com/Azure/qpid-proton-j-extensions)
- JOpt Simple (from http://pholser.github.io/jopt-simple)
- JUL to SLF4J bridge (from http://www.slf4j.org)
- Java Client Runtime for AutoRest (from https://github.com/Azure/autorest-clientruntime-for-java)
- Java JWT (from http://www.jwt.io)
- Lucene Core (from )
-- Lucene Core (from )
-- Lucene Sandbox (from )
- Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java)
@@ -912,10 +875,10 @@ The following software have components provided under the terms of this license:
- Microsoft Azure client library for KeyVault Secrets (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure common module for Storage (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure internal Avro module for Storage (from https://github.com/Azure/azure-sdk-for-java)
+- Mockito (from http://www.mockito.org)
- Mockito (from http://mockito.org)
- Mockito (from http://mockito.org)
- Mockito (from http://mockito.org)
-- Mockito (from http://www.mockito.org)
- Netty/Codec/HTTP (from )
- Netty/Common (from )
- Plexus :: Default Container (from )
diff --git a/devops/azure/chart/templates/deployment.yaml b/devops/azure/chart/templates/deployment.yaml
index 818f9caceecdbaa7d2492671fa8b8629c1ab6e11..1f8e620484e28c856b2ddad65a3ef0d1bf202156 100644
--- a/devops/azure/chart/templates/deployment.yaml
+++ b/devops/azure/chart/templates/deployment.yaml
@@ -56,7 +56,7 @@ spec:
readOnly: true
env:
- name: spring_application_name
- value: indexer
+ value: {{ .Chart.Name }}
- name: server.servlet.contextPath
value: /api/indexer/v2/
- name: server_port
@@ -100,7 +100,7 @@ spec:
- name: STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST
value: http://storage/api/storage/v2/query/records:batch
- name: STORAGE_QUERY_RECORD_HOST
- value: http://storage/api/storage/v2/records
+ value: http://storage/api/storage/v2/query/records
- name: partition_service_endpoint
value: http://partition/api/partition/v1
- name: azure_istioauth_enabled
diff --git a/devops/azure/chart/templates/hpa.yaml b/devops/azure/chart/templates/hpa.yaml
index d9d00bfc65e3df192ca241337772c937dbc1ff9e..68a00c86912a249472eec7cf49411f38b72322d4 100644
--- a/devops/azure/chart/templates/hpa.yaml
+++ b/devops/azure/chart/templates/hpa.yaml
@@ -24,7 +24,7 @@ spec:
kind: Deployment
name: {{ .Chart.Name }}
minReplicas: {{ .Values.global.replicaCount }}
- maxReplicas: 5
+ maxReplicas: 10
metrics:
- type: Resource
resource:
diff --git a/devops/azure/release.yaml b/devops/azure/release.yaml
index dca21e377c5a3c6d13d9dc59ff9e2f661851d2c2..dad51b35bebd9069044ac08b8848245a29165fb7 100644
--- a/devops/azure/release.yaml
+++ b/devops/azure/release.yaml
@@ -161,4 +161,4 @@ spec:
- name: STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST
value: http://storage/api/storage/v2/query/records:batch
- name: STORAGE_QUERY_RECORD_HOST
- value: http://storage/api/storage/v2/records
+ value: http://storage/api/storage/v2/query/records
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/error/GlobalExceptionMapperCore.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/error/GlobalExceptionMapperCore.java
new file mode 100644
index 0000000000000000000000000000000000000000..6e546e14cf7197a480bf84de084a3cf628373ad0
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/error/GlobalExceptionMapperCore.java
@@ -0,0 +1,88 @@
+// Copyright © Microsoft Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.error;
+
+import javax.validation.ValidationException;
+
+import javassist.NotFoundException;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.Ordered;
+import org.springframework.core.annotation.Order;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.security.access.AccessDeniedException;
+import org.springframework.web.bind.annotation.ControllerAdvice;
+import org.springframework.web.bind.annotation.ExceptionHandler;
+import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler;
+import org.opengroup.osdu.core.common.model.http.AppException;
+
+@Order(Ordered.HIGHEST_PRECEDENCE - 1)
+@ControllerAdvice
+public class GlobalExceptionMapperCore extends ResponseEntityExceptionHandler {
+
+ @Autowired
+ private JaxRsDpsLog logger;
+
+ @ExceptionHandler(AppException.class)
+ protected ResponseEntity<Object> handleAppException(AppException e) {
+ return this.getErrorResponse(e);
+ }
+
+ @ExceptionHandler(ValidationException.class)
+ protected ResponseEntity<Object> handleValidationException(ValidationException e) {
+ return this.getErrorResponse(
+ new AppException(HttpStatus.BAD_REQUEST.value(), "Validation error.", e.getMessage(), e));
+ }
+
+ @ExceptionHandler(NotFoundException.class)
+ protected ResponseEntity<Object> handleNotFoundException(NotFoundException e) {
+ return this.getErrorResponse(
+ new AppException(HttpStatus.NOT_FOUND.value(), "Resource not found.", e.getMessage(), e));
+ }
+
+ @ExceptionHandler(AccessDeniedException.class)
+ protected ResponseEntity<Object> handleAccessDeniedException(AccessDeniedException e) {
+ return this.getErrorResponse(
+ new AppException(HttpStatus.FORBIDDEN.value(), "Access denied", e.getMessage(), e));
+ }
+
+ @ExceptionHandler(Exception.class)
+ protected ResponseEntity<Object> handleGeneralException(Exception e) {
+ return this.getErrorResponse(
+ new AppException(HttpStatus.INTERNAL_SERVER_ERROR.value(), "Server error.",
+ "An unknown error has occurred.", e));
+ }
+
+ private ResponseEntity<Object> getErrorResponse(AppException e) {
+
+ String exceptionMsg = e.getOriginalException() != null
+ ? e.getOriginalException().getMessage()
+ : e.getError().getMessage();
+
+ if( e.getCause() instanceof Exception) {
+ Exception original = (Exception) e.getCause();
+ this.logger.error(original.getMessage(), original);
+ }
+
+ if (e.getError().getCode() > 499) {
+ this.logger.error(exceptionMsg, e);
+ } else {
+ this.logger.warning(exceptionMsg, e);
+ }
+
+ return new ResponseEntity<Object>(e.getError(), HttpStatus.resolve(e.getError().getCode()));
+ }
+}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
index 95f1c504f9eca6e03027fd4e14e8389fb8cb61bc..acd87b30e4a909d57832758b5260aec100421d41 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
@@ -15,12 +15,6 @@
package org.opengroup.osdu.indexer.service;
import com.google.gson.Gson;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-import javax.inject.Inject;
import org.apache.http.HttpStatus;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
@@ -38,28 +32,29 @@ import org.elasticsearch.index.reindex.UpdateByQueryRequest;
import org.opengroup.osdu.core.common.Constants;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.indexer.DEAnalyzerType;
-import org.opengroup.osdu.core.common.model.indexer.ElasticType;
import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
-import org.opengroup.osdu.core.common.model.indexer.Records;
-import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
import org.opengroup.osdu.core.common.search.Preconditions;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
import org.opengroup.osdu.indexer.util.ElasticClientHandler;
+import org.opengroup.osdu.indexer.util.TypeMapper;
import org.springframework.stereotype.Service;
+import javax.inject.Inject;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
@Service
public class IndexerMappingServiceImpl extends MappingServiceImpl implements IndexerMappingService {
- @Inject
- private IndexerConfigurationProperties configurationProperties;
@Inject
private JaxRsDpsLog log;
@Inject
private ElasticClientHandler elasticClientHandler;
private TimeValue REQUEST_TIMEOUT = TimeValue.timeValueMinutes(1);
-
+
/**
* Create a new type in Elasticsearch
*
@@ -81,7 +76,7 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
*
* @param schema Index schema
* @param type Mapping type
- * @return String JSON represnetation of type and elastic type
+ * @return String JSON representation of type and elastic type
*
* sample index mapping:
* "properties": {
@@ -111,25 +106,14 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
// meta attribute
Map<String, Object> metaMapping = new HashMap<>();
for (Map.Entry<String, Object> entry : schema.getMetaSchema().entrySet()) {
- String key = entry.getKey();
- if (key.equals(RecordMetaAttribute.ACL.getValue()) || key.equals(RecordMetaAttribute.LEGAL.getValue()) || key.equals(RecordMetaAttribute.ANCESTRY.getValue()) || key.equals(RecordMetaAttribute.INDEX_STATUS.getValue())) {
- metaMapping.put(key, entry.getValue());
- } else {
- metaMapping.put(key, Records.Type.builder().type(entry.getValue().toString()).build());
- }
+ metaMapping.put(entry.getKey(), TypeMapper.getMetaAttributeIndexerMapping(entry.getKey()));
}
// data-source attributes
Map<String, Object> dataMapping = new HashMap<>();
if (schema.getDataSchema() != null) {
for (Map.Entry<String, String> entry : schema.getDataSchema().entrySet()) {
- // Apply de_indexer_analyzer and de_search_analyzer to TEXT field
- if (configurationProperties.isPreDemo() && ElasticType.TEXT.getValue().equalsIgnoreCase(entry.getValue())) {
- log.info(String.format("indexing %s with custom analyzer", entry.getKey()));
- dataMapping.put(entry.getKey(), Records.Analyzer.builder().type(entry.getValue()).analyzer(DEAnalyzerType.INDEXER_ANALYZER.getValue()).search_analyzer(DEAnalyzerType.SEARCH_ANALYZER.getValue()).build());
- } else {
- dataMapping.put(entry.getKey(), Records.Type.builder().type(entry.getValue()).build());
- }
+ dataMapping.put(entry.getKey(), TypeMapper.getDataAttributeIndexerMapping(entry.getValue()));
}
// inner properties.data.properties block
@@ -159,7 +143,7 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
}
}
}
-
+
private boolean updateMappingToEnableKeywordIndexingForField(RestHighLevelClient client, Set<String> indicesSet, String fieldName) throws IOException {
String[] indices = indicesSet.toArray(new String[indicesSet.size()]);
Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> indexMappingMap = getIndexFieldMap(new String[]{"data."+fieldName}, client, indices);
@@ -190,7 +174,7 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
}
}
}
-
+
return indexMappingMap;
} catch (ElasticsearchException e) {
log.error(String.format("Failed to get indices: %s. | Error: %s", Arrays.toString(indices), e));
@@ -198,7 +182,9 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
}
}
- private boolean updateMappingForAllIndicesOfSameTypeToEnableKeywordIndexingForField(RestHighLevelClient client, String index, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> indexMapping, String fieldName) throws IOException {
+ private boolean updateMappingForAllIndicesOfSameTypeToEnableKeywordIndexingForField(
+ RestHighLevelClient client, String index, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> indexMapping, String fieldName) throws IOException {
+
PutMappingRequest request = new PutMappingRequest(index);
String type = indexMapping.keySet().iterator().next();
if(type.isEmpty()) {
@@ -219,7 +205,7 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
log.error(String.format("Could not find field: %s in the mapping of index: %s.", fieldName, index));
return false;
}
-
+
//Index the field with additional keyword type
Map<String, Object> keywordMap = new HashMap<>();
keywordMap.put(Constants.TYPE, "keyword");
@@ -235,22 +221,22 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
data.put(Constants.DATA,mapping);
Map<String, Object> properties = new HashMap<>();
properties.put(Constants.PROPERTIES, data);
-
+
request.source(new Gson().toJson(properties), XContentType.JSON);
try {
AcknowledgedResponse response = client.indices().putMapping(request, RequestOptions.DEFAULT);
boolean isIndicesUpdated = updateIndices(client, index);
return response.isAcknowledged() && isIndicesUpdated;
-
+
} catch (Exception e) {
log.error(String.format("Could not update mapping of index: %s. | Error: %s", index, e));
return false;
}
}
-
+
private boolean updateIndices(RestHighLevelClient client, String index) throws IOException {
- UpdateByQueryRequest request = new UpdateByQueryRequest(index);
+ UpdateByQueryRequest request = new UpdateByQueryRequest(index);
request.setConflicts("proceed");
BulkByScrollResponse response = client.updateByQuery(request, RequestOptions.DEFAULT);
if(!response.getBulkFailures().isEmpty()) {
@@ -259,7 +245,7 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
}
return true;
}
-
+
/**
* Create a new type in Elasticsearch
*
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
index 6bd98690beaf036862e0f8fdfe5aea2d35b29a3e..5fb8cba35b0886ce1e416f5e64f8d9952853ca9e 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
@@ -16,6 +16,7 @@ package org.opengroup.osdu.indexer.service;
import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
import org.apache.http.HttpStatus;
import org.elasticsearch.ElasticsearchStatusException;
import org.elasticsearch.action.bulk.BulkItemResponse;
@@ -64,7 +65,7 @@ public class IndexerServiceImpl implements IndexerService {
private static final List<RestStatus> RETRY_ELASTIC_EXCEPTION = new ArrayList<>(Arrays.asList(RestStatus.TOO_MANY_REQUESTS, RestStatus.BAD_GATEWAY, RestStatus.SERVICE_UNAVAILABLE));
- private final Gson gson = new Gson();
+ private final Gson gson = new GsonBuilder().serializeNulls().create();
@Inject
private JaxRsDpsLog jaxRsDpsLog;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java
index 0f3d5c4f18db650813ca096878925dc49a153921..70fa630e6ca33621db37c7b4f1a15344925fb3e1 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java
@@ -16,7 +16,10 @@ package org.opengroup.osdu.indexer.service;
import com.google.common.base.Strings;
import com.google.gson.Gson;
+
import java.util.Objects;
+
+import lombok.SneakyThrows;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.http.AppException;
@@ -53,74 +56,70 @@ public class ReindexServiceImpl implements ReindexService {
@Inject
private JaxRsDpsLog jaxRsDpsLog;
+ @SneakyThrows
@Override
public String reindexRecords(RecordReindexRequest recordReindexRequest, boolean forceClean) {
Long initialDelayMillis = 0l;
- try {
- DpsHeaders headers = this.requestInfo.getHeadersWithDwdAuthZ();
- if (forceClean) {
- this.indexSchemaService.syncIndexMappingWithStorageSchema(recordReindexRequest.getKind());
- initialDelayMillis = 30000l;
- }
+ DpsHeaders headers = this.requestInfo.getHeadersWithDwdAuthZ();
- RecordQueryResponse recordQueryResponse = this.storageService.getRecordsByKind(recordReindexRequest);
+ if (forceClean) {
+ this.indexSchemaService.syncIndexMappingWithStorageSchema(recordReindexRequest.getKind());
+ initialDelayMillis = 30000l;
+ }
- if (recordQueryResponse.getResults() != null && recordQueryResponse.getResults().size() != 0) {
+ RecordQueryResponse recordQueryResponse = this.storageService.getRecordsByKind(recordReindexRequest);
- List<RecordInfo> msgs = recordQueryResponse.getResults().stream()
- .map(record -> RecordInfo.builder().id(record).kind(recordReindexRequest.getKind()).op(OperationType.create.name()).build()).collect(Collectors.toList());
+ if (recordQueryResponse.getResults() != null && recordQueryResponse.getResults().size() != 0) {
- Map<String, String> attributes = new HashMap<>();
- attributes.put(DpsHeaders.ACCOUNT_ID, headers.getAccountId());
- attributes.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
- attributes.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
+ List<RecordInfo> msgs = recordQueryResponse.getResults().stream()
+ .map(record -> RecordInfo.builder().id(record).kind(recordReindexRequest.getKind()).op(OperationType.create.name()).build()).collect(Collectors.toList());
- Gson gson = new Gson();
- RecordChangedMessages recordChangedMessages = RecordChangedMessages.builder().data(gson.toJson(msgs)).attributes(attributes).build();
- String recordChangedMessagePayload = gson.toJson(recordChangedMessages);
- this.indexerQueueTaskBuilder.createWorkerTask(recordChangedMessagePayload, initialDelayMillis, headers);
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(DpsHeaders.ACCOUNT_ID, headers.getAccountId());
+ attributes.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
+ attributes.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
- // don't call reindex-worker endpoint if it's the last batch
- // previous storage query result size will be less then requested (limit param)
- if (!Strings.isNullOrEmpty(recordQueryResponse.getCursor()) && recordQueryResponse.getResults().size() == configurationProperties.getStorageRecordsBatchSize()) {
- String newPayLoad = gson.toJson(RecordReindexRequest.builder().cursor(recordQueryResponse.getCursor()).kind(recordReindexRequest.getKind()).build());
- this.indexerQueueTaskBuilder.createReIndexTask(newPayLoad, initialDelayMillis, headers);
- return newPayLoad;
- }
+ Gson gson = new Gson();
+ RecordChangedMessages recordChangedMessages = RecordChangedMessages.builder().data(gson.toJson(msgs)).attributes(attributes).build();
+ String recordChangedMessagePayload = gson.toJson(recordChangedMessages);
+ this.indexerQueueTaskBuilder.createWorkerTask(recordChangedMessagePayload, initialDelayMillis, headers);
- return recordChangedMessagePayload;
- } else {
- jaxRsDpsLog.info(String.format("kind: %s cannot be re-indexed, storage service cannot locate valid records", recordReindexRequest.getKind()));
+ // don't call reindex-worker endpoint if it's the last batch
+ // previous storage query result size will be less then requested (limit param)
+ if (!Strings.isNullOrEmpty(recordQueryResponse.getCursor()) && recordQueryResponse.getResults().size() == configurationProperties.getStorageRecordsBatchSize()) {
+ String newPayLoad = gson.toJson(RecordReindexRequest.builder().cursor(recordQueryResponse.getCursor()).kind(recordReindexRequest.getKind()).build());
+ this.indexerQueueTaskBuilder.createReIndexTask(newPayLoad, initialDelayMillis, headers);
+ return newPayLoad;
}
- return null;
- } catch (AppException e) {
- throw e;
- } catch (Exception e) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Unknown error", "An unknown error has occurred.", e);
+
+ return recordChangedMessagePayload;
+ } else {
+ jaxRsDpsLog.info(String.format("kind: %s cannot be re-indexed, storage service cannot locate valid records", recordReindexRequest.getKind()));
}
+ return null;
}
- @Override
- public void fullReindex(boolean forceClean) {
- List<String> allKinds = null;
- try {
- allKinds = storageService.getAllKinds();
- } catch (Exception e) {
- jaxRsDpsLog.error("storage service all kinds request failed",e);
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "storage service cannot respond with all kinds", "an unknown error has occurred.", e);
- }
- if (Objects.isNull(allKinds)){
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "storage service cannot respond with all kinds", "full reindex failed");
- }
- for (String kind : allKinds) {
- try {
- reindexRecords(new RecordReindexRequest(kind, ""), forceClean);
- } catch (Exception e) {
- jaxRsDpsLog.warning(String.format("kind: %s cannot be re-indexed", kind));
- continue;
- }
- }
- }
+ @Override
+ public void fullReindex(boolean forceClean) {
+ List<String> allKinds = null;
+ try {
+ allKinds = storageService.getAllKinds();
+ } catch (Exception e) {
+ jaxRsDpsLog.error("storage service all kinds request failed", e);
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "storage service cannot respond with all kinds", "an unknown error has occurred.", e);
+ }
+ if (Objects.isNull(allKinds)) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "storage service cannot respond with all kinds", "full reindex failed");
+ }
+ for (String kind : allKinds) {
+ try {
+ reindexRecords(new RecordReindexRequest(kind, ""), forceClean);
+ } catch (Exception e) {
+ jaxRsDpsLog.warning(String.format("kind: %s cannot be re-indexed", kind));
+ continue;
+ }
+ }
+ }
}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java
index fba62bb8ac5cd08549da9e7814720df38e526651..7ac887012e9e8218c65696457369c95981e43f59 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java
@@ -37,10 +37,10 @@ public class StorageIndexerPayloadMapper {
Object value = getPropertyValue(recordId, storageRecordData, name);
- if (value == null) continue;
-
ElasticType elasticType = ElasticType.forValue(entry.getValue());
+ if (value == null && !nullIndexedValueSupported(elasticType)) continue;
+
switch (elasticType) {
case KEYWORD:
case KEYWORD_ARRAY:
@@ -88,7 +88,7 @@ public class StorageIndexerPayloadMapper {
this.attributeParsingService.tryParseGeopoint(recordId, name, storageRecordData, dataMap);
break;
case GEO_SHAPE:
- this.attributeParsingService.tryParseGeojson(recordId, name, value, dataMap);
+ this.attributeParsingService.tryParseGeojson(recordId, name, value, dataMap);
break;
case NESTED:
case OBJECT:
@@ -123,4 +123,8 @@ public class StorageIndexerPayloadMapper {
}
return null;
}
+
+ private boolean nullIndexedValueSupported(ElasticType type) {
+ return type == ElasticType.TEXT;
+ }
}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java
index 2f7c9b7c0e9b3051d6a428fec359be7418f35f93..a893e1274bcb1d519e244ab3091747b64ecf6ba6 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java
@@ -22,6 +22,7 @@ import org.opengroup.osdu.core.common.model.indexer.StorageType;
import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
import org.apache.commons.lang3.StringUtils;
+
import java.util.HashMap;
import java.util.Map;
@@ -64,22 +65,45 @@ public class TypeMapper {
storageToIndexerType.put(StorageType.GEO_SHAPE.getValue(), ElasticType.GEO_SHAPE.getValue());
}
-
public static String getIndexerType(String storageType) {
- String indexedType = storageToIndexerType.getOrDefault(storageType, null);
- if (indexedType != null && indexedType.endsWith("_array")) {
- return StringUtils.substringBefore(indexedType, "_");
- }
- return indexedType;
+ return storageToIndexerType.getOrDefault(storageType, null);
}
public static Object getIndexerType(RecordMetaAttribute attribute) {
return metaAttributeIndexerType.getOrDefault(attribute.getValue(), null);
}
+ public static Object getMetaAttributeIndexerMapping(String key) {
+ if (key.equals(RecordMetaAttribute.ACL.getValue())
+ || key.equals(RecordMetaAttribute.LEGAL.getValue()) || key.equals(RecordMetaAttribute.ANCESTRY.getValue()) || key.equals(RecordMetaAttribute.INDEX_STATUS.getValue())) {
+ return metaAttributeIndexerType.get(key);
+ }
+ return Records.Type.builder().type(metaAttributeIndexerType.get(key).toString()).build();
+ }
+
+ public static Object getDataAttributeIndexerMapping(String indexerType) {
+ if (ElasticType.TEXT.getValue().equalsIgnoreCase(indexerType)) {
+ return getTextIndexerMapping();
+ }
+
+ if (isArray(indexerType)) {
+ return Records.Type.builder().type(getArrayMemberType(indexerType)).build();
+ }
+
+ return Records.Type.builder().type(indexerType).build();
+ }
+
+ private static boolean isArray(String indexerType) {
+ return indexerType != null && indexerType.endsWith("_array");
+ }
+
+ private static String getArrayMemberType(String indexerType) {
+ return StringUtils.substringBefore(indexerType, "_");
+ }
+
private static Object getAclIndexerMapping() {
Map<String, Object> aclRoleMapping = new HashMap<>();
- aclRoleMapping.put(AclRole.VIEWERS.getValue() , Records.Type.builder().type(ElasticType.KEYWORD.getValue()).build());
+ aclRoleMapping.put(AclRole.VIEWERS.getValue(), Records.Type.builder().type(ElasticType.KEYWORD.getValue()).build());
aclRoleMapping.put(AclRole.OWNERS.getValue(), Records.Type.builder().type(ElasticType.KEYWORD.getValue()).build());
Map<String, Object> aclProperties = new HashMap<>();
@@ -121,4 +145,22 @@ public class TypeMapper {
return indexStatusProperties;
}
+
+ private static Object getTextIndexerMapping() {
+ Map<String, Object> fieldIndexTypeMap = getKeywordMap();
+ Map<String, Object> textMap = new HashMap<>();
+ textMap.put("type", "text");
+ textMap.put("fields", fieldIndexTypeMap);
+ return textMap;
+ }
+
+ private static Map<String, Object> getKeywordMap() {
+ Map<String, Object> keywordMap = new HashMap<>();
+ keywordMap.put("type", "keyword");
+ keywordMap.put("ignore_above", 256);
+ keywordMap.put("null_value", "null");
+ Map<String, Object> fieldIndexTypeMap = new HashMap<>();
+ fieldIndexTypeMap.put("keyword", keywordMap);
+ return fieldIndexTypeMap;
+ }
}
\ No newline at end of file
diff --git a/provider/indexer-aws/pom.xml b/provider/indexer-aws/pom.xml
index d6a11e333aba7bc1168209b4489d02d6648909e3..1ee9eb488b579e264cb5916c96038282b1c26837 100644
--- a/provider/indexer-aws/pom.xml
+++ b/provider/indexer-aws/pom.xml
@@ -90,12 +90,6 @@
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-oauth2-jose</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-core</artifactId>
- <version>7.6.0</version>
- <scope>compile</scope>
- </dependency>
<dependency>
<groupId>javax.inject</groupId>
<artifactId>javax.inject</artifactId>
@@ -105,17 +99,14 @@
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
- <version>6.6.2</version>
</dependency>
<dependency>
- <groupId>org.elasticsearch.client</groupId>
- <artifactId>elasticsearch-rest-client</artifactId>
- <version>6.6.2</version>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-client</artifactId>
</dependency>
<dependency>
- <groupId>org.elasticsearch.client</groupId>
- <artifactId>elasticsearch-rest-high-level-client</artifactId>
- <version>6.6.2</version>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-high-level-client</artifactId>
</dependency>
<!-- Testing packages -->
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/service/ElasticClientHandlerAws.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/service/ElasticClientHandlerAws.java
index 3c7383a80aaa9aaf1a7fa8943673fa86dc0f371b..e6003130aad08cfdb49e57d70cee54f17eca625c 100644
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/service/ElasticClientHandlerAws.java
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/service/ElasticClientHandlerAws.java
@@ -44,8 +44,7 @@ public class ElasticClientHandlerAws extends ElasticClientHandler {
RestClientBuilder builder = RestClient.builder(new HttpHost(host, port, protocolScheme));
builder.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder
.setConnectTimeout(REST_CLIENT_CONNECT_TIMEOUT)
- .setSocketTimeout(REST_CLIENT_SOCKET_TIMEOUT));
- builder.setMaxRetryTimeoutMillis(REST_CLIENT_RETRY_TIMEOUT);
+ .setSocketTimeout(REST_CLIENT_SOCKET_TIMEOUT));
if(isLocalHost(host)) {
builder.setHttpClientConfigCallback(httpAsyncClientBuilder -> httpAsyncClientBuilder.setSSLHostnameVerifier((s, sslSession) -> true));
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java
new file mode 100644
index 0000000000000000000000000000000000000000..7210b4aaa06bb72b607ae31353835ca4353a19d1
--- /dev/null
+++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java
@@ -0,0 +1,168 @@
+// Copyright © Azure
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.azure.util;
+
+import com.google.common.base.Strings;
+import com.google.common.reflect.TypeToken;
+import com.google.gson.*;
+import com.microsoft.azure.servicebus.Message;
+import lombok.extern.java.Log;
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.azure.servicebus.ITopicClientFactory;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.indexer.OperationType;
+import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
+import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
+import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
+import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
+import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.opengroup.osdu.indexer.service.StorageService;
+import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Primary;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+import javax.inject.Inject;
+import javax.inject.Named;
+import java.lang.reflect.Type;
+import java.nio.charset.StandardCharsets;
+import java.util.stream.Collectors;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+
+@Log
+@Component
+@RequestScope
+@Primary
+public class IndexerQueueTaskBuilderAzure extends IndexerQueueTaskBuilder {
+
+ @Autowired
+ private ITopicClientFactory topicClientFactory;
+
+ @Inject
+ private IndexerConfigurationProperties configurationProperties;
+
+ @Inject
+ private JaxRsDpsLog logger;
+
+ @Inject
+ @Named("SERVICE_BUS_TOPIC")
+ private String serviceBusTopic;
+
+ @Inject
+ private StorageService storageService;
+
+
+ @Override
+ public void createWorkerTask(String payload, Long countdownMillis, DpsHeaders headers) {
+ headers.addCorrelationIdIfMissing();
+ createTask(payload, headers);
+ }
+
+ @Override
+ public void createReIndexTask(String payload, Long countdownMillis, DpsHeaders headers) {
+ headers.addCorrelationIdIfMissing();
+ publishAllRecordsToServiceBus(payload, headers);
+ }
+
+ private void publishAllRecordsToServiceBus(String payload, DpsHeaders headers) {
+ // fetch all the remaining records
+ // This logic is temporary and would be updated to call the storage service async.
+ // Currently the storage client can't be called out of request scope hence making the
+ // storage calls sync here
+ Gson gson = new Gson();
+ RecordReindexRequest recordReindexRequest = gson.fromJson(payload, RecordReindexRequest.class);
+ final String recordKind = recordReindexRequest.getKind();
+ RecordQueryResponse recordQueryResponse = null;
+
+ try {
+ do {
+ if (recordQueryResponse != null) {
+ recordReindexRequest = RecordReindexRequest.builder().cursor(recordQueryResponse.getCursor()).kind(recordKind).build();
+ }
+ recordQueryResponse = this.storageService.getRecordsByKind(recordReindexRequest);
+ if (recordQueryResponse.getResults() != null && recordQueryResponse.getResults().size() != 0) {
+
+ List<RecordInfo> records = recordQueryResponse.getResults().stream()
+ .map(record -> RecordInfo.builder().id(record).kind(recordKind).op(OperationType.create.name()).build()).collect(Collectors.toList());
+
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(DpsHeaders.ACCOUNT_ID, headers.getPartitionIdWithFallbackToAccountId());
+ attributes.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
+ attributes.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
+
+ RecordChangedMessages recordChangedMessages = RecordChangedMessages.builder().data(gson.toJson(records)).attributes(attributes).build();
+ String recordChangedMessagePayload = gson.toJson(recordChangedMessages);
+ createTask(recordChangedMessagePayload, headers);
+ }
+ } while (!Strings.isNullOrEmpty(recordQueryResponse.getCursor()) && recordQueryResponse.getResults().size() == configurationProperties.getStorageRecordsBatchSize());
+
+ } catch (AppException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Unknown error", "An unknown error has occurred.", e);
+ }
+ }
+
+ private void createTask(String payload, DpsHeaders headers) {
+ Gson gson = new Gson();
+ RecordChangedMessages receivedPayload = gson.fromJson(payload, RecordChangedMessages.class);
+
+ Message message = new Message();
+ Map<String, Object> properties = new HashMap<>();
+
+ // properties
+ properties.put(DpsHeaders.ACCOUNT_ID, headers.getPartitionIdWithFallbackToAccountId());
+ properties.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
+ headers.addCorrelationIdIfMissing();
+ properties.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
+ message.setProperties(properties);
+
+ // data
+ List<RecordInfo> recordInfos = parseRecordsAsJSON(receivedPayload.getData());
+
+ // add all to body {"message": {"data":[], "id":...}}
+ JsonObject jo = new JsonObject();
+ jo.add("data", gson.toJsonTree(recordInfos));
+ jo.addProperty(DpsHeaders.ACCOUNT_ID, headers.getPartitionIdWithFallbackToAccountId());
+ jo.addProperty(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
+ jo.addProperty(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
+ JsonObject jomsg = new JsonObject();
+ jomsg.add("message", jo);
+
+ message.setBody(jomsg.toString().getBytes(StandardCharsets.UTF_8));
+ message.setContentType("application/json");
+
+ try {
+ logger.info("Indexer publishes message to Service Bus " + headers.getCorrelationId());
+ topicClientFactory.getClient(headers.getPartitionId(), serviceBusTopic).send(message);
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ }
+ }
+
+ private List<RecordInfo> parseRecordsAsJSON(String inputPayload) {
+ Gson gson = new Gson();
+ Type type = new TypeToken<List<RecordInfo>>(){}.getType();
+ List<RecordInfo> recordInfoList = gson.fromJson(inputPayload, type);
+ return recordInfoList;
+ }
+}
diff --git a/provider/indexer-ibm/pom.xml b/provider/indexer-ibm/pom.xml
index a8ef4cf4cb9bc3d439b114778adc2cad62fc4fc6..f0a0c5f9249d3faba1e8b92e25fd97aced2ff4e8 100644
--- a/provider/indexer-ibm/pom.xml
+++ b/provider/indexer-ibm/pom.xml
@@ -89,17 +89,14 @@
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
- <version>6.6.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
- <version>6.6.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
- <version>6.6.2</version>
</dependency>
<!-- Test Dependencies -->
diff --git a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerMappingServiceTest.java b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerMappingServiceTest.java
index 7a6447711dda6b83ba8ba520c2480c45a276647f..d6982c113fc059e87626d2f931e4f2288f7fd3fd 100644
--- a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerMappingServiceTest.java
+++ b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerMappingServiceTest.java
@@ -1,12 +1,12 @@
/* Licensed Materials - Property of IBM */
/* (c) Copyright IBM Corp. 2020. All Rights Reserved.*/
-
package org.opengroup.osdu.indexer.ibm.service;
import org.apache.http.StatusLine;
import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest;
import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
-import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.*;
@@ -106,8 +106,7 @@ public class IndexerMappingServiceTest {
public void should_returnValidMapping_givenTrueMerge_createMappingTest() {
try {
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
-
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true);
assertEquals(this.mappingValid, mapping);
} catch (Exception e) {
@@ -119,8 +118,7 @@ public class IndexerMappingServiceTest {
public void should_returnValidMapping_givenExistType_createMappingTest() {
try {
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
-
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
IndexerMappingServiceImpl indexerMappingServiceLocal = PowerMockito.spy(new IndexerMappingServiceImpl());
doReturn(false).when(indexerMappingServiceLocal).isTypeExist(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any());
String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true);
@@ -129,7 +127,7 @@ public class IndexerMappingServiceTest {
fail("Should not throw this exception" + e.getMessage());
}
}
-
+
@Test
public void should_update_indices_field_with_keyword_when_valid_indices() throws Exception {
try {
@@ -137,33 +135,33 @@ public class IndexerMappingServiceTest {
indices.add("indices 1");
GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();
builder.field("any field", new HashMap());
builder.endObject();
BytesReference bytesReference = BytesReference.bytes(builder);
- FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
- Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
mappingBuilder.put("any index 1", mapBuilder);
mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
mapping.put("indices 1", mappingBuilder);
when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
- doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
BulkByScrollResponse response = mock(BulkByScrollResponse.class);
doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
-
+
this.sut.updateIndexMappingForIndicesOfSameType( indices,"any field");
} catch (Exception e) {
fail("Should not throw this exception" + e.getMessage());
}
}
-
+
@Test(expected = AppException.class)
public void should_throw_exception_if_someIndex_is_invalid_andWeIndexfield_with_keyword() throws Exception {
try {
@@ -171,28 +169,28 @@ public class IndexerMappingServiceTest {
indices.add("invalid 1");
GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();
builder.field("any field", new HashMap());
builder.endObject();
BytesReference bytesReference = BytesReference.bytes(builder);
- FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
- Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
mappingBuilder.put("any index 1", mapBuilder);
mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
mapping.put("indices 1", mappingBuilder);
when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
- doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
BulkByScrollResponse response = mock(BulkByScrollResponse.class);
doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
-
- this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
+
+ this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
} catch (Exception e) {
throw e;
}
@@ -205,22 +203,22 @@ public class IndexerMappingServiceTest {
indices.add("indices 1");
GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();
builder.field("any field", new HashMap());
builder.endObject();
BytesReference bytesReference = BytesReference.bytes(builder);
- FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
- Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
mappingBuilder.put("any index 1", mapBuilder);
mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
mapping.put("indices 1", mappingBuilder);
when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
- doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
BulkByScrollResponse response = mock(BulkByScrollResponse.class);
doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
@@ -240,22 +238,22 @@ public class IndexerMappingServiceTest {
indices.add("indices Invalid");
GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenThrow(new ElasticsearchException(""));
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenThrow(new ElasticsearchException(""));
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();
builder.field("any field", new HashMap());
builder.endObject();
BytesReference bytesReference = BytesReference.bytes(builder);
- FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
- Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
mappingBuilder.put("any index 1", mapBuilder);
mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
mapping.put("indices 1", mappingBuilder);
when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
- doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
BulkByScrollResponse response = mock(BulkByScrollResponse.class);
doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
@@ -274,26 +272,26 @@ public class IndexerMappingServiceTest {
indices.add("indices Invalid");
GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();
builder.field("any field", new HashMap());
builder.endObject();
BytesReference bytesReference = BytesReference.bytes(builder);
- FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
- Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
mappingBuilder.put("any index 1", mapBuilder);
mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
mapping.put("indices 1", mappingBuilder);
when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
- doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
BulkByScrollResponse response = mock(BulkByScrollResponse.class);
doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
- when(this.indicesClient.putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class))).thenThrow(new ElasticsearchException(""));
+ when(this.indicesClient.putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class))).thenThrow(new ElasticsearchException(""));
when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
} catch (AppException e) {
@@ -303,3 +301,4 @@ public class IndexerMappingServiceTest {
}
}
}
+
diff --git a/testing/indexer-test-aws/pom.xml b/testing/indexer-test-aws/pom.xml
index e8c2f455b1a331a3cff88dbb309e509e59b1cbb5..9335cf4e8d5323f6ad973d3114af68a272a27220 100644
--- a/testing/indexer-test-aws/pom.xml
+++ b/testing/indexer-test-aws/pom.xml
@@ -73,6 +73,22 @@
</dependency>
<!-- third party Apache 2.0 license packages -->
+ <!--Elasticsearch-->
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch</artifactId>
+ <version>7.8.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-client</artifactId>
+ <version>7.8.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-high-level-client</artifactId>
+ <version>7.8.1</version>
+ </dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
diff --git a/testing/indexer-test-core/src/main/resources/features/indexrecord/IndexRecord.feature b/testing/indexer-test-core/src/main/resources/features/indexrecord/IndexRecord.feature
index c281ec4a0b174ec98eb781f8e603eba3b3968ca1..a06fa2efcbcb9107b184934c0304bf105afbb85e 100644
--- a/testing/indexer-test-core/src/main/resources/features/indexrecord/IndexRecord.feature
+++ b/testing/indexer-test-core/src/main/resources/features/indexrecord/IndexRecord.feature
@@ -15,8 +15,8 @@ Feature: Indexing of the documents
Examples:
| kind | recordFile | number | index | type | acl | mapping |
- | "tenant1:testindex<timestamp>:well:1.0.0" | "index_records_1" | 5 | "tenant1-testindex<timestamp>-well-1.0.0" | "well" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
- | "tenant1:testindex<timestamp>:well:3.0.0" | "index_records_1" | 5 | "tenant1-testindex<timestamp>-well-3.0.0" | "well" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
+ | "tenant1:testindex<timestamp>:well:1.0.0" | "index_records_1" | 5 | "tenant1-testindex<timestamp>-well-1.0.0" | "well" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Country":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"County":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"EmptyAttribute":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Established":{"type":"date"},"Field":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellName":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellStatus":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellType":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
+ | "tenant1:testindex<timestamp>:well:3.0.0" | "index_records_1" | 5 | "tenant1-testindex<timestamp>-well-3.0.0" | "well" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Country":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"County":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"EmptyAttribute":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Established":{"type":"date"},"Field":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellName":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellStatus":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellType":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
Scenario Outline: Ingest the record and Index in the Elastic Search with bad attribute
When I ingest records with the <recordFile> with <acl> for a given <kind>
diff --git a/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature b/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
index 4152a53252458bd809ae9ffe5da447cf302060d7..9a340f29a585b112ce8a12481f840a9ac2ce8d6a 100644
--- a/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
+++ b/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
@@ -15,8 +15,8 @@ Feature: Indexing of the documents
Examples:
| kind | recordFile | number | index | type | acl | mapping |
- | "tenant1:indexer-int-test:sample-schema-1:1.0.4" | "index_records_1" | 5 | "tenant1-indexer-int-test-sample-schema-1-1.0.4" | "sample-schema-1" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
- | "tenant1:indexer-int-test:sample-schema-3:1.0.4" | "index_records_1" | 5 | "tenant1-indexer-int-test-sample-schema-3-1.0.4" | "sample-schema-3" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
+ | "tenant1:indexer-int-test:sample-schema-1:1.0.4" | "index_records_schema_1" | 5 | "tenant1-indexer-int-test-sample-schema-1-1.0.4" | "sample-schema-1" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Country":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"County":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"EmptyAttribute":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Established":{"type":"date"},"Field":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellName":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellStatus":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellType":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
+ | "tenant1:indexer-int-test:sample-schema-3:1.0.4" | "index_records_schema_1" | 5 | "tenant1-indexer-int-test-sample-schema-3-1.0.4" | "sample-schema-3" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Country":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"County":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"EmptyAttribute":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Established":{"type":"date"},"Field":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellName":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellStatus":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellType":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
Scenario Outline: Ingest the record and Index in the Elastic Search with bad attribute
When I ingest records with the <recordFile> with <acl> for a given <kind>
diff --git a/testing/indexer-test-ibm/pom.xml b/testing/indexer-test-ibm/pom.xml
index 1d0ec6267671426190b4f425241b0993597c5f75..f85b8b8ad1a2d3216ac246e5fe580596d3cf6e35 100644
--- a/testing/indexer-test-ibm/pom.xml
+++ b/testing/indexer-test-ibm/pom.xml
@@ -114,17 +114,17 @@
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
- <version>6.6.2</version>
+ <version>7.8.1</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
- <version>6.6.2</version>
+ <version>7.8.1</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
- <version>6.6.2</version>
+ <version>7.8.1</version>
</dependency>
<!--Logging-->