diff --git a/.fossa.yml b/.fossa.yml
index 005b660973c55b1ecb92b77bff52c5159978f3db..09b5fc51ab43d1499d01a0da799b7402cee5c50b 100644
--- a/.fossa.yml
+++ b/.fossa.yml
@@ -32,3 +32,7 @@ analyze:
type: mvn
target: provider/indexer-ibm/pom.xml
path: .
+ - name: indexer-reference
+ type: mvn
+ target: provider/indexer-reference/pom.xml
+ path: .
diff --git a/.gitignore b/.gitignore
index 9e269f1b78050908088da343128dea99d75bacdc..e8c005ae687055f7d4bcf5ada49f03d435dc6f64 100644
--- a/.gitignore
+++ b/.gitignore
@@ -41,3 +41,5 @@ provider/indexer-gcp/bin/*
# Environment configuration
*.env
.envrc
+/.gradle/checksums/checksums.lock
+/.gradle/6.7/fileHashes/fileHashes.lock
diff --git a/NOTICE b/NOTICE
index 313c46e4e329833ef5b24cb514cbf17b0456f536..a9257a173b8c7e34a8734ecb7d43f08e6b7e4e01 100644
--- a/NOTICE
+++ b/NOTICE
@@ -210,6 +210,7 @@ The following software have components provided under the terms of this license:
- AutoValue (from )
- Azure AD Spring Security Integration Spring Boot Starter (from https://github.com/Microsoft/azure-spring-boot)
- Azure Metrics Spring Boot Starter (from https://github.com/Microsoft/azure-spring-boot)
+- BSON (from http://bsonspec.org)
- Bean Validation API (from http://beanvalidation.org)
- Byte Buddy (without dependencies) (from )
- Byte Buddy Java agent (from )
@@ -236,18 +237,24 @@ The following software have components provided under the terms of this license:
- Elasticsearch: 5.0.0-alpha5 (from https://github.com/elastic/elasticsearch)
- FindBugs-jsr305 (from http://findbugs.sourceforge.net/)
- Google APIs Client Library for Java (from )
+- Google APIs Client Library for Java (from )
+- Google App Engine extensions to the Google HTTP Client Library for Java. (from )
- Google App Engine extensions to the Google HTTP Client Library for Java. (from )
- Google Cloud Core (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-core)
- Google Cloud Core HTTP (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-core-http)
+- Google Cloud Core HTTP (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-core-http)
- Google Cloud Core gRPC (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-core-grpc)
- Google Cloud Datastore (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-datastore)
+- Google Cloud Datastore (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-datastore)
- Google Cloud Key Management Service (KMS) API v1-rev22-1.23.0 (from )
- Google Cloud Logging (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-logging)
- Google Cloud Pub/Sub (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-pubsub)
+- Google Cloud Pub/Sub (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-pubsub)
- Google Cloud Storage (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-storage)
- Google HTTP Client Library for Java (from https://github.com/google/google-http-java-client.git)
- Google HTTP Client Library for Java (from https://github.com/google/google-http-java-client.git)
- Google OAuth Client Library for Java (from )
+- Google OAuth Client Library for Java (from )
- Gson (from https://github.com/google/gson)
- Gson (from https://github.com/google/gson)
- Guava InternalFutureFailureAccess and InternalFutures (from )
@@ -277,6 +284,7 @@ The following software have components provided under the terms of this license:
- Jackson (from http://jackson.codehaus.org)
- Jackson (from http://jackson.codehaus.org)
- Jackson 2 extensions to the Google HTTP Client Library for Java. (from https://github.com/google/google-http-java-client.git/google-http-client-jackson2)
+- Jackson 2 extensions to the Google HTTP Client Library for Java. (from https://github.com/google/google-http-java-client.git/google-http-client-jackson2)
- Jackson dataformat: CBOR (from http://github.com/FasterXML/jackson-dataformats-binary)
- Jackson dataformat: CBOR (from http://github.com/FasterXML/jackson-dataformats-binary)
- Jackson datatype: JSR310 (from http://wiki.fasterxml.com/JacksonModuleJSR310)
@@ -375,6 +383,8 @@ The following software have components provided under the terms of this license:
- Mockito (from http://www.mockito.org)
- Mockito (from http://mockito.org)
- Mojo's Maven plugin for Cobertura (from http://mojo.codehaus.org/cobertura-maven-plugin/)
+- MongoDB Driver (from http://www.mongodb.org)
+- MongoDB Java Driver Core (from http://www.mongodb.org)
- Netty Reactive Streams HTTP support (from )
- Netty Reactive Streams Implementation (from )
- Netty/Buffer (from http://netty.io/)
@@ -420,20 +430,24 @@ The following software have components provided under the terms of this license:
- PowerMock (from http://www.powermock.org)
- Protocol Buffer extensions to the Google HTTP Client Library for Java. (from )
- QpidJMS Client (from )
+- RabbitMQ Java Client (from http://www.rabbitmq.com)
- Reactive Streams Netty driver (from https://github.com/reactor/reactor-netty)
- Retrofit (from )
- Servlet Specification 2.5 API (from )
- Simple XML (from http://simple.sourceforge.net)
- SnakeYAML (from http://www.snakeyaml.org)
- Spatial4J (from http://www.locationtech.org/projects/locationtech.spatial4j)
+- Spring AMQP Core (from https://projects.spring.io/spring-amqp)
- Spring AOP (from https://github.com/spring-projects/spring-framework)
- Spring Beans (from https://github.com/spring-projects/spring-framework)
- Spring Boot (from http://projects.spring.io/spring-boot/)
+- Spring Boot AMQP Starter (from http://projects.spring.io/spring-boot/)
- Spring Boot Actuator (from http://projects.spring.io/spring-boot/)
- Spring Boot Actuator AutoConfigure (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-actuator-autoconfigure)
- Spring Boot Actuator Starter (from http://projects.spring.io/spring-boot/)
- Spring Boot AutoConfigure (from http://projects.spring.io/spring-boot/)
- Spring Boot Configuration Processor (from http://projects.spring.io/spring-boot/)
+- Spring Boot Data MongoDB Starter (from http://projects.spring.io/spring-boot/)
- Spring Boot Dependencies (from http://projects.spring.io/spring-boot/)
- Spring Boot Jersey Starter (from http://projects.spring.io/spring-boot/)
- Spring Boot Json Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-json)
@@ -452,11 +466,14 @@ The following software have components provided under the terms of this license:
- Spring Core (from https://github.com/spring-projects/spring-framework)
- Spring Data Core (from )
- Spring Data Core (from )
+- Spring Data MongoDB - Core (from )
- Spring Expression Language (SpEL) (from https://github.com/spring-projects/spring-framework)
- Spring JMS (from https://github.com/spring-projects/spring-framework)
- Spring Messaging (from https://github.com/spring-projects/spring-framework)
- Spring Plugin - Core (from )
- Spring Plugin - Metadata Extension (from )
+- Spring RabbitMQ Support (from https://projects.spring.io/spring-amqp)
+- Spring Retry (from http://www.springsource.org)
- Spring Security JWT Library (from http://github.com/spring-projects/spring-security-oauth)
- Spring Security JWT Library (from http://github.com/spring-projects/spring-security-oauth)
- Spring TestContext Framework (from https://github.com/spring-projects/spring-framework)
@@ -475,8 +492,10 @@ The following software have components provided under the terms of this license:
- cli (from https://github.com/elastic/elasticsearch)
- cli (from https://github.com/elastic/elasticsearch)
- com.google.api.grpc:grpc-google-cloud-pubsub-v1 (from https://github.com/googleapis/googleapis)
+- com.google.api.grpc:grpc-google-cloud-pubsub-v1 (from https://github.com/googleapis/googleapis)
- com.google.api.grpc:proto-google-cloud-logging-v2 (from https://github.com/googleapis/googleapis)
- com.google.api.grpc:proto-google-cloud-pubsub-v1 (from https://github.com/googleapis/googleapis)
+- com.google.api.grpc:proto-google-cloud-pubsub-v1 (from https://github.com/googleapis/googleapis)
- com.google.api.grpc:proto-google-common-protos (from https://github.com/googleapis/googleapis)
- com.google.api.grpc:proto-google-iam-v1 (from https://github.com/googleapis/googleapis)
- commons-collections (from )
@@ -534,6 +553,7 @@ The following software have components provided under the terms of this license:
- powermock-module-junit4-common (from )
- powermock-reflect (from )
- proto-google-cloud-datastore-v1 (from https://github.com/googleapis/api-client-staging)
+- proto-google-cloud-datastore-v1 (from https://github.com/googleapis/api-client-staging)
- proton-j (from )
- rank-eval (from https://github.com/elastic/elasticsearch)
- rank-eval (from https://github.com/elastic/elasticsearch)
@@ -579,6 +599,7 @@ The following software have components provided under the terms of this license:
- GAX (Google Api eXtensions) (from https://github.com/googleapis)
- GAX (Google Api eXtensions) (from https://github.com/googleapis)
- GAX (Google Api eXtensions) (from https://github.com/googleapis)
+- GAX (Google Api eXtensions) (from https://github.com/googleapis)
- Hamcrest Core (from http://hamcrest.org/)
- Lucene Common Analyzers (from )
- Lucene Common Analyzers (from )
@@ -607,6 +628,8 @@ The following software have components provided under the terms of this license:
- GAX (Google Api eXtensions) (from https://github.com/googleapis)
- GAX (Google Api eXtensions) (from https://github.com/googleapis)
- GAX (Google Api eXtensions) (from https://github.com/googleapis)
+- GAX (Google Api eXtensions) (from https://github.com/googleapis)
+- Google APIs Client Library for Java (from )
- Google APIs Client Library for Java (from )
- Google Auth Library for Java - Credentials (from )
- Google Auth Library for Java - OAuth2 HTTP (from )
@@ -653,6 +676,7 @@ The following software have components provided under the terms of this license:
- Checker Qual (from https://checkerframework.org)
- FindBugs-jsr305 (from http://findbugs.sourceforge.net/)
+- MongoDB Java Driver Core (from http://www.mongodb.org)
========================================================================
CC-BY-4.0
@@ -767,6 +791,7 @@ The following software have components provided under the terms of this license:
- JavaMail API (from )
- Mojo's Maven plugin for Cobertura (from http://mojo.codehaus.org/cobertura-maven-plugin/)
- OSGi resource locator (from )
+- RabbitMQ Java Client (from http://www.rabbitmq.com)
- Run Level Service (from )
- ServiceLocator Default Implementation (from git://java.net/hk2~git/hk2-locator)
- aopalliance-repackaged (from )
@@ -806,6 +831,7 @@ The following software have components provided under the terms of this license:
- JavaBeans Activation Framework API jar (from )
- JavaMail API (from )
- OSGi resource locator (from )
+- RabbitMQ Java Client (from http://www.rabbitmq.com)
- ServiceLocator Default Implementation (from git://java.net/hk2~git/hk2-locator)
- aopalliance-repackaged (from )
- javax.annotation-api (from http://jcp.org/en/jsr/detail?id=250)
@@ -872,6 +898,7 @@ The following software have components provided under the terms of this license:
- Apache Log4j API (from )
- Apache Log4j Core (from )
+- RabbitMQ Java Client (from http://www.rabbitmq.com)
========================================================================
MIT
@@ -938,6 +965,7 @@ The following software have components provided under the terms of this license:
- Cobertura code coverage (from http://cobertura.sourceforge.net)
- Javassist (from http://www.javassist.org/)
- Javassist (from http://www.javassist.org/)
+- RabbitMQ Java Client (from http://www.rabbitmq.com)
========================================================================
MPL-2.0
@@ -946,6 +974,7 @@ The following software have components provided under the terms of this license:
- Javassist (from http://www.javassist.org/)
- Javassist (from http://www.javassist.org/)
+- RabbitMQ Java Client (from http://www.rabbitmq.com)
========================================================================
PHP-3.01
@@ -1022,6 +1051,7 @@ The following software have components provided under the terms of this license:
- Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure client library for Blob Storage (from https://github.com/Azure/azure-sdk-for-java)
- Project Lombok (from https://projectlombok.org)
+- RabbitMQ Java Client (from http://www.rabbitmq.com)
- Spring Security JWT Library (from http://github.com/spring-projects/spring-security-oauth)
- Spring Security JWT Library (from http://github.com/spring-projects/spring-security-oauth)
- Spring Web (from https://github.com/spring-projects/spring-framework)
@@ -1042,6 +1072,7 @@ The following software have components provided under the terms of this license:
- Byte Buddy (without dependencies) (from )
- JUnit (from http://junit.org)
- JavaBeans Activation Framework API jar (from )
+- RabbitMQ Java Client (from http://www.rabbitmq.com)
- Servlet Specification 2.5 API (from )
- Spongy Castle (from http://rtyley.github.io/spongycastle/)
- System Rules (from http://stefanbirkner.github.io/system-rules/)
diff --git a/devops/azure/chart/templates/deployment.yaml b/devops/azure/chart/templates/deployment.yaml
index 818f9caceecdbaa7d2492671fa8b8629c1ab6e11..6ef26c036f8cb921e6ad534eef37cbc74127934a 100644
--- a/devops/azure/chart/templates/deployment.yaml
+++ b/devops/azure/chart/templates/deployment.yaml
@@ -56,7 +56,7 @@ spec:
readOnly: true
env:
- name: spring_application_name
- value: indexer
+ value: {{ .Chart.Name }}
- name: server.servlet.contextPath
value: /api/indexer/v2/
- name: server_port
@@ -88,7 +88,7 @@ spec:
- name: servicebus_topic_name
value: indexing-progress
- name: entitlements_service_endpoint
- value: http://entitlements-azure/entitlements/v1
+ value: http://entitlements/api/entitlements/v2
- name: entitlements_service_api_key
value: "OBSOLETE"
- name: schema_service_url
@@ -100,7 +100,7 @@ spec:
- name: STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST
value: http://storage/api/storage/v2/query/records:batch
- name: STORAGE_QUERY_RECORD_HOST
- value: http://storage/api/storage/v2/records
+ value: http://storage/api/storage/v2/query/records
- name: partition_service_endpoint
value: http://partition/api/partition/v1
- name: azure_istioauth_enabled
diff --git a/devops/azure/chart/templates/hpa.yaml b/devops/azure/chart/templates/hpa.yaml
index d9d00bfc65e3df192ca241337772c937dbc1ff9e..68a00c86912a249472eec7cf49411f38b72322d4 100644
--- a/devops/azure/chart/templates/hpa.yaml
+++ b/devops/azure/chart/templates/hpa.yaml
@@ -24,7 +24,7 @@ spec:
kind: Deployment
name: {{ .Chart.Name }}
minReplicas: {{ .Values.global.replicaCount }}
- maxReplicas: 5
+ maxReplicas: 10
metrics:
- type: Resource
resource:
diff --git a/devops/azure/release.yaml b/devops/azure/release.yaml
index dca21e377c5a3c6d13d9dc59ff9e2f661851d2c2..dad51b35bebd9069044ac08b8848245a29165fb7 100644
--- a/devops/azure/release.yaml
+++ b/devops/azure/release.yaml
@@ -161,4 +161,4 @@ spec:
- name: STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST
value: http://storage/api/storage/v2/query/records:batch
- name: STORAGE_QUERY_RECORD_HOST
- value: http://storage/api/storage/v2/records
+ value: http://storage/api/storage/v2/query/records
diff --git a/indexer-core/pom.xml b/indexer-core/pom.xml
index 919ad21103a071122b569c0ca33c2215a6ecaff6..3e01109f8b02130cedfbcb1d197b557fa2f9e7e9 100644
--- a/indexer-core/pom.xml
+++ b/indexer-core/pom.xml
@@ -4,18 +4,19 @@
<parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-service</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.8.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>indexer-core</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.8.0-SNAPSHOT</version>
<name>indexer-core</name>
<description>Indexer Service Core</description>
<packaging>jar</packaging>
<properties>
<commons-beanutils.version>1.9.4</commons-beanutils.version>
+ <osdu.oscorecommon.version>0.6.5</osdu.oscorecommon.version>
</properties>
<dependencies>
@@ -33,6 +34,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
+ <version>${osdu.oscorecommon.version}</version>
</dependency>
<!-- spring boot dependencies -->
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/error/GlobalExceptionMapperCore.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/error/GlobalExceptionMapperCore.java
new file mode 100644
index 0000000000000000000000000000000000000000..6e546e14cf7197a480bf84de084a3cf628373ad0
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/error/GlobalExceptionMapperCore.java
@@ -0,0 +1,88 @@
+// Copyright © Microsoft Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.error;
+
+import javax.validation.ValidationException;
+
+import javassist.NotFoundException;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.Ordered;
+import org.springframework.core.annotation.Order;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.security.access.AccessDeniedException;
+import org.springframework.web.bind.annotation.ControllerAdvice;
+import org.springframework.web.bind.annotation.ExceptionHandler;
+import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler;
+import org.opengroup.osdu.core.common.model.http.AppException;
+
+@Order(Ordered.HIGHEST_PRECEDENCE - 1)
+@ControllerAdvice
+public class GlobalExceptionMapperCore extends ResponseEntityExceptionHandler {
+
+ @Autowired
+ private JaxRsDpsLog logger;
+
+ @ExceptionHandler(AppException.class)
+ protected ResponseEntity<Object> handleAppException(AppException e) {
+ return this.getErrorResponse(e);
+ }
+
+ @ExceptionHandler(ValidationException.class)
+ protected ResponseEntity<Object> handleValidationException(ValidationException e) {
+ return this.getErrorResponse(
+ new AppException(HttpStatus.BAD_REQUEST.value(), "Validation error.", e.getMessage(), e));
+ }
+
+ @ExceptionHandler(NotFoundException.class)
+ protected ResponseEntity<Object> handleNotFoundException(NotFoundException e) {
+ return this.getErrorResponse(
+ new AppException(HttpStatus.NOT_FOUND.value(), "Resource not found.", e.getMessage(), e));
+ }
+
+ @ExceptionHandler(AccessDeniedException.class)
+ protected ResponseEntity<Object> handleAccessDeniedException(AccessDeniedException e) {
+ return this.getErrorResponse(
+ new AppException(HttpStatus.FORBIDDEN.value(), "Access denied", e.getMessage(), e));
+ }
+
+ @ExceptionHandler(Exception.class)
+ protected ResponseEntity<Object> handleGeneralException(Exception e) {
+ return this.getErrorResponse(
+ new AppException(HttpStatus.INTERNAL_SERVER_ERROR.value(), "Server error.",
+ "An unknown error has occurred.", e));
+ }
+
+ private ResponseEntity<Object> getErrorResponse(AppException e) {
+
+ String exceptionMsg = e.getOriginalException() != null
+ ? e.getOriginalException().getMessage()
+ : e.getError().getMessage();
+
+ if( e.getCause() instanceof Exception) {
+ Exception original = (Exception) e.getCause();
+ this.logger.error(original.getMessage(), original);
+ }
+
+ if (e.getError().getCode() > 499) {
+ this.logger.error(exceptionMsg, e);
+ } else {
+ this.logger.warning(exceptionMsg, e);
+ }
+
+ return new ResponseEntity<Object>(e.getError(), HttpStatus.resolve(e.getError().getCode()));
+ }
+}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java
index 3506c2ca092db2688a6f15a2eca790082ab9b05b..d98b1f77281138a9c3de3304350da461cda9ac08 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java
@@ -56,8 +56,10 @@ public class PropertiesProcessor {
public Stream<Map<String, Object>> processItem(AllOfItem allOfItem) {
Preconditions.checkNotNull(allOfItem, "allOfItem cannot be null");
- if (Objects.nonNull(allOfItem.getAllOf())) {
- return allOfItem.getAllOf().stream().flatMap(this::processItem);
+ Stream<Map<String, Object>> ofItems = processOfItems(allOfItem.getAllOf(), allOfItem.getAnyOf(), allOfItem.getOneOf());
+
+ if (Objects.nonNull(ofItems)) {
+ return ofItems;
}
String ref = allOfItem.getRef();
@@ -89,13 +91,34 @@ public class PropertiesProcessor {
new AppException(HttpStatus.SC_NOT_FOUND, "Failed to find definition:" + definitionSubRef,
"Unknown definition:" + definitionSubRef));
- if (Objects.nonNull(definition.getAllOf())) {
- return definition.getAllOf().stream().flatMap(this::processItem);
+ Stream<Map<String, Object>> ofItems =
+ processOfItems(definition.getAllOf(), definition.getAnyOf(), definition.getOneOf());
+
+ if (Objects.nonNull(ofItems)) {
+ return ofItems;
}
return processProperties(definition.getProperties());
}
+ private Stream<Map<String, Object>> processOfItems(List<AllOfItem> allOf, List<AllOfItem> anyOf, List<AllOfItem> oneOf) {
+ Stream<Map<String, Object>> ofItems = null;
+
+ if (Objects.nonNull(allOf)) {
+ ofItems = allOf.stream().flatMap(this::processItem);
+ }
+
+ if (Objects.nonNull(anyOf)) {
+ ofItems = Stream.concat(Optional.ofNullable(ofItems).orElseGet(Stream::empty), anyOf.stream().flatMap(this::processItem));
+ }
+
+ if (Objects.nonNull(oneOf)) {
+ ofItems = Stream.concat(Optional.ofNullable(ofItems).orElseGet(Stream::empty), oneOf.stream().flatMap(this::processItem));
+ }
+
+ return ofItems;
+ }
+
public Stream<Map<String, Object>> processProperties(Map<String, TypeProperty> properties){
return properties.entrySet().stream().flatMap(this::processPropertyEntry);
}
@@ -118,11 +141,10 @@ public class PropertiesProcessor {
return Stream.empty();
}
- if (Objects.nonNull(entry.getValue().getAllOf())) {
- PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey()
- , log, new SchemaConverterPropertiesConfig());
+ Stream<Map<String, Object>> ofItems = processOfItems(entry);
- return entry.getValue().getAllOf().stream().flatMap(propertiesProcessor::processItem);
+ if (Objects.nonNull(ofItems)) {
+ return ofItems;
}
if (Objects.nonNull(entry.getValue().getProperties())) {
@@ -139,6 +161,35 @@ public class PropertiesProcessor {
return storageSchemaEntry(getTypeByDefinitionProperty(entry.getValue()), pathPrefixWithDot + entry.getKey());
}
+ private Stream<Map<String, Object>> processOfItems(Map.Entry<String, TypeProperty> entry) {
+ Stream<Map<String, Object>> ofItems = null;
+
+ if (Objects.nonNull(entry.getValue().getAllOf())) {
+ PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey()
+ , log, new SchemaConverterPropertiesConfig());
+
+ ofItems = entry.getValue().getAllOf().stream().flatMap(propertiesProcessor::processItem);
+ }
+
+ if (Objects.nonNull(entry.getValue().getAnyOf())) {
+ PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey()
+ , log, new SchemaConverterPropertiesConfig());
+
+ ofItems = Stream.concat(Optional.ofNullable(ofItems).orElseGet(Stream::empty),
+ entry.getValue().getAnyOf().stream().flatMap(propertiesProcessor::processItem));
+ }
+
+ if (Objects.nonNull(entry.getValue().getOneOf())) {
+ PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey()
+ , log, new SchemaConverterPropertiesConfig());
+
+ ofItems = Stream.concat(Optional.ofNullable(ofItems).orElseGet(Stream::empty),
+ entry.getValue().getOneOf().stream().flatMap(propertiesProcessor::processItem));
+ }
+
+ return ofItems;
+ }
+
private Stream<Map<String, Object>> storageSchemaEntry(String kind, String path) {
Preconditions.checkNotNullOrEmpty(kind, "kind cannot be null or empty");
Preconditions.checkNotNullOrEmpty(path, "path cannot be null or empty");
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java
index 1dc63193b3db8c761f492165f9363390a709bdca..f613d9f9c77e433f62a333b4688ad47b8e67a245 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java
@@ -97,6 +97,18 @@ public class SchemaToStorageFormatImpl implements SchemaToStorageFormat {
.collect(Collectors.toList()));
}
+ if (schemaData.getAnyOf() != null) {
+ storageSchemaItems.addAll(schemaServiceSchema.getProperties().getData().getAnyOf().stream()
+ .flatMap(propertiesProcessor::processItem)
+ .collect(Collectors.toList()));
+ }
+
+ if (schemaData.getOneOf() != null) {
+ storageSchemaItems.addAll(schemaServiceSchema.getProperties().getData().getOneOf().stream()
+ .flatMap(propertiesProcessor::processItem)
+ .collect(Collectors.toList()));
+ }
+
if (schemaData.getRef() != null) {
storageSchemaItems.addAll(propertiesProcessor.processRef(schemaData.getRef())
.collect(Collectors.toList()));
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterPropertiesConfig.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterPropertiesConfig.java
index f530c842ba0bae004bc3e54e080cca143edb88c9..163664f5f04b0c7ee0b03f4fb4113eddee038235 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterPropertiesConfig.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterPropertiesConfig.java
@@ -24,7 +24,7 @@ public class SchemaConverterPropertiesConfig implements SchemaConverterConfig {
}
private Set<String> getDefaultSupportedArrayTypes() {
- return new HashSet<>(Arrays.asList("boolean", "integer", "number", "string"));
+ return new HashSet<>(Arrays.asList("boolean", "integer", "number", "string", "object"));
}
private Map<String, String> getDefaultSpecialDefinitionsMap() {
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/readme.md b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/readme.md
index 845325962f475bd6c02e3999d02a32b43b7fafba..3b5e146c5760cd3084651a612fae228d5a80c035 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/readme.md
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/readme.md
@@ -299,4 +299,67 @@ Ignored for now (array of references)
```
-\"kind\": \"long\"
\ No newline at end of file
+\"kind\": \"long\"
+
+Processing specifics
+----------------------------------------------------------------------------
+
+allOf, anyOf and oneOf tags are processed at the same way. All internal data(properties) are included into converted schema.
+
+For instance
+```json
+{
+ "definitions": {
+ "wellboreData1": {
+ "properties": {
+ "prop1": {
+ "type": "string"
+ }
+ }
+ },
+ "wellboreData2": {
+ "properties": {
+ "prop2": {
+ "type": "string"
+ }
+ }
+ }
+ },
+ "properties": {
+ "data": {
+ "allOf": [
+ {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/wellboreData1"
+ } ],
+ "oneOf": [
+ {
+ "$ref": "#/definitions/wellboreData2"
+ }
+ ]
+ }
+ ]
+ }
+ }
+}
+
+```
+
+is converted to
+
+```json
+{
+ "kind": "KIND_VAL",
+ "schema": [
+ {
+ "kind": "string",
+ "path": "prop1"
+ },
+ {
+ "kind": "string",
+ "path": "prop2"
+ }
+ ]
+}
+```
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/AllOfItem.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/AllOfItem.java
index bef9a14bc622a148ddb9d2603914015d1d1bad73..1354c3bd1b1dde40a19214ae1f3d5e56298f5b2a 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/AllOfItem.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/AllOfItem.java
@@ -27,4 +27,6 @@ public class AllOfItem {
private String type;
private Map<String, TypeProperty> properties;
private List<AllOfItem> allOf;
+ private List<AllOfItem> oneOf;
+ private List<AllOfItem> anyOf;
}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/Definition.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/Definition.java
index be8292a287a15885b64f7da6b4770451b5b4d088..4665c6a82c796dc96ec209f30e9257788b70b8f0 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/Definition.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/Definition.java
@@ -23,4 +23,6 @@ import java.util.Map;
public class Definition {
private Map<String, TypeProperty> properties;
private List<AllOfItem> allOf;
+ private List<AllOfItem> oneOf;
+ private List<AllOfItem> anyOf;
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/PropertiesData.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/PropertiesData.java
index 6c09c0116b6fbe269d93c13931a46493950fbc9a..61f1edb98ab5aa76438ece4eee8c0d15cac85fc1 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/PropertiesData.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/PropertiesData.java
@@ -23,6 +23,8 @@ import java.util.Map;
@Data
public class PropertiesData {
private List<AllOfItem> allOf;
+ private List<AllOfItem> oneOf;
+ private List<AllOfItem> anyOf;
@JsonProperty("$ref")
private String ref;
private Map<String, TypeProperty> properties;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java
index 3053e5d20a85ee5165d5436147b9e5c9a1bafc37..b929bad43f401e97fbc4f5b3ce8fd22344e14a76 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java
@@ -30,4 +30,6 @@ public class TypeProperty {
private Items items;
private Map<String, TypeProperty> properties;
private List<AllOfItem> allOf;
+ private List<AllOfItem> oneOf;
+ private List<AllOfItem> anyOf;
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java
index 9e9026c815055600e3b2a58da8e2c82317e812cd..fae0b4e6f31cd18ad960b841b6d9723e0162cf29 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java
@@ -208,6 +208,16 @@ public class AttributeParsingServiceImpl implements IAttributeParsingService {
}
}
+ @Override
+ public void tryParseNested(String recordId, String name, Object value, Map<String, Object> dataMap) {
+ dataMap.put(name,value);
+ }
+
+ @Override
+ public void tryParseObject(String recordId, String name, Object value, Map<String, Object> dataMap) {
+ dataMap.put(name,value);
+ }
+
private List<String> isArrayType(Object attributeVal) {
try {
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java
index 4050444a40b1bef8f184d8b09c80a078d9e73729..c4fb8ecf2966c3b354eb15e9fb32be8608554feb 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java
@@ -1,7 +1,5 @@
package org.opengroup.osdu.indexer.service;
-import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
-
import java.util.Map;
public interface IAttributeParsingService {
@@ -26,4 +24,8 @@ public interface IAttributeParsingService {
void tryParseGeopoint(String recordId, String attributeName, Map<String, Object> storageRecordData, Map<String, Object> dataMap);
void tryParseGeojson(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap);
+
+ void tryParseNested(String recordId, String name, Object value, Map<String, Object> dataMap);
+
+ void tryParseObject(String recordId, String name, Object value, Map<String, Object> dataMap);
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
index 113dc54bde6c9e4bf11e2fc36ab1725d0be77b72..a63848140947c2a425c69458671733c3beaae684 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
@@ -23,6 +23,7 @@ import org.elasticsearch.client.RestHighLevelClient;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.RequestStatus;
+import org.opengroup.osdu.core.common.model.indexer.ElasticType;
import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
import org.opengroup.osdu.core.common.model.indexer.OperationType;
import org.opengroup.osdu.core.common.model.indexer.StorageType;
@@ -47,6 +48,10 @@ import java.util.Map;
public class IndexSchemaServiceImpl implements IndexSchemaService {
private static final String FLATTENED_SCHEMA = "_flattened";
+ private static final String WELLBORE_MARKER_SET = "WellboreMarkerSet";
+ private static final String MARKERS = "Markers";
+ private static final String WELL_LOG = "WellLog";
+ private static final String CURVES = "Curves";
private final Gson gson = new Gson();
@@ -221,6 +226,7 @@ public class IndexSchemaServiceImpl implements IndexSchemaService {
meta.put(RecordMetaAttribute.TYPE.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.TYPE));
meta.put(RecordMetaAttribute.ACL.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.ACL));
meta.put(RecordMetaAttribute.X_ACL.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.X_ACL));
+ meta.put(RecordMetaAttribute.TAGS.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.TAGS));
meta.put(RecordMetaAttribute.LEGAL.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.LEGAL));
meta.put(RecordMetaAttribute.ANCESTRY.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.ANCESTRY));
meta.put(RecordMetaAttribute.INDEX_STATUS.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.INDEX_STATUS));
@@ -228,6 +234,14 @@ public class IndexSchemaServiceImpl implements IndexSchemaService {
String kind = schemaObj.getKind();
String type = kind.split(":")[2];
+ //TODO temporary fix for https://community.opengroup.org/osdu/platform/system/indexer-service/-/issues/1
+ if(data.get(MARKERS) != null){
+ data.put(MARKERS, ElasticType.NESTED.getValue());
+ }
+ if(data.get(CURVES) != null){
+ data.put(CURVES, ElasticType.NESTED.getValue());
+ }
+
return IndexSchema.builder().dataSchema(data).metaSchema(meta).kind(kind).type(type).build();
} catch (Exception e) {
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
index 95f1c504f9eca6e03027fd4e14e8389fb8cb61bc..acd87b30e4a909d57832758b5260aec100421d41 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
@@ -15,12 +15,6 @@
package org.opengroup.osdu.indexer.service;
import com.google.gson.Gson;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-import javax.inject.Inject;
import org.apache.http.HttpStatus;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
@@ -38,28 +32,29 @@ import org.elasticsearch.index.reindex.UpdateByQueryRequest;
import org.opengroup.osdu.core.common.Constants;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.indexer.DEAnalyzerType;
-import org.opengroup.osdu.core.common.model.indexer.ElasticType;
import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
-import org.opengroup.osdu.core.common.model.indexer.Records;
-import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
import org.opengroup.osdu.core.common.search.Preconditions;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
import org.opengroup.osdu.indexer.util.ElasticClientHandler;
+import org.opengroup.osdu.indexer.util.TypeMapper;
import org.springframework.stereotype.Service;
+import javax.inject.Inject;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
@Service
public class IndexerMappingServiceImpl extends MappingServiceImpl implements IndexerMappingService {
- @Inject
- private IndexerConfigurationProperties configurationProperties;
@Inject
private JaxRsDpsLog log;
@Inject
private ElasticClientHandler elasticClientHandler;
private TimeValue REQUEST_TIMEOUT = TimeValue.timeValueMinutes(1);
-
+
/**
* Create a new type in Elasticsearch
*
@@ -81,7 +76,7 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
*
* @param schema Index schema
* @param type Mapping type
- * @return String JSON represnetation of type and elastic type
+ * @return String JSON representation of type and elastic type
*
* sample index mapping:
* "properties": {
@@ -111,25 +106,14 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
// meta attribute
Map<String, Object> metaMapping = new HashMap<>();
for (Map.Entry<String, Object> entry : schema.getMetaSchema().entrySet()) {
- String key = entry.getKey();
- if (key.equals(RecordMetaAttribute.ACL.getValue()) || key.equals(RecordMetaAttribute.LEGAL.getValue()) || key.equals(RecordMetaAttribute.ANCESTRY.getValue()) || key.equals(RecordMetaAttribute.INDEX_STATUS.getValue())) {
- metaMapping.put(key, entry.getValue());
- } else {
- metaMapping.put(key, Records.Type.builder().type(entry.getValue().toString()).build());
- }
+ metaMapping.put(entry.getKey(), TypeMapper.getMetaAttributeIndexerMapping(entry.getKey()));
}
// data-source attributes
Map<String, Object> dataMapping = new HashMap<>();
if (schema.getDataSchema() != null) {
for (Map.Entry<String, String> entry : schema.getDataSchema().entrySet()) {
- // Apply de_indexer_analyzer and de_search_analyzer to TEXT field
- if (configurationProperties.isPreDemo() && ElasticType.TEXT.getValue().equalsIgnoreCase(entry.getValue())) {
- log.info(String.format("indexing %s with custom analyzer", entry.getKey()));
- dataMapping.put(entry.getKey(), Records.Analyzer.builder().type(entry.getValue()).analyzer(DEAnalyzerType.INDEXER_ANALYZER.getValue()).search_analyzer(DEAnalyzerType.SEARCH_ANALYZER.getValue()).build());
- } else {
- dataMapping.put(entry.getKey(), Records.Type.builder().type(entry.getValue()).build());
- }
+ dataMapping.put(entry.getKey(), TypeMapper.getDataAttributeIndexerMapping(entry.getValue()));
}
// inner properties.data.properties block
@@ -159,7 +143,7 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
}
}
}
-
+
private boolean updateMappingToEnableKeywordIndexingForField(RestHighLevelClient client, Set<String> indicesSet, String fieldName) throws IOException {
String[] indices = indicesSet.toArray(new String[indicesSet.size()]);
Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> indexMappingMap = getIndexFieldMap(new String[]{"data."+fieldName}, client, indices);
@@ -190,7 +174,7 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
}
}
}
-
+
return indexMappingMap;
} catch (ElasticsearchException e) {
log.error(String.format("Failed to get indices: %s. | Error: %s", Arrays.toString(indices), e));
@@ -198,7 +182,9 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
}
}
- private boolean updateMappingForAllIndicesOfSameTypeToEnableKeywordIndexingForField(RestHighLevelClient client, String index, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> indexMapping, String fieldName) throws IOException {
+ private boolean updateMappingForAllIndicesOfSameTypeToEnableKeywordIndexingForField(
+ RestHighLevelClient client, String index, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> indexMapping, String fieldName) throws IOException {
+
PutMappingRequest request = new PutMappingRequest(index);
String type = indexMapping.keySet().iterator().next();
if(type.isEmpty()) {
@@ -219,7 +205,7 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
log.error(String.format("Could not find field: %s in the mapping of index: %s.", fieldName, index));
return false;
}
-
+
//Index the field with additional keyword type
Map<String, Object> keywordMap = new HashMap<>();
keywordMap.put(Constants.TYPE, "keyword");
@@ -235,22 +221,22 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
data.put(Constants.DATA,mapping);
Map<String, Object> properties = new HashMap<>();
properties.put(Constants.PROPERTIES, data);
-
+
request.source(new Gson().toJson(properties), XContentType.JSON);
try {
AcknowledgedResponse response = client.indices().putMapping(request, RequestOptions.DEFAULT);
boolean isIndicesUpdated = updateIndices(client, index);
return response.isAcknowledged() && isIndicesUpdated;
-
+
} catch (Exception e) {
log.error(String.format("Could not update mapping of index: %s. | Error: %s", index, e));
return false;
}
}
-
+
private boolean updateIndices(RestHighLevelClient client, String index) throws IOException {
- UpdateByQueryRequest request = new UpdateByQueryRequest(index);
+ UpdateByQueryRequest request = new UpdateByQueryRequest(index);
request.setConflicts("proceed");
BulkByScrollResponse response = client.updateByQuery(request, RequestOptions.DEFAULT);
if(!response.getBulkFailures().isEmpty()) {
@@ -259,7 +245,7 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
}
return true;
}
-
+
/**
* Create a new type in Elasticsearch
*
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
index 6bd98690beaf036862e0f8fdfe5aea2d35b29a3e..af1e5aea27f0b3846c68ba76a0514fa2ae714c22 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
@@ -16,6 +16,7 @@ package org.opengroup.osdu.indexer.service;
import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
import org.apache.http.HttpStatus;
import org.elasticsearch.ElasticsearchStatusException;
import org.elasticsearch.action.bulk.BulkItemResponse;
@@ -47,6 +48,7 @@ import org.opengroup.osdu.indexer.util.ElasticClientHandler;
import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
import org.apache.commons.beanutils.PropertyUtils;
import org.apache.commons.beanutils.NestedNullException;
+import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Service;
import javax.inject.Inject;
@@ -58,13 +60,14 @@ import java.util.logging.Level;
import java.util.stream.Collectors;
@Service
+@Primary
public class IndexerServiceImpl implements IndexerService {
private static final TimeValue BULK_REQUEST_TIMEOUT = TimeValue.timeValueMinutes(1);
private static final List<RestStatus> RETRY_ELASTIC_EXCEPTION = new ArrayList<>(Arrays.asList(RestStatus.TOO_MANY_REQUESTS, RestStatus.BAD_GATEWAY, RestStatus.SERVICE_UNAVAILABLE));
- private final Gson gson = new Gson();
+ private final Gson gson = new GsonBuilder().serializeNulls().create();
@Inject
private JaxRsDpsLog jaxRsDpsLog;
@@ -308,6 +311,9 @@ public class IndexerServiceImpl implements IndexerService {
document.setVersion(storageRecord.getVersion());
document.setAcl(storageRecord.getAcl());
document.setLegal(storageRecord.getLegal());
+ if (storageRecord.getTags() != null) {
+ document.setTags(storageRecord.getTags());
+ }
RecordStatus recordStatus = this.jobStatus.getJobStatusByRecordId(storageRecord.getId());
if (recordStatus.getIndexProgress().getStatusCode() == 0) {
recordStatus.getIndexProgress().setStatusCode(HttpStatus.SC_OK);
@@ -462,6 +468,7 @@ public class IndexerServiceImpl implements IndexerService {
indexerPayload.put(RecordMetaAttribute.TYPE.getValue(), record.getType());
indexerPayload.put(RecordMetaAttribute.VERSION.getValue(), record.getVersion());
indexerPayload.put(RecordMetaAttribute.ACL.getValue(), record.getAcl());
+ indexerPayload.put(RecordMetaAttribute.TAGS.getValue(), record.getTags());
indexerPayload.put(RecordMetaAttribute.X_ACL.getValue(), Acl.flattenAcl(record.getAcl()));
indexerPayload.put(RecordMetaAttribute.LEGAL.getValue(), record.getLegal());
indexerPayload.put(RecordMetaAttribute.INDEX_STATUS.getValue(), record.getIndexProgress());
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java
index 0f3d5c4f18db650813ca096878925dc49a153921..70fa630e6ca33621db37c7b4f1a15344925fb3e1 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java
@@ -16,7 +16,10 @@ package org.opengroup.osdu.indexer.service;
import com.google.common.base.Strings;
import com.google.gson.Gson;
+
import java.util.Objects;
+
+import lombok.SneakyThrows;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.http.AppException;
@@ -53,74 +56,70 @@ public class ReindexServiceImpl implements ReindexService {
@Inject
private JaxRsDpsLog jaxRsDpsLog;
+ @SneakyThrows
@Override
public String reindexRecords(RecordReindexRequest recordReindexRequest, boolean forceClean) {
Long initialDelayMillis = 0l;
- try {
- DpsHeaders headers = this.requestInfo.getHeadersWithDwdAuthZ();
- if (forceClean) {
- this.indexSchemaService.syncIndexMappingWithStorageSchema(recordReindexRequest.getKind());
- initialDelayMillis = 30000l;
- }
+ DpsHeaders headers = this.requestInfo.getHeadersWithDwdAuthZ();
- RecordQueryResponse recordQueryResponse = this.storageService.getRecordsByKind(recordReindexRequest);
+ if (forceClean) {
+ this.indexSchemaService.syncIndexMappingWithStorageSchema(recordReindexRequest.getKind());
+ initialDelayMillis = 30000l;
+ }
- if (recordQueryResponse.getResults() != null && recordQueryResponse.getResults().size() != 0) {
+ RecordQueryResponse recordQueryResponse = this.storageService.getRecordsByKind(recordReindexRequest);
- List<RecordInfo> msgs = recordQueryResponse.getResults().stream()
- .map(record -> RecordInfo.builder().id(record).kind(recordReindexRequest.getKind()).op(OperationType.create.name()).build()).collect(Collectors.toList());
+ if (recordQueryResponse.getResults() != null && recordQueryResponse.getResults().size() != 0) {
- Map<String, String> attributes = new HashMap<>();
- attributes.put(DpsHeaders.ACCOUNT_ID, headers.getAccountId());
- attributes.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
- attributes.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
+ List<RecordInfo> msgs = recordQueryResponse.getResults().stream()
+ .map(record -> RecordInfo.builder().id(record).kind(recordReindexRequest.getKind()).op(OperationType.create.name()).build()).collect(Collectors.toList());
- Gson gson = new Gson();
- RecordChangedMessages recordChangedMessages = RecordChangedMessages.builder().data(gson.toJson(msgs)).attributes(attributes).build();
- String recordChangedMessagePayload = gson.toJson(recordChangedMessages);
- this.indexerQueueTaskBuilder.createWorkerTask(recordChangedMessagePayload, initialDelayMillis, headers);
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(DpsHeaders.ACCOUNT_ID, headers.getAccountId());
+ attributes.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
+ attributes.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
- // don't call reindex-worker endpoint if it's the last batch
- // previous storage query result size will be less then requested (limit param)
- if (!Strings.isNullOrEmpty(recordQueryResponse.getCursor()) && recordQueryResponse.getResults().size() == configurationProperties.getStorageRecordsBatchSize()) {
- String newPayLoad = gson.toJson(RecordReindexRequest.builder().cursor(recordQueryResponse.getCursor()).kind(recordReindexRequest.getKind()).build());
- this.indexerQueueTaskBuilder.createReIndexTask(newPayLoad, initialDelayMillis, headers);
- return newPayLoad;
- }
+ Gson gson = new Gson();
+ RecordChangedMessages recordChangedMessages = RecordChangedMessages.builder().data(gson.toJson(msgs)).attributes(attributes).build();
+ String recordChangedMessagePayload = gson.toJson(recordChangedMessages);
+ this.indexerQueueTaskBuilder.createWorkerTask(recordChangedMessagePayload, initialDelayMillis, headers);
- return recordChangedMessagePayload;
- } else {
- jaxRsDpsLog.info(String.format("kind: %s cannot be re-indexed, storage service cannot locate valid records", recordReindexRequest.getKind()));
+ // don't call reindex-worker endpoint if it's the last batch
+ // previous storage query result size will be less then requested (limit param)
+ if (!Strings.isNullOrEmpty(recordQueryResponse.getCursor()) && recordQueryResponse.getResults().size() == configurationProperties.getStorageRecordsBatchSize()) {
+ String newPayLoad = gson.toJson(RecordReindexRequest.builder().cursor(recordQueryResponse.getCursor()).kind(recordReindexRequest.getKind()).build());
+ this.indexerQueueTaskBuilder.createReIndexTask(newPayLoad, initialDelayMillis, headers);
+ return newPayLoad;
}
- return null;
- } catch (AppException e) {
- throw e;
- } catch (Exception e) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Unknown error", "An unknown error has occurred.", e);
+
+ return recordChangedMessagePayload;
+ } else {
+ jaxRsDpsLog.info(String.format("kind: %s cannot be re-indexed, storage service cannot locate valid records", recordReindexRequest.getKind()));
}
+ return null;
}
- @Override
- public void fullReindex(boolean forceClean) {
- List<String> allKinds = null;
- try {
- allKinds = storageService.getAllKinds();
- } catch (Exception e) {
- jaxRsDpsLog.error("storage service all kinds request failed",e);
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "storage service cannot respond with all kinds", "an unknown error has occurred.", e);
- }
- if (Objects.isNull(allKinds)){
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "storage service cannot respond with all kinds", "full reindex failed");
- }
- for (String kind : allKinds) {
- try {
- reindexRecords(new RecordReindexRequest(kind, ""), forceClean);
- } catch (Exception e) {
- jaxRsDpsLog.warning(String.format("kind: %s cannot be re-indexed", kind));
- continue;
- }
- }
- }
+ @Override
+ public void fullReindex(boolean forceClean) {
+ List<String> allKinds = null;
+ try {
+ allKinds = storageService.getAllKinds();
+ } catch (Exception e) {
+ jaxRsDpsLog.error("storage service all kinds request failed", e);
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "storage service cannot respond with all kinds", "an unknown error has occurred.", e);
+ }
+ if (Objects.isNull(allKinds)) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "storage service cannot respond with all kinds", "full reindex failed");
+ }
+ for (String kind : allKinds) {
+ try {
+ reindexRecords(new RecordReindexRequest(kind, ""), forceClean);
+ } catch (Exception e) {
+ jaxRsDpsLog.warning(String.format("kind: %s cannot be re-indexed", kind));
+ continue;
+ }
+ }
+ }
}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java
index fba62bb8ac5cd08549da9e7814720df38e526651..6d9d8b8cde2bf7664b1a84e2bc7a14c7551e0db3 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java
@@ -37,10 +37,10 @@ public class StorageIndexerPayloadMapper {
Object value = getPropertyValue(recordId, storageRecordData, name);
- if (value == null) continue;
-
ElasticType elasticType = ElasticType.forValue(entry.getValue());
+ if (value == null && !nullIndexedValueSupported(elasticType)) continue;
+
switch (elasticType) {
case KEYWORD:
case KEYWORD_ARRAY:
@@ -88,10 +88,14 @@ public class StorageIndexerPayloadMapper {
this.attributeParsingService.tryParseGeopoint(recordId, name, storageRecordData, dataMap);
break;
case GEO_SHAPE:
- this.attributeParsingService.tryParseGeojson(recordId, name, value, dataMap);
+ this.attributeParsingService.tryParseGeojson(recordId, name, value, dataMap);
break;
case NESTED:
+ this.attributeParsingService.tryParseNested(recordId, name, value, dataMap);
+ break;
case OBJECT:
+ this.attributeParsingService.tryParseObject(recordId, name, value, dataMap);
+ break;
case UNDEFINED:
// don't do anything for now
break;
@@ -123,4 +127,8 @@ public class StorageIndexerPayloadMapper {
}
return null;
}
+
+ private boolean nullIndexedValueSupported(ElasticType type) {
+ return type == ElasticType.TEXT;
+ }
}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/impl/SchemaProviderImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/impl/SchemaProviderImpl.java
index 70ef41dd6cef5e0c6687ad38f2fb1de8b6a08ffe..2323aeecdf9218aa1fa6eef5610d96803100d17c 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/impl/SchemaProviderImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/impl/SchemaProviderImpl.java
@@ -32,6 +32,7 @@ import java.io.UnsupportedEncodingException;
import java.net.URISyntaxException;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
+import java.util.Objects;
/**
* Provides implementation of the client service that retrieves schemas from the Schema Service
@@ -58,13 +59,9 @@ public class SchemaProviderImpl implements SchemaService {
@Override
public String getSchema(String kind) throws URISyntaxException, UnsupportedEncodingException {
- String schemaFromStorageService = getFromStorageService(kind);
+ String schemaServiceSchema = getFromSchemaService(kind);
- if (schemaFromStorageService != null) {
- return schemaFromStorageService;
- }
-
- return getFromSchemaService(kind);
+ return Objects.nonNull(schemaServiceSchema) ? schemaServiceSchema : getFromStorageService(kind);
}
protected String getFromSchemaService(String kind) throws UnsupportedEncodingException, URISyntaxException {
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java
index 2f7c9b7c0e9b3051d6a428fec359be7418f35f93..416728dc3cb6689e4486e41798f37d74b7e7e627 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java
@@ -22,6 +22,7 @@ import org.opengroup.osdu.core.common.model.indexer.StorageType;
import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
import org.apache.commons.lang3.StringUtils;
+
import java.util.HashMap;
import java.util.Map;
@@ -31,6 +32,8 @@ public class TypeMapper {
private static final Map<String, Object> metaAttributeIndexerType = new HashMap<>();
+ private static final String STORAGE_TYPE_OBJECTS = "[]object";
+
static {
metaAttributeIndexerType.put(RecordMetaAttribute.KIND.getValue(), ElasticType.KEYWORD.getValue());
@@ -40,6 +43,7 @@ public class TypeMapper {
metaAttributeIndexerType.put(RecordMetaAttribute.VERSION.getValue(), ElasticType.LONG.getValue());
metaAttributeIndexerType.put(RecordMetaAttribute.X_ACL.getValue(), ElasticType.KEYWORD.getValue());
metaAttributeIndexerType.put(RecordMetaAttribute.ACL.getValue(), getAclIndexerMapping());
+ metaAttributeIndexerType.put(RecordMetaAttribute.TAGS.getValue(), ElasticType.OBJECT.getValue());
metaAttributeIndexerType.put(RecordMetaAttribute.LEGAL.getValue(), getLegalIndexerMapping());
metaAttributeIndexerType.put(RecordMetaAttribute.ANCESTRY.getValue(), getAncestryIndexerMapping());
metaAttributeIndexerType.put(RecordMetaAttribute.INDEX_STATUS.getValue(), getIndexStatusMapping());
@@ -62,24 +66,50 @@ public class TypeMapper {
storageToIndexerType.put(StorageType.DATETIME_ARRAY.getValue(), ElasticType.DATE_ARRAY.getValue());
storageToIndexerType.put(StorageType.GEO_POINT.getValue(), ElasticType.GEO_POINT.getValue());
storageToIndexerType.put(StorageType.GEO_SHAPE.getValue(), ElasticType.GEO_SHAPE.getValue());
- }
+ //TODO temporary fix for https://community.opengroup.org/osdu/platform/system/indexer-service/-/issues/1
+ storageToIndexerType.put(STORAGE_TYPE_OBJECTS, ElasticType.OBJECT.getValue());
+ }
public static String getIndexerType(String storageType) {
- String indexedType = storageToIndexerType.getOrDefault(storageType, null);
- if (indexedType != null && indexedType.endsWith("_array")) {
- return StringUtils.substringBefore(indexedType, "_");
- }
- return indexedType;
+ return storageToIndexerType.getOrDefault(storageType, null);
}
public static Object getIndexerType(RecordMetaAttribute attribute) {
return metaAttributeIndexerType.getOrDefault(attribute.getValue(), null);
}
+ public static Object getMetaAttributeIndexerMapping(String key) {
+ if (key.equals(RecordMetaAttribute.ACL.getValue())
+ || key.equals(RecordMetaAttribute.LEGAL.getValue()) || key.equals(RecordMetaAttribute.ANCESTRY.getValue()) || key.equals(RecordMetaAttribute.INDEX_STATUS.getValue())) {
+ return metaAttributeIndexerType.get(key);
+ }
+ return Records.Type.builder().type(metaAttributeIndexerType.get(key).toString()).build();
+ }
+
+ public static Object getDataAttributeIndexerMapping(String indexerType) {
+ if (ElasticType.TEXT.getValue().equalsIgnoreCase(indexerType)) {
+ return getTextIndexerMapping();
+ }
+
+ if (isArray(indexerType)) {
+ return Records.Type.builder().type(getArrayMemberType(indexerType)).build();
+ }
+
+ return Records.Type.builder().type(indexerType).build();
+ }
+
+ private static boolean isArray(String indexerType) {
+ return indexerType != null && indexerType.endsWith("_array");
+ }
+
+ private static String getArrayMemberType(String indexerType) {
+ return StringUtils.substringBefore(indexerType, "_");
+ }
+
private static Object getAclIndexerMapping() {
Map<String, Object> aclRoleMapping = new HashMap<>();
- aclRoleMapping.put(AclRole.VIEWERS.getValue() , Records.Type.builder().type(ElasticType.KEYWORD.getValue()).build());
+ aclRoleMapping.put(AclRole.VIEWERS.getValue(), Records.Type.builder().type(ElasticType.KEYWORD.getValue()).build());
aclRoleMapping.put(AclRole.OWNERS.getValue(), Records.Type.builder().type(ElasticType.KEYWORD.getValue()).build());
Map<String, Object> aclProperties = new HashMap<>();
@@ -121,4 +151,22 @@ public class TypeMapper {
return indexStatusProperties;
}
+
+ private static Object getTextIndexerMapping() {
+ Map<String, Object> fieldIndexTypeMap = getKeywordMap();
+ Map<String, Object> textMap = new HashMap<>();
+ textMap.put("type", "text");
+ textMap.put("fields", fieldIndexTypeMap);
+ return textMap;
+ }
+
+ private static Map<String, Object> getKeywordMap() {
+ Map<String, Object> keywordMap = new HashMap<>();
+ keywordMap.put("type", "keyword");
+ keywordMap.put("ignore_above", 256);
+ keywordMap.put("null_value", "null");
+ Map<String, Object> fieldIndexTypeMap = new HashMap<>();
+ fieldIndexTypeMap.put("keyword", keywordMap);
+ return fieldIndexTypeMap;
+ }
}
\ No newline at end of file
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImplTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImplTest.java
index c90092879cf6047a76bb03a0f876abb59e40d358..07642442dc178ce37cf934fdfbdbf7a0d044171e 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImplTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImplTest.java
@@ -94,6 +94,21 @@ public class SchemaToStorageFormatImplTest {
testSingleFile("/converter/tags/allOf/indefinitions.json", KIND);
}
+ @Test
+ public void oneOfInDefinitions() {
+ testSingleFile("/converter/tags/oneOf/indefinitions.json", KIND);
+ }
+
+ @Test
+ public void anyOfInDefinitions() {
+ testSingleFile("/converter/tags/anyOf/indefinitions.json", KIND);
+ }
+
+ @Test
+ public void mixAllAnyOneOf() {
+ testSingleFile("/converter/tags/mixAllAnyOneOf/mix.json", KIND);
+ }
+
@Test
public void folderPassed() throws URISyntaxException, IOException {
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/impl/SchemaProviderImplTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/impl/SchemaProviderImplTest.java
index a3c5b46e73f938c8b5824fe484a7aa993919d5a0..870f2b76c7178ff6430521f3b90ebf2d39c16d55 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/impl/SchemaProviderImplTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/impl/SchemaProviderImplTest.java
@@ -77,7 +77,7 @@ public class SchemaProviderImplTest {
Assert.assertEquals("{\n" +
" \"kind\" : \"fake\",\n" +
" \"schema\" : [ ]\n" +
- "}", schema);
+ "}", schema.replaceAll("\r", ""));
}
@@ -136,7 +136,7 @@ public class SchemaProviderImplTest {
}
@Test
- public void should_call_Storage_then_Schema() throws Exception {
+ public void should_call_Schema_then_Storage() throws Exception {
String kind = "tenant:test:test:1.0.0";
SchemaProviderImpl schemaService = Mockito.mock(SchemaProviderImpl.class);
@@ -148,20 +148,20 @@ public class SchemaProviderImplTest {
assertNull(recordSchemaResponse);
inOrder.verify(schemaService).getSchema(any());
- inOrder.verify(schemaService).getFromStorageService(any());
inOrder.verify(schemaService).getFromSchemaService(any());
+ inOrder.verify(schemaService).getFromStorageService(any());
verify(schemaService, times(1)).getFromStorageService(any());
verify(schemaService, times(1)).getFromSchemaService(any());
}
@Test
- public void should_call_only_Storage_if_it_returns_result() throws Exception {
+ public void should_call_only_SchemaService_if_it_returns_result() throws Exception {
String kind = "tenant:test:test:1.0.0";
SchemaProviderImpl schemaService = Mockito.mock(SchemaProviderImpl.class);
when(schemaService.getSchema(any())).thenCallRealMethod();
String someSchema = "some schema";
- when(schemaService.getFromStorageService(any())).thenReturn(someSchema);
+ when(schemaService.getFromSchemaService(any())).thenReturn(someSchema);
InOrder inOrder = inOrder(schemaService);
@@ -169,9 +169,9 @@ public class SchemaProviderImplTest {
assertEquals(recordSchemaResponse, someSchema);
inOrder.verify(schemaService).getSchema(any());
- inOrder.verify(schemaService).getFromStorageService(any());
- verify(schemaService, times(0)).getFromSchemaService(any());
- verify(schemaService, times(1)).getFromStorageService(any());
+ inOrder.verify(schemaService).getFromSchemaService(any());
+ verify(schemaService, times(1)).getFromSchemaService(any());
+ verify(schemaService, times(0)).getFromStorageService(any());
}
}
\ No newline at end of file
diff --git a/indexer-core/src/test/resources/converter/R3-json-schema/Generated/file/File.1.0.0.json.res b/indexer-core/src/test/resources/converter/R3-json-schema/Generated/file/File.1.0.0.json.res
index b0f71fc6fd26e27eb5dc2362e7667aa1d17c481e..d0cc00a8d3fe76d6ff446a8433c4b895e8a88316 100644
--- a/indexer-core/src/test/resources/converter/R3-json-schema/Generated/file/File.1.0.0.json.res
+++ b/indexer-core/src/test/resources/converter/R3-json-schema/Generated/file/File.1.0.0.json.res
@@ -40,6 +40,10 @@
{
"kind": "string",
"path": "Checksum"
+ },
+ {
+ "kind": "[]object",
+ "path": "VectorHeaderMapping"
}
]
}
\ No newline at end of file
diff --git a/indexer-core/src/test/resources/converter/R3-json-schema/Generated/master-data/Agreement.1.0.0.json.res b/indexer-core/src/test/resources/converter/R3-json-schema/Generated/master-data/Agreement.1.0.0.json.res
index 136ed0b7ecb61b50f82b8d8e213db03ab410f8da..8728c3bc5da175c23046b1f86381ef5eb99cbfe3 100644
--- a/indexer-core/src/test/resources/converter/R3-json-schema/Generated/master-data/Agreement.1.0.0.json.res
+++ b/indexer-core/src/test/resources/converter/R3-json-schema/Generated/master-data/Agreement.1.0.0.json.res
@@ -32,6 +32,14 @@
{
"kind": "[]link",
"path": "Counterparties"
+ },
+ {
+ "kind": "[]object",
+ "path": "Terms"
+ },
+ {
+ "kind": "[]object",
+ "path": "RestrictedResources"
}
]
}
\ No newline at end of file
diff --git a/indexer-core/src/test/resources/converter/R3-json-schema/Generated/work-product-component/WellLog.1.0.0.json.res b/indexer-core/src/test/resources/converter/R3-json-schema/Generated/work-product-component/WellLog.1.0.0.json.res
index 517ca0395223bf432f8bc6b8c01bf7c768921532..359986f472e4ea54a08e6496b5230b914c3a80f0 100644
--- a/indexer-core/src/test/resources/converter/R3-json-schema/Generated/work-product-component/WellLog.1.0.0.json.res
+++ b/indexer-core/src/test/resources/converter/R3-json-schema/Generated/work-product-component/WellLog.1.0.0.json.res
@@ -76,6 +76,10 @@
{
"kind": "string",
"path": "VerticalMeasurementID"
+ },
+ {
+ "kind": "[]object",
+ "path": "Curves"
}
]
}
\ No newline at end of file
diff --git a/indexer-core/src/test/resources/converter/R3-json-schema/Generated/work-product/WorkProduct.1.0.0.json.res b/indexer-core/src/test/resources/converter/R3-json-schema/Generated/work-product/WorkProduct.1.0.0.json.res
index 0b807093bb76e1b01fa2c4d2753864bab6f52386..5852817c878ac55fcc522b6752635389135b40ae 100644
--- a/indexer-core/src/test/resources/converter/R3-json-schema/Generated/work-product/WorkProduct.1.0.0.json.res
+++ b/indexer-core/src/test/resources/converter/R3-json-schema/Generated/work-product/WorkProduct.1.0.0.json.res
@@ -44,6 +44,10 @@
{
"kind": "[]string",
"path": "Annotations"
+ },
+ {
+ "kind": "[]object",
+ "path": "LineageAssertions"
}
]
}
\ No newline at end of file
diff --git a/indexer-core/src/test/resources/converter/integration-tests/index_records_1.schema b/indexer-core/src/test/resources/converter/integration-tests/index_records_1.schema
index 672c3d965d50194ac5c80adecfddbeb24d51a856..779d2d20921f97aa86bed06943358d76c3d5f71f 100644
--- a/indexer-core/src/test/resources/converter/integration-tests/index_records_1.schema
+++ b/indexer-core/src/test/resources/converter/integration-tests/index_records_1.schema
@@ -20,7 +20,11 @@
},
"County": {
"type": "string"
- },
+ }}}],
+ "anyOf": [
+ {
+ "type": "object",
+ "properties": {
"State": {
"type": "string"
},
@@ -41,7 +45,11 @@
},
"EmptyAttribute": {
"type": "string"
- },
+ }}}],
+ "oneOf": [
+ {
+ "type": "object",
+ "properties": {
"Rank": {
"type": "integer"
},
diff --git a/indexer-core/src/test/resources/converter/tags/anyOf/indefinitions.json b/indexer-core/src/test/resources/converter/tags/anyOf/indefinitions.json
new file mode 100644
index 0000000000000000000000000000000000000000..4733286c10bd2d081ae85fa44ee22d499a86908d
--- /dev/null
+++ b/indexer-core/src/test/resources/converter/tags/anyOf/indefinitions.json
@@ -0,0 +1,29 @@
+{
+ "definitions": {
+ "wellboreData1": {
+ "properties": {
+ "prop1": {
+ "type": "string"
+ }
+ }
+ },
+ "wellboreData2": {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/wellboreData1"
+ }
+ ]
+ }
+ },
+ "properties": {
+ "data": {
+ "type": "object",
+ "properties": {
+ "Field": {
+ "$ref": "#/definitions/wellboreData2"
+ }
+ }
+ }
+ }
+}
+
diff --git a/indexer-core/src/test/resources/converter/tags/anyOf/indefinitions.json.res b/indexer-core/src/test/resources/converter/tags/anyOf/indefinitions.json.res
new file mode 100644
index 0000000000000000000000000000000000000000..e17ce90d066d023859f6b91d7756717b494a7a45
--- /dev/null
+++ b/indexer-core/src/test/resources/converter/tags/anyOf/indefinitions.json.res
@@ -0,0 +1,9 @@
+{
+ "kind": "KIND_VAL",
+ "schema": [
+ {
+ "kind": "string",
+ "path": "Field.prop1"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/indexer-core/src/test/resources/converter/tags/mixAllAnyOneOf/mix.json b/indexer-core/src/test/resources/converter/tags/mixAllAnyOneOf/mix.json
new file mode 100644
index 0000000000000000000000000000000000000000..2eb9de93a9911ad6407366a362fa0604219f7361
--- /dev/null
+++ b/indexer-core/src/test/resources/converter/tags/mixAllAnyOneOf/mix.json
@@ -0,0 +1,57 @@
+{
+ "definitions": {
+ "wellboreData1": {
+ "properties": {
+ "prop1": {
+ "type": "string"
+ }
+ }
+ },
+ "wellboreData2": {
+ "properties": {
+ "prop2": {
+ "type": "string"
+ }
+ }
+ },
+ "wellboreData3": {
+ "properties": {
+ "prop3": {
+ "type": "string"
+ }
+ }
+ },
+ "wellboreData4": {
+ "properties": {
+ "prop4": {
+ "type": "string"
+ }
+ }
+ }
+ },
+ "properties": {
+ "data": {
+ "allOf": [
+ {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/wellboreData1"
+ } ],
+ "oneOf": [
+ {
+ "$ref": "#/definitions/wellboreData2"
+ }
+ ]
+ },
+ {
+ "$ref": "#/definitions/wellboreData3"
+ },
+ {
+ "$ref": "#/definitions/wellboreData4"
+ }
+ ]
+ }
+ }
+}
+
+
diff --git a/indexer-core/src/test/resources/converter/tags/mixAllAnyOneOf/mix.json.res b/indexer-core/src/test/resources/converter/tags/mixAllAnyOneOf/mix.json.res
new file mode 100644
index 0000000000000000000000000000000000000000..22a8e6a806ba17e51adbd690f0ef56ac4f9f7bbe
--- /dev/null
+++ b/indexer-core/src/test/resources/converter/tags/mixAllAnyOneOf/mix.json.res
@@ -0,0 +1,21 @@
+{
+ "kind": "KIND_VAL",
+ "schema": [
+ {
+ "kind": "string",
+ "path": "prop1"
+ },
+ {
+ "kind": "string",
+ "path": "prop2"
+ },
+ {
+ "kind": "string",
+ "path": "prop3"
+ },
+ {
+ "kind": "string",
+ "path": "prop4"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/indexer-core/src/test/resources/converter/tags/oneOf/indefinitions.json b/indexer-core/src/test/resources/converter/tags/oneOf/indefinitions.json
new file mode 100644
index 0000000000000000000000000000000000000000..4733286c10bd2d081ae85fa44ee22d499a86908d
--- /dev/null
+++ b/indexer-core/src/test/resources/converter/tags/oneOf/indefinitions.json
@@ -0,0 +1,29 @@
+{
+ "definitions": {
+ "wellboreData1": {
+ "properties": {
+ "prop1": {
+ "type": "string"
+ }
+ }
+ },
+ "wellboreData2": {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/wellboreData1"
+ }
+ ]
+ }
+ },
+ "properties": {
+ "data": {
+ "type": "object",
+ "properties": {
+ "Field": {
+ "$ref": "#/definitions/wellboreData2"
+ }
+ }
+ }
+ }
+}
+
diff --git a/indexer-core/src/test/resources/converter/tags/oneOf/indefinitions.json.res b/indexer-core/src/test/resources/converter/tags/oneOf/indefinitions.json.res
new file mode 100644
index 0000000000000000000000000000000000000000..e17ce90d066d023859f6b91d7756717b494a7a45
--- /dev/null
+++ b/indexer-core/src/test/resources/converter/tags/oneOf/indefinitions.json.res
@@ -0,0 +1,9 @@
+{
+ "kind": "KIND_VAL",
+ "schema": [
+ {
+ "kind": "string",
+ "path": "Field.prop1"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index d24aad221589fd26fd867d4c1b9d8bd03f4d2c24..711526c64b74bffea49d64878eac270a9ab8d569 100644
--- a/pom.xml
+++ b/pom.xml
@@ -12,14 +12,14 @@
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-service</artifactId>
<packaging>pom</packaging>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.8.0-SNAPSHOT</version>
<description>Indexer Service</description>
<properties>
<java.version>1.8</java.version>
<springfox-version>2.7.0</springfox-version>
<spring-cloud.version>Greenwich.SR2</spring-cloud.version>
- <os-core-common.version>0.3.19</os-core-common.version>
+ <os-core-common.version>0.6.5</os-core-common.version>
<snakeyaml.version>1.26</snakeyaml.version>
<hibernate-validator.version>6.1.5.Final</hibernate-validator.version>
<jackson.version>2.11.2</jackson.version>
diff --git a/provider/indexer-aws/pom.xml b/provider/indexer-aws/pom.xml
index d6a11e333aba7bc1168209b4489d02d6648909e3..7fad557f0e8324b8df8cf15a3fd39878a40d0299 100644
--- a/provider/indexer-aws/pom.xml
+++ b/provider/indexer-aws/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-service</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.8.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
@@ -26,7 +26,7 @@
<artifactId>indexer-aws</artifactId>
<description>Storage service on AWS</description>
<packaging>jar</packaging>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.8.0-SNAPSHOT</version>
<properties>
<aws.version>1.11.637</aws.version>
@@ -43,7 +43,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-core</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.8.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
@@ -90,12 +90,6 @@
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-oauth2-jose</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-core</artifactId>
- <version>7.6.0</version>
- <scope>compile</scope>
- </dependency>
<dependency>
<groupId>javax.inject</groupId>
<artifactId>javax.inject</artifactId>
@@ -105,17 +99,14 @@
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
- <version>6.6.2</version>
</dependency>
<dependency>
- <groupId>org.elasticsearch.client</groupId>
- <artifactId>elasticsearch-rest-client</artifactId>
- <version>6.6.2</version>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-client</artifactId>
</dependency>
<dependency>
- <groupId>org.elasticsearch.client</groupId>
- <artifactId>elasticsearch-rest-high-level-client</artifactId>
- <version>6.6.2</version>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-high-level-client</artifactId>
</dependency>
<!-- Testing packages -->
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/service/ElasticClientHandlerAws.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/service/ElasticClientHandlerAws.java
index 3c7383a80aaa9aaf1a7fa8943673fa86dc0f371b..e6003130aad08cfdb49e57d70cee54f17eca625c 100644
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/service/ElasticClientHandlerAws.java
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/service/ElasticClientHandlerAws.java
@@ -44,8 +44,7 @@ public class ElasticClientHandlerAws extends ElasticClientHandler {
RestClientBuilder builder = RestClient.builder(new HttpHost(host, port, protocolScheme));
builder.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder
.setConnectTimeout(REST_CLIENT_CONNECT_TIMEOUT)
- .setSocketTimeout(REST_CLIENT_SOCKET_TIMEOUT));
- builder.setMaxRetryTimeoutMillis(REST_CLIENT_RETRY_TIMEOUT);
+ .setSocketTimeout(REST_CLIENT_SOCKET_TIMEOUT));
if(isLocalHost(host)) {
builder.setHttpClientConfigCallback(httpAsyncClientBuilder -> httpAsyncClientBuilder.setSSLHostnameVerifier((s, sslSession) -> true));
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java
index b93d3c94144d859cdf25cde6ff4e1cfcd7ff9358..cb47d53cd17fce4056d18d82a4371f8b1fe2d204 100644
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java
@@ -79,15 +79,38 @@ public class IndexerQueueTaskBuilderAws extends IndexerQueueTaskBuilder {
}
@Override
public void createReIndexTask(String payload,DpsHeaders headers) {
- this.createTask(payload, headers);
+ Map<String, MessageAttributeValue> messageAttributes = new HashMap<>();
+ messageAttributes.put(DpsHeaders.ACCOUNT_ID, new MessageAttributeValue()
+ .withDataType("String")
+ .withStringValue(headers.getPartitionIdWithFallbackToAccountId()));
+ messageAttributes.put(DpsHeaders.DATA_PARTITION_ID, new MessageAttributeValue()
+ .withDataType("String")
+ .withStringValue(headers.getPartitionIdWithFallbackToAccountId()));
+ headers.addCorrelationIdIfMissing();
+ messageAttributes.put(DpsHeaders.CORRELATION_ID, new MessageAttributeValue()
+ .withDataType("String")
+ .withStringValue(headers.getCorrelationId()));
+ messageAttributes.put(DpsHeaders.USER_EMAIL, new MessageAttributeValue()
+ .withDataType("String")
+ .withStringValue(headers.getUserEmail()));
+ messageAttributes.put(DpsHeaders.AUTHORIZATION, new MessageAttributeValue()
+ .withDataType("String")
+ .withStringValue(headers.getAuthorization()));
+ messageAttributes.put("ReIndexCursor", new MessageAttributeValue()
+ .withDataType("String")
+ .withStringValue("True"));
+ SendMessageRequest sendMessageRequest = new SendMessageRequest()
+ .withQueueUrl(storageQueue)
+ .withMessageBody(payload)
+ .withMessageAttributes(messageAttributes);
+ sqsClient.sendMessage(sendMessageRequest);
}
@Override
public void createReIndexTask(String payload, Long countDownMillis, DpsHeaders headers){
- this.createTask(payload, headers);
+ this.createReIndexTask(payload, headers);
}
private void createTask(String payload, DpsHeaders headers) {
-
Map<String, MessageAttributeValue> messageAttributes = new HashMap<>();
messageAttributes.put(DpsHeaders.ACCOUNT_ID, new MessageAttributeValue()
.withDataType("String")
diff --git a/provider/indexer-azure/pom.xml b/provider/indexer-azure/pom.xml
index f67a7fcbb3ab51a12eafb5120b8a56ffb4e64a16..c88af5429b37a4671ce3ec3e5ccd451c908f2dc7 100644
--- a/provider/indexer-azure/pom.xml
+++ b/provider/indexer-azure/pom.xml
@@ -21,12 +21,12 @@
<parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-service</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.8.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>indexer-azure</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.8.0-SNAPSHOT</version>
<name>indexer-azure</name>
<description>Indexer Service Azure</description>
<packaging>jar</packaging>
@@ -39,10 +39,10 @@
<azure.appservice.subscription />
<log4j.version>2.11.2</log4j.version>
<nimbus-jose-jwt.version>8.2</nimbus-jose-jwt.version>
- <indexer-core.version>0.6.0-SNAPSHOT</indexer-core.version>
+ <indexer-core.version>0.8.0-SNAPSHOT</indexer-core.version>
<spring-security-jwt.version>1.1.1.RELEASE</spring-security-jwt.version>
<osdu.corelibazure.version>0.0.42</osdu.corelibazure.version>
- <osdu.oscorecommon.version>0.3.12</osdu.oscorecommon.version>
+ <osdu.oscorecommon.version>0.6.5</osdu.oscorecommon.version>
<reactor-netty.version>0.9.12.RELEASE</reactor-netty.version>
<java-jwt.version>3.8.1</java-jwt.version>
<powermock.version>2.0.2</powermock.version>
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/config/AzureBootstrapConfig.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/config/AzureBootstrapConfig.java
index 570b932a29f49f8ba849488ed55d9e2f61d440ce..7be75c3be0c99a9f076f36f422b303e948aa035a 100644
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/config/AzureBootstrapConfig.java
+++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/config/AzureBootstrapConfig.java
@@ -19,6 +19,8 @@ import org.opengroup.osdu.azure.KeyVaultFacade;
import org.opengroup.osdu.core.common.entitlements.EntitlementsAPIConfig;
import org.opengroup.osdu.core.common.entitlements.EntitlementsFactory;
import org.opengroup.osdu.core.common.entitlements.IEntitlementsFactory;
+import org.opengroup.osdu.core.common.http.json.HttpResponseBodyMapper;
+import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Component;
@@ -90,12 +92,15 @@ public class AzureBootstrapConfig {
return String.format(urlFormat, tenant);
}
+ @Autowired
+ private HttpResponseBodyMapper httpResponseBodyMapper;
+
@Bean
public IEntitlementsFactory entitlementsFactory() {
EntitlementsAPIConfig apiConfig = EntitlementsAPIConfig.builder()
.apiKey(entitlementsAPIKey)
.rootUrl(entitlementsAPIEndpoint)
.build();
- return new EntitlementsFactory(apiConfig);
+ return new EntitlementsFactory(apiConfig, httpResponseBodyMapper);
}
}
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java
new file mode 100644
index 0000000000000000000000000000000000000000..7210b4aaa06bb72b607ae31353835ca4353a19d1
--- /dev/null
+++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java
@@ -0,0 +1,168 @@
+// Copyright © Azure
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.azure.util;
+
+import com.google.common.base.Strings;
+import com.google.common.reflect.TypeToken;
+import com.google.gson.*;
+import com.microsoft.azure.servicebus.Message;
+import lombok.extern.java.Log;
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.azure.servicebus.ITopicClientFactory;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.indexer.OperationType;
+import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
+import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
+import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
+import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
+import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.opengroup.osdu.indexer.service.StorageService;
+import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Primary;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+import javax.inject.Inject;
+import javax.inject.Named;
+import java.lang.reflect.Type;
+import java.nio.charset.StandardCharsets;
+import java.util.stream.Collectors;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+
+@Log
+@Component
+@RequestScope
+@Primary
+public class IndexerQueueTaskBuilderAzure extends IndexerQueueTaskBuilder {
+
+ @Autowired
+ private ITopicClientFactory topicClientFactory;
+
+ @Inject
+ private IndexerConfigurationProperties configurationProperties;
+
+ @Inject
+ private JaxRsDpsLog logger;
+
+ @Inject
+ @Named("SERVICE_BUS_TOPIC")
+ private String serviceBusTopic;
+
+ @Inject
+ private StorageService storageService;
+
+
+ @Override
+ public void createWorkerTask(String payload, Long countdownMillis, DpsHeaders headers) {
+ headers.addCorrelationIdIfMissing();
+ createTask(payload, headers);
+ }
+
+ @Override
+ public void createReIndexTask(String payload, Long countdownMillis, DpsHeaders headers) {
+ headers.addCorrelationIdIfMissing();
+ publishAllRecordsToServiceBus(payload, headers);
+ }
+
+ private void publishAllRecordsToServiceBus(String payload, DpsHeaders headers) {
+ // fetch all the remaining records
+ // This logic is temporary and would be updated to call the storage service async.
+ // Currently the storage client can't be called out of request scope hence making the
+ // storage calls sync here
+ Gson gson = new Gson();
+ RecordReindexRequest recordReindexRequest = gson.fromJson(payload, RecordReindexRequest.class);
+ final String recordKind = recordReindexRequest.getKind();
+ RecordQueryResponse recordQueryResponse = null;
+
+ try {
+ do {
+ if (recordQueryResponse != null) {
+ recordReindexRequest = RecordReindexRequest.builder().cursor(recordQueryResponse.getCursor()).kind(recordKind).build();
+ }
+ recordQueryResponse = this.storageService.getRecordsByKind(recordReindexRequest);
+ if (recordQueryResponse.getResults() != null && recordQueryResponse.getResults().size() != 0) {
+
+ List<RecordInfo> records = recordQueryResponse.getResults().stream()
+ .map(record -> RecordInfo.builder().id(record).kind(recordKind).op(OperationType.create.name()).build()).collect(Collectors.toList());
+
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(DpsHeaders.ACCOUNT_ID, headers.getPartitionIdWithFallbackToAccountId());
+ attributes.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
+ attributes.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
+
+ RecordChangedMessages recordChangedMessages = RecordChangedMessages.builder().data(gson.toJson(records)).attributes(attributes).build();
+ String recordChangedMessagePayload = gson.toJson(recordChangedMessages);
+ createTask(recordChangedMessagePayload, headers);
+ }
+ } while (!Strings.isNullOrEmpty(recordQueryResponse.getCursor()) && recordQueryResponse.getResults().size() == configurationProperties.getStorageRecordsBatchSize());
+
+ } catch (AppException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Unknown error", "An unknown error has occurred.", e);
+ }
+ }
+
+ private void createTask(String payload, DpsHeaders headers) {
+ Gson gson = new Gson();
+ RecordChangedMessages receivedPayload = gson.fromJson(payload, RecordChangedMessages.class);
+
+ Message message = new Message();
+ Map<String, Object> properties = new HashMap<>();
+
+ // properties
+ properties.put(DpsHeaders.ACCOUNT_ID, headers.getPartitionIdWithFallbackToAccountId());
+ properties.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
+ headers.addCorrelationIdIfMissing();
+ properties.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
+ message.setProperties(properties);
+
+ // data
+ List<RecordInfo> recordInfos = parseRecordsAsJSON(receivedPayload.getData());
+
+ // add all to body {"message": {"data":[], "id":...}}
+ JsonObject jo = new JsonObject();
+ jo.add("data", gson.toJsonTree(recordInfos));
+ jo.addProperty(DpsHeaders.ACCOUNT_ID, headers.getPartitionIdWithFallbackToAccountId());
+ jo.addProperty(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
+ jo.addProperty(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
+ JsonObject jomsg = new JsonObject();
+ jomsg.add("message", jo);
+
+ message.setBody(jomsg.toString().getBytes(StandardCharsets.UTF_8));
+ message.setContentType("application/json");
+
+ try {
+ logger.info("Indexer publishes message to Service Bus " + headers.getCorrelationId());
+ topicClientFactory.getClient(headers.getPartitionId(), serviceBusTopic).send(message);
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ }
+ }
+
+ private List<RecordInfo> parseRecordsAsJSON(String inputPayload) {
+ Gson gson = new Gson();
+ Type type = new TypeToken<List<RecordInfo>>(){}.getType();
+ List<RecordInfo> recordInfoList = gson.fromJson(inputPayload, type);
+ return recordInfoList;
+ }
+}
diff --git a/provider/indexer-gcp/pom.xml b/provider/indexer-gcp/pom.xml
index a37a4df8947684574ceb67a8389f5b658fef23c7..ff16763679abd9096948923f2ac504cd6e92ae57 100644
--- a/provider/indexer-gcp/pom.xml
+++ b/provider/indexer-gcp/pom.xml
@@ -5,12 +5,12 @@
<parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-service</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.8.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>indexer-gcp</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.8.0-SNAPSHOT</version>
<name>indexer-gcp</name>
<description>Indexer Service GCP App Engine</description>
<packaging>jar</packaging>
@@ -19,13 +19,13 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-core</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.8.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-gcp</artifactId>
- <version>0.6.1-SNAPSHOT</version>
+ <version>0.7.0</version>
</dependency>
<dependency>
diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/AppExceptionHandler.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/AppExceptionHandler.java
index 0e8293b1a59abd37ebdcd59bf3f2610a7c39e813..ebfe87ef077c4bbb244a1b68ba94d9138bfc0753 100644
--- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/AppExceptionHandler.java
+++ b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/AppExceptionHandler.java
@@ -23,12 +23,36 @@ public class AppExceptionHandler {
? e.getOriginalException().getMessage()
: e.getError().getMessage();
- if (e.getError().getCode() > 499) {
+ Integer errorCode = e.getError().getCode();
+
+ if (errorCode > 499) {
log.error(exceptionMsg, e.getOriginalException());
} else {
log.warn(exceptionMsg, e.getOriginalException());
}
- return new ResponseEntity<>(e.getError(), HttpStatus.resolve(e.getError().getCode()));
+ HttpStatus status = Objects.nonNull(HttpStatus.resolve(errorCode))
+ ? HttpStatus.resolve(errorCode)
+ : resolveNotSupportedStatus(errorCode);
+
+ return new ResponseEntity<>(e.getError(), status);
+ }
+
+ //Currently not all codes provided from core can be resolved by HttpStatus
+ //example org.opengroup.osdu.core.common.model.http.RequestStatus have not supported by HttpStatus codes
+ private HttpStatus resolveNotSupportedStatus(int statusCode) {
+ if (statusCode > 99 && statusCode < 200) {
+ return HttpStatus.CONTINUE;
+ }
+ if (statusCode > 199 && statusCode < 300) {
+ return HttpStatus.NO_CONTENT;
+ }
+ if (statusCode > 299 && statusCode < 400) {
+ return HttpStatus.MULTIPLE_CHOICES;
+ }
+ if (statusCode > 399 && statusCode < 500) {
+ return HttpStatus.BAD_REQUEST;
+ }
+ return HttpStatus.INTERNAL_SERVER_ERROR;
}
}
diff --git a/provider/indexer-gcp/src/main/resources/application.properties b/provider/indexer-gcp/src/main/resources/application.properties
index c2c74189250e40afae31e9f8c0174370a877c569..2213e4dca947f3ad90a5998299005c613a47ef61 100644
--- a/provider/indexer-gcp/src/main/resources/application.properties
+++ b/provider/indexer-gcp/src/main/resources/application.properties
@@ -42,4 +42,3 @@ security.https.certificate.trust=false
indexer.que.service.mail=default@iam.gserviceaccount.com
SCHEMA_HOST=${HOST}/api/schema-service/v1/schema
storage-query-kinds-host=https://${STORAGE_HOSTNAME}/api/storage/v2/query/kinds
-
diff --git a/provider/indexer-ibm/pom.xml b/provider/indexer-ibm/pom.xml
index a8ef4cf4cb9bc3d439b114778adc2cad62fc4fc6..abe6a5ad4769c95e854a1cc58e3c8702b8db3968 100644
--- a/provider/indexer-ibm/pom.xml
+++ b/provider/indexer-ibm/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-service</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.8.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
@@ -31,7 +31,7 @@
<packaging>jar</packaging>
<properties>
- <os-core-lib-ibm.version>0.3.8-SNAPSHOT</os-core-lib-ibm.version>
+ <os-core-lib-ibm.version>0.7.0</os-core-lib-ibm.version>
</properties>
<profiles>
@@ -53,7 +53,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-core</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.8.0-SNAPSHOT</version>
</dependency>
<dependency>
@@ -89,17 +89,14 @@
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
- <version>6.6.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
- <version>6.6.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
- <version>6.6.2</version>
</dependency>
<!-- Test Dependencies -->
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/di/EntitlementsClientFactory.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/di/EntitlementsClientFactory.java
new file mode 100644
index 0000000000000000000000000000000000000000..4a1938530b61f5a0530be55a47b30f7d453379bb
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/di/EntitlementsClientFactory.java
@@ -0,0 +1,44 @@
+
+package org.opengroup.osdu.indexer.ibm.di;
+
+import org.opengroup.osdu.core.common.entitlements.EntitlementsAPIConfig;
+import org.opengroup.osdu.core.common.entitlements.EntitlementsFactory;
+import org.opengroup.osdu.core.common.entitlements.IEntitlementsFactory;
+import org.opengroup.osdu.core.common.http.json.HttpResponseBodyMapper;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.beans.factory.config.AbstractFactoryBean;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+@Component
+@RequestScope
+@Lazy
+public class EntitlementsClientFactory extends AbstractFactoryBean<IEntitlementsFactory> {
+
+ @Value("${AUTHORIZE_API}")
+ private String AUTHORIZE_API;
+
+ @Value("${AUTHORIZE_API_KEY:}")
+ private String AUTHORIZE_API_KEY;
+
+ @Autowired
+ private HttpResponseBodyMapper mapper;
+
+ @Override
+ protected IEntitlementsFactory createInstance() throws Exception {
+
+ return new EntitlementsFactory(EntitlementsAPIConfig
+ .builder()
+ .rootUrl(AUTHORIZE_API)
+ .apiKey(AUTHORIZE_API_KEY)
+ .build(),
+ mapper);
+ }
+
+ @Override
+ public Class<?> getObjectType() {
+ return IEntitlementsFactory.class;
+ }
+}
\ No newline at end of file
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/IndexerQueueTaskBuilderIbm.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/IndexerQueueTaskBuilderIbm.java
index 48490a966c76462f23bf88b55e07a8c51b3c2dc9..2b4e88c68de8b4786c4d80b491ef418dfaf0acbc 100644
--- a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/IndexerQueueTaskBuilderIbm.java
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/IndexerQueueTaskBuilderIbm.java
@@ -3,6 +3,8 @@
package org.opengroup.osdu.indexer.ibm.util;
+import static org.opengroup.osdu.core.common.Constants.WORKER_RELATIVE_URL;
+
import java.util.Map;
import javax.inject.Inject;
@@ -46,6 +48,12 @@ public class IndexerQueueTaskBuilderIbm extends IndexerQueueTaskBuilder {
public void createReIndexTask(String payload, DpsHeaders headers) {
createTask(payload, headers);
}
+
+ //used by reindexer api
+ @Override
+ public void createWorkerTask(String payload, Long countdownMillis, DpsHeaders headers) {
+ createTask(payload, headers);
+ }
private void createTask(String payload, DpsHeaders headers) {
diff --git a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerMappingServiceTest.java b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerMappingServiceTest.java
index 7a6447711dda6b83ba8ba520c2480c45a276647f..d6982c113fc059e87626d2f931e4f2288f7fd3fd 100644
--- a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerMappingServiceTest.java
+++ b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerMappingServiceTest.java
@@ -1,12 +1,12 @@
/* Licensed Materials - Property of IBM */
/* (c) Copyright IBM Corp. 2020. All Rights Reserved.*/
-
package org.opengroup.osdu.indexer.ibm.service;
import org.apache.http.StatusLine;
import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest;
import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
-import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.*;
@@ -106,8 +106,7 @@ public class IndexerMappingServiceTest {
public void should_returnValidMapping_givenTrueMerge_createMappingTest() {
try {
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
-
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true);
assertEquals(this.mappingValid, mapping);
} catch (Exception e) {
@@ -119,8 +118,7 @@ public class IndexerMappingServiceTest {
public void should_returnValidMapping_givenExistType_createMappingTest() {
try {
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
-
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
IndexerMappingServiceImpl indexerMappingServiceLocal = PowerMockito.spy(new IndexerMappingServiceImpl());
doReturn(false).when(indexerMappingServiceLocal).isTypeExist(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any());
String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true);
@@ -129,7 +127,7 @@ public class IndexerMappingServiceTest {
fail("Should not throw this exception" + e.getMessage());
}
}
-
+
@Test
public void should_update_indices_field_with_keyword_when_valid_indices() throws Exception {
try {
@@ -137,33 +135,33 @@ public class IndexerMappingServiceTest {
indices.add("indices 1");
GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();
builder.field("any field", new HashMap());
builder.endObject();
BytesReference bytesReference = BytesReference.bytes(builder);
- FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
- Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
mappingBuilder.put("any index 1", mapBuilder);
mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
mapping.put("indices 1", mappingBuilder);
when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
- doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
BulkByScrollResponse response = mock(BulkByScrollResponse.class);
doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
-
+
this.sut.updateIndexMappingForIndicesOfSameType( indices,"any field");
} catch (Exception e) {
fail("Should not throw this exception" + e.getMessage());
}
}
-
+
@Test(expected = AppException.class)
public void should_throw_exception_if_someIndex_is_invalid_andWeIndexfield_with_keyword() throws Exception {
try {
@@ -171,28 +169,28 @@ public class IndexerMappingServiceTest {
indices.add("invalid 1");
GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();
builder.field("any field", new HashMap());
builder.endObject();
BytesReference bytesReference = BytesReference.bytes(builder);
- FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
- Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
mappingBuilder.put("any index 1", mapBuilder);
mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
mapping.put("indices 1", mappingBuilder);
when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
- doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
BulkByScrollResponse response = mock(BulkByScrollResponse.class);
doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
-
- this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
+
+ this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
} catch (Exception e) {
throw e;
}
@@ -205,22 +203,22 @@ public class IndexerMappingServiceTest {
indices.add("indices 1");
GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();
builder.field("any field", new HashMap());
builder.endObject();
BytesReference bytesReference = BytesReference.bytes(builder);
- FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
- Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
mappingBuilder.put("any index 1", mapBuilder);
mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
mapping.put("indices 1", mappingBuilder);
when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
- doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
BulkByScrollResponse response = mock(BulkByScrollResponse.class);
doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
@@ -240,22 +238,22 @@ public class IndexerMappingServiceTest {
indices.add("indices Invalid");
GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenThrow(new ElasticsearchException(""));
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenThrow(new ElasticsearchException(""));
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();
builder.field("any field", new HashMap());
builder.endObject();
BytesReference bytesReference = BytesReference.bytes(builder);
- FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
- Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
mappingBuilder.put("any index 1", mapBuilder);
mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
mapping.put("indices 1", mappingBuilder);
when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
- doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
BulkByScrollResponse response = mock(BulkByScrollResponse.class);
doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
@@ -274,26 +272,26 @@ public class IndexerMappingServiceTest {
indices.add("indices Invalid");
GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();
builder.field("any field", new HashMap());
builder.endObject();
BytesReference bytesReference = BytesReference.bytes(builder);
- FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
- Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+ Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
mapBuilder.put("data.any field", mappingMetaData);
- Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
mappingBuilder.put("any index 1", mapBuilder);
mappingBuilder.put("any index 2", mapBuilder);
- Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
mapping.put("indices 1", mappingBuilder);
when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
- doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
BulkByScrollResponse response = mock(BulkByScrollResponse.class);
doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
- when(this.indicesClient.putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class))).thenThrow(new ElasticsearchException(""));
+ when(this.indicesClient.putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class))).thenThrow(new ElasticsearchException(""));
when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
} catch (AppException e) {
@@ -303,3 +301,4 @@ public class IndexerMappingServiceTest {
}
}
}
+
diff --git a/provider/indexer-reference/pom.xml b/provider/indexer-reference/pom.xml
index 64212d38f47c19bfb2de3205a4b1fbf5617d84d5..4a20a160116a32dd5b35a133f3cd24295262b815 100644
--- a/provider/indexer-reference/pom.xml
+++ b/provider/indexer-reference/pom.xml
@@ -22,12 +22,12 @@
<parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-service</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.8.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>indexer-reference</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.8.0-SNAPSHOT</version>
<name>indexer-reference</name>
<description>Indexer Service GCP Anthos</description>
<packaging>jar</packaging>
@@ -36,7 +36,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-core</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.8.0-SNAPSHOT</version>
</dependency>
<dependency>
@@ -48,7 +48,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
- <version>0.0.18</version>
+ <version>0.6.5</version>
</dependency>
<dependency>
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/EntitlementsClientFactory.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/EntitlementsClientFactory.java
index 45131176cd1ff35eb2dc411b0073b245c0d3b8e8..2872c5c53405e12550c901064da117764208dc4f 100644
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/EntitlementsClientFactory.java
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/EntitlementsClientFactory.java
@@ -20,6 +20,7 @@ package org.opengroup.osdu.indexer.di;
import org.opengroup.osdu.core.common.entitlements.EntitlementsAPIConfig;
import org.opengroup.osdu.core.common.entitlements.EntitlementsFactory;
import org.opengroup.osdu.core.common.entitlements.IEntitlementsFactory;
+import org.opengroup.osdu.core.common.http.json.HttpResponseBodyMapper;
import org.opengroup.osdu.indexer.config.EntitlementsConfigProperties;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.config.AbstractFactoryBean;
@@ -34,12 +35,15 @@ public class EntitlementsClientFactory extends AbstractFactoryBean<IEntitlements
private EntitlementsConfigProperties entitlementsConfigProperties;
+ @Autowired
+ private HttpResponseBodyMapper mapper;
+
@Override
protected IEntitlementsFactory createInstance() throws Exception {
return new EntitlementsFactory(EntitlementsAPIConfig
.builder()
.rootUrl(entitlementsConfigProperties.getAuthorizeApi())
- .build());
+ .build(), mapper);
}
@Override
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
index 558e30bcc6d7c6653f51f1d023a56b6f015a5c18..1be2ba81868c6f3ed56cd4b56a25edfdbd576f99 100644
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
@@ -68,7 +68,6 @@ import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Service;
@Service
-@Primary
public class IndexerServiceImpl implements IndexerService {
private static final TimeValue BULK_REQUEST_TIMEOUT = TimeValue.timeValueMinutes(1);
diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java
index 9ec5fb7a30044f7f105c9a97229cb37322f218b9..bfc4f3a3330570ea73c59d2721954dd75aded7a5 100644
--- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java
+++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java
@@ -51,30 +51,17 @@ public class IndexFilterTest {
indexFilter.doFilter(httpServletRequest, httpServletResponse, filterChain);
- Mockito.verify(httpServletResponse)
- .addHeader("Access-Control-Allow-Origin", Collections.singletonList("*").toString());
- Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Headers", Collections
- .singletonList(
- "origin, content-type, accept, authorization, data-partition-id, correlation-id, appkey")
- .toString());
- Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Methods",
- Collections.singletonList("GET, POST, PUT, DELETE, OPTIONS, HEAD").toString());
- Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Credentials",
- Collections.singletonList("true").toString());
- Mockito.verify(httpServletResponse)
- .addHeader("X-Frame-Options", Collections.singletonList("DENY").toString());
- Mockito.verify(httpServletResponse)
- .addHeader("X-XSS-Protection", Collections.singletonList("1; mode=block").toString());
- Mockito.verify(httpServletResponse)
- .addHeader("X-Content-Type-Options", Collections.singletonList("nosniff").toString());
- Mockito.verify(httpServletResponse).addHeader("Cache-Control",
- Collections.singletonList("no-cache, no-store, must-revalidate").toString());
- Mockito.verify(httpServletResponse).addHeader("Content-Security-Policy",
- Collections.singletonList("default-src 'self'").toString());
- Mockito.verify(httpServletResponse).addHeader("Strict-Transport-Security",
- Collections.singletonList("max-age=31536000; includeSubDomains").toString());
- Mockito.verify(httpServletResponse)
- .addHeader("Expires", Collections.singletonList("0").toString());
+ Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Origin", Collections.singletonList("*").toString());
+ Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Headers", Collections.singletonList("origin, content-type, accept, authorization, data-partition-id, correlation-id, appkey").toString());
+ Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Methods", Collections.singletonList("GET, POST, PUT, DELETE, OPTIONS, HEAD, PATCH").toString());
+ Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Credentials", Collections.singletonList("true").toString());
+ Mockito.verify(httpServletResponse).addHeader("X-Frame-Options", Collections.singletonList("DENY").toString());
+ Mockito.verify(httpServletResponse).addHeader("X-XSS-Protection", Collections.singletonList("1; mode=block").toString());
+ Mockito.verify(httpServletResponse).addHeader("X-Content-Type-Options", Collections.singletonList("nosniff").toString());
+ Mockito.verify(httpServletResponse).addHeader("Cache-Control", Collections.singletonList("no-cache, no-store, must-revalidate").toString());
+ Mockito.verify(httpServletResponse).addHeader("Content-Security-Policy", Collections.singletonList("default-src 'self'").toString());
+ Mockito.verify(httpServletResponse).addHeader("Strict-Transport-Security", Collections.singletonList("max-age=31536000; includeSubDomains").toString());
+ Mockito.verify(httpServletResponse).addHeader("Expires", Collections.singletonList("0").toString());
Mockito.verify(httpServletResponse).addHeader("correlation-id", "correlation-id-value");
Mockito.verify(filterChain).doFilter(httpServletRequest, httpServletResponse);
}
diff --git a/testing/indexer-test-aws/pom.xml b/testing/indexer-test-aws/pom.xml
index e8c2f455b1a331a3cff88dbb309e509e59b1cbb5..2472c74b077f71aa6adfbdb750c091d8f86615bd 100644
--- a/testing/indexer-test-aws/pom.xml
+++ b/testing/indexer-test-aws/pom.xml
@@ -21,13 +21,13 @@
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>indexer-test</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.8.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-aws</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.8.0-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
@@ -42,7 +42,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.8.0-SNAPSHOT</version>
</dependency>
<!-- AWS specific packages -->
@@ -73,6 +73,22 @@
</dependency>
<!-- third party Apache 2.0 license packages -->
+ <!--Elasticsearch-->
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch</artifactId>
+ <version>7.8.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-client</artifactId>
+ <version>7.8.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-high-level-client</artifactId>
+ <version>7.8.1</version>
+ </dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
diff --git a/testing/indexer-test-azure/pom.xml b/testing/indexer-test-azure/pom.xml
index 2a6703142879b538f3f01319953cb9aa18833633..074e1cab971f0843dd57cf891727dba3873a821c 100644
--- a/testing/indexer-test-azure/pom.xml
+++ b/testing/indexer-test-azure/pom.xml
@@ -21,13 +21,13 @@
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>indexer-test</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.8.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-azure</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.8.0-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
@@ -45,14 +45,32 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.8.0-SNAPSHOT</version>
</dependency>
<!-- Azure dependencies -->
+ <!--Added dependency for utilizing azure service principle class-->
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-azure</artifactId>
<version>0.0.33</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>com.azure</groupId>
+ <artifactId>azure-core-http-netty</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ <version>4.1.50.Final</version>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport</artifactId>
+ <version>4.1.15.Final</version>
</dependency>
<dependency>
@@ -166,28 +184,5 @@
<version>3.0.0</version>
<scope>test</scope>
</dependency>
-<!--Added dependency for utilizing azure service principle class-->
- <dependency>
- <groupId>org.opengroup.osdu</groupId>
- <artifactId>core-lib-azure</artifactId>
- <version>0.0.33</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>com.azure</groupId>
- <artifactId>azure-core-http-netty</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-all</artifactId>
- <version>4.1.50.Final</version>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-transport</artifactId>
- <version>4.1.15.Final</version>
- </dependency>
</dependencies>
-</project>
\ No newline at end of file
+</project>
diff --git a/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
index ac04a614d3aea7ef3760a7f8e7d20d3afbe323b6..cacec759b027ddc62ee8451c0b5684cbd2e0bf9e 100644
--- a/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
+++ b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -69,4 +69,8 @@ public class Steps extends SchemaServiceRecordSteps {
super.iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(expectedCount, index, skippedAttributes);
}
+ @Then("^I should be able to search (\\d+) record with index \"([^\"]*)\" by tag \"([^\"]*)\" and value \"([^\"]*)\"$")
+ public void iShouldBeAbleToSearchRecordByTagKeyAndTagValue(int expectedNumber, String index, String tagKey, String tagValue) throws Throwable {
+ super.iShouldBeAbleToSearchRecordByTagKeyAndTagValue(index, tagKey, tagValue, expectedNumber);
+ }
}
\ No newline at end of file
diff --git a/testing/indexer-test-core/pom.xml b/testing/indexer-test-core/pom.xml
index 37659e5ba08a4f640dc0d678e5e453128833ba1c..0190d6b357070de949ae444d9b7d96aaba41ad9a 100644
--- a/testing/indexer-test-core/pom.xml
+++ b/testing/indexer-test-core/pom.xml
@@ -5,13 +5,13 @@
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>indexer-test</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.8.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.8.0-SNAPSHOT</version>
<properties>
<maven.compiler.target>1.8</maven.compiler.target>
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
index 1b420a262c01c5c1429ee8f0e7830b0b04bccb87..491bc22d2e31c2e47fe23e4af76d663bd2b538de 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
@@ -25,6 +25,7 @@ import java.lang.reflect.Type;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import static org.junit.Assert.*;
@@ -167,6 +168,13 @@ public class RecordSteps extends TestsBase {
assertEquals(expectedCount, documentCountByQuery);
}
+ public void iShouldBeAbleToSearchRecordByTagKeyAndTagValue(String index, String tagKey, String tagValue, int expectedNumber) throws Throwable {
+ TimeUnit.SECONDS.sleep(40);
+ index = generateActualName(index, timeStamp);
+ long actualNumberOfRecords = elasticUtils.fetchRecordsByTags(index, tagKey, tagValue);
+ assertEquals(expectedNumber, actualNumberOfRecords);
+ }
+
private long createIndex(String index) throws InterruptedException, IOException {
long numOfIndexedDocuments = 0;
int iterator;
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
index 452b8fd308a333af94707c3513d1018f1c8e6b59..cffca7e1e8644c68597a2827d59a8330dcac9ac0 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
@@ -59,9 +59,13 @@ import org.elasticsearch.search.builder.SearchSourceBuilder;
import java.io.IOException;
import java.util.*;
+import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.stream.Collectors;
+import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
+import static org.elasticsearch.index.query.QueryBuilders.termsQuery;
+
/**
* All util methods to use elastic apis for tests
@@ -215,6 +219,22 @@ public class ElasticUtils {
}
}
+ public long fetchRecordsByTags(String index, String tagKey, String tagValue) throws IOException {
+ try {
+ try (RestHighLevelClient client = this.createClient(username, password, host)) {
+ SearchRequest request = new SearchRequest(index);
+ SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
+ sourceBuilder.query(boolQuery().must(termsQuery(String.format("tags.%s", tagKey), tagValue)));
+ request.source(sourceBuilder);
+ SearchResponse searchResponse = client.search(request, RequestOptions.DEFAULT);
+ return searchResponse.getHits().getTotalHits().value;
+ }
+ } catch (ElasticsearchStatusException e) {
+ log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage()));
+ return -1;
+ }
+ }
+
public Map<String, Map<String, Object>> fetchRecordsData(String index) throws IOException {
try {
try (RestHighLevelClient client = this.createClient(username, password, host)) {
@@ -232,8 +252,9 @@ public class ElasticUtils {
}
}
- public long fetchRecordsByExistQuery(String index, String attributeName) throws IOException {
+ public long fetchRecordsByExistQuery(String index, String attributeName) throws Exception {
try {
+ TimeUnit.SECONDS.sleep(40);
try (RestHighLevelClient client = this.createClient(username, password, host)) {
SearchRequest searchRequest = new SearchRequest(index);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
diff --git a/testing/indexer-test-core/src/main/resources/features/indexrecord/IndexRecord.feature b/testing/indexer-test-core/src/main/resources/features/indexrecord/IndexRecord.feature
index c281ec4a0b174ec98eb781f8e603eba3b3968ca1..a06fa2efcbcb9107b184934c0304bf105afbb85e 100644
--- a/testing/indexer-test-core/src/main/resources/features/indexrecord/IndexRecord.feature
+++ b/testing/indexer-test-core/src/main/resources/features/indexrecord/IndexRecord.feature
@@ -15,8 +15,8 @@ Feature: Indexing of the documents
Examples:
| kind | recordFile | number | index | type | acl | mapping |
- | "tenant1:testindex<timestamp>:well:1.0.0" | "index_records_1" | 5 | "tenant1-testindex<timestamp>-well-1.0.0" | "well" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
- | "tenant1:testindex<timestamp>:well:3.0.0" | "index_records_1" | 5 | "tenant1-testindex<timestamp>-well-3.0.0" | "well" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
+ | "tenant1:testindex<timestamp>:well:1.0.0" | "index_records_1" | 5 | "tenant1-testindex<timestamp>-well-1.0.0" | "well" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Country":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"County":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"EmptyAttribute":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Established":{"type":"date"},"Field":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellName":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellStatus":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellType":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
+ | "tenant1:testindex<timestamp>:well:3.0.0" | "index_records_1" | 5 | "tenant1-testindex<timestamp>-well-3.0.0" | "well" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Country":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"County":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"EmptyAttribute":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Established":{"type":"date"},"Field":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellName":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellStatus":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellType":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
Scenario Outline: Ingest the record and Index in the Elastic Search with bad attribute
When I ingest records with the <recordFile> with <acl> for a given <kind>
diff --git a/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature b/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
index 920be422c258f89a160e7d2061b4796b0d3257bf..5c3d9e3bc9c5f8b3b71c86b753d1954c3b6bb23d 100644
--- a/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
+++ b/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
@@ -7,6 +7,7 @@ Feature: Indexing of the documents
| tenant1:indexer-int-test:sample-schema-1:2.0.4 | tenant1-indexer-int-test:sample-schema-1-2.0.4 | index_records_1 |
| tenant1:indexer-int-test:sample-schema-2:1.0.4 | tenant1-indexer-int-test:sample-schema-2-1.0.4 | index_records_2 |
| tenant1:indexer-int-test:sample-schema-3:1.0.4 | tenant1-indexer-int-test:sample-schema-3-1.0.4 | index_records_3 |
+ | tenant1:indexer-int-test:sample-schema-1:1.0.5 | tenant1-indexer-int-test:sample-schema-1-1.0.5 | index_records_1 |
Scenario Outline: Ingest the record and Index in the Elastic Search
When I ingest records with the <recordFile> with <acl> for a given <kind>
@@ -15,9 +16,9 @@ Feature: Indexing of the documents
Then I should get expected <elastic_data> values in the source section of Elastic Search results for the <index>
Examples:
- | kind | recordFile | number | index | type | acl | mapping | elastic_data |
- | "tenant1:indexer-int-test:sample-schema-1:2.0.4" | "index_records_1" | 5 | "tenant1-indexer-int-test-sample-schema-1-2.0.4" | "sample-schema-1" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" | "elastic_data_schema_1.json" |
- | "tenant1:indexer-int-test:sample-schema-3:1.0.4" | "index_records_1" | 5 | "tenant1-indexer-int-test-sample-schema-3-1.0.4" | "sample-schema-3" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" | "elastic_data_schema_3.json" |
+ | kind | recordFile | number | index | type | acl | mapping | elastic_data |
+ | "tenant1:indexer-int-test:sample-schema-1:2.0.4" | "index_records_schema_1" | 5 | "tenant1-indexer-int-test-sample-schema-1-2.0.4" | "sample-schema-1" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Country":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"County":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"EmptyAttribute":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Established":{"type":"date"},"Field":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellName":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellStatus":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellType":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" | "elastic_data_schema_1.json" |
+ | "tenant1:indexer-int-test:sample-schema-3:1.0.4" | "index_records_schema_1" | 5 | "tenant1-indexer-int-test-sample-schema-3-1.0.4" | "sample-schema-3" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Country":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"County":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"EmptyAttribute":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Established":{"type":"date"},"Field":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellName":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellStatus":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellType":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" | "elastic_data_schema_3.json" |
Scenario Outline: Ingest the record and Index in the Elastic Search with bad attribute
When I ingest records with the <recordFile> with <acl> for a given <kind>
@@ -27,3 +28,11 @@ Feature: Indexing of the documents
| kind | recordFile | number | index | skippedAttribute | acl |
| "tenant1:indexer-int-test:sample-schema-2:1.0.4" | "index_records_2" | 4 | "tenant1-indexer-int-test-sample-schema-2-1.0.4" | "data.Location" | "data.default.viewers@tenant1" |
| "tenant1:indexer-int-test:sample-schema-3:1.0.4" | "index_records_3" | 7 | "tenant1-indexer-int-test-sample-schema-3-1.0.4" | "data.GeoShape" | "data.default.viewers@tenant1" |
+
+ Scenario Outline: Ingest the record and Index in the Elastic Search with tags
+ When I ingest records with the <recordFile> with <acl> for a given <kind>
+ Then I should be able to search <number> record with index <index> by tag <tagKey> and value <tagValue>
+
+ Examples:
+ | kind | recordFile | index | acl | tagKey | tagValue | number |
+ | "tenant1:indexer-int-test:sample-schema-1:1.0.5" | "index_records_1" | "tenant1-indexer-int-test-sample-schema-1-1.0.5" | "data.default.viewers@tenant1" | "testtag" | "testvalue" | 5 |
diff --git a/testing/indexer-test-core/src/main/resources/testData/index_records_1.json b/testing/indexer-test-core/src/main/resources/testData/index_records_1.json
index 7affccb493fe398b9a8bc67fa4943d98416c3f32..d219115b89d9c1f2a461b472b0d6addd79d99ab8 100644
--- a/testing/indexer-test-core/src/main/resources/testData/index_records_1.json
+++ b/testing/indexer-test-core/src/main/resources/testData/index_records_1.json
@@ -1,6 +1,9 @@
[
{
"id": "tenant1:<kindSubType>:testIngest2<timestamp>",
+ "tags": {
+ "testtag": "testvalue"
+ },
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
@@ -24,6 +27,9 @@
},
{
"id": "tenant1:<kindSubType>:testIngest3<timestamp>",
+ "tags": {
+ "testtag": "testvalue"
+ },
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
@@ -47,6 +53,9 @@
},
{
"id": "tenant1:<kindSubType>:testIngest4<timestamp>",
+ "tags": {
+ "testtag": "testvalue"
+ },
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
@@ -69,6 +78,9 @@
},
{
"id": "tenant1:<kindSubType>:testIngest5<timestamp>",
+ "tags": {
+ "testtag": "testvalue"
+ },
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
@@ -91,6 +103,9 @@
},
{
"id": "tenant1:<kindSubType>:testIngest6<timestamp>",
+ "tags": {
+ "testtag": "testvalue"
+ },
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
diff --git a/testing/indexer-test-core/src/main/resources/testData/index_records_1.schema.json b/testing/indexer-test-core/src/main/resources/testData/index_records_1.schema.json
index 646155c8c9b09bf655edfa377d299e36fc0aa0b9..84cd895bd7d3014fb941a7f5d9b9db703fa97223 100644
--- a/testing/indexer-test-core/src/main/resources/testData/index_records_1.schema.json
+++ b/testing/indexer-test-core/src/main/resources/testData/index_records_1.schema.json
@@ -32,7 +32,14 @@
},
"County": {
"type": "string"
- },
+ }
+ }
+ }
+ ],
+ "anyOf": [
+ {
+ "type": "object",
+ "properties": {
"State": {
"type": "string"
},
@@ -53,7 +60,14 @@
},
"EmptyAttribute": {
"type": "string"
- },
+ }
+ }
+ }
+ ],
+ "oneOf": [
+ {
+ "type": "object",
+ "properties": {
"Rank": {
"type": "integer"
},
diff --git a/testing/indexer-test-gcp/pom.xml b/testing/indexer-test-gcp/pom.xml
index 74f56c7ace3750aa7165602b6e19c394229ed3a3..1f5f6b253efa1ab964b953f82112f6a213c33d86 100644
--- a/testing/indexer-test-gcp/pom.xml
+++ b/testing/indexer-test-gcp/pom.xml
@@ -6,13 +6,13 @@
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>indexer-test</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.8.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-gcp</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.8.0-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
@@ -37,7 +37,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.8.0-SNAPSHOT</version>
</dependency>
<!-- Cucumber -->
diff --git a/testing/indexer-test-ibm/pom.xml b/testing/indexer-test-ibm/pom.xml
index 1d0ec6267671426190b4f425241b0993597c5f75..21f69f828a8594c105ebe37e16e3c0c7e0e791c5 100644
--- a/testing/indexer-test-ibm/pom.xml
+++ b/testing/indexer-test-ibm/pom.xml
@@ -6,7 +6,7 @@
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>indexer-test</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.8.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -38,7 +38,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.8.0-SNAPSHOT</version>
</dependency>
<dependency>
@@ -114,17 +114,17 @@
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
- <version>6.6.2</version>
+ <version>7.8.1</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
- <version>6.6.2</version>
+ <version>7.8.1</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
- <version>6.6.2</version>
+ <version>7.8.1</version>
</dependency>
<!--Logging-->
diff --git a/testing/pom.xml b/testing/pom.xml
index 530619057e5638534ea2c8cb7bc0608cf1b8de1e..9d36987c79ff99bbea774dd4ebf269ff1d7fc156 100644
--- a/testing/pom.xml
+++ b/testing/pom.xml
@@ -18,7 +18,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.opengroup.osdu</groupId>
<artifactId>indexer-test</artifactId>
- <version>0.6.0-SNAPSHOT</version>
+ <version>0.8.0-SNAPSHOT</version>
<description>Indexer Service Integration Test Root Project</description>
<properties>
<spring.version>5.1.19.RELEASE</spring.version>