diff --git a/indexer-core/pom.xml b/indexer-core/pom.xml
index 308671c88808e2636b7cb90c5c1ac6f8e715be13..93252e1e089aaa7d508792ef4a4162720403f1d7 100644
--- a/indexer-core/pom.xml
+++ b/indexer-core/pom.xml
@@ -9,12 +9,27 @@
</parent>
<artifactId>indexer-core</artifactId>
- <version>1.0.5-SNAPSHOT</version>
+ <version>1.0.6-SNAPSHOT</version>
<name>indexer-core</name>
<description>Indexer Service Core</description>
<packaging>jar</packaging>
+ <properties>
+ <commons-beanutils.version>1.9.4</commons-beanutils.version>
+ </properties>
+
<dependencies>
+
+ <dependency>
+ <groupId>org.yaml</groupId>
+ <artifactId>snakeyaml</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.hibernate.validator</groupId>
+ <artifactId>hibernate-validator</artifactId>
+ </dependency>
+
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
@@ -52,7 +67,6 @@
<dependency>
<groupId>org.apache.tomcat.embed</groupId>
<artifactId>tomcat-embed-core</artifactId>
- <version>9.0.21</version>
</dependency>
<dependency>
@@ -65,17 +79,14 @@
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
- <version>6.6.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
- <version>6.6.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
- <version>6.6.2</version>
</dependency>
<dependency>
<groupId>org.locationtech.spatial4j</groupId>
@@ -108,17 +119,20 @@
<dependency>
<groupId>commons-beanutils</groupId>
<artifactId>commons-beanutils</artifactId>
- <version>1.9.3</version>
+ <version>${commons-beanutils.version}</version>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
- <version>1.12</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
- <version>2.9.9.3</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
</dependency>
<dependency>
<groupId>io.jsonwebtoken</groupId>
@@ -132,13 +146,6 @@
</exclusions>
</dependency>
- <!-- https://mvnrepository.com/artifact/commons-lang/commons-lang -->
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- <version>2.6</version>
- </dependency>
-
<!-- Test Dependencies -->
<dependency>
<groupId>org.springframework.boot</groupId>
diff --git a/pom.xml b/pom.xml
index 9fa0569762ee58fb04a9fdae10cacf0c227e4b2a..f666516546b5041cf58913e262477f6a8ec40b94 100644
--- a/pom.xml
+++ b/pom.xml
@@ -22,7 +22,15 @@
<springfox-version>2.7.0</springfox-version>
<spring-cloud.version>Greenwich.SR2</spring-cloud.version>
<os-core-common.version>0.3.6</os-core-common.version>
-<!-- <maven.compiler.target>1.8</maven.compiler.target>-->
+ <snakeyaml.version>1.26</snakeyaml.version>
+ <hibernate-validator.version>6.1.5.Final</hibernate-validator.version>
+ <jackson.version>2.11.2</jackson.version>
+ <tomcat-embed-core.version>9.0.37</tomcat-embed-core.version>
+ <common-codec.version>1.14</common-codec.version>
+ <elasticsearch.version>6.8.1</elasticsearch.version>
+ <netty.version>4.1.51.Final</netty.version>
+ <reactor-netty.version>0.8.20.RELEASE</reactor-netty.version>
+ <!-- <maven.compiler.target>1.8</maven.compiler.target>-->
<!-- <maven.compiler.source>1.8</maven.compiler.source>-->
<!-- <maven.war.plugin>2.6</maven.war.plugin>-->
<!-- <appengine.maven.plugin>1.0.0</appengine.maven.plugin>-->
@@ -60,11 +68,68 @@
<dependencyManagement>
<dependencies>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-bom</artifactId>
+ <version>${netty.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
<version>${os-core-common.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.yaml</groupId>
+ <artifactId>snakeyaml</artifactId>
+ <version>${snakeyaml.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.hibernate.validator</groupId>
+ <artifactId>hibernate-validator</artifactId>
+ <version>${hibernate-validator.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tomcat.embed</groupId>
+ <artifactId>tomcat-embed-core</artifactId>
+ <version>${tomcat-embed-core.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ <version>${common-codec.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch</artifactId>
+ <version>${elasticsearch.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-client</artifactId>
+ <version>${elasticsearch.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-high-level-client</artifactId>
+ <version>${elasticsearch.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.projectreactor.netty</groupId>
+ <artifactId>reactor-netty</artifactId>
+ <version>${reactor-netty.version}</version>
+ </dependency>
</dependencies>
</dependencyManagement>
diff --git a/provider/indexer-azure/pom.xml b/provider/indexer-azure/pom.xml
index 62d81b51380e5408f7afb21022f0ca0e68381d8d..343a293f710199d880e902d304534f1c2c7f8a64 100644
--- a/provider/indexer-azure/pom.xml
+++ b/provider/indexer-azure/pom.xml
@@ -39,9 +39,61 @@
<azure.appservice.plan></azure.appservice.plan>
<azure.appservice.appname></azure.appservice.appname>
<azure.appservice.subscription></azure.appservice.subscription>
+ <log4j.version>2.11.2</log4j.version>
+ <nimbus-jose-jwt.version>8.2</nimbus-jose-jwt.version>
+ <indexer-core.version>1.0.6-SNAPSHOT</indexer-core.version>
+ <spring-security-jwt.version>1.1.1.RELEASE</spring-security-jwt.version>
</properties>
<dependencies>
+ <dependency>
+ <groupId>org.yaml</groupId>
+ <artifactId>snakeyaml</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.hibernate.validator</groupId>
+ <artifactId>hibernate-validator</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ <version>${log4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-jul</artifactId>
+ <version>${log4j.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.nimbusds</groupId>
+ <artifactId>nimbus-jose-jwt</artifactId>
+ <version>${nimbus-jose-jwt.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.opengroup.osdu.indexer</groupId>
+ <artifactId>indexer-core</artifactId>
+ <version>${indexer-core.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </dependency>
+
<dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>azure-active-directory-spring-boot-starter</artifactId>
@@ -61,12 +113,18 @@
<groupId>org.springframework.security.oauth</groupId>
<artifactId>spring-security-oauth2</artifactId>
<version>2.3.6.RELEASE</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-mapper-asl</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-jwt</artifactId>
- <version>1.0.10.RELEASE</version>
+ <version>${spring-security-jwt.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.security</groupId>
@@ -90,11 +148,6 @@
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
</dependency>
- <dependency>
- <groupId>org.opengroup.osdu.indexer</groupId>
- <artifactId>indexer-core</artifactId>
- <version>1.0.5-SNAPSHOT</version>
- </dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
@@ -111,17 +164,14 @@
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
- <version>6.6.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
- <version>6.6.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
- <version>6.6.2</version>
</dependency>
<!-- Test Dependencies -->
@@ -253,7 +303,7 @@
<useSystemClassLoader>false</useSystemClassLoader>
<threadCount>1</threadCount>
</configuration>
- </plugin>
+ </plugin>
</plugins>
</build>
diff --git a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexerMappingServiceTest.java b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexerMappingServiceTest.java
index a391c1110ec40f5d395af88345f3723c0ce9d9ce..a83f2c7dab97d19426ad1e1800cf48dada775d66 100644
--- a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexerMappingServiceTest.java
+++ b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexerMappingServiceTest.java
@@ -16,8 +16,10 @@ package org.opengroup.osdu.indexer.azure.service;
import org.apache.http.StatusLine;
import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest;
import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.*;
@@ -72,7 +74,7 @@ public class IndexerMappingServiceTest {
private IndexerMappingServiceImpl sut;
@Mock
- private ElasticClientHandler elasticClientHandler;
+ private ElasticClientHandler elasticClientHandler;
@InjectMocks
private RestHighLevelClient restHighLevelClient;
@@ -117,7 +119,7 @@ public class IndexerMappingServiceTest {
public void should_returnValidMapping_givenTrueMerge_createMappingTest() {
try {
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true);
assertEquals(this.mappingValid, mapping);
@@ -130,7 +132,7 @@ public class IndexerMappingServiceTest {
public void should_returnValidMapping_givenExistType_createMappingTest() {
try {
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
IndexerMappingServiceImpl indexerMappingServiceLocal = PowerMockito.spy(new IndexerMappingServiceImpl());
doReturn(false).when(indexerMappingServiceLocal).isTypeExist(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any());
@@ -148,7 +150,7 @@ public class IndexerMappingServiceTest {
indices.add("indices 1");
GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();
builder.field("any field", new HashMap());
@@ -163,12 +165,12 @@ public class IndexerMappingServiceTest {
Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
mapping.put("indices 1", mappingBuilder);
when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
- doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
BulkByScrollResponse response = mock(BulkByScrollResponse.class);
doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
-
+
this.sut.updateIndexMappingForIndicesOfSameType( indices,"any field");
} catch (Exception e) {
fail("Should not throw this exception" + e.getMessage());
@@ -182,7 +184,7 @@ public class IndexerMappingServiceTest {
indices.add("invalid 1");
GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();
builder.field("any field", new HashMap());
@@ -197,13 +199,13 @@ public class IndexerMappingServiceTest {
Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
mapping.put("indices 1", mappingBuilder);
when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
- doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
BulkByScrollResponse response = mock(BulkByScrollResponse.class);
doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
-
- this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
+
+ this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
} catch (Exception e) {
throw e;
}
@@ -213,10 +215,10 @@ public class IndexerMappingServiceTest {
public void should_throw_exception_if_type_of_index_is_invalid_andWeIndexfield_with_keyword() throws Exception {
try {
Set<String> indices = new HashSet<String>();
- indices.add("indices 1");
+ indices.add("indices 1");
GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();
builder.field("any field", new HashMap());
@@ -231,7 +233,7 @@ public class IndexerMappingServiceTest {
Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
mapping.put("indices 1", mappingBuilder);
when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
- doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
BulkByScrollResponse response = mock(BulkByScrollResponse.class);
doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
@@ -251,7 +253,7 @@ public class IndexerMappingServiceTest {
indices.add("indices Invalid");
GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenThrow(new ElasticsearchException(""));
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenThrow(new ElasticsearchException(""));
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();
builder.field("any field", new HashMap());
@@ -266,7 +268,7 @@ public class IndexerMappingServiceTest {
Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
mapping.put("indices 1", mappingBuilder);
when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
- doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
BulkByScrollResponse response = mock(BulkByScrollResponse.class);
doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
@@ -285,7 +287,7 @@ public class IndexerMappingServiceTest {
indices.add("indices Invalid");
GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
- when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();
builder.field("any field", new HashMap());
@@ -300,11 +302,11 @@ public class IndexerMappingServiceTest {
Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
mapping.put("indices 1", mappingBuilder);
when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
- doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
BulkByScrollResponse response = mock(BulkByScrollResponse.class);
doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
- when(this.indicesClient.putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class))).thenThrow(new ElasticsearchException(""));
+ when(this.indicesClient.putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class))).thenThrow(new ElasticsearchException(""));
when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
} catch (AppException e) {