diff --git a/NOTICE b/NOTICE
index b27f17fac2923f619c9e40e0c530605d8c10066d..f1bf5bc49258adf89571d007a7dadd1a2ef00bcc 100644
--- a/NOTICE
+++ b/NOTICE
@@ -27,6 +27,7 @@ The following software have components provided under the terms of this license:
- ASM based accessors helper used by json-smart (from https://urielch.github.io/)
- AWS SDK For Java (from https://aws.amazon.com/sdkforjava)
- AWS SDK for Java - BOM (from https://aws.amazon.com/sdkforjava)
+- AWS SDK for Java - Models (from https://aws.amazon.com/sdkforjava)
- Apache Ant + JUnit (from http://ant.apache.org/, https://ant.apache.org/)
- Apache Ant Core
- Apache Ant Launcher (from http://ant.apache.org/)
@@ -110,6 +111,7 @@ The following software have components provided under the terms of this license:
- Nimbus Content Type (from https://bitbucket.org/connect2id/nimbus-content-type)
- Nimbus LangTag (from https://bitbucket.org/connect2id/nimbus-language-tags)
- Non-Blocking Reactive Foundation for the JVM (from https://github.com/reactor/reactor, https://github.com/reactor/reactor-core)
+- OAuth 2.0 SDK with OpenID Connect extensions (from https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions)
- Objenesis (from http://objenesis.org)
- OkHttp (from https://repo1.maven.org/maven2/com/squareup/okhttp3/okhttp, https://square.github.io/okhttp/)
- Okio (from https://github.com/square/okio/, https://repo1.maven.org/maven2/com/squareup/okio/okio)
@@ -117,6 +119,7 @@ The following software have components provided under the terms of this license:
- Plexus Velocity Component (from https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-velocity)
- PostgreSQL JDBC Driver
- PowerMock (from http://www.powermock.org, https://repo1.maven.org/maven2/org/powermock/powermock-api-mockito)
+- Prometheus Java Simpleclient (from https://repo1.maven.org/maven2/io/prometheus/simpleclient)
- Protocol Buffer extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-protobuf)
- RabbitMQ Java Client (from http://www.rabbitmq.com, https://www.rabbitmq.com)
- RxJava (from https://github.com/ReactiveX/RxJava)
@@ -157,6 +160,10 @@ BSD-2-Clause
========================================================================
The following software have components provided under the terms of this license:
+- ASM Analysis (from http://asm.ow2.io/)
+- ASM Commons (from http://asm.ow2.io/, https://repo1.maven.org/maven2/org/ow2/asm/asm-commons)
+- ASM Tree (from http://asm.ow2.io/, https://repo1.maven.org/maven2/org/ow2/asm/asm-tree)
+- ASM Util (from http://asm.ow2.io/)
- Apache Commons CLI (from https://commons.apache.org/proper/commons-cli/, https://repo1.maven.org/maven2/commons-cli/commons-cli)
- Hamcrest (from http://hamcrest.org/JavaHamcrest/)
- Hamcrest Core (from http://hamcrest.org/, https://repo1.maven.org/maven2/org/hamcrest/hamcrest-core)
@@ -364,8 +371,6 @@ The following software have components provided under the terms of this license:
========================================================================
LGPL-3.0-only
========================================================================
-The following software have components provided under the terms of this license:
-
- RabbitMQ Java Client (from http://www.rabbitmq.com, https://www.rabbitmq.com)
========================================================================
@@ -465,6 +470,7 @@ The following software have components provided under the terms of this license:
- LatencyUtils (from http://latencyutils.github.io/LatencyUtils/)
- PostgreSQL JDBC Driver
- Project Lombok (from http://projectlombok.org, https://projectlombok.org)
+- Prometheus Java Simpleclient (from https://repo1.maven.org/maven2/io/prometheus/simpleclient)
- RabbitMQ Java Client (from http://www.rabbitmq.com, https://www.rabbitmq.com)
- reactive-streams (from http://www.reactive-streams.org/)
- xml-apis
@@ -489,4 +495,4 @@ The following software have components provided under the terms of this license:
- Jakarta Activation API (from https://github.com/eclipse-ee4j/jaf, https://repo1.maven.org/maven2/jakarta/activation/jakarta.activation-api)
- Jakarta XML Binding API (from https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api, https://repo1.maven.org/maven2/org/jboss/spec/javax/xml/bind/jboss-jaxb-api_2.3_spec)
- jaxen (from http://jaxen.codehaus.org/, https://repo1.maven.org/maven2/jaxen/jaxen)
-- xml-apis
+- xml-apis
\ No newline at end of file
diff --git a/indexer-core/pom.xml b/indexer-core/pom.xml
index a12777a9faf3b82592af3e8f0881c16739e5e229..13e3c0672d9d49875ba95de9d293c550bee1995a 100644
--- a/indexer-core/pom.xml
+++ b/indexer-core/pom.xml
@@ -4,19 +4,18 @@
<parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-service</artifactId>
- <version>0.14.0-SNAPSHOT</version>
+ <version>0.15.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>indexer-core</artifactId>
- <version>0.14.0-SNAPSHOT</version>
+ <version>0.15.0-SNAPSHOT</version>
<name>indexer-core</name>
<description>Indexer Service Core</description>
<packaging>jar</packaging>
<properties>
<commons-beanutils.version>1.9.4</commons-beanutils.version>
- <osdu.oscorecommon.version>0.14.0-rc4</osdu.oscorecommon.version>
<spring-security-web.version>5.4.9</spring-security-web.version>
<gson.version>2.8.9</gson.version>
<netty.version>4.1.70.Final</netty.version>
@@ -49,7 +48,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
- <version>${osdu.oscorecommon.version}</version>
+ <version>${os-core-common.version}</version>
</dependency>
<!-- spring boot dependencies -->
@@ -217,7 +216,7 @@
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
- <version>3.0.0</version>
+ <version>3.12.0</version>
<scope>test</scope>
</dependency>
<dependency>
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/IndexerApplication.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/IndexerApplication.java
index 3b33fd2ebb76f058d9f4a41120383d1c48d1b738..416bf89dd51c5e78838ea5f6fcc86166c429d4a5 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/IndexerApplication.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/IndexerApplication.java
@@ -1,7 +1,7 @@
package org.opengroup.osdu.indexer;
import org.springframework.boot.SpringApplication;
-import org.springframework.boot.actuate.autoconfigure.elasticsearch.ElasticSearchRestHealthIndicatorAutoConfiguration;
+import org.springframework.boot.actuate.autoconfigure.elasticsearch.ElasticSearchRestHealthContributorAutoConfiguration;
import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
@@ -10,7 +10,7 @@ import org.springframework.context.annotation.Configuration;
@Configuration
@ComponentScan({"org.opengroup.osdu.core.common","org.opengroup.osdu.indexer", "org.opengroup.osdu.is"})
-@SpringBootApplication(exclude = {ElasticSearchRestHealthIndicatorAutoConfiguration.class, SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
+@SpringBootApplication(exclude = {ElasticSearchRestHealthContributorAutoConfiguration.class, SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
public class IndexerApplication {
public static void main( String[] args )
{
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/error/GlobalExceptionMapperCore.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/error/GlobalExceptionMapperCore.java
index c34001666b0c993dae73df6d792caa8d93e6067a..39f5a76b950163f1b283ccd9d8c5adc96a1e5b79 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/error/GlobalExceptionMapperCore.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/error/GlobalExceptionMapperCore.java
@@ -20,6 +20,7 @@ import javassist.NotFoundException;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.indexer.schema.converter.exeption.SchemaProcessingException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.Ordered;
import org.springframework.core.annotation.Order;
@@ -74,6 +75,12 @@ public class GlobalExceptionMapperCore extends ResponseEntityExceptionHandler {
}
}
+ @ExceptionHandler(SchemaProcessingException.class)
+ public ResponseEntity<Object> handleSchemaProcessingException(SchemaProcessingException e) {
+ return this.getErrorResponse(
+ new AppException(HttpStatus.BAD_REQUEST.value(), "Error processing schema.", e.getMessage(), e));
+ }
+
@ExceptionHandler(Exception.class)
protected ResponseEntity<Object> handleGeneralException(Exception e) {
return this.getErrorResponse(
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
index 844b310f1e5048782cd4ab69bde83f43e6375df3..24544bc2fb9e84fd2ed1b4691771f54fd7cfe457 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
@@ -366,7 +366,7 @@ public class IndexerServiceImpl implements IndexerService {
String index = this.elasticIndexNameResolver.getIndexNameFromKind(schema.getKind());
// check if index exist and sync meta attribute schema if required
- if (this.indicesService.isIndexExist(restClient, index)) {
+ if (this.indicesService.isIndexReady(restClient, index)) {
this.mappingService.syncIndexMappingIfRequired(restClient, index);
continue;
}
@@ -435,7 +435,9 @@ public class IndexerServiceImpl implements IndexerService {
Exception failedRequestCause = null;
try {
+ long startTime = System.currentTimeMillis();
BulkResponse bulkResponse = restClient.bulk(bulkRequest, RequestOptions.DEFAULT);
+ long stopTime = System.currentTimeMillis();
// log failed bulk requests
ArrayList<String> bulkFailures = new ArrayList<>();
@@ -464,7 +466,7 @@ public class IndexerServiceImpl implements IndexerService {
}
if (!bulkFailures.isEmpty()) this.jaxRsDpsLog.warning(bulkFailures);
- jaxRsDpsLog.info(String.format("records in elasticsearch service bulk request: %s | successful: %s | failed: %s", bulkRequest.numberOfActions(), succeededResponses, failedResponses));
+ jaxRsDpsLog.info(String.format("records in elasticsearch service bulk request: %s | successful: %s | failed: %s | time taken for bulk request: %d milliseconds", bulkRequest.numberOfActions(), succeededResponses, failedResponses, stopTime-startTime));
// retry entire message if all records are failing
if (bulkRequest.numberOfActions() == failureRecordIds.size()) throw new AppException(failedRequestStatus, "Elastic error", failedRequestCause.getMessage(), failedRequestCause);
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesService.java
index c09ff3e5744cb5ad52016499abdb8e55aff0b271..48fa3cfe46adf304beb772415b95b2b19c933450 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesService.java
@@ -36,4 +36,6 @@ public interface IndicesService {
boolean deleteIndex(String index) throws ElasticsearchException, IOException, AppException;
List<IndexInfo> getIndexInfo(RestHighLevelClient client, String indexPattern) throws IOException;
+
+ boolean isIndexReady(RestHighLevelClient client, String index) throws IOException;
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java
index ad1d20f43d6c72f8371331e70779f1e7dd18a791..25f5945654edc51b92cd8a53099298dca6d59d5f 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java
@@ -22,6 +22,8 @@ import org.apache.http.HttpStatus;
import org.apache.http.util.EntityUtils;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.Request;
@@ -92,13 +94,18 @@ public class IndicesServiceImpl implements IndicesService {
request.settings(settings != null ? settings : DEFAULT_INDEX_SETTINGS);
if (mapping != null) {
String mappingJsonString = new Gson().toJson(mapping, Map.class);
- request.mapping(mappingJsonString,XContentType.JSON);
+ request.mapping(mappingJsonString, XContentType.JSON);
}
request.setTimeout(REQUEST_TIMEOUT);
+ long startTime = System.currentTimeMillis();
CreateIndexResponse response = client.indices().create(request, RequestOptions.DEFAULT);
+ long stopTime = System.currentTimeMillis();
// cache the index status
boolean indexStatus = response.isAcknowledged() && response.isShardsAcknowledged();
- if (indexStatus) this.indexCache.put(index, true);
+ if (indexStatus) {
+ this.indexCache.put(index, true);
+ this.log.info(String.format("Time taken to successfully create new index %s : %d milliseconds", request.index(), stopTime-startTime));
+ }
return indexStatus;
} catch (ElasticsearchStatusException e) {
@@ -121,13 +128,7 @@ public class IndicesServiceImpl implements IndicesService {
*/
public boolean isIndexExist(RestHighLevelClient client, String index) throws IOException {
try {
- try {
- Boolean isIndexExist = (Boolean) this.indexCache.get(index);
- if (isIndexExist != null && isIndexExist) return true;
- } catch (RedisException ex) {
- //In case the format of cache changes then clean the cache
- this.indexCache.delete(index);
- }
+ if (this.indexExistInCache(index)) return true;
GetIndexRequest request = new GetIndexRequest(index);
boolean exists = client.indices().exists(request, RequestOptions.DEFAULT);
if (exists) this.indexCache.put(index, true);
@@ -142,6 +143,50 @@ public class IndicesServiceImpl implements IndicesService {
}
}
+ /**
+ * Check if an index ready for indexing
+ *
+ * @param index Index name
+ * @return index details if index already exists
+ * @throws IOException if request cannot be processed
+ */
+ public boolean isIndexReady(RestHighLevelClient client, String index) throws IOException {
+ try {
+ if (this.indexExistInCache(index)) return true;
+ GetIndexRequest request = new GetIndexRequest(index);
+ boolean exists = client.indices().exists(request, RequestOptions.DEFAULT);
+ if (!exists) return false;
+ ClusterHealthRequest indexHealthRequest = new ClusterHealthRequest();
+ indexHealthRequest.indices(index);
+ indexHealthRequest.timeout(REQUEST_TIMEOUT);
+ indexHealthRequest.waitForYellowStatus();
+ ClusterHealthResponse healthResponse = client.cluster().health(indexHealthRequest, RequestOptions.DEFAULT);
+ if (healthResponse.status() == RestStatus.OK) {
+ this.indexCache.put(index, true);
+ return true;
+ }
+ return false;
+ } catch (ElasticsearchException exception) {
+ if (exception.status() == RestStatus.NOT_FOUND) return false;
+ throw new AppException(
+ exception.status().getStatus(),
+ exception.getMessage(),
+ String.format("Error getting index: %s status", index),
+ exception);
+ }
+ }
+
+ private boolean indexExistInCache(String index) {
+ try {
+ Boolean isIndexExist = (Boolean) this.indexCache.get(index);
+ if (isIndexExist != null && isIndexExist) return true;
+ } catch (RedisException ex) {
+ //In case the format of cache changes then clean the cache
+ this.indexCache.delete(index);
+ }
+ return false;
+ }
+
/**
* Deletes index if user has required role: search.admin
*
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/error/GlobalExceptionMapperCoreTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/error/GlobalExceptionMapperCoreTest.java
index 992c0a01fea9b1892e20eaaa31b5674028741a85..1877df26e55b971991b6f222cc28a49926426c0f 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/error/GlobalExceptionMapperCoreTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/error/GlobalExceptionMapperCoreTest.java
@@ -23,6 +23,7 @@ import org.mockito.Mock;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.RequestStatus;
+import org.opengroup.osdu.indexer.schema.converter.exeption.SchemaProcessingException;
import org.powermock.modules.junit4.PowerMockRunner;
import org.springframework.http.ResponseEntity;
import org.springframework.security.access.AccessDeniedException;
@@ -107,4 +108,12 @@ public class GlobalExceptionMapperCoreTest {
ResponseEntity response = this.sut.handleIOException(ioException);
assertEquals(HttpStatus.SC_SERVICE_UNAVAILABLE, response.getStatusCodeValue());
}
+
+ @Test
+ public void should_returnAppException_when_SchemaProcessingExceptionIsThrown() {
+ SchemaProcessingException schemaProcessingException = new SchemaProcessingException("error processing schema");
+
+ ResponseEntity response = this.sut.handleSchemaProcessingException(schemaProcessingException);
+ assertEquals(HttpStatus.SC_BAD_REQUEST, response.getStatusCodeValue());
+ }
}
\ No newline at end of file
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndicesServiceTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndicesServiceTest.java
index 38c7a4f682e39182beaac68bac21ebd8ee34d3d9..17bce5783d76f793e848268d9f6a68de7f105a6a 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndicesServiceTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndicesServiceTest.java
@@ -22,8 +22,11 @@ import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.util.EntityUtils;
import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.*;
+import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.rest.RestStatus;
import org.junit.Before;
import org.junit.Test;
@@ -51,7 +54,7 @@ import static org.mockito.Mockito.*;
import static org.mockito.MockitoAnnotations.initMocks;
@RunWith(SpringRunner.class)
-@PrepareForTest({RestHighLevelClient.class, IndicesClient.class, EntityUtils.class})
+@PrepareForTest({RestHighLevelClient.class, IndicesClient.class, ClusterClient.class, EntityUtils.class})
public class IndicesServiceTest {
@Mock
private ElasticClientHandler elasticClientHandler;
@@ -73,11 +76,13 @@ public class IndicesServiceTest {
private RestHighLevelClient restHighLevelClient;
private IndicesClient indicesClient;
+ private ClusterClient clusterClient;
@Before
public void setup() {
initMocks(this);
indicesClient = PowerMockito.mock(IndicesClient.class);
+ clusterClient = PowerMockito.mock(ClusterClient.class);
restHighLevelClient = PowerMockito.mock(RestHighLevelClient.class);
}
@@ -218,4 +223,54 @@ public class IndicesServiceTest {
assertEquals("1", indicesList.get(0).getDocumentCount());
assertEquals("1551996907769", indicesList.get(0).getCreationDate());
}
+
+ @Test
+ public void should_returnTrue_indexExistInCache() throws IOException {
+ when(this.indicesExistCache.get("anyIndex")).thenReturn(true);
+
+ boolean result = this.sut.isIndexExist(any(RestHighLevelClient.class), "anyIndex");
+
+ assertTrue(result);
+ }
+
+ @Test
+ public void should_getIndexExist_whenIndexNotInCache() throws IOException {
+ when(this.indicesExistCache.get("anyIndex")).thenReturn(false);
+
+ doReturn(indicesClient).when(restHighLevelClient).indices();
+ doReturn(true).when(indicesClient).exists(any(GetIndexRequest.class), any(RequestOptions.class));
+
+ boolean result = this.sut.isIndexExist(restHighLevelClient, "anyIndex");
+
+ assertTrue(result);
+ verify(this.indicesExistCache, times(1)).get("anyIndex");
+ verify(this.indicesExistCache, times(1)).put("anyIndex", true);
+ }
+
+ @Test
+ public void should_getIndexReadyStatus_whenIndexInCache() throws IOException {
+ when(this.indicesExistCache.get("anyIndex")).thenReturn(true);
+
+ boolean result = this.sut.isIndexReady(any(RestHighLevelClient.class), "anyIndex");
+
+ assertTrue(result);
+ }
+
+ @Test
+ public void should_getIndexReadyStatus_whenIndexNotInCache() throws IOException {
+ when(this.indicesExistCache.get("anyIndex")).thenReturn(false);
+ doReturn(indicesClient).when(restHighLevelClient).indices();
+ doReturn(true).when(indicesClient).exists(any(GetIndexRequest.class), any(RequestOptions.class));
+
+ ClusterHealthResponse healthResponse = mock(ClusterHealthResponse.class);
+ when(healthResponse.status()).thenReturn(RestStatus.OK);
+ doReturn(clusterClient).when(restHighLevelClient).cluster();
+ doReturn(healthResponse).when(clusterClient).health(any(ClusterHealthRequest.class), any(RequestOptions.class));
+
+ boolean result = this.sut.isIndexReady(restHighLevelClient, "anyIndex");
+
+ assertTrue(result);
+ verify(this.indicesExistCache, times(1)).get("anyIndex");
+ verify(this.indicesExistCache, times(1)).put("anyIndex", true);
+ }
}
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index c148fd375a65588a7ce7a39dc0a1ff84c10c5f6c..2d5af651a216fc4e05d9af6e969c83cf26c44255 100644
--- a/pom.xml
+++ b/pom.xml
@@ -5,7 +5,7 @@
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-service</artifactId>
<packaging>pom</packaging>
- <version>0.14.0-SNAPSHOT</version>
+ <version>0.15.0-SNAPSHOT</version>
<description>Indexer Service</description>
<properties>
@@ -13,7 +13,7 @@
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
<spring-cloud.version>Greenwich.SR2</spring-cloud.version>
- <os-core-common.version>0.14.0-rc4</os-core-common.version>
+ <os-core-common.version>0.14.0-rc8</os-core-common.version>
<snakeyaml.version>1.26</snakeyaml.version>
<hibernate-validator.version>6.1.5.Final</hibernate-validator.version>
<jackson.version>2.11.4</jackson.version>
@@ -24,7 +24,7 @@
<reactor-netty.version>0.8.20.RELEASE</reactor-netty.version>
<woodstox-core.version>6.2.3</woodstox-core.version>
<log4j2.version>2.17.1</log4j2.version>
- <spring-boot.version>2.1.18.RELEASE</spring-boot.version>
+<!-- <spring-boot.version>2.1.18.RELEASE</spring-boot.version>-->
<json-smart.version>2.4.7</json-smart.version>
<!-- <maven.war.plugin>2.6</maven.war.plugin>-->
<!-- <appengine.maven.plugin>1.0.0</appengine.maven.plugin>-->
@@ -46,9 +46,9 @@
<dependencyManagement>
<dependencies>
<dependency>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-dependencies</artifactId>
- <version>${spring-boot.version}</version>
+ <groupId>org.opengroup.osdu</groupId>
+ <artifactId>os-core-common</artifactId>
+ <version>${os-core-common.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
@@ -69,11 +69,6 @@
<type>pom</type>
<scope>import</scope>
</dependency>
- <dependency>
- <groupId>org.opengroup.osdu</groupId>
- <artifactId>os-core-common</artifactId>
- <version>${os-core-common.version}</version>
- </dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
@@ -138,6 +133,11 @@
</dependencyManagement>
<dependencies>
+ <dependency>
+ <groupId>org.opengroup.osdu</groupId>
+ <artifactId>os-core-common</artifactId>
+ <version>${os-core-common.version}</version>
+ </dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
diff --git a/provider/indexer-aws/pom.xml b/provider/indexer-aws/pom.xml
index 2b16ef5eea4d596e8c287017bd8e3eef06a33239..a4e9c60910a9d8e4f6f785d26d94a3c3f6f250a8 100644
--- a/provider/indexer-aws/pom.xml
+++ b/provider/indexer-aws/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-service</artifactId>
- <version>0.14.0-SNAPSHOT</version>
+ <version>0.15.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
@@ -26,7 +26,7 @@
<artifactId>indexer-aws</artifactId>
<description>Indexer service on AWS</description>
<packaging>jar</packaging>
- <version>0.14.0-SNAPSHOT</version>
+ <version>0.15.0-SNAPSHOT</version>
<properties>
<aws.version>1.11.1018</aws.version>
@@ -59,11 +59,12 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
+ <version>${os-core-common.version}</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-core</artifactId>
- <version>0.14.0-SNAPSHOT</version>
+ <version>0.15.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
@@ -161,7 +162,6 @@
<dependency>
<groupId>org.springframework.data</groupId>
<artifactId>spring-data-commons</artifactId>
- <version>2.1.10.RELEASE</version>
<scope>compile</scope>
</dependency>
</dependencies>
diff --git a/provider/indexer-azure/pom.xml b/provider/indexer-azure/pom.xml
index 109f4eea0d2af31a8a849da4876421483fb9bfaa..c6772ec04587566053e8b2ad6e2f5aa01c602528 100644
--- a/provider/indexer-azure/pom.xml
+++ b/provider/indexer-azure/pom.xml
@@ -21,12 +21,12 @@
<parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-service</artifactId>
- <version>0.14.0-SNAPSHOT</version>
+ <version>0.15.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>indexer-azure</artifactId>
- <version>0.14.0-SNAPSHOT</version>
+ <version>0.15.0-SNAPSHOT</version>
<name>indexer-azure</name>
<description>Indexer Service Azure</description>
<packaging>jar</packaging>
@@ -39,10 +39,9 @@
<azure.appservice.subscription />
<log4j.version>2.17.1</log4j.version>
<nimbus-jose-jwt.version>8.2</nimbus-jose-jwt.version>
- <indexer-core.version>0.14.0-SNAPSHOT</indexer-core.version>
+ <indexer-core.version>0.15.0-SNAPSHOT</indexer-core.version>
<spring-security-jwt.version>1.1.1.RELEASE</spring-security-jwt.version>
<osdu.corelibazure.version>0.14.0-rc2</osdu.corelibazure.version>
- <osdu.oscorecommon.version>0.13.0</osdu.oscorecommon.version>
<reactor-netty.version>0.9.12.RELEASE</reactor-netty.version>
<java-jwt.version>3.8.1</java-jwt.version>
<powermock.version>2.0.2</powermock.version>
@@ -227,6 +226,14 @@
<version>1.7.0</version>
</dependency>
+ <!-- Prometheus Dependency -->
+ <dependency>
+ <groupId>io.micrometer</groupId>
+ <artifactId>micrometer-registry-prometheus</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+
+
<!-- Test Dependencies -->
<dependency>
<groupId>org.springframework.boot</groupId>
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java
index 6ee0289c6e1526316a71f3b94f372dcb6ed0c1bc..f7cabb70c42148fac7ddf806c811d3976b80aa91 100644
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java
+++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java
@@ -167,8 +167,10 @@ public class IndexerQueueTaskBuilderAzure extends IndexerQueueTaskBuilder {
message.setContentType("application/json");
try {
- logger.info("Indexer publishes message to Service Bus " + headers.getCorrelationId());
+ long startTime = System.currentTimeMillis();
topicClientFactory.getClient(headers.getPartitionId(), serviceBusReindexTopicName).send(message);
+ long stopTime = System.currentTimeMillis();
+ logger.info(String.format("Indexer publishes message to Service Bus, messageId: %s | time taken to send message: %d milliseconds ", message.getMessageId(), stopTime - startTime));
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
diff --git a/provider/indexer-azure/src/main/resources/application.properties b/provider/indexer-azure/src/main/resources/application.properties
index 8f479afcd82f6acbe812168eae39a0980b076310..f86f5149ad97e503a195f750f45463ac1165d7e4 100644
--- a/provider/indexer-azure/src/main/resources/application.properties
+++ b/provider/indexer-azure/src/main/resources/application.properties
@@ -93,6 +93,9 @@ management.server.port=8081
management.health.azure-key-vault.enabled=false
management.health.elasticsearch.enabled=false
+management.endpoints.web.exposure.include=${web_exposure_endpoints:health,info}
+management.metrics.enable.all=${enable_metrics:false}
+
#Redis
-redis.database=${REDIS_DATABASE}
\ No newline at end of file
+redis.database=${REDIS_DATABASE}
diff --git a/provider/indexer-gcp/pom.xml b/provider/indexer-gcp/pom.xml
index 311aa52ab24e9f07619c5db489bf3d1d2a78785f..3dcb625578684f95a7a1fac3d206cecae0ff961f 100644
--- a/provider/indexer-gcp/pom.xml
+++ b/provider/indexer-gcp/pom.xml
@@ -5,12 +5,12 @@
<parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-service</artifactId>
- <version>0.14.0-SNAPSHOT</version>
+ <version>0.15.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>indexer-gcp</artifactId>
- <version>0.14.0-SNAPSHOT</version>
+ <version>0.15.0-SNAPSHOT</version>
<name>indexer-gcp</name>
<description>Indexer Service GCP App Engine</description>
<packaging>jar</packaging>
@@ -24,7 +24,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-core</artifactId>
- <version>0.14.0-SNAPSHOT</version>
+ <version>0.15.0-SNAPSHOT</version>
<!-- excluded due to runtime conflict with latest core-lib-gcp transient dependencies -->
<exclusions>
<exclusion>
@@ -87,7 +87,7 @@
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
- <version>2.26.0</version>
+ <version>3.12.0</version>
<scope>test</scope>
</dependency>
<dependency>
diff --git a/provider/indexer-ibm/pom.xml b/provider/indexer-ibm/pom.xml
index 378bf7dace7c028b53b55110d81fced5a63d35e3..290e5b73fdf51d32e797b102f336e0769fd8ad71 100644
--- a/provider/indexer-ibm/pom.xml
+++ b/provider/indexer-ibm/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-service</artifactId>
- <version>0.14.0-SNAPSHOT</version>
+ <version>0.15.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
@@ -53,7 +53,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-core</artifactId>
- <version>0.14.0-SNAPSHOT</version>
+ <version>0.15.0-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>io.netty</groupId>
@@ -161,7 +161,7 @@
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
- <version>3.0.0</version>
+ <version>3.12.0</version>
<scope>test</scope>
</dependency>
diff --git a/provider/indexer-reference/pom.xml b/provider/indexer-reference/pom.xml
index 3e1359fa94dae1c660471bfd797f0dc482d8b961..02a9682b4c283cea4b3e5f56a70fa27940e0ab64 100644
--- a/provider/indexer-reference/pom.xml
+++ b/provider/indexer-reference/pom.xml
@@ -22,12 +22,12 @@
<parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-service</artifactId>
- <version>0.14.0-SNAPSHOT</version>
+ <version>0.15.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>indexer-reference</artifactId>
- <version>0.14.0-SNAPSHOT</version>
+ <version>0.15.0-SNAPSHOT</version>
<name>indexer-reference</name>
<description>Indexer Service GCP Anthos</description>
<packaging>jar</packaging>
@@ -36,7 +36,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-core</artifactId>
- <version>0.14.0-SNAPSHOT</version>
+ <version>0.15.0-SNAPSHOT</version>
</dependency>
<dependency>
@@ -48,6 +48,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
+ <version>${os-core-common.version}</version>
</dependency>
<dependency>
@@ -130,7 +131,7 @@
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
- <version>2.26.0</version>
+ <version>3.12.0</version>
<scope>test</scope>
</dependency>
<dependency>
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryMongoDB.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryMongoDB.java
index cbbfe46ec103f9532c8e1fd41899352568b8a701..ea6b2da9de66747da3fa72b57d00ea424dac4669 100644
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryMongoDB.java
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryMongoDB.java
@@ -18,7 +18,7 @@
package org.opengroup.osdu.indexer.persistence;
import static com.mongodb.client.model.Filters.eq;
-import static com.mongodb.util.JSON.serialize;
+import static com.mongodb.BasicDBObject.parse;
import com.google.gson.Gson;
import com.mongodb.client.MongoCollection;
@@ -75,7 +75,7 @@ public class ElasticRepositoryMongoDB implements IElasticRepository {
SETTINGS_FETCH_ERROR_MESSAGE);
}
- ElasticSettingsDoc elasticSettingsDoc = new Gson().fromJson(serialize(record),
+ ElasticSettingsDoc elasticSettingsDoc = new Gson().fromJson(String.valueOf(parse(record.toString())),
ElasticSettingsDoc.class);
try {
diff --git a/testing/indexer-test-aws/pom.xml b/testing/indexer-test-aws/pom.xml
index 31999a464e57031f6e09660c5aa51f53de2fb5f2..a9f53143f673edbad265d9e44a732eba465fb675 100644
--- a/testing/indexer-test-aws/pom.xml
+++ b/testing/indexer-test-aws/pom.xml
@@ -21,13 +21,13 @@
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>indexer-test</artifactId>
- <version>0.14.0-SNAPSHOT</version>
+ <version>0.15.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-aws</artifactId>
- <version>0.14.0-SNAPSHOT</version>
+ <version>0.15.0-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
@@ -42,7 +42,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
- <version>0.14.0-SNAPSHOT</version>
+ <version>0.15.0-SNAPSHOT</version>
</dependency>
<!-- AWS specific packages -->
diff --git a/testing/indexer-test-azure/pom.xml b/testing/indexer-test-azure/pom.xml
index 50f217f25c093c524e7a0308997c9da2ffebc12b..44c73414f92ea8b7fadfc71054e26acbdd887d65 100644
--- a/testing/indexer-test-azure/pom.xml
+++ b/testing/indexer-test-azure/pom.xml
@@ -21,13 +21,13 @@
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>indexer-test</artifactId>
- <version>0.14.0-SNAPSHOT</version>
+ <version>0.15.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-azure</artifactId>
- <version>0.14.0-SNAPSHOT</version>
+ <version>0.15.0-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
@@ -36,7 +36,7 @@
<cucumber.version>1.2.5</cucumber.version>
<junit.jupiter.version>5.6.0</junit.jupiter.version>
<elasticsearch.version>7.8.1</elasticsearch.version>
- <os-core-common.version>0.14.0-rc4</os-core-common.version>
+ <os-core-common.version>0.14.0-rc8</os-core-common.version>
</properties>
<dependencies>
@@ -45,7 +45,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
- <version>0.14.0-SNAPSHOT</version>
+ <version>0.15.0-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
diff --git a/testing/indexer-test-core/pom.xml b/testing/indexer-test-core/pom.xml
index 28dec9cf0cae26b8fafcf4779856283652a04ce7..3a9036ac64f1c563a1b292eaf6d36cdaed8a09b4 100644
--- a/testing/indexer-test-core/pom.xml
+++ b/testing/indexer-test-core/pom.xml
@@ -5,19 +5,19 @@
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>indexer-test</artifactId>
- <version>0.14.0-SNAPSHOT</version>
+ <version>0.15.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
- <version>0.14.0-SNAPSHOT</version>
+ <version>0.15.0-SNAPSHOT</version>
<properties>
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
<cucumber.version>1.2.5</cucumber.version>
- <os-core-common.version>0.14.0-rc4</os-core-common.version>
+ <os-core-common.version>0.14.0-rc8</os-core-common.version>
</properties>
<dependencies>
diff --git a/testing/indexer-test-gcp/pom.xml b/testing/indexer-test-gcp/pom.xml
index c32245324cbf5bf9290df99dab3b0ff2886c3bce..32ea67ded9eaf7846ce0d91961bf3abe9b71a8bc 100644
--- a/testing/indexer-test-gcp/pom.xml
+++ b/testing/indexer-test-gcp/pom.xml
@@ -6,13 +6,13 @@
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>indexer-test</artifactId>
- <version>0.14.0-SNAPSHOT</version>
+ <version>0.15.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-gcp</artifactId>
- <version>0.14.0-SNAPSHOT</version>
+ <version>0.15.0-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
@@ -37,7 +37,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
- <version>0.14.0-SNAPSHOT</version>
+ <version>0.15.0-SNAPSHOT</version>
</dependency>
<!-- Cucumber -->
diff --git a/testing/indexer-test-ibm/pom.xml b/testing/indexer-test-ibm/pom.xml
index 26f5ddcdcbe3c314759875f93995117f288ae372..7ec6d7df6a4767c3c8c7a9304ca2e601839536fa 100644
--- a/testing/indexer-test-ibm/pom.xml
+++ b/testing/indexer-test-ibm/pom.xml
@@ -6,13 +6,13 @@
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>indexer-test</artifactId>
- <version>0.14.0-SNAPSHOT</version>
+ <version>0.15.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-ibm</artifactId>
- <version>0.14.0-SNAPSHOT</version>
+ <version>0.15.0-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
@@ -38,13 +38,13 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
- <version>0.14.0-SNAPSHOT</version>
+ <version>0.15.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
- <version>0.14.0-rc4</version>
+ <version>0.14.0-rc8</version>
</dependency>
<dependency>
diff --git a/testing/pom.xml b/testing/pom.xml
index 08218f0dd1511159b46ec16efe599ca5d66ca7e2..047e96b2b66e33a975bdfae33bc5cb0c4210cbb6 100644
--- a/testing/pom.xml
+++ b/testing/pom.xml
@@ -18,7 +18,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.opengroup.osdu</groupId>
<artifactId>indexer-test</artifactId>
- <version>0.14.0-SNAPSHOT</version>
+ <version>0.15.0-SNAPSHOT</version>
<description>Indexer Service Integration Test Root Project</description>
<properties>
<spring.version>5.1.19.RELEASE</spring.version>