Commit 03c1d31a authored by Spencer Sutton's avatar Spencer Sutton
Browse files

merge

parents 4c4299e3 73d23837
......@@ -17,7 +17,7 @@ variables:
# --- osdu gcp specific variables for cloudrun ---
OSDU_GCP_SERVICE: search
OSDU_GCP_VENDOR: gcp
OSDU_GCP_ENV_VARS: ENVIRONMENT=$OSDU_GCP_SPRING_PROFILES_ACTIVE,GOOGLE_CLOUD_PROJECT=$OSDU_GCP_PROJECT,REDIS_SEARCH_HOST=$REDIS_SEARCH_HOST,REDIS_SEARCH_PORT=$OSDU_GCP_REDIS_SEARCH_PORT,REDIS_GROUP_HOST=$REDIS_GROUP_HOST,INDEXER_HOST=$OSDU_GCP_INDEXER_HOST_SEARCH,AUTHORIZE_API=$OSDU_GCP_AUTHORIZE_API,ENTITLEMENTS_HOST=$OSDU_GCP_ENTITLEMENTS_URL,SECURITY_HTTPS_CERTIFICATE_TRUST=$OSDU_SECURITY_HTTPS_CERTIFICATE_TRUST,GOOGLE_AUDIENCES=$GOOGLE_AUDIENCE,PARTITION_API=$OSDU_GCP_PARTITION_API,POLICY_API=$OSDU_GCP_POLICY_API,POLICY_ID=search --vpc-connector=$OSDU_GCP_VPC_CONNECTOR
OSDU_GCP_ENV_VARS: ENVIRONMENT=$OSDU_GCP_SPRING_PROFILES_ACTIVE,GOOGLE_CLOUD_PROJECT=$OSDU_GCP_PROJECT,REDIS_SEARCH_HOST=$REDIS_SEARCH_HOST,REDIS_SEARCH_PORT=$OSDU_GCP_REDIS_SEARCH_PORT,REDIS_GROUP_HOST=$REDIS_GROUP_HOST,INDEXER_HOST=$OSDU_GCP_INDEXER_HOST_SEARCH,AUTHORIZE_API=$OSDU_GCP_ENTITLEMENTS_V2_URL,ENTITLEMENTS_HOST=$OSDU_GCP_ENTITLEMENTS_V2_URL,SECURITY_HTTPS_CERTIFICATE_TRUST=$OSDU_SECURITY_HTTPS_CERTIFICATE_TRUST,GOOGLE_AUDIENCES=$GOOGLE_AUDIENCE,PARTITION_API=$OSDU_GCP_PARTITION_API,POLICY_API=$OSDU_GCP_POLICY_API,POLICY_ID=search --vpc-connector=$OSDU_GCP_VPC_CONNECTOR
# Variables should be partially removed after migration on cloudrun gcp
OSDU_GCP_BUILD_SUBDIR: provider/search-gcp
OSDU_GCP_INT_TEST_SUBDIR: testing/integration-tests/search-test-gcp
......
This diff is collapsed.
......@@ -14,6 +14,8 @@ spec:
metadata:
labels:
app: "{{ .Values.conf.app_name }}"
annotations:
rollme: {{ randAlphaNum 5 | quote }}
spec:
containers:
- name: "{{ .Values.conf.app_name }}"
......
......@@ -18,7 +18,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-search</artifactId>
<version>0.10.0-SNAPSHOT</version>
<version>0.11.0-SNAPSHOT</version>
<description>Root Search Service Project</description>
<packaging>pom</packaging>
......@@ -44,7 +44,7 @@
<log4j-core.version>2.13.2</log4j-core.version>
<google-oauth-client.version>1.31.0</google-oauth-client.version>
<commons-compress.version>1.20</commons-compress.version>
<osdu.oscorecommon.version>0.9.0</osdu.oscorecommon.version>
<osdu.oscorecommon.version>0.10.0</osdu.oscorecommon.version>
<tomcat-embed-core.version>9.0.45</tomcat-embed-core.version>
</properties>
......
......@@ -19,7 +19,7 @@
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-search</artifactId>
<version>0.10.0-SNAPSHOT</version>
<version>0.11.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
......@@ -27,7 +27,7 @@
<artifactId>search-aws</artifactId>
<description>Search Service on AWS</description>
<packaging>jar</packaging>
<version>0.10.0-SNAPSHOT</version>
<version>0.11.0-SNAPSHOT</version>
<properties>
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
......@@ -39,7 +39,7 @@
<aws.version>1.11.637</aws.version>
<deployment.environment>dev</deployment.environment>
<version.number>0.0.4-SNAPSHOT</version.number>
<osdu.oscorecommon.version>0.9.0</osdu.oscorecommon.version>
<osdu.oscorecommon.version>0.10.0</osdu.oscorecommon.version>
</properties>
<dependencies>
......@@ -47,7 +47,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>search-core</artifactId>
<version>0.10.0-SNAPSHOT</version>
<version>0.11.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
......
......@@ -36,6 +36,8 @@ import org.opengroup.osdu.search.logging.AuditLogger;
import org.opengroup.osdu.search.provider.interfaces.IScrollQueryService;
import org.opengroup.osdu.search.util.ElasticClientHandler;
import org.opengroup.osdu.search.util.QueryResponseUtil;
import org.opengroup.osdu.search.util.ResponseExceptionParser;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.inject.Inject;
......@@ -63,6 +65,8 @@ public class ScrollQueryServiceAwsImpl extends QueryBase implements IScrollQuery
private AuditLogger auditLogger;
@Inject
private QueryResponseUtil queryResponseUtil;
@Autowired
private ResponseExceptionParser exceptionParser;
private final MessageDigest digest;
......@@ -77,7 +81,14 @@ public class ScrollQueryServiceAwsImpl extends QueryBase implements IScrollQuery
try (RestHighLevelClient client = this.elasticClientHandler.createRestClient()) {
if (StringUtils.isEmpty(searchRequest.getCursor())) {
return this.initCursorQuery(searchRequest, client);
try {
return this.initCursorQuery(searchRequest, client);
} catch (AppException e) {
if (this.exceptionParser.parseException(e).stream().anyMatch(r -> r.contains("Trying to create too many scroll contexts. Must be less than or equal to:"))) {
throw new AppException(429, "Too many requests", "Too many cursor requests, please re-try after some time.", e);
}
throw e;
}
} else {
try {
CursorSettings cursorSettings = this.cursorCache.get(searchRequest.getCursor());
......
......@@ -20,13 +20,13 @@
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-search</artifactId>
<version>0.10.0-SNAPSHOT</version>
<version>0.11.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<packaging>jar</packaging>
<artifactId>search-azure</artifactId>
<version>0.10.0-SNAPSHOT</version>
<version>0.11.0-SNAPSHOT</version>
<properties>
<maven.compiler.target>1.8</maven.compiler.target>
......@@ -36,9 +36,9 @@
<failOnMissingWebXml>false</failOnMissingWebXml>
<project.main.basedir>${project.parent.basedir}</project.main.basedir>
<springboot.version>2.1.7.RELEASE</springboot.version>
<osdu.corelibazure.version>0.9.0</osdu.corelibazure.version>
<osdu.oscorecommon.version>0.9.0</osdu.oscorecommon.version>
<osdu.search-core.version>0.10.0-SNAPSHOT</osdu.search-core.version>
<osdu.corelibazure.version>0.10.1</osdu.corelibazure.version>
<osdu.oscorecommon.version>0.10.0</osdu.oscorecommon.version>
<osdu.search-core.version>0.11.0-SNAPSHOT</osdu.search-core.version>
<spatial4j.version>0.7</spatial4j.version>
<jts-io-common.version>1.15.0</jts-io-common.version>
<reactor-netty.version>1.0.7</reactor-netty.version>
......@@ -246,4 +246,4 @@
</plugin>
</plugins>
</build>
</project>
\ No newline at end of file
</project>
......@@ -37,6 +37,8 @@ import org.opengroup.osdu.search.logging.AuditLogger;
import org.opengroup.osdu.search.provider.interfaces.IScrollQueryService;
import org.opengroup.osdu.search.util.ElasticClientHandler;
import org.opengroup.osdu.search.util.QueryResponseUtil;
import org.opengroup.osdu.search.util.ResponseExceptionParser;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.inject.Inject;
......@@ -63,6 +65,8 @@ public class ScrollQueryServiceImpl extends QueryBase implements IScrollQuerySer
private AuditLogger auditLogger;
@Inject
private QueryResponseUtil queryResponseUtil;
@Autowired
private ResponseExceptionParser exceptionParser;
private final MessageDigest digest;
......@@ -77,7 +81,14 @@ public class ScrollQueryServiceImpl extends QueryBase implements IScrollQuerySer
try (RestHighLevelClient client = this.elasticClientHandler.createRestClient()) {
if (Strings.isNullOrEmpty(searchRequest.getCursor())) {
return this.initCursorQuery(searchRequest, client);
try {
return this.initCursorQuery(searchRequest, client);
} catch (AppException e) {
if (this.exceptionParser.parseException(e).stream().anyMatch(r -> r.contains("Trying to create too many scroll contexts. Must be less than or equal to:"))) {
throw new AppException(429, "Too many requests", "Too many cursor requests, please re-try after some time.", e);
}
throw e;
}
} else {
try {
CursorSettings cursorSettings = this.cursorCache.get(searchRequest.getCursor());
......
......@@ -41,6 +41,7 @@ import org.opengroup.osdu.search.provider.azure.config.ElasticLoggingConfig;
import org.opengroup.osdu.search.provider.interfaces.IProviderHeaderService;
import org.opengroup.osdu.search.util.CrossTenantUtils;
import org.opengroup.osdu.search.util.ElasticClientHandler;
import org.opengroup.osdu.search.util.ResponseExceptionParser;
import java.util.*;
......@@ -92,7 +93,8 @@ public class ScrollQueryServiceImplTest {
private CrossTenantUtils crossTenantUtils;
@Mock
private ElasticLoggingConfig elasticLoggingConfig;
@Mock
private ResponseExceptionParser exceptionParser;
@InjectMocks
private ScrollQueryServiceImpl sut;
......
......@@ -18,14 +18,14 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.opengroup.osdu</groupId>
<artifactId>search-byoc</artifactId>
<version>0.10.0-SNAPSHOT</version>
<version>0.11.0-SNAPSHOT</version>
<description>In memory implementation of Search service APIs</description>
<packaging>jar</packaging>
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-search</artifactId>
<version>0.10.0-SNAPSHOT</version>
<version>0.11.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
......@@ -43,12 +43,12 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>search-core</artifactId>
<version>0.10.0-SNAPSHOT</version>
<version>0.11.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
<version>0.9.0</version>
<version>0.10.0</version>
</dependency>
<dependency>
......
......@@ -39,6 +39,8 @@ import org.opengroup.osdu.core.common.model.search.Query;
import org.opengroup.osdu.search.provider.interfaces.IScrollQueryService;
import org.opengroup.osdu.search.util.CrossTenantUtils;
import org.opengroup.osdu.search.util.QueryResponseUtil;
import org.opengroup.osdu.search.util.ResponseExceptionParser;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.inject.Inject;
......@@ -70,6 +72,8 @@ public class ScrollQueryServiceImpl extends QueryBase implements IScrollQuerySer
@Inject
private QueryResponseUtil queryResponseUtil;
@Autowired
private ResponseExceptionParser exceptionParser;
private final MessageDigest digest;
......@@ -84,7 +88,14 @@ public class ScrollQueryServiceImpl extends QueryBase implements IScrollQuerySer
try (RestHighLevelClient client = this.elasticClientHandler.createRestClient()) {
if (Strings.isNullOrEmpty(searchRequest.getCursor())) {
return this.initCursorQuery(searchRequest, client);
try {
return this.initCursorQuery(searchRequest, client);
} catch (AppException e) {
if (this.exceptionParser.parseException(e).stream().anyMatch(r -> r.contains("Trying to create too many scroll contexts. Must be less than or equal to:"))) {
throw new AppException(429, "Too many requests", "Too many cursor requests, please re-try after some time.", e);
}
throw e;
}
} else {
try {
CursorSettings cursorSettings = null;//this.cursorCache.get(searchRequest.getCursor());
......
......@@ -18,14 +18,14 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.opengroup.osdu</groupId>
<artifactId>search-gcp</artifactId>
<version>0.10.0-SNAPSHOT</version>
<version>0.11.0-SNAPSHOT</version>
<description>Google cloud implementation of Search service APIs</description>
<packaging>jar</packaging>
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-search</artifactId>
<version>0.10.0-SNAPSHOT</version>
<version>0.11.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
......@@ -44,17 +44,17 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>search-core</artifactId>
<version>0.10.0-SNAPSHOT</version>
<version>0.11.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-gcp</artifactId>
<version>0.10.0-SNAPSHOT</version>
<version>0.10.0</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
<version>0.10.0-SNAPSHOT</version>
<version>0.10.0</version>
</dependency>
<dependency>
......
......@@ -37,6 +37,8 @@ import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.search.Scroll;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.opengroup.osdu.search.util.QueryResponseUtil;
import org.opengroup.osdu.search.util.ResponseExceptionParser;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.inject.Inject;
......@@ -63,6 +65,8 @@ public class ScrollQueryServiceImpl extends QueryBase implements IScrollQuerySer
private AuditLogger auditLogger;
@Inject
private QueryResponseUtil queryResponseUtil;
@Autowired
private ResponseExceptionParser exceptionParser;
private final MessageDigest digest;
......@@ -77,7 +81,14 @@ public class ScrollQueryServiceImpl extends QueryBase implements IScrollQuerySer
try (RestHighLevelClient client = this.elasticClientHandler.createRestClient()) {
if (Strings.isNullOrEmpty(searchRequest.getCursor())) {
return this.initCursorQuery(searchRequest, client);
try {
return this.initCursorQuery(searchRequest, client);
} catch (AppException e) {
if (this.exceptionParser.parseException(e).stream().anyMatch(r -> r.contains("Trying to create too many scroll contexts. Must be less than or equal to:"))) {
throw new AppException(429, "Too many requests", "Too many cursor requests, please re-try after some time.", e);
}
throw e;
}
} else {
try {
CursorSettings cursorSettings = this.cursorCache.get(searchRequest.getCursor());
......
......@@ -13,7 +13,7 @@
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-search</artifactId>
<version>0.10.0-SNAPSHOT</version>
<version>0.11.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
......@@ -23,9 +23,9 @@
<packaging>jar</packaging>
<properties>
<os-core-lib-ibm.version>0.10.0-SNAPSHOT</os-core-lib-ibm.version>
<os-core-lib-ibm.version>0.10.0</os-core-lib-ibm.version>
<start-class>org.opengroup.osdu.search.provider.ibm.app.SearchIBMApplication</start-class>
<osdu.oscorecommon.version>0.9.0</osdu.oscorecommon.version>
<osdu.oscorecommon.version>0.10.0</osdu.oscorecommon.version>
</properties>
......@@ -36,7 +36,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>search-core</artifactId>
<version>0.10.0-SNAPSHOT</version>
<version>0.11.0-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>com.nimbusds</groupId>
......
......@@ -35,6 +35,8 @@ import org.opengroup.osdu.search.logging.AuditLogger;
import org.opengroup.osdu.search.provider.interfaces.IScrollQueryService;
import org.opengroup.osdu.search.util.ElasticClientHandler;
import org.opengroup.osdu.search.util.QueryResponseUtil;
import org.opengroup.osdu.search.util.ResponseExceptionParser;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.google.common.base.Strings;
......@@ -55,6 +57,8 @@ public class ScrollQueryServiceImpl extends QueryBase implements IScrollQuerySer
private AuditLogger auditLogger;
@Inject
private QueryResponseUtil queryResponseUtil;
@Autowired
private ResponseExceptionParser exceptionParser;
private final MessageDigest digest;
......@@ -71,7 +75,14 @@ public class ScrollQueryServiceImpl extends QueryBase implements IScrollQuerySer
try (RestHighLevelClient client = this.elasticClientHandler.createRestClient()) {
if (Strings.isNullOrEmpty(searchRequest.getCursor())) {
return this.initCursorQuery(searchRequest, client);
try {
return this.initCursorQuery(searchRequest, client);
} catch (AppException e) {
if (this.exceptionParser.parseException(e).stream().anyMatch(r -> r.contains("Trying to create too many scroll contexts. Must be less than or equal to:"))) {
throw new AppException(429, "Too many requests", "Too many cursor requests, please re-try after some time.", e);
}
throw e;
}
} else {
try {
CursorSettings cursorSettings = this.cursorCache.get(searchRequest.getCursor());
......
......@@ -4,13 +4,13 @@
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-search</artifactId>
<version>0.10.0-SNAPSHOT</version>
<version>0.11.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>search-reference</artifactId>
<version>0.10.0-SNAPSHOT</version>
<version>0.11.0-SNAPSHOT</version>
<description>MongoDB implementation of Search service APIs</description>
<packaging>jar</packaging>
......@@ -28,12 +28,12 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>search-core</artifactId>
<version>0.10.0-SNAPSHOT</version>
<version>0.11.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
<version>0.9.0</version>
<version>0.10.0</version>
</dependency>
<dependency>
......
......@@ -44,6 +44,8 @@ import org.opengroup.osdu.search.logging.AuditLogger;
import org.opengroup.osdu.search.provider.interfaces.IScrollQueryService;
import org.opengroup.osdu.search.util.ElasticClientHandler;
import org.opengroup.osdu.search.util.QueryResponseUtil;
import org.opengroup.osdu.search.util.ResponseExceptionParser;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@Service
......@@ -59,6 +61,8 @@ public class ScrollQueryServiceImpl extends QueryBase implements IScrollQuerySer
private AuditLogger auditLogger;
@Inject
private QueryResponseUtil queryResponseUtil;
@Autowired
private ResponseExceptionParser exceptionParser;
private final MessageDigest digest;
......@@ -73,7 +77,14 @@ public class ScrollQueryServiceImpl extends QueryBase implements IScrollQuerySer
try (RestHighLevelClient client = this.elasticClientHandler.createRestClient()) {
if (searchRequest.getCursor() == null || searchRequest.getCursor().isEmpty()) {
return this.initCursorQuery(searchRequest, client);
try {
return this.initCursorQuery(searchRequest, client);
} catch (AppException e) {
if (this.exceptionParser.parseException(e).stream().anyMatch(r -> r.contains("Trying to create too many scroll contexts. Must be less than or equal to:"))) {
throw new AppException(429, "Too many requests", "Too many cursor requests, please re-try after some time.", e);
}
throw e;
}
} else {
try {
CursorSettings cursorSettings = this.cursorCache.get(searchRequest.getCursor());
......
......@@ -18,14 +18,14 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.opengroup.osdu</groupId>
<artifactId>search-core</artifactId>
<version>0.10.0-SNAPSHOT</version>
<version>0.11.0-SNAPSHOT</version>
<description>Core project for the Search service</description>
<packaging>jar</packaging>
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-search</artifactId>
<version>0.10.0-SNAPSHOT</version>
<version>0.11.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
......
......@@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import javassist.NotFoundException;
import org.elasticsearch.client.ResponseException;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.springframework.beans.factory.annotation.Autowired;
......@@ -42,7 +43,6 @@ import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExcep
import javax.validation.ValidationException;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
@Order(Ordered.HIGHEST_PRECEDENCE)
@ControllerAdvice
......@@ -127,6 +127,9 @@ public class GlobalExceptionMapper extends ResponseEntityExceptionHandler {
this.logger.warning(exceptionMsg, e);
}
// log suppressed exception from Elastic's ResponseException if any
this.logSuppressedElasticException(e);
// Support for non standard HttpStatus Codes
HttpStatus httpStatus = HttpStatus.resolve(e.getError().getCode());
if (httpStatus == null) {
......@@ -151,4 +154,14 @@ public class GlobalExceptionMapper extends ResponseEntityExceptionHandler {
}
return node;
}
private void logSuppressedElasticException(AppException e) {
Exception cause = e.getOriginalException();
if (cause != null && cause.getSuppressed() != null) {
for (Throwable t : cause.getSuppressed()) {
if (t instanceof ResponseException) this.logger.error(t.getMessage(), (ResponseException) t);
}
}
}
}
......@@ -149,7 +149,7 @@ public class EntitlementsAndCacheServiceImpl implements IEntitlementsExtensionSe
try {
groups = service.getGroups();
this.cache.put(cacheKey, groups);
this.logger.info("Entitlements cache miss");
this.logger.debug("Entitlements cache miss");
} catch (EntitlementsException e) {
e.printStackTrace();
HttpResponse response = e.getHttpResponse();
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment