throw 429 response if server cannot process new cursor request any more

provider/search-aws/src/main/java/org/opengroup/osdu/search/provider/aws/provider/impl/ScrollQueryServiceAwsImpl.java

@@ -36,6 +36,8 @@ import org.opengroup.osdu.search.logging.AuditLogger;
 import org.opengroup.osdu.search.provider.interfaces.IScrollQueryService;
 import org.opengroup.osdu.search.util.ElasticClientHandler;
 import org.opengroup.osdu.search.util.QueryResponseUtil;
+import org.opengroup.osdu.search.util.ResponseExceptionParser;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
 import javax.inject.Inject;
@@ -63,6 +65,8 @@ public class ScrollQueryServiceAwsImpl extends QueryBase implements IScrollQuery
     private AuditLogger auditLogger;
     @Inject
     private QueryResponseUtil queryResponseUtil;
+    @Autowired
+    private ResponseExceptionParser exceptionParser;
 
     private final MessageDigest digest;
 
@@ -77,7 +81,14 @@ public class ScrollQueryServiceAwsImpl extends QueryBase implements IScrollQuery
 
         try (RestHighLevelClient client = this.elasticClientHandler.createRestClient()) {
             if (StringUtils.isEmpty(searchRequest.getCursor())) {
-                return this.initCursorQuery(searchRequest, client);
+                try {
+                    return this.initCursorQuery(searchRequest, client);
+                } catch (AppException e) {
+                    if (this.exceptionParser.parseException(e).stream().anyMatch(r -> r.contains("Trying to create too many scroll contexts. Must be less than or equal to:"))) {
+                        throw new AppException(429, "Too many request", "Too many cursor request, please re-try after some time.", e);
+                    }
+                    throw e;
+                }
             } else {
                 try {
                     CursorSettings cursorSettings = this.cursorCache.get(searchRequest.getCursor());

provider/search-azure/src/main/java/org/opengroup/osdu/search/provider/azure/provider/impl/ScrollQueryServiceImpl.java

@@ -37,6 +37,8 @@ import org.opengroup.osdu.search.logging.AuditLogger;
 import org.opengroup.osdu.search.provider.interfaces.IScrollQueryService;
 import org.opengroup.osdu.search.util.ElasticClientHandler;
 import org.opengroup.osdu.search.util.QueryResponseUtil;
+import org.opengroup.osdu.search.util.ResponseExceptionParser;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
 import javax.inject.Inject;
@@ -63,6 +65,8 @@ public class ScrollQueryServiceImpl extends QueryBase implements IScrollQuerySer
     private AuditLogger auditLogger;
     @Inject
     private QueryResponseUtil queryResponseUtil;
+    @Autowired
+    private ResponseExceptionParser exceptionParser;
 
     private final MessageDigest digest;
 
@@ -77,7 +81,14 @@ public class ScrollQueryServiceImpl extends QueryBase implements IScrollQuerySer
 
         try (RestHighLevelClient client = this.elasticClientHandler.createRestClient()) {
             if (Strings.isNullOrEmpty(searchRequest.getCursor())) {
-                return this.initCursorQuery(searchRequest, client);
+                try {
+                    return this.initCursorQuery(searchRequest, client);
+                } catch (AppException e) {
+                    if (this.exceptionParser.parseException(e).stream().anyMatch(r -> r.contains("Trying to create too many scroll contexts. Must be less than or equal to:"))) {
+                        throw new AppException(429, "Too many request", "Too many cursor request, please re-try after some time.", e);
+                    }
+                    throw e;
+                }
             } else {
                 try {
                     CursorSettings cursorSettings = this.cursorCache.get(searchRequest.getCursor());

provider/search-azure/src/test/java/org/opengroup/osdu/search/provider/azure/provider.impl/ScrollQueryServiceImplTest.java

@@ -41,6 +41,7 @@ import org.opengroup.osdu.search.provider.azure.config.ElasticLoggingConfig;
 import org.opengroup.osdu.search.provider.interfaces.IProviderHeaderService;
 import org.opengroup.osdu.search.util.CrossTenantUtils;
 import org.opengroup.osdu.search.util.ElasticClientHandler;
+import org.opengroup.osdu.search.util.ResponseExceptionParser;
 
 import java.util.*;
 
@@ -92,7 +93,8 @@ public class ScrollQueryServiceImplTest {
     private CrossTenantUtils crossTenantUtils;
     @Mock
     private ElasticLoggingConfig elasticLoggingConfig;
-
+    @Mock
+    private ResponseExceptionParser exceptionParser;
     @InjectMocks
     private ScrollQueryServiceImpl sut;
 

provider/search-byoc/src/main/java/org/opengroup/osdu/search/provider/byoc/provider/impl/ScrollQueryServiceImpl.java

@@ -39,6 +39,8 @@ import org.opengroup.osdu.core.common.model.search.Query;
 import org.opengroup.osdu.search.provider.interfaces.IScrollQueryService;
 import org.opengroup.osdu.search.util.CrossTenantUtils;
 import org.opengroup.osdu.search.util.QueryResponseUtil;
+import org.opengroup.osdu.search.util.ResponseExceptionParser;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
 import javax.inject.Inject;
@@ -70,6 +72,8 @@ public class ScrollQueryServiceImpl extends QueryBase implements IScrollQuerySer
 
     @Inject
     private QueryResponseUtil queryResponseUtil;
+    @Autowired
+    private ResponseExceptionParser exceptionParser;
 
     private final MessageDigest digest;
 
@@ -84,7 +88,14 @@ public class ScrollQueryServiceImpl extends QueryBase implements IScrollQuerySer
 
         try (RestHighLevelClient client = this.elasticClientHandler.createRestClient()) {
             if (Strings.isNullOrEmpty(searchRequest.getCursor())) {
-                return this.initCursorQuery(searchRequest, client);
+                try {
+                    return this.initCursorQuery(searchRequest, client);
+                } catch (AppException e) {
+                    if (this.exceptionParser.parseException(e).stream().anyMatch(r -> r.contains("Trying to create too many scroll contexts. Must be less than or equal to:"))) {
+                        throw new AppException(429, "Too many request", "Too many cursor request, please re-try after some time.", e);
+                    }
+                    throw e;
+                }
             } else {
                 try {
                     CursorSettings cursorSettings = null;//this.cursorCache.get(searchRequest.getCursor());

provider/search-gcp/src/main/java/org/opengroup/osdu/search/provider/gcp/provider/impl/ScrollQueryServiceImpl.java

@@ -37,6 +37,8 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.opengroup.osdu.search.util.QueryResponseUtil;
+import org.opengroup.osdu.search.util.ResponseExceptionParser;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
 import javax.inject.Inject;
@@ -63,6 +65,8 @@ public class ScrollQueryServiceImpl extends QueryBase implements IScrollQuerySer
     private AuditLogger auditLogger;
     @Inject
     private QueryResponseUtil queryResponseUtil;
+    @Autowired
+    private ResponseExceptionParser exceptionParser;
 
     private final MessageDigest digest;
 
@@ -77,7 +81,14 @@ public class ScrollQueryServiceImpl extends QueryBase implements IScrollQuerySer
 
         try (RestHighLevelClient client = this.elasticClientHandler.createRestClient()) {
             if (Strings.isNullOrEmpty(searchRequest.getCursor())) {
-                return this.initCursorQuery(searchRequest, client);
+                try {
+                    return this.initCursorQuery(searchRequest, client);
+                } catch (AppException e) {
+                    if (this.exceptionParser.parseException(e).stream().anyMatch(r -> r.contains("Trying to create too many scroll contexts. Must be less than or equal to:"))) {
+                        throw new AppException(429, "Too many request", "Too many cursor request, please re-try after some time.", e);
+                    }
+                    throw e;
+                }
             } else {
                 try {
                     CursorSettings cursorSettings = this.cursorCache.get(searchRequest.getCursor());

provider/search-ibm/src/main/java/org/opengroup/osdu/search/provider/ibm/provider/impl/ScrollQueryServiceImpl.java

@@ -35,6 +35,8 @@ import org.opengroup.osdu.search.logging.AuditLogger;
 import org.opengroup.osdu.search.provider.interfaces.IScrollQueryService;
 import org.opengroup.osdu.search.util.ElasticClientHandler;
 import org.opengroup.osdu.search.util.QueryResponseUtil;
+import org.opengroup.osdu.search.util.ResponseExceptionParser;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
 import com.google.common.base.Strings;
@@ -55,6 +57,8 @@ public class ScrollQueryServiceImpl extends QueryBase implements IScrollQuerySer
     private AuditLogger auditLogger;
     @Inject
     private QueryResponseUtil queryResponseUtil;
+    @Autowired
+    private ResponseExceptionParser exceptionParser;
 
     private final MessageDigest digest;
 
@@ -71,7 +75,14 @@ public class ScrollQueryServiceImpl extends QueryBase implements IScrollQuerySer
 
         try (RestHighLevelClient client = this.elasticClientHandler.createRestClient()) {
             if (Strings.isNullOrEmpty(searchRequest.getCursor())) {
-                return this.initCursorQuery(searchRequest, client);
+                try {
+                    return this.initCursorQuery(searchRequest, client);
+                } catch (AppException e) {
+                    if (this.exceptionParser.parseException(e).stream().anyMatch(r -> r.contains("Trying to create too many scroll contexts. Must be less than or equal to:"))) {
+                        throw new AppException(429, "Too many request", "Too many cursor request, please re-try after some time.", e);
+                    }
+                    throw e;
+                }
             } else {
                 try {
                     CursorSettings cursorSettings = this.cursorCache.get(searchRequest.getCursor());

provider/search-reference/src/main/java/org/opengroup/osdu/search/provider/reference/provider/impl/ScrollQueryServiceImpl.java

@@ -44,6 +44,8 @@ import org.opengroup.osdu.search.logging.AuditLogger;
 import org.opengroup.osdu.search.provider.interfaces.IScrollQueryService;
 import org.opengroup.osdu.search.util.ElasticClientHandler;
 import org.opengroup.osdu.search.util.QueryResponseUtil;
+import org.opengroup.osdu.search.util.ResponseExceptionParser;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
 @Service
@@ -59,6 +61,8 @@ public class ScrollQueryServiceImpl extends QueryBase implements IScrollQuerySer
     private AuditLogger auditLogger;
     @Inject
     private QueryResponseUtil queryResponseUtil;
+    @Autowired
+    private ResponseExceptionParser exceptionParser;
 
     private final MessageDigest digest;
 
@@ -73,7 +77,14 @@ public class ScrollQueryServiceImpl extends QueryBase implements IScrollQuerySer
 
         try (RestHighLevelClient client = this.elasticClientHandler.createRestClient()) {
             if (searchRequest.getCursor() == null || searchRequest.getCursor().isEmpty()) {
-                return this.initCursorQuery(searchRequest, client);
+                try {
+                    return this.initCursorQuery(searchRequest, client);
+                } catch (AppException e) {
+                    if (this.exceptionParser.parseException(e).stream().anyMatch(r -> r.contains("Trying to create too many scroll contexts. Must be less than or equal to:"))) {
+                        throw new AppException(429, "Too many request", "Too many cursor request, please re-try after some time.", e);
+                    }
+                    throw e;
+                }
             } else {
                 try {
                     CursorSettings cursorSettings = this.cursorCache.get(searchRequest.getCursor());

search-core/src/main/java/org/opengroup/osdu/search/middleware/GlobalExceptionMapper.java

@@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import javassist.NotFoundException;
+import org.elasticsearch.client.ResponseException;
 import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
 import org.opengroup.osdu.core.common.model.http.AppException;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -42,7 +43,6 @@ import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExcep
 import javax.validation.ValidationException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Objects;
 
 @Order(Ordered.HIGHEST_PRECEDENCE)
 @ControllerAdvice
@@ -127,6 +127,9 @@ public class GlobalExceptionMapper extends ResponseEntityExceptionHandler {
             this.logger.warning(exceptionMsg, e);
         }
 
+        // log suppressed exception from Elastic's ResponseException if any
+        this.logSuppressedElasticException(e);
+
         // Support for non standard HttpStatus Codes
         HttpStatus httpStatus = HttpStatus.resolve(e.getError().getCode());
         if (httpStatus == null) {
@@ -151,4 +154,14 @@ public class GlobalExceptionMapper extends ResponseEntityExceptionHandler {
         }
         return node;
     }
+
+    private void logSuppressedElasticException(AppException e) {
+        Exception cause = e.getOriginalException();
+        if (cause != null && cause.getSuppressed() != null) {
+            for (Throwable t : cause.getSuppressed()) {
+                if (t instanceof ResponseException) this.logger.error(t.getMessage(), (ResponseException) t);
+            }
+        }
+    }
 }
+

search-core/src/main/java/org/opengroup/osdu/search/util/ResponseExceptionParser.java

+// Copyright © Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.search.util;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import org.elasticsearch.client.ResponseException;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+@Component
+public class ResponseExceptionParser {
+
+    @Autowired
+    private JaxRsDpsLog log;
+
+    private ObjectMapper objectMapper = new ObjectMapper();
+
+    public List<String> parseException(AppException e) {
+        Exception cause = e.getOriginalException();
+        if (cause == null || cause.getSuppressed() == null) {
+            return new ArrayList<>();
+        }
+
+        for (Throwable t : cause.getSuppressed()) {
+            if (!(t instanceof ResponseException)) continue;
+
+            ResponseException responseException = (ResponseException) t;
+            try {
+                JsonNode exceptionNode = objectMapper.readTree(
+                        responseException.getResponse().getEntity().getContent());
+                Optional<JsonNode> rootCause = Optional.ofNullable(exceptionNode.get("error"))
+                        .map(errorNode -> errorNode.get("root_cause"));
+
+                return getReason(rootCause);
+            } catch (IOException ioe) {
+                this.log.error("Unable to parse response exception content", ioe);
+            }
+        }
+        return new ArrayList<>();
+    }
+
+    private List<String> getReason(Optional<JsonNode> rootCause) {
+        List<String> out = new ArrayList<>();
+        if (rootCause.isPresent() && rootCause.get().isArray()) {
+            ArrayNode arrayNode = (ArrayNode) rootCause.get();
+            for (JsonNode c : arrayNode) out.add(c.get("reason").textValue());
+        }
+        return out;
+    }
+}
\ No newline at end of file