rebase

devops/azure/chart/helm-config.yaml

@@ -19,4 +19,6 @@ policy:
   enabled: #{POLICY_ENABLED}#
 
 schemaEndpoints:
-  disabled: #{SCHEMA_ENDPOINTS_DISABLED}#
\ No newline at end of file
+  disabled: #{SCHEMA_ENDPOINTS_DISABLED}#
+
+istioDnsHost: #{ISTIO_DNS_HOST}# 
\ No newline at end of file

devops/azure/chart/templates/virtual-service.yaml

+---
+# Source: /devops/azure/chart/templates/virtual-service.yaml
+apiVersion: networking.istio.io/v1alpha3
+kind: VirtualService
+metadata:
+  name: {{ .Chart.Name }}
+  namespace: osdu
+spec:
+  hosts:
+  - "{{ .Values.istioDnsHost }}"
+  gateways:
+  - istio-gateway
+  http:  
+  - match:
+    - uri:
+        prefix: "/api/{{ .Chart.Name }}/v2"
+    route:
+    - destination:
+        host: {{ .Chart.Name }}
+        port:
+          number: 80
+    corsPolicy:
+      maxAge: "60m"
+      allowCredentials: true
+      allowHeaders:
+      - Authorization
+      - Data-Partition-Id
+      - Correlation-Id
+      - Content-Type
+      allowMethods:
+      - POST
+      - GET
+      - PUT
+      - PATCH
+      - DELETE
+      allowOrigins:
+      - prefix: "*" 
\ No newline at end of file

devops/azure/chart/values.yaml

@@ -26,4 +26,6 @@ policy:
   enabled: true 
 
 schemaEndpoints:
-  disabled: false
\ No newline at end of file
+  disabled: false
+
+istioDnsHost: "" 
\ No newline at end of file

devops/gcp/deploy/templates/storage-deploy.yml

@@ -9,7 +9,7 @@ spec:
   selector:
     matchLabels:
       app: "{{ .Values.conf.app_name }}"
-  replicas: 1
+  replicas: {{ .Values.conf.replicas }}
   template:
     metadata:
       labels:
@@ -36,4 +36,3 @@ spec:
             cpu: "{{ .Values.data.limits_cpu }}"
             memory: "{{ .Values.data.limits_memory }}"
       serviceAccountName: "{{ .Values.data.serviceAccountName }}"
-

devops/gcp/deploy/values.yaml

@@ -13,3 +13,4 @@ data:
 conf:
   configmap: "storage-config"
   app_name: "storage"
+  replicas: 3

provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/cache/LegalTagCache.java

@@ -23,7 +23,7 @@ import org.springframework.stereotype.Component;
 
 import javax.inject.Inject;
 
-@Component
+@Component("LegalTagCache")
 public class LegalTagCache implements ICache<String, String> {
 
     @Inject

provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/cache/LegalTagCache.java

@@ -30,7 +30,7 @@ import org.springframework.stereotype.Component;
 import javax.inject.Inject;
 import java.util.Map;
 
-@Component
+@Component("LegalTagCache")
 public class LegalTagCache implements ICache<String, String> {
     @Value("${aws.elasticache.cluster.endpoint:null}")
     String REDIS_SEARCH_HOST;

provider/storage-azure/pom.xml

@@ -33,7 +33,7 @@
         <azure.appservice.plan />
         <azure.appservice.appname />
         <azure.appservice.subscription />
-        <osdu.corelibazure.version>0.11.0-rc2</osdu.corelibazure.version>
+        <osdu.corelibazure.version>0.12.0-rc6</osdu.corelibazure.version>
         <osdu.storage-core.version>0.12.0-SNAPSHOT</osdu.storage-core.version>
         <junit.version>4.12</junit.version>
         <mockito.version>1.10.19</mockito.version>

provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/util/GlobalAzureExceptionMapper.java → provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/cache/CursorRedisCache.java

@@ -12,32 +12,25 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package org.opengroup.osdu.storage.provider.azure.util;
-
-import com.azure.cosmos.implementation.RequestRateTooLargeException;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.storage.util.GlobalExceptionMapper;
-import org.springframework.http.HttpStatus;
-import org.springframework.http.ResponseEntity;
-import org.springframework.web.bind.annotation.ControllerAdvice;
-import org.springframework.web.bind.annotation.ExceptionHandler;
-
-@ControllerAdvice
-public class GlobalAzureExceptionMapper {
-
-    private GlobalExceptionMapper mapper;
-
-    public GlobalAzureExceptionMapper(GlobalExceptionMapper mapper) {
-        this.mapper = mapper;
+package org.opengroup.osdu.storage.provider.azure.cache;
+
+import org.opengroup.osdu.core.common.cache.RedisCache;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Component;
+
+import javax.inject.Named;
+
+@Component("CursorCache")
+@ConditionalOnProperty(value = "runtime.env.local", havingValue = "false", matchIfMissing = true)
+public class CursorRedisCache extends RedisCache<String, String> {
+
+    public CursorRedisCache(
+            final @Named("REDIS_HOST") String host,
+            final @Named("REDIS_PORT") int port,
+            final @Named("REDIS_PASSWORD") String password,
+            final @Named("CURSOR_REDIS_TTL") int timeout,
+            @Value("${redis.database}") final int database) {
+        super(host, port, password, timeout, database, String.class, String.class);
     }
-
-    @ExceptionHandler(RequestRateTooLargeException.class)
-    protected ResponseEntity<Object> handleCosmosdbException(Exception e) {
-        return mapper.getErrorResponse(
-                new AppException(HttpStatus.SERVICE_UNAVAILABLE.value(), "Too many requests on cosmosdb.",
-                        "Request rate is large. Please retry this request later", e));
-    }
-
 }
-
-

provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/cache/CursorVmCache.java

+// Copyright © Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.storage.provider.azure.cache;
+
+import org.opengroup.osdu.core.common.cache.VmCache;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Component;
+
+@Component("CursorCache")
+@ConditionalOnProperty(value = "runtime.env.local", havingValue = "true")
+public class CursorVmCache extends VmCache<String, String> {
+
+    public CursorVmCache() {
+        super(600, 1000);
+    }
+}
\ No newline at end of file

provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/di/RedisConfig.java

@@ -20,6 +20,9 @@ public class RedisConfig {
     @Value("${redis.group.ttl:30}")
     public int groupRedisTtl;
 
+    @Value("${redis.cursor.ttl:90}")
+    public int cursorRedisTtl;
+
     @Bean
     @Named("REDIS_PORT")
     public int getRedisPort() {
@@ -36,6 +39,10 @@ public class RedisConfig {
     @Named("GROUP_REDIS_TTL")
     public int getGroupRedisTtl() { return groupRedisTtl; }
 
+    @Bean
+    @Named("CURSOR_REDIS_TTL")
+    public int getCursorRedisTtl() { return cursorRedisTtl; }
+
     @Bean
     @Named("REDIS_HOST")
     public String redisHost(SecretClient kv) {

provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/QueryRepository.java

@@ -15,16 +15,21 @@
 package org.opengroup.osdu.storage.provider.azure.repository;
 
 import com.azure.cosmos.CosmosException;
+import com.google.common.base.Strings;
+import com.lambdaworks.redis.RedisException;
 import org.apache.http.HttpStatus;
 import org.opengroup.osdu.azure.query.CosmosStorePageRequest;
+import org.opengroup.osdu.core.common.cache.ICache;
 import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
 import org.opengroup.osdu.core.common.model.http.AppException;
 import org.opengroup.osdu.core.common.model.storage.DatastoreQueryResult;
 import org.opengroup.osdu.core.common.model.storage.RecordState;
+import org.opengroup.osdu.core.common.util.Crc32c;
 import org.opengroup.osdu.storage.provider.azure.RecordMetadataDoc;
 import org.opengroup.osdu.storage.provider.azure.SchemaDoc;
 import org.opengroup.osdu.storage.provider.interfaces.IQueryRepository;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.Pageable;
 import org.springframework.data.domain.Sort;
@@ -46,6 +51,10 @@ public class QueryRepository implements IQueryRepository {
     @Autowired
     private JaxRsDpsLog logger;
 
+    @Autowired
+    @Qualifier("CursorCache")
+    private ICache<String, String> cursorCache;
+
     @Override
     public DatastoreQueryResult getAllKinds(Integer limit, String cursor) {
 
@@ -96,18 +105,26 @@ public class QueryRepository implements IQueryRepository {
     }
 
     @Override
-    public DatastoreQueryResult getAllRecordIdsFromKind(String kind, Integer limit, String cursor) {
+    public DatastoreQueryResult getAllRecordIdsFromKind(String kind, Integer limit, String hashedCursorKey) {
         Assert.notNull(kind, "kind must not be null");
 
-        boolean paginated  = false;
+        boolean paginated = false;
         int numRecords = PAGE_SIZE;
         if (limit != null) {
             numRecords = limit > 0 ? limit : PAGE_SIZE;
             paginated = true;
         }
 
-        if (cursor != null && !cursor.isEmpty()) {
+        String cursor = null;
+        if (hashedCursorKey != null && !hashedCursorKey.isEmpty()) {
             paginated = true;
+            try {
+                cursor = this.cursorCache.get(hashedCursorKey);
+            } catch (RedisException ex) {
+                this.logger.error(String.format("Error getting key %s from redis: %s", hashedCursorKey, ex.getMessage()), ex);
+            }
+
+            if (Strings.isNullOrEmpty(cursor)) throw this.getInvalidCursorException();
         }
         String status = RecordState.active.toString();
         Sort sort = Sort.by(Sort.Direction.ASC, "id");
@@ -124,7 +141,12 @@ public class QueryRepository implements IQueryRepository {
                 if (pageable instanceof CosmosStorePageRequest) {
                     continuation = ((CosmosStorePageRequest) pageable).getRequestContinuation();
                 }
-                dqr.setCursor(continuation);
+
+                if (!Strings.isNullOrEmpty(continuation)) {
+                    String hashedCursor = Crc32c.hashToBase64EncodedString(continuation);
+                    this.cursorCache.put(hashedCursor, continuation);
+                    dqr.setCursor(hashedCursor);
+                }
                 docs = docPage.getContent();
             } else {
                 docs = record.findByMetadata_kindAndMetadata_status(kind, status);

provider/storage-azure/src/test/java/org/opengroup/osdu/storage/provider/azure/util/GlobalAzureExceptionMapperTest.java

-// Copyright © Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.storage.provider.azure.util;
-
-import com.azure.cosmos.implementation.RequestRateTooLargeException;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.mockito.Mockito;
-import org.mockito.runners.MockitoJUnitRunner;
-import org.opengroup.osdu.core.common.model.http.AppError;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.storage.util.GlobalExceptionMapper;
-import org.springframework.http.ResponseEntity;
-
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.when;
-import static org.springframework.http.HttpStatus.SERVICE_UNAVAILABLE;
-
-@RunWith(MockitoJUnitRunner.class)
-public class GlobalAzureExceptionMapperTest {
-
-    @InjectMocks
-    private GlobalAzureExceptionMapper sut;
-
-    @Mock
-    private GlobalExceptionMapper mapper;
-
-    @Test
-    public void should_returnServiceUnavailable_with_correct_reason_when_RequestRateTooLargeException_Is_Captured() {
-        RequestRateTooLargeException exception = Mockito.mock(RequestRateTooLargeException.class);
-        AppError expectedBody = new AppError(SERVICE_UNAVAILABLE.value(), "Too many requests on cosmosdb.", "Cosmosdb error.");
-
-        when(mapper.getErrorResponse(any(AppException.class))).thenReturn(new ResponseEntity<>(expectedBody, SERVICE_UNAVAILABLE));
-
-        ResponseEntity response = this.sut.handleCosmosdbException(exception);
-        assertEquals(503, response.getStatusCodeValue());
-        assertEquals("Too many requests on cosmosdb.", ((AppError)response.getBody()).getReason());
-
-    }
-
-}
-

provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/GoogleCloudStorage.java

@@ -208,8 +208,15 @@ public class GoogleCloudStorage implements ICloudStorage {
 				// inconsistency then cleanup and check the access again
 				// This makes all the APIs robust to inconsistent data, but will add some
 				// latency
-				if (!this.hasAccessRobustToDataCorruption(bucket, record, this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName(), properties.isEnableImpersonalization()))) {
-					return false;
+				try {
+					if (!this.hasAccessRobustToDataCorruption(bucket, record,
+							this.storageFactory.getStorage(this.headers.getUserEmail(),
+									tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName(),
+									properties.isEnableImpersonalization()))) {
+						return false;
+					}
+				} catch (StorageException exception) {
+					throw new AppException(HttpStatus.SC_FORBIDDEN, ACCESS_DENIED_ERROR_REASON, ACCESS_DENIED_ERROR_MSG, e);
 				}
 			}
 		}

provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/cache/LegalTagCache.java

@@ -22,7 +22,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 
-@Component
+@Component("LegalTagCache")
 public class LegalTagCache implements ICache<String, String> {
 
 	@Autowired

storage-core/src/main/java/org/opengroup/osdu/storage/service/LegalServiceImpl.java

@@ -31,6 +31,7 @@ import org.opengroup.osdu.core.common.model.storage.Record;
 import org.opengroup.osdu.core.common.model.storage.RecordIdWithVersion;
 import org.opengroup.osdu.core.common.model.storage.RecordMetadata;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.stereotype.Service;
 
 import java.util.List;
@@ -46,6 +47,7 @@ public class LegalServiceImpl implements ILegalService {
     @Autowired
     private DpsHeaders headers;
     @Autowired
+    @Qualifier("LegalTagCache")
     private ICache<String, String> cache;
     @Autowired
     private ILegalFactory factory;

storage-core/src/main/java/org/opengroup/osdu/storage/util/GlobalExceptionMapper.java

@@ -19,6 +19,8 @@ import javax.validation.ValidationException;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException;
 import javassist.NotFoundException;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.Ordered;
@@ -35,6 +37,8 @@ import org.springframework.web.context.request.WebRequest;
 import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler;
 import org.opengroup.osdu.core.common.model.http.AppException;
 
+import java.io.IOException;
+
 @Order(Ordered.HIGHEST_PRECEDENCE)
 @ControllerAdvice
 public class GlobalExceptionMapper extends ResponseEntityExceptionHandler {
@@ -77,6 +81,17 @@ public class GlobalExceptionMapper extends ResponseEntityExceptionHandler {
                 new AppException(HttpStatus.FORBIDDEN.value(), "Access denied", e.getMessage(), e));
     }
 
+    @ExceptionHandler(IOException.class)
+    public ResponseEntity<Object> handleIOException(IOException e) {
+        if (StringUtils.containsIgnoreCase(ExceptionUtils.getRootCauseMessage(e), "Broken pipe")) {
+            this.logger.warning("Client closed the connection while request still being processed");
+            return null;
+        } else {
+            return this.getErrorResponse(
+                    new AppException(HttpStatus.SERVICE_UNAVAILABLE.value(), "Unknown error", e.getMessage(), e));
+        }
+    }
+
     @Override
     @NonNull
     protected ResponseEntity<Object> handleHttpRequestMethodNotSupported(@NonNull HttpRequestMethodNotSupportedException e,

storage-core/src/test/java/org/opengroup/osdu/storage/util/GlobalExceptionMapperTest.java

@@ -16,6 +16,7 @@ package org.opengroup.osdu.storage.util;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
 import static org.mockito.Mockito.verify;
 
 import javax.validation.ValidationException;
@@ -36,6 +37,8 @@ import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
 import org.springframework.http.ResponseEntity;
 import org.mockito.runners.MockitoJUnitRunner;
 
+import java.io.IOException;
+
 @RunWith(MockitoJUnitRunner.class)
 public class GlobalExceptionMapperTest {
 
@@ -109,4 +112,22 @@ public class GlobalExceptionMapperTest {
 		assertEquals(AppError.class, response.getBody().getClass());
 		assertEquals("Unrecognized property.", ((AppError)response.getBody()).getReason());
 	}
+
+	@Test
+	public void should_returnNullResponse_when_BrokenPipeIOExceptionIsCaptured() {
+		IOException ioException = new IOException("Broken pipe");
+
+		ResponseEntity response = this.sut.handleIOException(ioException);
+
+		assertNull(response);
+	}
+
+	@Test
+	public void should_returnServiceUnavailable_when_IOExceptionIsCaptured() {
+		IOException ioException = new IOException("Not broken yet");
+
+		ResponseEntity response = this.sut.handleIOException(ioException);
+
+		assertEquals(HttpStatus.SC_SERVICE_UNAVAILABLE, response.getStatusCodeValue());
+	}
 }
\ No newline at end of file