rebase

provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/cache/LegalTagCache.java

@@ -23,7 +23,7 @@ import org.springframework.stereotype.Component;
 
 import javax.inject.Inject;
 
-@Component
+@Component("LegalTagCache")
 public class LegalTagCache implements ICache<String, String> {
 
     @Inject

provider/storage-aws/pom.xml

@@ -58,7 +58,7 @@
         <dependency>
             <groupId>org.opengroup.osdu.core.aws</groupId>
             <artifactId>os-core-lib-aws</artifactId>
-            <version>0.11.0-SNAPSHOT</version>
+            <version>0.11.0</version>
         </dependency>
         <dependency>
             <groupId>org.opengroup.osdu</groupId>

provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/cache/LegalTagCache.java

@@ -30,7 +30,7 @@ import org.springframework.stereotype.Component;
 import javax.inject.Inject;
 import java.util.Map;
 
-@Component
+@Component("LegalTagCache")
 public class LegalTagCache implements ICache<String, String> {
     @Value("${aws.elasticache.cluster.endpoint:null}")
     String REDIS_SEARCH_HOST;

provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/cache/CursorRedisCache.java

+// Copyright © Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.storage.provider.azure.cache;
+
+import org.opengroup.osdu.core.common.cache.RedisCache;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Component;
+
+import javax.inject.Named;
+
+@Component("CursorCache")
+@ConditionalOnProperty(value = "runtime.env.local", havingValue = "false", matchIfMissing = true)
+public class CursorRedisCache extends RedisCache<String, String> {
+
+    public CursorRedisCache(
+            final @Named("REDIS_HOST") String host,
+            final @Named("REDIS_PORT") int port,
+            final @Named("REDIS_PASSWORD") String password,
+            final @Named("CURSOR_REDIS_TTL") int timeout,
+            @Value("${redis.database}") final int database) {
+        super(host, port, password, timeout, database, String.class, String.class);
+    }
+}

provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/cache/CursorVmCache.java

+// Copyright © Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.storage.provider.azure.cache;
+
+import org.opengroup.osdu.core.common.cache.VmCache;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Component;
+
+@Component("CursorCache")
+@ConditionalOnProperty(value = "runtime.env.local", havingValue = "true")
+public class CursorVmCache extends VmCache<String, String> {
+
+    public CursorVmCache() {
+        super(600, 1000);
+    }
+}
\ No newline at end of file

provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/di/RedisConfig.java

@@ -20,6 +20,9 @@ public class RedisConfig {
     @Value("${redis.group.ttl:30}")
     public int groupRedisTtl;
 
+    @Value("${redis.cursor.ttl:90}")
+    public int cursorRedisTtl;
+
     @Bean
     @Named("REDIS_PORT")
     public int getRedisPort() {
@@ -36,6 +39,10 @@ public class RedisConfig {
     @Named("GROUP_REDIS_TTL")
     public int getGroupRedisTtl() { return groupRedisTtl; }
 
+    @Bean
+    @Named("CURSOR_REDIS_TTL")
+    public int getCursorRedisTtl() { return cursorRedisTtl; }
+
     @Bean
     @Named("REDIS_HOST")
     public String redisHost(SecretClient kv) {

provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/QueryRepository.java

@@ -15,16 +15,21 @@
 package org.opengroup.osdu.storage.provider.azure.repository;
 
 import com.azure.cosmos.CosmosException;
+import com.google.common.base.Strings;
+import com.lambdaworks.redis.RedisException;
 import org.apache.http.HttpStatus;
 import org.opengroup.osdu.azure.query.CosmosStorePageRequest;
+import org.opengroup.osdu.core.common.cache.ICache;
 import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
 import org.opengroup.osdu.core.common.model.http.AppException;
 import org.opengroup.osdu.core.common.model.storage.DatastoreQueryResult;
 import org.opengroup.osdu.core.common.model.storage.RecordState;
+import org.opengroup.osdu.core.common.util.Crc32c;
 import org.opengroup.osdu.storage.provider.azure.RecordMetadataDoc;
 import org.opengroup.osdu.storage.provider.azure.SchemaDoc;
 import org.opengroup.osdu.storage.provider.interfaces.IQueryRepository;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.Pageable;
 import org.springframework.data.domain.Sort;
@@ -46,6 +51,10 @@ public class QueryRepository implements IQueryRepository {
     @Autowired
     private JaxRsDpsLog logger;
 
+    @Autowired
+    @Qualifier("CursorCache")
+    private ICache<String, String> cursorCache;
+
     @Override
     public DatastoreQueryResult getAllKinds(Integer limit, String cursor) {
 
@@ -96,18 +105,26 @@ public class QueryRepository implements IQueryRepository {
     }
 
     @Override
-    public DatastoreQueryResult getAllRecordIdsFromKind(String kind, Integer limit, String cursor) {
+    public DatastoreQueryResult getAllRecordIdsFromKind(String kind, Integer limit, String hashedCursorKey) {
         Assert.notNull(kind, "kind must not be null");
 
-        boolean paginated  = false;
+        boolean paginated = false;
         int numRecords = PAGE_SIZE;
         if (limit != null) {
             numRecords = limit > 0 ? limit : PAGE_SIZE;
             paginated = true;
         }
 
-        if (cursor != null && !cursor.isEmpty()) {
+        String cursor = null;
+        if (hashedCursorKey != null && !hashedCursorKey.isEmpty()) {
             paginated = true;
+            try {
+                cursor = this.cursorCache.get(hashedCursorKey);
+            } catch (RedisException ex) {
+                this.logger.error(String.format("Error getting key %s from redis: %s", hashedCursorKey, ex.getMessage()), ex);
+            }
+
+            if (Strings.isNullOrEmpty(cursor)) throw this.getInvalidCursorException();
         }
         String status = RecordState.active.toString();
         Sort sort = Sort.by(Sort.Direction.ASC, "id");
@@ -124,7 +141,12 @@ public class QueryRepository implements IQueryRepository {
                 if (pageable instanceof CosmosStorePageRequest) {
                     continuation = ((CosmosStorePageRequest) pageable).getRequestContinuation();
                 }
-                dqr.setCursor(continuation);
+
+                if (!Strings.isNullOrEmpty(continuation)) {
+                    String hashedCursor = Crc32c.hashToBase64EncodedString(continuation);
+                    this.cursorCache.put(hashedCursor, continuation);
+                    dqr.setCursor(hashedCursor);
+                }
                 docs = docPage.getContent();
             } else {
                 docs = record.findByMetadata_kindAndMetadata_status(kind, status);

provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/cache/LegalTagCache.java

@@ -22,7 +22,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 
-@Component
+@Component("LegalTagCache")
 public class LegalTagCache implements ICache<String, String> {
 
 	@Autowired

storage-core/src/main/java/org/opengroup/osdu/storage/service/LegalServiceImpl.java

@@ -31,6 +31,7 @@ import org.opengroup.osdu.core.common.model.storage.Record;
 import org.opengroup.osdu.core.common.model.storage.RecordIdWithVersion;
 import org.opengroup.osdu.core.common.model.storage.RecordMetadata;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.stereotype.Service;
 
 import java.util.List;
@@ -46,6 +47,7 @@ public class LegalServiceImpl implements ILegalService {
     @Autowired
     private DpsHeaders headers;
     @Autowired
+    @Qualifier("LegalTagCache")
     private ICache<String, String> cache;
     @Autowired
     private ILegalFactory factory;