diff --git a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/QueryRepository.java b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/QueryRepository.java index e183c83c0cacdfc665dbfa8d7ffb72ea3eebd1fd..36636d91460d323d40c69589abc2cb7aab92d6af 100644 --- a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/QueryRepository.java +++ b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/QueryRepository.java @@ -116,7 +116,7 @@ public class QueryRepository implements IQueryRepository { } @Override - public DatastoreQueryResult getAllRecordIdsFromKind(String kind, Integer limit, String hashedCursorKey, Optional<CollaborationContext> collaborationContext) { + public DatastoreQueryResult getAllRecordIdsFromKind(String kind, Integer limit, String cursor, Optional<CollaborationContext> collaborationContext) { Assert.notNull(kind, "kind must not be null"); boolean paginated = false; @@ -126,16 +126,8 @@ public class QueryRepository implements IQueryRepository { paginated = true; } - String cursor = null; - if (hashedCursorKey != null && !hashedCursorKey.isEmpty()) { + if (cursor != null && !cursor.isEmpty()) { paginated = true; - try { - cursor = this.cursorCache.get(hashedCursorKey); - } catch (RedisException ex) { - this.logger.error(String.format("Error getting key %s from redis: %s", hashedCursorKey, ex.getMessage()), ex); - } - - if (Strings.isNullOrEmpty(cursor)) throw this.getInvalidCursorException(); } String status = RecordState.active.toString(); DatastoreQueryResult dqr = new DatastoreQueryResult(); @@ -169,11 +161,8 @@ public class QueryRepository implements IQueryRepository { } while (!Strings.isNullOrEmpty(continuation) && ids.size() < numRecords); - // Hash the continuationToken if (!Strings.isNullOrEmpty(continuation)) { - String hashedCursor = Crc32c.hashToBase64EncodedString(continuation); - this.cursorCache.put(hashedCursor, continuation); - dqr.setCursor(hashedCursor); + dqr.setCursor(continuation); } } else { diff --git a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java index 15c8e7670f8fed6c4e27abc3b2cc7aad26dfe241..9511a842a8c19f9e87847413090221716da99701 100644 --- a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java +++ b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java @@ -238,7 +238,9 @@ public class RecordMetadataRepository extends SimpleCosmosStoreRepository<Record Assert.notNull(kind, "kind must not be null"); Assert.notNull(status, "status must not be null"); SqlQuerySpec query = getIdsByMetadata_kindAndMetada_statusQuery(kind, status, collaborationContext); - return this.find(pageable, headers.getPartitionId(), cosmosDBName, recordMetadataCollection, query); + CosmosQueryRequestOptions queryOptions = new CosmosQueryRequestOptions(); + queryOptions.setResponseContinuationTokenLimitInKb(1); + return this.find(pageable, headers.getPartitionId(), cosmosDBName, recordMetadataCollection, query, queryOptions); } public int getMetadataDocumentCountForBlob(String path) { diff --git a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/SimpleCosmosStoreRepository.java b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/SimpleCosmosStoreRepository.java index db5de17ea9dbe82bdf7f4272595ddc08111f5937..66f2f223d57411f9f36fd01ce3e0e89f9bba375d 100644 --- a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/SimpleCosmosStoreRepository.java +++ b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/SimpleCosmosStoreRepository.java @@ -105,6 +105,11 @@ public class SimpleCosmosStoreRepository<T> implements CosmosStoreRepository<T> return this.operation.queryItemsPage(dataPartitionId, cosmosDBName, collection, query, clazz, pageSize, continuationToken); } + @Override + public <T> Page<T> queryItemsPage(String dataPartitionId, String cosmosDBName, String collection, SqlQuerySpec query, Class<T> clazz, int pageSize, String continuationToken, CosmosQueryRequestOptions options) { + return this.operation.queryItemsPage(dataPartitionId, cosmosDBName, collection, query, clazz, pageSize, continuationToken, options); + } + @Override public void createItem(String dataPartitionId, String cosmosDBName, String collection, @NonNull String partitionKey, T item) { Assert.notNull(item, ENTITY_MUST_NOT_BE_NULL); @@ -258,13 +263,17 @@ public class SimpleCosmosStoreRepository<T> implements CosmosStoreRepository<T> } public Page<T> find(@NonNull Pageable pageable, String dataPartitionId, String cosmosDBName, String collectionName, SqlQuerySpec query) { + return this.find(pageable, dataPartitionId, cosmosDBName, collectionName, query, new CosmosQueryRequestOptions()); + } + + public Page<T> find(@NonNull Pageable pageable, String dataPartitionId, String cosmosDBName, String collectionName, SqlQuerySpec query, CosmosQueryRequestOptions queryOptions) { Assert.notNull(pageable, PAGEABLE_MUST_NOT_BE_NULL); CosmosStoreQuery cosmosQuery = (new CosmosStoreQuery()).with(query.getQueryText()); if (pageable.getSort().isSorted()) { cosmosQuery.with(pageable.getSort()); } SqlQuerySpec sqlQuerySpec = (new FindQuerySpecGenerator()).generateCosmosWithQueryText(cosmosQuery, cosmosQuery.getQuery()); - return this.paginationQuery(pageable, sqlQuerySpec, domainClass, dataPartitionId, cosmosDBName, collectionName); + return this.paginationQuery(pageable, sqlQuerySpec, domainClass, dataPartitionId, cosmosDBName, collectionName, queryOptions); } @Override @@ -282,6 +291,10 @@ public class SimpleCosmosStoreRepository<T> implements CosmosStoreRepository<T> } public Page<T> paginationQuery(Pageable pageable, SqlQuerySpec query, Class<T> domainClass, String dataPartitionId, String cosmosDBName, String collectionName) { + return this.paginationQuery(pageable, query, domainClass, dataPartitionId, cosmosDBName, collectionName, new CosmosQueryRequestOptions()); + } + + public Page<T> paginationQuery(Pageable pageable, SqlQuerySpec query, Class<T> domainClass, String dataPartitionId, String cosmosDBName, String collectionName, CosmosQueryRequestOptions queryOptions) { Assert.isTrue(pageable.getPageSize() > 0, "pageable should have page size larger than 0"); Assert.hasText(collectionName, "collection should not be null, empty or only whitespaces"); String continuationToken = null; @@ -289,7 +302,7 @@ public class SimpleCosmosStoreRepository<T> implements CosmosStoreRepository<T> continuationToken = ((CosmosStorePageRequest)pageable).getRequestContinuation(); } int pageSize = pageable.getPageSize(); - return this.queryItemsPage(dataPartitionId, cosmosDBName, collectionName, query, domainClass, pageSize, continuationToken); + return this.queryItemsPage(dataPartitionId, cosmosDBName, collectionName, query, domainClass, pageSize, continuationToken, queryOptions); } protected BulkImportResponse bulkInsert(String dataPartitionId, String cosmosDBName, String collectionName, Collection<T> docs){ diff --git a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/interfaces/CosmosStoreRepository.java b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/interfaces/CosmosStoreRepository.java index 39aa08ee94a6ef0f65fb60b5e3cc1ee1aa016df9..0cac83f5943bf60c9c8941369eff80651f362b6c 100644 --- a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/interfaces/CosmosStoreRepository.java +++ b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/interfaces/CosmosStoreRepository.java @@ -58,5 +58,7 @@ public interface CosmosStoreRepository<T> extends PagingAndSortingRepository<T> <T> Page<T> queryItemsPage(String dataPartitionId, String cosmosDBName, String collection, SqlQuerySpec query, Class<T> clazz, int pageSize, String continuationToken); + <T> Page<T> queryItemsPage(String dataPartitionId, String cosmosDBName, String collection, SqlQuerySpec query, Class<T> clazz, int pageSize, String continuationToken, CosmosQueryRequestOptions options); + }