diff --git a/NOTICE b/NOTICE index 0d3ebe37a138a21494558260d29024c01557ba02..7e7b61c48a195f0ef7f954c11754e1ce7d61d6ce 100644 --- a/NOTICE +++ b/NOTICE @@ -333,7 +333,7 @@ The following software have components provided under the terms of this license: - Byte Buddy Java agent (from https://repo1.maven.org/maven2/net/bytebuddy/byte-buddy-agent) - ClassMate (from http://github.com/cowtowncoder/java-classmate) - Cloud Key Management Service (KMS) API (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-cloudkms) -- Cloud Storage JSON API (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-storage) +- Cloud Storage JSON API v1-rev20230301-2.0.0 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-storage) - CloudWatch Metrics for AWS Java SDK (from https://aws.amazon.com/sdkforjava) - Collections (from https://repo1.maven.org/maven2/commons-collections/commons-collections) - Commons Digester (from http://commons.apache.org/digester/) @@ -613,7 +613,6 @@ The following software have components provided under the terms of this license: - Google OAuth Client Library for Java (from https://repo1.maven.org/maven2/com/google/oauth-client/google-oauth-client) - Hamcrest (from http://hamcrest.org/JavaHamcrest/) - Hamcrest Core (from http://hamcrest.org/, http://hamcrest.org/JavaHamcrest/, https://repo1.maven.org/maven2/org/hamcrest/hamcrest-core) -- Jackson module: Afterburner (from http://wiki.fasterxml.com/JacksonHome, https://github.com/FasterXML/jackson-modules-base) - Jakarta Activation API (from https://github.com/eclipse-ee4j/jaf, https://github.com/jakartaee/jaf-api, https://repo1.maven.org/maven2/jakarta/activation/jakarta.activation-api) - Jakarta Annotations API (from https://projects.eclipse.org/projects/ee4j.ca) - Jakarta XML Binding API (from https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api, https://repo1.maven.org/maven2/org/jboss/spec/javax/xml/bind/jboss-jaxb-api_2.3_spec) @@ -984,6 +983,13 @@ The following software have components provided under the terms of this license: - HK2 Implementation Utilities (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-utils) - ServiceLocator Default Implementation (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-locator) +======================================================================== +mit-old-style-no-advert +======================================================================== +The following software have components provided under the terms of this license: + +- Brave (from https://repo1.maven.org/maven2/io/zipkin/brave/brave) + ======================================================================== public-domain ======================================================================== diff --git a/provider/storage-azure/pom.xml b/provider/storage-azure/pom.xml index 6ebdc2fd5776f65ba80aadad67409703673e15e7..7bd420443a5bc07fb27bd02e282a3170ba08aec4 100644 --- a/provider/storage-azure/pom.xml +++ b/provider/storage-azure/pom.xml @@ -33,8 +33,7 @@ <azure.appservice.plan /> <azure.appservice.appname /> <azure.appservice.subscription /> - <!-- below corelibazure version will be updated to a release candidate once the associated MR is merged --> - <osdu.corelibazure.version>0.19.0-rc5</osdu.corelibazure.version> + <osdu.corelibazure.version>0.20.0-rc6</osdu.corelibazure.version> <osdu.storage-core.version>0.20.0-SNAPSHOT</osdu.storage-core.version> <junit.version>4.12</junit.version> <mockito.version>1.10.19</mockito.version> diff --git a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/QueryRepository.java b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/QueryRepository.java index e183c83c0cacdfc665dbfa8d7ffb72ea3eebd1fd..36636d91460d323d40c69589abc2cb7aab92d6af 100644 --- a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/QueryRepository.java +++ b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/QueryRepository.java @@ -116,7 +116,7 @@ public class QueryRepository implements IQueryRepository { } @Override - public DatastoreQueryResult getAllRecordIdsFromKind(String kind, Integer limit, String hashedCursorKey, Optional<CollaborationContext> collaborationContext) { + public DatastoreQueryResult getAllRecordIdsFromKind(String kind, Integer limit, String cursor, Optional<CollaborationContext> collaborationContext) { Assert.notNull(kind, "kind must not be null"); boolean paginated = false; @@ -126,16 +126,8 @@ public class QueryRepository implements IQueryRepository { paginated = true; } - String cursor = null; - if (hashedCursorKey != null && !hashedCursorKey.isEmpty()) { + if (cursor != null && !cursor.isEmpty()) { paginated = true; - try { - cursor = this.cursorCache.get(hashedCursorKey); - } catch (RedisException ex) { - this.logger.error(String.format("Error getting key %s from redis: %s", hashedCursorKey, ex.getMessage()), ex); - } - - if (Strings.isNullOrEmpty(cursor)) throw this.getInvalidCursorException(); } String status = RecordState.active.toString(); DatastoreQueryResult dqr = new DatastoreQueryResult(); @@ -169,11 +161,8 @@ public class QueryRepository implements IQueryRepository { } while (!Strings.isNullOrEmpty(continuation) && ids.size() < numRecords); - // Hash the continuationToken if (!Strings.isNullOrEmpty(continuation)) { - String hashedCursor = Crc32c.hashToBase64EncodedString(continuation); - this.cursorCache.put(hashedCursor, continuation); - dqr.setCursor(hashedCursor); + dqr.setCursor(continuation); } } else { diff --git a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java index 14e39a7dad6ea317996c019cf10c4786350b580c..53207f1f668041c7aa0d9e8c02f1dd56ab63f34c 100644 --- a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java +++ b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java @@ -196,7 +196,9 @@ public class RecordMetadataRepository extends SimpleCosmosStoreRepository<Record Assert.notNull(kind, "kind must not be null"); Assert.notNull(status, "status must not be null"); SqlQuerySpec query = getIdsByMetadata_kindAndMetada_statusQuery(kind, status, collaborationContext); - return this.find(pageable, headers.getPartitionId(), cosmosDBName, recordMetadataCollection, query); + CosmosQueryRequestOptions queryOptions = new CosmosQueryRequestOptions(); + queryOptions.setResponseContinuationTokenLimitInKb(1); + return this.find(pageable, headers.getPartitionId(), cosmosDBName, recordMetadataCollection, query, queryOptions); } public int getMetadataDocumentCountForBlob(String path) { diff --git a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/SimpleCosmosStoreRepository.java b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/SimpleCosmosStoreRepository.java index db5de17ea9dbe82bdf7f4272595ddc08111f5937..66f2f223d57411f9f36fd01ce3e0e89f9bba375d 100644 --- a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/SimpleCosmosStoreRepository.java +++ b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/SimpleCosmosStoreRepository.java @@ -105,6 +105,11 @@ public class SimpleCosmosStoreRepository<T> implements CosmosStoreRepository<T> return this.operation.queryItemsPage(dataPartitionId, cosmosDBName, collection, query, clazz, pageSize, continuationToken); } + @Override + public <T> Page<T> queryItemsPage(String dataPartitionId, String cosmosDBName, String collection, SqlQuerySpec query, Class<T> clazz, int pageSize, String continuationToken, CosmosQueryRequestOptions options) { + return this.operation.queryItemsPage(dataPartitionId, cosmosDBName, collection, query, clazz, pageSize, continuationToken, options); + } + @Override public void createItem(String dataPartitionId, String cosmosDBName, String collection, @NonNull String partitionKey, T item) { Assert.notNull(item, ENTITY_MUST_NOT_BE_NULL); @@ -258,13 +263,17 @@ public class SimpleCosmosStoreRepository<T> implements CosmosStoreRepository<T> } public Page<T> find(@NonNull Pageable pageable, String dataPartitionId, String cosmosDBName, String collectionName, SqlQuerySpec query) { + return this.find(pageable, dataPartitionId, cosmosDBName, collectionName, query, new CosmosQueryRequestOptions()); + } + + public Page<T> find(@NonNull Pageable pageable, String dataPartitionId, String cosmosDBName, String collectionName, SqlQuerySpec query, CosmosQueryRequestOptions queryOptions) { Assert.notNull(pageable, PAGEABLE_MUST_NOT_BE_NULL); CosmosStoreQuery cosmosQuery = (new CosmosStoreQuery()).with(query.getQueryText()); if (pageable.getSort().isSorted()) { cosmosQuery.with(pageable.getSort()); } SqlQuerySpec sqlQuerySpec = (new FindQuerySpecGenerator()).generateCosmosWithQueryText(cosmosQuery, cosmosQuery.getQuery()); - return this.paginationQuery(pageable, sqlQuerySpec, domainClass, dataPartitionId, cosmosDBName, collectionName); + return this.paginationQuery(pageable, sqlQuerySpec, domainClass, dataPartitionId, cosmosDBName, collectionName, queryOptions); } @Override @@ -282,6 +291,10 @@ public class SimpleCosmosStoreRepository<T> implements CosmosStoreRepository<T> } public Page<T> paginationQuery(Pageable pageable, SqlQuerySpec query, Class<T> domainClass, String dataPartitionId, String cosmosDBName, String collectionName) { + return this.paginationQuery(pageable, query, domainClass, dataPartitionId, cosmosDBName, collectionName, new CosmosQueryRequestOptions()); + } + + public Page<T> paginationQuery(Pageable pageable, SqlQuerySpec query, Class<T> domainClass, String dataPartitionId, String cosmosDBName, String collectionName, CosmosQueryRequestOptions queryOptions) { Assert.isTrue(pageable.getPageSize() > 0, "pageable should have page size larger than 0"); Assert.hasText(collectionName, "collection should not be null, empty or only whitespaces"); String continuationToken = null; @@ -289,7 +302,7 @@ public class SimpleCosmosStoreRepository<T> implements CosmosStoreRepository<T> continuationToken = ((CosmosStorePageRequest)pageable).getRequestContinuation(); } int pageSize = pageable.getPageSize(); - return this.queryItemsPage(dataPartitionId, cosmosDBName, collectionName, query, domainClass, pageSize, continuationToken); + return this.queryItemsPage(dataPartitionId, cosmosDBName, collectionName, query, domainClass, pageSize, continuationToken, queryOptions); } protected BulkImportResponse bulkInsert(String dataPartitionId, String cosmosDBName, String collectionName, Collection<T> docs){ diff --git a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/interfaces/CosmosStoreRepository.java b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/interfaces/CosmosStoreRepository.java index 39aa08ee94a6ef0f65fb60b5e3cc1ee1aa016df9..0cac83f5943bf60c9c8941369eff80651f362b6c 100644 --- a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/interfaces/CosmosStoreRepository.java +++ b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/interfaces/CosmosStoreRepository.java @@ -58,5 +58,7 @@ public interface CosmosStoreRepository<T> extends PagingAndSortingRepository<T> <T> Page<T> queryItemsPage(String dataPartitionId, String cosmosDBName, String collection, SqlQuerySpec query, Class<T> clazz, int pageSize, String continuationToken); + <T> Page<T> queryItemsPage(String dataPartitionId, String cosmosDBName, String collection, SqlQuerySpec query, Class<T> clazz, int pageSize, String continuationToken, CosmosQueryRequestOptions options); + }