From b1cb9543fe0cc5bbc81aa68fa103e70d2e5d6e35 Mon Sep 17 00:00:00 2001 From: jsangiamo <69694815+jsangiamo@users.noreply.github.com> Date: Tue, 5 Jan 2021 09:50:41 -0800 Subject: [PATCH 01/18] adding initial code --- .../provider/azure/CloudStorageImpl.java | 2 ++ .../repository/RecordMetadataRepository.java | 27 +++++++++++++------ 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/CloudStorageImpl.java b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/CloudStorageImpl.java index f92aabab..9a778143 100644 --- a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/CloudStorageImpl.java +++ b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/CloudStorageImpl.java @@ -22,6 +22,7 @@ import org.apache.commons.lang3.ArrayUtils; import org.apache.http.HttpStatus; import org.opengroup.osdu.azure.blobstorage.IBlobContainerClientFactory; +import org.opengroup.osdu.azure.cosmosdb.CosmosBulkExecutorImpl; import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.core.common.model.entitlements.Acl; import org.opengroup.osdu.core.common.model.http.AppException; @@ -78,6 +79,7 @@ public class CloudStorageImpl implements ICloudStorage { private String containerName; + @Override public void write(RecordProcessing... recordsProcessing) { validateRecordAcls(recordsProcessing); diff --git a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java index da6cb44e..dbe89b8f 100644 --- a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java +++ b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java @@ -18,7 +18,11 @@ package org.opengroup.osdu.storage.provider.azure.repository; import com.azure.cosmos.CosmosException; import com.azure.cosmos.models.CosmosQueryRequestOptions; import com.azure.cosmos.models.SqlQuerySpec; +import com.microsoft.azure.documentdb.bulkexecutor.BulkImportResponse; import org.apache.http.HttpStatus; +import org.opengroup.osdu.azure.cosmosdb.CosmosBulkExecutorImpl; +import org.opengroup.osdu.azure.cosmosdb.CosmosStore; +import org.opengroup.osdu.azure.cosmosdb.CosmosStoreBulkOperations; import org.opengroup.osdu.azure.query.CosmosStorePageRequest; import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.core.common.model.http.AppException; @@ -59,21 +63,28 @@ public class RecordMetadataRepository extends SimpleCosmosStoreRepository createOrUpdate(List recordsMetadata) { + Assert.notNull(recordsMetadata, "recordsMetadata must not be null"); - if (recordsMetadata != null) { - for (RecordMetadata recordMetadata : recordsMetadata) { - RecordMetadataDoc doc = new RecordMetadataDoc(); - doc.setId(recordMetadata.getId()); - doc.setMetadata(recordMetadata); - this.save(doc); - } - } + BulkImportResponse response = cosmosBulkStore.bulkUpsert(headers.getPartitionId(), "osdu-db", "StorageRecord", recordsMetadata); + List errors = response.getErrors(); + List badDocs = response.getBadInputDocuments(); +// if (recordsMetadata != null) { +// for (RecordMetadata recordMetadata : recordsMetadata) { +// RecordMetadataDoc doc = new RecordMetadataDoc(); +// doc.setId(recordMetadata.getId()); +// doc.setMetadata(recordMetadata); +// this.save(doc); +// } +// } return recordsMetadata; } -- GitLab From 150a9a23bb6d0b8ca534ebc75ae4bcfd6d8e051d Mon Sep 17 00:00:00 2001 From: jsangiamo <69694815+jsangiamo@users.noreply.github.com> Date: Thu, 7 Jan 2021 09:00:14 -0800 Subject: [PATCH 02/18] first draft of code to be performance tested --- provider/storage-azure/pom.xml | 2 +- .../azure/di/AzureBootstrapConfig.java | 6 +++++ .../repository/RecordMetadataRepository.java | 25 ++++++++++--------- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/provider/storage-azure/pom.xml b/provider/storage-azure/pom.xml index 109c68f7..2155c91c 100644 --- a/provider/storage-azure/pom.xml +++ b/provider/storage-azure/pom.xml @@ -34,7 +34,7 @@ - 0.0.50 + 0.0.52-perf-SNAPSHOT 0.3.18 0.0.5-SNAPSHOT 4.12 diff --git a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/di/AzureBootstrapConfig.java b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/di/AzureBootstrapConfig.java index c563edcb..b75dc658 100644 --- a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/di/AzureBootstrapConfig.java +++ b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/di/AzureBootstrapConfig.java @@ -87,4 +87,10 @@ public class AzureBootstrapConfig { public String redisPassword(SecretClient kv) { return KeyVaultFacade.getSecretWithValidation(kv, "redis-password"); } + + @Bean + @Named("STORAGE_RECORD_COSMOS_COLLECTION_NAME") + public String storageRecordCosmosCollectionName() { + return "StorageRecord"; + } } diff --git a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java index dbe89b8f..51a882d1 100644 --- a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java +++ b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java @@ -28,6 +28,7 @@ import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.model.http.DpsHeaders; import org.opengroup.osdu.core.common.model.legal.LegalCompliance; +import org.opengroup.osdu.core.common.model.storage.Record; import org.opengroup.osdu.core.common.model.storage.RecordMetadata; import org.opengroup.osdu.storage.provider.azure.RecordMetadataDoc; import org.opengroup.osdu.storage.provider.azure.di.AzureBootstrapConfig; @@ -66,25 +67,25 @@ public class RecordMetadataRepository extends SimpleCosmosStoreRepository createOrUpdate(List recordsMetadata) { - Assert.notNull(recordsMetadata, "recordsMetadata must not be null"); - BulkImportResponse response = cosmosBulkStore.bulkUpsert(headers.getPartitionId(), "osdu-db", "StorageRecord", recordsMetadata); - List errors = response.getErrors(); - List badDocs = response.getBadInputDocuments(); -// if (recordsMetadata != null) { -// for (RecordMetadata recordMetadata : recordsMetadata) { -// RecordMetadataDoc doc = new RecordMetadataDoc(); -// doc.setId(recordMetadata.getId()); -// doc.setMetadata(recordMetadata); -// this.save(doc); -// } -// } + Collection docs = new ArrayList<>(); + for (RecordMetadata recordMetadata : recordsMetadata){ + RecordMetadataDoc doc = new RecordMetadataDoc(); + doc.setId(recordMetadata.getId()); + doc.setMetadata(recordMetadata); + docs.add(doc); + } + BulkImportResponse response = cosmosBulkStore.bulkInsert(headers.getPartitionId(), cosmosDBName, storageRecordCosmosCollectionName, docs, true, true, 100); + return recordsMetadata; } -- GitLab From bec73786104fb401a23e4e4d00fb327d6ce39b3a Mon Sep 17 00:00:00 2001 From: jsangiamo <69694815+jsangiamo@users.noreply.github.com> Date: Thu, 7 Jan 2021 09:08:22 -0800 Subject: [PATCH 03/18] reducing replica count for perf testing --- devops/azure/chart/helm-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devops/azure/chart/helm-config.yaml b/devops/azure/chart/helm-config.yaml index aa0d121b..85434607 100644 --- a/devops/azure/chart/helm-config.yaml +++ b/devops/azure/chart/helm-config.yaml @@ -2,7 +2,7 @@ global: # Service(s) Replica Count - replicaCount: 10 + replicaCount: 1 ################################################################################ # Specify the Gitlab branch being used for image creation -- GitLab From 3afbc3149f63b1294ed10ecca98f8261738a6e4d Mon Sep 17 00:00:00 2001 From: jsangiamo <69694815+jsangiamo@users.noreply.github.com> Date: Thu, 7 Jan 2021 09:29:53 -0800 Subject: [PATCH 04/18] incrementing dependency version to avoid getting old cached dependency --- provider/storage-azure/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provider/storage-azure/pom.xml b/provider/storage-azure/pom.xml index 2155c91c..7915ff5a 100644 --- a/provider/storage-azure/pom.xml +++ b/provider/storage-azure/pom.xml @@ -34,7 +34,7 @@ - 0.0.52-perf-SNAPSHOT + 0.0.52-perf-SNAPSHOT2 0.3.18 0.0.5-SNAPSHOT 4.12 -- GitLab From f9b0b9f406734c16276f2e81a8e36939aec8beee Mon Sep 17 00:00:00 2001 From: jsangiamo <69694815+jsangiamo@users.noreply.github.com> Date: Thu, 7 Jan 2021 20:32:11 -0800 Subject: [PATCH 05/18] adding timing log statements for debugging performance --- .../osdu/storage/provider/azure/CloudStorageImpl.java | 7 ++++++- .../azure/repository/RecordMetadataRepository.java | 5 +++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/CloudStorageImpl.java b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/CloudStorageImpl.java index 9a778143..381e97db 100644 --- a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/CloudStorageImpl.java +++ b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/CloudStorageImpl.java @@ -22,7 +22,6 @@ import org.apache.commons.lang3.ArrayUtils; import org.apache.http.HttpStatus; import org.opengroup.osdu.azure.blobstorage.IBlobContainerClientFactory; -import org.opengroup.osdu.azure.cosmosdb.CosmosBulkExecutorImpl; import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.core.common.model.entitlements.Acl; import org.opengroup.osdu.core.common.model.http.AppException; @@ -82,6 +81,10 @@ public class CloudStorageImpl implements ICloudStorage { @Override public void write(RecordProcessing... recordsProcessing) { + + long startTime = System.currentTimeMillis(); + + validateRecordAcls(recordsProcessing); List> tasks = new ArrayList<>(); @@ -102,6 +105,8 @@ public class CloudStorageImpl implements ICloudStorage { throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Error during record ingestion", "An unexpected error on writing the record has occurred", e); } + long endTime = System.currentTimeMillis(); + logger.info("TIMING: Wrote " + tasks.size() + " records to blob storage in " + (endTime - startTime)/1000 + " seconds"); } @Override diff --git a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java index 51a882d1..ade16163 100644 --- a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java +++ b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java @@ -77,6 +77,7 @@ public class RecordMetadataRepository extends SimpleCosmosStoreRepository createOrUpdate(List recordsMetadata) { Assert.notNull(recordsMetadata, "recordsMetadata must not be null"); + Collection docs = new ArrayList<>(); for (RecordMetadata recordMetadata : recordsMetadata){ RecordMetadataDoc doc = new RecordMetadataDoc(); @@ -84,8 +85,8 @@ public class RecordMetadataRepository extends SimpleCosmosStoreRepository Date: Thu, 7 Jan 2021 21:37:59 -0800 Subject: [PATCH 06/18] enhancing time logging --- .../osdu/storage/provider/azure/CloudStorageImpl.java | 2 +- .../provider/azure/repository/RecordMetadataRepository.java | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/CloudStorageImpl.java b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/CloudStorageImpl.java index 381e97db..0cdeffb5 100644 --- a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/CloudStorageImpl.java +++ b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/CloudStorageImpl.java @@ -106,7 +106,7 @@ public class CloudStorageImpl implements ICloudStorage { "An unexpected error on writing the record has occurred", e); } long endTime = System.currentTimeMillis(); - logger.info("TIMING: Wrote " + tasks.size() + " records to blob storage in " + (endTime - startTime)/1000 + " seconds"); + logger.info("TIMING: Wrote " + tasks.size() + " records to blob storage in " + (endTime - startTime) + " ms"); } @Override diff --git a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java index ade16163..950509a1 100644 --- a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java +++ b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java @@ -77,6 +77,7 @@ public class RecordMetadataRepository extends SimpleCosmosStoreRepository createOrUpdate(List recordsMetadata) { Assert.notNull(recordsMetadata, "recordsMetadata must not be null"); + long startTime = System.currentTimeMillis(); Collection docs = new ArrayList<>(); for (RecordMetadata recordMetadata : recordsMetadata){ @@ -87,6 +88,9 @@ public class RecordMetadataRepository extends SimpleCosmosStoreRepository Date: Mon, 11 Jan 2021 14:24:22 -0800 Subject: [PATCH 07/18] adding timing to batch get method --- .../provider/azure/repository/RecordMetadataRepository.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java index 950509a1..c8f15329 100644 --- a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java +++ b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java @@ -142,6 +142,7 @@ public class RecordMetadataRepository extends SimpleCosmosStoreRepository get(List ids) { + long startTime = System.currentTimeMillis(); Map output = new HashMap<>(); for (String id : ids) { RecordMetadataDoc doc = this.getOne(id); @@ -150,6 +151,8 @@ public class RecordMetadataRepository extends SimpleCosmosStoreRepository Date: Tue, 12 Jan 2021 15:22:20 -0800 Subject: [PATCH 08/18] adding in bulk get via using SQL query --- .../repository/RecordMetadataRepository.java | 34 ++++++++++++++----- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java index c8f15329..c02f214b 100644 --- a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java +++ b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java @@ -143,17 +143,20 @@ public class RecordMetadataRepository extends SimpleCosmosStoreRepository get(List ids) { long startTime = System.currentTimeMillis(); - Map output = new HashMap<>(); - for (String id : ids) { - RecordMetadataDoc doc = this.getOne(id); - if (doc == null) continue; - RecordMetadata rmd = doc.getMetadata(); - if (rmd == null) continue; - output.put(id, rmd); + String sqlQueryString = createCosmosBatchGetQueryById(ids); + SqlQuerySpec query = new SqlQuerySpec(sqlQueryString); + CosmosQueryRequestOptions options = new CosmosQueryRequestOptions(); + + List queryResults = this.queryItems(headers.getPartitionId(), cosmosDBName, recordMetadataCollection, query, options); + + Map results = new HashMap<>(); + for(RecordMetadataDoc doc : queryResults){ + if (doc.getMetadata() == null) continue; + results.put(doc.getId(), doc.getMetadata()); } long endTime = System.currentTimeMillis(); - logger.info("TIMING: batch get method took ms " + (endTime - startTime)); - return output; + logger.info("TIMING: bulk get method queried " + ids.size() + " records in ms " + (endTime - startTime)); + return results; } private AppException getInvalidCursorException() { @@ -198,4 +201,17 @@ public class RecordMetadataRepository extends SimpleCosmosStoreRepository ids){ + StringBuilder sb = new StringBuilder(); + sb.append("SELECT * FROM c WHERE c.id IN ("); + for(String id : ids){ + sb.append("\"" + id + "\","); + } + + // remove trailing comma, add closing parenthesis + sb.deleteCharAt(sb.lastIndexOf(",")); + sb.append(")"); + return sb.toString(); + } } -- GitLab From ce3aaa09f4c365888f43cb55fbcf19fd45b31db5 Mon Sep 17 00:00:00 2001 From: jsangiamo <69694815+jsangiamo@users.noreply.github.com> Date: Fri, 15 Jan 2021 13:09:07 -0800 Subject: [PATCH 09/18] cleaning up code --- provider/storage-azure/pom.xml | 2 +- .../azure/di/AzureBootstrapConfig.java | 13 ++++-- .../repository/RecordMetadataRepository.java | 41 ++++++++++++------- .../SimpleCosmosStoreRepository.java | 23 +++++++++++ 4 files changed, 61 insertions(+), 18 deletions(-) diff --git a/provider/storage-azure/pom.xml b/provider/storage-azure/pom.xml index 7915ff5a..b9847d42 100644 --- a/provider/storage-azure/pom.xml +++ b/provider/storage-azure/pom.xml @@ -34,7 +34,7 @@ - 0.0.52-perf-SNAPSHOT2 + 0.0.59 0.3.18 0.0.5-SNAPSHOT 4.12 diff --git a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/di/AzureBootstrapConfig.java b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/di/AzureBootstrapConfig.java index b75dc658..730cb261 100644 --- a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/di/AzureBootstrapConfig.java +++ b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/di/AzureBootstrapConfig.java @@ -88,9 +88,16 @@ public class AzureBootstrapConfig { return KeyVaultFacade.getSecretWithValidation(kv, "redis-password"); } + // How large the batch size has to be for the bulk executor to be used instead of uploading record in serial. @Bean - @Named("STORAGE_RECORD_COSMOS_COLLECTION_NAME") - public String storageRecordCosmosCollectionName() { - return "StorageRecord"; + public int minBatchSizeToUseBulkUpload() { + return Integer.parseInt(System.getProperty("MIN_BATCH_SIZE_TO_USE_BULK_UPLOAD", "50")); + } + + + // The maximum degree of concurrency per partition key range. The default value in the SDK is 20. + @Bean + public int bulkImportMaxConcurrencyPePartitionRange(){ + return Integer.parseInt(System.getProperty("BULK_IMPORT_MAX_CONCURRENCY_PER_PARTITION_RANGE", "20")); } } diff --git a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java index c02f214b..b1a38e75 100644 --- a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java +++ b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java @@ -20,15 +20,11 @@ import com.azure.cosmos.models.CosmosQueryRequestOptions; import com.azure.cosmos.models.SqlQuerySpec; import com.microsoft.azure.documentdb.bulkexecutor.BulkImportResponse; import org.apache.http.HttpStatus; -import org.opengroup.osdu.azure.cosmosdb.CosmosBulkExecutorImpl; -import org.opengroup.osdu.azure.cosmosdb.CosmosStore; -import org.opengroup.osdu.azure.cosmosdb.CosmosStoreBulkOperations; import org.opengroup.osdu.azure.query.CosmosStorePageRequest; import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.model.http.DpsHeaders; import org.opengroup.osdu.core.common.model.legal.LegalCompliance; -import org.opengroup.osdu.core.common.model.storage.Record; import org.opengroup.osdu.core.common.model.storage.RecordMetadata; import org.opengroup.osdu.storage.provider.azure.RecordMetadataDoc; import org.opengroup.osdu.storage.provider.azure.di.AzureBootstrapConfig; @@ -65,10 +61,7 @@ public class RecordMetadataRepository extends SimpleCosmosStoreRepository= minBatchSizeToUseBulkUpload) createOrUpdateParallel(recordsMetadata); + else createOrUpdateSerial(recordsMetadata); + + logger.info("TIMING: createOrUpdate method took ms " + (System.currentTimeMillis() - startTime)); + return recordsMetadata; + } + + /** + * Implementation of createOrUpdate that writes the records in serial one at a time to Cosmos. + * @param recordsMetadata records to write to cosmos. + */ + private void createOrUpdateSerial(List recordsMetadata){ + for (RecordMetadata recordMetadata : recordsMetadata) { + RecordMetadataDoc doc = new RecordMetadataDoc(); + doc.setId(recordMetadata.getId()); + doc.setMetadata(recordMetadata); + this.save(doc); + } + } + + /** + * Implementation of createOrUpdate that uses DocumentBulkExecutor to upload all records in parallel to Cosmos. + * @param recordsMetadata records to write to cosmos. + */ + private void createOrUpdateParallel(List recordsMetadata){ Collection docs = new ArrayList<>(); for (RecordMetadata recordMetadata : recordsMetadata){ RecordMetadataDoc doc = new RecordMetadataDoc(); @@ -86,12 +104,7 @@ public class RecordMetadataRepository extends SimpleCosmosStoreRepository implements CosmosStoreRepository @Autowired private CosmosStore operation; + @Autowired + private CosmosStoreBulkOperations cosmosBulkStore; + + @Autowired + private int bulkImportMaxConcurrencyPePartitionRange; + public SimpleCosmosStoreRepository(Class domainClass) { this.domainClass = domainClass; } @@ -275,4 +286,16 @@ public class SimpleCosmosStoreRepository implements CosmosStoreRepository int pageSize = pageable.getPageSize(); return this.queryItemsPage(dataPartitionId, cosmosDBName, collectionName, query, domainClass, pageSize, continuationToken); } + + protected BulkImportResponse bulkInsert(String dataPartitionId, String cosmosDBName, String collectionName, Collection docs){ + BulkImportResponse response = cosmosBulkStore.bulkInsert(dataPartitionId, cosmosDBName, collectionName, docs, true, true, bulkImportMaxConcurrencyPePartitionRange); + if(response.getNumberOfDocumentsImported() != docs.size()){ + List exceptions = new ArrayList<>(); + for(Exception e : response.getErrors()){ + exceptions.add(e.toString()); + } + throw new AppException(500, "Record creation has failed!", "Failed to import all documents to CosmosDB using bulk executor!", exceptions.toArray(new String[exceptions.size()])); + } + return response; + } } -- GitLab From 81d8fb05951a46674e84f9588b8e0e2c6de5e589 Mon Sep 17 00:00:00 2001 From: jsangiamo <69694815+jsangiamo@users.noreply.github.com> Date: Fri, 15 Jan 2021 14:10:37 -0800 Subject: [PATCH 10/18] comments and checking for null values --- .../provider/azure/repository/RecordMetadataRepository.java | 5 +++++ .../azure/repository/SimpleCosmosStoreRepository.java | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java index b1a38e75..5ee6e3c6 100644 --- a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java +++ b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java @@ -215,6 +215,11 @@ public class RecordMetadataRepository extends SimpleCosmosStoreRepository ids){ StringBuilder sb = new StringBuilder(); sb.append("SELECT * FROM c WHERE c.id IN ("); diff --git a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/SimpleCosmosStoreRepository.java b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/SimpleCosmosStoreRepository.java index b1365662..0f65924f 100644 --- a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/SimpleCosmosStoreRepository.java +++ b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/SimpleCosmosStoreRepository.java @@ -288,6 +288,10 @@ public class SimpleCosmosStoreRepository implements CosmosStoreRepository } protected BulkImportResponse bulkInsert(String dataPartitionId, String cosmosDBName, String collectionName, Collection docs){ + Assert.notNull(docs, "docs should not be null"); + Assert.notNull(dataPartitionId, "dataPartitionId should not be null"); + Assert.notNull(cosmosDBName, "cosmosDBName should not be null"); + Assert.notNull(collectionName, "collectionName should not be null"); BulkImportResponse response = cosmosBulkStore.bulkInsert(dataPartitionId, cosmosDBName, collectionName, docs, true, true, bulkImportMaxConcurrencyPePartitionRange); if(response.getNumberOfDocumentsImported() != docs.size()){ List exceptions = new ArrayList<>(); -- GitLab From 0609b3e4c2ab3ce8e85e0e6edd2075e60a596bbf Mon Sep 17 00:00:00 2001 From: jsangiamo <69694815+jsangiamo@users.noreply.github.com> Date: Sat, 16 Jan 2021 15:06:52 -0800 Subject: [PATCH 11/18] fixing comments --- .../storage/provider/azure/di/AzureBootstrapConfig.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/di/AzureBootstrapConfig.java b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/di/AzureBootstrapConfig.java index 730cb261..024077b0 100644 --- a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/di/AzureBootstrapConfig.java +++ b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/di/AzureBootstrapConfig.java @@ -88,14 +88,17 @@ public class AzureBootstrapConfig { return KeyVaultFacade.getSecretWithValidation(kv, "redis-password"); } - // How large the batch size has to be for the bulk executor to be used instead of uploading record in serial. + /** + * @return How large the batch size has to be for the bulk executor to be used instead of uploading record in serial. + */ @Bean public int minBatchSizeToUseBulkUpload() { return Integer.parseInt(System.getProperty("MIN_BATCH_SIZE_TO_USE_BULK_UPLOAD", "50")); } - - // The maximum degree of concurrency per partition key range. The default value in the SDK is 20. + /** + * @return The maximum degree of concurrency per partition key range. The default value in the SDK is 20. + */ @Bean public int bulkImportMaxConcurrencyPePartitionRange(){ return Integer.parseInt(System.getProperty("BULK_IMPORT_MAX_CONCURRENCY_PER_PARTITION_RANGE", "20")); -- GitLab From fc4434898180918203dfcb7c1b6699f9f9617c88 Mon Sep 17 00:00:00 2001 From: jsangiamo <69694815+jsangiamo@users.noreply.github.com> Date: Sun, 17 Jan 2021 18:50:08 -0800 Subject: [PATCH 12/18] upadting notice, small code tweaks --- NOTICE | 20 ++++- devops/azure-pipeline.yml | 62 --------------- devops/development-pipeline.yml | 62 --------------- devops/service-pipeline.yml | 77 ------------------- .../provider/azure/CloudStorageImpl.java | 6 +- .../repository/RecordMetadataRepository.java | 3 +- 6 files changed, 20 insertions(+), 210 deletions(-) delete mode 100644 devops/azure-pipeline.yml delete mode 100644 devops/development-pipeline.yml delete mode 100644 devops/service-pipeline.yml diff --git a/NOTICE b/NOTICE index 2d5439a3..239ca931 100644 --- a/NOTICE +++ b/NOTICE @@ -401,6 +401,7 @@ The following software have components provided under the terms of this license: - Adapter: RxJava (from ) - Apache Commons Codec (from http://commons.apache.org/proper/commons-codec/) - Apache Commons Codec (from http://commons.apache.org/proper/commons-codec/) +- Apache Commons Collections (from http://commons.apache.org/proper/commons-collections/) - Apache Commons FileUpload (from http://commons.apache.org/proper/commons-fileupload/) - Apache Commons Lang (from http://commons.apache.org/proper/commons-lang/) - Apache Commons Lang (from http://commons.apache.org/proper/commons-lang/) @@ -514,6 +515,7 @@ The following software have components provided under the terms of this license: - Java Native Access (from https://github.com/java-native-access/jna) - Java Native Access Platform (from https://github.com/java-native-access/jna) - Java Servlet API (from http://servlet-spec.java.net) +- Java UUID Generator (from http://wiki.fasterxml.com/JugHome) - Javassist (from http://www.javassist.org/) - Javassist (from http://www.javassist.org/) - Javassist (from http://www.javassist.org/) @@ -565,8 +567,8 @@ The following software have components provided under the terms of this license: - Microsoft Azure Java Core Library (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure Netty HTTP Client Library (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java) -- Mockito (from http://www.mockito.org) - Mockito (from http://mockito.org) +- Mockito (from http://www.mockito.org) - Mockito (from http://mockito.org) - Mockito (from http://www.mockito.org) - Netty Reactive Streams Implementation (from ) @@ -972,6 +974,13 @@ The following software have components provided under the terms of this license: - javax.ws.rs-api (from http://jax-rs-spec.java.net) - javax.ws.rs-api (from http://jax-rs-spec.java.net) +======================================================================== +JSON +======================================================================== +The following software have components provided under the terms of this license: + +- JSON in Java (from https://github.com/douglascrockford/JSON-java) + ======================================================================== LGPL-2.1-only ======================================================================== @@ -1023,6 +1032,7 @@ The following software have components provided under the terms of this license: - Animal Sniffer Annotations (from ) - Azure AD Spring Security Integration Spring Boot Starter (from https://github.com/Microsoft/azure-spring-boot) - Azure Java Client Authentication Library for AutoRest (from https://github.com/Azure/autorest-clientruntime-for-java) +- Azure Java Client Runtime for ARM (from https://github.com/Azure/autorest-clientruntime-for-java) - Azure Java Client Runtime for AutoRest (from https://github.com/Azure/autorest-clientruntime-for-java) - Azure Metrics Spring Boot Starter (from https://github.com/Microsoft/azure-spring-boot) - Azure Spring Boot AutoConfigure (from https://github.com/Microsoft/azure-spring-boot) @@ -1045,18 +1055,20 @@ The following software have components provided under the terms of this license: - Microsoft Azure Java Core Library (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure Netty HTTP Client Library (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure SDK annotations (from https://github.com/Microsoft/java-api-annotations) +- Microsoft Azure SDK for EventGrid Management (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure SDK for Service Bus (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure SDK for eventgrid (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure client library for Blob Storage (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure client library for Identity (from https://github.com/Azure/azure-sdk-for-java) +- Microsoft Azure client library for KeyVault Keys (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure client library for KeyVault Secrets (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure common module for Storage (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure internal Avro module for Storage (from https://github.com/Azure/azure-sdk-for-java) +- Mockito (from http://mockito.org) - Mockito (from http://www.mockito.org) - Mockito (from http://www.mockito.org) - Mockito (from http://mockito.org) -- Mockito (from http://mockito.org) - Netty/Codec/HTTP (from ) - Netty/Codec/HTTP (from ) - Netty/Common (from ) @@ -1068,6 +1080,8 @@ The following software have components provided under the terms of this license: - Spongy Castle (from http://rtyley.github.io/spongycastle/) - Spring Data for Azure Cosmos DB SQL API (from https://github.com/Azure/azure-sdk-for-java/tree/master/sdk/cosmos/azure-spring-data-cosmos) - adal4j (from https://github.com/AzureAD/azure-activedirectory-library-for-java) +- azure-documentdb (from https://azure.microsoft.com/en-us/services/cosmos-db/) +- documentdb-bulkexecutor (from http://azure.microsoft.com/en-us/services/documentdb/) - micrometer-core (from https://github.com/micrometer-metrics/micrometer) - mockito-junit-jupiter (from https://github.com/mockito/mockito) - msal4j (from https://github.com/AzureAD/microsoft-authentication-library-for-java) @@ -1142,6 +1156,7 @@ The following software have components provided under the terms of this license: - Joda-Time (from http://www.joda.org/joda-time/) - LatencyUtils (from http://latencyutils.github.io/LatencyUtils/) - Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java) +- Microsoft Azure SDK for EventGrid Management (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure client library for Blob Storage (from https://github.com/Azure/azure-sdk-for-java) - Project Lombok (from https://projectlombok.org) @@ -1149,6 +1164,7 @@ The following software have components provided under the terms of this license: - Spring Web (from https://github.com/spring-projects/spring-framework) - Spring Web (from https://github.com/spring-projects/spring-framework) - StAX API (from http://stax.codehaus.org/) +- azure-documentdb (from https://azure.microsoft.com/en-us/services/cosmos-db/) - msal4j (from https://github.com/AzureAD/microsoft-authentication-library-for-java) - reactive-streams (from http://www.reactive-streams.org/) diff --git a/devops/azure-pipeline.yml b/devops/azure-pipeline.yml deleted file mode 100644 index 8f96c7ac..00000000 --- a/devops/azure-pipeline.yml +++ /dev/null @@ -1,62 +0,0 @@ -##################### -# README: Defines a template to be used as a starting point for defining a service pipeline -##################### - -trigger: - batch: true - branches: - include: - - master - paths: - exclude: - - /**/*.md - - .gitignore - - images/ - -pr: - autoCancel: true - branches: - include: - - '*' - paths: - exclude: - - /**/*.md - - .gitignore - - images/ - -resources: - repositories: - - repository: osdu-infrastructure - type: git - name: osdu-r2/osdu-infrastructure - -variables: - - group: 'Azure Common Secrets' - - group: 'Azure - Common' - - - name: serviceName - value: 'storage' - -stages: - - template: devops/service-pipelines/build-stage.yml@osdu-infrastructure - parameters: - mavenGoal: 'package' - mavenPublishJUnitResults: true - mavenOptions: '-P storage-azure' - copyFileContents: | - pom.xml - provider/storage-azure/maven/settings.xml - provider/storage-azure/pom.xml - provider/storage-azure/target/*-spring-boot.jar - copyFileContentsToFlatten: '' - mavenSettingsFile: './maven/settings.xml' - serviceBase: ${{ variables.serviceName }} - testingRootFolder: 'testing' - - template: devops/service-pipelines/deploy-stages.yml@osdu-infrastructure - parameters: - serviceName: ${{ variables.serviceName }} - testCoreMavenPomFile: 'testing/storage-test-core/pom.xml' - testCoreMavenOptions: '--settings $(System.DefaultWorkingDirectory)/drop/deploy/testing/maven/settings.xml' - providers: - - name: Azure - environments: ['demo'] diff --git a/devops/development-pipeline.yml b/devops/development-pipeline.yml deleted file mode 100644 index bfaf59d0..00000000 --- a/devops/development-pipeline.yml +++ /dev/null @@ -1,62 +0,0 @@ -##################### -# README: Defines a template to be used as a starting point for defining a service pipeline -##################### - -trigger: - batch: true - branches: - include: - - master - paths: - exclude: - - /**/*.md - - .gitignore - - images/ - -pr: - autoCancel: true - branches: - include: - - "*" - paths: - exclude: - - /**/*.md - - .gitignore - - images/ - -resources: - repositories: - - repository: osdu-infrastructure - type: git - name: osdu-r2/osdu-infrastructure - -variables: - - group: "Azure Common Secrets" - - group: "Azure - Common" - - - name: serviceName - value: "storage" - -stages: - - template: devops/service-pipelines/build-stage.yml@osdu-infrastructure - parameters: - mavenGoal: "package" - mavenPublishJUnitResults: true - mavenOptions: "-P storage-azure" - copyFileContents: | - pom.xml - provider/storage-azure/maven/settings.xml - provider/storage-azure/pom.xml - provider/storage-azure/target/*-spring-boot.jar - copyFileContentsToFlatten: "" - mavenSettingsFile: "./maven/settings.xml" - serviceBase: ${{ variables.serviceName }} - testingRootFolder: "testing" - - template: devops/service-pipelines/deploy-stages.yml@osdu-infrastructure - parameters: - serviceName: ${{ variables.serviceName }} - testCoreMavenPomFile: "testing/storage-test-core/pom.xml" - testCoreMavenOptions: "--settings $(System.DefaultWorkingDirectory)/drop/deploy/testing/maven/settings.xml" - providers: - - name: Azure - environments: ["test"] diff --git a/devops/service-pipeline.yml b/devops/service-pipeline.yml deleted file mode 100644 index 1ace89be..00000000 --- a/devops/service-pipeline.yml +++ /dev/null @@ -1,77 +0,0 @@ -##################### -# README: Defines a template to be used as a starting point for defining a service pipeline -##################### - -trigger: - batch: true - branches: - include: - - master - paths: - exclude: - - /**/*.md - - .gitignore - - images/ - -pr: - autoCancel: true - branches: - include: - - '*' - exclude: - - master - paths: - exclude: - - /**/*.md - - .gitignore - - images/ - -resources: - repositories: - - repository: infrastructure-templates - type: git - name: open-data-ecosystem/infrastructure-templates - -variables: - - group: 'Azure Common Secrets' - - group: 'Azure - Common' - - - name: serviceName - value: 'storage' - -stages: - - template: devops/service-pipelines/build-stage.yml@infrastructure-templates - parameters: - mavenPublishJUnitResults: true - mavenOptions: '--settings ./storage-core/maven/settings.xml -DVSTS_FEED_TOKEN=$(VSTS_FEED_TOKEN)' - copyFileContents: | - pom.xml - provider/storage-azure/maven/settings.xml - provider/storage-azure/pom.xml - provider/storage-azure/target/*-spring-boot.jar - provider/storage-aws/CloudFormation - provider/storage-aws/Dockerfile - provider/storage-aws/buildspec-post-deploy.yml - provider/storage-aws/buildspec-pre-deploy.yml - provider/storage-aws/target/*-spring-boot.jar - copyFileContentsToFlatten: | - provider/storage-byoc/target/*-spring-boot.jar - provider/storage-gcp/target/*-spring-boot.jar - provider/storage-gcp/src/main/appengine/app.yaml - provider/storage-gcp/src/main/resources/application.properties - provider/storage-gcp/scripts/deploy.sh - serviceBase: ${{ variables.serviceName }} - testingRootFolder: 'testing' - - template: devops/service-pipelines/deploy-stages.yml@infrastructure-templates - parameters: - serviceName: ${{ variables.serviceName }} - testCoreMavenPomFile: 'testing/storage-test-core/pom.xml' - testCoreMavenOptions: '--settings $(System.DefaultWorkingDirectory)/drop/deploy/testing/maven/settings.xml -DVSTS_FEED_TOKEN=$(VSTS_FEED_TOKEN) ' - providers: - - name: Azure - # Merges into Master - ${{ if eq(variables['Build.SourceBranchName'], 'master') }}: - environments: ['devint', 'qa', 'prod'] - # PR updates / creations - ${{ if ne(variables['Build.SourceBranchName'], 'master') }}: - environments: ['devint'] diff --git a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/CloudStorageImpl.java b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/CloudStorageImpl.java index 0cdeffb5..c8f12a34 100644 --- a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/CloudStorageImpl.java +++ b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/CloudStorageImpl.java @@ -81,10 +81,7 @@ public class CloudStorageImpl implements ICloudStorage { @Override public void write(RecordProcessing... recordsProcessing) { - long startTime = System.currentTimeMillis(); - - validateRecordAcls(recordsProcessing); List> tasks = new ArrayList<>(); @@ -105,8 +102,7 @@ public class CloudStorageImpl implements ICloudStorage { throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Error during record ingestion", "An unexpected error on writing the record has occurred", e); } - long endTime = System.currentTimeMillis(); - logger.info("TIMING: Wrote " + tasks.size() + " records to blob storage in " + (endTime - startTime) + " ms"); + logger.info("TIMING: Wrote " + tasks.size() + " records to blob storage in " + (System.currentTimeMillis() - startTime) + " ms"); } @Override diff --git a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java index 5ee6e3c6..295e4287 100644 --- a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java +++ b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java @@ -167,8 +167,7 @@ public class RecordMetadataRepository extends SimpleCosmosStoreRepository Date: Mon, 18 Jan 2021 20:51:25 -0800 Subject: [PATCH 13/18] restoring replica count to 10 --- devops/azure/chart/helm-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devops/azure/chart/helm-config.yaml b/devops/azure/chart/helm-config.yaml index 85434607..aa0d121b 100644 --- a/devops/azure/chart/helm-config.yaml +++ b/devops/azure/chart/helm-config.yaml @@ -2,7 +2,7 @@ global: # Service(s) Replica Count - replicaCount: 1 + replicaCount: 10 ################################################################################ # Specify the Gitlab branch being used for image creation -- GitLab From 4f3ee6a68090c7bfeeba3a65fd859c479e00261e Mon Sep 17 00:00:00 2001 From: jsangiamo <69694815+jsangiamo@users.noreply.github.com> Date: Tue, 19 Jan 2021 10:52:24 -0800 Subject: [PATCH 14/18] updating README with new environment variables --- provider/storage-azure/README.md | 5 +++++ .../osdu/storage/provider/azure/CloudStorageImpl.java | 2 -- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/provider/storage-azure/README.md b/provider/storage-azure/README.md index 1c37a4c2..8302b45b 100644 --- a/provider/storage-azure/README.md +++ b/provider/storage-azure/README.md @@ -59,6 +59,11 @@ az keyvault secret show --vault-name $KEY_VAULT_NAME --name $KEY_VAULT_SECRET_NA | `AZURE_TENANT_ID` | `********` | AD tenant to authenticate users from | yes | keyvault secret: `$KEYVAULT_URI/secrets/app-dev-sp-tenant-id` | | `AZURE_CLIENT_SECRET` | `********` | Secret for `$AZURE_CLIENT_ID` | yes | keyvault secret: `$KEYVAULT_URI/secrets/app-dev-sp-password` | | `azure_istioauth_enabled` | `true` | Flag to Disable AAD auth | no | -- | +| `MIN_BATCH_SIZE_TO_USE_BULK_UPLOAD` | `50` | Minimum batch size to upload the records in bulk | no | Recommended to set to 50, but can be changed for specific use cases. NOTE: If not set, this will default to 50.| +| `BULK_IMPORT_MAX_CONCURRENCY_PER_PARTITION_RANGE` | `20` | Bulk uploader concurrency for each Cosmos data partition range | no | Recommended to set to 20, but can be changed for specific use cases. NOTE: If not set, this will default to 20.| +| `BULK_EXECUTOR_MAX_RUS` | `4000` | Maximum RU Consumption for bulk uploads in each storage service pod. NOTE: This is an attempted maximum and it may exceed this number. | no | Recommended to set to 4000, but can be changed for specific use cases. NOTE: If not set, this will default to 4000.| +| `DOCUMENT_CLIENT_MAX_POOL_SIZE` | See description | Connection pool size for bulk upload http client. Recommended to not set and allow for default value which is calculated based on the number of available processors. | no | Recommended to not set manually, but can be changed for specific use cases. NOTE: If not set, this will default to 100 * number of available processors.| + **Required to run integration tests** diff --git a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/CloudStorageImpl.java b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/CloudStorageImpl.java index c8f12a34..1a33a2be 100644 --- a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/CloudStorageImpl.java +++ b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/CloudStorageImpl.java @@ -77,8 +77,6 @@ public class CloudStorageImpl implements ICloudStorage { @Named("STORAGE_CONTAINER_NAME") private String containerName; - - @Override public void write(RecordProcessing... recordsProcessing) { long startTime = System.currentTimeMillis(); -- GitLab From 9bd4bf7cc9df2c3455190b6f802b9ba5426631a1 Mon Sep 17 00:00:00 2001 From: jsangiamo <69694815+jsangiamo@users.noreply.github.com> Date: Tue, 19 Jan 2021 11:08:45 -0800 Subject: [PATCH 15/18] removing log statements --- .../osdu/storage/provider/azure/CloudStorageImpl.java | 2 -- .../provider/azure/repository/RecordMetadataRepository.java | 4 ---- 2 files changed, 6 deletions(-) diff --git a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/CloudStorageImpl.java b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/CloudStorageImpl.java index 1a33a2be..2d4529d1 100644 --- a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/CloudStorageImpl.java +++ b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/CloudStorageImpl.java @@ -79,7 +79,6 @@ public class CloudStorageImpl implements ICloudStorage { @Override public void write(RecordProcessing... recordsProcessing) { - long startTime = System.currentTimeMillis(); validateRecordAcls(recordsProcessing); List> tasks = new ArrayList<>(); @@ -100,7 +99,6 @@ public class CloudStorageImpl implements ICloudStorage { throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Error during record ingestion", "An unexpected error on writing the record has occurred", e); } - logger.info("TIMING: Wrote " + tasks.size() + " records to blob storage in " + (System.currentTimeMillis() - startTime) + " ms"); } @Override diff --git a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java index 295e4287..b05fb0ff 100644 --- a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java +++ b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java @@ -70,12 +70,10 @@ public class RecordMetadataRepository extends SimpleCosmosStoreRepository createOrUpdate(List recordsMetadata) { Assert.notNull(recordsMetadata, "recordsMetadata must not be null"); - long startTime = System.currentTimeMillis(); if(recordsMetadata.size() >= minBatchSizeToUseBulkUpload) createOrUpdateParallel(recordsMetadata); else createOrUpdateSerial(recordsMetadata); - logger.info("TIMING: createOrUpdate method took ms " + (System.currentTimeMillis() - startTime)); return recordsMetadata; } @@ -155,7 +153,6 @@ public class RecordMetadataRepository extends SimpleCosmosStoreRepository get(List ids) { - long startTime = System.currentTimeMillis(); String sqlQueryString = createCosmosBatchGetQueryById(ids); SqlQuerySpec query = new SqlQuerySpec(sqlQueryString); CosmosQueryRequestOptions options = new CosmosQueryRequestOptions(); @@ -167,7 +164,6 @@ public class RecordMetadataRepository extends SimpleCosmosStoreRepository Date: Tue, 19 Jan 2021 11:12:16 -0800 Subject: [PATCH 16/18] switched to throwing 500 if there is an error rather than if the number of documents imported != number of documents in the request --- .../provider/azure/repository/SimpleCosmosStoreRepository.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/SimpleCosmosStoreRepository.java b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/SimpleCosmosStoreRepository.java index 0f65924f..632fff0c 100644 --- a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/SimpleCosmosStoreRepository.java +++ b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/SimpleCosmosStoreRepository.java @@ -293,7 +293,7 @@ public class SimpleCosmosStoreRepository implements CosmosStoreRepository Assert.notNull(cosmosDBName, "cosmosDBName should not be null"); Assert.notNull(collectionName, "collectionName should not be null"); BulkImportResponse response = cosmosBulkStore.bulkInsert(dataPartitionId, cosmosDBName, collectionName, docs, true, true, bulkImportMaxConcurrencyPePartitionRange); - if(response.getNumberOfDocumentsImported() != docs.size()){ + if(!response.getErrors().isEmpty()){ List exceptions = new ArrayList<>(); for(Exception e : response.getErrors()){ exceptions.add(e.toString()); -- GitLab From 580b52358ded32bdf2188da6033dcb35d33dd5a2 Mon Sep 17 00:00:00 2001 From: jsangiamo <69694815+jsangiamo@users.noreply.github.com> Date: Tue, 19 Jan 2021 12:05:10 -0800 Subject: [PATCH 17/18] fixing environment variable references to use environment variables rather than java properties --- .../storage/provider/azure/di/AzureBootstrapConfig.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/di/AzureBootstrapConfig.java b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/di/AzureBootstrapConfig.java index 024077b0..2e309122 100644 --- a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/di/AzureBootstrapConfig.java +++ b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/di/AzureBootstrapConfig.java @@ -93,7 +93,8 @@ public class AzureBootstrapConfig { */ @Bean public int minBatchSizeToUseBulkUpload() { - return Integer.parseInt(System.getProperty("MIN_BATCH_SIZE_TO_USE_BULK_UPLOAD", "50")); + if(System.getenv("MIN_BATCH_SIZE_TO_USE_BULK_UPLOAD") == null) return 50; + else return Integer.parseInt(System.getenv("MIN_BATCH_SIZE_TO_USE_BULK_UPLOAD")); } /** @@ -101,6 +102,7 @@ public class AzureBootstrapConfig { */ @Bean public int bulkImportMaxConcurrencyPePartitionRange(){ - return Integer.parseInt(System.getProperty("BULK_IMPORT_MAX_CONCURRENCY_PER_PARTITION_RANGE", "20")); + if(System.getenv("BULK_IMPORT_MAX_CONCURRENCY_PER_PARTITION_RANGE") == null) return 20; + else return Integer.parseInt(System.getenv("BULK_IMPORT_MAX_CONCURRENCY_PER_PARTITION_RANGE")); } } -- GitLab From a9f6e94b69786315850372b3551a3969e4ca4631 Mon Sep 17 00:00:00 2001 From: jsangiamo <69694815+jsangiamo@users.noreply.github.com> Date: Tue, 19 Jan 2021 19:15:48 -0800 Subject: [PATCH 18/18] removing hardcoded cosmos collection name --- .../provider/azure/repository/RecordMetadataRepository.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java index b05fb0ff..22fe8dcc 100644 --- a/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java +++ b/provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/RecordMetadataRepository.java @@ -102,7 +102,7 @@ public class RecordMetadataRepository extends SimpleCosmosStoreRepository