From f8e6e8ebee82988b7ac94d8af766e4fa26025ab7 Mon Sep 17 00:00:00 2001
From: Vibhuti Sharma <vibsharm@microsoft.com>
Date: Mon, 24 Jan 2022 15:18:06 +0530
Subject: [PATCH] Addressing MR comments
---
.../indexer/config/IndexerConfigurationProperties.java | 8 ++++----
.../osdu/indexer/service/ReindexServiceImpl.java | 2 +-
.../osdu/indexer/service/StorageServiceImpl.java | 4 +---
provider/indexer-azure/README.md | 2 ++
.../indexer/azure/util/IndexerQueueTaskBuilderAzure.java | 2 +-
.../src/main/resources/application.properties | 2 +-
6 files changed, 10 insertions(+), 10 deletions(-)
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/config/IndexerConfigurationProperties.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/config/IndexerConfigurationProperties.java
index 86dd9ecfa..cff663d6c 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/config/IndexerConfigurationProperties.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/config/IndexerConfigurationProperties.java
@@ -47,7 +47,7 @@ public class IndexerConfigurationProperties {
private String storageQueryRecordForConversionHost;
private String storageQueryRecordHost;
private Integer storageRecordsBatchSize;
- private Integer storageGetRecordsByKindBatchSize;
+ private Integer storageRecordsByKindBatchSize;
private String storageSchemaHost;
private String schemaHost;
private String entitlementsHost;
@@ -124,9 +124,9 @@ public class IndexerConfigurationProperties {
return Boolean.TRUE.toString().equalsIgnoreCase(smartSearchCcsDisabled);
}
- public Integer getStorageGetRecordsByKindBatchSize () {
- if (this.storageGetRecordsByKindBatchSize!=null) {
- return this.storageGetRecordsByKindBatchSize;
+ public Integer getStorageRecordsByKindBatchSize() {
+ if (this.storageRecordsByKindBatchSize != null) {
+ return this.storageRecordsByKindBatchSize;
}
// if property is not set, fall back to storageRecordsBatchSize property which is used by all CSPs to set batch size.
return this.storageRecordsBatchSize;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java
index d8fc6e3ce..af15d671f 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java
@@ -88,7 +88,7 @@ public class ReindexServiceImpl implements ReindexService {
// don't call reindex-worker endpoint if it's the last batch
// previous storage query result size will be less then requested (limit param)
- if (!Strings.isNullOrEmpty(recordQueryResponse.getCursor()) && recordQueryResponse.getResults().size() == configurationProperties.getStorageGetRecordsByKindBatchSize()) {
+ if (!Strings.isNullOrEmpty(recordQueryResponse.getCursor()) && recordQueryResponse.getResults().size() == configurationProperties.getStorageRecordsByKindBatchSize()) {
String newPayLoad = gson.toJson(RecordReindexRequest.builder().cursor(recordQueryResponse.getCursor()).kind(recordReindexRequest.getKind()).build());
this.indexerQueueTaskBuilder.createReIndexTask(newPayLoad, initialDelayMillis, headers);
return newPayLoad;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java
index 1135c6ccc..95d88bc77 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java
@@ -19,7 +19,6 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.api.client.http.HttpMethods;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
-import com.google.common.reflect.TypeToken;
import com.google.gson.Gson;
import com.google.gson.JsonElement;
@@ -43,7 +42,6 @@ import org.springframework.stereotype.Component;
import javax.inject.Inject;
import java.io.UnsupportedEncodingException;
-import java.lang.reflect.Type;
import java.net.URISyntaxException;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
@@ -189,7 +187,7 @@ public class StorageServiceImpl implements StorageService {
public RecordQueryResponse getRecordsByKind(RecordReindexRequest reindexRequest) throws URISyntaxException {
Map<String, String> queryParams = new HashMap<>();
queryParams.put(RecordMetaAttribute.KIND.getValue(), reindexRequest.getKind());
- queryParams.put("limit", configurationProperties.getStorageGetRecordsByKindBatchSize().toString());
+ queryParams.put("limit", configurationProperties.getStorageRecordsByKindBatchSize().toString());
if (!Strings.isNullOrEmpty(reindexRequest.getCursor())) {
queryParams.put("cursor", reindexRequest.getCursor());
}
diff --git a/provider/indexer-azure/README.md b/provider/indexer-azure/README.md
index bc6a63ec3..9f120e450 100644
--- a/provider/indexer-azure/README.md
+++ b/provider/indexer-azure/README.md
@@ -47,6 +47,8 @@ az keyvault secret show --vault-name $KEY_VAULT_NAME --name $KEY_VAULT_SECRET_NA
| `STORAGE_SCHEMA_HOST` | `${storage_service_url}/schemas` | Endpoint of schema API | no | - |
| `STORAGE_QUERY_RECORD_HOST` | `${storage_service_url}/query/records` | Endpoint of records API | no | - |
| `STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST` | `${storage_service_url}/query/records:batch` | Endpoint of records batch API | no | - |
+| `STORAGE_RECORDS_BATCH_SIZE` | 20 | Batch size for storage API `POST {endpoint}/query/records:batch` | no | - |
+| `STORAGE_RECORDS_BY_KIND_BATCH_SIZE` | 100 | Batch size for storage API `GET {endpoint}/query/records`. If this is not present, defaults to value of `STORAGE_RECORDS_BATCH_SIZE` | no | - |
| `KEYVAULT_URI` | ex `https://foo-kv.vault.azure.net/` | . | . | . |
| `appinsights_key` | `********` | App Insights key | yes | output of infrastructure deployments |
| `aad_client_id` | `********` | AAD client application ID | yes | output of infrastructure deployment |
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java
index 6d94cadda..6ee0289c6 100644
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java
+++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java
@@ -128,7 +128,7 @@ public class IndexerQueueTaskBuilderAzure extends IndexerQueueTaskBuilder {
String recordChangedMessagePayload = gson.toJson(recordChangedMessages);
createTask(recordChangedMessagePayload, headers);
}
- } while (!Strings.isNullOrEmpty(recordQueryResponse.getCursor()) && recordQueryResponse.getResults().size() == configurationProperties.getStorageGetRecordsByKindBatchSize());
+ } while (!Strings.isNullOrEmpty(recordQueryResponse.getCursor()) && recordQueryResponse.getResults().size() == configurationProperties.getStorageRecordsByKindBatchSize());
} catch (AppException e) {
throw e;
diff --git a/provider/indexer-azure/src/main/resources/application.properties b/provider/indexer-azure/src/main/resources/application.properties
index 4cfe826e6..8f479afcd 100644
--- a/provider/indexer-azure/src/main/resources/application.properties
+++ b/provider/indexer-azure/src/main/resources/application.properties
@@ -46,7 +46,7 @@ STORAGE_QUERY_RECORD_HOST=${storage_service_url}/query/records
STORAGE_QUERY_KINDS_HOST=${storage_service_url}/query/kinds
STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=${storage_service_url}/query/records:batch
STORAGE_RECORDS_BATCH_SIZE=20
-STORAGE_GET_RECORDS_BY_KIND_BATCH_SIZE=100
+STORAGE_RECORDS_BY_KIND_BATCH_SIZE=100
INDEXER_QUEUE_HOST=http://127.0.0.1:9000
--
GitLab