From 9df7ae6e27a7a784bd3dd13b8bb24ef97da9f67d Mon Sep 17 00:00:00 2001
From: Rustam_Lotsmanenko <Rustam_Lotsmanenko@epam.com>
Date: Thu, 21 Jan 2021 15:41:55 +0400
Subject: [PATCH] GONRG-1534 full-reindex Changes: *Added full reindex endpoint
*Added get all kinds method to StorageService *Added AppExceptionHandler to
GCP *Added property STORAGE_QUERY_KINDS_HOST
---
.../osdu/indexer/api/ReindexApi.java | 7 ++++
.../IndexerConfigurationProperties.java | 1 +
.../osdu/indexer/service/ReindexService.java | 2 ++
.../indexer/service/ReindexServiceImpl.java | 23 +++++++++++++
.../osdu/indexer/service/StorageService.java | 2 ++
.../indexer/service/StorageServiceImpl.java | 17 ++++++++++
.../src/main/resources/application.properties | 1 +
.../src/main/resources/application.properties | 1 +
.../indexer/util/AppExceptionHandler.java | 34 +++++++++++++++++++
.../src/main/resources/application.properties | 3 +-
.../src/main/resources/application.properties | 1 +
11 files changed, 91 insertions(+), 1 deletion(-)
create mode 100644 provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/AppExceptionHandler.java
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java
index 0a6f6ba7a..4f6a1f666 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java
@@ -54,4 +54,11 @@ public class ReindexApi {
this.auditLogger.getReindex(singletonList(recordReindexRequest.getKind()));
return new ResponseEntity<>(org.springframework.http.HttpStatus.OK);
}
+
+ @PreAuthorize("@authorizationFilter.hasPermission('" + SearchServiceRole.ADMIN + "')")
+ @PatchMapping
+ public ResponseEntity<String> fullReindex(@RequestParam(value = "force_clean", defaultValue = "false") boolean forceClean) throws IOException {
+ this.reIndexService.fullReindex(forceClean);
+ return new ResponseEntity<>(org.springframework.http.HttpStatus.OK);
+ }
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/config/IndexerConfigurationProperties.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/config/IndexerConfigurationProperties.java
index 13ed75f28..1123e5ce2 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/config/IndexerConfigurationProperties.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/config/IndexerConfigurationProperties.java
@@ -43,6 +43,7 @@ public class IndexerConfigurationProperties {
private String environment;
private String indexerHost;
private String searchHost;
+ private String storageQueryKindsHost;
private String storageQueryRecordForConversionHost;
private String storageQueryRecordHost;
private Integer storageRecordsBatchSize;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexService.java
index 39486f1c7..6569743e4 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexService.java
@@ -20,4 +20,6 @@ import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
public interface ReindexService {
String reindexRecords(RecordReindexRequest recordReindexRequest, boolean forceClean);
+
+ void fullReindex(boolean forceClean);
}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java
index 013fcb692..0f3d5c4f1 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java
@@ -16,6 +16,7 @@ package org.opengroup.osdu.indexer.service;
import com.google.common.base.Strings;
import com.google.gson.Gson;
+import java.util.Objects;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.http.AppException;
@@ -100,4 +101,26 @@ public class ReindexServiceImpl implements ReindexService {
throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Unknown error", "An unknown error has occurred.", e);
}
}
+
+ @Override
+ public void fullReindex(boolean forceClean) {
+ List<String> allKinds = null;
+ try {
+ allKinds = storageService.getAllKinds();
+ } catch (Exception e) {
+ jaxRsDpsLog.error("storage service all kinds request failed",e);
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "storage service cannot respond with all kinds", "an unknown error has occurred.", e);
+ }
+ if (Objects.isNull(allKinds)){
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "storage service cannot respond with all kinds", "full reindex failed");
+ }
+ for (String kind : allKinds) {
+ try {
+ reindexRecords(new RecordReindexRequest(kind, ""), forceClean);
+ } catch (Exception e) {
+ jaxRsDpsLog.warning(String.format("kind: %s cannot be re-indexed", kind));
+ continue;
+ }
+ }
+ }
}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageService.java
index ab8e99012..bfbb5c6d5 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageService.java
@@ -30,4 +30,6 @@ public interface StorageService {
RecordQueryResponse getRecordsByKind(RecordReindexRequest request) throws URISyntaxException;
String getStorageSchema(String kind) throws URISyntaxException, UnsupportedEncodingException;
+
+ List<String> getAllKinds() throws URISyntaxException;
}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java
index b8a3bf8aa..a390e617e 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java
@@ -20,6 +20,9 @@ import com.google.common.collect.Lists;
import com.google.common.reflect.TypeToken;
import com.google.gson.Gson;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
import org.opengroup.osdu.core.common.http.FetchServiceHttpRequest;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.http.AppException;
@@ -209,4 +212,18 @@ public class StorageServiceImpl implements StorageService {
HttpResponse response = this.urlFetchService.sendRequest(request);
return response.getResponseCode() != HttpStatus.SC_OK ? null : response.getBody();
}
+
+ @Override
+ public List<String> getAllKinds() throws URISyntaxException {
+ String url = configurationProperties.getStorageQueryKindsHost();
+ FetchServiceHttpRequest request = FetchServiceHttpRequest.builder()
+ .httpMethod(HttpMethods.GET)
+ .headers(this.requestInfo.getHeadersMap())
+ .url(url)
+ .build();
+ HttpResponse response = this.urlFetchService.sendRequest(request);
+ JsonObject asJsonObject = new JsonParser().parse(response.getBody()).getAsJsonObject();
+ JsonElement results = asJsonObject.get("results");
+ return response.getResponseCode() != HttpStatus.SC_OK ? null : this.gson.fromJson(results,List.class);
+ }
}
\ No newline at end of file
diff --git a/provider/indexer-aws/src/main/resources/application.properties b/provider/indexer-aws/src/main/resources/application.properties
index 77f80a34f..d29cc7a76 100644
--- a/provider/indexer-aws/src/main/resources/application.properties
+++ b/provider/indexer-aws/src/main/resources/application.properties
@@ -27,6 +27,7 @@ SCHEMA_HOST=${STORAGE_HOST}/api/schema-service/v1/schema
STORAGE_SCHEMA_HOST=${STORAGE_HOST}/api/storage/v2/schemas
STORAGE_QUERY_RECORD_HOST=${STORAGE_HOST}/api/storage/v2/query/records
+STORAGE_QUERY_KINDS_HOST=${STORAGE_HOST}/api/storage/v2/query/kinds
STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=${STORAGE_HOST}/api/storage/v2/query/records:batch
STORAGE_RECORDS_BATCH_SIZE=20
INDEXER_QUEUE_HOST=""
diff --git a/provider/indexer-azure/src/main/resources/application.properties b/provider/indexer-azure/src/main/resources/application.properties
index a334813af..619120f5e 100644
--- a/provider/indexer-azure/src/main/resources/application.properties
+++ b/provider/indexer-azure/src/main/resources/application.properties
@@ -41,6 +41,7 @@ SCHEMA_HOST=${schema_service_url}/schema
storage_service_url=${storage_service_endpoint}
STORAGE_SCHEMA_HOST=${storage_service_url}/schemas
STORAGE_QUERY_RECORD_HOST=${storage_service_url}/query/records
+STORAGE_QUERY_KINDS_HOST=${storage_service_url}/query/kinds
STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=${storage_service_url}/query/records:batch
STORAGE_RECORDS_BATCH_SIZE=20
diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/AppExceptionHandler.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/AppExceptionHandler.java
new file mode 100644
index 000000000..0e8293b1a
--- /dev/null
+++ b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/AppExceptionHandler.java
@@ -0,0 +1,34 @@
+package org.opengroup.osdu.indexer.util;
+
+import java.util.Objects;
+import lombok.extern.slf4j.Slf4j;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.ControllerAdvice;
+import org.springframework.web.bind.annotation.ExceptionHandler;
+
+@ControllerAdvice
+@Slf4j
+public class AppExceptionHandler {
+
+ @ExceptionHandler(AppException.class)
+ public ResponseEntity<Object> handleAppExceptions(AppException e) {
+ return this.getErrorResponse(e);
+ }
+
+ private ResponseEntity<Object> getErrorResponse(AppException e) {
+
+ String exceptionMsg = Objects.nonNull(e.getOriginalException())
+ ? e.getOriginalException().getMessage()
+ : e.getError().getMessage();
+
+ if (e.getError().getCode() > 499) {
+ log.error(exceptionMsg, e.getOriginalException());
+ } else {
+ log.warn(exceptionMsg, e.getOriginalException());
+ }
+
+ return new ResponseEntity<>(e.getError(), HttpStatus.resolve(e.getError().getCode()));
+ }
+}
diff --git a/provider/indexer-gcp/src/main/resources/application.properties b/provider/indexer-gcp/src/main/resources/application.properties
index 06ba9d941..55f828665 100644
--- a/provider/indexer-gcp/src/main/resources/application.properties
+++ b/provider/indexer-gcp/src/main/resources/application.properties
@@ -40,5 +40,6 @@ elastic-datastore-id=indexer-service
security.https.certificate.trust=false
indexer.que.service.mail=default@iam.gserviceaccount.com
+SCHEMA_HOST=https://os-schema-jvmvia5dea-uc.a.run.app/api/schema-service/v1/schema
+storage-query-kinds-host=https://${STORAGE_HOSTNAME}/api/storage/v2/query/kinds
-SCHEMA_HOST=${HOST}/api/schema-service/v1/schema
diff --git a/provider/indexer-ibm/src/main/resources/application.properties b/provider/indexer-ibm/src/main/resources/application.properties
index def415680..38a394adc 100644
--- a/provider/indexer-ibm/src/main/resources/application.properties
+++ b/provider/indexer-ibm/src/main/resources/application.properties
@@ -32,6 +32,7 @@ storage_service_url=http://localhost:8082
#storage_service_url=https://os-storage-ibm-osdu-r2.osduadev-a1c3eaf78a86806e299f5f3f207556f0-0000.us-south.containers.appdomain.cloud
STORAGE_SCHEMA_HOST=${storage_service_url}/api/storage/v2/schemas
STORAGE_QUERY_RECORD_HOST=${storage_service_url}/api/storage/v2/query/records
+STORAGE_QUERY_KINDS_HOST=${storage_service_url}/api/storage/v2/query/kinds
STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=${storage_service_url}/api/storage/v2/query/records:batch
STORAGE_RECORDS_BATCH_SIZE=20
--
GitLab