diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java index 0a6f6ba7a14c54fbf7b34f5e4acadf755a75109a..4f6a1f6669731c9a3602c3e73e79dc935387359d 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java @@ -54,4 +54,11 @@ public class ReindexApi { this.auditLogger.getReindex(singletonList(recordReindexRequest.getKind())); return new ResponseEntity<>(org.springframework.http.HttpStatus.OK); } + + @PreAuthorize("@authorizationFilter.hasPermission('" + SearchServiceRole.ADMIN + "')") + @PatchMapping + public ResponseEntity<String> fullReindex(@RequestParam(value = "force_clean", defaultValue = "false") boolean forceClean) throws IOException { + this.reIndexService.fullReindex(forceClean); + return new ResponseEntity<>(org.springframework.http.HttpStatus.OK); + } } diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/config/IndexerConfigurationProperties.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/config/IndexerConfigurationProperties.java index 13ed75f28a0ba814cd3af9b01e3a166a5b05ae7d..1123e5ce21712f4ceac5a47cc78b322f4ecca5a9 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/config/IndexerConfigurationProperties.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/config/IndexerConfigurationProperties.java @@ -43,6 +43,7 @@ public class IndexerConfigurationProperties { private String environment; private String indexerHost; private String searchHost; + private String storageQueryKindsHost; private String storageQueryRecordForConversionHost; private String storageQueryRecordHost; private Integer storageRecordsBatchSize; diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexService.java index 39486f1c7498260e7eda5fd83b9de379f79b1d4f..6569743e49fd39a381eec56123e14ea7b96c7232 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexService.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexService.java @@ -20,4 +20,6 @@ import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest; public interface ReindexService { String reindexRecords(RecordReindexRequest recordReindexRequest, boolean forceClean); + + void fullReindex(boolean forceClean); } \ No newline at end of file diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java index 013fcb6928d9f3a7a8d8ea27123b31bdd93d3832..0f3d5c4f18db650813ca096878925dc49a153921 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java @@ -16,6 +16,7 @@ package org.opengroup.osdu.indexer.service; import com.google.common.base.Strings; import com.google.gson.Gson; +import java.util.Objects; import org.apache.http.HttpStatus; import org.opengroup.osdu.core.common.model.http.DpsHeaders; import org.opengroup.osdu.core.common.model.http.AppException; @@ -100,4 +101,26 @@ public class ReindexServiceImpl implements ReindexService { throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Unknown error", "An unknown error has occurred.", e); } } + + @Override + public void fullReindex(boolean forceClean) { + List<String> allKinds = null; + try { + allKinds = storageService.getAllKinds(); + } catch (Exception e) { + jaxRsDpsLog.error("storage service all kinds request failed",e); + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "storage service cannot respond with all kinds", "an unknown error has occurred.", e); + } + if (Objects.isNull(allKinds)){ + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "storage service cannot respond with all kinds", "full reindex failed"); + } + for (String kind : allKinds) { + try { + reindexRecords(new RecordReindexRequest(kind, ""), forceClean); + } catch (Exception e) { + jaxRsDpsLog.warning(String.format("kind: %s cannot be re-indexed", kind)); + continue; + } + } + } } \ No newline at end of file diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageService.java index ab8e9901294893255b0f00a7632ee0145c37f1fa..bfbb5c6d5a0faf2cf1b62ec031697141f0f649ef 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageService.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageService.java @@ -30,4 +30,6 @@ public interface StorageService { RecordQueryResponse getRecordsByKind(RecordReindexRequest request) throws URISyntaxException; String getStorageSchema(String kind) throws URISyntaxException, UnsupportedEncodingException; + + List<String> getAllKinds() throws URISyntaxException; } \ No newline at end of file diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java index b8a3bf8aabbc4340309cbdaeb5cd29db22c60da1..a390e617ea31b9bb9f399deed3b9cc2b921ea48d 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java @@ -20,6 +20,9 @@ import com.google.common.collect.Lists; import com.google.common.reflect.TypeToken; import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import org.opengroup.osdu.core.common.http.FetchServiceHttpRequest; import org.opengroup.osdu.core.common.model.http.DpsHeaders; import org.opengroup.osdu.core.common.model.http.AppException; @@ -209,4 +212,18 @@ public class StorageServiceImpl implements StorageService { HttpResponse response = this.urlFetchService.sendRequest(request); return response.getResponseCode() != HttpStatus.SC_OK ? null : response.getBody(); } + + @Override + public List<String> getAllKinds() throws URISyntaxException { + String url = configurationProperties.getStorageQueryKindsHost(); + FetchServiceHttpRequest request = FetchServiceHttpRequest.builder() + .httpMethod(HttpMethods.GET) + .headers(this.requestInfo.getHeadersMap()) + .url(url) + .build(); + HttpResponse response = this.urlFetchService.sendRequest(request); + JsonObject asJsonObject = new JsonParser().parse(response.getBody()).getAsJsonObject(); + JsonElement results = asJsonObject.get("results"); + return response.getResponseCode() != HttpStatus.SC_OK ? null : this.gson.fromJson(results,List.class); + } } \ No newline at end of file diff --git a/provider/indexer-aws/src/main/resources/application.properties b/provider/indexer-aws/src/main/resources/application.properties index 77f80a34f9d23307e59202daf13fd6f047bd5ace..d29cc7a7661161f50377c9577cb7c4f418771c70 100644 --- a/provider/indexer-aws/src/main/resources/application.properties +++ b/provider/indexer-aws/src/main/resources/application.properties @@ -27,6 +27,7 @@ SCHEMA_HOST=${STORAGE_HOST}/api/schema-service/v1/schema STORAGE_SCHEMA_HOST=${STORAGE_HOST}/api/storage/v2/schemas STORAGE_QUERY_RECORD_HOST=${STORAGE_HOST}/api/storage/v2/query/records +STORAGE_QUERY_KINDS_HOST=${STORAGE_HOST}/api/storage/v2/query/kinds STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=${STORAGE_HOST}/api/storage/v2/query/records:batch STORAGE_RECORDS_BATCH_SIZE=20 INDEXER_QUEUE_HOST="" diff --git a/provider/indexer-azure/src/main/resources/application.properties b/provider/indexer-azure/src/main/resources/application.properties index a334813afb2b5cccfe9127b0cf95a9730f9e4f81..619120f5eeab512b3e0f3316e27156e70b6b9aef 100644 --- a/provider/indexer-azure/src/main/resources/application.properties +++ b/provider/indexer-azure/src/main/resources/application.properties @@ -41,6 +41,7 @@ SCHEMA_HOST=${schema_service_url}/schema storage_service_url=${storage_service_endpoint} STORAGE_SCHEMA_HOST=${storage_service_url}/schemas STORAGE_QUERY_RECORD_HOST=${storage_service_url}/query/records +STORAGE_QUERY_KINDS_HOST=${storage_service_url}/query/kinds STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=${storage_service_url}/query/records:batch STORAGE_RECORDS_BATCH_SIZE=20 diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/AppExceptionHandler.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/AppExceptionHandler.java new file mode 100644 index 0000000000000000000000000000000000000000..0e8293b1a59abd37ebdcd59bf3f2610a7c39e813 --- /dev/null +++ b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/AppExceptionHandler.java @@ -0,0 +1,34 @@ +package org.opengroup.osdu.indexer.util; + +import java.util.Objects; +import lombok.extern.slf4j.Slf4j; +import org.opengroup.osdu.core.common.model.http.AppException; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.ControllerAdvice; +import org.springframework.web.bind.annotation.ExceptionHandler; + +@ControllerAdvice +@Slf4j +public class AppExceptionHandler { + + @ExceptionHandler(AppException.class) + public ResponseEntity<Object> handleAppExceptions(AppException e) { + return this.getErrorResponse(e); + } + + private ResponseEntity<Object> getErrorResponse(AppException e) { + + String exceptionMsg = Objects.nonNull(e.getOriginalException()) + ? e.getOriginalException().getMessage() + : e.getError().getMessage(); + + if (e.getError().getCode() > 499) { + log.error(exceptionMsg, e.getOriginalException()); + } else { + log.warn(exceptionMsg, e.getOriginalException()); + } + + return new ResponseEntity<>(e.getError(), HttpStatus.resolve(e.getError().getCode())); + } +} diff --git a/provider/indexer-gcp/src/main/resources/application.properties b/provider/indexer-gcp/src/main/resources/application.properties index 06ba9d9417769af49ad5b9d364fa5cdecbb398da..b3d58b0c9b895a6ca38e8b29143edd258e7d8c8b 100644 --- a/provider/indexer-gcp/src/main/resources/application.properties +++ b/provider/indexer-gcp/src/main/resources/application.properties @@ -40,5 +40,6 @@ elastic-datastore-id=indexer-service security.https.certificate.trust=false indexer.que.service.mail=default@iam.gserviceaccount.com - SCHEMA_HOST=${HOST}/api/schema-service/v1/schema +storage-query-kinds-host=https://${STORAGE_HOSTNAME}/api/storage/v2/query/kinds + diff --git a/provider/indexer-ibm/src/main/resources/application.properties b/provider/indexer-ibm/src/main/resources/application.properties index def41568078ca252d54986a5eebde71a2fbc62d8..38a394adc7fc3c8230acc4a554212832c3e8861f 100644 --- a/provider/indexer-ibm/src/main/resources/application.properties +++ b/provider/indexer-ibm/src/main/resources/application.properties @@ -32,6 +32,7 @@ storage_service_url=http://localhost:8082 #storage_service_url=https://os-storage-ibm-osdu-r2.osduadev-a1c3eaf78a86806e299f5f3f207556f0-0000.us-south.containers.appdomain.cloud STORAGE_SCHEMA_HOST=${storage_service_url}/api/storage/v2/schemas STORAGE_QUERY_RECORD_HOST=${storage_service_url}/api/storage/v2/query/records +STORAGE_QUERY_KINDS_HOST=${storage_service_url}/api/storage/v2/query/kinds STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=${storage_service_url}/api/storage/v2/query/records:batch STORAGE_RECORDS_BATCH_SIZE=20