Merge branch 'gcp-full-reindex' into 'master'

Gcp full reindex (GONRG-1534)

See merge request !76

indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java

@@ -54,4 +54,11 @@ public class ReindexApi {
         this.auditLogger.getReindex(singletonList(recordReindexRequest.getKind()));
         return new ResponseEntity<>(org.springframework.http.HttpStatus.OK);
     }
+
+    @PreAuthorize("@authorizationFilter.hasPermission('" + SearchServiceRole.ADMIN + "')")
+    @PatchMapping
+    public ResponseEntity<String> fullReindex(@RequestParam(value = "force_clean", defaultValue = "false") boolean forceClean) throws IOException {
+        this.reIndexService.fullReindex(forceClean);
+        return new ResponseEntity<>(org.springframework.http.HttpStatus.OK);
+    }
 }

indexer-core/src/main/java/org/opengroup/osdu/indexer/config/IndexerConfigurationProperties.java

@@ -43,6 +43,7 @@ public class IndexerConfigurationProperties {
 	private String environment;
 	private String indexerHost;
 	private String searchHost;
+	private String storageQueryKindsHost;
 	private String storageQueryRecordForConversionHost;
 	private String storageQueryRecordHost;
 	private Integer storageRecordsBatchSize;

indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexService.java

@@ -20,4 +20,6 @@ import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
 public interface ReindexService {
 
     String reindexRecords(RecordReindexRequest recordReindexRequest, boolean forceClean);
+
+    void fullReindex(boolean forceClean);
 }
\ No newline at end of file

indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java

@@ -16,6 +16,7 @@ package org.opengroup.osdu.indexer.service;
 
 import com.google.common.base.Strings;
 import com.google.gson.Gson;
+import java.util.Objects;
 import org.apache.http.HttpStatus;
 import org.opengroup.osdu.core.common.model.http.DpsHeaders;
 import org.opengroup.osdu.core.common.model.http.AppException;
@@ -100,4 +101,26 @@ public class ReindexServiceImpl implements ReindexService {
             throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Unknown error", "An unknown error has occurred.", e);
         }
     }
+
+	@Override
+	public void fullReindex(boolean forceClean) {
+		List<String> allKinds = null;
+		try {
+			allKinds = storageService.getAllKinds();
+		} catch (Exception e) {
+			jaxRsDpsLog.error("storage service all kinds request failed",e);
+			throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "storage service cannot respond with all kinds", "an unknown error has occurred.", e);
+		}
+		if (Objects.isNull(allKinds)){
+			throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "storage service cannot respond with all kinds", "full reindex failed");
+		}
+		for (String kind : allKinds) {
+			try {
+				reindexRecords(new RecordReindexRequest(kind, ""), forceClean);
+			} catch (Exception e) {
+				jaxRsDpsLog.warning(String.format("kind: %s cannot be re-indexed", kind));
+				continue;
+			}
+		}
+	}
 }
\ No newline at end of file

indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageService.java

@@ -30,4 +30,6 @@ public interface StorageService {
     RecordQueryResponse getRecordsByKind(RecordReindexRequest request) throws URISyntaxException;
 
     String getStorageSchema(String kind) throws URISyntaxException, UnsupportedEncodingException;
+
+    List<String> getAllKinds() throws URISyntaxException;
 }
\ No newline at end of file

indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java

@@ -20,6 +20,9 @@ import com.google.common.collect.Lists;
 import com.google.common.reflect.TypeToken;
 import com.google.gson.Gson;
 
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
 import org.opengroup.osdu.core.common.http.FetchServiceHttpRequest;
 import org.opengroup.osdu.core.common.model.http.DpsHeaders;
 import org.opengroup.osdu.core.common.model.http.AppException;
@@ -209,4 +212,18 @@ public class StorageServiceImpl implements StorageService {
         HttpResponse response = this.urlFetchService.sendRequest(request);
         return response.getResponseCode() != HttpStatus.SC_OK ? null : response.getBody();
     }
+
+    @Override
+    public List<String> getAllKinds() throws URISyntaxException {
+        String url = configurationProperties.getStorageQueryKindsHost();
+        FetchServiceHttpRequest request = FetchServiceHttpRequest.builder()
+            .httpMethod(HttpMethods.GET)
+            .headers(this.requestInfo.getHeadersMap())
+            .url(url)
+            .build();
+        HttpResponse response = this.urlFetchService.sendRequest(request);
+        JsonObject asJsonObject = new JsonParser().parse(response.getBody()).getAsJsonObject();
+        JsonElement results = asJsonObject.get("results");
+        return response.getResponseCode() != HttpStatus.SC_OK ? null : this.gson.fromJson(results,List.class);
+    }
 }
\ No newline at end of file

provider/indexer-aws/src/main/resources/application.properties

@@ -27,6 +27,7 @@ SCHEMA_HOST=${STORAGE_HOST}/api/schema-service/v1/schema
 
 STORAGE_SCHEMA_HOST=${STORAGE_HOST}/api/storage/v2/schemas
 STORAGE_QUERY_RECORD_HOST=${STORAGE_HOST}/api/storage/v2/query/records
+STORAGE_QUERY_KINDS_HOST=${STORAGE_HOST}/api/storage/v2/query/kinds
 STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=${STORAGE_HOST}/api/storage/v2/query/records:batch
 STORAGE_RECORDS_BATCH_SIZE=20
 INDEXER_QUEUE_HOST=""

provider/indexer-azure/src/main/resources/application.properties

@@ -41,6 +41,7 @@ SCHEMA_HOST=${schema_service_url}/schema
 storage_service_url=${storage_service_endpoint}
 STORAGE_SCHEMA_HOST=${storage_service_url}/schemas
 STORAGE_QUERY_RECORD_HOST=${storage_service_url}/query/records
+STORAGE_QUERY_KINDS_HOST=${storage_service_url}/query/kinds
 STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=${storage_service_url}/query/records:batch
 STORAGE_RECORDS_BATCH_SIZE=20
 

provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/AppExceptionHandler.java

+package org.opengroup.osdu.indexer.util;
+
+import java.util.Objects;
+import lombok.extern.slf4j.Slf4j;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.ControllerAdvice;
+import org.springframework.web.bind.annotation.ExceptionHandler;
+
+@ControllerAdvice
+@Slf4j
+public class AppExceptionHandler {
+
+	@ExceptionHandler(AppException.class)
+	public ResponseEntity<Object> handleAppExceptions(AppException e) {
+		return this.getErrorResponse(e);
+	}
+
+	private ResponseEntity<Object> getErrorResponse(AppException e) {
+
+		String exceptionMsg = Objects.nonNull(e.getOriginalException())
+			? e.getOriginalException().getMessage()
+			: e.getError().getMessage();
+
+		if (e.getError().getCode() > 499) {
+			log.error(exceptionMsg, e.getOriginalException());
+		} else {
+			log.warn(exceptionMsg, e.getOriginalException());
+		}
+
+		return new ResponseEntity<>(e.getError(), HttpStatus.resolve(e.getError().getCode()));
+	}
+}

provider/indexer-gcp/src/main/resources/application.properties

@@ -40,5 +40,6 @@ elastic-datastore-id=indexer-service
 
 security.https.certificate.trust=false
 indexer.que.service.mail=default@iam.gserviceaccount.com
-
 SCHEMA_HOST=${HOST}/api/schema-service/v1/schema
+storage-query-kinds-host=https://${STORAGE_HOSTNAME}/api/storage/v2/query/kinds
+

provider/indexer-ibm/src/main/resources/application.properties

@@ -32,6 +32,7 @@ storage_service_url=http://localhost:8082
 #storage_service_url=https://os-storage-ibm-osdu-r2.osduadev-a1c3eaf78a86806e299f5f3f207556f0-0000.us-south.containers.appdomain.cloud
 STORAGE_SCHEMA_HOST=${storage_service_url}/api/storage/v2/schemas
 STORAGE_QUERY_RECORD_HOST=${storage_service_url}/api/storage/v2/query/records
+STORAGE_QUERY_KINDS_HOST=${storage_service_url}/api/storage/v2/query/kinds
 STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=${storage_service_url}/api/storage/v2/query/records:batch
 STORAGE_RECORDS_BATCH_SIZE=20