From ce5bd089db69e41417b0761ca04d24d41f251bce Mon Sep 17 00:00:00 2001
From: ZMai <zmai@slb.com>
Date: Thu, 7 Sep 2023 09:36:53 -0500
Subject: [PATCH] Always update ElasticSearch schema mapping when an index is
re-indexed
---
.../org/opengroup/osdu/indexer/api/RecordIndexerApi.java | 2 +-
.../java/org/opengroup/osdu/indexer/api/ReindexApi.java | 2 +-
.../osdu/indexer/service/IndexSchemaService.java | 2 ++
.../osdu/indexer/service/IndexSchemaServiceImpl.java | 9 +++++++++
.../opengroup/osdu/indexer/service/ReindexService.java | 2 +-
.../osdu/indexer/service/ReindexServiceImpl.java | 8 ++++++--
.../org/opengroup/osdu/indexer/api/ReindexApiTest.java | 6 +++---
.../osdu/indexer/azure/service/ReindexServiceTest.java | 8 ++++----
.../osdu/indexer/service/ReindexServiceTest.java | 8 ++++----
.../osdu/indexer/ibm/service/ReindexServiceTest.java | 8 ++++----
10 files changed, 35 insertions(+), 20 deletions(-)
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/RecordIndexerApi.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/RecordIndexerApi.java
index 4f08bbc54..c58170b0e 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/RecordIndexerApi.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/RecordIndexerApi.java
@@ -113,7 +113,7 @@ public class RecordIndexerApi {
public ResponseEntity<?> reindex(
@RequestBody @NotNull(message = SwaggerDoc.REQUEST_VALIDATION_NOT_NULL_BODY)
@Valid RecordReindexRequest recordReindexRequest) {
- return new ResponseEntity<>(reIndexService.reindexKind(recordReindexRequest, false), HttpStatus.OK);
+ return new ResponseEntity<>(reIndexService.reindexKind(recordReindexRequest, false, false), HttpStatus.OK);
}
// THIS IS AN INTERNAL USE API ONLY
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java
index c4ea75f99..73834c2bc 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java
@@ -104,7 +104,7 @@ public class ReindexApi {
public ResponseEntity<?> reindex(@NotNull @Valid @RequestBody RecordReindexRequest recordReindexRequest,
@Parameter(description = "Force Clean")
@RequestParam(value = "force_clean", defaultValue = "false") boolean forceClean) throws IOException {
- this.reIndexService.reindexKind(recordReindexRequest, this.indexSchemaService.isStorageSchemaSyncRequired(recordReindexRequest.getKind(), forceClean));
+ this.reIndexService.reindexKind(recordReindexRequest, this.indexSchemaService.isStorageSchemaSyncRequired(recordReindexRequest.getKind(), forceClean), true);
this.auditLogger.getReindex(singletonList(recordReindexRequest.getKind()));
return new ResponseEntity<>(org.springframework.http.HttpStatus.OK);
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaService.java
index 9b19787c3..f4242786d 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaService.java
@@ -35,6 +35,8 @@ public interface IndexSchemaService {
void processSchemaMessages(Map<String, OperationType> schemaMsgs) throws IOException;
+ void processSchemaUpsert(String kind) throws AppException;
+
void processSchemaUpsertEvent(RestHighLevelClient restClient, String kind) throws IOException, ElasticsearchStatusException, URISyntaxException;
void syncIndexMappingWithStorageSchema(String kind) throws ElasticsearchException, IOException, AppException, URISyntaxException;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
index 048e87a39..93faa0486 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
@@ -106,6 +106,15 @@ public class IndexSchemaServiceImpl implements IndexSchemaService {
}
}
+ @Override
+ public void processSchemaUpsert(String kind) throws AppException {
+ try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
+ processSchemaUpsertEvent(restClient, kind);
+ } catch (IOException | ElasticsearchStatusException | URISyntaxException e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "unable to process schema update", e.getMessage());
+ }
+ }
+
@Override
public void processSchemaUpsertEvent(RestHighLevelClient restClient, String kind) throws IOException, ElasticsearchStatusException, URISyntaxException {
String index = this.elasticIndexNameResolver.getIndexNameFromKind(kind);
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexService.java
index 231cbf2ad..59b799f52 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexService.java
@@ -22,7 +22,7 @@ import java.util.List;
public interface ReindexService {
- String reindexKind(RecordReindexRequest recordReindexRequest, boolean forceClean);
+ String reindexKind(RecordReindexRequest recordReindexRequest, boolean forceClean, boolean updateSchemaMapping);
Records reindexRecords(List<String> recordIds);
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java
index 062c48da5..e0bfb4b9b 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java
@@ -52,7 +52,7 @@ public class ReindexServiceImpl implements ReindexService {
@SneakyThrows
@Override
- public String reindexKind(RecordReindexRequest recordReindexRequest, boolean forceClean) {
+ public String reindexKind(RecordReindexRequest recordReindexRequest, boolean forceClean, boolean updateSchemaMapping) {
Long initialDelayMillis = 0l;
@@ -62,6 +62,10 @@ public class ReindexServiceImpl implements ReindexService {
this.indexSchemaService.syncIndexMappingWithStorageSchema(recordReindexRequest.getKind());
initialDelayMillis = 30000l;
}
+ else if(updateSchemaMapping){
+ this.indexSchemaService.processSchemaUpsert(recordReindexRequest.getKind());
+ initialDelayMillis = 15000l;
+ }
RecordQueryResponse recordQueryResponse = this.storageService.getRecordsByKind(recordReindexRequest);
@@ -113,7 +117,7 @@ public class ReindexServiceImpl implements ReindexService {
}
for (String kind : allKinds) {
try {
- reindexKind(new RecordReindexRequest(kind, ""), forceClean);
+ reindexKind(new RecordReindexRequest(kind, ""), forceClean, true);
} catch (Exception e) {
jaxRsDpsLog.warning(String.format("kind: %s cannot be re-indexed", kind));
continue;
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/ReindexApiTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/ReindexApiTest.java
index f5437fbd9..dadaed397 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/ReindexApiTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/ReindexApiTest.java
@@ -64,7 +64,7 @@ public class ReindexApiTest {
@Test
public void should_return200_when_valid_kind_provided() throws IOException {
- when(this.reIndexService.reindexKind(recordReindexRequest, false)).thenReturn("something");
+ when(this.reIndexService.reindexKind(recordReindexRequest, false, false)).thenReturn("something");
ResponseEntity<?> response = sut.reindex(recordReindexRequest, false);
@@ -73,14 +73,14 @@ public class ReindexApiTest {
@Test(expected = AppException.class)
public void should_throwAppException_ifUnknownExceptionCaught_reindexTest() throws IOException {
- when(this.reIndexService.reindexKind(recordReindexRequest, false)).thenThrow(new AppException(500, "", ""));
+ when(this.reIndexService.reindexKind(recordReindexRequest, false, true)).thenThrow(new AppException(500, "", ""));
sut.reindex(recordReindexRequest, false);
}
@Test(expected = NullPointerException.class)
public void should_throwAppException_ifNullPointerExceptionCaught_ReindexTest() throws IOException {
- when(this.reIndexService.reindexKind(recordReindexRequest, false)).thenThrow(new NullPointerException(""));
+ when(this.reIndexService.reindexKind(recordReindexRequest, false, true)).thenThrow(new NullPointerException(""));
sut.reindex(recordReindexRequest, false);
}
diff --git a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/ReindexServiceTest.java b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/ReindexServiceTest.java
index 6e6e52d89..6cdeb05c1 100644
--- a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/ReindexServiceTest.java
+++ b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/ReindexServiceTest.java
@@ -104,7 +104,7 @@ public class ReindexServiceTest {
recordQueryResponse.setResults(null);
when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
- String response = sut.reindexKind(recordReindexRequest, false);
+ String response = sut.reindexKind(recordReindexRequest, false, false);
Assert.assertNull(response);
} catch (Exception e) {
@@ -118,7 +118,7 @@ public class ReindexServiceTest {
recordQueryResponse.setResults(new ArrayList<>());
when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
- String response = sut.reindexKind(recordReindexRequest, false);
+ String response = sut.reindexKind(recordReindexRequest, false, false);
Assert.assertNull(response);
} catch (Exception e) {
@@ -138,7 +138,7 @@ public class ReindexServiceTest {
when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
- String taskQueuePayload = sut.reindexKind(recordReindexRequest, false);
+ String taskQueuePayload = sut.reindexKind(recordReindexRequest, false, false);
Assert.assertEquals("{\"kind\":\"tenant:test:test:1.0.0\",\"cursor\":\"100\"}", taskQueuePayload);
} catch (Exception e) {
@@ -154,7 +154,7 @@ public class ReindexServiceTest {
recordQueryResponse.setResults(results);
when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
- String taskQueuePayload = sut.reindexKind(recordReindexRequest, false);
+ String taskQueuePayload = sut.reindexKind(recordReindexRequest, false, false);
Assert.assertEquals(String.format("{\"data\":\"[{\\\"id\\\":\\\"test1\\\",\\\"kind\\\":\\\"tenant:test:test:1.0.0\\\",\\\"op\\\":\\\"create\\\"}]\",\"attributes\":{\"slb-correlation-id\":\"%s\"}}", correlationId), taskQueuePayload);
} catch (Exception e) {
diff --git a/provider/indexer-gc/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java b/provider/indexer-gc/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java
index 7fef8d1e9..34102903c 100644
--- a/provider/indexer-gc/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java
+++ b/provider/indexer-gc/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java
@@ -94,7 +94,7 @@ public class ReindexServiceTest {
recordQueryResponse.setResults(null);
when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
- String response = sut.reindexKind(recordReindexRequest, false);
+ String response = sut.reindexKind(recordReindexRequest, false, false);
Assert.assertNull(response);
} catch (Exception e) {
@@ -108,7 +108,7 @@ public class ReindexServiceTest {
recordQueryResponse.setResults(new ArrayList<>());
when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
- String response = sut.reindexKind(recordReindexRequest, false);
+ String response = sut.reindexKind(recordReindexRequest, false, false);
Assert.assertNull(response);
} catch (Exception e) {
@@ -128,7 +128,7 @@ public class ReindexServiceTest {
when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
- String taskQueuePayload = sut.reindexKind(recordReindexRequest, false);
+ String taskQueuePayload = sut.reindexKind(recordReindexRequest, false, false);
Assert.assertEquals("{\"kind\":\"tenant:test:test:1.0.0\",\"cursor\":\"100\"}", taskQueuePayload);
} catch (Exception e) {
@@ -144,7 +144,7 @@ public class ReindexServiceTest {
recordQueryResponse.setResults(results);
when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
- String taskQueuePayload = sut.reindexKind(recordReindexRequest, false);
+ String taskQueuePayload = sut.reindexKind(recordReindexRequest, false, false);
Assert.assertEquals(String.format("{\"data\":\"[{\\\"id\\\":\\\"test1\\\",\\\"kind\\\":\\\"tenant:test:test:1.0.0\\\",\\\"op\\\":\\\"create\\\"}]\",\"attributes\":{\"correlation-id\":\"%s\"}}", correlationId), taskQueuePayload);
} catch (Exception e) {
diff --git a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/ReindexServiceTest.java b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/ReindexServiceTest.java
index 1d562afe6..0af01628e 100644
--- a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/ReindexServiceTest.java
+++ b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/ReindexServiceTest.java
@@ -88,7 +88,7 @@ public class ReindexServiceTest {
recordQueryResponse.setResults(null);
when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
- String response = sut.reindexKind(recordReindexRequest, false);
+ String response = sut.reindexKind(recordReindexRequest, false, false);
Assert.assertNull(response);
} catch (Exception e) {
@@ -102,7 +102,7 @@ public class ReindexServiceTest {
recordQueryResponse.setResults(new ArrayList<>());
when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
- String response = sut.reindexKind(recordReindexRequest, false);
+ String response = sut.reindexKind(recordReindexRequest, false, false);
Assert.assertNull(response);
} catch (Exception e) {
@@ -122,7 +122,7 @@ public class ReindexServiceTest {
when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
- String taskQueuePayload = sut.reindexKind(recordReindexRequest, false);
+ String taskQueuePayload = sut.reindexKind(recordReindexRequest, false, false);
Assert.assertEquals("{\"kind\":\"tenant:test:test:1.0.0\",\"cursor\":\"100\"}", taskQueuePayload);
} catch (Exception e) {
@@ -138,7 +138,7 @@ public class ReindexServiceTest {
recordQueryResponse.setResults(results);
when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
- String taskQueuePayload = sut.reindexKind(recordReindexRequest, false);
+ String taskQueuePayload = sut.reindexKind(recordReindexRequest, false, false);
Assert.assertEquals(String.format("{\"data\":\"[{\\\"id\\\":\\\"test1\\\",\\\"kind\\\":\\\"tenant:test:test:1.0.0\\\",\\\"op\\\":\\\"create\\\"}]\",\"attributes\":{\"slb-correlation-id\":\"%s\"}}", correlationId), taskQueuePayload);
} catch (Exception e) {
--
GitLab