diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/SwaggerDoc.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/SwaggerDoc.java
index 713c981dc6dca4c3ecd29704038f59d43a8da885..a748ce00f0ebc0a9f9a524b8d331a5463f7f28d8 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/SwaggerDoc.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/SwaggerDoc.java
@@ -145,7 +145,6 @@ public final class SwaggerDoc {
// REQUEST VALIDATION
public static final String REQUEST_VALIDATION_NOT_NULL_BODY = "Request body can not be null";
- public static final String REQUEST_REINDEX_RECORDS_VALIDATION_EXCEEDS_LIMIT = "Maximum record ids supported in one API call is ";
// Azure Schema Request
public static final String SCHEMA_REQUEST_KIND = "Record kind for which the schema information is applied to.";
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java
index 166d4ae36aed7c5866d8a4de6ed1b0cc702e8e17..09c42d0c86608537cd8bf05c1a64e5b6009b1bb4 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java
@@ -78,9 +78,12 @@ public class ReindexApi {
})
@PreAuthorize("@authorizationFilter.hasPermission('" + SearchServiceRole.ADMIN + "')")
@PostMapping(path = "/records", consumes = "application/json")
- public ResponseEntity<?> reindexRecords(@NotNull @RequestBody ReindexRecordsRequest reindexRecordsRequest) {
+ public ResponseEntity<?> reindexRecords(@NotNull @Valid @RequestBody ReindexRecordsRequest reindexRecordsRequest) {
Records records = this.reIndexService.reindexRecords(reindexRecordsRequest.getRecordIds());
- this.auditLogger.getReindex(records.getRecords().stream().map(Records.Entity::getId).collect(Collectors.toList()));
+ List<String> reindexedRecords = records.getRecords().stream().map(Records.Entity::getId).collect(Collectors.toList());
+ if (!reindexedRecords.isEmpty()) {
+ this.auditLogger.getReindex(reindexedRecords);
+ }
return new ResponseEntity<>(ReindexRecordsResponse.builder().reIndexedRecords(records.getRecords().stream().map(Records.Entity::getId).collect(Collectors.toList())).notFoundRecords(records.getNotFound()).build(), HttpStatus.ACCEPTED);
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/ReindexRecordsRequest.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/ReindexRecordsRequest.java
index 5406656243ea6dbac4abc833983d52e27c4accc8..e0750605336760e68dfe91cdd9fbee0821cd72d8 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/ReindexRecordsRequest.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/ReindexRecordsRequest.java
@@ -2,6 +2,7 @@ package org.opengroup.osdu.indexer.model;
import lombok.AllArgsConstructor;
import lombok.Data;
+import lombok.NoArgsConstructor;
import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotNull;
@@ -10,8 +11,9 @@ import java.util.List;
@Data
@AllArgsConstructor
+@NoArgsConstructor
public class ReindexRecordsRequest {
@NotNull
- @Size(min = 1)
+ @Size(min = 1, max = 1000)
private List<@NotBlank String> recordIds;
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java
index 7e28543bca6a2cb031d7a0cad905efdf6d7e9bdb..062c48da5c31506fd443d9551191d1ac86cf6af0 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java
@@ -25,7 +25,6 @@ import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.indexer.*;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.indexer.SwaggerDoc;
import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
@@ -91,11 +90,7 @@ public class ReindexServiceImpl implements ReindexService {
@SneakyThrows
@Override
public Records reindexRecords(List<String> recordIds) {
- if (recordIds.size() > configurationProperties.getStorageRecordsBatchSize()) {
- throw new AppException(org.springframework.http.HttpStatus.BAD_REQUEST.value(), "Exceeds limit",
- SwaggerDoc.REQUEST_REINDEX_RECORDS_VALIDATION_EXCEEDS_LIMIT + configurationProperties.getStorageRecordsBatchSize());
- }
- Records records = this.storageService.getStorageRecords(recordIds, new ArrayList<>());
+ Records records = this.storageService.getStorageRecords(recordIds);
if (records.getRecords().size() > 0) {
List<RecordInfo> msgs = records.getRecords().stream()
.map(record -> RecordInfo.builder().id(record.getId()).kind(record.getKind()).op(OperationType.create.name()).build()).collect(Collectors.toList());
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageService.java
index 54a19e807079d6076b855e89ea2707f88308e038..d2833e9748c3ad8dcb98a0efb913d01ca68c090b 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageService.java
@@ -28,9 +28,11 @@ public interface StorageService {
Records getStorageRecords(List<String> ids, List<RecordInfo> recordChangedInfos) throws AppException, URISyntaxException;
+ Records getStorageRecords(List<String> ids) throws URISyntaxException;
+
RecordQueryResponse getRecordsByKind(RecordReindexRequest request) throws URISyntaxException;
String getStorageSchema(String kind) throws URISyntaxException, UnsupportedEncodingException;
List<String> getAllKinds() throws URISyntaxException;
-}
\ No newline at end of file
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java
index 736945e2988a1959691fa7e907ae3c0fdbda2b82..38c405d7af062ed239e0a111c72f92132f4f9d2a 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java
@@ -24,7 +24,11 @@ import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import org.apache.http.HttpStatus;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.entity.StringEntity;
import org.opengroup.osdu.core.common.http.FetchServiceHttpRequest;
+import org.opengroup.osdu.core.common.http.IHttpClientHandler;
import org.opengroup.osdu.core.common.http.IUrlFetchService;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
@@ -50,11 +54,7 @@ import java.io.UnsupportedEncodingException;
import java.net.URISyntaxException;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import java.util.stream.Collectors;
import static org.opengroup.osdu.core.common.Constants.SLB_FRAME_OF_REFERENCE_VALUE;
@@ -70,6 +70,8 @@ public class StorageServiceImpl implements StorageService {
@Inject
private IUrlFetchService urlFetchService;
@Inject
+ private IHttpClientHandler httpClientHandler;
+ @Inject
private JobStatus jobStatus;
@Inject
private IRequestInfo requestInfo;
@@ -98,6 +100,24 @@ public class StorageServiceImpl implements StorageService {
return Records.builder().records(valid).notFound(notFound).conversionStatuses(conversionStatuses).missingRetryRecords(missingRetryRecordIds).build();
}
+ @Override
+ public Records getStorageRecords(List<String> ids) throws URISyntaxException {
+ List<Records.Entity> valid = new ArrayList<>();
+ List<String> notFound = new ArrayList<>();
+ List<ConversionStatus> conversionStatuses = new ArrayList<>();
+ List<String> missingRetryRecordIds = new ArrayList<>();
+
+ List<List<String>> batch = Lists.partition(ids, configurationProperties.getStorageRecordsBatchSize());
+ for (List<String> recordsBatch : batch) {
+ Records storageOut = this.getRecords(recordsBatch);
+ valid.addAll(storageOut.getRecords());
+ notFound.addAll(storageOut.getNotFound());
+ conversionStatuses.addAll(storageOut.getConversionStatuses());
+ missingRetryRecordIds.addAll(storageOut.getMissingRetryRecords());
+ }
+ return Records.builder().records(valid).notFound(notFound).conversionStatuses(conversionStatuses).missingRetryRecords(missingRetryRecordIds).build();
+ }
+
protected Records getRecords(List<String> ids, Map<String, String> recordChangedMap, Map<String, String> validRecordKindPatchMap) throws URISyntaxException {
// e.g. {"records":["test:10"]}
String body = this.gson.toJson(RecordIds.builder().records(ids).build());
@@ -114,6 +134,25 @@ public class StorageServiceImpl implements StorageService {
return this.validateStorageResponse(response, ids, recordChangedMap, validRecordKindPatchMap);
}
+ protected Records getRecords(List<String> ids) throws URISyntaxException {
+ String body = this.gson.toJson(RecordIds.builder().records(ids).build());
+ DpsHeaders headers = this.requestInfo.getHeaders();
+ headers.put(FRAME_OF_REFERENCE, SLB_FRAME_OF_REFERENCE_VALUE);
+ URIBuilder builder = new URIBuilder(configurationProperties.getStorageQueryRecordForConversionHost());
+ HttpPost request = new HttpPost(builder.build());
+ request.setEntity(new StringEntity(body, StandardCharsets.UTF_8));
+ // we do not need retry on storage based on not found record
+ HttpResponse response = httpClientHandler.sendRequest(request, headers);
+ if (response.getResponseCode() > 299) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Internal Error", response.getBody());
+ }
+ try {
+ return this.objectMapper.readValue(response.getBody(), Records.class);
+ } catch (JsonProcessingException e) {
+ throw new AppException(RequestStatus.INVALID_RECORD, "Invalid request", "Successful Storage service response with wrong json", e);
+ }
+ }
+
private Records validateStorageResponse(HttpResponse response, List<String> ids, Map<String, String> recordChangedMap, Map<String, String> validRecordKindPatchMap) {
String bulkStorageData = response.getBody();
@@ -152,10 +191,7 @@ public class StorageServiceImpl implements StorageService {
}
// validate kind to avoid data duplication
- List<String> staleRecords = new ArrayList<>();
- if (recordChangedMap.size() > 0) {
- staleRecords = getStaleRecordsUpdate(recordChangedMap, validRecordKindPatchMap, validRecords);
- }
+ List<String> staleRecords = getStaleRecordsUpdate(recordChangedMap, validRecordKindPatchMap, validRecords);
List<Records.Entity> indexableRecords = validateKind(validRecords, staleRecords);
records.setRecords(indexableRecords);
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/ReindexApiTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/ReindexApiTest.java
index 719943ae2b5e3aeb4f49c34563732d5b0f2070a4..7e03dd7065e64e9706e80b10032eb3f9c00f38be 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/ReindexApiTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/ReindexApiTest.java
@@ -33,10 +33,12 @@ import org.springframework.test.context.junit4.SpringRunner;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.when;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.*;
@RunWith(SpringRunner.class)
public class ReindexApiTest {
@@ -85,11 +87,22 @@ public class ReindexApiTest {
@Test
public void should_return200_when_valid_record_id_list_provided() {
- when(this.reIndexService.reindexRecords(recordIds)).thenReturn(Records.builder().records(new ArrayList<>()).notFound(recordIds).build());
+ when(this.reIndexService.reindexRecords(recordIds)).thenReturn(Records.builder().records(new ArrayList<>()).records(Collections.singletonList(Records.Entity.builder().id("id1").build())).notFound(recordIds).build());
ResponseEntity<?> response = sut.reindexRecords(new ReindexRecordsRequest(recordIds));
assertEquals(HttpStatus.ACCEPTED, response.getStatusCode());
+ verify(auditLogger).getReindex(any());
+ }
+
+ @Test
+ public void should_notWriteAuditLog_when_no_valid_record_id_list_provided() {
+ when(this.reIndexService.reindexRecords(recordIds)).thenReturn(Records.builder().records(new ArrayList<>()).records(Collections.emptyList()).notFound(recordIds).build());
+
+ ResponseEntity<?> response = sut.reindexRecords(new ReindexRecordsRequest(recordIds));
+
+ assertEquals(HttpStatus.ACCEPTED, response.getStatusCode());
+ verify(auditLogger, never()).getReindex(any());
}
@Test(expected = AppException.class)
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceImplTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceImplTest.java
index 8514f1373f79e3c1567ffa3c1514a3aae598d2ff..6fef65e6183a074b38e5e2ba7e231aba765ca577 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceImplTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceImplTest.java
@@ -24,6 +24,7 @@ import org.mockito.ArgumentMatchers;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Spy;
+import org.opengroup.osdu.core.common.http.IHttpClientHandler;
import org.opengroup.osdu.core.common.http.IUrlFetchService;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
@@ -60,6 +61,8 @@ public class StorageServiceImplTest {
@Mock
private IUrlFetchService urlFetchService;
@Mock
+ private IHttpClientHandler httpClientHandler;
+ @Mock
private JobStatus jobStatus;
@Mock
private JaxRsDpsLog log;
@@ -169,33 +172,35 @@ public class StorageServiceImplTest {
}
@Test
- public void should_returnAllRecords_givenRecord_withoutValidation_getStorageRecordsTest() throws URISyntaxException {
+ public void should_returnZeroRecords_givenStaleMessage_getStorageRecordsTest() throws URISyntaxException {
- String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.2.0\"},{\"id\":\"testid2\", \"version\":1, \"kind\":\"tenant:test:test:2.2.0\"}],\"notFound\":[], \"conversionStatuses\": []}";
+ String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.2.0\"}],\"notFound\":[], \"conversionStatuses\": []}";
+ List<RecordInfo> recordChangeInfos = Arrays.asList(RecordInfo.builder().id("testid").kind("tenant:test:test:1.1.0").op(OperationType.update.getValue()).build());
HttpResponse httpResponse = mock(HttpResponse.class);
when(httpResponse.getBody()).thenReturn(validDataFromStorage);
when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
- Records storageRecords = this.sut.getStorageRecords(ids, new ArrayList<>());
+ Records storageRecords = this.sut.getStorageRecords(ids, recordChangeInfos);
- assertEquals(2, storageRecords.getRecords().size());
+ assertEquals(0, storageRecords.getRecords().size());
+ verify(this.log).warning("stale records found with older kind, skipping indexing | record ids: testid");
}
@Test
- public void should_returnZeroRecords_givenStaleMessage_getStorageRecordsTest() throws URISyntaxException {
+ public void should_returnStorageRecords_givenRecordIds_getValidStorageRecordsTest() throws URISyntaxException {
- String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.2.0\"}],\"notFound\":[], \"conversionStatuses\": []}";
- List<RecordInfo> recordChangeInfos = Arrays.asList(RecordInfo.builder().id("testid").kind("tenant:test:test:1.1.0").op(OperationType.update.getValue()).build());
+ String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"], \"conversionStatuses\": []}";
HttpResponse httpResponse = mock(HttpResponse.class);
when(httpResponse.getBody()).thenReturn(validDataFromStorage);
- when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
- Records storageRecords = this.sut.getStorageRecords(ids, recordChangeInfos);
+ when(configurationProperties.getStorageQueryRecordForConversionHost()).thenReturn("storageUrl");
+ when(this.httpClientHandler.sendRequest(any(), any())).thenReturn(httpResponse);
+ Records storageRecords = this.sut.getStorageRecords(ids);
- assertEquals(0, storageRecords.getRecords().size());
- verify(this.log).warning("stale records found with older kind, skipping indexing | record ids: testid");
+ assertEquals(1, storageRecords.getRecords().size());
+ assertEquals(1, storageRecords.getNotFound().size());
}
@Test
diff --git a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/ReindexServiceTest.java b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/ReindexServiceTest.java
index 59bba53ac308da40626b929373130e97f1ef9a82..4cca1d431c0374d040ab8945f15ca4fb35b57c2c 100644
--- a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/ReindexServiceTest.java
+++ b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/ReindexServiceTest.java
@@ -23,7 +23,6 @@ import org.junit.runner.RunWith;
import org.mockito.ArgumentMatchers;
import org.mockito.InjectMocks;
import org.mockito.Mock;
-import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
@@ -157,20 +156,13 @@ public class ReindexServiceTest {
}
}
- @Test(expected = AppException.class)
- public void should_throwException_givenRecordIdsExceedsTheLimit_reIndexRecordsTest() {
- when(configurationProperties.getStorageRecordsBatchSize()).thenReturn(2);
- List<String> recordIds = Arrays.asList("id1", "id2", "id3");
- sut.reindexRecords(recordIds);
- }
-
@Test
public void should_createReindexTaskForValidRecords_givenValidRecordIds_reIndexRecordsTest() throws URISyntaxException {
DpsHeaders headers = new DpsHeaders();
when(requestInfo.getHeadersWithDwdAuthZ()).thenReturn(headers);
when(configurationProperties.getStorageRecordsBatchSize()).thenReturn(2);
List<String> recordIds = Arrays.asList("id1", "id2");
- when(storageService.getStorageRecords(recordIds, new ArrayList<>())).thenReturn(
+ when(storageService.getStorageRecords(recordIds)).thenReturn(
Records.builder().records(Collections.singletonList(Records.Entity.builder().id("id1").kind("kind1").build())).notFound(Collections.singletonList("id2")).build()
);
Records records = sut.reindexRecords(recordIds);