From d6a2de6798dd7d768e61a8fc335628b68555fbb7 Mon Sep 17 00:00:00 2001
From: NThakur4 <nthakur4@slb.com>
Date: Thu, 16 Sep 2021 16:06:42 -0500
Subject: [PATCH] add retries for schema service and all other calls
---
.../indexer/azure/service/RetryPolicy.java | 35 +++++++++++--
.../service/UrlFetchServiceAzureImpl.java | 21 ++++----
.../azure/service/RetryPolicyTest.java | 6 +--
.../service/UrlFetchServiceAzureImplTest.java | 52 ++++++++-----------
4 files changed, 69 insertions(+), 45 deletions(-)
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/service/RetryPolicy.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/service/RetryPolicy.java
index 47d906c9a..0bbcb3905 100644
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/service/RetryPolicy.java
+++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/service/RetryPolicy.java
@@ -28,6 +28,7 @@ import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
import java.time.Duration;
+import java.util.function.Predicate;
/**
* This class handles retry configuration logic for calls made to <prefix>/storage/v2/query/records:batch
@@ -50,23 +51,25 @@ public class RetryPolicy {
/**
* @return RetryConfig with 3 attempts and 1 sec wait time
*/
- public RetryConfig retryConfig() {
+ public RetryConfig retryConfig(Predicate<HttpResponse> predicate) {
return RetryConfig.<HttpResponse>custom()
.maxAttempts(attempts)
.waitDuration(Duration.ofMillis(waitDuration))
- .retryOnResult(response -> isRetryRequired(response))
+ .retryOnResult(predicate)
.build();
}
/**
* Unfound records get listed under a JsonArray "notFound" in the http json response
+ *
* @param response
* @return if there are elements in "notFound" returns true, else false
*/
- private boolean isRetryRequired(HttpResponse response) {
+ public boolean batchRetryPolicy(HttpResponse response) {
if (response == null || response.getBody().isEmpty()) {
return false;
}
+
JsonObject jsonObject = new JsonParser().parse(response.getBody()).getAsJsonObject();
JsonElement notFoundElement = (JsonArray) jsonObject.get(RECORD_NOT_FOUND);
if (notFoundElement == null ||
@@ -78,4 +81,30 @@ public class RetryPolicy {
log.info("Retry is set true");
return true;
}
+
+ public boolean schemaRetryPolicy(HttpResponse response) {
+ if (response == null || response.getBody().isEmpty()) {
+ return false;
+ }
+
+ if (response.getResponseCode() == 404) {
+ log.info("Retry is set true");
+ return true;
+ }
+
+ return false;
+ }
+
+ public boolean defaultRetryPolicy(HttpResponse response) {
+ if (response == null || response.getBody().isEmpty()) {
+ return false;
+ }
+
+ if (response.getResponseCode() > 501) {
+ log.info("Retry is set true");
+ return true;
+ }
+
+ return false;
+ }
}
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/service/UrlFetchServiceAzureImpl.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/service/UrlFetchServiceAzureImpl.java
index d56b888fa..c0c56ccd8 100644
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/service/UrlFetchServiceAzureImpl.java
+++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/service/UrlFetchServiceAzureImpl.java
@@ -41,6 +41,7 @@ import java.util.function.Supplier;
public class UrlFetchServiceAzureImpl implements IUrlFetchService {
public static final String STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST_URL = "storage/v2/query/records:batch";
+ public static final String SCHEMA_SERVICE_HOST_URL = "api/schema-service/v1/schema";
@Autowired
private RetryPolicy policy;
@@ -50,6 +51,7 @@ public class UrlFetchServiceAzureImpl implements IUrlFetchService {
@Autowired
private JaxRsDpsLog logger;
+
/**
* this method invokes retryFunction only for <prefix>/storage/v2/query/records:batch
* calls otherwise invokes UrlFetchService.sendRequest(FetchServiceHttpRequest request)
@@ -60,14 +62,7 @@ public class UrlFetchServiceAzureImpl implements IUrlFetchService {
*/
@Override
public HttpResponse sendRequest(FetchServiceHttpRequest httpRequest) throws URISyntaxException {
- HttpResponse output;
- if (httpRequest.getUrl().contains(STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST_URL)) {
- output = this.retryFunction(httpRequest);
- if (output != null) {
- return output;
- }
- }
- return this.urlFetchService.sendRequest(httpRequest);
+ return this.retryFunction(httpRequest);
}
/**
@@ -78,7 +73,15 @@ public class UrlFetchServiceAzureImpl implements IUrlFetchService {
* @return null if URISyntaxException is caught else returns HttpResponse
*/
private HttpResponse retryFunction(FetchServiceHttpRequest request) {
- RetryConfig config = this.policy.retryConfig();
+ RetryConfig config;
+ if (request.getUrl().contains(STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST_URL)) {
+ config = this.policy.retryConfig(response -> this.policy.batchRetryPolicy(response));
+ } else if (request.getUrl().contains(SCHEMA_SERVICE_HOST_URL)) {
+ config = this.policy.retryConfig(response -> this.policy.schemaRetryPolicy(response));
+ } else {
+ config = this.policy.retryConfig(response -> this.policy.defaultRetryPolicy(response));
+ }
+
RetryRegistry registry = RetryRegistry.of(config);
Retry retry = registry.retry("retryPolicy", config);
diff --git a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/RetryPolicyTest.java b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/RetryPolicyTest.java
index 70a7fcb9e..a27c40932 100644
--- a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/RetryPolicyTest.java
+++ b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/RetryPolicyTest.java
@@ -131,7 +131,7 @@ public class RetryPolicyTest {
@Test
public void retry_should_be_true_for_jsonResponseWithNotFound() {
- RetryConfig config = this.retryPolicy.retryConfig();
+ RetryConfig config = this.retryPolicy.retryConfig(response -> this.retryPolicy.batchRetryPolicy(response));
Predicate<HttpResponse> retry = config.getResultPredicate();
response.setBody(JSON_RESPONSE_WITH_NOT_FOUND);
assert retry != null;
@@ -142,7 +142,7 @@ public class RetryPolicyTest {
@Test
public void retry_should_be_false_for_jsonResponse1WithOut_NotFound() {
- RetryConfig config = this.retryPolicy.retryConfig();
+ RetryConfig config = this.retryPolicy.retryConfig(response -> this.retryPolicy.batchRetryPolicy(response));
Predicate<HttpResponse> retry = config.getResultPredicate();
response.setBody(JSON_RESPONSE1_WITHOUT_NOT_FOUND);
boolean value = retry.test(response);
@@ -152,7 +152,7 @@ public class RetryPolicyTest {
@Test
public void retry_should_be_false_for_jsonResponse2WithOut_NotFound() {
- RetryConfig config = this.retryPolicy.retryConfig();
+ RetryConfig config = this.retryPolicy.retryConfig(response -> this.retryPolicy.batchRetryPolicy(response));
Predicate<HttpResponse> retry = config.getResultPredicate();
response.setBody(JSON_RESPONSE2_WITHOUT_NOT_FOUND);
boolean value = retry.test(response);
diff --git a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/UrlFetchServiceAzureImplTest.java b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/UrlFetchServiceAzureImplTest.java
index 916c5b8bc..18adccbfe 100644
--- a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/UrlFetchServiceAzureImplTest.java
+++ b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/UrlFetchServiceAzureImplTest.java
@@ -14,8 +14,6 @@
package org.opengroup.osdu.indexer.azure.service;
-import io.github.resilience4j.retry.RetryConfig;
-import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
@@ -26,12 +24,8 @@ import org.opengroup.osdu.core.common.http.UrlFetchServiceImpl;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.HttpResponse;
-import java.time.Duration;
-
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.atMost;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.when;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.*;
@RunWith(MockitoJUnitRunner.class)
@@ -95,45 +89,43 @@ public class UrlFetchServiceAzureImplTest {
" \"conversionStatuses\":[]\n" +
"}";
- private static final String url = "https://demo/api/storage/v2/query/records:batch";
- private static final String url2 = "https://demo/api/storage/v2/schemas";
-
- @Before
- public void setUp() {
- when(this.retryPolicy.retryConfig()).thenReturn(new RetryPolicy().retryConfig());
- }
+ private static final String BATCH_API_URL = "https://demo/api/storage/v2/query/records:batch";
+ private static final String STORAGE_API_URL = "https://demo/api/storage/v2/schemas";
+ private static final String SCHEMA_API_URL = "https://demo/api/schema-service/v1/schema/osdu:file:gom:1.0.0";
@Test
public void shouldRetry_ForJSON1_when_storageQueryRecordCallIsMade() throws Exception {
response.setBody(JSON1);
- httpRequest.setUrl(url);
-
- when(urlFetchServiceAzure.sendRequest(httpRequest)).thenReturn(response);
+ httpRequest.setUrl(BATCH_API_URL);
+ when(this.retryPolicy.retryConfig(any())).thenReturn(new RetryPolicy().retryConfig(response -> this.retryPolicy.batchRetryPolicy(response)));
+ when(urlFetchService.sendRequest(httpRequest)).thenReturn(response);
urlFetchServiceAzure.sendRequest(httpRequest);
+
verify(urlFetchService, atMost(4)).sendRequest(httpRequest);
}
@Test
- public void shouldNotRetry_ForJSON2_when_storageQueryRecordCallIsMade() throws Exception {
- response.setBody(JSON2);
- httpRequest.setUrl(url);
-
- when(urlFetchServiceAzure.sendRequest(httpRequest)).thenReturn(response);
+ public void shouldRetry_ForJSON1_when_schemaRecordCallIsMade() throws Exception {
+ response.setBody(JSON1);
+ httpRequest.setUrl(SCHEMA_API_URL);
+ when(this.retryPolicy.retryConfig(any())).thenReturn(new RetryPolicy().retryConfig(response -> this.retryPolicy.schemaRetryPolicy(response)));
+ when(urlFetchService.sendRequest(httpRequest)).thenReturn(response);
urlFetchServiceAzure.sendRequest(httpRequest);
- verify(urlFetchService, atMost(2)).sendRequest(httpRequest);
- }
+ verify(urlFetchService, atMost(4)).sendRequest(httpRequest);
+ }
@Test
- public void retryFunction_shouldNotBeCalled() throws Exception {
- httpRequest.setUrl(url2);
-
+ public void shouldRetry_when_anyOtherCallIsMade() throws Exception {
+ response.setBody(JSON2);
+ httpRequest.setUrl(STORAGE_API_URL);
+ when(this.retryPolicy.retryConfig(any())).thenReturn(new RetryPolicy().retryConfig(response -> this.retryPolicy.defaultRetryPolicy(response)));
when(urlFetchService.sendRequest(httpRequest)).thenReturn(response);
urlFetchServiceAzure.sendRequest(httpRequest);
- verify(urlFetchService, times(1)).sendRequest(httpRequest);
- }
+ verify(urlFetchService, atMost(4)).sendRequest(httpRequest);
+ }
}
--
GitLab