Commit ecbd1856 authored by Muskan Srivastava's avatar Muskan Srivastava
Browse files

changes for retry

parent 264f172b
Pipeline #41857 failed with stage
in 1 minute and 53 seconds
...@@ -6,6 +6,8 @@ import org.opengroup.osdu.core.common.http.json.HttpResponseBodyMapper; ...@@ -6,6 +6,8 @@ import org.opengroup.osdu.core.common.http.json.HttpResponseBodyMapper;
import org.opengroup.osdu.core.common.model.http.DpsHeaders; import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import java.util.ArrayList;
public class EntitlementsFactoryAzure implements IEntitlementsFactory { public class EntitlementsFactoryAzure implements IEntitlementsFactory {
...@@ -24,6 +26,10 @@ public class EntitlementsFactoryAzure implements IEntitlementsFactory { ...@@ -24,6 +26,10 @@ public class EntitlementsFactoryAzure implements IEntitlementsFactory {
} }
RetryAndTimeoutConfiguration config = new RetryAndTimeoutConfiguration(); RetryAndTimeoutConfiguration config = new RetryAndTimeoutConfiguration();
config.setMaxRetryAttempts(10); config.setMaxRetryAttempts(10);
ArrayList<Integer> statusCode = new ArrayList<>();
statusCode.add(403);
statusCode.add(200);
config.setStatusCodesToBeRetried(statusCode);
return new EntitlementsService(this.config, return new EntitlementsService(this.config,
new HttpClientAzure(config), new HttpClientAzure(config),
headers, mapper); headers, mapper);
......
package org.opengroup.osdu.azure.retry; package org.opengroup.osdu.azure.retry;
import com.google.api.client.http.HttpMethods;
import io.github.resilience4j.retry.Retry; import io.github.resilience4j.retry.Retry;
import io.github.resilience4j.retry.RetryConfig; import io.github.resilience4j.retry.RetryConfig;
import io.github.resilience4j.retry.RetryRegistry; import io.github.resilience4j.retry.RetryRegistry;
...@@ -8,23 +7,18 @@ import org.apache.http.Header; ...@@ -8,23 +7,18 @@ import org.apache.http.Header;
import org.apache.http.HttpHeaders; import org.apache.http.HttpHeaders;
import org.apache.http.HttpStatus; import org.apache.http.HttpStatus;
import org.apache.http.client.config.RequestConfig; import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.*; import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.utils.URIBuilder; import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.entity.ContentType; import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.entity.StringEntity; import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicHeader; import org.apache.http.message.BasicHeader;
import org.apache.http.message.HeaderGroup;
import org.opengroup.osdu.core.common.http.HttpRequest; import org.opengroup.osdu.core.common.http.HttpRequest;
import org.opengroup.osdu.core.common.http.HttpResponse; import org.opengroup.osdu.core.common.http.HttpResponse;
import org.opengroup.osdu.core.common.http.IHttpClient; import org.opengroup.osdu.core.common.http.IHttpClient;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException; import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.http.RequestStatus; import org.opengroup.osdu.core.common.model.http.RequestStatus;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.http.MediaType; import org.springframework.http.MediaType;
import java.io.BufferedReader; import java.io.BufferedReader;
...@@ -33,33 +27,27 @@ import java.io.InputStreamReader; ...@@ -33,33 +27,27 @@ import java.io.InputStreamReader;
import java.net.SocketTimeoutException; import java.net.SocketTimeoutException;
import java.net.URISyntaxException; import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Supplier; import java.util.function.Supplier;
import java.util.logging.Logger;
public class HttpClientAzure implements IHttpClient { public class HttpClientAzure implements IHttpClient {
@Autowired
@Lazy
private JaxRsDpsLog log;
@Autowired private final static Logger LOGGER =
@Lazy Logger.getLogger(HttpClientAzure.class.getName());
private RetryAndTimeoutConfiguration configuration;
private final RequestConfig REQUEST_CONFIG = RequestConfig.custom()
.setConnectTimeout(60000)
.setConnectionRequestTimeout(60000)
.setSocketTimeout(60000).build();
private RetryConfig retryConfig; private RetryAndTimeoutConfiguration configuration;
public HttpClientAzure() { public HttpClientAzure() {
this.retryConfig = (new RetryAndTimeoutConfiguration()).getRetryConfig(); this.configuration = new RetryAndTimeoutConfiguration();
} }
public HttpClientAzure(RetryAndTimeoutConfiguration retryAndTimeoutConfiguration) { public HttpClientAzure(RetryAndTimeoutConfiguration retryAndTimeoutConfiguration) {
this.retryConfig = retryAndTimeoutConfiguration.getRetryConfig(); this.configuration = retryAndTimeoutConfiguration;
} }
@Override @Override
...@@ -68,13 +56,11 @@ public class HttpClientAzure implements IHttpClient { ...@@ -68,13 +56,11 @@ public class HttpClientAzure implements IHttpClient {
HttpResponse output = new HttpResponse(); HttpResponse output = new HttpResponse();
output.setRequest(request); output.setRequest(request);
Long curTimeStamp = System.currentTimeMillis();
HttpUriRequest req = null; HttpUriRequest req = null;
try { try {
req = getRequest(request); req = getRequest(request);
} catch (URISyntaxException e) { } catch (URISyntaxException e) {
//TODO Better handling of exception throw new AppException(HttpStatus.SC_BAD_REQUEST, "URL Syntax is in correct", "URL Syntax is incorrect", e);
e.printStackTrace();
} }
List<Header> httpHeaders = new ArrayList<>(); List<Header> httpHeaders = new ArrayList<>();
...@@ -88,12 +74,19 @@ public class HttpClientAzure implements IHttpClient { ...@@ -88,12 +74,19 @@ public class HttpClientAzure implements IHttpClient {
try { try {
CloseableHttpClient httpclient = HttpClients.custom() CloseableHttpClient httpclient = HttpClients.custom()
.setDefaultHeaders(httpHeaders) .setDefaultHeaders(httpHeaders)
// .setDefaultRequestConfig(REQUEST_CONFIG) .setDefaultRequestConfig(RequestConfig.custom()
.setConnectTimeout(this.configuration.getConnectTimeoutInMillis())
.setConnectionRequestTimeout(this.configuration.getConnectionRequestTimeoutInMillis())
.setSocketTimeout(this.configuration.getSocketTimeoutInMillis()).build())
.build(); .build();
try (CloseableHttpResponse response = this.getHttpClientWithRetry(httpclient, req)) { try (CloseableHttpResponse response = this.getHttpClientWithRetry(httpclient, req)) {
output.setResponseCode(response.getStatusLine().getStatusCode());
if (response.getStatusLine().getStatusCode() == 204) {
return output;
}
StringBuilder responseBuilder = new StringBuilder(); StringBuilder responseBuilder = new StringBuilder();
try (BufferedReader br = new BufferedReader(new InputStreamReader(response.getEntity().getContent()))) { try (BufferedReader br = new BufferedReader(new InputStreamReader(response.getEntity().getContent()))) {
String responsePayloadLine; String responsePayloadLine;
...@@ -101,17 +94,11 @@ public class HttpClientAzure implements IHttpClient { ...@@ -101,17 +94,11 @@ public class HttpClientAzure implements IHttpClient {
responseBuilder.append(responsePayloadLine); responseBuilder.append(responsePayloadLine);
} }
} }
String responseBody = responseBuilder.toString(); String responseBody = responseBuilder.toString();
// handle case where upstream server is running out of resources and throwing generic exception
// checkResponseMediaType(response, responseBody);
output.setResponseCode(response.getStatusLine().getStatusCode());
output.setBody(responseBody); output.setBody(responseBody);
if (output.getResponseCode() != 200) { if (output.getResponseCode() != 200) {
// log.info(String.format("method: %s | response code: %s | url: %s | error message: %s", request.getHttpMethod(), output.getResponseCode(), request.getUrl().toString(), responseBody)); LOGGER.info(String.format("method: %s | response code: %s | url: %s | error message: %s", request.getHttpMethod(), output.getResponseCode(), request.getUrl(), responseBody));
} }
return output; return output;
} }
...@@ -119,28 +106,47 @@ public class HttpClientAzure implements IHttpClient { ...@@ -119,28 +106,47 @@ public class HttpClientAzure implements IHttpClient {
throw new AppException(RequestStatus.SOCKET_TIMEOUT, "Socket time out", "Request cannot be completed in specified time", e); throw new AppException(RequestStatus.SOCKET_TIMEOUT, "Socket time out", "Request cannot be completed in specified time", e);
} catch (IOException e) { } catch (IOException e) {
throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Internal communication failure", "Internal communication failure", e); throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Internal communication failure", "Internal communication failure", e);
} finally {
// Long latency = System.currentTimeMillis() - curTimeStamp;
// log.info(String.format("method: %s | latency: %s | url: %s | correlation id: %s", req.getMethod(), latency, req.getURI().toString(), request.getHeaders().get(DpsHeaders.CORRELATION_ID)));
} }
} }
private CloseableHttpResponse getHttpClientWithRetry(CloseableHttpClient httpClient, HttpUriRequest request) { private CloseableHttpResponse getHttpClientWithRetry(CloseableHttpClient httpClient, HttpUriRequest request) {
RetryConfig config = this.retryConfig;
RetryRegistry registry = RetryRegistry.of(config); AtomicInteger retryCounter = new AtomicInteger();
Retry retry = registry.retry("retryPolicy", config); retryCounter.set(0);
RetryConfig retryConfig = RetryConfig
.<CloseableHttpResponse>custom()
.waitDuration(Duration.ofMillis(this.configuration.getWaitDurationInMillis()))
.maxAttempts(this.configuration.getMaxRetryAttempts()).retryOnResult(s -> {
try {
if (this.configuration.isRetryRequired() && this.configuration.getStatusCodesToBeRetried().contains(s.getStatusLine().getStatusCode())) {
if (retryCounter.get() < this.configuration.getMaxRetryAttempts() - 1) {
s.getEntity().getContent().close();
}
return true;
} else {
return false;
}
} catch (Exception e1) {
return false;
}
})
.build();
RetryRegistry registry = RetryRegistry.of(retryConfig);
Retry retry = registry.retry("Http client");
retry.getEventPublisher().onRetry(e -> {
retryCounter.getAndIncrement();
LOGGER.info("Retrying the request ");
});
Supplier<CloseableHttpResponse> httpClientSupplier = () -> { Supplier<CloseableHttpResponse> httpClientSupplier = () -> {
try { try {
return httpClient.execute(request); CloseableHttpResponse response = httpClient.execute(request);
return response;
} catch (IOException e) { } catch (IOException e) {
//TODO Better handling of exception throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Internal communication failure", "Internal communication failure", e);
e.printStackTrace();
} }
return null;
}; };
Supplier<CloseableHttpResponse> supplierWithRetry = Retry.decorateSupplier(retry, httpClientSupplier); return Retry.decorateSupplier(retry, httpClientSupplier).get();
return supplierWithRetry.get();
} }
private HttpUriRequest getRequest(HttpRequest request) throws URISyntaxException { private HttpUriRequest getRequest(HttpRequest request) throws URISyntaxException {
...@@ -148,7 +154,6 @@ public class HttpClientAzure implements IHttpClient { ...@@ -148,7 +154,6 @@ public class HttpClientAzure implements IHttpClient {
RequestBuilder requestBuilder = RequestBuilder.create(request.getHttpMethod().toUpperCase()) RequestBuilder requestBuilder = RequestBuilder.create(request.getHttpMethod().toUpperCase())
.setUri(request.getUrl()); .setUri(request.getUrl());
Map<String, String> queryParams = request.getQueryParams(); Map<String, String> queryParams = request.getQueryParams();
if (queryParams != null && !queryParams.isEmpty()) { if (queryParams != null && !queryParams.isEmpty()) {
for (String param : queryParams.keySet()) { for (String param : queryParams.keySet()) {
......
package org.opengroup.osdu.azure.retry; package org.opengroup.osdu.azure.retry;
import io.github.resilience4j.retry.RetryConfig;
import lombok.Data; import lombok.Data;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.opengroup.osdu.core.common.http.HttpResponse;
import org.springframework.stereotype.Component;
import org.springframework.web.context.annotation.RequestScope;
import java.time.Duration; import java.util.ArrayList;
@Data @Data
@Component
@RequestScope
public class RetryAndTimeoutConfiguration { public class RetryAndTimeoutConfiguration {
private int maxRetryAttempts = 3; private int maxRetryAttempts = 3;
private int waitDurationInMillis = 1000; private int waitDurationInMillis = 1000;
private boolean exponentialBackOff = false; private boolean exponentialBackOff = false;
private boolean retryRequired = true;
private int connectTimeoutInMillis = 60000;
private int connectionRequestTimeoutInMillis = 60000;
private int socketTimeoutInMillis = 60000;
private ArrayList<Integer> statusCodesToBeRetried;
public RetryConfig getRetryConfig() public RetryAndTimeoutConfiguration(){
{ statusCodesToBeRetried = new ArrayList<>();
return RetryConfig.<CloseableHttpResponse>custom()
.maxAttempts(10)
.waitDuration(Duration.ofMillis(1000))
.retryOnResult(response -> isRetryRequired(response))
.build();
} }
private boolean isRetryRequired(CloseableHttpResponse response)
{
if(response.getStatusLine().getStatusCode()==200 ||
response.getStatusLine().getStatusCode()==403 ||
response.getStatusLine().getStatusCode()==404 ||
response.getStatusLine().getStatusCode()==500)
// && retryConfigParameters.isRetryEnabled())
{
return true;
}
return false;
}
} }
...@@ -4,10 +4,7 @@ import org.apache.http.HttpEntity; ...@@ -4,10 +4,7 @@ import org.apache.http.HttpEntity;
import org.apache.http.HttpStatus; import org.apache.http.HttpStatus;
import org.apache.http.StatusLine; import org.apache.http.StatusLine;
import org.apache.http.client.ServiceUnavailableRetryStrategy; import org.apache.http.client.ServiceUnavailableRetryStrategy;
import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.*;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.entity.ContentType; import org.apache.http.entity.ContentType;
import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.HttpClientBuilder;
...@@ -18,7 +15,7 @@ import org.junit.jupiter.api.Test; ...@@ -18,7 +15,7 @@ import org.junit.jupiter.api.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.mockito.InjectMocks; import org.mockito.InjectMocks;
import org.mockito.Mock; import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner; import org.mockito.junit.MockitoJUnitRunner;
import org.opengroup.osdu.core.common.http.HttpClientHandler; import org.opengroup.osdu.core.common.http.HttpClientHandler;
import org.opengroup.osdu.core.common.http.HttpRequest; import org.opengroup.osdu.core.common.http.HttpRequest;
import org.opengroup.osdu.core.common.http.HttpResponse; import org.opengroup.osdu.core.common.http.HttpResponse;
...@@ -36,52 +33,41 @@ import java.nio.charset.StandardCharsets; ...@@ -36,52 +33,41 @@ import java.nio.charset.StandardCharsets;
import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.fail; import static org.junit.jupiter.api.Assertions.fail;;
import static org.mockito.Matchers.any; import static org.mockito.Mockito.*;
import static org.mockito.Mockito.mock; import static org.powermock.api.mockito.PowerMockito.mockStatic;
import static org.mockito.Mockito.when;
@Ignore
@RunWith(MockitoJUnitRunner.class) @RunWith(MockitoJUnitRunner.class)
@PrepareForTest({HttpClients.class}) @PrepareForTest({HttpClients.class})
public class HttpClientAzureTest { public class HttpClientAzureTest {
private static final String GET = "GET";
private static final String URL = "https://jsonplaceholder.typicode.com/posts/1";
private static final String RESPONSE = "{" +
" \"userId\": 1," +
" \"id\": 1," +
" \"title\": \"sunt aut facere repellat provident occaecati excepturi optio reprehenderit\"," +
" \"body\": \"quia et suscipit\\nsuscipit recusandae consequuntur expedita et cum\\nreprehenderit molestiae ut ut quas totam\\nnostrum rerum est autem sunt rem eveniet architecto\"" +
"}";
private static final String POST = "POST"; private static final String POST = "POST";
private static final String HEADER_NAME = "ANY_HEADER"; private static final String URL2 ="https://jsonplaceholder.typicode.com/posts/1";
private static final String HEADER_VALUE = "ANY_VALUE";
private static final String URL = "https://test.com";
private static final String RESPONSE = "hello world";
// @Mock
// private static DpsHeaders HEADERS;
//
//
//
// @Before
// public void setup() {
// HEADERS.put(HEADER_NAME, HEADER_VALUE);
//// mockStatic(HttpClients.class);
// }
@Test
public void demo()
{
HttpRequest request = HttpRequest.get().url(URL).build();
HttpClientAzure client =new HttpClientAzure();
HttpResponse response = client.send(request);
}
@Test @Test
public void demoTest() throws IOException, URISyntaxException { public void demoTest() throws IOException, URISyntaxException {
RetryAndTimeoutConfiguration config = mock(RetryAndTimeoutConfiguration.class); RetryAndTimeoutConfiguration config = mock(RetryAndTimeoutConfiguration.class);
assertNotNull(config); assertNotNull(config);
when(config.getMaxRetryAttempts()).thenReturn(5);
config.getStatusCodesToBeRetried().add(200);
HttpRequest httpRequest = mock(HttpRequest.class); HttpRequest httpRequest = mock(HttpRequest.class);
when(httpRequest.getHttpMethod()).thenReturn(POST); when(httpRequest.getHttpMethod()).thenReturn(GET);
when(httpRequest.getUrl()).thenReturn(URL); when(httpRequest.getUrl()).thenReturn(URL);
HttpUriRequest httpUriRequest = mock(HttpUriRequest.class); HttpGet httpUriRequest = mock(HttpGet.class);
when(httpUriRequest.getMethod()).thenReturn(POST); when(httpUriRequest.getMethod()).thenReturn(GET);
when(httpUriRequest.getURI()).thenReturn(new URI(URL)); when(httpUriRequest.getURI()).thenReturn(new URI(URL));
InputStream stream = new ByteArrayInputStream(RESPONSE.getBytes(StandardCharsets.UTF_8)); InputStream stream = new ByteArrayInputStream(RESPONSE.getBytes(StandardCharsets.UTF_8));
...@@ -91,22 +77,24 @@ public class HttpClientAzureTest { ...@@ -91,22 +77,24 @@ public class HttpClientAzureTest {
HttpEntity entity = mock(HttpEntity.class); HttpEntity entity = mock(HttpEntity.class);
when(entity.getContent()).thenReturn(stream); when(entity.getContent()).thenReturn(stream);
//
CloseableHttpResponse response = mock(CloseableHttpResponse.class); CloseableHttpResponse response = mock(CloseableHttpResponse.class);
when(response.getStatusLine()).thenReturn(statusLine); when(response.getStatusLine()).thenReturn(statusLine);
when(response.getEntity()).thenReturn(entity); when(response.getEntity()).thenReturn(entity);
CloseableHttpClient httpClient = mock(CloseableHttpClient.class); CloseableHttpClient httpClient = mock(CloseableHttpClient.class);
when(httpClient.execute(any(HttpUriRequest.class))).thenReturn(response); when(httpClient.execute(any(HttpPost.class))).thenReturn(response);
when(config.getRetryConfig()).thenReturn((new RetryAndTimeoutConfiguration()).getRetryConfig());
HttpClientAzure client = new HttpClientAzure(config); HttpClientAzure client = new HttpClientAzure(config);
org.opengroup.osdu.core.common.http.HttpResponse result = client.send(httpRequest); org.opengroup.osdu.core.common.http.HttpResponse result = client.send(httpRequest);
assertEquals(HttpStatus.SC_OK, result.getResponseCode()); assertEquals(HttpStatus.SC_OK, result.getResponseCode());
assertEquals(RESPONSE, result.getBody()); assertEquals(RESPONSE, result.getBody());
//verify(httpClient,times(5)).execute(httpUriRequest);
}
public void demoTest2(){
} }
} }
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment