Commit ecbd1856 authored by Muskan Srivastava's avatar Muskan Srivastava
Browse files

changes for retry

parent 264f172b
Pipeline #41857 failed with stage
in 1 minute and 53 seconds
......@@ -6,6 +6,8 @@ import org.opengroup.osdu.core.common.http.json.HttpResponseBodyMapper;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
public class EntitlementsFactoryAzure implements IEntitlementsFactory {
......@@ -24,6 +26,10 @@ public class EntitlementsFactoryAzure implements IEntitlementsFactory {
}
RetryAndTimeoutConfiguration config = new RetryAndTimeoutConfiguration();
config.setMaxRetryAttempts(10);
ArrayList<Integer> statusCode = new ArrayList<>();
statusCode.add(403);
statusCode.add(200);
config.setStatusCodesToBeRetried(statusCode);
return new EntitlementsService(this.config,
new HttpClientAzure(config),
headers, mapper);
......
package org.opengroup.osdu.azure.retry;
import com.google.api.client.http.HttpMethods;
import io.github.resilience4j.retry.Retry;
import io.github.resilience4j.retry.RetryConfig;
import io.github.resilience4j.retry.RetryRegistry;
......@@ -8,23 +7,18 @@ import org.apache.http.Header;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpStatus;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.*;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.entity.ContentType;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicHeader;
import org.apache.http.message.HeaderGroup;
import org.opengroup.osdu.core.common.http.HttpRequest;
import org.opengroup.osdu.core.common.http.HttpResponse;
import org.opengroup.osdu.core.common.http.IHttpClient;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.http.RequestStatus;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.http.MediaType;
import java.io.BufferedReader;
......@@ -33,33 +27,27 @@ import java.io.InputStreamReader;
import java.net.SocketTimeoutException;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Supplier;
import java.util.logging.Logger;
public class HttpClientAzure implements IHttpClient {
@Autowired
@Lazy
private JaxRsDpsLog log;
@Autowired
@Lazy
private RetryAndTimeoutConfiguration configuration;
private final RequestConfig REQUEST_CONFIG = RequestConfig.custom()
.setConnectTimeout(60000)
.setConnectionRequestTimeout(60000)
.setSocketTimeout(60000).build();
private final static Logger LOGGER =
Logger.getLogger(HttpClientAzure.class.getName());
private RetryConfig retryConfig;
private RetryAndTimeoutConfiguration configuration;
public HttpClientAzure() {
this.retryConfig = (new RetryAndTimeoutConfiguration()).getRetryConfig();
this.configuration = new RetryAndTimeoutConfiguration();
}
public HttpClientAzure(RetryAndTimeoutConfiguration retryAndTimeoutConfiguration) {
this.retryConfig = retryAndTimeoutConfiguration.getRetryConfig();
this.configuration = retryAndTimeoutConfiguration;
}
@Override
......@@ -68,13 +56,11 @@ public class HttpClientAzure implements IHttpClient {
HttpResponse output = new HttpResponse();
output.setRequest(request);
Long curTimeStamp = System.currentTimeMillis();
HttpUriRequest req = null;
try {
req = getRequest(request);
} catch (URISyntaxException e) {
//TODO Better handling of exception
e.printStackTrace();
throw new AppException(HttpStatus.SC_BAD_REQUEST, "URL Syntax is in correct", "URL Syntax is incorrect", e);
}
List<Header> httpHeaders = new ArrayList<>();
......@@ -88,12 +74,19 @@ public class HttpClientAzure implements IHttpClient {
try {
CloseableHttpClient httpclient = HttpClients.custom()
.setDefaultHeaders(httpHeaders)
// .setDefaultRequestConfig(REQUEST_CONFIG)
.setDefaultRequestConfig(RequestConfig.custom()
.setConnectTimeout(this.configuration.getConnectTimeoutInMillis())
.setConnectionRequestTimeout(this.configuration.getConnectionRequestTimeoutInMillis())
.setSocketTimeout(this.configuration.getSocketTimeoutInMillis()).build())
.build();
try (CloseableHttpResponse response = this.getHttpClientWithRetry(httpclient, req)) {
output.setResponseCode(response.getStatusLine().getStatusCode());
if (response.getStatusLine().getStatusCode() == 204) {
return output;
}
StringBuilder responseBuilder = new StringBuilder();
try (BufferedReader br = new BufferedReader(new InputStreamReader(response.getEntity().getContent()))) {
String responsePayloadLine;
......@@ -101,17 +94,11 @@ public class HttpClientAzure implements IHttpClient {
responseBuilder.append(responsePayloadLine);
}
}
String responseBody = responseBuilder.toString();
// handle case where upstream server is running out of resources and throwing generic exception
// checkResponseMediaType(response, responseBody);
output.setResponseCode(response.getStatusLine().getStatusCode());
output.setBody(responseBody);
if (output.getResponseCode() != 200) {
// log.info(String.format("method: %s | response code: %s | url: %s | error message: %s", request.getHttpMethod(), output.getResponseCode(), request.getUrl().toString(), responseBody));
LOGGER.info(String.format("method: %s | response code: %s | url: %s | error message: %s", request.getHttpMethod(), output.getResponseCode(), request.getUrl(), responseBody));
}
return output;
}
......@@ -119,28 +106,47 @@ public class HttpClientAzure implements IHttpClient {
throw new AppException(RequestStatus.SOCKET_TIMEOUT, "Socket time out", "Request cannot be completed in specified time", e);
} catch (IOException e) {
throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Internal communication failure", "Internal communication failure", e);
} finally {
// Long latency = System.currentTimeMillis() - curTimeStamp;
// log.info(String.format("method: %s | latency: %s | url: %s | correlation id: %s", req.getMethod(), latency, req.getURI().toString(), request.getHeaders().get(DpsHeaders.CORRELATION_ID)));
}
}
private CloseableHttpResponse getHttpClientWithRetry(CloseableHttpClient httpClient, HttpUriRequest request) {
RetryConfig config = this.retryConfig;
RetryRegistry registry = RetryRegistry.of(config);
Retry retry = registry.retry("retryPolicy", config);
AtomicInteger retryCounter = new AtomicInteger();
retryCounter.set(0);
RetryConfig retryConfig = RetryConfig
.<CloseableHttpResponse>custom()
.waitDuration(Duration.ofMillis(this.configuration.getWaitDurationInMillis()))
.maxAttempts(this.configuration.getMaxRetryAttempts()).retryOnResult(s -> {
try {
if (this.configuration.isRetryRequired() && this.configuration.getStatusCodesToBeRetried().contains(s.getStatusLine().getStatusCode())) {
if (retryCounter.get() < this.configuration.getMaxRetryAttempts() - 1) {
s.getEntity().getContent().close();
}
return true;
} else {
return false;
}
} catch (Exception e1) {
return false;
}
})
.build();
RetryRegistry registry = RetryRegistry.of(retryConfig);
Retry retry = registry.retry("Http client");
retry.getEventPublisher().onRetry(e -> {
retryCounter.getAndIncrement();
LOGGER.info("Retrying the request ");
});
Supplier<CloseableHttpResponse> httpClientSupplier = () -> {
try {
return httpClient.execute(request);
CloseableHttpResponse response = httpClient.execute(request);
return response;
} catch (IOException e) {
//TODO Better handling of exception
e.printStackTrace();
throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Internal communication failure", "Internal communication failure", e);
}
return null;
};
Supplier<CloseableHttpResponse> supplierWithRetry = Retry.decorateSupplier(retry, httpClientSupplier);
return supplierWithRetry.get();
return Retry.decorateSupplier(retry, httpClientSupplier).get();
}
private HttpUriRequest getRequest(HttpRequest request) throws URISyntaxException {
......@@ -148,7 +154,6 @@ public class HttpClientAzure implements IHttpClient {
RequestBuilder requestBuilder = RequestBuilder.create(request.getHttpMethod().toUpperCase())
.setUri(request.getUrl());
Map<String, String> queryParams = request.getQueryParams();
if (queryParams != null && !queryParams.isEmpty()) {
for (String param : queryParams.keySet()) {
......
package org.opengroup.osdu.azure.retry;
import io.github.resilience4j.retry.RetryConfig;
import lombok.Data;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.opengroup.osdu.core.common.http.HttpResponse;
import org.springframework.stereotype.Component;
import org.springframework.web.context.annotation.RequestScope;
import java.time.Duration;
import java.util.ArrayList;
@Data
@Component
@RequestScope
public class RetryAndTimeoutConfiguration {
private int maxRetryAttempts = 3;
private int waitDurationInMillis = 1000;
private boolean exponentialBackOff = false;
private boolean retryRequired = true;
private int connectTimeoutInMillis = 60000;
private int connectionRequestTimeoutInMillis = 60000;
private int socketTimeoutInMillis = 60000;
private ArrayList<Integer> statusCodesToBeRetried;
public RetryConfig getRetryConfig()
{
return RetryConfig.<CloseableHttpResponse>custom()
.maxAttempts(10)
.waitDuration(Duration.ofMillis(1000))
.retryOnResult(response -> isRetryRequired(response))
.build();
public RetryAndTimeoutConfiguration(){
statusCodesToBeRetried = new ArrayList<>();
}
private boolean isRetryRequired(CloseableHttpResponse response)
{
if(response.getStatusLine().getStatusCode()==200 ||
response.getStatusLine().getStatusCode()==403 ||
response.getStatusLine().getStatusCode()==404 ||
response.getStatusLine().getStatusCode()==500)
// && retryConfigParameters.isRetryEnabled())
{
return true;
}
return false;
}
}
......@@ -4,10 +4,7 @@ import org.apache.http.HttpEntity;
import org.apache.http.HttpStatus;
import org.apache.http.StatusLine;
import org.apache.http.client.ServiceUnavailableRetryStrategy;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.*;
import org.apache.http.entity.ContentType;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
......@@ -18,7 +15,7 @@ import org.junit.jupiter.api.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
import org.mockito.junit.MockitoJUnitRunner;
import org.opengroup.osdu.core.common.http.HttpClientHandler;
import org.opengroup.osdu.core.common.http.HttpRequest;
import org.opengroup.osdu.core.common.http.HttpResponse;
......@@ -36,52 +33,41 @@ import java.nio.charset.StandardCharsets;
import static org.junit.Assert.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.fail;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.junit.jupiter.api.Assertions.fail;;
import static org.mockito.Mockito.*;
import static org.powermock.api.mockito.PowerMockito.mockStatic;
@Ignore
@RunWith(MockitoJUnitRunner.class)
@PrepareForTest({HttpClients.class})
public class HttpClientAzureTest {
private static final String GET = "GET";
private static final String URL = "https://jsonplaceholder.typicode.com/posts/1";
private static final String RESPONSE = "{" +
" \"userId\": 1," +
" \"id\": 1," +
" \"title\": \"sunt aut facere repellat provident occaecati excepturi optio reprehenderit\"," +
" \"body\": \"quia et suscipit\\nsuscipit recusandae consequuntur expedita et cum\\nreprehenderit molestiae ut ut quas totam\\nnostrum rerum est autem sunt rem eveniet architecto\"" +
"}";
private static final String POST = "POST";
private static final String HEADER_NAME = "ANY_HEADER";
private static final String HEADER_VALUE = "ANY_VALUE";
private static final String URL = "https://test.com";
private static final String RESPONSE = "hello world";
private static final String URL2 ="https://jsonplaceholder.typicode.com/posts/1";
// @Mock
// private static DpsHeaders HEADERS;
//
//
//
// @Before
// public void setup() {
// HEADERS.put(HEADER_NAME, HEADER_VALUE);
//// mockStatic(HttpClients.class);
// }
@Test
public void demo()
{
HttpRequest request = HttpRequest.get().url(URL).build();
HttpClientAzure client =new HttpClientAzure();
HttpResponse response = client.send(request);
}
@Test
public void demoTest() throws IOException, URISyntaxException {
RetryAndTimeoutConfiguration config = mock(RetryAndTimeoutConfiguration.class);
assertNotNull(config);
when(config.getMaxRetryAttempts()).thenReturn(5);
config.getStatusCodesToBeRetried().add(200);
HttpRequest httpRequest = mock(HttpRequest.class);
when(httpRequest.getHttpMethod()).thenReturn(POST);
when(httpRequest.getHttpMethod()).thenReturn(GET);
when(httpRequest.getUrl()).thenReturn(URL);
HttpUriRequest httpUriRequest = mock(HttpUriRequest.class);
when(httpUriRequest.getMethod()).thenReturn(POST);
HttpGet httpUriRequest = mock(HttpGet.class);
when(httpUriRequest.getMethod()).thenReturn(GET);
when(httpUriRequest.getURI()).thenReturn(new URI(URL));
InputStream stream = new ByteArrayInputStream(RESPONSE.getBytes(StandardCharsets.UTF_8));
......@@ -91,22 +77,24 @@ public class HttpClientAzureTest {
HttpEntity entity = mock(HttpEntity.class);
when(entity.getContent()).thenReturn(stream);
//
CloseableHttpResponse response = mock(CloseableHttpResponse.class);
when(response.getStatusLine()).thenReturn(statusLine);
when(response.getEntity()).thenReturn(entity);
CloseableHttpClient httpClient = mock(CloseableHttpClient.class);
when(httpClient.execute(any(HttpUriRequest.class))).thenReturn(response);
when(httpClient.execute(any(HttpPost.class))).thenReturn(response);
when(config.getRetryConfig()).thenReturn((new RetryAndTimeoutConfiguration()).getRetryConfig());
HttpClientAzure client = new HttpClientAzure(config);
org.opengroup.osdu.core.common.http.HttpResponse result = client.send(httpRequest);
assertEquals(HttpStatus.SC_OK, result.getResponseCode());
assertEquals(RESPONSE, result.getBody());
//verify(httpClient,times(5)).execute(httpUriRequest);
}
public void demoTest2(){
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment