Commit b488ada1 authored by Mingyang Zhu's avatar Mingyang Zhu Committed by Jason
Browse files

return 503 instead 401 if querying rebuilt cache timeout and adjust the Retry...

return 503 instead 401 if querying rebuilt cache timeout and adjust the Retry parameter based on the load test cosmos latency
parent 0c7c175c
......@@ -56,6 +56,8 @@ The following software have components provided under the terms of this license:
- Asynchronous Http Client (from )
- Asynchronous Http Client Netty Utils (from )
- AutoValue Annotations (from )
- Awaitility (from http://awaitility.org)
- Awaitility Proxy (from http://awaitility.org)
- Azure Metrics Spring Boot Starter (from https://github.com/Microsoft/azure-spring-boot)
- Bean Validation API (from http://beanvalidation.org)
- Byte Buddy (without dependencies) (from )
......@@ -64,6 +66,7 @@ The following software have components provided under the terms of this license:
- ClassMate (from http://github.com/cowtowncoder/java-classmate)
- Cloud Storage JSON API v1-rev58-1.21.0 (from )
- Commons IO (from http://commons.apache.org/io/)
- Commons IO (from http://commons.apache.org/io/)
- Commons Lang (from http://commons.apache.org/lang/)
- Converter: Jackson (from )
- Elastic JNA Distribution (from https://github.com/java-native-access/jna)
......@@ -185,6 +188,7 @@ The following software have components provided under the terms of this license:
- Non-Blocking Reactive Foundation for the JVM (from https://github.com/reactor/reactor)
- OAuth 2.0 SDK with OpenID Connect extensions (from https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions)
- Objenesis (from http://objenesis.org)
- Objenesis (from http://objenesis.org)
- OkHttp (from )
- OkHttp Logging Interceptor (from )
- OkHttp URLConnection (from )
......@@ -198,6 +202,7 @@ The following software have components provided under the terms of this license:
- Protocol Buffer extensions to the Google HTTP Client Library for Java. (from )
- Reactive Streams Netty driver (from https://github.com/reactor/reactor-netty)
- Redisson (from http://redisson.org)
- Redisson (from http://redisson.org)
- Retrofit (from )
- Simple XML (from http://simple.sourceforge.net)
- SnakeYAML (from http://www.snakeyaml.org)
......@@ -258,6 +263,7 @@ The following software have components provided under the terms of this license:
- error-prone annotations (from )
- error-prone annotations (from )
- exp4j (from http://www.objecthunter.net/exp4j)
- fst (from http://ruedigermoeller.github.io/fast-serialization/)
- io.grpc:grpc-alts (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-api (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-auth (from https://github.com/grpc/grpc-java)
......@@ -296,6 +302,8 @@ The following software have components provided under the terms of this license:
- rank-eval (from https://github.com/elastic/elasticsearch)
- resilience4j (from https://github.com/resilience4j/resilience4j)
- resilience4j (from https://github.com/resilience4j/resilience4j)
- resilience4j (from https://github.com/resilience4j/resilience4j)
- resilience4j (from https://github.com/resilience4j/resilience4j)
- rest (from https://github.com/elastic/elasticsearch)
- rest-high-level (from https://github.com/elastic/elasticsearch)
- rxjava (from https://github.com/ReactiveX/RxJava)
......@@ -337,6 +345,7 @@ The following software have components provided under the terms of this license:
- GAX (Google Api eXtensions) (from https://github.com/googleapis)
- Hamcrest (from http://hamcrest.org/JavaHamcrest/)
- Hamcrest Core (from http://hamcrest.org/)
- Hamcrest library (from )
- JLine (from )
- Jodd BeanUtil (from http://jodd.org)
- Jodd Core (from http://jodd.org)
......@@ -365,6 +374,7 @@ The following software have components provided under the terms of this license:
- Google Auth Library for Java - OAuth2 HTTP (from )
- Hamcrest (from http://hamcrest.org/JavaHamcrest/)
- Hamcrest Core (from http://hamcrest.org/)
- Hamcrest library (from )
- JLine (from )
- JavaBeans Activation Framework API jar (from )
- Jodd BeanUtil (from http://jodd.org)
......@@ -383,6 +393,7 @@ The following software have components provided under the terms of this license:
- Protocol Buffer Java API (from https://developers.google.com/protocol-buffers/)
- Protocol Buffers [Util] (from )
- Redisson (from http://redisson.org)
- Redisson (from http://redisson.org)
- Reflections (from http://github.com/ronmamo/reflections)
- SnakeYAML (from http://www.snakeyaml.org)
- SnakeYAML (from http://www.snakeyaml.org)
......@@ -567,6 +578,7 @@ The following software have components provided under the terms of this license:
- Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
- fst (from http://ruedigermoeller.github.io/fast-serialization/)
========================================================================
LGPL-2.1-or-later
......
......@@ -19,6 +19,11 @@
<springfox-version>2.7.0</springfox-version>
<tomcat-embed-core.version>9.0.37</tomcat-embed-core.version>
<gremlin.version>3.4.10</gremlin.version>
<redisson.version>3.11.6</redisson.version>
<resilience4j.version>1.7.0</resilience4j.version>
<embedded-resdis.version>0.7.1</embedded-resdis.version>
<awaitility.version>3.0.0</awaitility.version>
<awaitility.proxy.version>3.0.0</awaitility.proxy.version>
</properties>
<dependencies>
......@@ -151,6 +156,18 @@
<version>${springfox-version}</version>
</dependency>
<dependency>
<groupId>org.redisson</groupId>
<artifactId>redisson</artifactId>
<version>${redisson.version}</version>
</dependency>
<dependency>
<groupId>io.github.resilience4j</groupId>
<artifactId>resilience4j-retry</artifactId>
<version>${resilience4j.version}</version>
</dependency>
<!-- test scope dependencies -->
<dependency>
<groupId>org.springframework.boot</groupId>
......@@ -187,12 +204,30 @@
<version>2.0.2-beta</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>it.ozimov</groupId>
<artifactId>embedded-redis</artifactId>
<version>${embedded-resdis.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.tinkerpop</groupId>
<artifactId>tinkergraph-gremlin</artifactId>
<version>${gremlin.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.awaitility</groupId>
<artifactId>awaitility</artifactId>
<version>${awaitility.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.awaitility</groupId>
<artifactId>awaitility-proxy</artifactId>
<version>${awaitility.proxy.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
......
......@@ -4,6 +4,9 @@ import com.azure.security.keyvault.secrets.SecretClient;
import org.opengroup.osdu.azure.KeyVaultFacade;
import org.opengroup.osdu.core.common.cache.RedisCache;
import org.opengroup.osdu.entitlements.v2.model.ParentReferences;
import org.redisson.Redisson;
import org.redisson.api.RedissonClient;
import org.redisson.config.Config;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
......@@ -15,15 +18,21 @@ public class CacheConfig {
@Autowired
private SecretClient secretClient;
@Value("${redis.port:6380}")
@Value("${redis.port}")
private int redisPort;
@Value("${redis.database:8}")
@Value("${redis.database}")
private int redisDatabase;
@Value("${redisson.connection.timeout}")
private int redissonConnectionTimeout;
@Value("${spring.application.name}")
private String applicationName;
@Bean
public int getRedisTtlSeconds() {
if(System.getenv("REDIS_TTL_SECONDS") == null) return 1;
if (System.getenv("REDIS_TTL_SECONDS") == null) return 1;
else return Integer.parseInt(System.getenv("REDIS_TTL_SECONDS"));
}
......@@ -33,6 +42,17 @@ public class CacheConfig {
ParentReferences.class);
}
@Bean
public RedissonClient getRedissonClient() {
Config config = new Config();
config.useSingleServer().setAddress(String.format("rediss://%s:%d",getRedisHostname(), redisPort))
.setPassword(getRedisPassword())
.setDatabase(redisDatabase)
.setTimeout(redissonConnectionTimeout)
.setClientName(applicationName);
return Redisson.create(config);
}
public String getRedisHostname() {
return KeyVaultFacade.getSecretWithValidation(secretClient, "redis-hostname");
}
......
package org.opengroup.osdu.entitlements.v2.azure.config;
import io.github.resilience4j.core.IntervalFunction;
import io.github.resilience4j.retry.Retry;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import java.util.Objects;
@Configuration
public class RetryConfig {
@Value("${spring.application.name}")
private String applicationName;
@Value("${cache.retry.max}")
private int maxRetry;
@Value("${cache.retry.interval}")
private int retryInterval;
@Value("${cache.retry.random.factor}")
private float randomFactor;
/**
Retry client with internal of retryInterval +/- random(randomFactor*retryInterval) for maximum maxRetry times.
This client is used for the concurrent cache rebuild thread which do not acquire the lock to wait and get the rebuilt cache result
*/
@Bean
public Retry getRetryClient() {
return Retry.of(applicationName, io.github.resilience4j.retry.RetryConfig.custom()
.maxAttempts(maxRetry)
.intervalFunction(IntervalFunction.ofRandomized(retryInterval, randomFactor))
.retryOnResult(Objects::isNull)
.build());
}
}
package org.opengroup.osdu.entitlements.v2.azure.service;
import io.github.resilience4j.retry.Retry;
import lombok.RequiredArgsConstructor;
import org.opengroup.osdu.core.common.cache.ICache;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.entitlements.v2.azure.service.metrics.hitsnmisses.HitsNMissesMetricService;
import org.opengroup.osdu.entitlements.v2.model.EntityNode;
import org.opengroup.osdu.entitlements.v2.model.ParentReference;
import org.opengroup.osdu.entitlements.v2.model.ParentReferences;
import org.opengroup.osdu.entitlements.v2.service.GroupCacheService;
import org.opengroup.osdu.entitlements.v2.azure.service.metrics.hitsnmisses.HitsNMissesMetricService;
import org.opengroup.osdu.entitlements.v2.spi.retrievegroup.RetrieveGroupRepo;
import org.redisson.api.RLock;
import org.redisson.api.RedissonClient;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
import java.util.Collections;
import java.util.Set;
import java.util.concurrent.TimeUnit;
@Service
@RequiredArgsConstructor
public class GroupCacheServiceAzure implements GroupCacheService {
private final JaxRsDpsLog log;
private final RetrieveGroupRepo retrieveGroupRepo;
private final ICache<String, ParentReferences> redisGroupCache;
private final HitsNMissesMetricService metricService;
private final RedissonClient redissonClient;
private final Retry retry;
@Value("${redisson.lock.acquisition.timeout}")
private int redissonLockAcquisitionTimeOut;
@Value("${redisson.lock.expiration}")
private int redissonLockExpiration;
@Override
public Set<ParentReference> getFromPartitionCache(String requesterId, String partitionId) {
String key = String.format("%s-%s", requesterId, partitionId);
ParentReferences parentReferences = redisGroupCache.get(key);
if (parentReferences == null) {
metricService.sendMissesMetric();
EntityNode entityNode = EntityNode.createMemberNodeForNewUser(requesterId, partitionId);
Set<ParentReference> allParents = retrieveGroupRepo.loadAllParents(entityNode).getParentReferences();
parentReferences = new ParentReferences();
parentReferences.setParentReferencesOfUser(allParents);
redisGroupCache.put(key, parentReferences);
RLock cacheEntryLock = redissonClient.getLock(key);
return lockCacheEntryAndRebuild(cacheEntryLock, key, requesterId, partitionId);
} else {
metricService.sendHitsMetric();
return parentReferences.getParentReferencesOfUser();
}
}
/**
The unblock function may throw exception when cache update takes longer than the lock expiration time,
so when the time it tries to unlock the lock has already expired or re-acquired by another thread. In this case, since the lock is already released, we just
log the error message without doing anything further. The log is for the tracking purpose to understand the possibility so we can adjust parameters accordingly.
Refer to: https://github.com/redisson/redisson/issues/581
*/
private Set<ParentReference> lockCacheEntryAndRebuild(RLock cacheEntryLock, String key, String requesterId, String partitionId) {
boolean locked = false;
try {
locked = cacheEntryLock.tryLock(redissonLockAcquisitionTimeOut, redissonLockExpiration, TimeUnit.MILLISECONDS);
if (locked) {
metricService.sendMissesMetric();
ParentReferences parentReferences = rebuildCache(requesterId, partitionId);
redisGroupCache.put(key, parentReferences);
return parentReferences.getParentReferencesOfUser();
} else {
ParentReferences parentReferences = Retry.decorateSupplier(retry, () -> redisGroupCache.get(key)).get();
if (parentReferences == null) {
metricService.sendMissesMetric();
} else {
metricService.sendHitsMetric();
return parentReferences.getParentReferencesOfUser();
}
}
} catch (InterruptedException ex) {
log.error(String.format("InterruptedException caught when lock the cache key %s: %s", key, ex));
Thread.currentThread().interrupt();
} finally {
if (locked) {
try {
cacheEntryLock.unlock();
} catch (Exception ex) {
log.warning(String.format("unlock exception: %s", ex));
}
}
}
return parentReferences.getParentReferencesOfUser();
throw new AppException(
HttpStatus.SERVICE_UNAVAILABLE.value(),
HttpStatus.SERVICE_UNAVAILABLE.getReasonPhrase(),
"Failed to get the groups");
}
private ParentReferences rebuildCache(String requesterId, String partitionId) {
EntityNode entityNode = EntityNode.createMemberNodeForNewUser(requesterId, partitionId);
Set<ParentReference> allParents = retrieveGroupRepo.loadAllParents(entityNode).getParentReferences();
ParentReferences parentReferences = new ParentReferences();
parentReferences.setParentReferencesOfUser(allParents);
return parentReferences;
}
}
......@@ -45,3 +45,13 @@ app.graph.db.sslEnabled=true
app.projectId=evd-ddl-us-services
app.domain=${service_domain_name}
app.quota.users.data.root=${root_data_group_quota}
# Cache
redis.port=6380
redis.database=8
redisson.connection.timeout=100000
redisson.lock.acquisition.timeout=10
redisson.lock.expiration=5000
cache.retry.max=15
cache.retry.interval=200
cache.retry.random.factor=0.1
package org.opengroup.osdu.entitlements.v2.azure.service;
import io.github.resilience4j.retry.Retry;
import org.awaitility.Duration;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.AdditionalAnswers;
import org.mockito.Mock;
import org.mockito.junit.MockitoJUnitRunner;
import org.mockito.stubbing.Answer;
import org.opengroup.osdu.core.common.cache.ICache;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.entitlements.v2.azure.service.metrics.hitsnmisses.HitsNMissesMetricService;
import org.opengroup.osdu.entitlements.v2.model.EntityNode;
import org.opengroup.osdu.entitlements.v2.model.ParentReference;
import org.opengroup.osdu.entitlements.v2.model.ParentReferences;
import org.opengroup.osdu.entitlements.v2.model.ParentTreeDto;
import org.opengroup.osdu.entitlements.v2.azure.service.metrics.hitsnmisses.HitsNMissesMetricService;
import org.opengroup.osdu.entitlements.v2.spi.retrievegroup.RetrieveGroupRepo;
import org.redisson.Redisson;
import org.redisson.api.RedissonClient;
import org.redisson.config.Config;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.context.TestConfiguration;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Primary;
import org.springframework.test.context.TestPropertySource;
import org.springframework.test.context.junit4.SpringRunner;
import redis.embedded.RedisServer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import static org.awaitility.Awaitility.await;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class)
@SpringBootTest
@TestPropertySource(properties = {"spring.application.name=test",
"redis.redisson.lock.acquisition.timeout=10",
"redisson.lock.expiration=5000",
"cache.retry.max=15",
"cache.retry.interval=200",
"cache.retry.random.factor=0.1"})
@RunWith(SpringRunner.class)
public class GroupCacheServiceAzureTest {
@TestConfiguration
static class TestConfig {
@Bean
@Primary
public RedissonClient mockRedissonClient() {
Config config = new Config();
config.useSingleServer().setAddress(String.format("redis://%s:%d", "localhost", 7000))
.setPassword("pass")
.setDatabase(0)
.setTimeout(100000)
.setClientName("test");
return Redisson.create(config);
}
}
private Set<ParentReference> parents = new HashSet<>();
private ParentReferences parentReferences = new ParentReferences();
private EntityNode requester;
@Mock
@MockBean
private RetrieveGroupRepo retrieveGroupRepo;
@Mock
@MockBean
private HitsNMissesMetricService metricService;
@MockBean
private ICache<String, ParentReferences> redisGroupCache;
@MockBean
private JaxRsDpsLog log;
@Mock
private ParentTreeDto parentTreeDto;
@Mock
private HitsNMissesMetricService metricService;
@InjectMocks
@Autowired
private RedissonClient redissonClient;
@Autowired
private Retry retry;
@Autowired
private GroupCacheServiceAzure sut;
private static RedisServer redisServer;
@BeforeClass
public static void setupClass() {
redisServer = RedisServer.builder().port(7000).setting("requirepass pass").build();
redisServer.start();
}
@AfterClass
public static void end() {
redisServer.stop();
}
@Before
public void setup() {
ParentReference parent1 = ParentReference.builder()
......@@ -84,4 +152,82 @@ public class GroupCacheServiceAzureTest {
verify(this.retrieveGroupRepo, times(0)).loadAllParents(this.requester);
verify(this.metricService, times(1)).sendHitsMetric();
}
@Test
public void shouldOnlyOneThreadRebuildCacheInConcurrentScenarioAndOtherThreadWaitAndReturnTheRebuiltCache() throws InterruptedException {
List<ParentReferences> cacheValues = new ArrayList<>();
for (int i = 0; i < 15; i++) {
cacheValues.add(null);
}
cacheValues.add(this.parentReferences);
when(this.redisGroupCache.get("requesterId-dp")).thenAnswer(AdditionalAnswers.returnsElementsOf(cacheValues));
when(this.retrieveGroupRepo.loadAllParents(this.requester)).thenAnswer((Answer<ParentTreeDto>) invocationOnMock -> {
await().atLeast(Duration.FIVE_SECONDS);
return parentTreeDto;
});
when(this.parentTreeDto.getParentReferences()).thenReturn(this.parents);
int threads = 3;
ExecutorService executor = Executors.newFixedThreadPool(threads);
List<Callable<Set<ParentReference>>> tasks = new ArrayList<>();
for (int i = 0; i < threads; i++) {
Callable<Set<ParentReference>> task = () -> sut.getFromPartitionCache("requesterId", "dp");
tasks.add(task);
}
List<Future<Set<ParentReference>>> responses = executor.invokeAll(tasks);
executor.shutdown();
verify(this.retrieveGroupRepo, times(1)).loadAllParents(this.requester);
responses.forEach(result -> {
try {
assertEquals(this.parents, result.get());
} catch (InterruptedException | ExecutionException e) {
fail("No exception expected");
}
});
}
@Test
public void shouldReturnEmptyIfTimeout() throws InterruptedException {
when(this.redisGroupCache.get("requesterId-dp")).thenReturn(null);
when(this.retrieveGroupRepo.loadAllParents(this.requester)).thenAnswer((Answer<ParentTreeDto>) invocationOnMock -> {
await().atLeast(Duration.FIVE_SECONDS);
return parentTreeDto;
});
when(this.parentTreeDto.getParentReferences()).thenReturn(this.parents);
int threads = 3;
ExecutorService executor = Executors.newFixedThreadPool(threads);
List<Callable<Set<ParentReference>>> tasks = new ArrayList<>();
for (int i = 0; i < threads; i++) {
Callable<Set<ParentReference>> task = () -> sut.getFromPartitionCache("requesterId", "dp");
tasks.add(task);
}
List<Future<Set<ParentReference>>> responses = executor.invokeAll(tasks);
executor.shutdown();
verify(this.retrieveGroupRepo, times(1)).loadAllParents(this.requester);
assertEquals(1, responses.stream().filter(result -> {
try {
return this.parents.equals(result.get());
} catch (InterruptedException | ExecutionException e) {
}
return false;
}).count());
assertEquals(2, responses.stream().filter(result -> {
try {
result.get();
} catch (InterruptedException | ExecutionException e) {
AppException appEx = (AppException)e.getCause();
if (appEx.getError().getCode() == 503) {
return true;
}
}
return false;
}).count());
}
}