Commit 9e2dac26 authored by Tika Lestari [SLB]'s avatar Tika Lestari [SLB] Committed by Jason
Browse files

Implement redis cache and add metrics

parent ff613c50
......@@ -4,7 +4,7 @@ import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.entitlements.v2.model.ParentReference;
import org.opengroup.osdu.entitlements.v2.service.IGroupCacheService;
import org.opengroup.osdu.entitlements.v2.service.GroupCacheService;
import org.opengroup.osdu.entitlements.v2.util.RequestInfoUtilService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
......@@ -21,7 +21,7 @@ public class AuthorizationServiceEntitlements implements AuthorizationService {
@Autowired
private JaxRsDpsLog log;
@Autowired
private IGroupCacheService groupCacheService;
private GroupCacheService groupCacheService;
@Autowired
private RequestInfoUtilService requestInfoUtilService;
......
package org.opengroup.osdu.entitlements.v2.config;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import com.fasterxml.jackson.databind.MapperFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.springframework.context.annotation.Primary;
@Configuration
public class ConfigMapper {
@Bean
@Primary
public ObjectMapper objectMapper() {
ObjectMapper mapper = new ObjectMapper();
mapper.configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_PROPERTIES, true);
return mapper;
}
}
\ No newline at end of file
package org.opengroup.osdu.entitlements.v2.model;
import lombok.Data;
import lombok.Generated;
import lombok.NoArgsConstructor;
import java.util.Set;
@Data
@Generated
@NoArgsConstructor
public class ParentReferences {
private Set<ParentReference> parentReferencesOfUser;
}
\ No newline at end of file
package org.opengroup.osdu.entitlements.v2.service;
import org.opengroup.osdu.entitlements.v2.model.ParentReference;
import java.util.Set;
public interface GroupCache {
Set<ParentReference> getGroupCache(String requesterId);
void addGroupCache(String requesterId, Set<ParentReference> parents);
}
package org.opengroup.osdu.entitlements.v2.service;
import lombok.RequiredArgsConstructor;
import org.opengroup.osdu.entitlements.v2.model.EntityNode;
import org.opengroup.osdu.entitlements.v2.model.ParentReference;
import org.opengroup.osdu.entitlements.v2.spi.retrievegroup.RetrieveGroupRepo;
import org.springframework.stereotype.Service;
import java.util.Set;
@Service
@RequiredArgsConstructor
public class GroupCacheService implements IGroupCacheService {
private final RetrieveGroupRepo retrieveGroupRepo;
private final GroupCache vmGroupCache;
public interface GroupCacheService {
@Override
public Set<ParentReference> getFromPartitionCache(String requesterId, String partitionId) {
String key = String.format("%s-%s", requesterId, partitionId);
Set<ParentReference> result = vmGroupCache.getGroupCache(key);
if (result == null) {
EntityNode entityNode = EntityNode.createMemberNodeForNewUser(requesterId, partitionId);
result = retrieveGroupRepo.loadAllParents(entityNode).getParentReferences();
vmGroupCache.addGroupCache(key, result);
}
return result;
}
Set<ParentReference> getFromPartitionCache(String requesterId, String partitionId);
}
package org.opengroup.osdu.entitlements.v2.service;
import org.opengroup.osdu.entitlements.v2.model.ParentReference;
import java.util.Set;
public interface IGroupCacheService {
Set<ParentReference> getFromPartitionCache(String requesterId, String partitionId);
}
......@@ -24,7 +24,7 @@ public class ListGroupService {
private final AuditLogger auditLogger;
private final RequestInfo requestInfo;
private final RetrieveGroupRepo retrieveGroupRepo;
private final IGroupCacheService groupCacheService;
private final GroupCacheService groupCacheService;
public Set<ParentReference> getGroups(ListGroupServiceDto listGroupServiceDto) {
log.info(String.format("ListGroupService#run timestamp: %d", System.currentTimeMillis()));
......
......@@ -9,7 +9,7 @@ import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.entitlements.v2.model.ParentReference;
import org.opengroup.osdu.entitlements.v2.service.IGroupCacheService;
import org.opengroup.osdu.entitlements.v2.service.GroupCacheService;
import org.opengroup.osdu.entitlements.v2.util.RequestInfoUtilService;
import org.powermock.modules.junit4.PowerMockRunner;
......@@ -29,7 +29,7 @@ public class AuthorizationServiceEntitlementsTest {
@Mock
private JaxRsDpsLog log;
@Mock
private IGroupCacheService groupCacheService;
private GroupCacheService groupCacheService;
@Mock
private RequestInfoUtilService requestInfoUtilService;
@Mock
......
package org.opengroup.osdu.entitlements.v2.service;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
......
......@@ -3,12 +3,11 @@
<modelVersion>4.0.0</modelVersion>
<properties>
<os-core-common.version>0.3.6</os-core-common.version>
<os-core-common.version>0.6.9</os-core-common.version>
<java.version>1.8</java.version>
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<os-core-common.version>0.3.12</os-core-common.version>
<spring-boot-dependencies.version>2.4.0</spring-boot-dependencies.version>
</properties>
......
package org.opengroup.osdu.entitlements.v2.azure.config;
import com.azure.security.keyvault.secrets.SecretClient;
import org.opengroup.osdu.azure.KeyVaultFacade;
import org.opengroup.osdu.core.common.cache.RedisCache;
import org.opengroup.osdu.entitlements.v2.model.ParentReferences;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class CacheConfig {
@Autowired
private SecretClient secretClient;
@Value("${redis.port:6380}")
private int redisPort;
@Value("${redis.database:8}")
private int redisDatabase;
@Bean
public int getRedisTtlSeconds() {
if(System.getenv("REDIS_TTL_SECONDS") == null) return 1;
else return Integer.parseInt(System.getenv("REDIS_TTL_SECONDS"));
}
@Bean
public RedisCache<String, ParentReferences> groupCacheRedis() {
return new RedisCache<>(getRedisHostname(), redisPort, getRedisPassword(), getRedisTtlSeconds(), redisDatabase, String.class,
ParentReferences.class);
}
public String getRedisHostname() {
return KeyVaultFacade.getSecretWithValidation(secretClient, "redis-hostname");
}
public String getRedisPassword() {
return KeyVaultFacade.getSecretWithValidation(secretClient, "redis-password");
}
}
package org.opengroup.osdu.entitlements.v2.azure.service;
import lombok.RequiredArgsConstructor;
import org.opengroup.osdu.core.common.cache.ICache;
import org.opengroup.osdu.entitlements.v2.model.EntityNode;
import org.opengroup.osdu.entitlements.v2.model.ParentReference;
import org.opengroup.osdu.entitlements.v2.model.ParentReferences;
import org.opengroup.osdu.entitlements.v2.service.GroupCacheService;
import org.opengroup.osdu.entitlements.v2.azure.service.metrics.hitsnmisses.HitsNMissesMetricService;
import org.opengroup.osdu.entitlements.v2.spi.retrievegroup.RetrieveGroupRepo;
import org.springframework.stereotype.Service;
import java.util.Set;
@Service
@RequiredArgsConstructor
public class GroupCacheServiceAzure implements GroupCacheService {
private final RetrieveGroupRepo retrieveGroupRepo;
private final ICache<String, ParentReferences> redisGroupCache;
private final HitsNMissesMetricService metricService;
@Override
public Set<ParentReference> getFromPartitionCache(String requesterId, String partitionId) {
String key = String.format("%s-%s", requesterId, partitionId);
ParentReferences parentReferences = redisGroupCache.get(key);
if (parentReferences == null) {
metricService.sendMissesMetric();
EntityNode entityNode = EntityNode.createMemberNodeForNewUser(requesterId, partitionId);
Set<ParentReference> allParents = retrieveGroupRepo.loadAllParents(entityNode).getParentReferences();
parentReferences = new ParentReferences();
parentReferences.setParentReferencesOfUser(allParents);
redisGroupCache.put(key, parentReferences);
} else {
metricService.sendHitsMetric();
}
return parentReferences.getParentReferencesOfUser();
}
}
package org.opengroup.osdu.entitlements.v2.azure.service.metrics.hitsnmisses;
import com.microsoft.applicationinsights.TelemetryClient;
import com.microsoft.applicationinsights.telemetry.MetricTelemetry;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
public abstract class AbstractHitsNMissesMetricService implements HitsNMissesMetricService {
private static final int DEFAULT_METRIC_VALUE = 1;
private final TelemetryClient telemetryClient;
/**
* This value will be used to send the 'hits' metric to application insights.
* Based on this name, it will be possible to filter metrics in the Metrics Explorer.
*
* @return The name of the 'hits' metric.
*/
protected abstract String hitsMetricName();
/**
* This value will be used to send the 'misses' metric to application insights.
* Based on this name, it will be possible to filter metrics in the Metrics Explorer.
*
* @return The name of the 'misses' metric.
*/
protected abstract String missesMetricName();
@Override
public void sendHitsMetric() {
sendMetric(hitsMetricName());
}
@Override
public void sendMissesMetric() {
sendMetric(missesMetricName());
}
private void sendMetric(String name) {
MetricTelemetry metric = new MetricTelemetry();
metric.setName(name);
metric.setValue(DEFAULT_METRIC_VALUE);
telemetryClient.trackMetric(metric);
telemetryClient.flush();
}
}
\ No newline at end of file
package org.opengroup.osdu.entitlements.v2.azure.service.metrics.hitsnmisses;
public interface HitsNMissesMetricService {
/**
* Sends one 'hits' metric to application insights,
* to send several such metrics, you need to call this method exactly as many times as metrics you expect to send.
* <p>
* 'hits' refers to the number of times an action has reached the destination.
*/
void sendHitsMetric();
/**
* Sends one 'misses' metric to application insights,
* to send several such metrics, you need to call this method exactly as many times as metrics you expect to send.
* <p>
* 'misses' refers to the number of times an action has NOT reached the destination.
*/
void sendMissesMetric();
}
\ No newline at end of file
package org.opengroup.osdu.entitlements.v2.azure.service.metrics.hitsnmisses;
import com.microsoft.applicationinsights.TelemetryClient;
import org.springframework.stereotype.Service;
@Service
public class RedisCacheHitsNMissesMetricService extends AbstractHitsNMissesMetricService {
private static final String HITS_METRIC_NAME = "[Entitlements service] Redis cache HITS";
private static final String MISSES_METRIC_NAME = "[Entitlements service] Redis cache MISSES";
public RedisCacheHitsNMissesMetricService(TelemetryClient telemetryClient) {
super(telemetryClient);
}
@Override
protected String hitsMetricName() {
return HITS_METRIC_NAME;
}
@Override
protected String missesMetricName() {
return MISSES_METRIC_NAME;
}
}
package org.opengroup.osdu.entitlements.v2.service;
package org.opengroup.osdu.entitlements.v2.azure.service;
import org.junit.Before;
import org.junit.Test;
......@@ -6,9 +6,12 @@ import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.MockitoJUnitRunner;
import org.opengroup.osdu.core.common.cache.ICache;
import org.opengroup.osdu.entitlements.v2.model.EntityNode;
import org.opengroup.osdu.entitlements.v2.model.ParentReference;
import org.opengroup.osdu.entitlements.v2.model.ParentReferences;
import org.opengroup.osdu.entitlements.v2.model.ParentTreeDto;
import org.opengroup.osdu.entitlements.v2.azure.service.metrics.hitsnmisses.HitsNMissesMetricService;
import org.opengroup.osdu.entitlements.v2.spi.retrievegroup.RetrieveGroupRepo;
import java.util.HashSet;
......@@ -20,20 +23,23 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class)
public class GroupCacheServiceTests {
public class GroupCacheServiceAzureTest {
private Set<ParentReference> parents = new HashSet<>();
private ParentReferences parentReferences = new ParentReferences();
private EntityNode requester;
@Mock
private RetrieveGroupRepo retrieveGroupRepo;
@Mock
private GroupCache vmGroupCache;
private ICache<String, ParentReferences> redisGroupCache;
@Mock
private ParentTreeDto parentTreeDto;
@Mock
private HitsNMissesMetricService metricService;
@InjectMocks
private GroupCacheService sut;
private GroupCacheServiceAzure sut;
@Before
public void setup() {
......@@ -51,28 +57,31 @@ public class GroupCacheServiceTests {
.build();
parents.add(parent1);
parents.add(parent2);
parentReferences.setParentReferencesOfUser(parents);
requester = EntityNode.createMemberNodeForNewUser("requesterId", "dp");
}
@Test
public void shouldGetAllParentsFromRepoForTheFirstTime() {
when(this.vmGroupCache.getGroupCache("requesterId-dp")).thenReturn(null);
when(this.redisGroupCache.get("requesterId-dp")).thenReturn(null);
when(this.retrieveGroupRepo.loadAllParents(this.requester)).thenReturn(this.parentTreeDto);
when(this.parentTreeDto.getParentReferences()).thenReturn(this.parents);
Set<ParentReference> result = this.sut.getFromPartitionCache("requesterId", "dp");
assertEquals(this.parents, result);
verify(this.retrieveGroupRepo, times(1)).loadAllParents(this.requester);
verify(this.vmGroupCache, times(1)).addGroupCache("requesterId-dp", this.parents);
verify(this.redisGroupCache, times(1)).put("requesterId-dp", this.parentReferences);
verify(this.metricService, times(1)).sendMissesMetric();
}
@Test
public void shouldGetAllParentsFromCacheForTheSecondTime() {
when(this.vmGroupCache.getGroupCache("requesterId-dp")).thenReturn(this.parents);
when(this.redisGroupCache.get("requesterId-dp")).thenReturn(this.parentReferences);
Set<ParentReference> result = this.sut.getFromPartitionCache("requesterId", "dp");
assertEquals(this.parents, result);
verify(this.retrieveGroupRepo, times(0)).loadAllParents(this.requester);
verify(this.metricService, times(1)).sendHitsMetric();
}
}
package org.opengroup.osdu.entitlements.v2.azure.service.metrics.hitsnmisses;
import com.microsoft.applicationinsights.TelemetryClient;
import com.microsoft.applicationinsights.telemetry.MetricTelemetry;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.ArgumentCaptor;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.MockitoJUnitRunner;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@RunWith(MockitoJUnitRunner.class)
public class RedisCacheHitsNMissesMetricServiceTest {
@Mock
private TelemetryClient telemetryClient;
@InjectMocks
private RedisCacheHitsNMissesMetricService redisCacheHitsNMissesMetricService;
@Test
public void shouldSendHitsMetricSuccessfully() {
ArgumentCaptor<MetricTelemetry> metricCaptor = ArgumentCaptor.forClass(MetricTelemetry.class);
redisCacheHitsNMissesMetricService.sendHitsMetric();
verify(telemetryClient, times(1)).trackMetric(metricCaptor.capture());
assertEquals("[Entitlements service] Redis cache HITS", metricCaptor.getValue().getName());
assertEquals(1, metricCaptor.getValue().getValue(), 0);
verify(telemetryClient, times(1)).flush();
}
@Test
public void shouldSendMissesMetricSuccessfully() {
ArgumentCaptor<MetricTelemetry> metricCaptor = ArgumentCaptor.forClass(MetricTelemetry.class);
redisCacheHitsNMissesMetricService.sendMissesMetric();
verify(telemetryClient, times(1)).trackMetric(metricCaptor.capture());
assertEquals("[Entitlements service] Redis cache MISSES", metricCaptor.getValue().getName());
assertEquals(1, metricCaptor.getValue().getValue(), 0);
verify(telemetryClient, times(1)).flush();
}
}
......@@ -9,6 +9,7 @@ import org.junit.Test;
import org.junit.jupiter.api.Assertions;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.opengroup.osdu.core.common.cache.ICache;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
......@@ -20,6 +21,7 @@ import org.opengroup.osdu.entitlements.v2.auth.AuthorizationService;
import org.opengroup.osdu.entitlements.v2.azure.AzureAppProperties;
import org.opengroup.osdu.entitlements.v2.logging.AuditLogger;
import org.opengroup.osdu.entitlements.v2.model.ParentReference;
import org.opengroup.osdu.entitlements.v2.model.ParentReferences;
import org.opengroup.osdu.entitlements.v2.model.Role;
import org.opengroup.osdu.entitlements.v2.model.addmember.AddMemberDto;
import org.opengroup.osdu.entitlements.v2.model.creategroup.CreateGroupDto;
......@@ -27,6 +29,7 @@ import org.opengroup.osdu.entitlements.v2.model.listgroup.ListGroupResponseDto;
import org.opengroup.osdu.entitlements.v2.model.listmember.ListMemberResponseDto;
import org.opengroup.osdu.entitlements.v2.model.listmember.MemberDto;
import org.opengroup.osdu.entitlements.v2.model.updategroup.UpdateGroupOperation;
import org.opengroup.osdu.entitlements.v2.azure.service.metrics.hitsnmisses.HitsNMissesMetricService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest;
import org.springframework.boot.test.mock.mockito.MockBean;
......@@ -117,6 +120,10 @@ public class CreateMembershipsWorkflowSinglePartitionTest {
private ITenantFactory tenantFactory;
@MockBean
private AuthorizationService authService;
@MockBean
private ICache<String, ParentReferences> redisGroupCache;
@MockBean
private HitsNMissesMetricService metricService;
@Before
public void before() {
......
package org.opengroup.osdu.entitlements.v2.gcp.service;
import lombok.RequiredArgsConstructor;
import org.opengroup.osdu.entitlements.v2.model.EntityNode;
import org.opengroup.osdu.entitlements.v2.model.ParentReference;
import org.opengroup.osdu.entitlements.v2.service.GroupCacheService;
import org.opengroup.osdu.entitlements.v2.spi.retrievegroup.RetrieveGroupRepo;
import org.springframework.stereotype.Service;
import java.util.Set;
@Service
@RequiredArgsConstructor
public class GroupCacheServiceGcp implements GroupCacheService {
private final RetrieveGroupRepo retrieveGroupRepo;
private final VmGroupCache vmGroupCache;
@Override
public Set<ParentReference> getFromPartitionCache(String requesterId, String partitionId) {
String key = String.format("%s-%s", requesterId, partitionId);
Set<ParentReference> result = vmGroupCache.getGroupCache(key);
if (result == null) {
EntityNode entityNode = EntityNode.createMemberNodeForNewUser(requesterId, partitionId);
result = retrieveGroupRepo.loadAllParents(entityNode).getParentReferences();
vmGroupCache.addGroupCache(key, result);
}
return result;
}
}
package org.opengroup.osdu.entitlements.v2.service;
package org.opengroup.osdu.entitlements.v2.gcp.service;
import org.opengroup.osdu.entitlements.v2.model.ParentReference;
import org.opengroup.osdu.entitlements.v2.service.GroupCache;
import org.springframework.stereotype.Component;
import org.springframework.web.context.annotation.RequestScope;
......@@ -11,15 +10,13 @@ import java.util.Set;
@RequestScope
@Component
public class VmGroupCache implements GroupCache {