Commit f5ba9a41 authored by Hema Vishnu Pola [Microsoft]'s avatar Hema Vishnu Pola [Microsoft]
Browse files

Merge branch 'StoragePerf' into 'master'

Changes identified in Storage performance testing for legal tag update

See merge request !238
parents 63f7c5b7 98307c9d
Pipeline #56767 passed with stages
in 39 minutes and 54 seconds
This diff is collapsed.
......@@ -123,12 +123,6 @@ spec:
value: "true"
- name: azure_activedirectory_AppIdUri
value: "api://$(aad_client_id)"
- name: max_concurrent_calls
value: "3"
- name: executor_n_threads
value: "32"
- name: max_lock_renew_duration_seconds
value: "1800"
- name: JAVA_OPTS
value: "-XX:InitialRAMPercentage=25.0 -XX:MaxRAMPercentage=50.0"
- name: "SERVER_TOMCAT_MAXTHREADS"
......
......@@ -33,43 +33,64 @@ import java.util.stream.Collectors;
/**
* Thread scope which allows putting data in thread scope and clearing up afterwards.
*/
public class ThreadScope implements Scope, DisposableBean {
@Autowired
HttpServletRequest request;
/**
* Get bean for given name in the "ThreadScope".
* Get bean for the given name in the "ThreadScope"
* This is called for creating beans of DpsHeaders and ThreadDpsHeaders type in "ThreadScope"
*
* The two types are distinguished on the basis of Request Attributes in Request Context Holder.
*
* If Request Attributes is not null, it is api request thread.
*
* For a new Api request thread MDC context is cleared and then this function is called
* before extracting and setting headers.
* For api request, with MDC context map of size zero, we clear the thread context, to ensure
* ThreadLocal variable values are not reused, by new threads.
*
* Next for new, api request, the function returns new bean of DPSHeader type, after adding it to the context.
*
* If the Request Attributes is null, it is a subscriber thread, here if the bean does not already
* exists in object factory, we create a new bean of type ThreadDpsHeader and return , after adding it to the context.
*
* For both cases, if it is not a new api request or if it is not a new subscriber thread request,
* we return the existing object from the object factory.
*
*/
public Object get(String name, ObjectFactory<?> factory) {
ThreadScopeContext context = ThreadScopeContextHolder.getContext();
RequestAttributes att = RequestContextHolder.getRequestAttributes();
RequestAttributes requestAttributes = RequestContextHolder.getRequestAttributes();
Map<String, String> contextMap = MDC.getCopyOfContextMap();
if (null != requestAttributes && contextMap.size() == 0) {
context.clear();
}
Object result = context.getBean(name);
if (null == result) {
if (att != null) {
DpsHeaders headers = new DpsHeaders();
HttpServletRequest request = ((ServletRequestAttributes) att).getRequest();
Map<String, String> header = Collections
.list(request.getHeaderNames())
.stream()
.collect(Collectors.toMap(h -> h, request::getHeader));
for (Map.Entry<String, String> entry : header.entrySet()) {
headers.put(entry.getKey(), entry.getValue());
}
context.setBean(name, headers);
MDC.setContextMap(header);
return headers;
} else {
result = factory.getObject();
context.setBean(name, result);
return result;
if (null == result && null != requestAttributes) {
DpsHeaders headers = new DpsHeaders();
HttpServletRequest request = ((ServletRequestAttributes) requestAttributes).getRequest();
Map<String, String> header = Collections
.list(request.getHeaderNames())
.stream()
.collect(Collectors.toMap(h -> h, request::getHeader));
for (Map.Entry<String, String> entry : header.entrySet()) {
headers.put(entry.getKey(), entry.getValue());
}
context.setBean(name, headers);
MDC.setContextMap(header);
return headers;
} else if (null == result) {
result = factory.getObject();
context.setBean(name, result);
return result;
} else {
return result;
}
return result;
}
/**
* Removes bean from scope.
*/
......
......@@ -34,6 +34,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
import java.util.HashMap;
......@@ -42,6 +43,7 @@ import java.util.Map;
import static java.nio.charset.StandardCharsets.UTF_8;
@Component
@ConditionalOnProperty(value = "azure.feature.legaltag-compliance-update.enabled", havingValue = "true", matchIfMissing = false)
public class LegalComplianceChangeUpdate{
private final static Logger LOGGER = LoggerFactory.getLogger(LegalComplianceChangeUpdate.class);
......@@ -49,10 +51,8 @@ public class LegalComplianceChangeUpdate{
private IRecordsMetadataRepository recordsRepo;
@Autowired
private StorageAuditLogger auditLogger;
// @Autowired
// private ThreadDpsHeaders headers; //to be used when azure.feature.legaltag-compliance-update.enabled is set
@Autowired
private DpsHeaders headers;
private ThreadDpsHeaders headers; //to be used when azure.feature.legaltag-compliance-update.enabled is set
@Autowired
private MDCContextMap mdcContextMap;
@Autowired
......@@ -68,8 +68,7 @@ public class LegalComplianceChangeUpdate{
LegalTagChangedCollection tags = gson.fromJson(legalTagsChangedData.getData(), LegalTagChangedCollection.class);
message.setMessageId(legalTagsChangedRequest.getId());
//uncomment when azure.feature.legaltag-compliance-update.enabled is enabled
//headers.setThreadContext(legalTagsChangedData.getDataPartitionId(), legalTagsChangedData.getCorrelationId(), legalTagsChangedData.getUser());
headers.setThreadContext(legalTagsChangedData.getDataPartitionId(), legalTagsChangedData.getCorrelationId(), legalTagsChangedData.getUser());
MDC.setContextMap(mdcContextMap.getContextMap(headers.getCorrelationId(), headers.getCorrelationId()));
complianceMessagePullReceiver.receiveMessage(tags, headers);
......
......@@ -22,9 +22,11 @@ import org.opengroup.osdu.storage.provider.azure.di.AzureBootstrapConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
@Component
@ConditionalOnProperty(value = "azure.feature.legaltag-compliance-update.enabled", havingValue = "true", matchIfMissing = false)
public class LegalTagSubscriptionClientFactory {
private final static Logger LOGGER = LoggerFactory.getLogger(LegalTagSubscriptionClientFactory.class);
......
......@@ -25,6 +25,7 @@ import org.opengroup.osdu.storage.provider.azure.interfaces.ILegalTagSubscriptio
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
import java.time.Duration;
......@@ -34,6 +35,7 @@ import java.util.concurrent.Executors;
import java.util.stream.Collectors;
@Component
@ConditionalOnProperty(value = "azure.feature.legaltag-compliance-update.enabled", havingValue = "true", matchIfMissing = false)
public class LegalTagSubscriptionManagerImpl implements ILegalTagSubscriptionManager {
private final static Logger LOGGER = LoggerFactory.getLogger(LegalTagSubscriptionManagerImpl.class);
......
......@@ -22,9 +22,11 @@ import org.opengroup.osdu.core.common.model.legal.jobs.ComplianceUpdateStoppedEx
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import java.util.concurrent.CompletableFuture;
@ConditionalOnProperty(value = "azure.feature.legaltag-compliance-update.enabled", havingValue = "true", matchIfMissing = false)
public class LegalTagSubscriptionMessageHandler implements IMessageHandler {
@Autowired
private final static Logger LOGGER = LoggerFactory.getLogger(LegalTagSubscriptionMessageHandler.class);
......
......@@ -19,6 +19,7 @@ import org.opengroup.osdu.storage.provider.azure.pubsub.LegalTagSubscriptionMana
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.beans.factory.config.BeanFactoryPostProcessor;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.ApplicationListener;
import org.springframework.context.annotation.Bean;
import org.springframework.context.event.ContextRefreshedEvent;
......@@ -26,6 +27,7 @@ import org.springframework.stereotype.Component;
@Component
@ConditionalOnProperty(value = "azure.feature.legaltag-compliance-update.enabled", havingValue = "true", matchIfMissing = false)
public class LegalTagSubscriberSetUp implements ApplicationListener<ContextRefreshedEvent> {
@Autowired
private LegalTagSubscriptionManagerImpl legalTagSubscriptionManager;
......
......@@ -93,7 +93,7 @@ record-id.max.length=1024
executor-n-threads=32
# Specifies the maximum number of concurrent calls to the callback the message pump should initiate
max-concurrent-calls=3
max-concurrent-calls=1
# Specifies the maximum duration in seconds within which the lock will be renewed automatically
max-lock-renew=1800
......
......@@ -4,7 +4,20 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.slf4j.MDC;
import org.springframework.beans.factory.ObjectFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.mock.web.MockHttpServletRequest;
import org.springframework.web.context.request.RequestContextHolder;
import org.springframework.web.context.request.ServletRequestAttributes;
import javax.servlet.http.HttpServletRequest;
import java.util.Collections;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
......@@ -13,6 +26,7 @@ import static org.junit.jupiter.api.Assertions.*;
@ExtendWith(MockitoExtension.class)
class ThreadScopeTest {
private String name = "name";
private ThreadScope threadScope = new ThreadScope();
@Mock
ObjectFactory<?> factory = null;
@Mock
......@@ -21,6 +35,14 @@ class ThreadScopeTest {
ThreadScope scope;
@Mock
ThreadScopeContext context;
@Mock
ThreadDpsHeaders threadDpsHeaders;
@Mock
DpsHeaders dpsHeaders;
@Mock
MockHttpServletRequest request;
@Mock
MDC mdcContext;
@Test
void shouldGetObject() {
......@@ -34,5 +56,15 @@ class ThreadScopeTest {
obj = scope.get(any(),any());
assertNull(obj);
}
}
\ No newline at end of file
@Test
void shouldGetObjectofDpsHeaderClass(){
Map<String, String> headers = new HashMap<>();
headers.put("header1", "value1");
headers.put("Content-Type", "text/html");
Enumeration<String> headerNames = Collections.enumeration(headers.keySet());
when(request.getHeaderNames()).thenReturn(headerNames);
RequestContextHolder.setRequestAttributes(new ServletRequestAttributes(request));
when(request.getHeader(any())).thenReturn(any());
assertEquals(threadScope.get(name,factory).getClass(),dpsHeaders.getClass());
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment