Commit bb05c3e8 authored by Alok Joshi's avatar Alok Joshi
Browse files

Merge branch 'master' of...

Merge branch 'master' of https://community.opengroup.org/osdu/platform/system/lib/cloud/azure/os-core-lib-azure into use_debug_log
parents cb41053f 33180eed
Pipeline #46765 passed with stages
in 5 minutes and 35 seconds
......@@ -17,10 +17,13 @@ package org.opengroup.osdu.azure.blobstorage;
import com.azure.identity.DefaultAzureCredential;
import com.azure.storage.blob.BlobServiceClient;
import com.azure.storage.blob.BlobServiceClientBuilder;
import com.azure.storage.common.policy.RequestRetryOptions;
import org.opengroup.osdu.azure.cache.BlobServiceClientCache;
import org.opengroup.osdu.azure.di.BlobStoreRetryConfiguration;
import org.opengroup.osdu.azure.partition.PartitionInfoAzure;
import org.opengroup.osdu.azure.partition.PartitionServiceClient;
import org.opengroup.osdu.common.Validators;
import org.springframework.beans.factory.annotation.Autowired;
/**
* Implementation for IBlobServiceClientFactory.
......@@ -30,6 +33,9 @@ public class BlobServiceClientFactoryImpl implements IBlobServiceClientFactory {
private PartitionServiceClient partitionService;
private BlobServiceClientCache clientCache;
@Autowired
private BlobStoreRetryConfiguration blobStoreRetryConfiguration;
/**
* Constructor to initialize instance of {@link BlobServiceClientFactoryImpl}.
* @param credentials Azure Credentials to use
......@@ -61,9 +67,12 @@ public class BlobServiceClientFactoryImpl implements IBlobServiceClientFactory {
PartitionInfoAzure pi = this.partitionService.getPartition(dataPartitionId);
String endpoint = String.format("https://%s.blob.core.windows.net", pi.getStorageAccountName());
RequestRetryOptions requestRetryOptions = blobStoreRetryConfiguration.getRequestRetryOptions();
BlobServiceClient blobServiceClient = new BlobServiceClientBuilder()
.endpoint(endpoint)
.credential(defaultAzureCredential)
.retryOptions(requestRetryOptions)
.buildClient();
this.clientCache.put(cacheKey, blobServiceClient);
......
......@@ -2,11 +2,14 @@ package org.opengroup.osdu.azure.cosmosdb;
import com.azure.cosmos.CosmosClient;
import com.azure.cosmos.CosmosClientBuilder;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import javax.annotation.PostConstruct;
import com.azure.cosmos.ThrottlingRetryOptions;
import org.opengroup.osdu.azure.logging.CoreLoggerFactory;
import org.opengroup.osdu.azure.di.CosmosRetryConfiguration;
import org.opengroup.osdu.azure.partition.PartitionInfoAzure;
import org.opengroup.osdu.azure.partition.PartitionServiceClient;
import org.opengroup.osdu.common.Validators;
......@@ -27,6 +30,9 @@ public class CosmosClientFactoryImpl implements ICosmosClientFactory {
private Map<String, CosmosClient> cosmosClientMap;
@Autowired
private CosmosRetryConfiguration cosmosRetryConfiguration;
/**
* Initializes the private variables as required.
*/
......@@ -58,9 +64,13 @@ public class CosmosClientFactoryImpl implements ICosmosClientFactory {
*/
private CosmosClient createCosmosClient(final String dataPartitionId) {
PartitionInfoAzure pi = this.partitionService.getPartition(dataPartitionId);
ThrottlingRetryOptions throttlingRetryOptions = cosmosRetryConfiguration.getThrottlingRetryOptions();
CosmosClient cosmosClient = new CosmosClientBuilder()
.endpoint(pi.getCosmosEndpoint())
.key(pi.getCosmosPrimaryKey())
.throttlingRetryOptions(throttlingRetryOptions)
.buildClient();
CoreLoggerFactory.getInstance().getLogger(LOGGER_NAME)
.info("Created CosmosClient for dataPartition {}.", dataPartitionId);
......
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.azure.di;
import com.azure.storage.common.policy.RequestRetryOptions;
import com.azure.storage.common.policy.RetryPolicyType;
import lombok.Getter;
import lombok.Setter;
import org.opengroup.osdu.azure.logging.CoreLoggerFactory;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
import java.time.Duration;
/**
* Config for BlogStorage Retry.
*/
@Configuration
@ConfigurationProperties("azure.blobstore")
@Getter
@Setter
public class BlobStoreRetryConfiguration {
public static final String LOGGER_NAME = BlobStoreRetryConfiguration.class.getName();
private final RequestRetryOptions defaultRequestRetryOptions = new RequestRetryOptions();
private static final String DEFAULT_STRING_VALUE = "";
private int maxTries = defaultRequestRetryOptions.getMaxTries();
private long tryTimeoutInSeconds = defaultRequestRetryOptions.getTryTimeoutDuration().getSeconds();
private long retryDelayInMs = defaultRequestRetryOptions.getRetryDelay().toMillis();
private long maxRetryDelayInMs = defaultRequestRetryOptions.getMaxRetryDelay().toMillis();
private String retryPolicyTypeValue = DEFAULT_STRING_VALUE;
/**
* Checks whether an string variable value is configured or not.
* @param val string value to be checked
* @return true if value is configured in app.properties
*/
private boolean valueConfigured(final String val) {
if (val == null || val.equals(DEFAULT_STRING_VALUE)) {
return false;
}
return true;
}
/**
* Method to get RequestRetryOptions object based on configuration set in applicaiton.properties.
* @return RequestRetryOption object with appropriate configurations.
*/
public RequestRetryOptions getRequestRetryOptions() {
// Check whether the variables have been set, else keep them as null.
// Value has to be sent as null incase where they are not configured to use the default configurations (As specified in RequestRetryOptions.class)
// https://azure.github.io/azure-storage-java-async/com/microsoft/azure/storage/blob/RequestRetryOptions.html
RetryPolicyType retryPolicyType;
try {
retryPolicyType = valueConfigured(retryPolicyTypeValue) ? RetryPolicyType.valueOf(retryPolicyTypeValue) : RetryPolicyType.EXPONENTIAL;
} catch (Exception ex) { // For wrong values of Retry Policy, it will default to Exponential.
retryPolicyType = RetryPolicyType.EXPONENTIAL;
}
RequestRetryOptions requestRetryOptions = new RequestRetryOptions(retryPolicyType, maxTries, Duration.ofSeconds(tryTimeoutInSeconds), Duration.ofMillis(retryDelayInMs), Duration.ofMillis(maxRetryDelayInMs), null);
CoreLoggerFactory.getInstance().getLogger(LOGGER_NAME).info(String.format("Retry Options on BlobStorage with RetryPolicyType = %s , maxTries = %d , tryTimeout = %d , retryDelay = %d , maxRetryDelay = %d .",
retryPolicyType, requestRetryOptions.getMaxTries(), requestRetryOptions.getTryTimeoutDuration().getSeconds(), requestRetryOptions.getRetryDelay().toMillis(), requestRetryOptions.getMaxRetryDelay().toMillis()));
return requestRetryOptions;
}
}
/** Copyright © Microsoft Corporation
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
**/
package org.opengroup.osdu.azure.di;
import com.azure.cosmos.ThrottlingRetryOptions;
import lombok.Getter;
import lombok.Setter;
import org.opengroup.osdu.azure.logging.CoreLoggerFactory;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
import java.time.Duration;
/**
* CosmosRetryConfiguration settings.
*/
@Configuration
@ConfigurationProperties("azure.cosmos")
@Getter
@Setter
public class CosmosRetryConfiguration {
public static final String LOGGER_NAME = CosmosRetryConfiguration.class.getName();
private ThrottlingRetryOptions defaultThrottlingRetryOptions = new ThrottlingRetryOptions();
/**
* Value for max Retry Count on Throttled Requests for Cosmos.
*/
private int maxRetryCount = defaultThrottlingRetryOptions.getMaxRetryAttemptsOnThrottledRequests(); // Sets the default value for axRetryAttemptsOnThrottledRequests
/**
* Value for max retry wait time for Cosmos (Value in seconds).
*/
private long retryWaitTimeout = defaultThrottlingRetryOptions.getMaxRetryWaitTime().getSeconds(); // Sets the default value for MaxRetryWaitTime in seconds
/**
* Set's the Throttling retry options based on application.properties configuration.
* @return Throttling retry options
*/
public ThrottlingRetryOptions getThrottlingRetryOptions() {
ThrottlingRetryOptions throttlingRetryOptions = new ThrottlingRetryOptions();
throttlingRetryOptions.setMaxRetryAttemptsOnThrottledRequests(this.getMaxRetryCount());
throttlingRetryOptions.setMaxRetryWaitTime(Duration.ofSeconds(this.getRetryWaitTimeout()));
CoreLoggerFactory.getInstance().getLogger(LOGGER_NAME).info("Retry Options on CosmosClient with maxRetryAttempts = {} , MaxRetryWaitTime = {}.", throttlingRetryOptions.getMaxRetryAttemptsOnThrottledRequests(), throttlingRetryOptions.getMaxRetryWaitTime());
return throttlingRetryOptions;
}
}
......@@ -16,6 +16,7 @@ package org.opengroup.osdu.azure.blobstorage;
import com.azure.identity.DefaultAzureCredential;
import com.azure.storage.blob.BlobServiceClient;
import com.azure.storage.common.policy.RequestRetryOptions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
......@@ -24,8 +25,10 @@ import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.opengroup.osdu.azure.cache.BlobServiceClientCache;
import org.opengroup.osdu.azure.di.BlobStoreConfiguration;
import org.opengroup.osdu.azure.di.BlobStoreRetryConfiguration;
import org.opengroup.osdu.azure.partition.PartitionInfoAzure;
import org.opengroup.osdu.azure.partition.PartitionServiceClient;
import org.opengroup.osdu.core.common.logging.ILogger;
import org.opengroup.osdu.core.common.partition.Property;
import static org.junit.jupiter.api.Assertions.*;
......@@ -43,6 +46,10 @@ public class BlobServiceClientFactoryImplTest {
private BlobServiceClientCache clientCache;
@Mock
private BlobStoreConfiguration configuration;
@Mock
private BlobStoreRetryConfiguration blobStoreRetryConfiguration;
@Mock
private ILogger logger;
@InjectMocks
private BlobServiceClientFactoryImpl sut;
......@@ -83,6 +90,7 @@ public class BlobServiceClientFactoryImplTest {
PartitionInfoAzure.builder()
.idConfig(Property.builder().value(PARTITION_ID).build())
.storageAccountNameConfig(Property.builder().value(ACCOUNT_NAME).build()).build());
when(this.blobStoreRetryConfiguration.getRequestRetryOptions()).thenReturn(new RequestRetryOptions());
BlobServiceClient serviceClient = this.sut.getBlobServiceClient(PARTITION_ID);
assertNotNull(serviceClient);
......@@ -94,6 +102,7 @@ public class BlobServiceClientFactoryImplTest {
PartitionInfoAzure.builder()
.idConfig(Property.builder().value(PARTITION_ID).build())
.storageAccountNameConfig(Property.builder().value(ACCOUNT_NAME).build()).build());
when(this.blobStoreRetryConfiguration.getRequestRetryOptions()).thenReturn(new RequestRetryOptions());
BlobServiceClient serviceClient = this.sut.getBlobServiceClient(PARTITION_ID);
assertNotNull(serviceClient);
......
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.azure.di;
import com.azure.storage.common.policy.RequestRetryOptions;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.opengroup.osdu.azure.logging.CoreLogger;
import org.opengroup.osdu.azure.logging.CoreLoggerFactory;
import org.opengroup.osdu.core.common.logging.ILogger;
import java.lang.reflect.Field;
import java.time.Duration;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.when;
import static org.mockito.MockitoAnnotations.openMocks;
@ExtendWith(MockitoExtension.class)
public class BlobStoreRetryConfigurationTest {
@InjectMocks
private BlobStoreRetryConfiguration blobStoreRetryConfiguration;
@Mock
private ILogger logger;
RequestRetryOptions defaultRequestRetryOptions = new RequestRetryOptions();
@Mock
private CoreLoggerFactory coreLoggerFactory;
@Mock
private CoreLogger coreLogger;
/**
* Workaround for inability to mock static methods like getInstance().
*
* @param mock CoreLoggerFactory mock instance
*/
private void mockSingleton(CoreLoggerFactory mock) {
try {
Field instance = CoreLoggerFactory.class.getDeclaredField("instance");
instance.setAccessible(true);
instance.set(null, mock);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
/**
* Reset workaround for inability to mock static methods like getInstance().
*/
private void resetSingleton() {
try {
Field instance = CoreLoggerFactory.class.getDeclaredField("instance");
instance.setAccessible(true);
instance.set(null, null);
instance.setAccessible(false);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@BeforeEach
void setUp() {
openMocks(this);
mockSingleton(coreLoggerFactory);
when(coreLoggerFactory.getLogger(anyString())).thenReturn(coreLogger);
}
@AfterEach
public void takeDown() {
resetSingleton();
}
@Test
public void should_set_default_values() {
RequestRetryOptions requestRetryOptions = blobStoreRetryConfiguration.getRequestRetryOptions();
assertEquals(requestRetryOptions.getMaxTries(), defaultRequestRetryOptions.getMaxTries());
assertEquals(requestRetryOptions.getTryTimeoutDuration(), defaultRequestRetryOptions.getTryTimeoutDuration());
assertEquals(requestRetryOptions.getRetryDelay(), defaultRequestRetryOptions.getRetryDelay());
assertEquals(requestRetryOptions.getMaxRetryDelay(), defaultRequestRetryOptions.getMaxRetryDelay());
}
@Test
public void should_set_maxtries() {
int maxTriesValue = 10;
blobStoreRetryConfiguration.setMaxTries(maxTriesValue);
RequestRetryOptions requestRetryOptions = blobStoreRetryConfiguration.getRequestRetryOptions();
assertEquals(requestRetryOptions.getMaxTries(), maxTriesValue);
assertEquals(requestRetryOptions.getTryTimeoutDuration(), defaultRequestRetryOptions.getTryTimeoutDuration());
assertEquals(requestRetryOptions.getRetryDelay(), defaultRequestRetryOptions.getRetryDelay());
assertEquals(requestRetryOptions.getMaxRetryDelay(), defaultRequestRetryOptions.getMaxRetryDelay());
}
@Test
public void should_set_try_timeout() {
int tryTimeoutValue = 50;
blobStoreRetryConfiguration.setTryTimeoutInSeconds(tryTimeoutValue);
RequestRetryOptions requestRetryOptions = blobStoreRetryConfiguration.getRequestRetryOptions();
assertEquals(requestRetryOptions.getMaxTries(), defaultRequestRetryOptions.getMaxTries());
assertEquals(requestRetryOptions.getTryTimeoutDuration(), Duration.ofSeconds(tryTimeoutValue));
assertEquals(requestRetryOptions.getRetryDelay(), defaultRequestRetryOptions.getRetryDelay());
assertEquals(requestRetryOptions.getMaxRetryDelay(), defaultRequestRetryOptions.getMaxRetryDelay());
}
@Test
public void should_set_RetryDelay() {
int retryDelayValue = 50;
int maxRetryDelayValue = 100;
blobStoreRetryConfiguration.setRetryDelayInMs(retryDelayValue);
blobStoreRetryConfiguration.setMaxRetryDelayInMs(maxRetryDelayValue);
RequestRetryOptions requestRetryOptions = blobStoreRetryConfiguration.getRequestRetryOptions();
assertEquals(requestRetryOptions.getMaxTries(), defaultRequestRetryOptions.getMaxTries());
assertEquals(requestRetryOptions.getTryTimeoutDuration(), defaultRequestRetryOptions.getTryTimeoutDuration());
assertEquals(requestRetryOptions.getRetryDelay(), Duration.ofMillis(retryDelayValue));
assertEquals(requestRetryOptions.getMaxRetryDelay(), Duration.ofMillis(maxRetryDelayValue));
}
}
/** Copyright © Microsoft Corporation
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
**/
package org.opengroup.osdu.azure.di;
import com.azure.cosmos.ThrottlingRetryOptions;
import io.jsonwebtoken.lang.Assert;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.opengroup.osdu.azure.logging.CoreLogger;
import org.opengroup.osdu.azure.logging.CoreLoggerFactory;
import java.lang.reflect.Field;
import java.time.Duration;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.when;
import static org.mockito.MockitoAnnotations.openMocks;
@ExtendWith(MockitoExtension.class)
public class CosmosRetryConfigurationTest {
@InjectMocks
CosmosRetryConfiguration cosmosRetryConfiguration;
@Mock
private CoreLoggerFactory coreLoggerFactory;
@Mock
private CoreLogger coreLogger;
ThrottlingRetryOptions defaultRetryOptions = new ThrottlingRetryOptions();
/**
* Workaround for inability to mock static methods like getInstance().
*
* @param mock CoreLoggerFactory mock instance
*/
private void mockSingleton(CoreLoggerFactory mock) {
try {
Field instance = CoreLoggerFactory.class.getDeclaredField("instance");
instance.setAccessible(true);
instance.set(null, mock);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
/**
* Reset workaround for inability to mock static methods like getInstance().
*/
private void resetSingleton() {
try {
Field instance = CoreLoggerFactory.class.getDeclaredField("instance");
instance.setAccessible(true);
instance.set(null, null);
instance.setAccessible(false);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@BeforeEach
void setUp() {
openMocks(this);
mockSingleton(coreLoggerFactory);
when(coreLoggerFactory.getLogger(anyString())).thenReturn(coreLogger);
}
@AfterEach
public void takeDown() {
resetSingleton();
}
@Test
public void should_set_maxRetryCount_ThrottlingOptions() {
int maxRetryValue = 1;
cosmosRetryConfiguration.setMaxRetryCount(maxRetryValue);
ThrottlingRetryOptions throttlingRetryOptions = cosmosRetryConfiguration.getThrottlingRetryOptions();
Assert.isTrue(throttlingRetryOptions.getMaxRetryAttemptsOnThrottledRequests() == maxRetryValue);
Assert.isTrue(throttlingRetryOptions.getMaxRetryWaitTime().equals(defaultRetryOptions.getMaxRetryWaitTime()));
}
@Test
public void should_set_RetryWaitTimeout_ThrottlingOptions() {
int retryWaitTimeoutValue = 20;
cosmosRetryConfiguration.setRetryWaitTimeout(retryWaitTimeoutValue);
ThrottlingRetryOptions throttlingRetryOptions = cosmosRetryConfiguration.getThrottlingRetryOptions();
Assert.isTrue(throttlingRetryOptions.getMaxRetryWaitTime().equals(Duration.ofSeconds(retryWaitTimeoutValue)));
Assert.isTrue(throttlingRetryOptions.getMaxRetryAttemptsOnThrottledRequests() == defaultRetryOptions.getMaxRetryAttemptsOnThrottledRequests());
}
@Test
public void should_set_RetryWaitTimeout_MaxRetry_ThrottlingOptions() {
int retryWaitTimeoutValue = 20;
int maxRetryValue = 1;
cosmosRetryConfiguration.setRetryWaitTimeout(retryWaitTimeoutValue);
cosmosRetryConfiguration.setMaxRetryCount(maxRetryValue);
ThrottlingRetryOptions throttlingRetryOptions = cosmosRetryConfiguration.getThrottlingRetryOptions();
Assert.isTrue(throttlingRetryOptions.getMaxRetryAttemptsOnThrottledRequests() == maxRetryValue);
Assert.isTrue(throttlingRetryOptions.getMaxRetryWaitTime().equals(Duration.ofSeconds(retryWaitTimeoutValue)));
}
@Test
public void should_not_set_RetryWaitTimeout_MaxRetry_ThrottlingOptions() {
ThrottlingRetryOptions throttlingRetryOptions = cosmosRetryConfiguration.getThrottlingRetryOptions();
Assert.isTrue(throttlingRetryOptions.getMaxRetryAttemptsOnThrottledRequests() == defaultRetryOptions.getMaxRetryAttemptsOnThrottledRequests());
Assert.isTrue(throttlingRetryOptions.getMaxRetryWaitTime().equals(defaultRetryOptions.getMaxRetryWaitTime()));
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment