Commit 33180eed authored by Hema Vishnu Pola [Microsoft]'s avatar Hema Vishnu Pola [Microsoft]
Browse files

Merge branch 'rosakhuj/blobstore2' into 'master'

Blobstore Retry Configuration

See merge request !104
parents cc072fa5 dd5cd8ad
Pipeline #46747 passed with stages
in 7 minutes and 42 seconds
......@@ -17,10 +17,13 @@ package org.opengroup.osdu.azure.blobstorage;
import com.azure.identity.DefaultAzureCredential;
import com.azure.storage.blob.BlobServiceClient;
import com.azure.storage.blob.BlobServiceClientBuilder;
import com.azure.storage.common.policy.RequestRetryOptions;
import org.opengroup.osdu.azure.cache.BlobServiceClientCache;
import org.opengroup.osdu.azure.di.BlobStoreRetryConfiguration;
import org.opengroup.osdu.azure.partition.PartitionInfoAzure;
import org.opengroup.osdu.azure.partition.PartitionServiceClient;
import org.opengroup.osdu.common.Validators;
import org.springframework.beans.factory.annotation.Autowired;
/**
* Implementation for IBlobServiceClientFactory.
......@@ -30,6 +33,9 @@ public class BlobServiceClientFactoryImpl implements IBlobServiceClientFactory {
private PartitionServiceClient partitionService;
private BlobServiceClientCache clientCache;
@Autowired
private BlobStoreRetryConfiguration blobStoreRetryConfiguration;
/**
* Constructor to initialize instance of {@link BlobServiceClientFactoryImpl}.
* @param credentials Azure Credentials to use
......@@ -61,9 +67,12 @@ public class BlobServiceClientFactoryImpl implements IBlobServiceClientFactory {
PartitionInfoAzure pi = this.partitionService.getPartition(dataPartitionId);
String endpoint = String.format("https://%s.blob.core.windows.net", pi.getStorageAccountName());
RequestRetryOptions requestRetryOptions = blobStoreRetryConfiguration.getRequestRetryOptions();
BlobServiceClient blobServiceClient = new BlobServiceClientBuilder()
.endpoint(endpoint)
.credential(defaultAzureCredential)
.retryOptions(requestRetryOptions)
.buildClient();
this.clientCache.put(cacheKey, blobServiceClient);
......
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.azure.di;
import com.azure.storage.common.policy.RequestRetryOptions;
import com.azure.storage.common.policy.RetryPolicyType;
import lombok.Getter;
import lombok.Setter;
import org.opengroup.osdu.azure.logging.CoreLoggerFactory;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
import java.time.Duration;
/**
* Config for BlogStorage Retry.
*/
@Configuration
@ConfigurationProperties("azure.blobstore")
@Getter
@Setter
public class BlobStoreRetryConfiguration {
public static final String LOGGER_NAME = BlobStoreRetryConfiguration.class.getName();
private final RequestRetryOptions defaultRequestRetryOptions = new RequestRetryOptions();
private static final String DEFAULT_STRING_VALUE = "";
private int maxTries = defaultRequestRetryOptions.getMaxTries();
private long tryTimeoutInSeconds = defaultRequestRetryOptions.getTryTimeoutDuration().getSeconds();
private long retryDelayInMs = defaultRequestRetryOptions.getRetryDelay().toMillis();
private long maxRetryDelayInMs = defaultRequestRetryOptions.getMaxRetryDelay().toMillis();
private String retryPolicyTypeValue = DEFAULT_STRING_VALUE;
/**
* Checks whether an string variable value is configured or not.
* @param val string value to be checked
* @return true if value is configured in app.properties
*/
private boolean valueConfigured(final String val) {
if (val == null || val.equals(DEFAULT_STRING_VALUE)) {
return false;
}
return true;
}
/**
* Method to get RequestRetryOptions object based on configuration set in applicaiton.properties.
* @return RequestRetryOption object with appropriate configurations.
*/
public RequestRetryOptions getRequestRetryOptions() {
// Check whether the variables have been set, else keep them as null.
// Value has to be sent as null incase where they are not configured to use the default configurations (As specified in RequestRetryOptions.class)
// https://azure.github.io/azure-storage-java-async/com/microsoft/azure/storage/blob/RequestRetryOptions.html
RetryPolicyType retryPolicyType;
try {
retryPolicyType = valueConfigured(retryPolicyTypeValue) ? RetryPolicyType.valueOf(retryPolicyTypeValue) : RetryPolicyType.EXPONENTIAL;
} catch (Exception ex) { // For wrong values of Retry Policy, it will default to Exponential.
retryPolicyType = RetryPolicyType.EXPONENTIAL;
}
RequestRetryOptions requestRetryOptions = new RequestRetryOptions(retryPolicyType, maxTries, Duration.ofSeconds(tryTimeoutInSeconds), Duration.ofMillis(retryDelayInMs), Duration.ofMillis(maxRetryDelayInMs), null);
CoreLoggerFactory.getInstance().getLogger(LOGGER_NAME).info(String.format("Retry Options on BlobStorage with RetryPolicyType = %s , maxTries = %d , tryTimeout = %d , retryDelay = %d , maxRetryDelay = %d .",
retryPolicyType, requestRetryOptions.getMaxTries(), requestRetryOptions.getTryTimeoutDuration().getSeconds(), requestRetryOptions.getRetryDelay().toMillis(), requestRetryOptions.getMaxRetryDelay().toMillis()));
return requestRetryOptions;
}
}
......@@ -16,6 +16,7 @@ package org.opengroup.osdu.azure.blobstorage;
import com.azure.identity.DefaultAzureCredential;
import com.azure.storage.blob.BlobServiceClient;
import com.azure.storage.common.policy.RequestRetryOptions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
......@@ -24,8 +25,10 @@ import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.opengroup.osdu.azure.cache.BlobServiceClientCache;
import org.opengroup.osdu.azure.di.BlobStoreConfiguration;
import org.opengroup.osdu.azure.di.BlobStoreRetryConfiguration;
import org.opengroup.osdu.azure.partition.PartitionInfoAzure;
import org.opengroup.osdu.azure.partition.PartitionServiceClient;
import org.opengroup.osdu.core.common.logging.ILogger;
import org.opengroup.osdu.core.common.partition.Property;
import static org.junit.jupiter.api.Assertions.*;
......@@ -43,6 +46,10 @@ public class BlobServiceClientFactoryImplTest {
private BlobServiceClientCache clientCache;
@Mock
private BlobStoreConfiguration configuration;
@Mock
private BlobStoreRetryConfiguration blobStoreRetryConfiguration;
@Mock
private ILogger logger;
@InjectMocks
private BlobServiceClientFactoryImpl sut;
......@@ -83,6 +90,7 @@ public class BlobServiceClientFactoryImplTest {
PartitionInfoAzure.builder()
.idConfig(Property.builder().value(PARTITION_ID).build())
.storageAccountNameConfig(Property.builder().value(ACCOUNT_NAME).build()).build());
when(this.blobStoreRetryConfiguration.getRequestRetryOptions()).thenReturn(new RequestRetryOptions());
BlobServiceClient serviceClient = this.sut.getBlobServiceClient(PARTITION_ID);
assertNotNull(serviceClient);
......@@ -94,6 +102,7 @@ public class BlobServiceClientFactoryImplTest {
PartitionInfoAzure.builder()
.idConfig(Property.builder().value(PARTITION_ID).build())
.storageAccountNameConfig(Property.builder().value(ACCOUNT_NAME).build()).build());
when(this.blobStoreRetryConfiguration.getRequestRetryOptions()).thenReturn(new RequestRetryOptions());
BlobServiceClient serviceClient = this.sut.getBlobServiceClient(PARTITION_ID);
assertNotNull(serviceClient);
......
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.azure.di;
import com.azure.storage.common.policy.RequestRetryOptions;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.opengroup.osdu.azure.logging.CoreLogger;
import org.opengroup.osdu.azure.logging.CoreLoggerFactory;
import org.opengroup.osdu.core.common.logging.ILogger;
import java.lang.reflect.Field;
import java.time.Duration;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.when;
import static org.mockito.MockitoAnnotations.openMocks;
@ExtendWith(MockitoExtension.class)
public class BlobStoreRetryConfigurationTest {
@InjectMocks
private BlobStoreRetryConfiguration blobStoreRetryConfiguration;
@Mock
private ILogger logger;
RequestRetryOptions defaultRequestRetryOptions = new RequestRetryOptions();
@Mock
private CoreLoggerFactory coreLoggerFactory;
@Mock
private CoreLogger coreLogger;
/**
* Workaround for inability to mock static methods like getInstance().
*
* @param mock CoreLoggerFactory mock instance
*/
private void mockSingleton(CoreLoggerFactory mock) {
try {
Field instance = CoreLoggerFactory.class.getDeclaredField("instance");
instance.setAccessible(true);
instance.set(null, mock);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
/**
* Reset workaround for inability to mock static methods like getInstance().
*/
private void resetSingleton() {
try {
Field instance = CoreLoggerFactory.class.getDeclaredField("instance");
instance.setAccessible(true);
instance.set(null, null);
instance.setAccessible(false);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@BeforeEach
void setUp() {
openMocks(this);
mockSingleton(coreLoggerFactory);
when(coreLoggerFactory.getLogger(anyString())).thenReturn(coreLogger);
}
@AfterEach
public void takeDown() {
resetSingleton();
}
@Test
public void should_set_default_values() {
RequestRetryOptions requestRetryOptions = blobStoreRetryConfiguration.getRequestRetryOptions();
assertEquals(requestRetryOptions.getMaxTries(), defaultRequestRetryOptions.getMaxTries());
assertEquals(requestRetryOptions.getTryTimeoutDuration(), defaultRequestRetryOptions.getTryTimeoutDuration());
assertEquals(requestRetryOptions.getRetryDelay(), defaultRequestRetryOptions.getRetryDelay());
assertEquals(requestRetryOptions.getMaxRetryDelay(), defaultRequestRetryOptions.getMaxRetryDelay());
}
@Test
public void should_set_maxtries() {
int maxTriesValue = 10;
blobStoreRetryConfiguration.setMaxTries(maxTriesValue);
RequestRetryOptions requestRetryOptions = blobStoreRetryConfiguration.getRequestRetryOptions();
assertEquals(requestRetryOptions.getMaxTries(), maxTriesValue);
assertEquals(requestRetryOptions.getTryTimeoutDuration(), defaultRequestRetryOptions.getTryTimeoutDuration());
assertEquals(requestRetryOptions.getRetryDelay(), defaultRequestRetryOptions.getRetryDelay());
assertEquals(requestRetryOptions.getMaxRetryDelay(), defaultRequestRetryOptions.getMaxRetryDelay());
}
@Test
public void should_set_try_timeout() {
int tryTimeoutValue = 50;
blobStoreRetryConfiguration.setTryTimeoutInSeconds(tryTimeoutValue);
RequestRetryOptions requestRetryOptions = blobStoreRetryConfiguration.getRequestRetryOptions();
assertEquals(requestRetryOptions.getMaxTries(), defaultRequestRetryOptions.getMaxTries());
assertEquals(requestRetryOptions.getTryTimeoutDuration(), Duration.ofSeconds(tryTimeoutValue));
assertEquals(requestRetryOptions.getRetryDelay(), defaultRequestRetryOptions.getRetryDelay());
assertEquals(requestRetryOptions.getMaxRetryDelay(), defaultRequestRetryOptions.getMaxRetryDelay());
}
@Test
public void should_set_RetryDelay() {
int retryDelayValue = 50;
int maxRetryDelayValue = 100;
blobStoreRetryConfiguration.setRetryDelayInMs(retryDelayValue);
blobStoreRetryConfiguration.setMaxRetryDelayInMs(maxRetryDelayValue);
RequestRetryOptions requestRetryOptions = blobStoreRetryConfiguration.getRequestRetryOptions();
assertEquals(requestRetryOptions.getMaxTries(), defaultRequestRetryOptions.getMaxTries());
assertEquals(requestRetryOptions.getTryTimeoutDuration(), defaultRequestRetryOptions.getTryTimeoutDuration());
assertEquals(requestRetryOptions.getRetryDelay(), Duration.ofMillis(retryDelayValue));
assertEquals(requestRetryOptions.getMaxRetryDelay(), Duration.ofMillis(maxRetryDelayValue));
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment