Merge branch 'rosakhuj/blobstore2' into 'master'

Blobstore Retry Configuration

See merge request !104

src/main/java/org/opengroup/osdu/azure/blobstorage/BlobServiceClientFactoryImpl.java

@@ -17,10 +17,13 @@ package org.opengroup.osdu.azure.blobstorage;
 import com.azure.identity.DefaultAzureCredential;
 import com.azure.storage.blob.BlobServiceClient;
 import com.azure.storage.blob.BlobServiceClientBuilder;
+import com.azure.storage.common.policy.RequestRetryOptions;
 import org.opengroup.osdu.azure.cache.BlobServiceClientCache;
+import org.opengroup.osdu.azure.di.BlobStoreRetryConfiguration;
 import org.opengroup.osdu.azure.partition.PartitionInfoAzure;
 import org.opengroup.osdu.azure.partition.PartitionServiceClient;
 import org.opengroup.osdu.common.Validators;
+import org.springframework.beans.factory.annotation.Autowired;
 
 /**
  * Implementation for IBlobServiceClientFactory.
@@ -30,6 +33,9 @@ public class BlobServiceClientFactoryImpl implements IBlobServiceClientFactory {
     private PartitionServiceClient partitionService;
     private BlobServiceClientCache clientCache;
 
+    @Autowired
+    private BlobStoreRetryConfiguration blobStoreRetryConfiguration;
+
     /**
      * Constructor to initialize instance of {@link BlobServiceClientFactoryImpl}.
      * @param credentials Azure Credentials to use
@@ -61,9 +67,12 @@ public class BlobServiceClientFactoryImpl implements IBlobServiceClientFactory {
         PartitionInfoAzure pi = this.partitionService.getPartition(dataPartitionId);
         String endpoint = String.format("https://%s.blob.core.windows.net", pi.getStorageAccountName());
 
+        RequestRetryOptions requestRetryOptions = blobStoreRetryConfiguration.getRequestRetryOptions();
+
         BlobServiceClient blobServiceClient = new BlobServiceClientBuilder()
                 .endpoint(endpoint)
                 .credential(defaultAzureCredential)
+                .retryOptions(requestRetryOptions)
                 .buildClient();
 
         this.clientCache.put(cacheKey, blobServiceClient);

src/main/java/org/opengroup/osdu/azure/di/BlobStoreRetryConfiguration.java

+// Copyright © Microsoft Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.azure.di;
+
+import com.azure.storage.common.policy.RequestRetryOptions;
+import com.azure.storage.common.policy.RetryPolicyType;
+import lombok.Getter;
+import lombok.Setter;
+import org.opengroup.osdu.azure.logging.CoreLoggerFactory;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+import java.time.Duration;
+
+/**
+ * Config for BlogStorage Retry.
+ */
+@Configuration
+@ConfigurationProperties("azure.blobstore")
+@Getter
+@Setter
+public class BlobStoreRetryConfiguration {
+
+    public static final String LOGGER_NAME = BlobStoreRetryConfiguration.class.getName();
+
+    private final RequestRetryOptions defaultRequestRetryOptions = new RequestRetryOptions();
+    private static final String DEFAULT_STRING_VALUE = "";
+
+    private int maxTries = defaultRequestRetryOptions.getMaxTries();
+    private long tryTimeoutInSeconds = defaultRequestRetryOptions.getTryTimeoutDuration().getSeconds();
+    private long retryDelayInMs = defaultRequestRetryOptions.getRetryDelay().toMillis();
+    private long maxRetryDelayInMs = defaultRequestRetryOptions.getMaxRetryDelay().toMillis();
+    private String retryPolicyTypeValue = DEFAULT_STRING_VALUE;
+
+    /**
+     * Checks whether an string variable value is configured or not.
+     * @param val string value to be checked
+     * @return true if value is configured in app.properties
+     */
+    private boolean valueConfigured(final String val) {
+        if (val == null || val.equals(DEFAULT_STRING_VALUE)) {
+            return false;
+        }
+        return true;
+    }
+
+    /**
+     * Method to get RequestRetryOptions object based on configuration set in applicaiton.properties.
+     * @return RequestRetryOption object with appropriate configurations.
+     */
+    public RequestRetryOptions getRequestRetryOptions() {
+
+        // Check whether the variables have been set, else keep them as null.
+        // Value has to be sent as null incase where they are not configured to use the default configurations (As specified in RequestRetryOptions.class)
+        // https://azure.github.io/azure-storage-java-async/com/microsoft/azure/storage/blob/RequestRetryOptions.html
+        RetryPolicyType retryPolicyType;
+        try {
+            retryPolicyType = valueConfigured(retryPolicyTypeValue) ? RetryPolicyType.valueOf(retryPolicyTypeValue) : RetryPolicyType.EXPONENTIAL;
+        } catch (Exception ex) { // For wrong values of Retry Policy, it will default to Exponential.
+            retryPolicyType = RetryPolicyType.EXPONENTIAL;
+        }
+
+
+        RequestRetryOptions requestRetryOptions = new RequestRetryOptions(retryPolicyType, maxTries, Duration.ofSeconds(tryTimeoutInSeconds), Duration.ofMillis(retryDelayInMs), Duration.ofMillis(maxRetryDelayInMs), null);
+
+
+        CoreLoggerFactory.getInstance().getLogger(LOGGER_NAME).info(String.format("Retry Options on BlobStorage with RetryPolicyType = %s , maxTries = %d , tryTimeout = %d , retryDelay = %d , maxRetryDelay = %d .",
+                retryPolicyType, requestRetryOptions.getMaxTries(), requestRetryOptions.getTryTimeoutDuration().getSeconds(), requestRetryOptions.getRetryDelay().toMillis(), requestRetryOptions.getMaxRetryDelay().toMillis()));
+
+        return requestRetryOptions;
+    }
+}

src/test/java/org/opengroup/osdu/azure/blobstorage/BlobServiceClientFactoryImplTest.java

@@ -16,6 +16,7 @@ package org.opengroup.osdu.azure.blobstorage;
 
 import com.azure.identity.DefaultAzureCredential;
 import com.azure.storage.blob.BlobServiceClient;
+import com.azure.storage.common.policy.RequestRetryOptions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
@@ -24,8 +25,10 @@ import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
 import org.opengroup.osdu.azure.cache.BlobServiceClientCache;
 import org.opengroup.osdu.azure.di.BlobStoreConfiguration;
+import org.opengroup.osdu.azure.di.BlobStoreRetryConfiguration;
 import org.opengroup.osdu.azure.partition.PartitionInfoAzure;
 import org.opengroup.osdu.azure.partition.PartitionServiceClient;
+import org.opengroup.osdu.core.common.logging.ILogger;
 import org.opengroup.osdu.core.common.partition.Property;
 
 import static org.junit.jupiter.api.Assertions.*;
@@ -43,6 +46,10 @@ public class BlobServiceClientFactoryImplTest {
     private BlobServiceClientCache clientCache;
     @Mock
     private BlobStoreConfiguration configuration;
+    @Mock
+    private BlobStoreRetryConfiguration blobStoreRetryConfiguration;
+    @Mock
+    private ILogger logger;
     @InjectMocks
     private BlobServiceClientFactoryImpl sut;
 
@@ -83,6 +90,7 @@ public class BlobServiceClientFactoryImplTest {
                 PartitionInfoAzure.builder()
                         .idConfig(Property.builder().value(PARTITION_ID).build())
                         .storageAccountNameConfig(Property.builder().value(ACCOUNT_NAME).build()).build());
+        when(this.blobStoreRetryConfiguration.getRequestRetryOptions()).thenReturn(new RequestRetryOptions());
 
         BlobServiceClient serviceClient = this.sut.getBlobServiceClient(PARTITION_ID);
         assertNotNull(serviceClient);
@@ -94,6 +102,7 @@ public class BlobServiceClientFactoryImplTest {
                 PartitionInfoAzure.builder()
                         .idConfig(Property.builder().value(PARTITION_ID).build())
                         .storageAccountNameConfig(Property.builder().value(ACCOUNT_NAME).build()).build());
+        when(this.blobStoreRetryConfiguration.getRequestRetryOptions()).thenReturn(new RequestRetryOptions());
 
         BlobServiceClient serviceClient = this.sut.getBlobServiceClient(PARTITION_ID);
         assertNotNull(serviceClient);

src/test/java/org/opengroup/osdu/azure/di/BlobStoreRetryConfigurationTest.java

+// Copyright © Microsoft Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.azure.di;
+
+import com.azure.storage.common.policy.RequestRetryOptions;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.opengroup.osdu.azure.logging.CoreLogger;
+import org.opengroup.osdu.azure.logging.CoreLoggerFactory;
+import org.opengroup.osdu.core.common.logging.ILogger;
+
+import java.lang.reflect.Field;
+import java.time.Duration;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.when;
+import static org.mockito.MockitoAnnotations.openMocks;
+
+@ExtendWith(MockitoExtension.class)
+public class BlobStoreRetryConfigurationTest {
+    @InjectMocks
+    private BlobStoreRetryConfiguration blobStoreRetryConfiguration;
+    @Mock
+    private ILogger logger;
+    RequestRetryOptions defaultRequestRetryOptions = new RequestRetryOptions();
+    @Mock
+    private CoreLoggerFactory coreLoggerFactory;
+    @Mock
+    private CoreLogger coreLogger;
+
+    /**
+     * Workaround for inability to mock static methods like getInstance().
+     *
+     * @param mock CoreLoggerFactory mock instance
+     */
+    private void mockSingleton(CoreLoggerFactory mock) {
+        try {
+            Field instance = CoreLoggerFactory.class.getDeclaredField("instance");
+            instance.setAccessible(true);
+            instance.set(null, mock);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * Reset workaround for inability to mock static methods like getInstance().
+     */
+    private void resetSingleton() {
+        try {
+            Field instance = CoreLoggerFactory.class.getDeclaredField("instance");
+            instance.setAccessible(true);
+            instance.set(null, null);
+            instance.setAccessible(false);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @BeforeEach
+    void setUp() {
+        openMocks(this);
+        mockSingleton(coreLoggerFactory);
+        when(coreLoggerFactory.getLogger(anyString())).thenReturn(coreLogger);
+    }
+
+    @AfterEach
+    public void takeDown() {
+        resetSingleton();
+    }
+
+    @Test
+    public void should_set_default_values() {
+        RequestRetryOptions requestRetryOptions = blobStoreRetryConfiguration.getRequestRetryOptions();
+
+        assertEquals(requestRetryOptions.getMaxTries(), defaultRequestRetryOptions.getMaxTries());
+        assertEquals(requestRetryOptions.getTryTimeoutDuration(), defaultRequestRetryOptions.getTryTimeoutDuration());
+        assertEquals(requestRetryOptions.getRetryDelay(), defaultRequestRetryOptions.getRetryDelay());
+        assertEquals(requestRetryOptions.getMaxRetryDelay(), defaultRequestRetryOptions.getMaxRetryDelay());
+    }
+
+    @Test
+    public void should_set_maxtries() {
+        int maxTriesValue = 10;
+        blobStoreRetryConfiguration.setMaxTries(maxTriesValue);
+        RequestRetryOptions requestRetryOptions = blobStoreRetryConfiguration.getRequestRetryOptions();
+
+        assertEquals(requestRetryOptions.getMaxTries(), maxTriesValue);
+        assertEquals(requestRetryOptions.getTryTimeoutDuration(), defaultRequestRetryOptions.getTryTimeoutDuration());
+        assertEquals(requestRetryOptions.getRetryDelay(), defaultRequestRetryOptions.getRetryDelay());
+        assertEquals(requestRetryOptions.getMaxRetryDelay(), defaultRequestRetryOptions.getMaxRetryDelay());
+    }
+
+    @Test
+    public void should_set_try_timeout() {
+        int tryTimeoutValue = 50;
+        blobStoreRetryConfiguration.setTryTimeoutInSeconds(tryTimeoutValue);
+        RequestRetryOptions requestRetryOptions = blobStoreRetryConfiguration.getRequestRetryOptions();
+
+        assertEquals(requestRetryOptions.getMaxTries(), defaultRequestRetryOptions.getMaxTries());
+        assertEquals(requestRetryOptions.getTryTimeoutDuration(), Duration.ofSeconds(tryTimeoutValue));
+        assertEquals(requestRetryOptions.getRetryDelay(), defaultRequestRetryOptions.getRetryDelay());
+        assertEquals(requestRetryOptions.getMaxRetryDelay(), defaultRequestRetryOptions.getMaxRetryDelay());
+    }
+
+    @Test
+    public void should_set_RetryDelay() {
+        int retryDelayValue = 50;
+        int maxRetryDelayValue = 100;
+        blobStoreRetryConfiguration.setRetryDelayInMs(retryDelayValue);
+        blobStoreRetryConfiguration.setMaxRetryDelayInMs(maxRetryDelayValue);
+        RequestRetryOptions requestRetryOptions = blobStoreRetryConfiguration.getRequestRetryOptions();
+
+        assertEquals(requestRetryOptions.getMaxTries(), defaultRequestRetryOptions.getMaxTries());
+        assertEquals(requestRetryOptions.getTryTimeoutDuration(), defaultRequestRetryOptions.getTryTimeoutDuration());
+        assertEquals(requestRetryOptions.getRetryDelay(), Duration.ofMillis(retryDelayValue));
+        assertEquals(requestRetryOptions.getMaxRetryDelay(), Duration.ofMillis(maxRetryDelayValue));
+    }
+
+}