Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
Menu
Open sidebar
Open Subsurface Data Universe Software
Platform
System
Lib
cloud
azure
OS Core Lib Azure
Commits
33180eed
Commit
33180eed
authored
Jun 17, 2021
by
Hema Vishnu Pola [Microsoft]
Browse files
Merge branch 'rosakhuj/blobstore2' into 'master'
Blobstore Retry Configuration See merge request
!104
parents
cc072fa5
dd5cd8ad
Pipeline
#46747
passed with stages
in 7 minutes and 42 seconds
Changes
4
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
src/main/java/org/opengroup/osdu/azure/blobstorage/BlobServiceClientFactoryImpl.java
View file @
33180eed
...
...
@@ -17,10 +17,13 @@ package org.opengroup.osdu.azure.blobstorage;
import
com.azure.identity.DefaultAzureCredential
;
import
com.azure.storage.blob.BlobServiceClient
;
import
com.azure.storage.blob.BlobServiceClientBuilder
;
import
com.azure.storage.common.policy.RequestRetryOptions
;
import
org.opengroup.osdu.azure.cache.BlobServiceClientCache
;
import
org.opengroup.osdu.azure.di.BlobStoreRetryConfiguration
;
import
org.opengroup.osdu.azure.partition.PartitionInfoAzure
;
import
org.opengroup.osdu.azure.partition.PartitionServiceClient
;
import
org.opengroup.osdu.common.Validators
;
import
org.springframework.beans.factory.annotation.Autowired
;
/**
* Implementation for IBlobServiceClientFactory.
...
...
@@ -30,6 +33,9 @@ public class BlobServiceClientFactoryImpl implements IBlobServiceClientFactory {
private
PartitionServiceClient
partitionService
;
private
BlobServiceClientCache
clientCache
;
@Autowired
private
BlobStoreRetryConfiguration
blobStoreRetryConfiguration
;
/**
* Constructor to initialize instance of {@link BlobServiceClientFactoryImpl}.
* @param credentials Azure Credentials to use
...
...
@@ -61,9 +67,12 @@ public class BlobServiceClientFactoryImpl implements IBlobServiceClientFactory {
PartitionInfoAzure
pi
=
this
.
partitionService
.
getPartition
(
dataPartitionId
);
String
endpoint
=
String
.
format
(
"https://%s.blob.core.windows.net"
,
pi
.
getStorageAccountName
());
RequestRetryOptions
requestRetryOptions
=
blobStoreRetryConfiguration
.
getRequestRetryOptions
();
BlobServiceClient
blobServiceClient
=
new
BlobServiceClientBuilder
()
.
endpoint
(
endpoint
)
.
credential
(
defaultAzureCredential
)
.
retryOptions
(
requestRetryOptions
)
.
buildClient
();
this
.
clientCache
.
put
(
cacheKey
,
blobServiceClient
);
...
...
src/main/java/org/opengroup/osdu/azure/di/BlobStoreRetryConfiguration.java
0 → 100644
View file @
33180eed
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package
org.opengroup.osdu.azure.di
;
import
com.azure.storage.common.policy.RequestRetryOptions
;
import
com.azure.storage.common.policy.RetryPolicyType
;
import
lombok.Getter
;
import
lombok.Setter
;
import
org.opengroup.osdu.azure.logging.CoreLoggerFactory
;
import
org.springframework.boot.context.properties.ConfigurationProperties
;
import
org.springframework.context.annotation.Configuration
;
import
java.time.Duration
;
/**
* Config for BlogStorage Retry.
*/
@Configuration
@ConfigurationProperties
(
"azure.blobstore"
)
@Getter
@Setter
public
class
BlobStoreRetryConfiguration
{
public
static
final
String
LOGGER_NAME
=
BlobStoreRetryConfiguration
.
class
.
getName
();
private
final
RequestRetryOptions
defaultRequestRetryOptions
=
new
RequestRetryOptions
();
private
static
final
String
DEFAULT_STRING_VALUE
=
""
;
private
int
maxTries
=
defaultRequestRetryOptions
.
getMaxTries
();
private
long
tryTimeoutInSeconds
=
defaultRequestRetryOptions
.
getTryTimeoutDuration
().
getSeconds
();
private
long
retryDelayInMs
=
defaultRequestRetryOptions
.
getRetryDelay
().
toMillis
();
private
long
maxRetryDelayInMs
=
defaultRequestRetryOptions
.
getMaxRetryDelay
().
toMillis
();
private
String
retryPolicyTypeValue
=
DEFAULT_STRING_VALUE
;
/**
* Checks whether an string variable value is configured or not.
* @param val string value to be checked
* @return true if value is configured in app.properties
*/
private
boolean
valueConfigured
(
final
String
val
)
{
if
(
val
==
null
||
val
.
equals
(
DEFAULT_STRING_VALUE
))
{
return
false
;
}
return
true
;
}
/**
* Method to get RequestRetryOptions object based on configuration set in applicaiton.properties.
* @return RequestRetryOption object with appropriate configurations.
*/
public
RequestRetryOptions
getRequestRetryOptions
()
{
// Check whether the variables have been set, else keep them as null.
// Value has to be sent as null incase where they are not configured to use the default configurations (As specified in RequestRetryOptions.class)
// https://azure.github.io/azure-storage-java-async/com/microsoft/azure/storage/blob/RequestRetryOptions.html
RetryPolicyType
retryPolicyType
;
try
{
retryPolicyType
=
valueConfigured
(
retryPolicyTypeValue
)
?
RetryPolicyType
.
valueOf
(
retryPolicyTypeValue
)
:
RetryPolicyType
.
EXPONENTIAL
;
}
catch
(
Exception
ex
)
{
// For wrong values of Retry Policy, it will default to Exponential.
retryPolicyType
=
RetryPolicyType
.
EXPONENTIAL
;
}
RequestRetryOptions
requestRetryOptions
=
new
RequestRetryOptions
(
retryPolicyType
,
maxTries
,
Duration
.
ofSeconds
(
tryTimeoutInSeconds
),
Duration
.
ofMillis
(
retryDelayInMs
),
Duration
.
ofMillis
(
maxRetryDelayInMs
),
null
);
CoreLoggerFactory
.
getInstance
().
getLogger
(
LOGGER_NAME
).
info
(
String
.
format
(
"Retry Options on BlobStorage with RetryPolicyType = %s , maxTries = %d , tryTimeout = %d , retryDelay = %d , maxRetryDelay = %d ."
,
retryPolicyType
,
requestRetryOptions
.
getMaxTries
(),
requestRetryOptions
.
getTryTimeoutDuration
().
getSeconds
(),
requestRetryOptions
.
getRetryDelay
().
toMillis
(),
requestRetryOptions
.
getMaxRetryDelay
().
toMillis
()));
return
requestRetryOptions
;
}
}
src/test/java/org/opengroup/osdu/azure/blobstorage/BlobServiceClientFactoryImplTest.java
View file @
33180eed
...
...
@@ -16,6 +16,7 @@ package org.opengroup.osdu.azure.blobstorage;
import
com.azure.identity.DefaultAzureCredential
;
import
com.azure.storage.blob.BlobServiceClient
;
import
com.azure.storage.common.policy.RequestRetryOptions
;
import
org.junit.jupiter.api.BeforeEach
;
import
org.junit.jupiter.api.Test
;
import
org.junit.jupiter.api.extension.ExtendWith
;
...
...
@@ -24,8 +25,10 @@ import org.mockito.Mock;
import
org.mockito.junit.jupiter.MockitoExtension
;
import
org.opengroup.osdu.azure.cache.BlobServiceClientCache
;
import
org.opengroup.osdu.azure.di.BlobStoreConfiguration
;
import
org.opengroup.osdu.azure.di.BlobStoreRetryConfiguration
;
import
org.opengroup.osdu.azure.partition.PartitionInfoAzure
;
import
org.opengroup.osdu.azure.partition.PartitionServiceClient
;
import
org.opengroup.osdu.core.common.logging.ILogger
;
import
org.opengroup.osdu.core.common.partition.Property
;
import
static
org
.
junit
.
jupiter
.
api
.
Assertions
.*;
...
...
@@ -43,6 +46,10 @@ public class BlobServiceClientFactoryImplTest {
private
BlobServiceClientCache
clientCache
;
@Mock
private
BlobStoreConfiguration
configuration
;
@Mock
private
BlobStoreRetryConfiguration
blobStoreRetryConfiguration
;
@Mock
private
ILogger
logger
;
@InjectMocks
private
BlobServiceClientFactoryImpl
sut
;
...
...
@@ -83,6 +90,7 @@ public class BlobServiceClientFactoryImplTest {
PartitionInfoAzure
.
builder
()
.
idConfig
(
Property
.
builder
().
value
(
PARTITION_ID
).
build
())
.
storageAccountNameConfig
(
Property
.
builder
().
value
(
ACCOUNT_NAME
).
build
()).
build
());
when
(
this
.
blobStoreRetryConfiguration
.
getRequestRetryOptions
()).
thenReturn
(
new
RequestRetryOptions
());
BlobServiceClient
serviceClient
=
this
.
sut
.
getBlobServiceClient
(
PARTITION_ID
);
assertNotNull
(
serviceClient
);
...
...
@@ -94,6 +102,7 @@ public class BlobServiceClientFactoryImplTest {
PartitionInfoAzure
.
builder
()
.
idConfig
(
Property
.
builder
().
value
(
PARTITION_ID
).
build
())
.
storageAccountNameConfig
(
Property
.
builder
().
value
(
ACCOUNT_NAME
).
build
()).
build
());
when
(
this
.
blobStoreRetryConfiguration
.
getRequestRetryOptions
()).
thenReturn
(
new
RequestRetryOptions
());
BlobServiceClient
serviceClient
=
this
.
sut
.
getBlobServiceClient
(
PARTITION_ID
);
assertNotNull
(
serviceClient
);
...
...
src/test/java/org/opengroup/osdu/azure/di/BlobStoreRetryConfigurationTest.java
0 → 100644
View file @
33180eed
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package
org.opengroup.osdu.azure.di
;
import
com.azure.storage.common.policy.RequestRetryOptions
;
import
org.junit.jupiter.api.AfterEach
;
import
org.junit.jupiter.api.BeforeEach
;
import
org.junit.jupiter.api.Test
;
import
org.junit.jupiter.api.extension.ExtendWith
;
import
org.mockito.InjectMocks
;
import
org.mockito.Mock
;
import
org.mockito.junit.jupiter.MockitoExtension
;
import
org.opengroup.osdu.azure.logging.CoreLogger
;
import
org.opengroup.osdu.azure.logging.CoreLoggerFactory
;
import
org.opengroup.osdu.core.common.logging.ILogger
;
import
java.lang.reflect.Field
;
import
java.time.Duration
;
import
static
org
.
junit
.
jupiter
.
api
.
Assertions
.
assertEquals
;
import
static
org
.
mockito
.
ArgumentMatchers
.
anyString
;
import
static
org
.
mockito
.
Mockito
.
when
;
import
static
org
.
mockito
.
MockitoAnnotations
.
openMocks
;
@ExtendWith
(
MockitoExtension
.
class
)
public
class
BlobStoreRetryConfigurationTest
{
@InjectMocks
private
BlobStoreRetryConfiguration
blobStoreRetryConfiguration
;
@Mock
private
ILogger
logger
;
RequestRetryOptions
defaultRequestRetryOptions
=
new
RequestRetryOptions
();
@Mock
private
CoreLoggerFactory
coreLoggerFactory
;
@Mock
private
CoreLogger
coreLogger
;
/**
* Workaround for inability to mock static methods like getInstance().
*
* @param mock CoreLoggerFactory mock instance
*/
private
void
mockSingleton
(
CoreLoggerFactory
mock
)
{
try
{
Field
instance
=
CoreLoggerFactory
.
class
.
getDeclaredField
(
"instance"
);
instance
.
setAccessible
(
true
);
instance
.
set
(
null
,
mock
);
}
catch
(
Exception
e
)
{
throw
new
RuntimeException
(
e
);
}
}
/**
* Reset workaround for inability to mock static methods like getInstance().
*/
private
void
resetSingleton
()
{
try
{
Field
instance
=
CoreLoggerFactory
.
class
.
getDeclaredField
(
"instance"
);
instance
.
setAccessible
(
true
);
instance
.
set
(
null
,
null
);
instance
.
setAccessible
(
false
);
}
catch
(
Exception
e
)
{
throw
new
RuntimeException
(
e
);
}
}
@BeforeEach
void
setUp
()
{
openMocks
(
this
);
mockSingleton
(
coreLoggerFactory
);
when
(
coreLoggerFactory
.
getLogger
(
anyString
())).
thenReturn
(
coreLogger
);
}
@AfterEach
public
void
takeDown
()
{
resetSingleton
();
}
@Test
public
void
should_set_default_values
()
{
RequestRetryOptions
requestRetryOptions
=
blobStoreRetryConfiguration
.
getRequestRetryOptions
();
assertEquals
(
requestRetryOptions
.
getMaxTries
(),
defaultRequestRetryOptions
.
getMaxTries
());
assertEquals
(
requestRetryOptions
.
getTryTimeoutDuration
(),
defaultRequestRetryOptions
.
getTryTimeoutDuration
());
assertEquals
(
requestRetryOptions
.
getRetryDelay
(),
defaultRequestRetryOptions
.
getRetryDelay
());
assertEquals
(
requestRetryOptions
.
getMaxRetryDelay
(),
defaultRequestRetryOptions
.
getMaxRetryDelay
());
}
@Test
public
void
should_set_maxtries
()
{
int
maxTriesValue
=
10
;
blobStoreRetryConfiguration
.
setMaxTries
(
maxTriesValue
);
RequestRetryOptions
requestRetryOptions
=
blobStoreRetryConfiguration
.
getRequestRetryOptions
();
assertEquals
(
requestRetryOptions
.
getMaxTries
(),
maxTriesValue
);
assertEquals
(
requestRetryOptions
.
getTryTimeoutDuration
(),
defaultRequestRetryOptions
.
getTryTimeoutDuration
());
assertEquals
(
requestRetryOptions
.
getRetryDelay
(),
defaultRequestRetryOptions
.
getRetryDelay
());
assertEquals
(
requestRetryOptions
.
getMaxRetryDelay
(),
defaultRequestRetryOptions
.
getMaxRetryDelay
());
}
@Test
public
void
should_set_try_timeout
()
{
int
tryTimeoutValue
=
50
;
blobStoreRetryConfiguration
.
setTryTimeoutInSeconds
(
tryTimeoutValue
);
RequestRetryOptions
requestRetryOptions
=
blobStoreRetryConfiguration
.
getRequestRetryOptions
();
assertEquals
(
requestRetryOptions
.
getMaxTries
(),
defaultRequestRetryOptions
.
getMaxTries
());
assertEquals
(
requestRetryOptions
.
getTryTimeoutDuration
(),
Duration
.
ofSeconds
(
tryTimeoutValue
));
assertEquals
(
requestRetryOptions
.
getRetryDelay
(),
defaultRequestRetryOptions
.
getRetryDelay
());
assertEquals
(
requestRetryOptions
.
getMaxRetryDelay
(),
defaultRequestRetryOptions
.
getMaxRetryDelay
());
}
@Test
public
void
should_set_RetryDelay
()
{
int
retryDelayValue
=
50
;
int
maxRetryDelayValue
=
100
;
blobStoreRetryConfiguration
.
setRetryDelayInMs
(
retryDelayValue
);
blobStoreRetryConfiguration
.
setMaxRetryDelayInMs
(
maxRetryDelayValue
);
RequestRetryOptions
requestRetryOptions
=
blobStoreRetryConfiguration
.
getRequestRetryOptions
();
assertEquals
(
requestRetryOptions
.
getMaxTries
(),
defaultRequestRetryOptions
.
getMaxTries
());
assertEquals
(
requestRetryOptions
.
getTryTimeoutDuration
(),
defaultRequestRetryOptions
.
getTryTimeoutDuration
());
assertEquals
(
requestRetryOptions
.
getRetryDelay
(),
Duration
.
ofMillis
(
retryDelayValue
));
assertEquals
(
requestRetryOptions
.
getMaxRetryDelay
(),
Duration
.
ofMillis
(
maxRetryDelayValue
));
}
}
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment