Commit 09cd4931 authored by Ronak Sakhuja's avatar Ronak Sakhuja
Browse files

Merge branch 'master' of...

Merge branch 'master' of https://community.opengroup.org/osdu/platform/system/lib/cloud/azure/os-core-lib-azure into rosakhuj/blobstore2
parents afb539b4 9c273dcc
Pipeline #46606 failed with stage
in 12 seconds
......@@ -87,3 +87,11 @@ Enabled transaction logger and slf4jlogger
| --- | --- | --- |
| `azure.blobStore.required` | `true` | - |
| `azure.storage.account-name` | ex `testStorage` | storage account name |
# Default retry and timeout values for service-to-service communication
| name | default value |
| --- | --- |
| `maxRetry` | `3` |
| `connectTimeout` | `60000` |
| `requestTimeout` | `60000` |
| `socketTimeout` | `60000` |
......@@ -20,7 +20,7 @@
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-azure</artifactId>
<packaging>jar</packaging>
<version>0.9.0-SNAPSHOT</version>
<version>0.10.0-SNAPSHOT</version>
<name>core-lib-azure</name>
<properties>
......@@ -37,7 +37,7 @@
<azure.appinsights.log4j.version>2.6.3</azure.appinsights.log4j.version>
<azure.spring.boot.version>3.4.0</azure.spring.boot.version>
<lombok.version>1.18.16</lombok.version>
<osdu.oscorecommon.version>0.9.0-rc13</osdu.oscorecommon.version>
<osdu.oscorecommon.version>0.9.0</osdu.oscorecommon.version>
<azure-mgmt-eventgrid.version>1.0.0-beta-3</azure-mgmt-eventgrid.version>
<azure-security-keyvault-keys.version>4.2.3</azure-security-keyvault-keys.version>
<documentdb-bulkexecutor.version>2.12.0</documentdb-bulkexecutor.version>
......
......@@ -21,6 +21,7 @@ import com.azure.storage.blob.models.BlobCopyInfo;
import com.azure.storage.blob.models.BlobErrorCode;
import com.azure.storage.blob.models.BlobStorageException;
import com.azure.storage.blob.models.CopyStatusType;
import com.azure.storage.blob.models.UserDelegationKey;
import com.azure.storage.blob.sas.BlobContainerSasPermission;
import com.azure.storage.blob.sas.BlobSasPermission;
import com.azure.storage.blob.sas.BlobServiceSasSignatureValues;
......@@ -253,6 +254,8 @@ public class BlobStore {
}
/**
* This method is used to generate pre-signed url for file (blob).
* NOTE: Using the below method will require BlobServiceClient to be instantiated using StorageSharedKeyCredential
* @param dataPartitionId Data partition id
* @param filePath Path of file (blob) for which SAS token needs to be generated
* @param containerName Name of the storage container
......@@ -267,8 +270,8 @@ public class BlobStore {
}
/**
* Generates pre-signed url to a blob container.
*
* This method is used to generate pre-signed url for blob container.
* NOTE: Using the below method will require BlobServiceClient to be instantiated using StorageSharedKeyCredential
* @param dataPartitionId data partition id
* @param containerName Name of the storage container
* @param expiryTime Time after which the token expires
......@@ -280,6 +283,30 @@ public class BlobStore {
return blobContainerClient.getBlobContainerUrl() + "?" + generateSASToken(blobContainerClient, expiryTime, permissions);
}
/**
* Generates pre-signed url to a blob container using the user delegation key.
*
* @param dataPartitionId data partition id
* @param containerName Name of the storage container
* @param startTime Time after which the token is activated (null in case of instant activation)
* @param expiryTime Time after which the token expires
* @param permissions permissions for the given container
* @return Generates pre-signed url for a given container
*/
public String generatePreSignedUrlWithUserDelegationSas(final String dataPartitionId, final String containerName, final OffsetDateTime startTime, final OffsetDateTime expiryTime, final BlobContainerSasPermission permissions) {
BlobContainerClient blobContainerClient = getBlobContainerClient(dataPartitionId, containerName);
BlobServiceClient blobServiceClient = blobServiceClientFactory.getBlobServiceClient(dataPartitionId);
UserDelegationKey userDelegationKey = blobServiceClient.getUserDelegationKey(startTime, expiryTime);
BlobServiceSasSignatureValues blobServiceSasSignatureValues = new BlobServiceSasSignatureValues(expiryTime, permissions).setStartTime(startTime);
final long start = System.currentTimeMillis();
String sasToken = blobContainerClient.generateUserDelegationSas(blobServiceSasSignatureValues, userDelegationKey);
final long timeTaken = System.currentTimeMillis() - start;
logDependency("GENERATE_PRESIGNED_URL_USER_DELEGATION_SAS", blobContainerClient.getBlobContainerName(), blobContainerClient.getBlobContainerUrl(), timeTaken, String.valueOf(HttpStatus.SC_OK), true);
return blobContainerClient.getBlobContainerUrl() + "?" + sasToken;
}
/**
* Method is used to copy a file specified at Source URL to the provided destination.
*
......
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License
package org.opengroup.osdu.azure.di;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
/**
* Configuration bean for setting up retry and timeouts variables.
*/
@Data
@Component
@ConfigurationProperties(prefix = "azure.service.retry.config")
public class RetryAndTimeoutConfiguration {
private int maxRetry = 3;
private int connectTimeout = 60000;
private int requestTimeout = 60000;
private int socketTimeout = 60000;
}
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License
package org.opengroup.osdu.azure.entitlements;
import lombok.Data;
import org.opengroup.osdu.core.common.entitlements.EntitlementsAPIConfig;
import org.springframework.beans.factory.config.AbstractFactoryBean;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
/**
* Creates bean of EntitlementsAPIConfig.
*/
@Data
@Component
@ConfigurationProperties
public class EntilementsAPIConfigBean extends AbstractFactoryBean<EntitlementsAPIConfig> {
private String authorizeAPI;
private String authorizeAPIKey;
/**
* Abstract method of AbstractBeanFactory<T>.
*
* @return class type
*/
@Override
public Class<?> getObjectType() {
return EntitlementsAPIConfig.class;
}
/**
* Abstract method of AbstractBeanFactory<T> type.
*
* @return EntitlementsAPIConfig
* @throws Exception
*/
@Override
protected EntitlementsAPIConfig createInstance() throws Exception {
return EntitlementsAPIConfig
.builder()
.rootUrl(authorizeAPI)
.apiKey(authorizeAPIKey)
.build();
}
}
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License
package org.opengroup.osdu.azure.entitlements;
import org.opengroup.osdu.core.common.entitlements.EntitlementsAPIConfig;
import org.opengroup.osdu.core.common.entitlements.EntitlementsService;
import org.opengroup.osdu.core.common.entitlements.IEntitlementsFactory;
import org.opengroup.osdu.core.common.entitlements.IEntitlementsService;
import org.opengroup.osdu.core.common.http.IHttpClient;
import org.opengroup.osdu.core.common.http.json.HttpResponseBodyMapper;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
import java.util.Objects;
/**
* Implements IEntitlementsFactory.
*/
@Component
@Primary
@ConditionalOnProperty(value = "azure.entitlements.factory.enabled", havingValue = "true", matchIfMissing = true)
public class EntitlementsFactoryAzure implements IEntitlementsFactory {
private final EntitlementsAPIConfig config;
private final HttpResponseBodyMapper mapper;
private final IHttpClient client;
/**
* Constructor Injection for above 3 fields.
*
* @param entitlementsConfig EntitlementsAPIConfig
* @param httpMapper HttpResponseBodyMapper
* @param httpClient IHttpClient
*/
@Autowired
public EntitlementsFactoryAzure(final EntitlementsAPIConfig entitlementsConfig, final HttpResponseBodyMapper httpMapper, final IHttpClient httpClient) {
this.config = entitlementsConfig;
this.mapper = httpMapper;
this.client = httpClient;
}
/**
* returns instance of EntitlementsService.
*
* @param headers DpsHeaders
* @return IEntitlementsService
*/
@Override
public IEntitlementsService create(final DpsHeaders headers) {
Objects.requireNonNull(headers, "headers cannot be null");
return new EntitlementsService(this.config,
this.client,
headers,
this.mapper);
}
}
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License
package org.opengroup.osdu.azure.httpconfig;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.http.FetchServiceHttpRequest;
import org.opengroup.osdu.core.common.http.HttpRequest;
import org.opengroup.osdu.core.common.http.IHttpClient;
import org.opengroup.osdu.core.common.http.HttpResponse;
import org.opengroup.osdu.core.common.http.UrlFetchServiceImpl;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
import java.net.URISyntaxException;
/**
* Extends URlFetchService and implements IHttpClient to send requests.
*/
@Primary
@Component
public class HttpClientAzure extends UrlFetchServiceImpl implements IHttpClient {
/**
* calls urlfetchservice's send request.
*
* @param httpRequest made by user class
* @return HttpResponse
*/
@Override
public HttpResponse send(final HttpRequest httpRequest) {
org.opengroup.osdu.core.common.model.http.HttpResponse response = null;
try {
response = super.sendRequest(FetchServiceHttpRequest.builder()
.body(httpRequest.getBody())
.httpMethod(httpRequest.getHttpMethod())
.queryParams(httpRequest.getQueryParams())
.url(httpRequest.getUrl())
.headers(httpRequest.getHeaders())
.build());
} catch (URISyntaxException e) {
throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getReason(), "URI Syntax is not correct", e);
}
HttpResponse httpResponse = new HttpResponse();
httpResponse.setBody(response.getBody());
httpResponse.setResponseCode(response.getResponseCode());
httpResponse.setContentType(response.getContentType());
httpResponse.setRequest(httpRequest);
return httpResponse;
}
}
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License
package org.opengroup.osdu.azure.httpconfig;
import org.apache.http.client.config.RequestConfig;
import org.opengroup.osdu.azure.di.RetryAndTimeoutConfiguration;
import org.opengroup.osdu.core.common.http.HttpClientHandler;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
/**
* Extends HttpClientHandler.
*/
@Primary
@Component
public class HttpClientHandlerAzure extends HttpClientHandler {
private RetryAndTimeoutConfiguration configuration;
/**
* Constuctor injection for RetryAndTimeoutConfiguration.
*
* @param retryAndTimeoutConfiguration of type RetryAndTimeoutConfiguration
*/
@Autowired
public HttpClientHandlerAzure(final RetryAndTimeoutConfiguration retryAndTimeoutConfiguration) {
this.configuration = retryAndTimeoutConfiguration;
super.REQUEST_CONFIG = RequestConfig.custom()
.setConnectTimeout(configuration.getConnectTimeout())
.setConnectionRequestTimeout(configuration.getRequestTimeout())
.setSocketTimeout(configuration.getSocketTimeout()).build();
super.RETRY_COUNT = configuration.getMaxRetry();
}
}
......@@ -58,6 +58,12 @@ public class PartitionInfoAzure {
@SerializedName("storage-account-name")
private Property storageAccountNameConfig;
@SerializedName("ingest-storage-account-key")
private Property ingestStorageAccountKeyConfig;
@SerializedName("ingest-storage-account-name")
private Property ingestStorageAccountNameConfig;
@SerializedName("sb-namespace")
private Property sbNamespaceConfig;
......@@ -230,6 +236,26 @@ public class PartitionInfoAzure {
return String.valueOf(this.getStorageAccountNameConfig().getValue());
}
/**
* @return ingestion storage account key
*/
public String getIngestStorageAccountKey() {
if (this.getIngestStorageAccountKeyConfig().isSensitive()) {
return getSecret(this.getIngestStorageAccountKeyConfig());
}
return String.valueOf(this.getIngestStorageAccountKeyConfig().getValue());
}
/**
* @return ingestion storage account name
*/
public String getIngestStorageAccountName() {
if (this.getIngestStorageAccountNameConfig().isSensitive()) {
return getSecret(this.getIngestStorageAccountNameConfig());
}
return String.valueOf(this.getIngestStorageAccountNameConfig().getValue());
}
/**
* @return partition event grid topic endpoint
*/
......
......@@ -71,7 +71,7 @@ public class PartitionServiceEventGridClient {
Validators.checkNotNullAndNotEmpty(partitionId, "partitionId");
Validators.checkNotNullAndNotEmpty(topicName, "topicName");
Map<String, EventGridTopicPartitionInfoAzure> eventGridTopicPartitionInfoAzure = getAllEventGridTopicsInPartition(partitionId);
Map<String, EventGridTopicPartitionInfoAzure> eventGridTopicPartitionInfoAzure = getAllRelevantEventGridTopicsInPartition(partitionId, topicName);
if (!eventGridTopicPartitionInfoAzure.containsKey(topicName)) {
throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Invalid EventGrid Partition configuration for the partition " + partitionId, "Please refer to wiki here <>");
......@@ -80,20 +80,21 @@ public class PartitionServiceEventGridClient {
}
/**
* Get partition info.
* Get relevant event grid topics.
*
* @param partitionId Partition Id
* @return Partition info
* @param topicName Topic Name
* @return Event Grid Topics
* @throws AppException Exception thrown by {@link IPartitionFactory}
* @throws PartitionException Exception thrown by {@link IPartitionFactory}
*/
Map<String, EventGridTopicPartitionInfoAzure> getAllEventGridTopicsInPartition(final String partitionId) throws AppException, PartitionException {
Map<String, EventGridTopicPartitionInfoAzure> getAllRelevantEventGridTopicsInPartition(final String partitionId, final String topicName) throws AppException, PartitionException {
PartitionInfo partitionInfo = getPartitionInfo(partitionId);
Map<String, Property> propertyMap = partitionInfo.getProperties();
Map<String, EventGridTopicPartitionInfoAzure> topics = new HashMap<>();
for (Map.Entry<String, Property> property : propertyMap.entrySet()) {
if (isEventGridProperty(property)) {
if (isEventGridProperty(property) && property.getKey().contains(topicName)) {
StringTokenizer stringTokenizer = new StringTokenizer(property.getKey(), "-");
if (stringTokenizer.countTokens() == 2) {
addEventGridTopicName(topics, property, stringTokenizer);
......
# Copyright © Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This file is place holder for all the common configuration across services. The properties
# which are common across all services can be placed here.
#
# In order to use the properties in this file (i.e., common.properties) in any service, configure
# the environment variable SPRING_CONFIG_NAME = common,application in that service. Doing this will
# configure the service to use properties from this file along with application.properties of that service.
#Enable tomcat metrics
server.tomcat.mbeanregistry.enabled=true
#Tomcat threads configuration
server.tomcat.min-spare-threads=100
\ No newline at end of file
......@@ -24,6 +24,7 @@ import com.azure.storage.blob.sas.BlobContainerSasPermission;
import com.azure.storage.blob.sas.BlobSasPermission;
import com.azure.storage.blob.sas.BlobServiceSasSignatureValues;
import com.azure.storage.blob.specialized.BlockBlobClient;
import org.apache.catalina.User;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
......@@ -423,6 +424,45 @@ public class BlobStoreTest {
assertEquals(containerPreSignedUrl, obtainedPreSignedUrl);
}
@Test
public void generatePreSignedUrlWithUserDelegationSas_NullPreSignedTokenObtained() {
int expiryDays = 1;
OffsetDateTime startTime = OffsetDateTime.now();
OffsetDateTime expiryTime = OffsetDateTime.now().plusDays(expiryDays);
BlobContainerSasPermission blobContainerSasPermission = (new BlobContainerSasPermission()).setReadPermission(true).setCreatePermission(true);
String obtainedPreSignedUrl = blobStore.generatePreSignedUrlWithUserDelegationSas(PARTITION_ID, STORAGE_CONTAINER_NAME, startTime, expiryTime, blobContainerSasPermission);
assertEquals("null?null", obtainedPreSignedUrl);
}
@Test
public void generatePreSignedURLlWithUserDelegationSas_whenContainerPreSignedUrl_thenReturnsValidSasToken() {
UserDelegationKey userDelegationKey = mock(UserDelegationKey.class);
String containerSasToken = "containerSasToken";
String containerUrl = "containerUrl";
String containerPreSignedUrl = containerUrl + "?" + containerSasToken;
doReturn(userDelegationKey).when(blobServiceClient).getUserDelegationKey(any(OffsetDateTime.class), any(OffsetDateTime.class));
doReturn(containerUrl).when(blobContainerClient).getBlobContainerUrl();
doReturn(containerSasToken).when(blobContainerClient).generateUserDelegationSas(any(BlobServiceSasSignatureValues.class), any(UserDelegationKey.class));
int expiryDays = 1;
OffsetDateTime startTime = OffsetDateTime.now();
OffsetDateTime expiryTime = OffsetDateTime.now().plusDays(expiryDays);
BlobContainerSasPermission blobContainerSasPermission = (new BlobContainerSasPermission()).setReadPermission(true).setCreatePermission(true);
String obtainedPreSignedUrl = blobStore.generatePreSignedUrlWithUserDelegationSas(PARTITION_ID, STORAGE_CONTAINER_NAME, startTime, expiryTime, blobContainerSasPermission);
ArgumentCaptor<BlobServiceSasSignatureValues> blobServiceSasSignatureValuesArgumentCaptor = ArgumentCaptor.forClass(BlobServiceSasSignatureValues.class);
ArgumentCaptor<UserDelegationKey> userDelegationKeyArgumentCaptor = ArgumentCaptor.forClass(UserDelegationKey.class);
verify(blobContainerClient).generateUserDelegationSas(blobServiceSasSignatureValuesArgumentCaptor.capture(), userDelegationKeyArgumentCaptor.capture());
assertEquals(blobContainerSasPermission.toString(), blobServiceSasSignatureValuesArgumentCaptor.getValue().getPermissions());
assertEquals(userDelegationKey, userDelegationKeyArgumentCaptor.getValue());
assertEquals(startTime, blobServiceSasSignatureValuesArgumentCaptor.getValue().getStartTime());
assertEquals(expiryTime, blobServiceSasSignatureValuesArgumentCaptor.getValue().getExpiryTime());
assertEquals(containerPreSignedUrl, obtainedPreSignedUrl);
}
private BlobStorageException mockStorageException(BlobErrorCode errorCode) {
BlobStorageException mockException = mock(BlobStorageException.class);
lenient().when(mockException.getErrorCode()).thenReturn(errorCode);
......
......@@ -66,7 +66,6 @@ public class PartitionServiceEventGridClientTest {
final String eventGridTopicName2 = "testEventGridTopicName2";
final String eventGridTopicAccessKey2 = "testEventGridTopicAccessKey2";
final String topicId1 = "recordstopic";
final String topicId2 = "testtopic";
Map<String, Property> properties = new HashMap<>();
properties.put("id", Property.builder().value(PARTITION_ID).build());
......@@ -85,18 +84,15 @@ public class PartitionServiceEventGridClientTest {
// Act
Map<String, EventGridTopicPartitionInfoAzure> eventGridTopicPartitionInfoAzureMap =
partitionServiceClientSpy.getAllEventGridTopicsInPartition("tenant1");
partitionServiceClientSpy.getAllRelevantEventGridTopicsInPartition("tenant1", "recordstopic");
// Assert
assertEquals(eventGridTopicPartitionInfoAzureMap.size(), 2);
assertEquals(eventGridTopicPartitionInfoAzureMap.size(), 1);
assertTrue(eventGridTopicPartitionInfoAzureMap.containsKey(topicId1));
assertTrue(eventGridTopicPartitionInfoAzureMap.containsKey(topicId2));
// Validate that the EventGridTopicPartitionInfo is mapped correctly.
assertEquals(eventGridTopicPartitionInfoAzureMap.get(topicId1).getTopicName(), eventGridTopicName1);
assertEquals(eventGridTopicPartitionInfoAzureMap.get(topicId1).getTopicAccessKey(), eventGridTopicAccessKey1);
assertEquals(eventGridTopicPartitionInfoAzureMap.get(topicId2).getTopicName(), eventGridTopicName2);
assertEquals(eventGridTopicPartitionInfoAzureMap.get(topicId2).getTopicAccessKey(), eventGridTopicAccessKey2);
}
@Test
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment