Commit c92d7c24 authored by Ronak Sakhuja's avatar Ronak Sakhuja
Browse files

Merge readmee

parents fee409d2 9c273dcc
Pipeline #46358 failed with stage
in 1 minute and 39 seconds
......@@ -15,13 +15,13 @@ limitations under the License.
# Introduction
This repository houses code that is used across the Microsoft Azure hosted OSDU and OpenDES platforms. The intention of this repository is to minimize code duplication for common scenarios such as dependency configuration for services in Azure (KeyVault, Cosmos, Storage and others).
# Pre-requisites
You need
1. Maven 3.6.x
2. Java 1.8
1. Maven 3.6.x
2. Java 1.8
# Local Usage
......@@ -72,7 +72,7 @@ which can be fixed by excluding some of the logger dependencies.
Note: Below are reference PRs for exclusion and might change from service to service
Refer this [MR](https://community.opengroup.org/osdu/platform/security-and-compliance/entitlements-azure/-/merge_requests/13) as reference on how to exclude dependencies along with how to enable the
Refer this [MR](https://community.opengroup.org/osdu/platform/security-and-compliance/entitlements-azure/-/merge_requests/13) as reference on how to exclude dependencies along with how to enable the
Enabled transaction logger and slf4jlogger
## Environment variables to be added in application.properties to consume the TenantFactoryImpl
......@@ -87,3 +87,11 @@ Enabled transaction logger and slf4jlogger
| --- | --- | --- |
| `azure.blobStore.required` | `true` | - |
| `azure.storage.account-name` | ex `testStorage` | storage account name |
# Default retry and timeout values for service-to-service communication
| name | default value |
| --- | --- |
| `maxRetry` | `3` |
| `connectTimeout` | `60000` |
| `requestTimeout` | `60000` |
| `socketTimeout` | `60000` |
......@@ -21,6 +21,7 @@ import com.azure.storage.blob.models.BlobCopyInfo;
import com.azure.storage.blob.models.BlobErrorCode;
import com.azure.storage.blob.models.BlobStorageException;
import com.azure.storage.blob.models.CopyStatusType;
import com.azure.storage.blob.models.UserDelegationKey;
import com.azure.storage.blob.sas.BlobContainerSasPermission;
import com.azure.storage.blob.sas.BlobSasPermission;
import com.azure.storage.blob.sas.BlobServiceSasSignatureValues;
......@@ -253,6 +254,8 @@ public class BlobStore {
}
/**
* This method is used to generate pre-signed url for file (blob).
* NOTE: Using the below method will require BlobServiceClient to be instantiated using StorageSharedKeyCredential
* @param dataPartitionId Data partition id
* @param filePath Path of file (blob) for which SAS token needs to be generated
* @param containerName Name of the storage container
......@@ -267,8 +270,8 @@ public class BlobStore {
}
/**
* Generates pre-signed url to a blob container.
*
* This method is used to generate pre-signed url for blob container.
* NOTE: Using the below method will require BlobServiceClient to be instantiated using StorageSharedKeyCredential
* @param dataPartitionId data partition id
* @param containerName Name of the storage container
* @param expiryTime Time after which the token expires
......@@ -280,6 +283,30 @@ public class BlobStore {
return blobContainerClient.getBlobContainerUrl() + "?" + generateSASToken(blobContainerClient, expiryTime, permissions);
}
/**
* Generates pre-signed url to a blob container using the user delegation key.
*
* @param dataPartitionId data partition id
* @param containerName Name of the storage container
* @param startTime Time after which the token is activated (null in case of instant activation)
* @param expiryTime Time after which the token expires
* @param permissions permissions for the given container
* @return Generates pre-signed url for a given container
*/
public String generatePreSignedUrlWithUserDelegationSas(final String dataPartitionId, final String containerName, final OffsetDateTime startTime, final OffsetDateTime expiryTime, final BlobContainerSasPermission permissions) {
BlobContainerClient blobContainerClient = getBlobContainerClient(dataPartitionId, containerName);
BlobServiceClient blobServiceClient = blobServiceClientFactory.getBlobServiceClient(dataPartitionId);
UserDelegationKey userDelegationKey = blobServiceClient.getUserDelegationKey(startTime, expiryTime);
BlobServiceSasSignatureValues blobServiceSasSignatureValues = new BlobServiceSasSignatureValues(expiryTime, permissions).setStartTime(startTime);
final long start = System.currentTimeMillis();
String sasToken = blobContainerClient.generateUserDelegationSas(blobServiceSasSignatureValues, userDelegationKey);
final long timeTaken = System.currentTimeMillis() - start;
logDependency("GENERATE_PRESIGNED_URL_USER_DELEGATION_SAS", blobContainerClient.getBlobContainerName(), blobContainerClient.getBlobContainerUrl(), timeTaken, String.valueOf(HttpStatus.SC_OK), true);
return blobContainerClient.getBlobContainerUrl() + "?" + sasToken;
}
/**
* Method is used to copy a file specified at Source URL to the provided destination.
*
......
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License
package org.opengroup.osdu.azure.di;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
/**
* Configuration bean for setting up retry and timeouts variables.
*/
@Data
@Component
@ConfigurationProperties(prefix = "azure.service.retry.config")
public class RetryAndTimeoutConfiguration {
private int maxRetry = 3;
private int connectTimeout = 60000;
private int requestTimeout = 60000;
private int socketTimeout = 60000;
}
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License
package org.opengroup.osdu.azure.entitlements;
import lombok.Data;
import org.opengroup.osdu.core.common.entitlements.EntitlementsAPIConfig;
import org.springframework.beans.factory.config.AbstractFactoryBean;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
/**
* Creates bean of EntitlementsAPIConfig.
*/
@Data
@Component
@ConfigurationProperties
public class EntilementsAPIConfigBean extends AbstractFactoryBean<EntitlementsAPIConfig> {
private String authorizeAPI;
private String authorizeAPIKey;
/**
* Abstract method of AbstractBeanFactory<T>.
*
* @return class type
*/
@Override
public Class<?> getObjectType() {
return EntitlementsAPIConfig.class;
}
/**
* Abstract method of AbstractBeanFactory<T> type.
*
* @return EntitlementsAPIConfig
* @throws Exception
*/
@Override
protected EntitlementsAPIConfig createInstance() throws Exception {
return EntitlementsAPIConfig
.builder()
.rootUrl(authorizeAPI)
.apiKey(authorizeAPIKey)
.build();
}
}
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License
package org.opengroup.osdu.azure.entitlements;
import org.opengroup.osdu.core.common.entitlements.EntitlementsAPIConfig;
import org.opengroup.osdu.core.common.entitlements.EntitlementsService;
import org.opengroup.osdu.core.common.entitlements.IEntitlementsFactory;
import org.opengroup.osdu.core.common.entitlements.IEntitlementsService;
import org.opengroup.osdu.core.common.http.IHttpClient;
import org.opengroup.osdu.core.common.http.json.HttpResponseBodyMapper;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
import java.util.Objects;
/**
* Implements IEntitlementsFactory.
*/
@Component
@Primary
@ConditionalOnProperty(value = "azure.entitlements.factory.enabled", havingValue = "true", matchIfMissing = true)
public class EntitlementsFactoryAzure implements IEntitlementsFactory {
private final EntitlementsAPIConfig config;
private final HttpResponseBodyMapper mapper;
private final IHttpClient client;
/**
* Constructor Injection for above 3 fields.
*
* @param entitlementsConfig EntitlementsAPIConfig
* @param httpMapper HttpResponseBodyMapper
* @param httpClient IHttpClient
*/
@Autowired
public EntitlementsFactoryAzure(final EntitlementsAPIConfig entitlementsConfig, final HttpResponseBodyMapper httpMapper, final IHttpClient httpClient) {
this.config = entitlementsConfig;
this.mapper = httpMapper;
this.client = httpClient;
}
/**
* returns instance of EntitlementsService.
*
* @param headers DpsHeaders
* @return IEntitlementsService
*/
@Override
public IEntitlementsService create(final DpsHeaders headers) {
Objects.requireNonNull(headers, "headers cannot be null");
return new EntitlementsService(this.config,
this.client,
headers,
this.mapper);
}
}
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License
package org.opengroup.osdu.azure.httpconfig;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.http.FetchServiceHttpRequest;
import org.opengroup.osdu.core.common.http.HttpRequest;
import org.opengroup.osdu.core.common.http.IHttpClient;
import org.opengroup.osdu.core.common.http.HttpResponse;
import org.opengroup.osdu.core.common.http.UrlFetchServiceImpl;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
import java.net.URISyntaxException;
/**
* Extends URlFetchService and implements IHttpClient to send requests.
*/
@Primary
@Component
public class HttpClientAzure extends UrlFetchServiceImpl implements IHttpClient {
/**
* calls urlfetchservice's send request.
*
* @param httpRequest made by user class
* @return HttpResponse
*/
@Override
public HttpResponse send(final HttpRequest httpRequest) {
org.opengroup.osdu.core.common.model.http.HttpResponse response = null;
try {
response = super.sendRequest(FetchServiceHttpRequest.builder()
.body(httpRequest.getBody())
.httpMethod(httpRequest.getHttpMethod())
.queryParams(httpRequest.getQueryParams())
.url(httpRequest.getUrl())
.headers(httpRequest.getHeaders())
.build());
} catch (URISyntaxException e) {
throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getReason(), "URI Syntax is not correct", e);
}
HttpResponse httpResponse = new HttpResponse();
httpResponse.setBody(response.getBody());
httpResponse.setResponseCode(response.getResponseCode());
httpResponse.setContentType(response.getContentType());
httpResponse.setRequest(httpRequest);
return httpResponse;
}
}
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License
package org.opengroup.osdu.azure.httpconfig;
import org.apache.http.client.config.RequestConfig;
import org.opengroup.osdu.azure.di.RetryAndTimeoutConfiguration;
import org.opengroup.osdu.core.common.http.HttpClientHandler;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
/**
* Extends HttpClientHandler.
*/
@Primary
@Component
public class HttpClientHandlerAzure extends HttpClientHandler {
private RetryAndTimeoutConfiguration configuration;
/**
* Constuctor injection for RetryAndTimeoutConfiguration.
*
* @param retryAndTimeoutConfiguration of type RetryAndTimeoutConfiguration
*/
@Autowired
public HttpClientHandlerAzure(final RetryAndTimeoutConfiguration retryAndTimeoutConfiguration) {
this.configuration = retryAndTimeoutConfiguration;
super.REQUEST_CONFIG = RequestConfig.custom()
.setConnectTimeout(configuration.getConnectTimeout())
.setConnectionRequestTimeout(configuration.getRequestTimeout())
.setSocketTimeout(configuration.getSocketTimeout()).build();
super.RETRY_COUNT = configuration.getMaxRetry();
}
}
......@@ -58,6 +58,12 @@ public class PartitionInfoAzure {
@SerializedName("storage-account-name")
private Property storageAccountNameConfig;
@SerializedName("ingest-storage-account-key")
private Property ingestStorageAccountKeyConfig;
@SerializedName("ingest-storage-account-name")
private Property ingestStorageAccountNameConfig;
@SerializedName("sb-namespace")
private Property sbNamespaceConfig;
......@@ -230,6 +236,26 @@ public class PartitionInfoAzure {
return String.valueOf(this.getStorageAccountNameConfig().getValue());
}
/**
* @return ingestion storage account key
*/
public String getIngestStorageAccountKey() {
if (this.getIngestStorageAccountKeyConfig().isSensitive()) {
return getSecret(this.getIngestStorageAccountKeyConfig());
}
return String.valueOf(this.getIngestStorageAccountKeyConfig().getValue());
}
/**
* @return ingestion storage account name
*/
public String getIngestStorageAccountName() {
if (this.getIngestStorageAccountNameConfig().isSensitive()) {
return getSecret(this.getIngestStorageAccountNameConfig());
}
return String.valueOf(this.getIngestStorageAccountNameConfig().getValue());
}
/**
* @return partition event grid topic endpoint
*/
......
......@@ -24,6 +24,7 @@ import com.azure.storage.blob.sas.BlobContainerSasPermission;
import com.azure.storage.blob.sas.BlobSasPermission;
import com.azure.storage.blob.sas.BlobServiceSasSignatureValues;
import com.azure.storage.blob.specialized.BlockBlobClient;
import org.apache.catalina.User;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
......@@ -423,6 +424,45 @@ public class BlobStoreTest {
assertEquals(containerPreSignedUrl, obtainedPreSignedUrl);
}
@Test
public void generatePreSignedUrlWithUserDelegationSas_NullPreSignedTokenObtained() {
int expiryDays = 1;
OffsetDateTime startTime = OffsetDateTime.now();
OffsetDateTime expiryTime = OffsetDateTime.now().plusDays(expiryDays);
BlobContainerSasPermission blobContainerSasPermission = (new BlobContainerSasPermission()).setReadPermission(true).setCreatePermission(true);
String obtainedPreSignedUrl = blobStore.generatePreSignedUrlWithUserDelegationSas(PARTITION_ID, STORAGE_CONTAINER_NAME, startTime, expiryTime, blobContainerSasPermission);
assertEquals("null?null", obtainedPreSignedUrl);
}
@Test
public void generatePreSignedURLlWithUserDelegationSas_whenContainerPreSignedUrl_thenReturnsValidSasToken() {
UserDelegationKey userDelegationKey = mock(UserDelegationKey.class);
String containerSasToken = "containerSasToken";
String containerUrl = "containerUrl";
String containerPreSignedUrl = containerUrl + "?" + containerSasToken;
doReturn(userDelegationKey).when(blobServiceClient).getUserDelegationKey(any(OffsetDateTime.class), any(OffsetDateTime.class));
doReturn(containerUrl).when(blobContainerClient).getBlobContainerUrl();
doReturn(containerSasToken).when(blobContainerClient).generateUserDelegationSas(any(BlobServiceSasSignatureValues.class), any(UserDelegationKey.class));
int expiryDays = 1;
OffsetDateTime startTime = OffsetDateTime.now();
OffsetDateTime expiryTime = OffsetDateTime.now().plusDays(expiryDays);
BlobContainerSasPermission blobContainerSasPermission = (new BlobContainerSasPermission()).setReadPermission(true).setCreatePermission(true);
String obtainedPreSignedUrl = blobStore.generatePreSignedUrlWithUserDelegationSas(PARTITION_ID, STORAGE_CONTAINER_NAME, startTime, expiryTime, blobContainerSasPermission);
ArgumentCaptor<BlobServiceSasSignatureValues> blobServiceSasSignatureValuesArgumentCaptor = ArgumentCaptor.forClass(BlobServiceSasSignatureValues.class);
ArgumentCaptor<UserDelegationKey> userDelegationKeyArgumentCaptor = ArgumentCaptor.forClass(UserDelegationKey.class);
verify(blobContainerClient).generateUserDelegationSas(blobServiceSasSignatureValuesArgumentCaptor.capture(), userDelegationKeyArgumentCaptor.capture());
assertEquals(blobContainerSasPermission.toString(), blobServiceSasSignatureValuesArgumentCaptor.getValue().getPermissions());
assertEquals(userDelegationKey, userDelegationKeyArgumentCaptor.getValue());
assertEquals(startTime, blobServiceSasSignatureValuesArgumentCaptor.getValue().getStartTime());
assertEquals(expiryTime, blobServiceSasSignatureValuesArgumentCaptor.getValue().getExpiryTime());
assertEquals(containerPreSignedUrl, obtainedPreSignedUrl);
}
private BlobStorageException mockStorageException(BlobErrorCode errorCode) {
BlobStorageException mockException = mock(BlobStorageException.class);
lenient().when(mockException.getErrorCode()).thenReturn(errorCode);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment