Commit ed74304a authored by Aalekh Jain's avatar Aalekh Jain Committed by Kishore Battula
Browse files

Multi partition changes for ingestion-workflow

parent 10b07b14
......@@ -21,6 +21,7 @@ import com.azure.storage.blob.models.BlobCopyInfo;
import com.azure.storage.blob.models.BlobErrorCode;
import com.azure.storage.blob.models.BlobStorageException;
import com.azure.storage.blob.models.CopyStatusType;
import com.azure.storage.blob.models.UserDelegationKey;
import com.azure.storage.blob.sas.BlobContainerSasPermission;
import com.azure.storage.blob.sas.BlobSasPermission;
import com.azure.storage.blob.sas.BlobServiceSasSignatureValues;
......@@ -253,6 +254,8 @@ public class BlobStore {
}
/**
* This method is used to generate pre-signed url for file (blob).
* NOTE: Using the below method will require BlobServiceClient to be instantiated using StorageSharedKeyCredential
* @param dataPartitionId Data partition id
* @param filePath Path of file (blob) for which SAS token needs to be generated
* @param containerName Name of the storage container
......@@ -267,8 +270,8 @@ public class BlobStore {
}
/**
* Generates pre-signed url to a blob container.
*
* This method is used to generate pre-signed url for blob container.
* NOTE: Using the below method will require BlobServiceClient to be instantiated using StorageSharedKeyCredential
* @param dataPartitionId data partition id
* @param containerName Name of the storage container
* @param expiryTime Time after which the token expires
......@@ -280,6 +283,30 @@ public class BlobStore {
return blobContainerClient.getBlobContainerUrl() + "?" + generateSASToken(blobContainerClient, expiryTime, permissions);
}
/**
* Generates pre-signed url to a blob container using the user delegation key.
*
* @param dataPartitionId data partition id
* @param containerName Name of the storage container
* @param startTime Time after which the token is activated (null in case of instant activation)
* @param expiryTime Time after which the token expires
* @param permissions permissions for the given container
* @return Generates pre-signed url for a given container
*/
public String generatePreSignedUrlWithUserDelegationSas(final String dataPartitionId, final String containerName, final OffsetDateTime startTime, final OffsetDateTime expiryTime, final BlobContainerSasPermission permissions) {
BlobContainerClient blobContainerClient = getBlobContainerClient(dataPartitionId, containerName);
BlobServiceClient blobServiceClient = blobServiceClientFactory.getBlobServiceClient(dataPartitionId);
UserDelegationKey userDelegationKey = blobServiceClient.getUserDelegationKey(startTime, expiryTime);
BlobServiceSasSignatureValues blobServiceSasSignatureValues = new BlobServiceSasSignatureValues(expiryTime, permissions).setStartTime(startTime);
final long start = System.currentTimeMillis();
String sasToken = blobContainerClient.generateUserDelegationSas(blobServiceSasSignatureValues, userDelegationKey);
final long timeTaken = System.currentTimeMillis() - start;
logDependency("GENERATE_PRESIGNED_URL_USER_DELEGATION_SAS", blobContainerClient.getBlobContainerName(), blobContainerClient.getBlobContainerUrl(), timeTaken, String.valueOf(HttpStatus.SC_OK), true);
return blobContainerClient.getBlobContainerUrl() + "?" + sasToken;
}
/**
* Method is used to copy a file specified at Source URL to the provided destination.
*
......
......@@ -58,6 +58,12 @@ public class PartitionInfoAzure {
@SerializedName("storage-account-name")
private Property storageAccountNameConfig;
@SerializedName("ingest-storage-account-key")
private Property ingestStorageAccountKeyConfig;
@SerializedName("ingest-storage-account-name")
private Property ingestStorageAccountNameConfig;
@SerializedName("sb-namespace")
private Property sbNamespaceConfig;
......@@ -230,6 +236,26 @@ public class PartitionInfoAzure {
return String.valueOf(this.getStorageAccountNameConfig().getValue());
}
/**
* @return ingestion storage account key
*/
public String getIngestStorageAccountKey() {
if (this.getIngestStorageAccountKeyConfig().isSensitive()) {
return getSecret(this.getIngestStorageAccountKeyConfig());
}
return String.valueOf(this.getIngestStorageAccountKeyConfig().getValue());
}
/**
* @return ingestion storage account name
*/
public String getIngestStorageAccountName() {
if (this.getIngestStorageAccountNameConfig().isSensitive()) {
return getSecret(this.getIngestStorageAccountNameConfig());
}
return String.valueOf(this.getIngestStorageAccountNameConfig().getValue());
}
/**
* @return partition event grid topic endpoint
*/
......
......@@ -24,6 +24,7 @@ import com.azure.storage.blob.sas.BlobContainerSasPermission;
import com.azure.storage.blob.sas.BlobSasPermission;
import com.azure.storage.blob.sas.BlobServiceSasSignatureValues;
import com.azure.storage.blob.specialized.BlockBlobClient;
import org.apache.catalina.User;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
......@@ -423,6 +424,45 @@ public class BlobStoreTest {
assertEquals(containerPreSignedUrl, obtainedPreSignedUrl);
}
@Test
public void generatePreSignedUrlWithUserDelegationSas_NullPreSignedTokenObtained() {
int expiryDays = 1;
OffsetDateTime startTime = OffsetDateTime.now();
OffsetDateTime expiryTime = OffsetDateTime.now().plusDays(expiryDays);
BlobContainerSasPermission blobContainerSasPermission = (new BlobContainerSasPermission()).setReadPermission(true).setCreatePermission(true);
String obtainedPreSignedUrl = blobStore.generatePreSignedUrlWithUserDelegationSas(PARTITION_ID, STORAGE_CONTAINER_NAME, startTime, expiryTime, blobContainerSasPermission);
assertEquals("null?null", obtainedPreSignedUrl);
}
@Test
public void generatePreSignedURLlWithUserDelegationSas_whenContainerPreSignedUrl_thenReturnsValidSasToken() {
UserDelegationKey userDelegationKey = mock(UserDelegationKey.class);
String containerSasToken = "containerSasToken";
String containerUrl = "containerUrl";
String containerPreSignedUrl = containerUrl + "?" + containerSasToken;
doReturn(userDelegationKey).when(blobServiceClient).getUserDelegationKey(any(OffsetDateTime.class), any(OffsetDateTime.class));
doReturn(containerUrl).when(blobContainerClient).getBlobContainerUrl();
doReturn(containerSasToken).when(blobContainerClient).generateUserDelegationSas(any(BlobServiceSasSignatureValues.class), any(UserDelegationKey.class));
int expiryDays = 1;
OffsetDateTime startTime = OffsetDateTime.now();
OffsetDateTime expiryTime = OffsetDateTime.now().plusDays(expiryDays);
BlobContainerSasPermission blobContainerSasPermission = (new BlobContainerSasPermission()).setReadPermission(true).setCreatePermission(true);
String obtainedPreSignedUrl = blobStore.generatePreSignedUrlWithUserDelegationSas(PARTITION_ID, STORAGE_CONTAINER_NAME, startTime, expiryTime, blobContainerSasPermission);
ArgumentCaptor<BlobServiceSasSignatureValues> blobServiceSasSignatureValuesArgumentCaptor = ArgumentCaptor.forClass(BlobServiceSasSignatureValues.class);
ArgumentCaptor<UserDelegationKey> userDelegationKeyArgumentCaptor = ArgumentCaptor.forClass(UserDelegationKey.class);
verify(blobContainerClient).generateUserDelegationSas(blobServiceSasSignatureValuesArgumentCaptor.capture(), userDelegationKeyArgumentCaptor.capture());
assertEquals(blobContainerSasPermission.toString(), blobServiceSasSignatureValuesArgumentCaptor.getValue().getPermissions());
assertEquals(userDelegationKey, userDelegationKeyArgumentCaptor.getValue());
assertEquals(startTime, blobServiceSasSignatureValuesArgumentCaptor.getValue().getStartTime());
assertEquals(expiryTime, blobServiceSasSignatureValuesArgumentCaptor.getValue().getExpiryTime());
assertEquals(containerPreSignedUrl, obtainedPreSignedUrl);
}
private BlobStorageException mockStorageException(BlobErrorCode errorCode) {
BlobStorageException mockException = mock(BlobStorageException.class);
lenient().when(mockException.getErrorCode()).thenReturn(errorCode);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment