Commit d81f8695 authored by Aman Verma's avatar Aman Verma Committed by Kishore Battula
Browse files

adding blobStore class to facilitate interaction with azure blob storage

parent 42da376b
......@@ -21,7 +21,7 @@
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-azure</artifactId>
<packaging>jar</packaging>
<version>0.0.13</version>
<version>0.0.14</version>
<name>core-lib-azure</name>
<properties>
......
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.azure.blobstorage;
import com.azure.storage.blob.BlobContainerClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
/**
* Implementation for IBlobContainerClientFactory.
*/
@Component
@Lazy
public class BlobContainerClientFactoryImpl implements IBlobContainerClientFactory {
@Lazy
@Autowired
private BlobContainerClient blobContainerClient;
/**
*
* @param dataPartitionId Data partition id
* @return the blob container client instance.
*/
@Override
public BlobContainerClient getClient(final String dataPartitionId) {
return blobContainerClient;
}
}
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.azure.blobstorage;
import com.azure.storage.blob.BlobContainerClient;
import com.azure.storage.blob.models.BlobErrorCode;
import com.azure.storage.blob.models.BlobStorageException;
import com.azure.storage.blob.specialized.BlockBlobClient;
import org.opengroup.osdu.core.common.logging.ILogger;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
import java.io.ByteArrayOutputStream;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
import java.util.Collections;;
/**
* A simpler interface to interact with Azure blob storage.
* Usage examples:
* <pre>
* {@code
* @Autowired
* private BlobStore blobStore;
*
* String readFromBlobExample()
* {
* String content = blobStorage.readFromBlob("dataPartitionId", "filePath");
* if (content != null)
* return content;
* }
*
* void writeToBlobExample()
* {
* blobStorage.writeToBlob("dataPartitionId", "filePath", "content");
* }
*
* void deleteFromBlobExample()
* {
* Boolean success = blobStorage.deleteFromBlob("dataPartitionId", "filePath");
* }
* }
* </pre>
*/
@Component
@Lazy
public class BlobStore {
@Autowired
private IBlobContainerClientFactory blobContainerClientFactory;
@Autowired
private ILogger logger;
private static final String LOG_PREFIX = "azure-core-lib";
/**
*
* @param filePath Path of file to be read.
* @param dataPartitionId Data partition id
* @return the content of file with provided file path.
*/
public String readFromBlob(final String dataPartitionId, final String filePath) {
BlobContainerClient blobContainerClient = getBlobContainerClient(dataPartitionId);
BlockBlobClient blockBlobClient = blobContainerClient.getBlobClient(filePath).getBlockBlobClient();
try (ByteArrayOutputStream downloadStream = new ByteArrayOutputStream()) {
blockBlobClient.download(downloadStream);
return downloadStream.toString(StandardCharsets.UTF_8.name());
} catch (BlobStorageException ex) {
if (ex.getErrorCode().equals(BlobErrorCode.BLOB_NOT_FOUND)) {
String errorMessage = "Specified blob was not found";
logger.warning(LOG_PREFIX, errorMessage, Collections.<String, String>emptyMap());
throw new AppException(404, errorMessage, ex.getMessage(), ex);
} else {
String errorMessage = "Failed to read specified blob";
logger.warning(LOG_PREFIX, errorMessage, Collections.<String, String>emptyMap());
throw new AppException(500, errorMessage, ex.getMessage(), ex);
}
} catch (UnsupportedEncodingException ex) {
String errorMessage = String.format("Encoding was not correct for item with name=%s", filePath);
logger.warning(LOG_PREFIX, errorMessage, Collections.<String, String>emptyMap());
throw new AppException(400, errorMessage, ex.getMessage(), ex);
} catch (IOException ex) {
String errorMessage = String.format("Malformed document for item with name=%s", filePath);
logger.warning(LOG_PREFIX, errorMessage, Collections.<String, String>emptyMap());
throw new AppException(500, errorMessage, ex.getMessage(), ex);
}
}
/**
*
* @param filePath Path of file to be deleted.
* @param dataPartitionId Data partition id
* @return boolean indicating whether the deletion of given file was successful or not.
*/
public boolean deleteFromBlob(final String dataPartitionId, final String filePath) {
BlobContainerClient blobContainerClient = getBlobContainerClient(dataPartitionId);
BlockBlobClient blockBlobClient = blobContainerClient.getBlobClient(filePath).getBlockBlobClient();
try {
blockBlobClient.delete();
return true;
} catch (BlobStorageException ex) {
if (ex.getErrorCode().equals(BlobErrorCode.BLOB_NOT_FOUND)) {
String errorMessage = "Specified blob was not found";
logger.warning(LOG_PREFIX, errorMessage, Collections.<String, String>emptyMap());
throw new AppException(404, errorMessage, ex.getMessage(), ex);
} else {
String errorMessage = "Failed to delete blob";
logger.warning(LOG_PREFIX, errorMessage, Collections.<String, String>emptyMap());
throw new AppException(500, errorMessage, ex.getMessage(), ex);
}
}
}
/**
*
* @param filePath Path of file to be written at.
* @param content Content to be written in the file.
* @param dataPartitionId Data partition id
*/
public void writeToBlob(final String dataPartitionId,
final String filePath,
final String content) {
byte[] bytes = content.getBytes(StandardCharsets.UTF_8);
int bytesSize = bytes.length;
BlobContainerClient blobContainerClient = getBlobContainerClient(dataPartitionId);
BlockBlobClient blockBlobClient = blobContainerClient.getBlobClient(filePath).getBlockBlobClient();
try (ByteArrayInputStream dataStream = new ByteArrayInputStream(bytes)) {
blockBlobClient.upload(dataStream, bytesSize, true);
} catch (BlobStorageException ex) {
String errorMessage = "Failed to upload file content.";
logger.warning(LOG_PREFIX, errorMessage, Collections.<String, String>emptyMap());
throw new AppException(500, errorMessage, ex.getMessage(), ex);
} catch (IOException ex) {
String errorMessage = String.format("Malformed document for item with name=%s", filePath);
logger.warning(LOG_PREFIX, errorMessage, Collections.<String, String>emptyMap());
throw new AppException(500, errorMessage, ex.getMessage(), ex);
}
}
/**
*
* @param dataPartitionId Data partition id
* @return blob container client corresponding to the dataPartitionId.
*/
private BlobContainerClient getBlobContainerClient(final String dataPartitionId) {
try {
return blobContainerClientFactory.getClient(dataPartitionId);
} catch (Exception ex) {
String errorMessage = "Error creating creating blob container client.";
logger.warning(LOG_PREFIX, errorMessage, Collections.<String, String>emptyMap());
throw new AppException(500, errorMessage, ex.getMessage(), ex);
}
}
}
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.azure.blobstorage;
import com.azure.storage.blob.BlobContainerClient;
/**
* Interface for BlobContainer client factory to return appropriate
* blobContainerClient based on the data partition id.
*/
public interface IBlobContainerClientFactory {
/**
*
* @param dataPartitionId Data partition id
* @return blobContainerClient for given data partition id.
*/
BlobContainerClient getClient(String dataPartitionId);
}
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.azure.blobstorage;
import com.azure.storage.blob.BlobClient;
import com.azure.storage.blob.BlobContainerClient;
import com.azure.storage.blob.models.BlobErrorCode;
import com.azure.storage.blob.models.BlobStorageException;
import com.azure.storage.blob.models.BlockBlobItem;
import com.azure.storage.blob.specialized.BlockBlobClient;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.opengroup.osdu.core.common.logging.ILogger;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import java.io.ByteArrayOutputStream;
import java.util.HashMap;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.*;
import static org.mockito.MockitoAnnotations.initMocks;
@ExtendWith(MockitoExtension.class)
public class BlobStoreTest {
private static final String PARTITION_ID = "dataPartitionId";
private static final String FILE_PATH = "filePath";
private static final String CONTENT = "hello world";
@InjectMocks
BlobStore blobStore;
@Mock
IBlobContainerClientFactory blobContainerClientFactory;
@Mock
BlobContainerClient blobContainerClient;
@Mock
BlobClient blobClient;
@Mock
BlockBlobClient blockBlobClient;
@Mock
BlockBlobItem blockBlobItem;
@Mock
ILogger logger;
@BeforeEach
void init() {
initMocks(this);
lenient().doReturn(blobContainerClient).when(blobContainerClientFactory).getClient(PARTITION_ID);
lenient().doReturn(blobClient).when(blobContainerClient).getBlobClient(FILE_PATH);
lenient().doReturn(blockBlobClient).when(blobClient).getBlockBlobClient();
lenient().doNothing().when(logger).warning(eq("azure-core-lib"), any(), anyMap());
}
@Test
public void readFromBlob_ErrorCreatingBlobContainerClient()
{
doThrow(AppException.class).when(blobContainerClientFactory).getClient(eq(PARTITION_ID));
try {
String content = blobStore.readFromBlob(PARTITION_ID, FILE_PATH);
} catch (AppException ex) {
assertEquals(500, ex.getError().getCode());
} catch (Exception ex) {
fail("should not get different error code");
}
}
@Test
public void readFromBlob_Success()
{
String content = blobStore.readFromBlob(PARTITION_ID, FILE_PATH);
ArgumentCaptor<ByteArrayOutputStream> outputStream = ArgumentCaptor.forClass(ByteArrayOutputStream.class);
// validate that the download method is being invoked appropriately.
verify(blockBlobClient).download(outputStream.capture());
}
@Test
public void readFromBlob_BlobNotFound()
{
BlobStorageException exception = mockStorageException(BlobErrorCode.BLOB_NOT_FOUND);
doThrow(exception).when(blockBlobClient).download(any());
try {
String content = blobStore.readFromBlob(PARTITION_ID, FILE_PATH);
} catch (AppException ex) {
assertEquals(404, ex.getError().getCode());
} catch (Exception ex) {
fail("should not get different error code");
}
}
@Test
public void readFromBlob_InternalError()
{
BlobStorageException exception = mockStorageException(BlobErrorCode.INTERNAL_ERROR);
doThrow(exception).when(blockBlobClient).download(any());
try {
String content = blobStore.readFromBlob(PARTITION_ID, FILE_PATH);
} catch (AppException ex) {
assertEquals(500, ex.getError().getCode());
} catch (Exception ex) {
fail("should not get different error code");
}
}
@Test
public void deleteFromBlob_ErrorCreatingBlobContainerClient()
{
doThrow(AppException.class).when(blobContainerClientFactory).getClient(eq(PARTITION_ID));
try {
blobStore.deleteFromBlob(PARTITION_ID, FILE_PATH);
} catch (AppException ex) {
assertEquals(500, ex.getError().getCode());
} catch (Exception ex) {
fail("should not get different error code");
}
}
@Test
public void deleteFromBlob_BlobNotFound()
{
BlobStorageException exception = mockStorageException(BlobErrorCode.BLOB_NOT_FOUND);
doThrow(exception).when(blockBlobClient).delete();
try {
blobStore.deleteFromBlob(PARTITION_ID, FILE_PATH);
} catch (AppException ex) {
assertEquals(404, ex.getError().getCode());
} catch (Exception ex) {
fail("should not get different error code");
}
}
@Test
public void deleteFromBlob_InternalError()
{
BlobStorageException exception = mockStorageException(BlobErrorCode.INTERNAL_ERROR);
doThrow(exception).when(blockBlobClient).delete();
try {
blobStore.deleteFromBlob(PARTITION_ID, FILE_PATH);
} catch (AppException ex) {
assertEquals(500, ex.getError().getCode());
} catch (Exception ex) {
fail("should not get different error code");
}
}
@Test
public void deleteFromBlob_Success()
{
doNothing().when(blockBlobClient).delete();
try {
blobStore.deleteFromBlob(PARTITION_ID, FILE_PATH);
} catch (Exception ex) {
fail("should not get any exception.");
}
}
@Test
public void writeToBlob_ErrorCreatingBlobContainerClient()
{
doThrow(AppException.class).when(blobContainerClientFactory).getClient(eq(PARTITION_ID));
try {
blobStore.writeToBlob(PARTITION_ID, FILE_PATH, CONTENT);
} catch (AppException ex) {
assertEquals(500, ex.getError().getCode());
} catch (Exception ex) {
fail("should not get different error code");
}
}
@Test
public void writeToBlob_InternalError()
{
BlobStorageException exception = mockStorageException(BlobErrorCode.INTERNAL_ERROR);
doThrow(exception).when(blockBlobClient).upload(any(), anyLong(), eq(true));
try {
blobStore.writeToBlob(PARTITION_ID, FILE_PATH, CONTENT);
} catch (AppException ex) {
assertEquals(500, ex.getError().getCode());
} catch (Exception ex) {
fail("should not get different error code");
}
}
@Test
public void writeToBlob_Success()
{
doReturn(blockBlobItem).when(blockBlobClient).upload(any(), anyLong(), eq(true));
try {
blobStore.writeToBlob(PARTITION_ID, FILE_PATH, CONTENT);
} catch (Exception ex) {
fail("should not get any exception.");
}
}
private BlobStorageException mockStorageException(BlobErrorCode errorCode) {
BlobStorageException mockException = mock(BlobStorageException.class);
lenient().when(mockException.getErrorCode()).thenReturn(errorCode);
return mockException;
}
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment