diff --git a/pom.xml b/pom.xml index 4e8ae7943cd21679d762d449af4de5c983f4b4bf..ee95754734ddd3a0fe79b46f2116a974ecdb859d 100644 --- a/pom.xml +++ b/pom.xml @@ -21,7 +21,7 @@ org.opengroup.osdu core-lib-azure jar - 0.0.13 + 0.0.14 core-lib-azure diff --git a/src/main/java/org/opengroup/osdu/azure/blobstorage/BlobContainerClientFactoryImpl.java b/src/main/java/org/opengroup/osdu/azure/blobstorage/BlobContainerClientFactoryImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..81e6c93c3c8f62f9e99251e38fe9c0a430a61151 --- /dev/null +++ b/src/main/java/org/opengroup/osdu/azure/blobstorage/BlobContainerClientFactoryImpl.java @@ -0,0 +1,42 @@ +// Copyright © Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.opengroup.osdu.azure.blobstorage; + +import com.azure.storage.blob.BlobContainerClient; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; + +/** + * Implementation for IBlobContainerClientFactory. + */ +@Component +@Lazy +public class BlobContainerClientFactoryImpl implements IBlobContainerClientFactory { + + @Lazy + @Autowired + private BlobContainerClient blobContainerClient; + + /** + * + * @param dataPartitionId Data partition id + * @return the blob container client instance. + */ + @Override + public BlobContainerClient getClient(final String dataPartitionId) { + return blobContainerClient; + } +} diff --git a/src/main/java/org/opengroup/osdu/azure/blobstorage/BlobStore.java b/src/main/java/org/opengroup/osdu/azure/blobstorage/BlobStore.java new file mode 100644 index 0000000000000000000000000000000000000000..cf88d09af438db950fc79f1b13198da724e4f10b --- /dev/null +++ b/src/main/java/org/opengroup/osdu/azure/blobstorage/BlobStore.java @@ -0,0 +1,173 @@ +// Copyright © Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.opengroup.osdu.azure.blobstorage; + +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.models.BlobErrorCode; +import com.azure.storage.blob.models.BlobStorageException; +import com.azure.storage.blob.specialized.BlockBlobClient; +import org.opengroup.osdu.core.common.logging.ILogger; +import org.opengroup.osdu.core.common.model.http.AppException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; + +import java.io.ByteArrayOutputStream; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; +import java.util.Collections;; + +/** + * A simpler interface to interact with Azure blob storage. + * Usage examples: + *
+ * {@code
+ *      @Autowired
+ *      private BlobStore blobStore;
+ *
+ *      String readFromBlobExample()
+ *      {
+ *          String content = blobStorage.readFromBlob("dataPartitionId", "filePath");
+ *             if (content != null)
+ *                 return content;
+ *      }
+ *
+ *      void writeToBlobExample()
+ *      {
+ *          blobStorage.writeToBlob("dataPartitionId", "filePath", "content");
+ *      }
+ *
+ *      void deleteFromBlobExample()
+ *      {
+ *          Boolean success = blobStorage.deleteFromBlob("dataPartitionId", "filePath");
+ *      }
+ * }
+ * 
+ */ + +@Component +@Lazy +public class BlobStore { + + @Autowired + private IBlobContainerClientFactory blobContainerClientFactory; + + @Autowired + private ILogger logger; + + private static final String LOG_PREFIX = "azure-core-lib"; + + /** + * + * @param filePath Path of file to be read. + * @param dataPartitionId Data partition id + * @return the content of file with provided file path. + */ + public String readFromBlob(final String dataPartitionId, final String filePath) { + BlobContainerClient blobContainerClient = getBlobContainerClient(dataPartitionId); + BlockBlobClient blockBlobClient = blobContainerClient.getBlobClient(filePath).getBlockBlobClient(); + try (ByteArrayOutputStream downloadStream = new ByteArrayOutputStream()) { + blockBlobClient.download(downloadStream); + return downloadStream.toString(StandardCharsets.UTF_8.name()); + } catch (BlobStorageException ex) { + if (ex.getErrorCode().equals(BlobErrorCode.BLOB_NOT_FOUND)) { + String errorMessage = "Specified blob was not found"; + logger.warning(LOG_PREFIX, errorMessage, Collections.emptyMap()); + throw new AppException(404, errorMessage, ex.getMessage(), ex); + } else { + String errorMessage = "Failed to read specified blob"; + logger.warning(LOG_PREFIX, errorMessage, Collections.emptyMap()); + throw new AppException(500, errorMessage, ex.getMessage(), ex); + } + } catch (UnsupportedEncodingException ex) { + String errorMessage = String.format("Encoding was not correct for item with name=%s", filePath); + logger.warning(LOG_PREFIX, errorMessage, Collections.emptyMap()); + throw new AppException(400, errorMessage, ex.getMessage(), ex); + } catch (IOException ex) { + String errorMessage = String.format("Malformed document for item with name=%s", filePath); + logger.warning(LOG_PREFIX, errorMessage, Collections.emptyMap()); + throw new AppException(500, errorMessage, ex.getMessage(), ex); + } + } + + /** + * + * @param filePath Path of file to be deleted. + * @param dataPartitionId Data partition id + * @return boolean indicating whether the deletion of given file was successful or not. + */ + public boolean deleteFromBlob(final String dataPartitionId, final String filePath) { + BlobContainerClient blobContainerClient = getBlobContainerClient(dataPartitionId); + BlockBlobClient blockBlobClient = blobContainerClient.getBlobClient(filePath).getBlockBlobClient(); + try { + blockBlobClient.delete(); + return true; + } catch (BlobStorageException ex) { + if (ex.getErrorCode().equals(BlobErrorCode.BLOB_NOT_FOUND)) { + String errorMessage = "Specified blob was not found"; + logger.warning(LOG_PREFIX, errorMessage, Collections.emptyMap()); + throw new AppException(404, errorMessage, ex.getMessage(), ex); + } else { + String errorMessage = "Failed to delete blob"; + logger.warning(LOG_PREFIX, errorMessage, Collections.emptyMap()); + throw new AppException(500, errorMessage, ex.getMessage(), ex); + } + } + } + + /** + * + * @param filePath Path of file to be written at. + * @param content Content to be written in the file. + * @param dataPartitionId Data partition id + */ + public void writeToBlob(final String dataPartitionId, + final String filePath, + final String content) { + byte[] bytes = content.getBytes(StandardCharsets.UTF_8); + int bytesSize = bytes.length; + BlobContainerClient blobContainerClient = getBlobContainerClient(dataPartitionId); + BlockBlobClient blockBlobClient = blobContainerClient.getBlobClient(filePath).getBlockBlobClient(); + try (ByteArrayInputStream dataStream = new ByteArrayInputStream(bytes)) { + blockBlobClient.upload(dataStream, bytesSize, true); + } catch (BlobStorageException ex) { + String errorMessage = "Failed to upload file content."; + logger.warning(LOG_PREFIX, errorMessage, Collections.emptyMap()); + throw new AppException(500, errorMessage, ex.getMessage(), ex); + } catch (IOException ex) { + String errorMessage = String.format("Malformed document for item with name=%s", filePath); + logger.warning(LOG_PREFIX, errorMessage, Collections.emptyMap()); + throw new AppException(500, errorMessage, ex.getMessage(), ex); + } + } + + /** + * + * @param dataPartitionId Data partition id + * @return blob container client corresponding to the dataPartitionId. + */ + private BlobContainerClient getBlobContainerClient(final String dataPartitionId) { + try { + return blobContainerClientFactory.getClient(dataPartitionId); + } catch (Exception ex) { + String errorMessage = "Error creating creating blob container client."; + logger.warning(LOG_PREFIX, errorMessage, Collections.emptyMap()); + throw new AppException(500, errorMessage, ex.getMessage(), ex); + } + } +} + diff --git a/src/main/java/org/opengroup/osdu/azure/blobstorage/IBlobContainerClientFactory.java b/src/main/java/org/opengroup/osdu/azure/blobstorage/IBlobContainerClientFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..24a06be32bd8a1ab36be7161a699a502c86cb39f --- /dev/null +++ b/src/main/java/org/opengroup/osdu/azure/blobstorage/IBlobContainerClientFactory.java @@ -0,0 +1,30 @@ +// Copyright © Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.opengroup.osdu.azure.blobstorage; + +import com.azure.storage.blob.BlobContainerClient; + +/** + * Interface for BlobContainer client factory to return appropriate + * blobContainerClient based on the data partition id. + */ +public interface IBlobContainerClientFactory { + /** + * + * @param dataPartitionId Data partition id + * @return blobContainerClient for given data partition id. + */ + BlobContainerClient getClient(String dataPartitionId); +} diff --git a/src/test/java/org/opengroup/osdu/azure/blobstorage/BlobStoreTest.java b/src/test/java/org/opengroup/osdu/azure/blobstorage/BlobStoreTest.java new file mode 100644 index 0000000000000000000000000000000000000000..453367ae50d18744888cafb457f2623b6493e9a7 --- /dev/null +++ b/src/test/java/org/opengroup/osdu/azure/blobstorage/BlobStoreTest.java @@ -0,0 +1,224 @@ +// Copyright © Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.opengroup.osdu.azure.blobstorage; + +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.models.BlobErrorCode; +import com.azure.storage.blob.models.BlobStorageException; +import com.azure.storage.blob.models.BlockBlobItem; +import com.azure.storage.blob.specialized.BlockBlobClient; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opengroup.osdu.core.common.logging.ILogger; +import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.core.common.model.http.DpsHeaders; + +import java.io.ByteArrayOutputStream; +import java.util.HashMap; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; +import static org.mockito.MockitoAnnotations.initMocks; + +@ExtendWith(MockitoExtension.class) +public class BlobStoreTest { + private static final String PARTITION_ID = "dataPartitionId"; + private static final String FILE_PATH = "filePath"; + private static final String CONTENT = "hello world"; + + @InjectMocks + BlobStore blobStore; + + @Mock + IBlobContainerClientFactory blobContainerClientFactory; + + @Mock + BlobContainerClient blobContainerClient; + + @Mock + BlobClient blobClient; + + @Mock + BlockBlobClient blockBlobClient; + + @Mock + BlockBlobItem blockBlobItem; + + @Mock + ILogger logger; + + @BeforeEach + void init() { + initMocks(this); + lenient().doReturn(blobContainerClient).when(blobContainerClientFactory).getClient(PARTITION_ID); + lenient().doReturn(blobClient).when(blobContainerClient).getBlobClient(FILE_PATH); + lenient().doReturn(blockBlobClient).when(blobClient).getBlockBlobClient(); + lenient().doNothing().when(logger).warning(eq("azure-core-lib"), any(), anyMap()); + } + + @Test + public void readFromBlob_ErrorCreatingBlobContainerClient() + { + doThrow(AppException.class).when(blobContainerClientFactory).getClient(eq(PARTITION_ID)); + try { + String content = blobStore.readFromBlob(PARTITION_ID, FILE_PATH); + } catch (AppException ex) { + assertEquals(500, ex.getError().getCode()); + } catch (Exception ex) { + fail("should not get different error code"); + } + } + + @Test + public void readFromBlob_Success() + { + String content = blobStore.readFromBlob(PARTITION_ID, FILE_PATH); + ArgumentCaptor outputStream = ArgumentCaptor.forClass(ByteArrayOutputStream.class); + + // validate that the download method is being invoked appropriately. + verify(blockBlobClient).download(outputStream.capture()); + } + + @Test + public void readFromBlob_BlobNotFound() + { + BlobStorageException exception = mockStorageException(BlobErrorCode.BLOB_NOT_FOUND); + doThrow(exception).when(blockBlobClient).download(any()); + try { + String content = blobStore.readFromBlob(PARTITION_ID, FILE_PATH); + } catch (AppException ex) { + assertEquals(404, ex.getError().getCode()); + } catch (Exception ex) { + fail("should not get different error code"); + } + } + + @Test + public void readFromBlob_InternalError() + { + BlobStorageException exception = mockStorageException(BlobErrorCode.INTERNAL_ERROR); + doThrow(exception).when(blockBlobClient).download(any()); + try { + String content = blobStore.readFromBlob(PARTITION_ID, FILE_PATH); + } catch (AppException ex) { + assertEquals(500, ex.getError().getCode()); + } catch (Exception ex) { + fail("should not get different error code"); + } + } + + @Test + public void deleteFromBlob_ErrorCreatingBlobContainerClient() + { + doThrow(AppException.class).when(blobContainerClientFactory).getClient(eq(PARTITION_ID)); + try { + blobStore.deleteFromBlob(PARTITION_ID, FILE_PATH); + } catch (AppException ex) { + assertEquals(500, ex.getError().getCode()); + } catch (Exception ex) { + fail("should not get different error code"); + } + } + + @Test + public void deleteFromBlob_BlobNotFound() + { + BlobStorageException exception = mockStorageException(BlobErrorCode.BLOB_NOT_FOUND); + doThrow(exception).when(blockBlobClient).delete(); + try { + blobStore.deleteFromBlob(PARTITION_ID, FILE_PATH); + } catch (AppException ex) { + assertEquals(404, ex.getError().getCode()); + } catch (Exception ex) { + fail("should not get different error code"); + } + } + + @Test + public void deleteFromBlob_InternalError() + { + BlobStorageException exception = mockStorageException(BlobErrorCode.INTERNAL_ERROR); + doThrow(exception).when(blockBlobClient).delete(); + try { + blobStore.deleteFromBlob(PARTITION_ID, FILE_PATH); + } catch (AppException ex) { + assertEquals(500, ex.getError().getCode()); + } catch (Exception ex) { + fail("should not get different error code"); + } + } + + @Test + public void deleteFromBlob_Success() + { + doNothing().when(blockBlobClient).delete(); + try { + blobStore.deleteFromBlob(PARTITION_ID, FILE_PATH); + } catch (Exception ex) { + fail("should not get any exception."); + } + } + + @Test + public void writeToBlob_ErrorCreatingBlobContainerClient() + { + doThrow(AppException.class).when(blobContainerClientFactory).getClient(eq(PARTITION_ID)); + try { + blobStore.writeToBlob(PARTITION_ID, FILE_PATH, CONTENT); + } catch (AppException ex) { + assertEquals(500, ex.getError().getCode()); + } catch (Exception ex) { + fail("should not get different error code"); + } + } + + @Test + public void writeToBlob_InternalError() + { + BlobStorageException exception = mockStorageException(BlobErrorCode.INTERNAL_ERROR); + doThrow(exception).when(blockBlobClient).upload(any(), anyLong(), eq(true)); + try { + blobStore.writeToBlob(PARTITION_ID, FILE_PATH, CONTENT); + } catch (AppException ex) { + assertEquals(500, ex.getError().getCode()); + } catch (Exception ex) { + fail("should not get different error code"); + } + } + + @Test + public void writeToBlob_Success() + { + doReturn(blockBlobItem).when(blockBlobClient).upload(any(), anyLong(), eq(true)); + try { + blobStore.writeToBlob(PARTITION_ID, FILE_PATH, CONTENT); + } catch (Exception ex) { + fail("should not get any exception."); + } + } + + private BlobStorageException mockStorageException(BlobErrorCode errorCode) { + BlobStorageException mockException = mock(BlobStorageException.class); + lenient().when(mockException.getErrorCode()).thenReturn(errorCode); + return mockException; + } +}