Skip to content

GitLab

Commit 891c6537 authored by MANISH KUMAR's avatar MANISH KUMAR
Browse files

Merge branch 'log-blobstore-calls' into 'master' 

Log external calls from BlobStore

See merge request !57
parents 7a70410b 89cfdff5
Pipeline #26932 passed with stages
in 10 minutes and 35 seconds
Hide whitespace changes
Inline Side-by-side
src/main/java/org/opengroup/osdu/azure/blobstorage/BlobStore.java
View file @ 891c6537
......@@ -20,12 +20,17 @@ import com.azure.storage.blob.BlobServiceClient;
import com.azure.storage.blob.models.BlobCopyInfo;
import com.azure.storage.blob.models.BlobErrorCode;
import com.azure.storage.blob.models.BlobStorageException;
import com.azure.storage.blob.models.CopyStatusType;
import com.azure.storage.blob.sas.BlobContainerSasPermission;
import com.azure.storage.blob.sas.BlobSasPermission;
import com.azure.storage.blob.sas.BlobServiceSasSignatureValues;
import com.azure.storage.blob.specialized.BlockBlobClient;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.azure.logging.CoreLoggerFactory;
import org.opengroup.osdu.azure.logging.DependencyPayload;
import org.opengroup.osdu.core.common.logging.ILogger;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.slf4j.helpers.MessageFormatter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
......@@ -34,7 +39,6 @@ import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.OffsetDateTime;
import java.util.Collections;
/**
* A simpler interface to interact with Azure blob storage.
......@@ -78,12 +82,15 @@ import java.util.Collections;
* </pre>
*/
public class BlobStore {
private static final String LOGGER_NAME = BlobStore.class.getName();
private IBlobServiceClientFactory blobServiceClientFactory;
private ILogger logger;
/**
* Constructor to create BlobStore.
* @param factory Factory that provides blob client.
*
* @param factory Factory that provides blob client.
* @param loggerInstance logger instance to be used for logging.
*/
public BlobStore(final IBlobServiceClientFactory factory, final ILogger loggerInstance) {
......@@ -91,8 +98,6 @@ public class BlobStore {
this.logger = loggerInstance;
}
private static final String LOG_PREFIX = "azure-core-lib";
/**
* @param filePath Path of file to be read.
* @param dataPartitionId Data partition id
......@@ -105,19 +110,28 @@ public class BlobStore {
final String containerName) {
BlobContainerClient blobContainerClient = getBlobContainerClient(dataPartitionId, containerName);
BlockBlobClient blockBlobClient = blobContainerClient.getBlobClient(filePath).getBlockBlobClient();
final long start = System.currentTimeMillis();
int statusCode = HttpStatus.SC_OK;
try (ByteArrayOutputStream downloadStream = new ByteArrayOutputStream()) {
blockBlobClient.download(downloadStream);
CoreLoggerFactory.getInstance().getLogger(LOGGER_NAME).info("{}", MessageFormatter.format("Done reading from {}", filePath).getMessage());
return downloadStream.toString(StandardCharsets.UTF_8.name());
} catch (BlobStorageException ex) {
statusCode = ex.getStatusCode();
if (ex.getErrorCode().equals(BlobErrorCode.BLOB_NOT_FOUND)) {
throw handleBlobStoreException(404, "Specified blob was not found", ex);
}
throw handleBlobStoreException(500, "Failed to read specified blob", ex);
} catch (UnsupportedEncodingException ex) {
throw handleBlobStoreException(400, String.format("Encoding was not correct for item with name=%s", filePath), ex);
statusCode = HttpStatus.SC_BAD_REQUEST;
throw handleBlobStoreException(400, MessageFormatter.format("Encoding was not correct for item with name={}", filePath).getMessage(), ex);
} catch (IOException ex) {
throw handleBlobStoreException(500, String.format("Malformed document for item with name=%s", filePath), ex);
statusCode = HttpStatus.SC_INTERNAL_SERVER_ERROR;
throw handleBlobStoreException(500, MessageFormatter.format("Malformed document for item with name={}", filePath).getMessage(), ex);
} finally {
final long timeTaken = System.currentTimeMillis() - start;
final String dependencyData = MessageFormatter.arrayFormat("{}:{}/{}", new String[]{dataPartitionId, containerName, filePath}).getMessage();
logDependency("READ_FROM_STORAGE_CONTAINER", dependencyData, dependencyData, timeTaken, String.valueOf(statusCode), statusCode == HttpStatus.SC_OK);
}
}
......@@ -133,14 +147,22 @@ public class BlobStore {
final String containerName) {
BlobContainerClient blobContainerClient = getBlobContainerClient(dataPartitionId, containerName);
BlockBlobClient blockBlobClient = blobContainerClient.getBlobClient(filePath).getBlockBlobClient();
final long start = System.currentTimeMillis();
int statusCode = HttpStatus.SC_OK;
try {
blockBlobClient.delete();
CoreLoggerFactory.getInstance().getLogger(LOGGER_NAME).info("{}", MessageFormatter.format("Done deleting blob at {}", filePath).getMessage());
return true;
} catch (BlobStorageException ex) {
statusCode = ex.getStatusCode();
if (ex.getErrorCode().equals(BlobErrorCode.BLOB_NOT_FOUND)) {
throw handleBlobStoreException(404, "Specified blob was not found", ex);
}
throw handleBlobStoreException(500, "Failed to delete blob", ex);
} finally {
final long timeTaken = System.currentTimeMillis() - start;
final String dependencyData = MessageFormatter.arrayFormat("{}:{}/{}", new String[]{dataPartitionId, containerName, filePath}).getMessage();
logDependency("DELETE_FROM_STORAGE_CONTAINER", dependencyData, dependencyData, timeTaken, String.valueOf(statusCode), statusCode == HttpStatus.SC_OK);
}
}
......@@ -159,12 +181,22 @@ public class BlobStore {
int bytesSize = bytes.length;
BlobContainerClient blobContainerClient = getBlobContainerClient(dataPartitionId, containerName);
BlockBlobClient blockBlobClient = blobContainerClient.getBlobClient(filePath).getBlockBlobClient();
final long start = System.currentTimeMillis();
int statusCode = HttpStatus.SC_OK;
try (ByteArrayInputStream dataStream = new ByteArrayInputStream(bytes)) {
blockBlobClient.upload(dataStream, bytesSize, true);
CoreLoggerFactory.getInstance().getLogger(LOGGER_NAME).info("{}", MessageFormatter.format("Done uploading file content to %s", filePath).getMessage());
} catch (BlobStorageException ex) {
statusCode = ex.getStatusCode();
throw handleBlobStoreException(500, "Failed to upload file content.", ex);
} catch (IOException ex) {
throw handleBlobStoreException(500, String.format("Malformed document for item with name=%s", filePath), ex);
statusCode = HttpStatus.SC_INTERNAL_SERVER_ERROR;
throw handleBlobStoreException(500, MessageFormatter.format("Malformed document for item with name={}", filePath).getMessage(), ex);
} finally {
final long timeTaken = System.currentTimeMillis() - start;
final String dependencyData = MessageFormatter.format("{}:{}/{}", new String[]{dataPartitionId, containerName, filePath}).getMessage();
logDependency("WRITE_TO_STORAGE_CONTAINER", dependencyData, dependencyData, timeTaken, String.valueOf(statusCode), statusCode == HttpStatus.SC_OK);
}
}
......@@ -198,10 +230,11 @@ public class BlobStore {
/**
* Generates pre-signed url to a blob container.
*
* @param dataPartitionId data partition id
* @param containerName Name of the storage container
* @param expiryTime Time after which the token expires
* @param permissions permissions for the given container
* @param containerName Name of the storage container
* @param expiryTime Time after which the token expires
* @param permissions permissions for the given container
* @return Generates pre-signed url for a given container
*/
public String generatePreSignedURL(final String dataPartitionId, final String containerName, final OffsetDateTime expiryTime, final BlobContainerSasPermission permissions) {
......@@ -211,6 +244,7 @@ public class BlobStore {
/**
* Method is used to copy a file specified at Source URL to the provided destination.
*
* @param dataPartitionId Data partition id
* @param filePath Path of file (blob) to which the file has to be copied
* @param containerName Name of the storage container
......@@ -218,34 +252,49 @@ public class BlobStore {
* @return Blob Copy Final Result.
*/
public BlobCopyInfo copyFile(final String dataPartitionId, final String filePath, final String containerName,
final String sourceUrl) {
final String sourceUrl) {
BlobContainerClient blobContainerClient = getBlobContainerClient(dataPartitionId, containerName);
BlockBlobClient blockBlobClient = blobContainerClient.getBlobClient(filePath).getBlockBlobClient();
final long start = System.currentTimeMillis();
SyncPoller<BlobCopyInfo, Void> result = blockBlobClient.beginCopy(sourceUrl, Duration.ofSeconds(1));
return result.waitForCompletion().getValue();
BlobCopyInfo blobCopyInfo = result.waitForCompletion().getValue();
final long timeTaken = System.currentTimeMillis() - start;
final String target = MessageFormatter.arrayFormat("{}:{}/{}", new String[]{dataPartitionId, containerName, filePath}).getMessage();
CopyStatusType status = blobCopyInfo == null ? CopyStatusType.FAILED : blobCopyInfo.getCopyStatus();
logDependency("COPY_FILE", sourceUrl, target, timeTaken, status.toString(), status == CopyStatusType.SUCCESS);
return blobCopyInfo;
}
/**
*
* @param blockBlobClient Blob client
* @param expiryTime Time after which SAS Token expires
* @param permissions Permissions for the given blob
* @param expiryTime Time after which SAS Token expires
* @param permissions Permissions for the given blob
* @return Generates SAS Token.
*/
private String generateSASToken(final BlockBlobClient blockBlobClient, final OffsetDateTime expiryTime, final BlobSasPermission permissions) {
BlobServiceSasSignatureValues blobServiceSasSignatureValues = new BlobServiceSasSignatureValues(expiryTime, permissions);
return blockBlobClient.generateSas(blobServiceSasSignatureValues);
final long start = System.currentTimeMillis();
String sasToken = blockBlobClient.generateSas(blobServiceSasSignatureValues);
final long timeTaken = System.currentTimeMillis() - start;
logDependency("GENERATE_SAS_TOKEN", blockBlobClient.getBlobName(), blockBlobClient.getBlobUrl(), timeTaken, String.valueOf(HttpStatus.SC_OK), true);
return sasToken;
}
/**
* @param client Container client
* @param expiryTime Time after which SAS Token expires
* @param client Container client
* @param expiryTime Time after which SAS Token expires
* @param permissions Permissions for the given container
* @return Generates SAS Token.
*/
private String generateSASToken(final BlobContainerClient client, final OffsetDateTime expiryTime, final BlobContainerSasPermission permissions) {
BlobServiceSasSignatureValues blobServiceSasSignatureValues = new BlobServiceSasSignatureValues(expiryTime, permissions);
return client.generateSas(blobServiceSasSignatureValues);
final long start = System.currentTimeMillis();
String sasToken = client.generateSas(blobServiceSasSignatureValues);
final long timeTaken = System.currentTimeMillis() - start;
logDependency("GENERATE_SAS_TOKEN", client.getBlobContainerName(), client.getBlobContainerUrl(), timeTaken, String.valueOf(HttpStatus.SC_OK), true);
return sasToken;
}
/**
......@@ -273,7 +322,25 @@ public class BlobStore {
* @return Instance of AppException
*/
private AppException handleBlobStoreException(final int status, final String errorMessage, final Exception ex) {
logger.warning(LOG_PREFIX, errorMessage, Collections.<String, String>emptyMap());
CoreLoggerFactory.getInstance().getLogger(LOGGER_NAME).warn(MessageFormatter.format("{}", errorMessage).getMessage(), ex);
return new AppException(status, errorMessage, ex.getMessage(), ex);
}
/**
* Log dependency.
*
* @param name the name of the command initiated with this dependency call
* @param data the command initiated by this dependency call
* @param target the target of this dependency call
* @param timeTakenInMs the request duration in milliseconds
* @param resultCode the result code of the call
* @param success indication of successful or unsuccessful call
*/
private void logDependency(final String name, final String data, final String target, final long timeTakenInMs, final String resultCode, final boolean success) {
DependencyPayload payload = new DependencyPayload(name, data, Duration.ofMillis(timeTakenInMs), resultCode, success);
payload.setType("BlobStore");
payload.setTarget(target);
CoreLoggerFactory.getInstance().getLogger(LOGGER_NAME).logDependency(payload);
}
}
src/test/java/org/opengroup/osdu/azure/KeyVaultFacadeTest.java
View file @ 891c6537
......@@ -16,22 +16,79 @@ package org.opengroup.osdu.azure;
import com.azure.security.keyvault.secrets.SecretClient;
import com.azure.security.keyvault.secrets.models.KeyVaultSecret;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
import org.opengroup.osdu.azure.logging.CoreLogger;
import org.opengroup.osdu.azure.logging.CoreLoggerFactory;
import java.lang.reflect.Field;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyMap;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.*;
import static org.mockito.Mockito.lenient;
import static org.mockito.MockitoAnnotations.initMocks;
@ExtendWith(MockitoExtension.class)
class KeyVaultFacadeTest {
@Mock
private CoreLoggerFactory coreLoggerFactory;
@Mock
private CoreLogger coreLogger;
@Mock
private SecretClient kv;
/**
* Workaround for inability to mock static methods like getInstance().
*
* @param mock CoreLoggerFactory mock instance
*/
private void mockSingleton(CoreLoggerFactory mock) {
try {
Field instance = CoreLoggerFactory.class.getDeclaredField("instance");
instance.setAccessible(true);
instance.set(null, mock);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
/**
* Reset workaround for inability to mock static methods like getInstance().
*/
private void resetSingleton() {
try {
Field instance = CoreLoggerFactory.class.getDeclaredField("instance");
instance.setAccessible(true);
instance.set(null, null);
instance.setAccessible(false);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@BeforeEach
void init() {
mockSingleton(coreLoggerFactory);
when(coreLoggerFactory.getLogger(anyString())).thenReturn(coreLogger);
}
@AfterEach
public void takeDown() {
resetSingleton();
}
@Test
void getSecretWithValidation_checksForNullResponse() {
// the null response is the default behavior, but its included here to be explicit
......
src/test/java/org/opengroup/osdu/azure/blobstorage/BlobStoreTest.java
View file @ 891c6537
......@@ -19,14 +19,12 @@ import com.azure.core.util.polling.SyncPoller;
import com.azure.storage.blob.BlobClient;
import com.azure.storage.blob.BlobContainerClient;
import com.azure.storage.blob.BlobServiceClient;
import com.azure.storage.blob.models.BlobCopyInfo;
import com.azure.storage.blob.models.BlobErrorCode;
import com.azure.storage.blob.models.BlobStorageException;
import com.azure.storage.blob.models.BlockBlobItem;
import com.azure.storage.blob.models.*;
import com.azure.storage.blob.sas.BlobContainerSasPermission;
import com.azure.storage.blob.sas.BlobSasPermission;
import com.azure.storage.blob.sas.BlobServiceSasSignatureValues;
import com.azure.storage.blob.specialized.BlockBlobClient;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
......@@ -34,13 +32,15 @@ import org.mockito.ArgumentCaptor;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.opengroup.osdu.azure.logging.CoreLogger;
import org.opengroup.osdu.azure.logging.CoreLoggerFactory;
import org.opengroup.osdu.core.common.logging.ILogger;
import org.opengroup.osdu.core.common.model.http.AppException;
import java.io.ByteArrayOutputStream;
import java.lang.reflect.Field;
import java.time.Duration;
import java.time.OffsetDateTime;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
......@@ -57,8 +57,11 @@ public class BlobStoreTest {
private static final String STORAGE_CONTAINER_NAME = "containerName";
private static final String SOURCE_FILE_URL = "http://someURL";
@InjectMocks
private BlobStore blobStore;
@Mock
private CoreLoggerFactory coreLoggerFactory;
@Mock
private CoreLogger coreLogger;
@Mock
private IBlobServiceClientFactory blobServiceClientFactory;
......@@ -93,9 +96,45 @@ public class BlobStoreTest {
@Mock
private BlobSasPermission blobSasPermission;
@InjectMocks
private BlobStore blobStore;
/**
* Workaround for inability to mock static methods like getInstance().
*
* @param mock CoreLoggerFactory mock instance
*/
private void mockSingleton(CoreLoggerFactory mock) {
try {
Field instance = CoreLoggerFactory.class.getDeclaredField("instance");
instance.setAccessible(true);
instance.set(null, mock);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
/**
* Reset workaround for inability to mock static methods like getInstance().
*/
private void resetSingleton() {
try {
Field instance = CoreLoggerFactory.class.getDeclaredField("instance");
instance.setAccessible(true);
instance.set(null, null);
instance.setAccessible(false);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@BeforeEach
void init() {
initMocks(this);
mockSingleton(coreLoggerFactory);
when(coreLoggerFactory.getLogger(anyString())).thenReturn(coreLogger);
lenient().when(blobClient.getBlockBlobClient()).thenReturn(blockBlobClient);
lenient().when(blobContainerClient.getBlobClient(FILE_PATH)).thenReturn(blobClient);
lenient().when(blobServiceClient.getBlobContainerClient(STORAGE_CONTAINER_NAME)).thenReturn(blobContainerClient);
......@@ -103,6 +142,11 @@ public class BlobStoreTest {
lenient().doNothing().when(logger).warning(eq("azure-core-lib"), any(), anyMap());
}
@AfterEach
public void takeDown() {
resetSingleton();
}
@Test
public void readFromStorageContainer_ErrorCreatingBlobContainerClient() {
doThrow(BlobStorageException.class).when(blobServiceClientFactory).getBlobServiceClient(eq(PARTITION_ID));
......@@ -249,6 +293,7 @@ public class BlobStoreTest {
public void copyFile_Success() {
String copyId = "copyId";
doReturn(copyId).when(blobCopyInfo).getCopyId();
doReturn(CopyStatusType.SUCCESS).when(blobCopyInfo).getCopyStatus();
doReturn(blobCopyInfo).when(pollResponse).getValue();
doReturn(pollResponse).when(syncPoller).waitForCompletion();
doReturn(syncPoller).when(blockBlobClient).beginCopy(SOURCE_FILE_URL, Duration.ofSeconds(1));
......
src/test/java/org/opengroup/osdu/azure/cosmosdb/CosmosStoreTest.java
View file @ 891c6537
......@@ -106,7 +106,7 @@ class CosmosStoreTest {
try {
Field instance = CoreLoggerFactory.class.getDeclaredField("instance");
instance.setAccessible(true);
instance.set(instance, mock);
instance.set(null, mock);
} catch (Exception e) {
throw new RuntimeException(e);
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment