From 588799ecbf849c9b9834cc1cde00a4d26e6dd5a2 Mon Sep 17 00:00:00 2001 From: Harshit Aggarwal Date: Wed, 17 Aug 2022 17:44:31 +0530 Subject: [PATCH 1/5] init --- .../DataLakeClientFactoryImpl.java | 305 ++++++++++-------- .../azure/datalakestorage/DataLakeStore.java | 46 +++ .../IDataLakeClientFactory.java | 96 +++--- .../publisherFacade/ServiceBusPublisher.java | 2 +- 4 files changed, 263 insertions(+), 186 deletions(-) diff --git a/src/main/java/org/opengroup/osdu/azure/datalakestorage/DataLakeClientFactoryImpl.java b/src/main/java/org/opengroup/osdu/azure/datalakestorage/DataLakeClientFactoryImpl.java index ed98004e..61538b59 100644 --- a/src/main/java/org/opengroup/osdu/azure/datalakestorage/DataLakeClientFactoryImpl.java +++ b/src/main/java/org/opengroup/osdu/azure/datalakestorage/DataLakeClientFactoryImpl.java @@ -1,143 +1,164 @@ -// Copyright © Microsoft Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.opengroup.osdu.azure.datalakestorage; - -import com.azure.identity.DefaultAzureCredential; -import com.azure.storage.common.StorageSharedKeyCredential; -import com.azure.storage.common.policy.RequestRetryOptions; -import com.azure.storage.file.datalake.DataLakeDirectoryClient; -import com.azure.storage.file.datalake.DataLakeFileSystemClient; -import com.azure.storage.file.datalake.DataLakeServiceClient; -import com.azure.storage.file.datalake.DataLakeServiceClientBuilder; -import org.opengroup.osdu.azure.di.BlobStoreRetryConfiguration; -import org.opengroup.osdu.azure.di.MSIConfiguration; -import org.opengroup.osdu.azure.partition.PartitionInfoAzure; -import org.opengroup.osdu.azure.partition.PartitionServiceClient; -import org.opengroup.osdu.common.Validators; -import org.springframework.beans.factory.annotation.Autowired; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - - -/** - * Implementation for IDataLakeClientFactory. - */ -public final class DataLakeClientFactoryImpl implements IDataLakeClientFactory { - - private Map dataLakeContainerClientMap; - - @Autowired - private DefaultAzureCredential defaultAzureCredential; - - @Autowired - private PartitionServiceClient partitionService; - - @Autowired - private BlobStoreRetryConfiguration blobStoreRetryConfiguration; - - @Autowired - private MSIConfiguration msiConfiguration; - - /** - * Constructor to initialize instance of {@link DataLakeClientFactoryImpl}. - * @param credentials Azure Credentials to use - * @param partitionServiceClient Partition service client to use - */ - public DataLakeClientFactoryImpl(final DefaultAzureCredential credentials, - final PartitionServiceClient partitionServiceClient) { - this.defaultAzureCredential = credentials; - this.partitionService = partitionServiceClient; - dataLakeContainerClientMap = new ConcurrentHashMap<>(); - } - /** - * create DataLakeDirectoryClient. - * @param dataPartitionId - * @param directoryName - * @param containerName - * @return DataLakeServiceClient - */ - @Override - public DataLakeDirectoryClient getDataLakeDirectoryClient( - final String dataPartitionId, - final String directoryName, - final String containerName) { - DataLakeFileSystemClient fileSystemClient = getFileSystemClient(dataPartitionId, containerName); - return fileSystemClient.getDirectoryClient(directoryName); - } - - /** - * - * @param dataPartitionId dataPartitionId - * @param fileSystemName file System Name - * @return DataLakeFileSystemClient - */ - private DataLakeFileSystemClient getFileSystemClient( - final String dataPartitionId, - final String fileSystemName) { - Validators.checkNotNullAndNotEmpty(dataPartitionId, "dataPartitionId"); - Validators.checkNotNullAndNotEmpty(fileSystemName, "fileSystemName"); - - String cacheKey = String.format("%s-%s", dataPartitionId, fileSystemName); - DataLakeServiceClient dataLakeServiceClient; - if (this.dataLakeContainerClientMap.containsKey(cacheKey)) { - dataLakeServiceClient = this.dataLakeContainerClientMap.get(cacheKey); - } else { - dataLakeServiceClient = getDataLakeServiceClient(dataPartitionId); - this.dataLakeContainerClientMap.put(cacheKey, dataLakeServiceClient); - } - return dataLakeServiceClient.getFileSystemClient(fileSystemName); - } - - /** - * - * @param dataPartitionId dataPartitionId - * @return DataLakeServiceClient - */ - @Override - public DataLakeServiceClient getDataLakeServiceClient( - final String dataPartitionId) { - PartitionInfoAzure pi = this.partitionService.getPartition(dataPartitionId); - - String endpoint = String.format("https://%s.dfs.core.windows.net", pi.getHierarchicalStorageAccountName()); - - DataLakeServiceClientBuilder dataLakeServiceClientBuilder = getDataLakeServiceClientBuilder(endpoint); - if (msiConfiguration.getIsEnabled()) { - return dataLakeServiceClientBuilder.credential(defaultAzureCredential) - .buildClient(); - } else { - StorageSharedKeyCredential storageSharedKeyCredential = new StorageSharedKeyCredential( - pi.getHierarchicalStorageAccountName(), - pi.getHierarchicalStorageAccountKey() - ); - - return dataLakeServiceClientBuilder - .credential(storageSharedKeyCredential) - .buildClient(); - } - } - - /** - * - * @param endpoint Azure DataLake endpoint - * @return DataLakeServiceClientBuilder - */ - private DataLakeServiceClientBuilder getDataLakeServiceClientBuilder(final String endpoint) { - RequestRetryOptions requestRetryOptions = blobStoreRetryConfiguration.getRequestRetryOptions(); - return new DataLakeServiceClientBuilder() - .endpoint(endpoint) - .retryOptions(requestRetryOptions); - } +// Copyright © Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.opengroup.osdu.azure.datalakestorage; + +import com.azure.identity.DefaultAzureCredential; +import com.azure.storage.common.StorageSharedKeyCredential; +import com.azure.storage.common.policy.RequestRetryOptions; +import com.azure.storage.file.datalake.DataLakeDirectoryClient; +import com.azure.storage.file.datalake.DataLakeFileSystemClient; +import com.azure.storage.file.datalake.DataLakeServiceClient; +import com.azure.storage.file.datalake.DataLakeServiceClientBuilder; +import org.opengroup.osdu.azure.di.BlobStoreRetryConfiguration; +import org.opengroup.osdu.azure.di.MSIConfiguration; +import org.opengroup.osdu.azure.partition.PartitionInfoAzure; +import org.opengroup.osdu.azure.partition.PartitionServiceClient; +import org.opengroup.osdu.common.Validators; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + + +/** + * Implementation for IDataLakeClientFactory. + */ +public final class DataLakeClientFactoryImpl implements IDataLakeClientFactory { + + private Map dataLakeContainerClientMap; + + @Autowired + private DefaultAzureCredential defaultAzureCredential; + + @Autowired + private PartitionServiceClient partitionService; + + @Autowired + private BlobStoreRetryConfiguration blobStoreRetryConfiguration; + + @Autowired + private MSIConfiguration msiConfiguration; + + /** + * Constructor to initialize instance of {@link DataLakeClientFactoryImpl}. + * @param credentials Azure Credentials to use + * @param partitionServiceClient Partition service client to use + */ + public DataLakeClientFactoryImpl(final DefaultAzureCredential credentials, + final PartitionServiceClient partitionServiceClient) { + this.defaultAzureCredential = credentials; + this.partitionService = partitionServiceClient; + dataLakeContainerClientMap = new ConcurrentHashMap<>(); + } + /** + * create DataLakeDirectoryClient. + * @param dataPartitionId + * @param directoryName + * @param containerName + * @return DataLakeServiceClient + */ + @Override + public DataLakeDirectoryClient getDataLakeDirectoryClient( + final String dataPartitionId, + final String directoryName, + final String containerName) { + DataLakeFileSystemClient fileSystemClient = getFileSystemClient(dataPartitionId, containerName); + return fileSystemClient.getDirectoryClient(directoryName); + } + + /** + * + * @param dataPartitionId dataPartitionId + * @param fileSystemName file System Name + * @return DataLakeFileSystemClient + */ + private DataLakeFileSystemClient getFileSystemClient( + final String dataPartitionId, + final String fileSystemName) { + Validators.checkNotNullAndNotEmpty(dataPartitionId, "dataPartitionId"); + Validators.checkNotNullAndNotEmpty(fileSystemName, "fileSystemName"); + + String cacheKey = String.format("%s-%s", dataPartitionId, fileSystemName); + DataLakeServiceClient dataLakeServiceClient; + if (this.dataLakeContainerClientMap.containsKey(cacheKey)) { + dataLakeServiceClient = this.dataLakeContainerClientMap.get(cacheKey); + } else { + dataLakeServiceClient = getDataLakeServiceClient(dataPartitionId); + this.dataLakeContainerClientMap.put(cacheKey, dataLakeServiceClient); + } + return dataLakeServiceClient.getFileSystemClient(fileSystemName); + } + + /** + * + * @param dataPartitionId dataPartitionId + * @return DataLakeServiceClient + */ + @Override + public DataLakeServiceClient getDataLakeServiceClient( + final String dataPartitionId) { + PartitionInfoAzure pi = this.partitionService.getPartition(dataPartitionId); + + String endpoint = String.format("https://%s.dfs.core.windows.net", pi.getHierarchicalStorageAccountName()); + + DataLakeServiceClientBuilder dataLakeServiceClientBuilder = getDataLakeServiceClientBuilder(endpoint); + if (msiConfiguration.getIsEnabled()) { + return dataLakeServiceClientBuilder.credential(defaultAzureCredential) + .buildClient(); + } else { + StorageSharedKeyCredential storageSharedKeyCredential = new StorageSharedKeyCredential( + pi.getHierarchicalStorageAccountName(), + pi.getHierarchicalStorageAccountKey() + ); + + return dataLakeServiceClientBuilder + .credential(storageSharedKeyCredential) + .buildClient(); + } + } + + /** + * + * @param dataPartitionId dataPartitionId + * @param fileSystemName fileSystemName + * @return DataLakeServiceClient + */ + @Override + public DataLakeServiceClient getDataLakeServiceClient( + final String dataPartitionId, + final String fileSystemName) { + String cacheKey = String.format("%s-%s", dataPartitionId, fileSystemName); + DataLakeServiceClient dataLakeServiceClient; + if (this.dataLakeContainerClientMap.containsKey(cacheKey)) { + dataLakeServiceClient = this.dataLakeContainerClientMap.get(cacheKey); + } else { + dataLakeServiceClient = getDataLakeServiceClient(dataPartitionId); + this.dataLakeContainerClientMap.put(cacheKey, dataLakeServiceClient); + } + return dataLakeServiceClient; + } + + /** + * + * @param endpoint Azure DataLake endpoint + * @return DataLakeServiceClientBuilder + */ + private DataLakeServiceClientBuilder getDataLakeServiceClientBuilder(final String endpoint) { + RequestRetryOptions requestRetryOptions = blobStoreRetryConfiguration.getRequestRetryOptions(); + return new DataLakeServiceClientBuilder() + .endpoint(endpoint) + .retryOptions(requestRetryOptions); + } } \ No newline at end of file diff --git a/src/main/java/org/opengroup/osdu/azure/datalakestorage/DataLakeStore.java b/src/main/java/org/opengroup/osdu/azure/datalakestorage/DataLakeStore.java index 6fc4dc92..1a016acb 100644 --- a/src/main/java/org/opengroup/osdu/azure/datalakestorage/DataLakeStore.java +++ b/src/main/java/org/opengroup/osdu/azure/datalakestorage/DataLakeStore.java @@ -16,6 +16,8 @@ package org.opengroup.osdu.azure.datalakestorage; import com.azure.storage.common.sas.SasProtocol; import com.azure.storage.file.datalake.DataLakeDirectoryClient; +import com.azure.storage.file.datalake.DataLakeServiceClient; +import com.azure.storage.file.datalake.models.UserDelegationKey; import com.azure.storage.file.datalake.sas.DataLakeServiceSasSignatureValues; import com.azure.storage.file.datalake.sas.FileSystemSasPermission; import org.apache.http.HttpStatus; @@ -76,6 +78,22 @@ public class DataLakeStore { return String.format("%s?%s", dataLakeClient.getDirectoryUrl(), generateSASToken(dataLakeClient, expiryTime, permissions)); } + /** + * Generates pre-signed url to a DataLake Directory using the user delegation key. + * + * @param dataPartitionId dataPartitionId + * @param containerName containerName + * @param directoryName fileName + * @param expiryTime expiryTime + * @param permissions permissions + * @return string + */ + public String generatePreSignedURLWithUserDelegationSas(final String dataPartitionId, final String containerName, final String directoryName, + final OffsetDateTime expiryTime, final FileSystemSasPermission permissions) { + DataLakeDirectoryClient dataLakeClient = createDataLakeDirectoryClient(dataPartitionId, directoryName, containerName); + return String.format("%s?%s", dataLakeClient.getDirectoryUrl(), generateSASTokenWithUserDelegationSas(dataPartitionId, containerName, dataLakeClient, expiryTime, permissions)); + } + /** * Generate SaS token to interact with Azure DataLake Gen 2. * @@ -97,6 +115,34 @@ public class DataLakeStore { return sasToken; } + /** + * Generate SaS token to interact with Azure DataLake Gen 2. + * + * @param dataPartitionId dataPartitionId + * @param fileSystemName fileSystemName + * @param client client + * @param expiryTime expiryTime + * @param permissions permissions + * @return string + */ + private String generateSASTokenWithUserDelegationSas(final String dataPartitionId, final String fileSystemName, final DataLakeDirectoryClient client, final OffsetDateTime expiryTime, + final FileSystemSasPermission permissions) { + DataLakeServiceSasSignatureValues sign = new DataLakeServiceSasSignatureValues(expiryTime, permissions) + .setStartTime(OffsetDateTime.now()) + .setProtocol(SasProtocol.HTTPS_ONLY); + + OffsetDateTime startTime = OffsetDateTime.now(); + DataLakeServiceClient dataLakeServiceClient = dataLakeClientFactory.getDataLakeServiceClient(dataPartitionId, fileSystemName); + + UserDelegationKey userDelegationKey = dataLakeServiceClient.getUserDelegationKey(startTime, expiryTime); + final long start = System.currentTimeMillis(); + String sasToken = client.generateUserDelegationSas(sign, userDelegationKey); + + final long timeTaken = System.currentTimeMillis() - start; + logDependency("GENERATE_SAS_TOKEN", client.getDirectoryName(), client.getDirectoryUrl(), timeTaken, String.valueOf(HttpStatus.SC_OK), true); + return sasToken; + } + /** * Create DataLakeDirectoryClient. * diff --git a/src/main/java/org/opengroup/osdu/azure/datalakestorage/IDataLakeClientFactory.java b/src/main/java/org/opengroup/osdu/azure/datalakestorage/IDataLakeClientFactory.java index 1573a3ac..e4d152cc 100644 --- a/src/main/java/org/opengroup/osdu/azure/datalakestorage/IDataLakeClientFactory.java +++ b/src/main/java/org/opengroup/osdu/azure/datalakestorage/IDataLakeClientFactory.java @@ -1,43 +1,53 @@ -// Copyright © Microsoft Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.opengroup.osdu.azure.datalakestorage; - -import com.azure.storage.file.datalake.DataLakeDirectoryClient; -import com.azure.storage.file.datalake.DataLakeServiceClient; - -/** - * Foo. - */ -public interface IDataLakeClientFactory { - /** - * - * @param dataPartitionId dataPartitionId - * @param pathName pathName - * @param containerName containerName - * @return DataLakeDirectoryClient - */ - DataLakeDirectoryClient getDataLakeDirectoryClient( - String dataPartitionId, - String pathName, - String containerName); - - /** - * - * @param dataPartitionId dataPartitionId - * @return Get DataLakeServiceClient. - */ - DataLakeServiceClient getDataLakeServiceClient( - String dataPartitionId); -} +// Copyright © Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.opengroup.osdu.azure.datalakestorage; + +import com.azure.storage.file.datalake.DataLakeDirectoryClient; +import com.azure.storage.file.datalake.DataLakeServiceClient; + +/** + * Foo. + */ +public interface IDataLakeClientFactory { + /** + * + * @param dataPartitionId dataPartitionId + * @param pathName pathName + * @param containerName containerName + * @return DataLakeDirectoryClient + */ + DataLakeDirectoryClient getDataLakeDirectoryClient( + String dataPartitionId, + String pathName, + String containerName); + + /** + * + * @param dataPartitionId dataPartitionId + * @return Get DataLakeServiceClient. + */ + DataLakeServiceClient getDataLakeServiceClient( + String dataPartitionId); + + /** + * + * @param dataPartitionId dataPartitionId + * @param fileSystemName fileSystemName + * @return Get DataLakeServiceClient. + */ + DataLakeServiceClient getDataLakeServiceClient( + String dataPartitionId, + String fileSystemName); +} diff --git a/src/main/java/org/opengroup/osdu/azure/publisherFacade/ServiceBusPublisher.java b/src/main/java/org/opengroup/osdu/azure/publisherFacade/ServiceBusPublisher.java index 55dff4c4..aa0c12ee 100644 --- a/src/main/java/org/opengroup/osdu/azure/publisherFacade/ServiceBusPublisher.java +++ b/src/main/java/org/opengroup/osdu/azure/publisherFacade/ServiceBusPublisher.java @@ -81,7 +81,7 @@ public class ServiceBusPublisher { retryCount--; if (retryCount < 0) { LOGGER.error("Retry limit Exceeded.Unable to publish message with message id {}", message.getMessageId()); - throw new AppException(501,"Internal Server Error" , "Failed to publish message in service bus", e); + throw new AppException(501, "Internal Server Error", "Failed to publish message in service bus", e); } } } -- GitLab From 14a6861c47f1a82322cdefc73dd4ca325be72106 Mon Sep 17 00:00:00 2001 From: Harshit Aggarwal Date: Wed, 17 Aug 2022 18:10:34 +0530 Subject: [PATCH 2/5] init --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 241b4d8f..68bf5a37 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ org.opengroup.osdu core-lib-azure jar - 0.17.0-SNAPSHOT + 0.17.0-v1 core-lib-azure -- GitLab From 3b7c51424bf0a024f60ef694ecc6bf07b99b60be Mon Sep 17 00:00:00 2001 From: Harshit Aggarwal Date: Thu, 18 Aug 2022 07:09:19 +0530 Subject: [PATCH 3/5] minor fix in pom --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 68bf5a37..241b4d8f 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ org.opengroup.osdu core-lib-azure jar - 0.17.0-v1 + 0.17.0-SNAPSHOT core-lib-azure -- GitLab From 12664da96975583ed17d00264e3d9b6b2848c6ff Mon Sep 17 00:00:00 2001 From: Harshit Aggarwal Date: Thu, 18 Aug 2022 07:35:49 +0530 Subject: [PATCH 4/5] add ut --- .../DataLakeClientFactoryImplTest.java | 376 ++++++++------ .../azure/blobstorage/DataLakeStoreTest.java | 480 ++++++++++-------- 2 files changed, 467 insertions(+), 389 deletions(-) diff --git a/src/test/java/org/opengroup/osdu/azure/blobstorage/DataLakeClientFactoryImplTest.java b/src/test/java/org/opengroup/osdu/azure/blobstorage/DataLakeClientFactoryImplTest.java index 62bebc3d..0ab9f0f5 100644 --- a/src/test/java/org/opengroup/osdu/azure/blobstorage/DataLakeClientFactoryImplTest.java +++ b/src/test/java/org/opengroup/osdu/azure/blobstorage/DataLakeClientFactoryImplTest.java @@ -1,167 +1,209 @@ -// Copyright © Microsoft Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.opengroup.osdu.azure.blobstorage; - -import com.azure.identity.DefaultAzureCredential; -import com.azure.storage.common.policy.RequestRetryOptions; -import com.azure.storage.file.datalake.DataLakeDirectoryClient; -import com.azure.storage.file.datalake.DataLakeFileSystemClient; -import com.azure.storage.file.datalake.DataLakeServiceClient; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; -import org.mockito.junit.jupiter.MockitoExtension; -import org.opengroup.osdu.azure.datalakestorage.DataLakeClientFactoryImpl; -import org.opengroup.osdu.azure.di.BlobStoreRetryConfiguration; -import org.opengroup.osdu.azure.di.MSIConfiguration; -import org.opengroup.osdu.azure.partition.PartitionInfoAzure; -import org.opengroup.osdu.azure.partition.PartitionServiceClient; - -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.Mockito.lenient; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -public class DataLakeClientFactoryImplTest { - - private static final String PARTITION_ID = "dataPartitionId"; - private static final String DIRECTORY_NAME = "directoryName"; - private static final String FILE_SYSTEM_NAME = "fileSystemName"; - private static final String ACCOUNT_NAME = "accountName"; - private static final String ACCOUNT_KEY = "accountKey"; - - @Mock - private DataLakeServiceClient mockDataLakeServiceClient; - - @Mock - private DataLakeFileSystemClient mockDataLakeFileSystemClient; - - @Mock - private DataLakeDirectoryClient mockDataLakeDirectoryClient; - - @Mock - private PartitionServiceClient mockPartitionServiceClient; - - @Mock - private PartitionInfoAzure mockPartitionInfoAzure; - - @Mock - private Map dataLakeContainerClientMap; - - @Mock - private BlobStoreRetryConfiguration mockBlobStoreRetryConfiguration; - - @Mock - private RequestRetryOptions mockRequestRetryOptions; - - @Mock - private MSIConfiguration mockMsiConfiguration; - - @Mock - private DefaultAzureCredential mockDefaultAzureCredential; - - @InjectMocks - private DataLakeClientFactoryImpl dataLakeClientFactoryImpl; - - - @BeforeEach - void init() { - MockitoAnnotations.openMocks(this); - - lenient().when(mockDataLakeServiceClient.getFileSystemClient(FILE_SYSTEM_NAME)) - .thenReturn(mockDataLakeFileSystemClient); - lenient().when(mockDataLakeFileSystemClient.getDirectoryClient(DIRECTORY_NAME)) - .thenReturn(mockDataLakeDirectoryClient); - lenient().when(mockPartitionServiceClient.getPartition(PARTITION_ID)) - .thenReturn(mockPartitionInfoAzure); - lenient().when(mockPartitionInfoAzure.getHierarchicalStorageAccountName()) - .thenReturn(ACCOUNT_NAME); - lenient().when(mockPartitionInfoAzure.getHierarchicalStorageAccountKey()) - .thenReturn(ACCOUNT_KEY); - } - - @Test - public void getDataLakeDirectoryClient_Success() { - when(mockBlobStoreRetryConfiguration.getRequestRetryOptions()) - .thenReturn(mockRequestRetryOptions); - - DataLakeDirectoryClient dataLakeDirectoryClient = dataLakeClientFactoryImpl.getDataLakeDirectoryClient( - PARTITION_ID, DIRECTORY_NAME, FILE_SYSTEM_NAME); - - assertNotNull(dataLakeDirectoryClient); - String cacheKey = String.format("%s-%s", PARTITION_ID, FILE_SYSTEM_NAME); - verify(dataLakeContainerClientMap, times(0)).get(cacheKey); - } - - @Test - public void getDataLakeDirectoryClient_MsiEnabled() { - when(mockBlobStoreRetryConfiguration.getRequestRetryOptions()) - .thenReturn(mockRequestRetryOptions); - when(mockMsiConfiguration.getIsEnabled()).thenReturn(true); - - DataLakeDirectoryClient dataLakeDirectoryClient = dataLakeClientFactoryImpl.getDataLakeDirectoryClient( - PARTITION_ID, DIRECTORY_NAME, FILE_SYSTEM_NAME); - - assertNotNull(dataLakeDirectoryClient); - String cacheKey = String.format("%s-%s", PARTITION_ID, FILE_SYSTEM_NAME); - verify(dataLakeContainerClientMap, times(0)).get(cacheKey); - } - - @Test - public void getDataLakeDirectoryClient_should_throwException_given_nullDataPartitionId() { - try { - dataLakeClientFactoryImpl.getDataLakeDirectoryClient(null, DIRECTORY_NAME, FILE_SYSTEM_NAME); - } catch (NullPointerException ex) { - assertEquals("dataPartitionId cannot be null!", ex.getMessage()); - } catch (Exception ex) { - fail("Should not get any other exception. Received " + ex.getClass()); - } - } - - @Test - public void getDataLakeDirectoryClient_should_throwException_given_emptyDataPartitionId() { - try { - dataLakeClientFactoryImpl.getDataLakeDirectoryClient("", DIRECTORY_NAME, FILE_SYSTEM_NAME); - } catch (IllegalArgumentException ex) { - assertEquals("dataPartitionId cannot be empty!", ex.getMessage()); - } catch (Exception ex) { - fail("Should not get any other exception. Received " + ex.getClass()); - } - } - - @Test - public void getDataLakeDirectoryClient_should_return_cachedContainer_when_cachedEarlier() { - String cacheKey = String.format("%s-%s", PARTITION_ID, FILE_SYSTEM_NAME); - - when(this.dataLakeContainerClientMap.containsKey(cacheKey)).thenReturn(true); - when(this.dataLakeContainerClientMap.get(cacheKey)).thenReturn(mockDataLakeServiceClient); - - DataLakeDirectoryClient dataLakeDirectoryClient = dataLakeClientFactoryImpl.getDataLakeDirectoryClient( - PARTITION_ID, DIRECTORY_NAME, FILE_SYSTEM_NAME); - - assertNotNull(dataLakeDirectoryClient); - verify(this.dataLakeContainerClientMap).containsKey(cacheKey); - verify(this.dataLakeContainerClientMap).get(cacheKey); - } -} +// Copyright © Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.opengroup.osdu.azure.blobstorage; + +import com.azure.identity.DefaultAzureCredential; +import com.azure.storage.common.policy.RequestRetryOptions; +import com.azure.storage.file.datalake.DataLakeDirectoryClient; +import com.azure.storage.file.datalake.DataLakeFileSystemClient; +import com.azure.storage.file.datalake.DataLakeServiceClient; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opengroup.osdu.azure.datalakestorage.DataLakeClientFactoryImpl; +import org.opengroup.osdu.azure.di.BlobStoreRetryConfiguration; +import org.opengroup.osdu.azure.di.MSIConfiguration; +import org.opengroup.osdu.azure.partition.PartitionInfoAzure; +import org.opengroup.osdu.azure.partition.PartitionServiceClient; + +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class DataLakeClientFactoryImplTest { + + private static final String PARTITION_ID = "dataPartitionId"; + private static final String DIRECTORY_NAME = "directoryName"; + private static final String FILE_SYSTEM_NAME = "fileSystemName"; + private static final String ACCOUNT_NAME = "accountName"; + private static final String ACCOUNT_KEY = "accountKey"; + + @Mock + private DataLakeServiceClient mockDataLakeServiceClient; + + @Mock + private DataLakeFileSystemClient mockDataLakeFileSystemClient; + + @Mock + private DataLakeDirectoryClient mockDataLakeDirectoryClient; + + @Mock + private PartitionServiceClient mockPartitionServiceClient; + + @Mock + private PartitionInfoAzure mockPartitionInfoAzure; + + @Mock + private Map dataLakeContainerClientMap; + + @Mock + private BlobStoreRetryConfiguration mockBlobStoreRetryConfiguration; + + @Mock + private RequestRetryOptions mockRequestRetryOptions; + + @Mock + private MSIConfiguration mockMsiConfiguration; + + @Mock + private DefaultAzureCredential mockDefaultAzureCredential; + + @InjectMocks + private DataLakeClientFactoryImpl dataLakeClientFactoryImpl; + + + @BeforeEach + void init() { + MockitoAnnotations.openMocks(this); + + lenient().when(mockDataLakeServiceClient.getFileSystemClient(FILE_SYSTEM_NAME)) + .thenReturn(mockDataLakeFileSystemClient); + lenient().when(mockDataLakeFileSystemClient.getDirectoryClient(DIRECTORY_NAME)) + .thenReturn(mockDataLakeDirectoryClient); + lenient().when(mockPartitionServiceClient.getPartition(PARTITION_ID)) + .thenReturn(mockPartitionInfoAzure); + lenient().when(mockPartitionInfoAzure.getHierarchicalStorageAccountName()) + .thenReturn(ACCOUNT_NAME); + lenient().when(mockPartitionInfoAzure.getHierarchicalStorageAccountKey()) + .thenReturn(ACCOUNT_KEY); + } + + @Test + public void getDataLakeDirectoryClient_Success() { + when(mockBlobStoreRetryConfiguration.getRequestRetryOptions()) + .thenReturn(mockRequestRetryOptions); + + DataLakeDirectoryClient dataLakeDirectoryClient = dataLakeClientFactoryImpl.getDataLakeDirectoryClient( + PARTITION_ID, DIRECTORY_NAME, FILE_SYSTEM_NAME); + + assertNotNull(dataLakeDirectoryClient); + String cacheKey = String.format("%s-%s", PARTITION_ID, FILE_SYSTEM_NAME); + verify(dataLakeContainerClientMap, times(0)).get(cacheKey); + } + + @Test + public void getDataLakeDirectoryClient_MsiEnabled() { + when(mockBlobStoreRetryConfiguration.getRequestRetryOptions()) + .thenReturn(mockRequestRetryOptions); + when(mockMsiConfiguration.getIsEnabled()).thenReturn(true); + + DataLakeDirectoryClient dataLakeDirectoryClient = dataLakeClientFactoryImpl.getDataLakeDirectoryClient( + PARTITION_ID, DIRECTORY_NAME, FILE_SYSTEM_NAME); + + assertNotNull(dataLakeDirectoryClient); + String cacheKey = String.format("%s-%s", PARTITION_ID, FILE_SYSTEM_NAME); + verify(dataLakeContainerClientMap, times(0)).get(cacheKey); + } + + @Test + public void getDataLakeDirectoryClient_should_throwException_given_nullDataPartitionId() { + try { + dataLakeClientFactoryImpl.getDataLakeDirectoryClient(null, DIRECTORY_NAME, FILE_SYSTEM_NAME); + } catch (NullPointerException ex) { + assertEquals("dataPartitionId cannot be null!", ex.getMessage()); + } catch (Exception ex) { + fail("Should not get any other exception. Received " + ex.getClass()); + } + } + + @Test + public void getDataLakeDirectoryClient_should_throwException_given_emptyDataPartitionId() { + try { + dataLakeClientFactoryImpl.getDataLakeDirectoryClient("", DIRECTORY_NAME, FILE_SYSTEM_NAME); + } catch (IllegalArgumentException ex) { + assertEquals("dataPartitionId cannot be empty!", ex.getMessage()); + } catch (Exception ex) { + fail("Should not get any other exception. Received " + ex.getClass()); + } + } + + @Test + public void getDataLakeDirectoryClient_should_return_cachedContainer_when_cachedEarlier() { + String cacheKey = String.format("%s-%s", PARTITION_ID, FILE_SYSTEM_NAME); + + when(this.dataLakeContainerClientMap.containsKey(cacheKey)).thenReturn(true); + when(this.dataLakeContainerClientMap.get(cacheKey)).thenReturn(mockDataLakeServiceClient); + + DataLakeDirectoryClient dataLakeDirectoryClient = dataLakeClientFactoryImpl.getDataLakeDirectoryClient( + PARTITION_ID, DIRECTORY_NAME, FILE_SYSTEM_NAME); + + assertNotNull(dataLakeDirectoryClient); + verify(this.dataLakeContainerClientMap).containsKey(cacheKey); + verify(this.dataLakeContainerClientMap).get(cacheKey); + } + + @Test + public void getDataLakeServiceClient_should_return_cachedContainer_when_cachedEarlier() { + String cacheKey = String.format("%s-%s", PARTITION_ID, FILE_SYSTEM_NAME); + + when(this.dataLakeContainerClientMap.containsKey(cacheKey)).thenReturn(true); + when(this.dataLakeContainerClientMap.get(cacheKey)).thenReturn(mockDataLakeServiceClient); + + DataLakeServiceClient dataLakeServiceClient = dataLakeClientFactoryImpl.getDataLakeServiceClient( + PARTITION_ID, FILE_SYSTEM_NAME); + + assertNotNull(dataLakeServiceClient); + verify(this.dataLakeContainerClientMap).containsKey(cacheKey); + verify(this.dataLakeContainerClientMap).get(cacheKey); + } + + @Test + public void getDataLakeServiceClient_MsiEnabled() { + when(mockBlobStoreRetryConfiguration.getRequestRetryOptions()) + .thenReturn(mockRequestRetryOptions); + when(mockMsiConfiguration.getIsEnabled()).thenReturn(true); + + DataLakeServiceClient dataLakeServiceClient = dataLakeClientFactoryImpl.getDataLakeServiceClient( + PARTITION_ID, FILE_SYSTEM_NAME); + + assertNotNull(dataLakeServiceClient); + String cacheKey = String.format("%s-%s", PARTITION_ID, FILE_SYSTEM_NAME); + verify(dataLakeContainerClientMap, times(0)).get(cacheKey); + } + + @Test + public void getDataLakeServiceClient_Success() { + when(mockBlobStoreRetryConfiguration.getRequestRetryOptions()) + .thenReturn(mockRequestRetryOptions); + + DataLakeServiceClient dataLakeServiceClient = dataLakeClientFactoryImpl.getDataLakeServiceClient( + PARTITION_ID, FILE_SYSTEM_NAME); + + assertNotNull(dataLakeServiceClient); + String cacheKey = String.format("%s-%s", PARTITION_ID, FILE_SYSTEM_NAME); + verify(dataLakeContainerClientMap, times(0)).get(cacheKey); + } +} diff --git a/src/test/java/org/opengroup/osdu/azure/blobstorage/DataLakeStoreTest.java b/src/test/java/org/opengroup/osdu/azure/blobstorage/DataLakeStoreTest.java index 72076f17..a56b5afe 100644 --- a/src/test/java/org/opengroup/osdu/azure/blobstorage/DataLakeStoreTest.java +++ b/src/test/java/org/opengroup/osdu/azure/blobstorage/DataLakeStoreTest.java @@ -1,222 +1,258 @@ -// Copyright © Microsoft Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package org.opengroup.osdu.azure.blobstorage; - -import com.azure.storage.file.datalake.DataLakeDirectoryClient; -import com.azure.storage.file.datalake.models.DataLakeStorageException; -import com.azure.storage.file.datalake.models.PathInfo; -import com.azure.storage.file.datalake.sas.DataLakeServiceSasSignatureValues; -import com.azure.storage.file.datalake.sas.FileSystemSasPermission; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.ArgumentCaptor; -import org.mockito.Captor; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; -import org.mockito.junit.jupiter.MockitoExtension; -import org.opengroup.osdu.azure.datalakestorage.DataLakeStore; -import org.opengroup.osdu.azure.datalakestorage.IDataLakeClientFactory; -import org.opengroup.osdu.azure.logging.CoreLogger; -import org.opengroup.osdu.azure.logging.CoreLoggerFactory; -import org.opengroup.osdu.core.common.logging.ILogger; -import org.opengroup.osdu.core.common.model.http.AppException; - -import java.lang.reflect.Field; -import java.time.OffsetDateTime; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyMap; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.lenient; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -public class DataLakeStoreTest { - - private static final String PARTITION_ID = "dataPartitionId"; - private static final String DIRECTORY_NAME = "directoryName"; - private static final String FILE_SYSTEM_NAME = "fileSystemName"; - private static final String DESTINATION_FILE_SYSTEM = "destinationSystemName"; - - @Mock - private CoreLoggerFactory coreLoggerFactory; - - @Mock - private IDataLakeClientFactory dataLakeClientFactory; - - @Mock - private DataLakeDirectoryClient dataLakeDirectoryClient; - - @InjectMocks - private DataLakeStore dataLakeStore; - - @InjectMocks - private PathInfo pathInfo; - - @Mock - private CoreLogger coreLogger; - - @Captor - private ArgumentCaptor dataLakeServiceSasSignatureValuesCaptor; - - @BeforeEach - void init() { - MockitoAnnotations.openMocks(this); - - mockSingleton(coreLoggerFactory); - lenient().when(coreLoggerFactory.getLogger(anyString())).thenReturn(coreLogger); - lenient().when(dataLakeClientFactory.getDataLakeDirectoryClient( - PARTITION_ID,DIRECTORY_NAME, FILE_SYSTEM_NAME)).thenReturn(dataLakeDirectoryClient); - } - - @AfterEach - public void takeDown() { - resetSingleton(); - } - - /** - * Workaround for inability to mock static methods like getInstance(). - * - * @param mock CoreLoggerFactory mock instance - */ - private void mockSingleton(CoreLoggerFactory mock) { - try { - Field instance = CoreLoggerFactory.class.getDeclaredField("instance"); - instance.setAccessible(true); - instance.set(null, mock); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - /** - * Reset workaround for inability to mock static methods like getInstance(). - */ - private void resetSingleton() { - try { - Field instance = CoreLoggerFactory.class.getDeclaredField("instance"); - instance.setAccessible(true); - instance.set(null, null); - instance.setAccessible(false); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - @Test - public void createDirectory_Success(){ - - when(dataLakeDirectoryClient.create()).thenReturn(pathInfo); - dataLakeStore.createDirectory(PARTITION_ID, FILE_SYSTEM_NAME,DIRECTORY_NAME); - - verify(dataLakeClientFactory).getDataLakeDirectoryClient(PARTITION_ID, DIRECTORY_NAME, FILE_SYSTEM_NAME); - verify(dataLakeDirectoryClient).create(); - } - - @Test - public void createDirectory_AppException(){ - doThrow(DataLakeStorageException.class).when(dataLakeClientFactory).getDataLakeDirectoryClient(PARTITION_ID,DIRECTORY_NAME, FILE_SYSTEM_NAME); - try { - dataLakeStore.createDirectory(PARTITION_ID, FILE_SYSTEM_NAME, DIRECTORY_NAME); - } catch (AppException exception) { - assertEquals(500, exception.getError().getCode()); - } catch (Exception exception) { - fail("should not get different error code"); - } - } - - @Test - public void generatePreSignedURL_ReturnsValidSasToken() { - String containerSasToken = "containerSasToken"; - String containerUrl = "containerUrl"; - String containerPreSignedUrl = String.format("%s?%s",containerUrl,containerSasToken); - - doReturn(containerUrl).when(dataLakeDirectoryClient).getDirectoryUrl(); - doReturn(DIRECTORY_NAME).when(dataLakeDirectoryClient).getDirectoryName(); - doReturn(containerSasToken).when(dataLakeDirectoryClient).generateSas(any(DataLakeServiceSasSignatureValues.class)); - - int expiryDays = 1; - OffsetDateTime expiryTime = OffsetDateTime.now().plusDays(expiryDays); - - FileSystemSasPermission fileSystemSasPermission = - (new FileSystemSasPermission()).setReadPermission(true).setCreatePermission(true); - - String obtainedPreSignedUrl = dataLakeStore.generatePreSignedURL(PARTITION_ID, FILE_SYSTEM_NAME, DIRECTORY_NAME, - expiryTime, fileSystemSasPermission); - - verify(dataLakeDirectoryClient).generateSas(dataLakeServiceSasSignatureValuesCaptor.capture()); - - assertEquals(fileSystemSasPermission.toString(), dataLakeServiceSasSignatureValuesCaptor.getValue().getPermissions()); - assertEquals(expiryTime, dataLakeServiceSasSignatureValuesCaptor.getValue().getExpiryTime()); - assertEquals(containerPreSignedUrl, obtainedPreSignedUrl); - } - - @Test - public void generatePreSignedURL_ThrowException() { - doThrow(DataLakeStorageException.class).when(dataLakeDirectoryClient).generateSas(any(DataLakeServiceSasSignatureValues.class)); - int expiryDays = 1; - OffsetDateTime expiryTime = OffsetDateTime.now().plusDays(expiryDays); - - FileSystemSasPermission fileSystemSasPermission = - (new FileSystemSasPermission()).setReadPermission(true).setCreatePermission(true); - - try { - String obtainedPreSignedUrl = dataLakeStore.generatePreSignedURL(PARTITION_ID, FILE_SYSTEM_NAME, DIRECTORY_NAME, - expiryTime, fileSystemSasPermission); - } catch (DataLakeStorageException ex) { - verify(dataLakeDirectoryClient, times(1)).generateSas(any(DataLakeServiceSasSignatureValues.class)); - } catch (Exception ex) { - fail("should not get different error code"); - } - } - - @Test - public void moveDirectory_success() { - DataLakeDirectoryClient mockDirectoryClient = mock(DataLakeDirectoryClient.class); - when(dataLakeDirectoryClient.rename(DESTINATION_FILE_SYSTEM, DIRECTORY_NAME)) - .thenReturn(mockDirectoryClient); - - DataLakeDirectoryClient directoryClient = dataLakeStore.moveDirectory(PARTITION_ID, - FILE_SYSTEM_NAME, DIRECTORY_NAME, DESTINATION_FILE_SYSTEM); - - assertEquals(mockDirectoryClient, directoryClient); - } - - @Test - public void moveDirectory_throwException() { - doThrow(DataLakeStorageException.class).when(dataLakeDirectoryClient) - .rename(DESTINATION_FILE_SYSTEM, DIRECTORY_NAME); - - try { - dataLakeStore.moveDirectory(PARTITION_ID, - FILE_SYSTEM_NAME, DIRECTORY_NAME, DESTINATION_FILE_SYSTEM); - } catch (DataLakeStorageException ex) { - verify(dataLakeDirectoryClient).rename(DESTINATION_FILE_SYSTEM, DIRECTORY_NAME); - } catch (Exception ex) { - fail("should not get different error"); - } - } - -} +// Copyright © Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package org.opengroup.osdu.azure.blobstorage; + +import com.azure.storage.file.datalake.DataLakeDirectoryClient; +import com.azure.storage.file.datalake.DataLakeServiceClient; +import com.azure.storage.file.datalake.models.DataLakeStorageException; +import com.azure.storage.file.datalake.models.PathInfo; +import com.azure.storage.file.datalake.models.UserDelegationKey; +import com.azure.storage.file.datalake.sas.DataLakeServiceSasSignatureValues; +import com.azure.storage.file.datalake.sas.FileSystemSasPermission; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opengroup.osdu.azure.datalakestorage.DataLakeStore; +import org.opengroup.osdu.azure.datalakestorage.IDataLakeClientFactory; +import org.opengroup.osdu.azure.logging.CoreLogger; +import org.opengroup.osdu.azure.logging.CoreLoggerFactory; +import org.opengroup.osdu.core.common.logging.ILogger; +import org.opengroup.osdu.core.common.model.http.AppException; + +import java.lang.reflect.Field; +import java.time.OffsetDateTime; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class DataLakeStoreTest { + + private static final String PARTITION_ID = "dataPartitionId"; + private static final String DIRECTORY_NAME = "directoryName"; + private static final String FILE_SYSTEM_NAME = "fileSystemName"; + private static final String DESTINATION_FILE_SYSTEM = "destinationSystemName"; + + @Mock + private CoreLoggerFactory coreLoggerFactory; + + @Mock + private IDataLakeClientFactory dataLakeClientFactory; + + @Mock + private DataLakeDirectoryClient dataLakeDirectoryClient; + + @InjectMocks + private DataLakeStore dataLakeStore; + + @InjectMocks + private PathInfo pathInfo; + + @Mock + private CoreLogger coreLogger; + + @Mock + private DataLakeServiceClient dataLakeServiceClient; + + @Mock + private UserDelegationKey userDelegationKey; + + @Captor + private ArgumentCaptor dataLakeServiceSasSignatureValuesCaptor; + + @BeforeEach + void init() { + MockitoAnnotations.openMocks(this); + + mockSingleton(coreLoggerFactory); + lenient().when(coreLoggerFactory.getLogger(anyString())).thenReturn(coreLogger); + lenient().when(dataLakeClientFactory.getDataLakeDirectoryClient( + PARTITION_ID,DIRECTORY_NAME, FILE_SYSTEM_NAME)).thenReturn(dataLakeDirectoryClient); + } + + @AfterEach + public void takeDown() { + resetSingleton(); + } + + /** + * Workaround for inability to mock static methods like getInstance(). + * + * @param mock CoreLoggerFactory mock instance + */ + private void mockSingleton(CoreLoggerFactory mock) { + try { + Field instance = CoreLoggerFactory.class.getDeclaredField("instance"); + instance.setAccessible(true); + instance.set(null, mock); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * Reset workaround for inability to mock static methods like getInstance(). + */ + private void resetSingleton() { + try { + Field instance = CoreLoggerFactory.class.getDeclaredField("instance"); + instance.setAccessible(true); + instance.set(null, null); + instance.setAccessible(false); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Test + public void createDirectory_Success(){ + + when(dataLakeDirectoryClient.create()).thenReturn(pathInfo); + dataLakeStore.createDirectory(PARTITION_ID, FILE_SYSTEM_NAME,DIRECTORY_NAME); + + verify(dataLakeClientFactory).getDataLakeDirectoryClient(PARTITION_ID, DIRECTORY_NAME, FILE_SYSTEM_NAME); + verify(dataLakeDirectoryClient).create(); + } + + @Test + public void createDirectory_AppException(){ + doThrow(DataLakeStorageException.class).when(dataLakeClientFactory).getDataLakeDirectoryClient(PARTITION_ID,DIRECTORY_NAME, FILE_SYSTEM_NAME); + try { + dataLakeStore.createDirectory(PARTITION_ID, FILE_SYSTEM_NAME, DIRECTORY_NAME); + } catch (AppException exception) { + assertEquals(500, exception.getError().getCode()); + } catch (Exception exception) { + fail("should not get different error code"); + } + } + + @Test + public void generatePreSignedURL_ReturnsValidSasToken() { + String containerSasToken = "containerSasToken"; + String containerUrl = "containerUrl"; + String containerPreSignedUrl = String.format("%s?%s",containerUrl,containerSasToken); + + doReturn(containerUrl).when(dataLakeDirectoryClient).getDirectoryUrl(); + doReturn(DIRECTORY_NAME).when(dataLakeDirectoryClient).getDirectoryName(); + doReturn(containerSasToken).when(dataLakeDirectoryClient).generateSas(any(DataLakeServiceSasSignatureValues.class)); + + int expiryDays = 1; + OffsetDateTime expiryTime = OffsetDateTime.now().plusDays(expiryDays); + + FileSystemSasPermission fileSystemSasPermission = + (new FileSystemSasPermission()).setReadPermission(true).setCreatePermission(true); + + String obtainedPreSignedUrl = dataLakeStore.generatePreSignedURL(PARTITION_ID, FILE_SYSTEM_NAME, DIRECTORY_NAME, + expiryTime, fileSystemSasPermission); + + verify(dataLakeDirectoryClient).generateSas(dataLakeServiceSasSignatureValuesCaptor.capture()); + + assertEquals(fileSystemSasPermission.toString(), dataLakeServiceSasSignatureValuesCaptor.getValue().getPermissions()); + assertEquals(expiryTime, dataLakeServiceSasSignatureValuesCaptor.getValue().getExpiryTime()); + assertEquals(containerPreSignedUrl, obtainedPreSignedUrl); + } + + @Test + public void generatePreSignedURLWithUserDelegationSas_ReturnsValidSasToken() { + String containerSasToken = "containerSasToken"; + String containerUrl = "containerUrl"; + String containerPreSignedUrl = String.format("%s?%s",containerUrl, containerSasToken); + int expiryDays = 1; + OffsetDateTime expiryTime = OffsetDateTime.now().plusDays(expiryDays); + + ArgumentCaptor userDelegationKeyArgumentCaptor = ArgumentCaptor.forClass(UserDelegationKey.class); + + doReturn(containerUrl).when(dataLakeDirectoryClient).getDirectoryUrl(); + doReturn(DIRECTORY_NAME).when(dataLakeDirectoryClient).getDirectoryName(); + doReturn(containerSasToken).when(dataLakeDirectoryClient).generateUserDelegationSas(any(DataLakeServiceSasSignatureValues.class), any(UserDelegationKey.class)); + doReturn(dataLakeServiceClient).when(dataLakeClientFactory).getDataLakeServiceClient(PARTITION_ID, FILE_SYSTEM_NAME); + doReturn(userDelegationKey).when(dataLakeServiceClient).getUserDelegationKey(any(OffsetDateTime.class), any(OffsetDateTime.class)); + + FileSystemSasPermission fileSystemSasPermission = + (new FileSystemSasPermission()).setReadPermission(true).setCreatePermission(true); + + String obtainedPreSignedUrl = dataLakeStore.generatePreSignedURLWithUserDelegationSas(PARTITION_ID, FILE_SYSTEM_NAME, DIRECTORY_NAME, + expiryTime, fileSystemSasPermission); + + verify(dataLakeDirectoryClient).generateUserDelegationSas(dataLakeServiceSasSignatureValuesCaptor.capture(), userDelegationKeyArgumentCaptor.capture()); + + assertEquals(fileSystemSasPermission.toString(), dataLakeServiceSasSignatureValuesCaptor.getValue().getPermissions()); + assertEquals(containerPreSignedUrl, obtainedPreSignedUrl); + } + + @Test + public void generatePreSignedURL_ThrowException() { + doThrow(DataLakeStorageException.class).when(dataLakeDirectoryClient).generateSas(any(DataLakeServiceSasSignatureValues.class)); + int expiryDays = 1; + OffsetDateTime expiryTime = OffsetDateTime.now().plusDays(expiryDays); + + FileSystemSasPermission fileSystemSasPermission = + (new FileSystemSasPermission()).setReadPermission(true).setCreatePermission(true); + + try { + String obtainedPreSignedUrl = dataLakeStore.generatePreSignedURL(PARTITION_ID, FILE_SYSTEM_NAME, DIRECTORY_NAME, + expiryTime, fileSystemSasPermission); + } catch (DataLakeStorageException ex) { + verify(dataLakeDirectoryClient, times(1)).generateSas(any(DataLakeServiceSasSignatureValues.class)); + } catch (Exception ex) { + fail("should not get different error code"); + } + } + + @Test + public void moveDirectory_success() { + DataLakeDirectoryClient mockDirectoryClient = mock(DataLakeDirectoryClient.class); + when(dataLakeDirectoryClient.rename(DESTINATION_FILE_SYSTEM, DIRECTORY_NAME)) + .thenReturn(mockDirectoryClient); + + DataLakeDirectoryClient directoryClient = dataLakeStore.moveDirectory(PARTITION_ID, + FILE_SYSTEM_NAME, DIRECTORY_NAME, DESTINATION_FILE_SYSTEM); + + assertEquals(mockDirectoryClient, directoryClient); + } + + @Test + public void moveDirectory_throwException() { + doThrow(DataLakeStorageException.class).when(dataLakeDirectoryClient) + .rename(DESTINATION_FILE_SYSTEM, DIRECTORY_NAME); + + try { + dataLakeStore.moveDirectory(PARTITION_ID, + FILE_SYSTEM_NAME, DIRECTORY_NAME, DESTINATION_FILE_SYSTEM); + } catch (DataLakeStorageException ex) { + verify(dataLakeDirectoryClient).rename(DESTINATION_FILE_SYSTEM, DIRECTORY_NAME); + } catch (Exception ex) { + fail("should not get different error"); + } + } + +} -- GitLab From 8705ca1c54990dde5ae8fd2e9eebc5b632e0cdf1 Mon Sep 17 00:00:00 2001 From: Harshit Aggarwal Date: Thu, 18 Aug 2022 10:57:18 +0530 Subject: [PATCH 5/5] PR comments --- .../opengroup/osdu/azure/datalakestorage/DataLakeStore.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/opengroup/osdu/azure/datalakestorage/DataLakeStore.java b/src/main/java/org/opengroup/osdu/azure/datalakestorage/DataLakeStore.java index 1a016acb..bf25611b 100644 --- a/src/main/java/org/opengroup/osdu/azure/datalakestorage/DataLakeStore.java +++ b/src/main/java/org/opengroup/osdu/azure/datalakestorage/DataLakeStore.java @@ -111,7 +111,7 @@ public class DataLakeStore { final long start = System.currentTimeMillis(); String sasToken = client.generateSas(sign); final long timeTaken = System.currentTimeMillis() - start; - logDependency("GENERATE_SAS_TOKEN", client.getDirectoryName(), client.getDirectoryUrl(), timeTaken, String.valueOf(HttpStatus.SC_OK), true); + logDependency("GENERATE_SAS_TOKEN_DATALAKE", client.getDirectoryName(), client.getDirectoryUrl(), timeTaken, String.valueOf(HttpStatus.SC_OK), true); return sasToken; } @@ -139,7 +139,7 @@ public class DataLakeStore { String sasToken = client.generateUserDelegationSas(sign, userDelegationKey); final long timeTaken = System.currentTimeMillis() - start; - logDependency("GENERATE_SAS_TOKEN", client.getDirectoryName(), client.getDirectoryUrl(), timeTaken, String.valueOf(HttpStatus.SC_OK), true); + logDependency("GENERATE_SAS_TOKEN_DATALAKE", client.getDirectoryName(), client.getDirectoryUrl(), timeTaken, String.valueOf(HttpStatus.SC_OK), true); return sasToken; } -- GitLab