Commit 35e7574d authored by Aman Verma's avatar Aman Verma
Browse files

Merge remote-tracking branch 'origin/master' into users/amaverma/updatedMRTemplate

parents 3d93bef8 1b54a891
Pipeline #9067 passed with stages
in 5 minutes and 58 seconds
......@@ -74,8 +74,16 @@ Note: Below are reference PRs for exclusion and might change from service to ser
Refer this [MR](https://community.opengroup.org/osdu/platform/security-and-compliance/entitlements-azure/-/merge_requests/13) as reference on how to exclude dependencies along with how to enable the
Enabled transaction logger and slf4jlogger
## Environment variables to be added in application.properties to consume the TenantFactoryImpl
| name | value | description |
| --- | --- | --- |
| `tenantInfo.container.name` | `TenantInfo` | cosmos container name |
| `azure.cosmosdb.database` | ex `dev-osdu-r2-db` | cosmos databse name |
\ No newline at end of file
| `azure.cosmosdb.database` | ex `dev-osdu-r2-db` | cosmos database name |
| `tenantFactoryImpl.required` | ex `true` | Set this property to true in order to consume TenantFactoryImpl class from core-lib-azure |
## Settings to be added in application.properties to consume the BlobStore
| name | value | description |
| --- | --- | --- |
| `azure.blobStore.required` | `true` | - |
| `azure.storage.account-name` | ex `testStorage` | storage account name |
......@@ -21,7 +21,7 @@
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-azure</artifactId>
<packaging>jar</packaging>
<version>0.0.18</version>
<version>0.0.28</version>
<name>core-lib-azure</name>
<properties>
......
......@@ -39,8 +39,8 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* A simpler interface for interacting with CosmosDB.
......@@ -82,7 +82,7 @@ import java.util.logging.Logger;
@Lazy
public class CosmosStore {
private static final Logger LOGGER = Logger.getLogger(CosmosStore.class.getName());
private static final Logger LOGGER = LoggerFactory.getLogger(CosmosStore.class.getName());
@Autowired
private ICosmosClientFactory cosmosClientFactory;
......@@ -105,11 +105,11 @@ public class CosmosStore {
findItem(cosmosContainer, id, partitionKey).delete(new CosmosItemRequestOptions(partitionKey));
} catch (NotFoundException e) {
String errorMessage = "Item was unexpectedly not found";
LOGGER.log(Level.WARNING, errorMessage, e);
LOGGER.warn(errorMessage, e);
throw new AppException(404, errorMessage, e.getMessage(), e);
} catch (CosmosClientException e) {
String errorMessage = "Unexpectedly failed to delete item from CosmosDB";
LOGGER.log(Level.WARNING, errorMessage, e);
LOGGER.warn(errorMessage, e);
throw new AppException(500, errorMessage, e.getMessage(), e);
}
}
......@@ -139,14 +139,16 @@ public class CosmosStore {
.getObject(clazz);
return Optional.ofNullable(item);
} catch (NotFoundException e) {
LOGGER.info(String.format("Unable to find item with ID=%s and PK=%s", id, partitionKey));
LOGGER.warn(String.format("Unable to find item with ID=%s and PK=%s", id, partitionKey));
return Optional.empty();
} catch (IOException e) {
LOGGER.warning(String.format("Malformed document for item with ID=%s and PK=%s", id, partitionKey));
return Optional.empty();
} catch (CosmosClientException e) {
String errorMessage = "Unexpectedly encountered error calling CosmosDB";
LOGGER.log(Level.WARNING, errorMessage, e);
} catch (IOException | CosmosClientException e) {
String errorMessage;
if (e instanceof IOException) {
errorMessage = String.format("Malformed document for item with ID=%s and PK=%s", id, partitionKey);
} else {
errorMessage = "Unexpectedly encountered error calling CosmosDB";
}
LOGGER.warn(errorMessage, e);
throw new AppException(500, errorMessage, e.getMessage(), e);
}
}
......@@ -193,7 +195,7 @@ public class CosmosStore {
results.add(properties.getObject(clazz));
} catch (IOException e) {
String errorMessage = String.format("Malformed document for item with ID=%s", properties.getId());
LOGGER.log(Level.WARNING, errorMessage, e);
LOGGER.warn(errorMessage, e);
throw new AppException(500, errorMessage, e.getMessage(), e);
}
}
......@@ -316,7 +318,7 @@ public class CosmosStore {
cosmosContainer.upsertItem(item);
} catch (CosmosClientException e) {
String errorMessage = "Unexpectedly failed to put item into CosmosDB";
LOGGER.log(Level.WARNING, errorMessage, e);
LOGGER.warn(errorMessage, e);
throw new AppException(500, errorMessage, e.getMessage(), e);
}
}
......@@ -338,11 +340,11 @@ public class CosmosStore {
cosmosContainer.createItem(item);
} catch (ConflictException e) {
String errorMessage = "Resource with specified id or name already exists.";
LOGGER.log(Level.WARNING, errorMessage, e);
LOGGER.warn(errorMessage, e);
throw new AppException(409, errorMessage, e.getMessage(), e);
} catch (CosmosClientException e) {
String errorMessage = "Unexpectedly failed to insert item into CosmosDB";
LOGGER.log(Level.WARNING, errorMessage, e);
LOGGER.warn(errorMessage, e);
throw new AppException(500, errorMessage, e.getMessage(), e);
}
}
......@@ -376,7 +378,7 @@ public class CosmosStore {
.getContainer(collection);
} catch (Exception e) {
String errorMessage = "Error creating creating Cosmos Client";
LOGGER.log(Level.WARNING, errorMessage, e);
LOGGER.warn(errorMessage, e);
throw new AppException(500, errorMessage, e.getMessage(), e);
}
......
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.azure.blobstorage;
import com.azure.identity.DefaultAzureCredential;
import com.azure.storage.blob.BlobServiceClient;
import com.azure.storage.blob.BlobServiceClientBuilder;
import org.opengroup.osdu.azure.di.BlobStoreConfiguration;
import org.opengroup.osdu.common.Validators;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
/**
* Implementation for IBlobServiceClientFactory.
*/
@Component
@ConditionalOnProperty(value = "azure.blobStore.required", havingValue = "true", matchIfMissing = false)
public class BlobServiceClientFactoryImpl implements IBlobServiceClientFactory {
private BlobServiceClient blobServiceClient;
/**
* Constructor to initialize blobServiceClient.
* @param defaultAzureCredential Default azure credentials.
* @param blobStoreConfiguration Configuration details for blob storage.
*/
@Autowired
public BlobServiceClientFactoryImpl(
final DefaultAzureCredential defaultAzureCredential,
final BlobStoreConfiguration blobStoreConfiguration) {
Validators.checkNotNull(defaultAzureCredential, "Default credentials");
Validators.checkNotNullAndNotEmpty(blobStoreConfiguration.getStorageAccountName(), "Storage account name cannot be null");
String endpoint = String.format("https://%s.blob.core.windows.net", blobStoreConfiguration.getStorageAccountName());
blobServiceClient = new BlobServiceClientBuilder()
.endpoint(endpoint)
.credential(defaultAzureCredential)
.buildClient();
}
/**
* @param dataPartitionId data partition id.
* @return BlobServiceClient corresponding to the given data partition id.
*/
@Override
public BlobServiceClient getBlobServiceClient(final String dataPartitionId) {
return blobServiceClient;
}
}
......@@ -15,13 +15,14 @@
package org.opengroup.osdu.azure.blobstorage;
import com.azure.storage.blob.BlobContainerClient;
import com.azure.storage.blob.BlobServiceClient;
import com.azure.storage.blob.models.BlobErrorCode;
import com.azure.storage.blob.models.BlobStorageException;
import com.azure.storage.blob.specialized.BlockBlobClient;
import org.opengroup.osdu.core.common.logging.ILogger;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
import java.io.ByteArrayOutputStream;
......@@ -39,32 +40,32 @@ import java.util.Collections;;
* @Autowired
* private BlobStore blobStore;
*
* String readFromBlobExample()
* String readFromStorageContainerExamole()
* {
* String content = blobStorage.readFromBlob("dataPartitionId", "filePath");
* if (content != null)
* String content = blobStorage.readFromStorageContainer("dataPartitionId", "filePath", "containerName");
* if (content != null)
* return content;
* }
*
* void writeToBlobExample()
* void writeToStorageContainerExample()
* {
* blobStorage.writeToBlob("dataPartitionId", "filePath", "content");
* blobStorage.writeToStorageContainer("dataPartitionId", "filePath", "content", "containerName");
* }
*
* void deleteFromBlobExample()
* void deleteFromStorageContainerExample()
* {
* Boolean success = blobStorage.deleteFromBlob("dataPartitionId", "filePath");
* Boolean success = blobStorage.deleteFromStorageContainer("dataPartitionId", "filePath", "containerName");
* }
* }
* </pre>
*/
@Component
@Lazy
@ConditionalOnProperty(value = "azure.blobStore.required", havingValue = "true", matchIfMissing = false)
public class BlobStore {
@Autowired
private IBlobContainerClientFactory blobContainerClientFactory;
private IBlobServiceClientFactory blobServiceClientFactory;
@Autowired
private ILogger logger;
......@@ -75,10 +76,14 @@ public class BlobStore {
*
* @param filePath Path of file to be read.
* @param dataPartitionId Data partition id
* @param containerName Name of the storage container
* @return the content of file with provided file path.
*/
public String readFromBlob(final String dataPartitionId, final String filePath) {
BlobContainerClient blobContainerClient = getBlobContainerClient(dataPartitionId);
public String readFromStorageContainer(
final String dataPartitionId,
final String filePath,
final String containerName) {
BlobContainerClient blobContainerClient = getBlobContainerClient(dataPartitionId, containerName);
BlockBlobClient blockBlobClient = blobContainerClient.getBlobClient(filePath).getBlockBlobClient();
try (ByteArrayOutputStream downloadStream = new ByteArrayOutputStream()) {
blockBlobClient.download(downloadStream);
......@@ -108,10 +113,14 @@ public class BlobStore {
*
* @param filePath Path of file to be deleted.
* @param dataPartitionId Data partition id
* @param containerName Name of the storage container
* @return boolean indicating whether the deletion of given file was successful or not.
*/
public boolean deleteFromBlob(final String dataPartitionId, final String filePath) {
BlobContainerClient blobContainerClient = getBlobContainerClient(dataPartitionId);
public boolean deleteFromStorageContainer(
final String dataPartitionId,
final String filePath,
final String containerName) {
BlobContainerClient blobContainerClient = getBlobContainerClient(dataPartitionId, containerName);
BlockBlobClient blockBlobClient = blobContainerClient.getBlobClient(filePath).getBlockBlobClient();
try {
blockBlobClient.delete();
......@@ -134,13 +143,16 @@ public class BlobStore {
* @param filePath Path of file to be written at.
* @param content Content to be written in the file.
* @param dataPartitionId Data partition id
* @param containerName Name of the storage container
*/
public void writeToBlob(final String dataPartitionId,
final String filePath,
final String content) {
public void writeToStorageContainer(
final String dataPartitionId,
final String filePath,
final String content,
final String containerName) {
byte[] bytes = content.getBytes(StandardCharsets.UTF_8);
int bytesSize = bytes.length;
BlobContainerClient blobContainerClient = getBlobContainerClient(dataPartitionId);
BlobContainerClient blobContainerClient = getBlobContainerClient(dataPartitionId, containerName);
BlockBlobClient blockBlobClient = blobContainerClient.getBlobClient(filePath).getBlockBlobClient();
try (ByteArrayInputStream dataStream = new ByteArrayInputStream(bytes)) {
blockBlobClient.upload(dataStream, bytesSize, true);
......@@ -157,17 +169,19 @@ public class BlobStore {
/**
*
* @param dataPartitionId Data partition id
* @param dataPartitionId Data partition id.
* @param containerName Name of storage container.
* @return blob container client corresponding to the dataPartitionId.
*/
private BlobContainerClient getBlobContainerClient(final String dataPartitionId) {
private BlobContainerClient getBlobContainerClient(final String dataPartitionId, final String containerName) {
try {
return blobContainerClientFactory.getClient(dataPartitionId);
BlobServiceClient serviceClient = blobServiceClientFactory.getBlobServiceClient(dataPartitionId);
return serviceClient.getBlobContainerClient(containerName);
} catch (Exception ex) {
String errorMessage = "Error creating creating blob container client.";
logger.warning(LOG_PREFIX, errorMessage, Collections.<String, String>emptyMap());
throw new AppException(500, errorMessage, ex.getMessage(), ex);
}
}
}
}
......@@ -14,17 +14,17 @@
package org.opengroup.osdu.azure.blobstorage;
import com.azure.storage.blob.BlobContainerClient;
import com.azure.storage.blob.BlobServiceClient;
/**
* Interface for BlobContainer client factory to return appropriate
* blobContainerClient based on the data partition id.
* Interface for Blob service client factory to return appropriate
* blobServiceClient based on the data partition id.
*/
public interface IBlobContainerClientFactory {
public interface IBlobServiceClientFactory {
/**
*
* @param dataPartitionId Data partition id
* @return blobContainerClient for given data partition id.
* @param dataPartitionId data partition id.
* @return BlobServiceClient corresponding to the given data partition id.
*/
BlobContainerClient getClient(String dataPartitionId);
BlobServiceClient getBlobServiceClient(String dataPartitionId);
}
......@@ -12,30 +12,21 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.azure.blobstorage;
package org.opengroup.osdu.azure.di;
import com.azure.storage.blob.BlobContainerClient;
import org.springframework.beans.factory.annotation.Autowired;
import lombok.Getter;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
/**
* Implementation for IBlobContainerClientFactory.
* A configuration bean class to set up blob store related variables.
*/
@Component
@Configuration
@Getter
@Lazy
public class BlobContainerClientFactoryImpl implements IBlobContainerClientFactory {
public class BlobStoreConfiguration {
@Autowired
private BlobContainerClient blobContainerClient;
/**
*
* @param dataPartitionId Data partition id
* @return the blob container client instance.
*/
@Override
public BlobContainerClient getClient(final String dataPartitionId) {
return blobContainerClient;
}
@Value("${azure.storage.account-name}")
private String storageAccountName;
}
package org.opengroup.osdu.azure.filters;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.slf4j.MDC;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
import javax.servlet.Filter;
import javax.servlet.FilterChain;
import javax.servlet.ServletException;
import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
/**
* MDC Filter for logging.
*/
@Component
@ConditionalOnProperty(value = "logging.mdccontext.enabled", havingValue = "true", matchIfMissing = false)
public class Slf4jMDCFilter implements Filter {
@Autowired
private DpsHeaders dpsHeaders;
/**
* Filter logic.
* @param servletRequest Request object.
* @param servletResponse Response object.
* @param filterChain Filter Chain object.
* @throws IOException
* @throws ServletException
*/
@Override
public void doFilter(final ServletRequest servletRequest, final ServletResponse servletResponse,
final FilterChain filterChain) throws IOException, ServletException {
MDC.setContextMap(getContextMap());
filterChain.doFilter(servletRequest, servletResponse);
MDC.clear();
}
/**
* Method to create context map for mdc.
* @return Context map.
*/
private Map<String, String> getContextMap() {
final Map<String, String> contextMap = new HashMap<>();
contextMap.put(DpsHeaders.CORRELATION_ID, dpsHeaders.getCorrelationId());
contextMap.put(DpsHeaders.DATA_PARTITION_ID, dpsHeaders.getPartitionId());
return contextMap;
}
}
......@@ -21,6 +21,7 @@ import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
......@@ -32,7 +33,7 @@ import java.util.Collection;
* Implementation for ITenantFactory.
*/
@Component
@Lazy
@ConditionalOnProperty(value = "tenantFactoryImpl.required", havingValue = "true", matchIfMissing = false)
public class TenantFactoryImpl implements ITenantFactory {
@Autowired
......@@ -112,6 +113,7 @@ public class TenantFactoryImpl implements ITenantFactory {
ti.setName(tenantName);
ti.setComplianceRuleSet(tenantInfoDoc.getComplianceRuleSet());
ti.setDataPartitionId(tenantName);
ti.setServiceAccount(tenantInfoDoc.getServiceprincipalAppId());
this.tenants.put(tenantName, ti);
}
);
......
......@@ -18,12 +18,12 @@
<Configuration packages="com.microsoft.applicationinsights.log4j.v2">
<Properties>
<Property name="LOG_PATTERN">
%d{yyyy-MM-dd HH:mm:ss.SSS} %5p ${hostName} --- [%15.15t] %-40.40c{1.} : %m%n%ex
%d{yyyy-MM-dd HH:mm:ss.SSS} %5p ${hostName} --- [%15.15t] %-40.40c{1.} correlation-id=%X{correlation-id} data-partition-id=%X{data-partition-id}: %m%n%ex
</Property>
</Properties>
<Appenders>
<Console name="Console" target="SYSTEM_OUT">
<PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} %5p ${hostName} --- [%15.15t] %-40.40c{1.} : %m%n%ex"/>
<PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} %5p ${hostName} --- [%15.15t] %-40.40c{1.} correlation-id=%X{correlation-id} data-partition-id=%X{data-partition-id}: %m%n%ex"/>
</Console>
<ApplicationInsightsAppender name="aiAppender">
</ApplicationInsightsAppender>
......
......@@ -137,9 +137,12 @@ class CosmosStoreTest {
}
@Test
void findItem_returnsEmpty_ifMalformedDocument() throws IOException {
void findItem_throws500_ifMalformedDocument() throws IOException {
doThrow(IOException.class).when(cosmosItemProperties).getObject(any());
assertFalse(cosmosStore.findItem(DATA_PARTITION_ID, COSMOS_DB, COLLECTION, ID, PARTITION_KEY, String.class).isPresent());
AppException exception = assertThrows(AppException.class, () -> {
cosmosStore.findItem(DATA_PARTITION_ID, COSMOS_DB, COLLECTION, ID, PARTITION_KEY, String.class);
});
assertEquals(500, exception.getError().getCode());
}
@Test
......
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.azure.blobstorage;
import com.azure.identity.DefaultAzureCredential;
import com.azure.storage.blob.BlobServiceClient;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.opengroup.osdu.azure.di.BlobStoreConfiguration;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.fail;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.lenient;
import static org.mockito.MockitoAnnotations.initMocks;
@ExtendWith(MockitoExtension.class)
public class BlobServiceClientFactoryImplTest {
@Mock
DefaultAzureCredential credential;
@Mock
BlobStoreConfiguration configuration;
BlobServiceClientFactoryImpl clientFactory;
private static final String ACCOUNT_NAME = "testAccount";
private static final String PARTITION_ID = "dataPartitionId";
@BeforeEach
void init() {
initMocks(this);
lenient().doReturn(ACCOUNT_NAME).when(configuration).getStorageAccountName();
}
@Test
public void ConstructorThrowsException_IfDefaultAzureCredentialIsNull() {
try {
clientFactory = new BlobServiceClientFactoryImpl(null, configuration);
} catch (NullPointerException ex) {
assertEquals("Default credentials cannot be null!", ex.getMessage());
} catch (Exception ex) {
fail("Should not get any other exception. Received " + ex.getClass());
}
}
@Test
public void ConstructorThrowsException_IfBlobStoreConfigurationIsNull() {
doReturn("").when(configuration).getStorageAccountName();
try {
clientFactory = new BlobServiceClientFactoryImpl(credential, configuration);
} catch (IllegalArgumentException ex) {
assertEquals("Storage account name cannot be null cannot be empty!", ex.getMessage());
} catch (Exception ex) {
fail("Should not get any other exception. Received " + ex.getClass());
}