Commit cdd5029f authored by Dania Kodeih (Microsoft)'s avatar Dania Kodeih (Microsoft)
Browse files

Merge branch 'partition-svc' into 'master'

update client lib to support multi-tenancy

See merge request osdu/platform/system/lib/cloud/azure/os-core-lib-azure!24
parents 8683e419 8651999d
......@@ -19,6 +19,7 @@ The following software have components provided under the terms of this license:
- Apache Commons Validator (from http://commons.apache.org/proper/commons-validator/)
- Apache HttpAsyncClient (from http://hc.apache.org/httpcomponents-asyncclient)
- Apache HttpClient (from http://hc.apache.org/httpcomponents-client)
- Apache HttpClient Cache (from http://hc.apache.org/httpcomponents-client)
- Apache HttpCore (from http://hc.apache.org/httpcomponents-core-ga)
- Apache HttpCore NIO (from http://hc.apache.org/httpcomponents-core-ga)
- Apache Log4j API (from )
......@@ -342,7 +343,6 @@ The following software have components provided under the terms of this license:
- Azure Spring Boot AutoConfigure (from https://github.com/Microsoft/azure-spring-boot)
- Azure Spring Boot Starter (from https://github.com/Microsoft/azure-spring-boot)
- Checker Qual (from https://checkerframework.org)
- Elasticsearch: 5.0.0-alpha5 (from https://github.com/elastic/elasticsearch)
- Extensions on Apache Proton-J library (from https://github.com/Azure/qpid-proton-j-extensions)
- JOpt Simple (from http://pholser.github.io/jopt-simple)
- JUL to SLF4J bridge (from http://www.slf4j.org)
......
......@@ -21,7 +21,7 @@
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-azure</artifactId>
<packaging>jar</packaging>
<version>0.0.29</version>
<version>0.0.32</version>
<name>core-lib-azure</name>
<properties>
......@@ -85,7 +85,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
<version>0.3.4</version>
<version>0.3.12</version>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
......
package org.opengroup.osdu.azure;
import com.azure.cosmos.CosmosClient;
import com.azure.cosmos.internal.AsyncDocumentClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
/**
* Implementation for ICosmosClientFactory.
*/
@Component
@Lazy
public class CosmosClientFactoryImpl implements ICosmosClientFactory {
@Lazy
@Autowired
private CosmosClient cosmosClient;
@Lazy
@Autowired
private AsyncDocumentClient asyncDocumentClient;
/**
* @param dataPartitionId Data Partition Id
* @return Cosmos Client instance
*/
@Override
public CosmosClient getClient(final String dataPartitionId) {
return cosmosClient;
}
/**
* @param dataPartitionId Data Partition Id
* @return Async Document Client instance
*/
@Override
public AsyncDocumentClient getAsyncClient(final String dataPartitionId) {
return asyncDocumentClient;
}
}
......@@ -14,6 +14,7 @@
package org.opengroup.osdu.azure;
import com.azure.core.exception.ResourceNotFoundException;
import com.azure.security.keyvault.secrets.SecretClient;
import com.azure.security.keyvault.secrets.models.KeyVaultSecret;
import org.opengroup.osdu.common.Validators;
......@@ -52,8 +53,32 @@ public final class KeyVaultFacade {
public static String getSecretWithValidation(final SecretClient kv, final String secretName) {
KeyVaultSecret secret = kv.getSecret(secretName);
Validators.checkNotNull(secret, secretName);
String secretValue = secret.getValue();
Validators.checkNotNullAndNotEmpty(secretValue, secretName);
return secretValue;
}
/**
* Get the secret with a default value. If the secret is not found or is null return the default value.
*
* @param kv Client configured to the correct vault
* @param secretName name of secret
* @param defaultValue to be used in case the secret is null or empty.
* @return Secret value. It is guaranteed to be returned with either default value or a non null, non empty secret.
*/
public String getSecretWithDefault(final SecretClient kv, final String secretName, final String defaultValue) {
Validators.checkNotNull(secretName, "Secret with name " + secretName);
KeyVaultSecret secret;
try {
secret = kv.getSecret(secretName);
if (secret == null || secret.getValue() == null || secret.getValue().isEmpty()) {
return defaultValue;
}
} catch (ResourceNotFoundException secretNotFound) {
return defaultValue;
}
return secret.getValue();
}
}
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.azure.blobstorage;
import com.azure.identity.DefaultAzureCredential;
import com.azure.storage.blob.BlobContainerClient;
import com.azure.storage.blob.BlobContainerClientBuilder;
import org.opengroup.osdu.azure.cache.BlobContainerClientCache;
import org.opengroup.osdu.azure.partition.PartitionInfoAzure;
import org.opengroup.osdu.azure.partition.PartitionServiceClient;
import org.opengroup.osdu.common.Validators;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
/**
* Implementation for IBlobContainerClientFactory.
*/
@Component
@Lazy
public class BlobContainerClientFactoryImpl implements IBlobContainerClientFactory {
@Autowired
private DefaultAzureCredential defaultAzureCredential;
@Autowired
private PartitionServiceClient partitionService;
@Autowired
private BlobContainerClientCache clientCache;
/**
* @param dataPartitionId Data partition id
* @param containerName Blob container name
* @return the blob container client instance.
*/
@Override
public BlobContainerClient getClient(final String dataPartitionId, final String containerName) {
Validators.checkNotNull(defaultAzureCredential, "Credential");
Validators.checkNotNullAndNotEmpty(dataPartitionId, "dataPartitionId");
Validators.checkNotNullAndNotEmpty(containerName, "containerName");
String cacheKey = String.format("%s-%s", dataPartitionId, containerName);
if (this.clientCache.containsKey(cacheKey)) {
return this.clientCache.get(cacheKey);
}
PartitionInfoAzure pi = this.partitionService.getPartition(dataPartitionId);
String endpoint = String.format("https://%s.blob.core.windows.net", pi.getStorageAccountName());
BlobContainerClient blobContainerClient = new BlobContainerClientBuilder()
.endpoint(endpoint)
.credential(defaultAzureCredential)
.containerName(containerName)
.buildClient();
this.clientCache.put(cacheKey, blobContainerClient);
return blobContainerClient;
}
}
......@@ -17,39 +17,31 @@ package org.opengroup.osdu.azure.blobstorage;
import com.azure.identity.DefaultAzureCredential;
import com.azure.storage.blob.BlobServiceClient;
import com.azure.storage.blob.BlobServiceClientBuilder;
import org.opengroup.osdu.azure.di.BlobStoreConfiguration;
import org.opengroup.osdu.azure.cache.BlobServiceClientCache;
import org.opengroup.osdu.azure.partition.PartitionInfoAzure;
import org.opengroup.osdu.azure.partition.PartitionServiceClient;
import org.opengroup.osdu.common.Validators;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
/**
* Implementation for IBlobServiceClientFactory.
* Implementation for IBlobServiceClientFactory.
*/
@Component
@Lazy
@ConditionalOnProperty(value = "azure.blobStore.required", havingValue = "true", matchIfMissing = false)
public class BlobServiceClientFactoryImpl implements IBlobServiceClientFactory {
private BlobServiceClient blobServiceClient;
@Autowired
private DefaultAzureCredential defaultAzureCredential;
/**
* Constructor to initialize blobServiceClient.
* @param defaultAzureCredential Default azure credentials.
* @param blobStoreConfiguration Configuration details for blob storage.
*/
@Autowired
public BlobServiceClientFactoryImpl(
final DefaultAzureCredential defaultAzureCredential,
final BlobStoreConfiguration blobStoreConfiguration) {
Validators.checkNotNull(defaultAzureCredential, "Default credentials");
Validators.checkNotNullAndNotEmpty(blobStoreConfiguration.getStorageAccountName(), "Storage account name cannot be null");
String endpoint = String.format("https://%s.blob.core.windows.net", blobStoreConfiguration.getStorageAccountName());
blobServiceClient = new BlobServiceClientBuilder()
.endpoint(endpoint)
.credential(defaultAzureCredential)
.buildClient();
}
private PartitionServiceClient partitionService;
@Autowired
private BlobServiceClientCache clientCache;
/**
* @param dataPartitionId data partition id.
......@@ -57,6 +49,24 @@ public class BlobServiceClientFactoryImpl implements IBlobServiceClientFactory {
*/
@Override
public BlobServiceClient getBlobServiceClient(final String dataPartitionId) {
Validators.checkNotNull(defaultAzureCredential, "Credential");
Validators.checkNotNullAndNotEmpty(dataPartitionId, "dataPartitionId");
String cacheKey = String.format("%s-blobServiceClient", dataPartitionId);
if (this.clientCache.containsKey(cacheKey)) {
return this.clientCache.get(cacheKey);
}
PartitionInfoAzure pi = this.partitionService.getPartition(dataPartitionId);
String endpoint = String.format("https://%s.blob.core.windows.net", pi.getStorageAccountName());
BlobServiceClient blobServiceClient = new BlobServiceClientBuilder()
.endpoint(endpoint)
.credential(defaultAzureCredential)
.buildClient();
this.clientCache.put(cacheKey, blobServiceClient);
return blobServiceClient;
}
}
......@@ -25,12 +25,12 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
import java.io.ByteArrayOutputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
import java.util.Collections;;
import java.util.Collections;
/**
* A simpler interface to interact with Azure blob storage.
......@@ -73,10 +73,9 @@ public class BlobStore {
private static final String LOG_PREFIX = "azure-core-lib";
/**
*
* @param filePath Path of file to be read.
* @param dataPartitionId Data partition id
* @param containerName Name of the storage container
* @param filePath Path of file to be read.
* @param dataPartitionId Data partition id
* @param containerName Name of the storage container
* @return the content of file with provided file path.
*/
public String readFromStorageContainer(
......@@ -90,30 +89,21 @@ public class BlobStore {
return downloadStream.toString(StandardCharsets.UTF_8.name());
} catch (BlobStorageException ex) {
if (ex.getErrorCode().equals(BlobErrorCode.BLOB_NOT_FOUND)) {
String errorMessage = "Specified blob was not found";
logger.warning(LOG_PREFIX, errorMessage, Collections.<String, String>emptyMap());
throw new AppException(404, errorMessage, ex.getMessage(), ex);
} else {
String errorMessage = "Failed to read specified blob";
logger.warning(LOG_PREFIX, errorMessage, Collections.<String, String>emptyMap());
throw new AppException(500, errorMessage, ex.getMessage(), ex);
throw handleBlobStoreException(404, "Specified blob was not found", ex);
}
throw handleBlobStoreException(500, "Failed to read specified blob", ex);
} catch (UnsupportedEncodingException ex) {
String errorMessage = String.format("Encoding was not correct for item with name=%s", filePath);
logger.warning(LOG_PREFIX, errorMessage, Collections.<String, String>emptyMap());
throw new AppException(400, errorMessage, ex.getMessage(), ex);
throw handleBlobStoreException(400, String.format("Encoding was not correct for item with name=%s", filePath), ex);
} catch (IOException ex) {
String errorMessage = String.format("Malformed document for item with name=%s", filePath);
logger.warning(LOG_PREFIX, errorMessage, Collections.<String, String>emptyMap());
throw new AppException(500, errorMessage, ex.getMessage(), ex);
throw handleBlobStoreException(500, String.format("Malformed document for item with name=%s", filePath), ex);
}
}
/**
*
* @param filePath Path of file to be deleted.
* @param dataPartitionId Data partition id
* @param containerName Name of the storage container
* @param filePath Path of file to be deleted.
* @param dataPartitionId Data partition id
* @param containerName Name of the storage container
* @return boolean indicating whether the deletion of given file was successful or not.
*/
public boolean deleteFromStorageContainer(
......@@ -127,23 +117,17 @@ public class BlobStore {
return true;
} catch (BlobStorageException ex) {
if (ex.getErrorCode().equals(BlobErrorCode.BLOB_NOT_FOUND)) {
String errorMessage = "Specified blob was not found";
logger.warning(LOG_PREFIX, errorMessage, Collections.<String, String>emptyMap());
throw new AppException(404, errorMessage, ex.getMessage(), ex);
} else {
String errorMessage = "Failed to delete blob";
logger.warning(LOG_PREFIX, errorMessage, Collections.<String, String>emptyMap());
throw new AppException(500, errorMessage, ex.getMessage(), ex);
throw handleBlobStoreException(404, "Specified blob was not found", ex);
}
throw handleBlobStoreException(500, "Failed to delete blob", ex);
}
}
/**
*
* @param filePath Path of file to be written at.
* @param content Content to be written in the file.
* @param dataPartitionId Data partition id
* @param containerName Name of the storage container
* @param filePath Path of file to be written at.
* @param content Content to be written in the file.
* @param dataPartitionId Data partition id
* @param containerName Name of the storage container
*/
public void writeToStorageContainer(
final String dataPartitionId,
......@@ -157,31 +141,38 @@ public class BlobStore {
try (ByteArrayInputStream dataStream = new ByteArrayInputStream(bytes)) {
blockBlobClient.upload(dataStream, bytesSize, true);
} catch (BlobStorageException ex) {
String errorMessage = "Failed to upload file content.";
logger.warning(LOG_PREFIX, errorMessage, Collections.<String, String>emptyMap());
throw new AppException(500, errorMessage, ex.getMessage(), ex);
throw handleBlobStoreException(500, "Failed to upload file content.", ex);
} catch (IOException ex) {
String errorMessage = String.format("Malformed document for item with name=%s", filePath);
logger.warning(LOG_PREFIX, errorMessage, Collections.<String, String>emptyMap());
throw new AppException(500, errorMessage, ex.getMessage(), ex);
throw handleBlobStoreException(500, String.format("Malformed document for item with name=%s", filePath), ex);
}
}
/**
*
* @param dataPartitionId Data partition id.
* @param containerName Name of storage container.
* @param dataPartitionId Data partition id.
* @param containerName Name of storage container.
* @return blob container client corresponding to the dataPartitionId.
*/
private BlobContainerClient getBlobContainerClient(final String dataPartitionId, final String containerName) {
try {
BlobServiceClient serviceClient = blobServiceClientFactory.getBlobServiceClient(dataPartitionId);
return serviceClient.getBlobContainerClient(containerName);
} catch (AppException ex) {
throw handleBlobStoreException(ex.getError().getCode(), "Error creating creating blob container client.", ex);
} catch (Exception ex) {
String errorMessage = "Error creating creating blob container client.";
logger.warning(LOG_PREFIX, errorMessage, Collections.<String, String>emptyMap());
throw new AppException(500, errorMessage, ex.getMessage(), ex);
throw handleBlobStoreException(500, "Error creating creating blob container client.", ex);
}
}
/**
* Logs and returns instance of AppException.
*
* @param status Response status code
* @param errorMessage Error message
* @param ex Original exception
* @return Instance of AppException
*/
private AppException handleBlobStoreException(final int status, final String errorMessage, final Exception ex) {
logger.warning(LOG_PREFIX, errorMessage, Collections.<String, String>emptyMap());
return new AppException(status, errorMessage, ex.getMessage(), ex);
}
}
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.azure.blobstorage;
import com.azure.storage.blob.BlobContainerClient;
/**
* Interface for BlobContainer client factory to return appropriate
* blobContainerClient based on the data partition id.
*/
public interface IBlobContainerClientFactory {
/**
* @param dataPartitionId Data partition id
* @param containerName Container name
* @return blobContainerClient for given data partition id.
*/
BlobContainerClient getClient(String dataPartitionId, String containerName);
}
package org.opengroup.osdu.azure.cache;
import com.azure.cosmos.internal.AsyncDocumentClient;
import org.opengroup.osdu.core.common.cache.VmCache;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
/**
* Implementation of ICache for AsyncDocumentClient.
*/
@Component
@Lazy
public class AsyncCosmosClientCache extends VmCache<String, AsyncDocumentClient> {
/**
* Default cache constructor.
*/
public AsyncCosmosClientCache() {
super(60 * 60, 1000);
}
/**
* @param key cache key
* @return true if found in cache
*/
public boolean containsKey(final String key) {
return this.get(key) != null;
}
}
package org.opengroup.osdu.azure.cache;
import com.azure.storage.blob.BlobContainerClient;
import org.opengroup.osdu.core.common.cache.VmCache;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
/**
* Implementation of ICache for BlobContainerClient.
*/
@Component
@Lazy
public class BlobContainerClientCache extends VmCache<String, BlobContainerClient> {
/**
* Default cache constructor.
*/
public BlobContainerClientCache() {
super(60 * 60, 1000);
}
/**
* @param key cache key
* @return true if found in cache
*/
public boolean containsKey(final String key) {
return this.get(key) != null;
}
}
\ No newline at end of file
package org.opengroup.osdu.azure.cache;
import com.azure.storage.blob.BlobServiceClient;
import org.opengroup.osdu.core.common.cache.VmCache;
/**
* Implementation of ICache for BlobServiceClient.
*/
public class BlobServiceClientCache extends VmCache<String, BlobServiceClient> {
/**
* Default cache constructor.
*/
public BlobServiceClientCache() {
super(60 * 60, 1000);
}
/**
* @param key cache key
* @return true if found in cache
*/
public boolean containsKey(final String key) {
return this.get(key) != null;
}
}
package org.opengroup.osdu.azure.cache;
import com.azure.cosmos.CosmosClient;
import org.opengroup.osdu.core.common.cache.VmCache;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
/**
* Implementation of ICache for CosmosClient.
*/
@Component
@Lazy
public class CosmosClientCache extends VmCache<String, CosmosClient> {
/**
* Default cache constructor.
*/
public CosmosClientCache() {
super(60 * 60, 1000);
}
/**
* @param key cache key
* @return true if found in cache
*/
public boolean containsKey(final String key) {
return this.get(key) != null;
}
}
\ No newline at end of file
......@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.elastic.dependencies;
package org.opengroup.osdu.azure.cache;
import org.opengroup.osdu.core.common.cache.VmCache;
import org.opengroup.osdu.core.common.model.search.ClusterSettings;
......@@ -41,7 +41,7 @@ public class ElasticCredentialsCache implements IElasticCredentialsCache<String,
public ElasticCredentialsCache(
@Named("ELASTIC_CACHE_EXPIRATION") final Integer cacheExpirationMinutes,
@Named("MAX_CACHE_VALUE_SIZE") final Integer maxCachedObjectEntries) {
cache = new VmCache<>(cacheExpirationMinutes, maxCachedObjectEntries);
cache = new VmCache<>(cacheExpirationMinutes * 60, maxCachedObjectEntries);
}