Merge branch 'partition-svc' into 'master'

update client lib to support multi-tenancy

See merge request osdu/platform/system/lib/cloud/azure/os-core-lib-azure!24

NOTICE

@@ -19,6 +19,7 @@ The following software have components provided under the terms of this license:
 - Apache Commons Validator (from http://commons.apache.org/proper/commons-validator/)
 - Apache HttpAsyncClient (from http://hc.apache.org/httpcomponents-asyncclient)
 - Apache HttpClient (from http://hc.apache.org/httpcomponents-client)
+- Apache HttpClient Cache (from http://hc.apache.org/httpcomponents-client)
 - Apache HttpCore (from http://hc.apache.org/httpcomponents-core-ga)
 - Apache HttpCore NIO (from http://hc.apache.org/httpcomponents-core-ga)
 - Apache Log4j API (from )
@@ -342,7 +343,6 @@ The following software have components provided under the terms of this license:
 - Azure Spring Boot AutoConfigure (from https://github.com/Microsoft/azure-spring-boot)
 - Azure Spring Boot Starter (from https://github.com/Microsoft/azure-spring-boot)
 - Checker Qual (from https://checkerframework.org)
-- Elasticsearch: 5.0.0-alpha5 (from https://github.com/elastic/elasticsearch)
 - Extensions on Apache Proton-J library (from https://github.com/Azure/qpid-proton-j-extensions)
 - JOpt Simple (from http://pholser.github.io/jopt-simple)
 - JUL to SLF4J bridge (from http://www.slf4j.org)

pom.xml

@@ -21,7 +21,7 @@
   <groupId>org.opengroup.osdu</groupId>
   <artifactId>core-lib-azure</artifactId>
   <packaging>jar</packaging>
-  <version>0.0.29</version>
+  <version>0.0.32</version>
   <name>core-lib-azure</name>
 
   <properties>
@@ -85,7 +85,7 @@
     <dependency>
       <groupId>org.opengroup.osdu</groupId>
       <artifactId>os-core-common</artifactId>
-      <version>0.3.4</version>
+      <version>0.3.12</version>
       <exclusions>
         <exclusion>
           <groupId>org.springframework.boot</groupId>

src/main/java/org/opengroup/osdu/azure/CosmosClientFactoryImpl.java

-package org.opengroup.osdu.azure;
-
-import com.azure.cosmos.CosmosClient;
-import com.azure.cosmos.internal.AsyncDocumentClient;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.context.annotation.Lazy;
-import org.springframework.stereotype.Component;
-
-/**
- *  Implementation for ICosmosClientFactory.
- */
-@Component
-@Lazy
-public class CosmosClientFactoryImpl implements ICosmosClientFactory {
-
-    @Lazy
-    @Autowired
-    private CosmosClient cosmosClient;
-
-    @Lazy
-    @Autowired
-    private AsyncDocumentClient asyncDocumentClient;
-
-    /**
-     * @param dataPartitionId Data Partition Id
-     * @return Cosmos Client instance
-     */
-    @Override
-    public CosmosClient getClient(final String dataPartitionId) {
-        return cosmosClient;
-    }
-
-    /**
-     * @param dataPartitionId Data Partition Id
-     * @return Async Document Client instance
-     */
-    @Override
-    public AsyncDocumentClient getAsyncClient(final String dataPartitionId) {
-        return asyncDocumentClient;
-    }
-}

src/main/java/org/opengroup/osdu/azure/KeyVaultFacade.java

@@ -14,6 +14,7 @@
 
 package org.opengroup.osdu.azure;
 
+import com.azure.core.exception.ResourceNotFoundException;
 import com.azure.security.keyvault.secrets.SecretClient;
 import com.azure.security.keyvault.secrets.models.KeyVaultSecret;
 import org.opengroup.osdu.common.Validators;
@@ -52,8 +53,32 @@ public final class KeyVaultFacade {
     public static String getSecretWithValidation(final SecretClient kv, final String secretName) {
         KeyVaultSecret secret = kv.getSecret(secretName);
         Validators.checkNotNull(secret, secretName);
+
         String secretValue = secret.getValue();
         Validators.checkNotNullAndNotEmpty(secretValue, secretName);
+
         return secretValue;
     }
+
+    /**
+     * Get the secret with a default value. If the secret is not found or is null return the default value.
+     *
+     * @param kv           Client configured to the correct vault
+     * @param secretName   name of secret
+     * @param defaultValue to be used in case the secret is null or empty.
+     * @return Secret value. It is guaranteed to be returned with either default value or a non null, non empty secret.
+     */
+    public String getSecretWithDefault(final SecretClient kv, final String secretName, final String defaultValue) {
+        Validators.checkNotNull(secretName, "Secret with name " + secretName);
+        KeyVaultSecret secret;
+        try {
+            secret = kv.getSecret(secretName);
+            if (secret == null || secret.getValue() == null || secret.getValue().isEmpty()) {
+                return defaultValue;
+            }
+        } catch (ResourceNotFoundException secretNotFound) {
+            return defaultValue;
+        }
+        return secret.getValue();
+    }
 }

src/main/java/org/opengroup/osdu/azure/blobstorage/BlobContainerClientFactoryImpl.java

+// Copyright © Microsoft Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.azure.blobstorage;
+
+import com.azure.identity.DefaultAzureCredential;
+import com.azure.storage.blob.BlobContainerClient;
+import com.azure.storage.blob.BlobContainerClientBuilder;
+import org.opengroup.osdu.azure.cache.BlobContainerClientCache;
+import org.opengroup.osdu.azure.partition.PartitionInfoAzure;
+import org.opengroup.osdu.azure.partition.PartitionServiceClient;
+import org.opengroup.osdu.common.Validators;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.stereotype.Component;
+
+/**
+ * Implementation for IBlobContainerClientFactory.
+ */
+@Component
+@Lazy
+public class BlobContainerClientFactoryImpl implements IBlobContainerClientFactory {
+
+    @Autowired
+    private DefaultAzureCredential defaultAzureCredential;
+
+    @Autowired
+    private PartitionServiceClient partitionService;
+
+    @Autowired
+    private BlobContainerClientCache clientCache;
+
+    /**
+     * @param dataPartitionId Data partition id
+     * @param containerName   Blob container name
+     * @return the blob container client instance.
+     */
+    @Override
+    public BlobContainerClient getClient(final String dataPartitionId, final String containerName) {
+        Validators.checkNotNull(defaultAzureCredential, "Credential");
+        Validators.checkNotNullAndNotEmpty(dataPartitionId, "dataPartitionId");
+        Validators.checkNotNullAndNotEmpty(containerName, "containerName");
+
+        String cacheKey = String.format("%s-%s", dataPartitionId, containerName);
+        if (this.clientCache.containsKey(cacheKey)) {
+            return this.clientCache.get(cacheKey);
+        }
+
+        PartitionInfoAzure pi = this.partitionService.getPartition(dataPartitionId);
+        String endpoint = String.format("https://%s.blob.core.windows.net", pi.getStorageAccountName());
+
+        BlobContainerClient blobContainerClient = new BlobContainerClientBuilder()
+                .endpoint(endpoint)
+                .credential(defaultAzureCredential)
+                .containerName(containerName)
+                .buildClient();
+
+        this.clientCache.put(cacheKey, blobContainerClient);
+
+        return blobContainerClient;
+    }
+}

src/main/java/org/opengroup/osdu/azure/blobstorage/BlobServiceClientFactoryImpl.java

@@ -17,39 +17,31 @@ package org.opengroup.osdu.azure.blobstorage;
 import com.azure.identity.DefaultAzureCredential;
 import com.azure.storage.blob.BlobServiceClient;
 import com.azure.storage.blob.BlobServiceClientBuilder;
-import org.opengroup.osdu.azure.di.BlobStoreConfiguration;
+import org.opengroup.osdu.azure.cache.BlobServiceClientCache;
+import org.opengroup.osdu.azure.partition.PartitionInfoAzure;
+import org.opengroup.osdu.azure.partition.PartitionServiceClient;
 import org.opengroup.osdu.common.Validators;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Lazy;
 import org.springframework.stereotype.Component;
 
 /**
- *  Implementation for IBlobServiceClientFactory.
+ * Implementation for IBlobServiceClientFactory.
  */
 @Component
+@Lazy
 @ConditionalOnProperty(value = "azure.blobStore.required", havingValue = "true", matchIfMissing = false)
 public class BlobServiceClientFactoryImpl implements IBlobServiceClientFactory {
 
-    private BlobServiceClient blobServiceClient;
+    @Autowired
+    private DefaultAzureCredential defaultAzureCredential;
 
-    /**
-     * Constructor to initialize blobServiceClient.
-     * @param defaultAzureCredential            Default azure credentials.
-     * @param blobStoreConfiguration         Configuration details for blob storage.
-     */
     @Autowired
-    public BlobServiceClientFactoryImpl(
-            final DefaultAzureCredential defaultAzureCredential,
-            final BlobStoreConfiguration blobStoreConfiguration) {
-        Validators.checkNotNull(defaultAzureCredential, "Default credentials");
-        Validators.checkNotNullAndNotEmpty(blobStoreConfiguration.getStorageAccountName(), "Storage account name cannot be null");
-
-        String endpoint = String.format("https://%s.blob.core.windows.net", blobStoreConfiguration.getStorageAccountName());
-        blobServiceClient = new BlobServiceClientBuilder()
-                .endpoint(endpoint)
-                .credential(defaultAzureCredential)
-                .buildClient();
-    }
+    private PartitionServiceClient partitionService;
+
+    @Autowired
+    private BlobServiceClientCache clientCache;
 
     /**
      * @param dataPartitionId data partition id.
@@ -57,6 +49,24 @@ public class BlobServiceClientFactoryImpl implements IBlobServiceClientFactory {
      */
     @Override
     public BlobServiceClient getBlobServiceClient(final String dataPartitionId) {
+        Validators.checkNotNull(defaultAzureCredential, "Credential");
+        Validators.checkNotNullAndNotEmpty(dataPartitionId, "dataPartitionId");
+
+        String cacheKey = String.format("%s-blobServiceClient", dataPartitionId);
+        if (this.clientCache.containsKey(cacheKey)) {
+            return this.clientCache.get(cacheKey);
+        }
+
+        PartitionInfoAzure pi = this.partitionService.getPartition(dataPartitionId);
+        String endpoint = String.format("https://%s.blob.core.windows.net", pi.getStorageAccountName());
+
+        BlobServiceClient blobServiceClient = new BlobServiceClientBuilder()
+                .endpoint(endpoint)
+                .credential(defaultAzureCredential)
+                .buildClient();
+
+        this.clientCache.put(cacheKey, blobServiceClient);
+
         return blobServiceClient;
     }
 }

src/main/java/org/opengroup/osdu/azure/blobstorage/BlobStore.java

@@ -25,12 +25,12 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.stereotype.Component;
 
-import java.io.ByteArrayOutputStream;
 import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 import java.nio.charset.StandardCharsets;
-import java.util.Collections;;
+import java.util.Collections;
 
 /**
  * A simpler interface to interact with Azure blob storage.
@@ -73,10 +73,9 @@ public class BlobStore {
     private static final String LOG_PREFIX = "azure-core-lib";
 
     /**
-     *
-     * @param filePath              Path of file to be read.
-     * @param dataPartitionId       Data partition id
-     * @param containerName         Name of the storage container
+     * @param filePath        Path of file to be read.
+     * @param dataPartitionId Data partition id
+     * @param containerName   Name of the storage container
      * @return the content of file with provided file path.
      */
     public String readFromStorageContainer(
@@ -90,30 +89,21 @@ public class BlobStore {
             return downloadStream.toString(StandardCharsets.UTF_8.name());
         } catch (BlobStorageException ex) {
             if (ex.getErrorCode().equals(BlobErrorCode.BLOB_NOT_FOUND)) {
-                String errorMessage = "Specified blob was not found";
-                logger.warning(LOG_PREFIX, errorMessage, Collections.<String, String>emptyMap());
-                throw new AppException(404, errorMessage, ex.getMessage(), ex);
-            } else {
-                String errorMessage = "Failed to read specified blob";
-                logger.warning(LOG_PREFIX, errorMessage, Collections.<String, String>emptyMap());
-                throw new AppException(500, errorMessage, ex.getMessage(), ex);
+                throw handleBlobStoreException(404, "Specified blob was not found", ex);
             }
+            throw handleBlobStoreException(500, "Failed to read specified blob", ex);
         } catch (UnsupportedEncodingException ex) {
-            String errorMessage = String.format("Encoding was not correct for item with name=%s", filePath);
-            logger.warning(LOG_PREFIX, errorMessage, Collections.<String, String>emptyMap());
-            throw new AppException(400, errorMessage, ex.getMessage(), ex);
+            throw handleBlobStoreException(400, String.format("Encoding was not correct for item with name=%s", filePath), ex);
         } catch (IOException ex) {
-            String errorMessage = String.format("Malformed document for item with name=%s", filePath);
-            logger.warning(LOG_PREFIX, errorMessage, Collections.<String, String>emptyMap());
-            throw new AppException(500, errorMessage, ex.getMessage(), ex);
+
+            throw handleBlobStoreException(500, String.format("Malformed document for item with name=%s", filePath), ex);
         }
     }
 
     /**
-     *
-     * @param filePath              Path of file to be deleted.
-     * @param dataPartitionId       Data partition id
-     * @param containerName         Name of the storage container
+     * @param filePath        Path of file to be deleted.
+     * @param dataPartitionId Data partition id
+     * @param containerName   Name of the storage container
      * @return boolean indicating whether the deletion of given file was successful or not.
      */
     public boolean deleteFromStorageContainer(
@@ -127,23 +117,17 @@ public class BlobStore {
             return true;
         } catch (BlobStorageException ex) {
             if (ex.getErrorCode().equals(BlobErrorCode.BLOB_NOT_FOUND)) {
-                String errorMessage = "Specified blob was not found";
-                logger.warning(LOG_PREFIX, errorMessage, Collections.<String, String>emptyMap());
-                throw new AppException(404, errorMessage, ex.getMessage(), ex);
-            } else {
-                String errorMessage = "Failed to delete blob";
-                logger.warning(LOG_PREFIX, errorMessage, Collections.<String, String>emptyMap());
-                throw new AppException(500, errorMessage, ex.getMessage(), ex);
+                throw handleBlobStoreException(404, "Specified blob was not found", ex);
             }
+            throw handleBlobStoreException(500, "Failed to delete blob", ex);
         }
     }
 
     /**
-     *
-     * @param filePath              Path of file to be written at.
-     * @param content               Content to be written in the file.
-     * @param dataPartitionId       Data partition id
-     * @param containerName         Name of the storage container
+     * @param filePath        Path of file to be written at.
+     * @param content         Content to be written in the file.
+     * @param dataPartitionId Data partition id
+     * @param containerName   Name of the storage container
      */
     public void writeToStorageContainer(
             final String dataPartitionId,
@@ -157,31 +141,38 @@ public class BlobStore {
         try (ByteArrayInputStream dataStream = new ByteArrayInputStream(bytes)) {
             blockBlobClient.upload(dataStream, bytesSize, true);
         } catch (BlobStorageException ex) {
-            String errorMessage = "Failed to upload file content.";
-            logger.warning(LOG_PREFIX, errorMessage, Collections.<String, String>emptyMap());
-            throw new AppException(500, errorMessage, ex.getMessage(), ex);
+            throw handleBlobStoreException(500, "Failed to upload file content.", ex);
         } catch (IOException ex) {
-            String errorMessage = String.format("Malformed document for item with name=%s", filePath);
-            logger.warning(LOG_PREFIX, errorMessage, Collections.<String, String>emptyMap());
-            throw new AppException(500, errorMessage, ex.getMessage(), ex);
+            throw handleBlobStoreException(500, String.format("Malformed document for item with name=%s", filePath), ex);
         }
     }
 
     /**
-     *
-     * @param dataPartitionId       Data partition id.
-     * @param containerName         Name of storage container.
+     * @param dataPartitionId Data partition id.
+     * @param containerName   Name of storage container.
      * @return blob container client corresponding to the dataPartitionId.
      */
     private BlobContainerClient getBlobContainerClient(final String dataPartitionId, final String containerName) {
         try {
             BlobServiceClient serviceClient = blobServiceClientFactory.getBlobServiceClient(dataPartitionId);
             return serviceClient.getBlobContainerClient(containerName);
+        } catch (AppException ex) {
+            throw handleBlobStoreException(ex.getError().getCode(), "Error creating creating blob container client.", ex);
         } catch (Exception ex) {
-            String errorMessage = "Error creating creating blob container client.";
-            logger.warning(LOG_PREFIX, errorMessage, Collections.<String, String>emptyMap());
-            throw new AppException(500, errorMessage, ex.getMessage(), ex);
+            throw handleBlobStoreException(500, "Error creating creating blob container client.", ex);
         }
     }
 
+    /**
+     * Logs and returns instance of AppException.
+     *
+     * @param status       Response status code
+     * @param errorMessage Error message
+     * @param ex           Original exception
+     * @return Instance of AppException
+     */
+    private AppException handleBlobStoreException(final int status, final String errorMessage, final Exception ex) {
+        logger.warning(LOG_PREFIX, errorMessage, Collections.<String, String>emptyMap());
+        return new AppException(status, errorMessage, ex.getMessage(), ex);
+    }
 }

src/main/java/org/opengroup/osdu/azure/blobstorage/IBlobContainerClientFactory.java

+// Copyright © Microsoft Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.azure.blobstorage;
+
+import com.azure.storage.blob.BlobContainerClient;
+
+/**
+ * Interface for BlobContainer client factory to return appropriate
+ * blobContainerClient based on the data partition id.
+ */
+public interface IBlobContainerClientFactory {
+    /**
+     * @param dataPartitionId Data partition id
+     * @param containerName   Container name
+     * @return blobContainerClient for given data partition id.
+     */
+    BlobContainerClient getClient(String dataPartitionId, String containerName);
+}

src/main/java/org/opengroup/osdu/azure/cache/AsyncCosmosClientCache.java

+package org.opengroup.osdu.azure.cache;
+
+import com.azure.cosmos.internal.AsyncDocumentClient;
+import org.opengroup.osdu.core.common.cache.VmCache;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.stereotype.Component;
+
+/**
+ * Implementation of ICache for AsyncDocumentClient.
+ */
+@Component
+@Lazy
+public class AsyncCosmosClientCache extends VmCache<String, AsyncDocumentClient> {
+
+    /**
+     *  Default cache constructor.
+     */
+    public AsyncCosmosClientCache() {
+        super(60 * 60, 1000);
+    }
+
+    /**
+     * @param key cache key
+     * @return true if found in cache
+     */
+    public boolean containsKey(final String key) {
+        return this.get(key) != null;
+    }
+}

src/main/java/org/opengroup/osdu/azure/cache/BlobContainerClientCache.java

+package org.opengroup.osdu.azure.cache;
+
+import com.azure.storage.blob.BlobContainerClient;
+import org.opengroup.osdu.core.common.cache.VmCache;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.stereotype.Component;
+
+/**
+ * Implementation of ICache for BlobContainerClient.
+ */
+@Component
+@Lazy
+public class BlobContainerClientCache extends VmCache<String, BlobContainerClient> {
+
+    /**
+     *  Default cache constructor.
+     */
+    public BlobContainerClientCache() {
+        super(60 * 60, 1000);
+    }
+
+    /**
+     * @param key cache key
+     * @return true if found in cache
+     */
+    public boolean containsKey(final String key) {
+        return this.get(key) != null;
+    }
+}
\ No newline at end of file

src/main/java/org/opengroup/osdu/azure/cache/BlobServiceClientCache.java

+package org.opengroup.osdu.azure.cache;
+
+import com.azure.storage.blob.BlobServiceClient;
+import org.opengroup.osdu.core.common.cache.VmCache;
+
+/**
+ * Implementation of ICache for BlobServiceClient.
+ */
+public class BlobServiceClientCache extends VmCache<String, BlobServiceClient> {
+
+    /**
+     *  Default cache constructor.
+     */
+    public BlobServiceClientCache() {
+        super(60 * 60, 1000);
+    }
+
+    /**
+     * @param key cache key
+     * @return true if found in cache
+     */
+    public boolean containsKey(final String key) {
+        return this.get(key) != null;
+    }
+}

src/main/java/org/opengroup/osdu/azure/cache/CosmosClientCache.java

+package org.opengroup.osdu.azure.cache;
+
+import com.azure.cosmos.CosmosClient;
+import org.opengroup.osdu.core.common.cache.VmCache;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.stereotype.Component;
+
+/**
+ * Implementation of ICache for CosmosClient.
+ */
+@Component
+@Lazy
+public class CosmosClientCache extends VmCache<String, CosmosClient> {
+
+    /**
+     *  Default cache constructor.
+     */
+    public CosmosClientCache() {
+        super(60 * 60, 1000);
+    }
+
+    /**
+     * @param key cache key
+     * @return true if found in cache
+     */
+    public boolean containsKey(final String key) {
+        return this.get(key) != null;
+    }
+}
\ No newline at end of file

src/main/java/org/opengroup/osdu/elastic/dependencies/ElasticCredentialsCache.java → src/main/java/org/opengroup/osdu/azure/cache/ElasticCredentialsCache.java

@@ -12,7 +12,7 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-package org.opengroup.osdu.elastic.dependencies;
+package org.opengroup.osdu.azure.cache;
 
 import org.opengroup.osdu.core.common.cache.VmCache;
 import org.opengroup.osdu.core.common.model.search.ClusterSettings;
@@ -41,7 +41,7 @@ public class ElasticCredentialsCache implements IElasticCredentialsCache<String,
     public ElasticCredentialsCache(
             @Named("ELASTIC_CACHE_EXPIRATION") final Integer cacheExpirationMinutes,
             @Named("MAX_CACHE_VALUE_SIZE") final Integer maxCachedObjectEntries) {
-        cache = new VmCache<>(cacheExpirationMinutes, maxCachedObjectEntries);
+        cache = new VmCache<>(cacheExpirationMinutes * 60, maxCachedObjectEntries);
     }