Commit add609d0 authored by Aalekh Jain's avatar Aalekh Jain
Browse files

Merge branch 'master' of...

Merge branch 'master' of https://community.opengroup.org/osdu/platform/system/lib/cloud/azure/os-core-lib-azure into containerCreationAndDeletion
parents 03ecb05d 93afc761
Pipeline #23719 failed with stage
in 1 minute and 55 seconds
......@@ -13,7 +13,7 @@ include:
- project: 'osdu/platform/ci-cd-pipelines'
ref: 'master'
file: 'scanners/fossa.yml'
file: 'scanners/fossa-maven.yml'
- project: 'osdu/platform/ci-cd-pipelines'
ref: 'master'
......
......@@ -18,6 +18,7 @@ The following software have components provided under the terms of this license:
- ASM based accessors helper used by json-smart (from )
- Adapter: RxJava (from )
- Apache Commons Codec (from http://commons.apache.org/proper/commons-codec/)
- Apache Commons Collections (from http://commons.apache.org/proper/commons-collections/)
- Apache Commons Lang (from http://commons.apache.org/proper/commons-lang/)
- Apache Commons Logging (from http://commons.apache.org/proper/commons-logging/)
- Apache HttpAsyncClient (from http://hc.apache.org/httpcomponents-asyncclient)
......@@ -69,6 +70,7 @@ The following software have components provided under the terms of this license:
- Jackson-module-parameter-names (from )
- Java Native Access (from https://github.com/java-native-access/jna)
- Java Native Access Platform (from https://github.com/java-native-access/jna)
- Java UUID Generator (from http://wiki.fasterxml.com/JugHome)
- Javassist (from http://www.javassist.org/)
- Joda-Time (from http://www.joda.org/joda-time/)
- KeePassJava2 :: All (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2)
......@@ -149,6 +151,7 @@ The following software have components provided under the terms of this license:
- Spring Core (from https://github.com/spring-projects/spring-framework)
- Spring Data Core (from )
- Spring Expression Language (SpEL) (from https://github.com/spring-projects/spring-framework)
- Spring TestContext Framework (from https://github.com/spring-projects/spring-framework)
- Spring Transaction (from https://github.com/spring-projects/spring-framework)
- Spring Web (from https://github.com/spring-projects/spring-framework)
- Spring Web MVC (from https://github.com/spring-projects/spring-framework)
......@@ -332,6 +335,13 @@ The following software have components provided under the terms of this license:
- Project Lombok (from https://projectlombok.org)
- jakarta.annotation-api (from https://projects.eclipse.org/projects/ee4j.ca)
========================================================================
JSON
========================================================================
The following software have components provided under the terms of this license:
- JSON in Java (from https://github.com/douglascrockford/JSON-java)
========================================================================
LGPL-2.1-only
========================================================================
......@@ -369,6 +379,7 @@ MIT
The following software have components provided under the terms of this license:
- Azure Java Client Authentication Library for AutoRest (from https://github.com/Azure/autorest-clientruntime-for-java)
- Azure Java Client Runtime for ARM (from https://github.com/Azure/autorest-clientruntime-for-java)
- Azure Java Client Runtime for AutoRest (from https://github.com/Azure/autorest-clientruntime-for-java)
- Azure Metrics Spring Boot Starter (from https://github.com/Microsoft/azure-spring-boot)
- Azure Spring Boot AutoConfigure (from https://github.com/Microsoft/azure-spring-boot)
......@@ -387,11 +398,13 @@ The following software have components provided under the terms of this license:
- Microsoft Azure Java Core Library (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure Netty HTTP Client Library (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure SDK annotations (from https://github.com/Microsoft/java-api-annotations)
- Microsoft Azure SDK for EventGrid Management (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure SDK for Service Bus (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure SDK for eventgrid (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure client library for Blob Storage (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure client library for Identity (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure client library for KeyVault Keys (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure client library for KeyVault Secrets (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure common module for Storage (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure internal Avro module for Storage (from https://github.com/Azure/azure-sdk-for-java)
......@@ -403,6 +416,8 @@ The following software have components provided under the terms of this license:
- Spongy Castle (from http://rtyley.github.io/spongycastle/)
- Spring Data for Azure Cosmos DB SQL API (from https://github.com/Azure/azure-sdk-for-java/tree/master/sdk/cosmos/azure-spring-data-cosmos)
- adal4j (from https://github.com/AzureAD/azure-activedirectory-library-for-java)
- azure-documentdb (from https://azure.microsoft.com/en-us/services/cosmos-db/)
- documentdb-bulkexecutor (from http://azure.microsoft.com/en-us/services/documentdb/)
- micrometer-core (from https://github.com/micrometer-metrics/micrometer)
- mockito-junit-jupiter (from https://github.com/mockito/mockito)
- msal4j (from https://github.com/AzureAD/microsoft-authentication-library-for-java)
......@@ -464,11 +479,13 @@ The following software have components provided under the terms of this license:
- Joda-Time (from http://www.joda.org/joda-time/)
- LatencyUtils (from http://latencyutils.github.io/LatencyUtils/)
- Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Azure SDK for EventGrid Management (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure client library for Blob Storage (from https://github.com/Azure/azure-sdk-for-java)
- Project Lombok (from https://projectlombok.org)
- Spring Web (from https://github.com/spring-projects/spring-framework)
- StAX API (from http://stax.codehaus.org/)
- azure-documentdb (from https://azure.microsoft.com/en-us/services/cosmos-db/)
- msal4j (from https://github.com/AzureAD/microsoft-authentication-library-for-java)
- reactive-streams (from http://www.reactive-streams.org/)
......
......@@ -20,7 +20,7 @@
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-azure</artifactId>
<packaging>jar</packaging>
<version>0.0.57</version>
<version>0.0.63</version>
<name>core-lib-azure</name>
<properties>
......@@ -49,9 +49,12 @@
<reactor.version>Dysprosium-SR12</reactor.version>
<netty.version>4.1.51.Final</netty.version>
<lombok.version>1.18.16</lombok.version>
<osdu.oscorecommon.version>0.3.18</osdu.oscorecommon.version>
<osdu.oscorecommon.version>0.3.23</osdu.oscorecommon.version>
<mockito-junit-jupiter.version>2.23.0</mockito-junit-jupiter.version>
<spring-boot-starter-log4j2.version>2.3.4.RELEASE</spring-boot-starter-log4j2.version>
<azure-mgmt-eventgrid.version>1.0.0-beta-3</azure-mgmt-eventgrid.version>
<azure-security-keyvault-keys.version>4.2.3</azure-security-keyvault-keys.version>
<documentdb-bulkexecutor.version>2.12.0</documentdb-bulkexecutor.version>
</properties>
<licenses>
......@@ -247,6 +250,39 @@
<groupId>com.microsoft.azure</groupId>
<artifactId>azure-eventgrid</artifactId>
<version>1.2.0</version>
<exclusions>
<exclusion>
<artifactId>azure-client-runtime</artifactId>
<groupId>com.microsoft.azure</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.microsoft.azure.eventgrid.v2020_04_01_preview</groupId>
<artifactId>azure-mgmt-eventgrid</artifactId>
<version>${azure-mgmt-eventgrid.version}</version>
</dependency>
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-security-keyvault-keys</artifactId>
<version>${azure-security-keyvault-keys.version}</version>
</dependency>
<dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>documentdb-bulkexecutor</artifactId>
<version>${documentdb-bulkexecutor.version}</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-test</artifactId>
<version>5.2.8.RELEASE</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
......
package org.opengroup.osdu.azure.cache;
import com.microsoft.azure.documentdb.bulkexecutor.DocumentBulkExecutor;
import org.opengroup.osdu.core.common.cache.VmCache;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
/**
* Implementation of ICache for DocumentBulkExecutor.
*/
@Component
@Lazy
public class CosmosBulkExecutorCache extends VmCache<String, DocumentBulkExecutor> {
/**
* Default cache constructor.
*/
public CosmosBulkExecutorCache() {
super(60 * 60, 1000);
}
/**
* @param key cache key
* @return true if found in cache
*/
public boolean containsKey(final String key) {
return this.get(key) != null;
}
}
\ No newline at end of file
package org.opengroup.osdu.azure.cache;
import org.opengroup.osdu.core.common.cache.VmCache;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
import com.azure.security.keyvault.keys.cryptography.CryptographyClient;
/**
* Implementation of ICache for CryptographyClient.
*/
@Lazy
@Component
public class CryptographyClientCache extends VmCache<String, CryptographyClient> {
/**
* Default cache constructor.
*/
public CryptographyClientCache() {
super(60 * 60, 1000);
}
/**
* @param key cache key
* @return true if found in cache
*/
public boolean containsKey(final String key) {
return this.get(key) != null;
}
}
package org.opengroup.osdu.azure.cache;
import org.opengroup.osdu.core.common.cache.VmCache;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
import com.microsoft.azure.management.eventgrid.v2020_04_01_preview.implementation.EventGridManager;
/**
* Implementation of ICache for CryptographyClient.
*/
@Lazy
@Component
public class EventGridManagerCache extends VmCache<String, EventGridManager> {
/**
* Default cache constructor.
*/
public EventGridManagerCache() {
super(60 * 60, 1000);
}
/**
* @param key cache key
* @return true if found in cache
*/
public boolean containsKey(final String key) {
return this.get(key) != null;
}
}
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.azure.concurrency;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
/***
* Custom Executors class that returns a custom thread pool executor {@link CustomThreadPoolExecutor}.
*/
public final class CustomExecutors {
/***
* Private constructor -- this class should never be instantiated.
*/
private CustomExecutors() {
}
/***
* Returning new custom thread pool executor.
* @param nThreads corresponding to corePoolSize and maxPoolSize.
* @return instance of {@link CustomThreadPoolExecutor}
*/
public static ExecutorService newFixedThreadPool(final int nThreads) {
return new CustomThreadPoolExecutor(nThreads, nThreads,
0L, TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<Runnable>());
}
}
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.azure.concurrency;
import org.springframework.web.context.request.RequestContextHolder;
import javax.validation.constraints.NotNull;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
/***
* A custom thread pool executor that executes code in the new thread after copying the MDC and request context.
*/
public class CustomThreadPoolExecutor extends ThreadPoolExecutor {
/***
* Calling constructor of the super class.
* @param corePoolSize the number of threads to keep in the pool, even
* if they are idle, unless {@code allowCoreThreadTimeOut} is set
* @param maximumPoolSize the maximum number of threads to allow in the pool
* @param keepAliveTime when the number of threads is greater than
* the core, this is the maximum time that excess idle threads
* will wait for new tasks before terminating.
* @param unit the time unit for the {@code keepAliveTime} argument
* @param workQueue the queue to use for holding tasks before they are
* executed. This queue will hold only the {@code Runnable}
* tasks submitted by the {@code execute} method.
*/
public CustomThreadPoolExecutor(
final int corePoolSize,
final int maximumPoolSize,
final long keepAliveTime,
final TimeUnit unit,
final BlockingQueue<Runnable> workQueue
) {
super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue);
}
/**
* Calling execute method of super class after wrapping the executable code with MDC and thread context.
* @param command
*/
@Override
public void execute(final @NotNull Runnable command) {
super.execute(CustomThreadPoolExecutorUtil.wrapWithContext(command, RequestContextHolder.currentRequestAttributes()));
}
}
package org.opengroup.osdu.azure.concurrency;
import org.jetbrains.annotations.NotNull;
import org.opengroup.osdu.azure.logging.CoreLoggerFactory;
import org.slf4j.MDC;
import org.springframework.web.context.request.RequestAttributes;
import org.springframework.web.context.request.RequestContextHolder;
import java.util.Map;
/***
* Utils class for CustomThreadPoolExecutor {@link CustomThreadPoolExecutor}.
*/
public final class CustomThreadPoolExecutorUtil {
private static final String LOGGER_NAME = CustomThreadPoolExecutorUtil.class.getName();
/***
* Constructor marked as private so that this class is never instantiated.
*/
private CustomThreadPoolExecutorUtil() {
}
/***
* Copy the MDC and request context to the Runnable object being returned.
* @param task instance of Runnable class.
* @param context the RequestAttributes to be copied to new thread.
* @return Runnable instance wrapped with request context and MDC.
*/
public static Runnable wrapWithContext(final @NotNull Runnable task, final RequestAttributes context) {
Map<String, String> contextMap = MDC.getCopyOfContextMap();
return () -> {
//save the current MDC context
setMDCContext(contextMap);
//save the current request attributes
setRequestContext(context);
try {
task.run();
} finally {
// once the task is complete, clear MDC
MDC.clear();
RequestContextHolder.resetRequestAttributes();
}
};
}
/***
* Set the MDC.
* @param contextMap the MDC map to be copied.
*/
private static void setMDCContext(final Map<String, String> contextMap) {
MDC.clear();
if (contextMap != null) {
MDC.setContextMap(contextMap);
} else {
CoreLoggerFactory.getInstance().getLogger(LOGGER_NAME).warn("Cannot set MDC as it is null");
}
}
/***
* Set the Request Attributes.
* @param context the Attributes to be copied.
*/
private static void setRequestContext(final RequestAttributes context) {
RequestContextHolder.resetRequestAttributes();
if (context != null) {
RequestContextHolder.setRequestAttributes(context);
} else {
CoreLoggerFactory.getInstance().getLogger(LOGGER_NAME).warn("Cannot set RequestAttributes as they are null");
}
}
}
\ No newline at end of file
package org.opengroup.osdu.azure.cosmosdb;
import com.microsoft.azure.documentdb.ConnectionPolicy;
import com.microsoft.azure.documentdb.ConsistencyLevel;
import com.microsoft.azure.documentdb.DocumentClient;
import com.microsoft.azure.documentdb.DocumentClientException;
import com.microsoft.azure.documentdb.DocumentCollection;
import com.microsoft.azure.documentdb.bulkexecutor.DocumentBulkExecutor;
import org.opengroup.osdu.azure.cache.CosmosBulkExecutorCache;
import org.opengroup.osdu.azure.partition.PartitionInfoAzure;
import org.opengroup.osdu.azure.partition.PartitionServiceClient;
import org.opengroup.osdu.common.Validators;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
/**
* A factory class to generate DocumentBulkExecutor objects to perform bulk operations.
*/
@Component
@Lazy
public class CosmosBulkExecutorFactoryImpl implements ICosmosBulkExecutorFactory {
private static final Logger LOGGER = LoggerFactory.getLogger(CosmosBulkExecutorFactoryImpl.class.getName());
@Lazy
@Autowired
private PartitionServiceClient partitionService;
@Lazy
@Autowired
private CosmosBulkExecutorCache cosmosBulkExecutorCache;
@Autowired
private int documentClientMaxPoolSize;
@Autowired
private int bulkExecutorMaxRUs;
private final String unformattedCollectionLink = "/dbs/%s/colls/%s";
private final String unformattedCosmosBulkExecutorCacheKey = "%s-%s-%s-cosmosBulkExecutor";
private final String unformattedDocumentClientCacheKey = "%s-documentClient";
/**
*
* @param dataPartitionId name of data partition.
* @param cosmosDBName name of cosmos db.
* @param collectionName name of collection in cosmos.
* @return DocumentBulkExecutor to perform bulk Cosmos opartions.
* @throws Exception if there is an error creating the DocumentBulkExecutor object.
*/
public DocumentBulkExecutor getClient(final String dataPartitionId,
final String cosmosDBName,
final String collectionName) {
Validators.checkNotNullAndNotEmpty(dataPartitionId, "dataPartitionId");
Validators.checkNotNullAndNotEmpty(cosmosDBName, "cosmosDBName");
Validators.checkNotNullAndNotEmpty(collectionName, "collectionName");
String cacheKey = String.format(unformattedCosmosBulkExecutorCacheKey, dataPartitionId, cosmosDBName, collectionName);
if (this.cosmosBulkExecutorCache.containsKey(cacheKey)) {
return this.cosmosBulkExecutorCache.get(cacheKey);
}
PartitionInfoAzure pi = this.partitionService.getPartition(dataPartitionId);
DocumentClient client = getDocumentClient(pi.getCosmosEndpoint(),
pi.getCosmosPrimaryKey());
String collectionLink = String.format(unformattedCollectionLink, cosmosDBName, collectionName);
try {
DocumentCollection collection = client.readCollection(collectionLink, null).getResource();
DocumentBulkExecutor executor = DocumentBulkExecutor.builder().from(
client,
cosmosDBName,
collectionName,
collection.getPartitionKey(),
bulkExecutorMaxRUs
).build();
cosmosBulkExecutorCache.put(String.format(unformattedCosmosBulkExecutorCacheKey, dataPartitionId, cosmosDBName, collectionName), executor);
// Set client retry options to 0 because retries are handled by DocumentBulkExecutor class.
client.getConnectionPolicy().getRetryOptions().setMaxRetryAttemptsOnThrottledRequests(0);
client.getConnectionPolicy().getRetryOptions().setMaxRetryWaitTimeInSeconds(0);
return executor;
} catch (DocumentClientException e) {
String errorMessage = "Unexpectedly failed to create DocumentCollection object";
LOGGER.warn(errorMessage, e);
throw new AppException(500, errorMessage, e.getMessage(), e);
} catch (Exception e) {
String errorMessage = "Unexpectedly failed create DocumentBulkExecutor";
LOGGER.warn(errorMessage, e);
throw new AppException(500, errorMessage, e.getMessage(), e);
}
}
/**
*
* @param cosmosEndpoint endpoint to Cosmos db.
* @param cosmosPrimaryKey primary key for connection to Cosmos db.
* @return DocumentClient object.
*/
private DocumentClient getDocumentClient(final String cosmosEndpoint,
final String cosmosPrimaryKey) {
ConnectionPolicy policy = new ConnectionPolicy();
policy.setMaxPoolSize(documentClientMaxPoolSize);
// Initialize with these values for retries. These are overridden once the DocumentBulkExecutor object is created
policy.getRetryOptions().setMaxRetryWaitTimeInSeconds(30);
policy.getRetryOptions().setMaxRetryAttemptsOnThrottledRequests(9);
DocumentClient client = new DocumentClient(
cosmosEndpoint,
cosmosPrimaryKey,
policy,
ConsistencyLevel.Session
);
return client;
}
}
......@@ -110,11 +110,10 @@ public class CosmosStore {
T item = container.readItem(id, key, options, clazz).getItem();
return Optional.ofNullable((T) item);
} catch (NotFoundException e) {
LOGGER.warn(String.format("Unable to find item with ID=%s and PK=%s", id, partitionKey));
LOGGER.warn(String.format("Unable to find item with ID=%s and PK=%s", id, partitionKey), e);
return Optional.empty();
} catch (CosmosException e) {
String errorMessage;
errorMessage = "Unexpectedly encountered error calling CosmosDB";
String errorMessage = "Unexpectedly encountered error calling CosmosDB";
LOGGER.warn(errorMessage, e);
throw new AppException(500, errorMessage, e.getMessage(), e);
}
......
package org.opengroup.osdu.azure.cosmosdb;
import com.google.gson.Gson;
import com.microsoft.azure.documentdb.DocumentClientException;
import com.microsoft.azure.documentdb.bulkexecutor.BulkImportResponse;
import com.microsoft.azure.documentdb.bulkexecutor.DocumentBulkExecutor;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.Collection;
/**
* Class to perform bulk Cosmos operations using DocumentBulkExecutor.
*/
@Component
@Lazy
public class CosmosStoreBulkOperations {
private static final Logger LOGGER = LoggerFactory.getLogger(CosmosStoreBulkOperations.class.getName());