Commit 0bc4e188 authored by Abhiman Neelakanteswara's avatar Abhiman Neelakanteswara
Browse files

Merge branch 'master' into bug/fix_swagger_issue

parents ed36b944 3b0ebfd7
Pipeline #46876 failed with stages
in 17 seconds
......@@ -27,7 +27,7 @@
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-workflow</artifactId>
<version>0.9.0-SNAPSHOT</version>
<version>0.10.0-SNAPSHOT</version>
<packaging>pom</packaging>
<name>os-workflow</name>
......
......@@ -19,12 +19,12 @@
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-workflow</artifactId>
<version>0.9.0-SNAPSHOT</version>
<version>0.10.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>workflow-aws</artifactId>
<version>0.9.0-SNAPSHOT</version>
<version>0.10.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>workflow-aws</name>
......@@ -34,7 +34,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>workflow-core</artifactId>
<version>0.9.0-SNAPSHOT</version>
<version>0.10.0-SNAPSHOT</version>
</dependency>
<dependency>
......
......@@ -20,7 +20,7 @@
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-workflow</artifactId>
<version>0.9.0-SNAPSHOT</version>
<version>0.10.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
......@@ -31,7 +31,7 @@
<properties>
<azure.version>2.1.7</azure.version>
<osdu.corelibazure.version>0.9.0</osdu.corelibazure.version>
<osdu.corelibazure.version>0.10.0-rc4</osdu.corelibazure.version>
<azure.appservice.resourcegroup />
<azure.appservice.plan />
<azure.appservice.appname />
......@@ -88,7 +88,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>workflow-core</artifactId>
<version>0.9.0-SNAPSHOT</version>
<version>0.10.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
......
package org.opengroup.osdu.workflow.provider.azure.cache;
import com.azure.storage.blob.BlobServiceClient;
import org.opengroup.osdu.core.common.cache.VmCache;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
@Component
@Lazy
public class IngestBlobServiceClientCache extends VmCache<String, BlobServiceClient> {
public IngestBlobServiceClientCache() {
super(60 * 60, 1000);
}
public boolean containsKey(final String key) {
return this.get(key) != null;
}
}
......@@ -14,13 +14,6 @@
package org.opengroup.osdu.workflow.provider.azure.config;
import com.azure.cosmos.CosmosClient;
import com.azure.cosmos.CosmosClientBuilder;
import com.azure.security.keyvault.secrets.SecretClient;
import com.azure.storage.blob.BlobServiceClient;
import com.azure.storage.blob.BlobServiceClientBuilder;
import com.azure.storage.common.StorageSharedKeyCredential;
import org.opengroup.osdu.azure.KeyVaultFacade;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
......@@ -36,16 +29,6 @@ public class AzureBootstrapConfig {
@Value("${osdu.azure.partitionId}")
private String partitionId;
@Bean
public BlobServiceClient buildBlobServiceClient(SecretClient kv) {
final String partitionId = getPartitionId();
final String accountName = KeyVaultFacade.getSecretWithValidation(kv, String.format("%s-ingest-storage", partitionId));
final String accountKey = KeyVaultFacade.getSecretWithValidation(kv, String.format("%s-ingest-storage-key", partitionId));
final StorageSharedKeyCredential credential = new StorageSharedKeyCredential(accountName, accountKey);
String endpoint = String.format("https://%s.blob.core.windows.net", accountName);
return new BlobServiceClientBuilder().endpoint(endpoint).credential(credential).buildClient();
}
@Bean
@Named("KEY_VAULT_URL")
public String keyVaultURL() {
......
package org.opengroup.osdu.workflow.provider.azure.config;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
@ConfigurationProperties("azure.storage")
@Configuration
@Getter
@Setter
public class BlobConfig {
private String tasksSharingStorageAccount;
}
package org.opengroup.osdu.workflow.provider.azure.config;
import com.azure.identity.DefaultAzureCredential;
import com.azure.storage.blob.BlobServiceClient;
import com.azure.storage.blob.BlobServiceClientBuilder;
import org.opengroup.osdu.azure.blobstorage.IBlobServiceClientFactory;
import org.opengroup.osdu.azure.partition.PartitionInfoAzure;
import org.opengroup.osdu.azure.partition.PartitionServiceClient;
import org.opengroup.osdu.common.Validators;
import org.opengroup.osdu.workflow.provider.azure.cache.IngestBlobServiceClientCache;
class BlobServiceIngestClientFactory implements IBlobServiceClientFactory {
private DefaultAzureCredential defaultAzureCredential;
private PartitionServiceClient partitionService;
private IngestBlobServiceClientCache clientCache;
public BlobServiceIngestClientFactory(final DefaultAzureCredential credentials,
final PartitionServiceClient partitionServiceClient,
final IngestBlobServiceClientCache blobServiceClientCache) {
this.defaultAzureCredential = credentials;
this.partitionService = partitionServiceClient;
this.clientCache = blobServiceClientCache;
}
@Override
public BlobServiceClient getBlobServiceClient(final String dataPartitionId) {
Validators.checkNotNull(defaultAzureCredential, "Credential");
Validators.checkNotNullAndNotEmpty(dataPartitionId, "dataPartitionId");
String cacheKey = String.format("%s-ingest-blobServiceClient", dataPartitionId);
if (this.clientCache.containsKey(cacheKey)) {
return this.clientCache.get(cacheKey);
}
PartitionInfoAzure pi = this.partitionService.getPartition(dataPartitionId);
String endpoint = String.format("https://%s.blob.core.windows.net", pi.getIngestStorageAccountName());
BlobServiceClient blobServiceClient = new BlobServiceClientBuilder()
.endpoint(endpoint)
.credential(defaultAzureCredential)
.buildClient();
this.clientCache.put(cacheKey, blobServiceClient);
return blobServiceClient;
}
}
package org.opengroup.osdu.workflow.provider.azure.config;
import com.azure.identity.DefaultAzureCredential;
import org.opengroup.osdu.azure.blobstorage.BlobStore;
import org.opengroup.osdu.azure.blobstorage.IBlobServiceClientFactory;
import org.opengroup.osdu.azure.partition.PartitionServiceClient;
import org.opengroup.osdu.core.common.logging.ILogger;
import org.opengroup.osdu.workflow.provider.azure.cache.IngestBlobServiceClientCache;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class BlobStoreProviderIngest {
@Autowired
DefaultAzureCredential defaultAzureCredential;
@Autowired
PartitionServiceClient partitionService;
@Autowired
IngestBlobServiceClientCache clientCache;
@Bean
@Qualifier("IngestBlobServiceClientFactory")
public IBlobServiceClientFactory buildBlobClientFactory(final DefaultAzureCredential defaultAzureCredential,
final PartitionServiceClient partitionServiceClient,
final IngestBlobServiceClientCache blobServiceClientCache) {
return new BlobServiceIngestClientFactory(defaultAzureCredential, partitionServiceClient, blobServiceClientCache);
}
@Bean
@Qualifier("IngestBlobStore")
public BlobStore buildBlobStore(@Qualifier("IngestBlobServiceClientFactory") final IBlobServiceClientFactory blobServiceClientFactory, final ILogger logger) {
return new BlobStore(blobServiceClientFactory, logger);
}
}
package org.opengroup.osdu.workflow.provider.azure.config;
import com.azure.storage.blob.BlobServiceClient;
import org.opengroup.osdu.azure.blobstorage.IBlobServiceClientFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
@Component
@Primary
class SinglePartitionBlobServiceClientFactory implements IBlobServiceClientFactory {
@Autowired
private BlobServiceClient blobServiceClient;
@Override
public BlobServiceClient getBlobServiceClient(String dataPartitionId) {
return blobServiceClient;
}
}
......@@ -3,7 +3,6 @@ package org.opengroup.osdu.workflow.provider.azure.repository;
import com.azure.cosmos.CosmosException;
import com.azure.cosmos.models.SqlParameter;
import com.azure.cosmos.models.SqlQuerySpec;
import org.opengroup.osdu.azure.blobstorage.BlobStore;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.azure.cosmosdb.CosmosStore;
import org.opengroup.osdu.azure.query.CosmosStorePageRequest;
......@@ -15,21 +14,21 @@ import org.opengroup.osdu.workflow.model.WorkflowRun;
import org.opengroup.osdu.workflow.model.WorkflowRunsPage;
import org.opengroup.osdu.workflow.model.WorkflowStatusType;
import org.opengroup.osdu.workflow.provider.azure.config.CosmosConfig;
import org.opengroup.osdu.workflow.provider.azure.consts.WorkflowRunConstants;
import org.opengroup.osdu.workflow.provider.azure.model.WorkflowRunDoc;
import org.opengroup.osdu.workflow.provider.azure.utils.CursorUtils;
import org.opengroup.osdu.workflow.provider.interfaces.IWorkflowRunRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.stereotype.Component;
import org.opengroup.osdu.workflow.provider.azure.consts.WorkflowRunConstants;
import static org.opengroup.osdu.workflow.model.WorkflowStatusType.getCompletedStatusTypes;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import static org.opengroup.osdu.workflow.model.WorkflowStatusType.getCompletedStatusTypes;
@Component
public class WorkflowRunRepository implements IWorkflowRunRepository {
......@@ -53,9 +52,6 @@ public class WorkflowRunRepository implements IWorkflowRunRepository {
@Autowired
private WorkflowTasksSharingRepository workflowTasksSharingRepository;
@Autowired
private BlobStore blobStore;
@Override
public WorkflowRun saveWorkflowRun(final WorkflowRun workflowRun) {
final WorkflowRunDoc workflowRunDoc = buildWorkflowRunDoc(workflowRun);
......
......@@ -7,9 +7,10 @@ import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.workflow.exception.WorkflowNotFoundException;
import org.opengroup.osdu.workflow.provider.azure.config.CosmosConfig;
import org.opengroup.osdu.workflow.provider.azure.model.WorkflowTasksSharingDoc;
import org.opengroup.osdu.workflow.provider.azure.interfaces.IWorkflowTasksSharingRepository;
import org.opengroup.osdu.workflow.provider.azure.model.WorkflowTasksSharingDoc;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component;
import java.time.OffsetDateTime;
......@@ -22,6 +23,7 @@ public class WorkflowTasksSharingRepository implements IWorkflowTasksSharingRepo
private static final String LOGGER_NAME = WorkflowTasksSharingRepository.class.getName();
@Autowired
@Qualifier("IngestBlobStore")
BlobStore blobStore;
@Autowired
......@@ -40,6 +42,7 @@ public class WorkflowTasksSharingRepository implements IWorkflowTasksSharingRepo
public String getSignedUrl(String workflowName, String runId) {
final String dataPartitionId = dpsHeaders.getPartitionId();
// TODO : Add support for using user provided expiry time and permissions (?)
final OffsetDateTime startTime = OffsetDateTime.now();
final int expiryDays = 7;
final OffsetDateTime expiryTime = OffsetDateTime.now().plusDays(expiryDays);
// TODO : Add support for custom permission (?)
......@@ -63,7 +66,7 @@ public class WorkflowTasksSharingRepository implements IWorkflowTasksSharingRepo
cosmosStore.createItem(dataPartitionId, cosmosConfig.getDatabase(),
cosmosConfig.getWorkflowTasksSharingCollection(), workflowTasksSharingDocNewContainer.getPartitionKey(), workflowTasksSharingDocNewContainer);
}
return blobStore.generatePreSignedURL(dataPartitionId, containerId, expiryTime, permissions);
return blobStore.generatePreSignedUrlWithUserDelegationSas(dataPartitionId, containerId, startTime, expiryTime, permissions);
}
WorkflowTasksSharingDoc workflowTasksSharingDocBuilder(String workflowName, String runId, String containerId) {
......
......@@ -64,8 +64,4 @@ logging.mdccontext.enabled=true
osdu.entitlements.url=${entitlements_service_endpoint}
osdu.entitlements.appKey=${entitlements_service_api_key}
# Blobstore config
azure.blobStore.required=true
azure.storage.tasksSharingStorageAccount=${tasks_sharing_storage_account}
osdu.azure.partitionId=opendes
......@@ -81,7 +81,7 @@ public class WorkflowTasksSharingRepositoryTest {
ArgumentCaptor<String> containerIdCaptor = ArgumentCaptor.forClass(String.class);
ArgumentCaptor<BlobContainerSasPermission> blobContainerSasPermissionArgumentCaptor = ArgumentCaptor.forClass(BlobContainerSasPermission.class);
verify(blobStore).generatePreSignedURL(eq(PARTITION_ID), containerIdCaptor.capture(), any(), blobContainerSasPermissionArgumentCaptor.capture());
verify(blobStore).generatePreSignedUrlWithUserDelegationSas(eq(PARTITION_ID), containerIdCaptor.capture(), any(), any(), blobContainerSasPermissionArgumentCaptor.capture());
String containerId = containerIdCaptor.getValue();
BlobContainerSasPermission blobContainerSasPermission = blobContainerSasPermissionArgumentCaptor.getValue();
......@@ -100,7 +100,7 @@ public class WorkflowTasksSharingRepositoryTest {
verify(blobStore).createBlobContainer(eq(PARTITION_ID), containerIdCaptor.capture());
String containerId = containerIdCaptor.getValue();
verify(blobStore).generatePreSignedURL(eq(PARTITION_ID), eq(containerId), any(), blobContainerSasPermissionArgumentCaptor.capture());
verify(blobStore).generatePreSignedUrlWithUserDelegationSas(eq(PARTITION_ID), eq(containerId), any(), any(), blobContainerSasPermissionArgumentCaptor.capture());
BlobContainerSasPermission blobContainerSasPermission = blobContainerSasPermissionArgumentCaptor.getValue();
verify(cosmosStore).createItem(
eq(PARTITION_ID),
......
......@@ -64,10 +64,7 @@ logging.mdccontext.enabled=true
osdu.entitlements.url=${entitlements_service_endpoint}
osdu.entitlements.appKey=${entitlements_service_api_key}
# Blobstore config
azure.blobStore.required=true
azure.storage.tasksSharingStorageAccount=${tasks_sharing_storage_account}
osdu.azure.partitionId=opendes
spring.main.allow-bean-definition-overriding=true
......@@ -21,12 +21,12 @@
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-workflow</artifactId>
<version>0.9.0-SNAPSHOT</version>
<version>0.10.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>workflow-gcp</artifactId>
<version>0.9.0-SNAPSHOT</version>
<version>0.10.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>workflow-gcp</name>
......@@ -40,7 +40,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>workflow-core</artifactId>
<version>0.9.0-SNAPSHOT</version>
<version>0.10.0-SNAPSHOT</version>
</dependency>
<dependency>
......
package org.opengroup.osdu.workflow.provider.gcp.di;
/*
* Copyright 2020 Google LLC
* Copyright 2020 EPAM Systems, Inc
......@@ -17,6 +15,8 @@ package org.opengroup.osdu.workflow.provider.gcp.di;
* limitations under the License.
*/
package org.opengroup.osdu.workflow.provider.gcp.di;
import org.opengroup.osdu.core.common.cache.ICache;
import org.opengroup.osdu.core.common.cache.VmCache;
import org.opengroup.osdu.core.gcp.multitenancy.credentials.DatastoreCredential;
......
......@@ -203,7 +203,7 @@ public class GcpWorkflowMetadataRepository implements IWorkflowMetadataRepositor
txn.commit();
workflowMetadata.setWorkflowId(entity.getKey().getName());
} else {
workflowMetadata.setWorkflowId(tasks.next().getKey().getName());
throw new AppException(HttpStatus.CONFLICT.value(), "Conflict", String.format("Workflow with name %s already exists.", workflowName));
}
} catch (DatastoreException ex) {
throw new PersistenceException(ex.getCode(), ex.getMessage(), ex.getReason());
......
......@@ -16,7 +16,6 @@
package org.opengroup.osdu.workflow.provider.gcp.security;
import org.springframework.context.annotation.Configuration;
import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity;
import org.springframework.security.config.annotation.web.builders.HttpSecurity;
import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
......
......@@ -6,12 +6,12 @@
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-workflow</artifactId>
<version>0.9.0-SNAPSHOT</version>
<version>0.10.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>workflow-ibm</artifactId>
<version>0.9.0-SNAPSHOT</version>
<version>0.10.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>workflow-ibm</name>
......@@ -27,7 +27,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>workflow-core</artifactId>
<version>0.9.0-SNAPSHOT</version>
<version>0.10.0-SNAPSHOT</version>
</dependency>
<dependency>
......
......@@ -19,7 +19,7 @@
<groupId>org.opengroup.osdu</groupId>
<artifactId>workflow-test</artifactId>
<version>0.9.0-SNAPSHOT</version>
<version>0.10.0-SNAPSHOT</version>
<description>Workflow Service Integration Test Root Project</description>
<packaging>pom</packaging>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment