Commit f22160e1 authored by Vineeth Guna [Microsoft]'s avatar Vineeth Guna [Microsoft]
Browse files

Merge branch 'SystemDags-Core' into 'master'

Adding system dags support in core

See merge request !146
parents 3b6dc2ee 31a0a518
Pipeline #68540 failed with stages
in 55 minutes and 34 seconds
......@@ -28,9 +28,6 @@ The following software have components provided under the terms of this license:
- Netty/Codec (from https://repo1.maven.org/maven2/io/netty/netty-codec)
- Netty/TomcatNative [BoringSSL - Static] (from https://repo1.maven.org/maven2/io/netty/netty-tcnative-boringssl-static)
- SnakeYAML (from http://www.snakeyaml.org)
- Spring Boot Security Starter (from http://projects.spring.io/spring-boot/)
- Spring Boot Test Starter (from http://projects.spring.io/spring-boot/)
- Spring Boot Web Starter (from http://projects.spring.io/spring-boot/)
- Spring Core (from https://github.com/spring-projects/spring-framework)
- Spring Data Core (from https://repo1.maven.org/maven2/org/springframework/data/spring-data-commons)
- Spring Web (from https://github.com/spring-projects/spring-framework)
......@@ -39,15 +36,18 @@ The following software have components provided under the terms of this license:
- jackson-databind (from http://github.com/FasterXML/jackson)
- spring-boot-configuration-processor (from https://spring.io/projects/spring-boot)
- spring-boot-starter-json (from https://spring.io/projects/spring-boot)
- spring-cloud-starter (from http://projects.spring.io/spring-cloud)
- spring-boot-starter-security (from https://spring.io/projects/spring-boot)
- spring-boot-starter-test (from https://spring.io/projects/spring-boot)
- spring-boot-starter-web (from https://spring.io/projects/spring-boot)
- spring-cloud-starter (from https://projects.spring.io/spring-cloud)
- spring-security-oauth2-client (from https://spring.io/spring-security)
- spring-security-oauth2-jose (from https://spring.io/spring-security)
- spring-security-test (from https://spring.io/projects/spring-security)
- spring-security-web (from https://spring.io/projects/spring-security)
- tomcat (from http://tomcat.apache.org/)
- spring-security-test (from https://spring.io/spring-security)
- spring-security-web (from https://spring.io/spring-security)
- tomcat (from https://tomcat.apache.org/)
- tomcat-catalina (from https://tomcat.apache.org/)
- tomcat-embed-core (from http://tomcat.apache.org/)
- tomcat-embed-websocket (from http://tomcat.apache.org/)
- tomcat-embed-websocket (from https://tomcat.apache.org/)
- tomcat-websocket (from http://tomcat.apache.org/)
========================================================================
......
......@@ -49,7 +49,7 @@
<javax.inject.version>1</javax.inject.version>
<org.mapstruct.version>1.3.1.Final</org.mapstruct.version>
<maven-surefire-plugin.version>3.0.0-M4</maven-surefire-plugin.version>
<os-core-common.version>0.11.0-SNAPSHOT</os-core-common.version>
<os-core-common.version>0.12.0-SNAPSHOT</os-core-common.version>
<springfox.version>3.0.0</springfox.version>
</properties>
......
package org.opengroup.osdu.workflow.aws.repository;
import org.opengroup.osdu.workflow.exception.WorkflowNotFoundException;
import org.opengroup.osdu.workflow.model.WorkflowMetadata;
import org.opengroup.osdu.workflow.provider.interfaces.IWorkflowSystemMetadataRepository;
import org.springframework.stereotype.Repository;
import org.springframework.web.context.annotation.RequestScope;
import java.util.ArrayList;
import java.util.List;
@Repository
@RequestScope
public class AwsWorkflowSystemMetadataRepository implements IWorkflowSystemMetadataRepository {
/**
* Returns workflow metadata based on workflowName
*
* @param workflowName Name of the workflow for which metadata should be retrieved.
* @return Workflow metadata
*/
@Override
public WorkflowMetadata getSystemWorkflow(String workflowName) {
throw new WorkflowNotFoundException(String.format("Workflow: '%s' not found", workflowName));
}
/**
* Creates workflow metadata record in persistence store.
*
* @param workflowMetadata Workflow metadata object to save in persistence store.
* @return Workflow metadata
*/
@Override
public WorkflowMetadata createSystemWorkflow(WorkflowMetadata workflowMetadata) {
return null;
}
/**
* Deletes workflow metadata based on workflowName
*
* @param workflowName Name of the workflow for which metadata should be deleted.
*/
@Override
public void deleteSystemWorkflow(String workflowName) { }
/**
* Get all system workflows metadata based on prefix
*
* @param prefix Name of the system workflow for which metadata should be deleted.
*/
@Override
public List<WorkflowMetadata> getAllSystemWorkflow(String prefix) {
return new ArrayList<>();
}
}
package org.opengroup.osdu.workflow.aws.service;
import org.opengroup.osdu.workflow.provider.interfaces.IAdminAuthorizationService;
import org.springframework.stereotype.Component;
@Component
public class AdminAuthorizationServiceImpl implements IAdminAuthorizationService {
@Override
public boolean isDomainAdminServiceAccount() {
return false;
}
}
package org.opengroup.osdu.workflow.provider.azure.interfaces;
public interface IAuthorizationServiceSP {
boolean isDomainAdminServiceAccount();
}
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.workflow.provider.azure.interfaces;
import org.opengroup.osdu.workflow.model.WorkflowMetadata;
import org.opengroup.osdu.workflow.model.CreateWorkflowRequest;
public interface IWorkflowSystemManagerService {
/**
* Creates workflow with given request.
* @param request Request object which has information to create workflow.
* @return Workflow metadata.
*/
WorkflowMetadata createSystemWorkflow(final CreateWorkflowRequest request);
/**
* Deletes workflow based on workflowName
* @param workflowName Id of the workflow which needs to be deleted.
*/
void deleteSystemWorkflow(final String workflowName);
}
......@@ -2,8 +2,9 @@ package org.opengroup.osdu.workflow.provider.azure.repository;
import com.azure.cosmos.CosmosException;
import com.azure.cosmos.models.CosmosQueryRequestOptions;
import com.azure.cosmos.models.SqlParameter;
import com.azure.cosmos.models.SqlQuerySpec;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.opengroup.osdu.azure.cosmosdb.CosmosStore;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
......@@ -15,144 +16,83 @@ import org.opengroup.osdu.workflow.provider.azure.config.AzureWorkflowEngineConf
import org.opengroup.osdu.workflow.provider.azure.config.CosmosConfig;
import org.opengroup.osdu.workflow.provider.azure.model.WorkflowMetadataDoc;
import org.opengroup.osdu.workflow.provider.interfaces.IWorkflowMetadataRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.apache.commons.lang3.StringUtils;
import org.opengroup.osdu.workflow.provider.azure.utils.ThreadLocalUtils;
import org.springframework.stereotype.Repository;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.ArrayList;
@Component
import static org.opengroup.osdu.workflow.provider.azure.utils.WorkflowMetadataUtils.*;
@Slf4j
@Repository
@RequiredArgsConstructor
public class WorkflowMetadataRepository implements IWorkflowMetadataRepository {
private static final String LOGGER_NAME = WorkflowMetadataRepository.class.getName();
private static final String KEY_DAG_CONTENT = "dagContent";
private static final String KEY_DAG_TYPE = "dagType";
@Autowired
private CosmosConfig cosmosConfig;
private final CosmosConfig cosmosConfig;
@Autowired
private CosmosStore cosmosStore;
private final CosmosStore cosmosStore;
@Autowired
private DpsHeaders dpsHeaders;
private final DpsHeaders dpsHeaders;
@Autowired
private JaxRsDpsLog logger;
private final JaxRsDpsLog logger;
@Autowired
private AzureWorkflowEngineConfig workflowEngineConfig;
private final AzureWorkflowEngineConfig workflowEngineConfig;
@Override
public WorkflowMetadata createWorkflow(final WorkflowMetadata workflowMetadata) {
final WorkflowMetadataDoc workflowMetadataDoc = buildWorkflowMetadataDoc(workflowMetadata);
final WorkflowMetadataDoc workflowMetadataDoc = buildWorkflowMetadataDoc(workflowEngineConfig, workflowMetadata);
try {
if(StringUtils.isEmpty(dpsHeaders.getPartitionId())) {
cosmosStore.createItem( cosmosConfig.getSystemdatabase(),
cosmosConfig.getWorkflowMetadataCollection(), workflowMetadataDoc.getPartitionKey(),
workflowMetadataDoc);
}
else {
cosmosStore.createItem(dpsHeaders.getPartitionId(), cosmosConfig.getDatabase(),
cosmosConfig.getWorkflowMetadataCollection(), workflowMetadataDoc.getPartitionKey(),
workflowMetadataDoc);
}
return buildWorkflowMetadata(workflowMetadataDoc);
} catch (AppException e) {
String workflowName = workflowMetadataDoc.getWorkflowName();
if(e.getError().getCode() == 409) {
final String errorMessage = String.format("Workflow with name %s already exists",
workflowMetadataDoc.getWorkflowName());
logger.error(errorMessage, e);
throw new ResourceConflictException(workflowMetadataDoc.getWorkflowName(), errorMessage);
final String errorMessage = String.format("Workflow with name %s already exists", workflowName);
logger.error(LOGGER_NAME, errorMessage);
throw new ResourceConflictException(workflowName, errorMessage);
} else {
throw e;
}
}
return buildWorkflowMetadata(workflowMetadataDoc);
}
@Override
public WorkflowMetadata getWorkflow(final String workflowName) {
Optional<WorkflowMetadataDoc> workflowMetadataDoc=null;
//this means this call is from deleteWorkflow system endpoint
if(StringUtils.isEmpty(dpsHeaders.getPartitionId())) {
workflowMetadataDoc= getSystemWorkflow(workflowName);
}
else {
workflowMetadataDoc= getPrivateWorkflow(workflowName);
if(null == workflowMetadataDoc || !workflowMetadataDoc.isPresent()) {
workflowMetadataDoc= getSystemWorkflow(workflowName);
}
}
Optional<WorkflowMetadataDoc> workflowMetadataDoc =
cosmosStore.findItem(
dpsHeaders.getPartitionId(),
cosmosConfig.getDatabase(),
cosmosConfig.getWorkflowMetadataCollection(),
workflowName,
workflowName,
WorkflowMetadataDoc.class
);
if (null == workflowMetadataDoc || !workflowMetadataDoc.isPresent()) {
final String errorMessage = String.format("Workflow: %s doesn't exist", workflowName);
logger.error(LOGGER_NAME, errorMessage);
throw new WorkflowNotFoundException(errorMessage);
}
WorkflowMetadata workflowMetadata= buildWorkflowMetadata(workflowMetadataDoc.get());
return workflowMetadata;
}
private Optional<WorkflowMetadataDoc> getSystemWorkflow(final String workflowName) {
final Optional<WorkflowMetadataDoc> workflowSystemMetadataDoc =
cosmosStore.findItem(
cosmosConfig.getSystemdatabase(),
cosmosConfig.getWorkflowMetadataCollection(),
workflowName,
workflowName,
WorkflowMetadataDoc.class);
if(null != workflowSystemMetadataDoc && workflowSystemMetadataDoc.isPresent()) {
ThreadLocalUtils.setSystemDagFlag(true);
Map<String, Object> registrationInstructions = workflowSystemMetadataDoc.get().getRegistrationInstructions();
registrationInstructions.put(KEY_DAG_TYPE, "system");
}
return workflowSystemMetadataDoc;
return buildWorkflowMetadata(workflowMetadataDoc.get());
}
private Optional<WorkflowMetadataDoc> getPrivateWorkflow(final String workflowName) {
final Optional<WorkflowMetadataDoc> workflowMetadataDoc =
cosmosStore.findItem(dpsHeaders.getPartitionId(),
cosmosConfig.getDatabase(),
cosmosConfig.getWorkflowMetadataCollection(),
workflowName,
workflowName,
WorkflowMetadataDoc.class);
if(null != workflowMetadataDoc && workflowMetadataDoc.isPresent()) {
ThreadLocalUtils.setSystemDagFlag(false);
Map<String, Object> registrationInstructions = workflowMetadataDoc.get().getRegistrationInstructions();
registrationInstructions.put(KEY_DAG_TYPE, "private");
}
return workflowMetadataDoc;
}
@Override
public void deleteWorkflow(String workflowName) {
if(StringUtils.isEmpty(dpsHeaders.getPartitionId())) {
cosmosStore.deleteItem( cosmosConfig.getSystemdatabase(),
cosmosConfig.getWorkflowMetadataCollection(), workflowName, workflowName);
}
else {
cosmosStore.deleteItem(dpsHeaders.getPartitionId(), cosmosConfig.getDatabase(),
cosmosConfig.getWorkflowMetadataCollection(), workflowName, workflowName);
}
cosmosStore.deleteItem(
dpsHeaders.getPartitionId(),
cosmosConfig.getDatabase(),
cosmosConfig.getWorkflowMetadataCollection(),
workflowName,
workflowName
);
}
@Override
public List<WorkflowMetadata> getAllWorkflowForTenant(String prefix) {
try {
SqlQuerySpec sqlQuerySpec;
if(prefix != null && !(prefix.isEmpty())) {
SqlParameter prefixParameter = new SqlParameter("@prefix", prefix);
sqlQuerySpec = new SqlQuerySpec("SELECT * FROM c " +
"where STARTSWITH(c.workflowName, @prefix, true) " +
"ORDER BY c._ts DESC", prefixParameter);
}
else {
sqlQuerySpec = new SqlQuerySpec("SELECT * FROM c " +
"ORDER BY c._ts DESC");
}
SqlQuerySpec sqlQuerySpec = buildSqlQuerySpecForGetAllWorkflow(prefix);
final List<WorkflowMetadataDoc> workflowMetadataDocs = cosmosStore.queryItems(
dpsHeaders.getPartitionId(),
cosmosConfig.getDatabase(),
......@@ -160,59 +100,9 @@ public class WorkflowMetadataRepository implements IWorkflowMetadataRepository {
sqlQuerySpec,
new CosmosQueryRequestOptions(),
WorkflowMetadataDoc.class);
//looking for shared workflows
final List<WorkflowMetadataDoc> workflowSystemMetadataDocs = cosmosStore.queryItems(
cosmosConfig.getSystemdatabase(),
cosmosConfig.getWorkflowMetadataCollection(),
sqlQuerySpec,
new CosmosQueryRequestOptions(),
WorkflowMetadataDoc.class);
workflowMetadataDocs.addAll(workflowSystemMetadataDocs);
List<WorkflowMetadata> workflowMetadataList = new ArrayList<>();
for(WorkflowMetadataDoc workflowMetadataDoc: workflowMetadataDocs)
{
workflowMetadataList.add(buildWorkflowMetadata(workflowMetadataDoc));
}
return workflowMetadataList;
return convertWorkflowMetadataDocsToWorkflowMetadataList(workflowMetadataDocs);
} catch (CosmosException e) {
throw new AppException(e.getStatusCode(), e.getMessage(), e.getMessage(), e);
}
}
private WorkflowMetadataDoc buildWorkflowMetadataDoc(final WorkflowMetadata workflowMetadata) {
// If we need to save multiple versions of workflow, then choose id as guid and get becomes a query.
// This is to avoid conflicts. Only one combination of Id and partition key should exist.
Map<String, Object> registrationInstructionForMetadata =
new HashMap<>(workflowMetadata.getRegistrationInstructions());
String dagContent =
(String) registrationInstructionForMetadata.remove(KEY_DAG_CONTENT);
if (workflowEngineConfig.getIgnoreDagContent()) {
dagContent = "";
}
return WorkflowMetadataDoc.builder()
.id(workflowMetadata.getWorkflowName())
.partitionKey(workflowMetadata.getWorkflowName())
.workflowName(workflowMetadata.getWorkflowName())
.description(workflowMetadata.getDescription())
.createdBy(workflowMetadata.getCreatedBy())
.creationTimestamp(workflowMetadata.getCreationTimestamp())
.version(workflowMetadata.getVersion())
.isRegisteredByWorkflowService(
dagContent != null && !dagContent.isEmpty())
.registrationInstructions(registrationInstructionForMetadata).build();
}
private WorkflowMetadata buildWorkflowMetadata(final WorkflowMetadataDoc workflowMetadataDoc) {
return WorkflowMetadata.builder()
.workflowId(workflowMetadataDoc.getId())
.workflowName(workflowMetadataDoc.getWorkflowName())
.description(workflowMetadataDoc.getDescription())
.createdBy(workflowMetadataDoc.getCreatedBy())
.creationTimestamp(workflowMetadataDoc.getCreationTimestamp())
.version(workflowMetadataDoc.getVersion())
.isDeployedThroughWorkflowService(workflowMetadataDoc.getIsRegisteredByWorkflowService())
.registrationInstructions(workflowMetadataDoc.getRegistrationInstructions()).build();
}
}
package org.opengroup.osdu.workflow.provider.azure.repository;
import com.azure.cosmos.models.CosmosQueryRequestOptions;
import com.azure.cosmos.models.SqlQuerySpec;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.opengroup.osdu.azure.cosmosdb.CosmosStore;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.workflow.exception.ResourceConflictException;
import org.opengroup.osdu.workflow.exception.WorkflowNotFoundException;
import org.opengroup.osdu.workflow.model.WorkflowMetadata;
import org.opengroup.osdu.workflow.provider.azure.config.AzureWorkflowEngineConfig;
import org.opengroup.osdu.workflow.provider.azure.config.CosmosConfig;
import org.opengroup.osdu.workflow.provider.azure.model.WorkflowMetadataDoc;
import org.opengroup.osdu.workflow.provider.interfaces.IWorkflowSystemMetadataRepository;
import org.springframework.stereotype.Repository;
import java.util.List;
import java.util.Optional;
import static org.opengroup.osdu.workflow.provider.azure.utils.WorkflowMetadataUtils.*;
@Slf4j
@Repository
@RequiredArgsConstructor
public class WorkflowSystemMetadataRepository implements IWorkflowSystemMetadataRepository {
private static final String LOGGER_NAME = WorkflowMetadataRepository.class.getName();
private final CosmosConfig cosmosConfig;
private final CosmosStore cosmosStore;
private final JaxRsDpsLog logger;
private final AzureWorkflowEngineConfig workflowEngineConfig;
@Override
public WorkflowMetadata getSystemWorkflow(final String workflowName) {
Optional<WorkflowMetadataDoc> workflowSystemMetadataDoc =
cosmosStore.findItem(
cosmosConfig.getSystemdatabase(),
cosmosConfig.getWorkflowMetadataCollection(),
workflowName,
workflowName,
WorkflowMetadataDoc.class
);
if (null == workflowSystemMetadataDoc || !workflowSystemMetadataDoc.isPresent()) {
final String errorMessage = String.format("Workflow: %s doesn't exist", workflowName);
logger.error(LOGGER_NAME, errorMessage);
throw new WorkflowNotFoundException(errorMessage);
}
return buildWorkflowMetadata(workflowSystemMetadataDoc.get());
}
@Override
public WorkflowMetadata createSystemWorkflow(final WorkflowMetadata workflowMetadata) {
final WorkflowMetadataDoc workflowMetadataDoc = buildWorkflowMetadataDoc(workflowEngineConfig, workflowMetadata);
try {
cosmosStore.createItem(
cosmosConfig.getSystemdatabase(),
cosmosConfig.getWorkflowMetadataCollection(),
workflowMetadataDoc.getPartitionKey(),
workflowMetadataDoc
);
} catch (AppException e) {
String workflowName = workflowMetadataDoc.getWorkflowName();
if(e.getError().getCode() == 409) {
final String errorMessage = String.format("Workflow with name %s already exists", workflowName);
logger.error(LOGGER_NAME, errorMessage);
throw new ResourceConflictException(workflowName, errorMessage);
} else {
throw e;
}
}
return buildWorkflowMetadata(workflowMetadataDoc);
}
@Override
public void deleteSystemWorkflow(String workflowName) {
cosmosStore.deleteItem(
cosmosConfig.getSystemdatabase(),
cosmosConfig.getWorkflowMetadataCollection(),
workflowName,
workflowName
);
}
@Override
public List<WorkflowMetadata> getAllSystemWorkflow(String prefix) {
SqlQuerySpec sqlQuerySpec = buildSqlQuerySpecForGetAllWorkflow(prefix);
final List<WorkflowMetadataDoc> workflowSystemMetadataDocs = cosmosStore.queryItems(
cosmosConfig.getSystemdatabase(),
cosmosConfig.getWorkflowMetadataCollection(),
sqlQuerySpec,
new CosmosQueryRequestOptions(),
WorkflowMetadataDoc.class);
return convertWorkflowMetadataDocsToWorkflowMetadataList(workflowSystemMetadataDocs);
}
}
package org.opengroup.osdu.workflow.provider.azure.security;
import org.apache.commons.lang3.StringUtils;
import org.opengroup.osdu.core.common.exception.BadRequestException;
import org.opengroup.osdu.core.common.exception.UnauthorizedException;
import org.opengroup.osdu.workflow.provider.azure.interfaces.IAuthorizationServiceSP;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.web.context.annotation.RequestScope;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@Component("authorizationFilterSP")
@RequestScope
public class AuthorizationFilterSP {
@Autowired
private IAuthorizationServiceSP authorizationService;
final DpsHeaders headers;
public boolean hasPermissions() {
validateMandatoryHeaders();
headers.put(DpsHeaders.USER_EMAIL, "ServicePrincipalUser");
return authorizationService.isDomainAdminServiceAccount();
}
private void validateMandatoryHeaders() {
if (StringUtils.isEmpty(this.headers.getAuthorization())) {
throw new UnauthorizedException("Authorization header is mandatory");
}
if (!StringUtils.isEmpty(this.headers.getPartitionId())) {