Commit f5c7e6c2 authored by Pingjiang Wang's avatar Pingjiang Wang
Browse files

workflow serivce - azure implementation.

parent d315c9eb
Pipeline #2802 passed with stages
in 43 minutes and 30 seconds
......@@ -37,6 +37,7 @@
<modules>
<module>workflow-core</module>
<module>provider/workflow-azure</module>
<module>provider/workflow-gcp</module>
<!-- <module>provider/workflow-ibm</module> Fix: Missing classes-->
<module>provider/workflow-gcp-datastore</module>
......
##
# Needed to run the service
##
export LOG_PREFIX=
export entitlements_service_endpoint=
export entitlements_service_api_key=
export azure.activedirectory.session-stateless=
export aad_client_id=
export azure.activedirectory.AppIdUri=
export cosmosdb_database=
export KEYVAULT_URI=
export AZURE_CLIENT_ID=
export AZURE_TENANT_ID=
export AZURE_CLIENT_SECRET=
export appinsights_key=
export airflow_url=
export airflow_username=
export airflow_password=
##
# Needed to run the integration tests
##
export INTEGRATION_TESTER=
export MY_TENANT=
export AZURE_TESTER_SERVICEPRINCIPAL_SECRET=
export AZURE_AD_TENANT_ID=
export AZURE_AD_APP_RESOURCE_ID=
export HOST_URL=
export ENTITLEMENT_URL=
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright © Microsoft Corporation
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
<servers>
<server>
<id>azure-auth</id>
<configuration>
<tenant>${AZURE_DEPLOY_TENANT}</tenant>
<client>${AZURE_DEPLOY_CLIENT_ID}</client>
<key>${AZURE_DEPLOY_CLIENT_SECRET}</key>
<environment>AZURE</environment>
</configuration>
</server>
</servers>
</settings>
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright © Microsoft Corporation
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-workflow</artifactId>
<version>0.0.1-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>workflow-azure</artifactId>
<description>Workflow service on Azure</description>
<packaging>jar</packaging>
<properties>
<azure.version>2.1.7</azure.version>
<osdu.azurecore.version>0.0.6-SNAPSHOT</osdu.azurecore.version>
<azure.appservice.resourcegroup></azure.appservice.resourcegroup>
<azure.appservice.plan></azure.appservice.plan>
<azure.appservice.appname></azure.appservice.appname>
<azure.appservice.subscription></azure.appservice.subscription>
</properties>
<dependencies>
<dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>azure-active-directory-spring-boot-starter</artifactId>
<version>${azure.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-security</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-oauth2-client</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-oauth2-jose</artifactId>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-azure</artifactId>
<version>${osdu.azurecore.version}</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>workflow-core</artifactId>
<version>0.0.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-client</artifactId>
<version>1.19.4</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>1.10.19</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-module-junit4</artifactId>
<version>2.0.2</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-test</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.json</groupId>
<artifactId>json</artifactId>
<version>20140107</version>
</dependency>
</dependencies>
<repositories>
<repository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</url>
</repository>
</repositories>
<distributionManagement>
<repository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/projects/44/packages/maven</url>
</repository>
<snapshotRepository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/projects/44/packages/maven</url>
</snapshotRepository>
</distributionManagement>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
<configuration>
<classifier>spring-boot</classifier>
<mainClass>org.opengroup.osdu.workflow.provider.azure.WorkflowApplication</mainClass>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>com.microsoft.azure</groupId>
<artifactId>azure-webapp-maven-plugin</artifactId>
<version>1.7.0</version>
<configuration>
<schemaVersion>V2</schemaVersion>
<authentication>
<serverId>azure-auth</serverId>
</authentication>
<subscriptionId>${azure.appservice.subscription}</subscriptionId>
<resourceGroup>${azure.appservice.resourcegroup}</resourceGroup>
<appServicePlanName>${azure.appservice.plan}</appServicePlanName>
<appName>${azure.appservice.appname}</appName>
<appSettings>
<property>
<name>JAVA_OPTS</name>
<value>-Dserver.port=80</value>
</property>
</appSettings>
<deployment>
<resources>
<resource>
<directory>${project.basedir}/target</directory>
<includes>
<include>*spring-boot.jar</include>
</includes>
</resource>
</resources>
</deployment>
</configuration>
</plugin>
</plugins>
</build>
</project>
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.workflow.provider.azure;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.FilterType;
/**
* Note: these exclusions are the result of duplicate dependencies being introduced in the
* {@link org.opengroup.osdu.is} package, which is pulled in through the os-core-lib-azure
* mvn project. These duplicate beans are not needed by this application and so they are explicity
* ignored.
*/
@ComponentScan(
basePackages = {"org.opengroup.osdu"},
excludeFilters = {
@ComponentScan.Filter(type = FilterType.REGEX, pattern = "org.opengroup.osdu.is.*"),
}
)
@SpringBootApplication
public class WorkflowApplication {
public static void main(String[] args) {
SpringApplication.run(WorkflowApplication.class, args);
}
}
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.workflow.provider.azure.config;
import com.azure.cosmos.CosmosClient;
import com.azure.cosmos.CosmosContainer;
import com.azure.security.keyvault.secrets.SecretClient;
import org.opengroup.osdu.azure.KeyVaultFacade;
import org.opengroup.osdu.common.Validators;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import sun.misc.BASE64Encoder;
import javax.inject.Named;
@Configuration
public class AzureBootstrapConfig {
@Value("${azure.keyvault.url}")
private String keyVaultURL;
@Value("${azure.cosmosdb.database}")
private String cosmosDBName;
@Value("${azure.cosmosdb.ingestionstrategy.collection}")
private String ingestionStrategyCollectionName;
@Value("${azure.cosmosdb.workflowstatus.collection}")
private String workflowStatusCollectionName;
@Value("${azure.airflow.url}")
private String airflowURL;
@Value("${azure.airflow.username}")
private String airflowUsername;
@Value("${azure.airflow.password}")
private String airflowPassword;
@Bean
@Named("AIRFLOW_URL")
public String airflowURL() {
return airflowURL;
}
@Bean
@Named("AIRFLOW_APP_KEY")
public String airflowAppKey() {
Validators.checkNotNull(airflowUsername, "Airflow username cannot be null");
Validators.checkNotNull(airflowPassword, "Airflow password cannot be null");
String airflowAuthString = airflowUsername + ":" + airflowPassword;
return new BASE64Encoder().encode(airflowAuthString.getBytes());
}
@Bean
@Named("KEY_VAULT_URL")
public String keyVaultURL() {
return keyVaultURL;
}
@Bean
@Named("COSMOS_ENDPOINT")
public String cosmosEndpoint(SecretClient kv) {
return KeyVaultFacade.getSecretWithValidation(kv, "cosmos-endpoint");
}
@Bean
@Named("COSMOS_KEY")
public String cosmosKey(SecretClient kv) {
return KeyVaultFacade.getSecretWithValidation(kv, "cosmos-primary-key");
}
@Bean
@Named("INGESTION_STRATEGY_CONTAINER")
CosmosContainer ingestionStrategyContainer(final CosmosClient cosmosClient) {
Validators.checkNotNull(cosmosClient, "Cosmos client cannot be null");
return cosmosClient.getDatabase(cosmosDBName).getContainer(ingestionStrategyCollectionName);
}
@Bean
@Named("WORKFLOW_STATUS_CONTAINER")
CosmosContainer workflowStatusContainer(final CosmosClient cosmosClient) {
Validators.checkNotNull(cosmosClient, "Cosmos client cannot be null");
return cosmosClient.getDatabase(cosmosDBName).getContainer(workflowStatusCollectionName);
}
}
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.workflow.provider.azure.model;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@AllArgsConstructor
@NoArgsConstructor
public class IngestionStrategyDoc {
private String id;
public String workflowType;
public String dataType;
public String userId;
public String dagName;
}
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.workflow.provider.azure.model;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.opengroup.osdu.workflow.model.WorkflowStatusType;
import java.util.Date;
@Data
@AllArgsConstructor
@NoArgsConstructor
@Builder
public class WorkflowStatusDoc {
private String id;
public String workflowId;
public String airflowRunId;
public WorkflowStatusType workflowStatusType;
public Date submittedAt;
public String submittedBy;
}
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.workflow.provider.azure.repository;
import java.util.List;
import java.util.Optional;
import java.util.logging.Level;
import java.util.logging.Logger;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.opengroup.osdu.azure.CosmosFacade;
import org.opengroup.osdu.core.common.model.WorkflowType;
import org.opengroup.osdu.workflow.model.IngestionStrategy;
import org.opengroup.osdu.workflow.provider.azure.model.IngestionStrategyDoc;
import org.opengroup.osdu.workflow.provider.interfaces.IIngestionStrategyRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Repository;
import com.azure.cosmos.CosmosContainer;
import javax.inject.Named;
@Repository
@Slf4j
@RequiredArgsConstructor
public class IngestionStrategyRepository implements IIngestionStrategyRepository {
private static Logger logger = Logger.getLogger(IngestionStrategyRepository.class.getName());
@Autowired
@Named("INGESTION_STRATEGY_CONTAINER")
private CosmosContainer ingestionStrategyContainer;
@Override
public IngestionStrategy findByWorkflowTypeAndDataTypeAndUserId(WorkflowType workflowType,
String dataType, String userId) {
logger.log(Level.INFO, String.format("Requesting dag selection. Workflow type: {%s}, Data type: {%s}, User id: {%s}",
workflowType, dataType, userId));
Optional<IngestionStrategyDoc> document = CosmosFacade.findItem(
ingestionStrategyContainer,
String.format("%s-%s", workflowType.toString().toLowerCase(), dataType.toLowerCase()),
workflowType.toString().toLowerCase(),
IngestionStrategyDoc.class);
IngestionStrategyDoc ingestionStrategyDoc = !document.isPresent() ? null : document.get();
IngestionStrategy ingestionStrategy = null;
if(ingestionStrategyDoc != null) {
logger.log(Level.INFO, String.format("Found dag: {%s}", ingestionStrategyDoc.dagName));
ingestionStrategy = buildIngestionStrategy(ingestionStrategyDoc);
}
return ingestionStrategy;
}
private IngestionStrategy buildIngestionStrategy(IngestionStrategyDoc ingestionStrategyDoc) {
logger.log(Level.INFO, String.format("Build ingestion strategy. Ingestion Strategy Doc: {%s}",
ingestionStrategyDoc.toString()));
return IngestionStrategy.builder()
.workflowType(WorkflowType.valueOf(ingestionStrategyDoc.workflowType.toUpperCase()))
.dataType(ingestionStrategyDoc.dataType)
.userId(ingestionStrategyDoc.userId)
.dagName(ingestionStrategyDoc.dagName)
.build();
}
}
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.workflow.provider.azure.repository;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.Optional;