Merge branch 'pinwang/2841' into 'master'

workflow serivce - azure implementation.

See merge request !1

pom.xml

@@ -37,6 +37,7 @@
 
   <modules>
     <module>workflow-core</module>
+    <module>provider/workflow-azure</module>
     <module>provider/workflow-gcp</module>
 <!--    <module>provider/workflow-ibm</module> Fix: Missing classes-->
     <module>provider/workflow-gcp-datastore</module>

provider/workflow-azure/.envrc.template

+##
+# Needed to run the service
+##
+export LOG_PREFIX=
+export entitlements_service_endpoint=
+export entitlements_service_api_key=
+export azure.activedirectory.session-stateless=
+export aad_client_id=
+export azure.activedirectory.AppIdUri=
+export cosmosdb_database=
+export KEYVAULT_URI=
+export AZURE_CLIENT_ID=
+export AZURE_TENANT_ID=
+export AZURE_CLIENT_SECRET=
+export appinsights_key=
+export airflow_url=
+export airflow_username=
+export airflow_password=
+
+
+##
+# Needed to run the integration tests
+##
+export INTEGRATION_TESTER=
+export MY_TENANT=
+export AZURE_TESTER_SERVICEPRINCIPAL_SECRET=
+export AZURE_AD_TENANT_ID=
+export AZURE_AD_APP_RESOURCE_ID=
+export HOST_URL=
+export ENTITLEMENT_URL=

provider/workflow-azure/maven/settings.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Copyright © Microsoft Corporation
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
+          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+          xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
+  <servers>
+    <server>
+      <id>azure-auth</id>
+      <configuration>
+        <tenant>${AZURE_DEPLOY_TENANT}</tenant>
+        <client>${AZURE_DEPLOY_CLIENT_ID}</client>
+        <key>${AZURE_DEPLOY_CLIENT_SECRET}</key>
+        <environment>AZURE</environment>
+      </configuration>
+    </server>
+  </servers>
+</settings>

provider/workflow-azure/pom.xml

+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  Copyright © Microsoft Corporation
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <parent>
+    <groupId>org.opengroup.osdu</groupId>
+    <artifactId>os-workflow</artifactId>
+    <version>0.0.1-SNAPSHOT</version>
+    <relativePath>../../pom.xml</relativePath>
+  </parent>
+
+  <modelVersion>4.0.0</modelVersion>  
+  <artifactId>workflow-azure</artifactId>  
+  <description>Workflow service on Azure</description>
+  <packaging>jar</packaging>
+
+  <properties> 
+    <azure.version>2.1.7</azure.version>
+    <osdu.azurecore.version>0.0.6-SNAPSHOT</osdu.azurecore.version>
+    <azure.appservice.resourcegroup></azure.appservice.resourcegroup>
+    <azure.appservice.plan></azure.appservice.plan>
+    <azure.appservice.appname></azure.appservice.appname>
+    <azure.appservice.subscription></azure.appservice.subscription>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>com.microsoft.azure</groupId>
+      <artifactId>azure-active-directory-spring-boot-starter</artifactId>
+      <version>${azure.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.springframework.boot</groupId>
+      <artifactId>spring-boot-starter-security</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.springframework.security</groupId>
+      <artifactId>spring-security-oauth2-client</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.springframework.security</groupId>
+      <artifactId>spring-security-oauth2-jose</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.opengroup.osdu</groupId>
+      <artifactId>core-lib-azure</artifactId>
+      <version>${osdu.azurecore.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.opengroup.osdu</groupId>
+      <artifactId>workflow-core</artifactId>
+      <version>0.0.1-SNAPSHOT</version>
+    </dependency>
+    <dependency>
+      <groupId>com.sun.jersey</groupId>
+      <artifactId>jersey-client</artifactId>
+      <version>1.19.4</version>
+    </dependency>
+    <dependency>
+      <groupId>org.projectlombok</groupId>
+      <artifactId>lombok</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>4.12</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-all</artifactId>
+      <version>1.10.19</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.powermock</groupId>
+      <artifactId>powermock-module-junit4</artifactId>
+      <version>2.0.2</version>
+    </dependency>
+    <dependency>
+      <groupId>org.springframework.boot</groupId>
+      <artifactId>spring-boot-starter-test</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.springframework.security</groupId>
+      <artifactId>spring-security-test</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.json</groupId>
+      <artifactId>json</artifactId>
+      <version>20140107</version>
+    </dependency>
+  </dependencies>
+
+  <repositories>
+    <repository>
+      <id>${gitlab-server}</id>
+      <url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</url>
+    </repository>
+  </repositories>
+
+  <distributionManagement>
+    <repository>
+      <id>${gitlab-server}</id>
+      <url>https://community.opengroup.org/api/v4/projects/44/packages/maven</url>
+    </repository>
+    <snapshotRepository>
+      <id>${gitlab-server}</id>
+      <url>https://community.opengroup.org/api/v4/projects/44/packages/maven</url>
+    </snapshotRepository>
+  </distributionManagement>
+  <build> 
+    <plugins> 
+      <plugin> 
+        <groupId>org.springframework.boot</groupId>  
+        <artifactId>spring-boot-maven-plugin</artifactId>  
+        <executions> 
+          <execution> 
+            <goals> 
+              <goal>repackage</goal> 
+            </goals>  
+            <configuration> 
+              <classifier>spring-boot</classifier>  
+              <mainClass>org.opengroup.osdu.workflow.provider.azure.WorkflowApplication</mainClass>
+            </configuration> 
+          </execution> 
+        </executions> 
+      </plugin>
+      <plugin>
+        <groupId>com.microsoft.azure</groupId>  
+        <artifactId>azure-webapp-maven-plugin</artifactId>  
+        <version>1.7.0</version>  
+        <configuration>
+          <schemaVersion>V2</schemaVersion>
+          <authentication>
+            <serverId>azure-auth</serverId>
+          </authentication>
+          <subscriptionId>${azure.appservice.subscription}</subscriptionId>
+          <resourceGroup>${azure.appservice.resourcegroup}</resourceGroup>
+          <appServicePlanName>${azure.appservice.plan}</appServicePlanName>
+          <appName>${azure.appservice.appname}</appName>
+          <appSettings>
+            <property>
+              <name>JAVA_OPTS</name>
+              <value>-Dserver.port=80</value>
+            </property>
+          </appSettings>
+          <deployment>
+            <resources>
+              <resource>
+                <directory>${project.basedir}/target</directory>
+                <includes>
+                  <include>*spring-boot.jar</include>
+                </includes>
+              </resource>
+            </resources>
+          </deployment>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build> 
+</project>

provider/workflow-azure/src/main/java/org/opengroup/osdu/workflow/provider/azure/WorkflowApplication.java

+//  Copyright © Microsoft Corporation
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+package org.opengroup.osdu.workflow.provider.azure;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.FilterType;
+
+/**
+ * Note: these exclusions are the result of duplicate dependencies being introduced in the
+ * {@link org.opengroup.osdu.is} package, which is pulled in through the os-core-lib-azure
+ * mvn project. These duplicate beans are not needed by this application and so they are explicity
+ * ignored.
+ */
+@ComponentScan(
+        basePackages = {"org.opengroup.osdu"},
+        excludeFilters = {
+                @ComponentScan.Filter(type = FilterType.REGEX, pattern = "org.opengroup.osdu.is.*"),
+        }
+)
+
+@SpringBootApplication
+public class WorkflowApplication {
+    public static void main(String[] args) {
+      SpringApplication.run(WorkflowApplication.class, args);
+    }
+}

provider/workflow-azure/src/main/java/org/opengroup/osdu/workflow/provider/azure/config/AzureBootstrapConfig.java

+// Copyright © Microsoft Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.workflow.provider.azure.config;
+
+import com.azure.cosmos.CosmosClient;
+import com.azure.cosmos.CosmosContainer;
+import com.azure.security.keyvault.secrets.SecretClient;
+
+import org.opengroup.osdu.azure.KeyVaultFacade;
+import org.opengroup.osdu.common.Validators;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import sun.misc.BASE64Encoder;
+
+import javax.inject.Named;
+
+@Configuration
+public class AzureBootstrapConfig {
+
+  @Value("${azure.keyvault.url}")
+  private String keyVaultURL;
+
+  @Value("${azure.cosmosdb.database}")
+  private String cosmosDBName;
+
+  @Value("${azure.cosmosdb.ingestionstrategy.collection}")
+  private String ingestionStrategyCollectionName;
+
+  @Value("${azure.cosmosdb.workflowstatus.collection}")
+  private String workflowStatusCollectionName;
+
+  @Value("${azure.airflow.url}")
+  private String airflowURL;
+
+  @Value("${azure.airflow.username}")
+  private String airflowUsername;
+
+  @Value("${azure.airflow.password}")
+  private String airflowPassword;
+
+  @Bean
+  @Named("AIRFLOW_URL")
+  public String airflowURL() {
+    return airflowURL;
+  }
+
+  @Bean
+  @Named("AIRFLOW_APP_KEY")
+  public String airflowAppKey() {
+    Validators.checkNotNull(airflowUsername, "Airflow username cannot be null");
+    Validators.checkNotNull(airflowPassword, "Airflow password cannot be null");
+    String airflowAuthString = airflowUsername + ":" + airflowPassword;
+    return new BASE64Encoder().encode(airflowAuthString.getBytes());
+  }
+
+  @Bean
+  @Named("KEY_VAULT_URL")
+  public String keyVaultURL() {
+    return keyVaultURL;
+  }
+
+  @Bean
+  @Named("COSMOS_ENDPOINT")
+  public String cosmosEndpoint(SecretClient kv) {
+    return KeyVaultFacade.getSecretWithValidation(kv, "cosmos-endpoint");
+  }
+
+  @Bean
+  @Named("COSMOS_KEY")
+  public String cosmosKey(SecretClient kv) {
+    return KeyVaultFacade.getSecretWithValidation(kv, "cosmos-primary-key");
+  }
+
+  @Bean
+  @Named("INGESTION_STRATEGY_CONTAINER")
+  CosmosContainer ingestionStrategyContainer(final CosmosClient cosmosClient) {
+    Validators.checkNotNull(cosmosClient, "Cosmos client cannot be null");
+    return cosmosClient.getDatabase(cosmosDBName).getContainer(ingestionStrategyCollectionName);
+  }
+
+  @Bean
+  @Named("WORKFLOW_STATUS_CONTAINER")
+  CosmosContainer workflowStatusContainer(final CosmosClient cosmosClient) {
+    Validators.checkNotNull(cosmosClient, "Cosmos client cannot be null");
+    return cosmosClient.getDatabase(cosmosDBName).getContainer(workflowStatusCollectionName);
+  }
+}

provider/workflow-azure/src/main/java/org/opengroup/osdu/workflow/provider/azure/model/IngestionStrategyDoc.java

+// Copyright © Microsoft Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.workflow.provider.azure.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class IngestionStrategyDoc {
+  private String id;
+
+  public String workflowType;
+
+  public String dataType;
+
+  public String userId;
+
+  public String dagName;
+}

provider/workflow-azure/src/main/java/org/opengroup/osdu/workflow/provider/azure/model/WorkflowStatusDoc.java

+// Copyright © Microsoft Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.workflow.provider.azure.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.opengroup.osdu.workflow.model.WorkflowStatusType;
+
+import java.util.Date;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Builder
+public class WorkflowStatusDoc {
+  private String id;
+
+  public String workflowId;
+
+  public String airflowRunId;
+
+  public WorkflowStatusType workflowStatusType;
+
+  public Date submittedAt;
+
+  public String submittedBy;
+}

provider/workflow-azure/src/main/java/org/opengroup/osdu/workflow/provider/azure/repository/IngestionStrategyRepository.java

+//  Copyright © Microsoft Corporation
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+package org.opengroup.osdu.workflow.provider.azure.repository;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import org.opengroup.osdu.azure.CosmosFacade;
+import org.opengroup.osdu.core.common.model.WorkflowType;
+import org.opengroup.osdu.workflow.model.IngestionStrategy;
+import org.opengroup.osdu.workflow.provider.azure.model.IngestionStrategyDoc;
+import org.opengroup.osdu.workflow.provider.interfaces.IIngestionStrategyRepository;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Repository;
+
+import com.azure.cosmos.CosmosContainer;
+
+import javax.inject.Named;
+
+@Repository
+@Slf4j
+@RequiredArgsConstructor
+public class IngestionStrategyRepository implements IIngestionStrategyRepository {
+
+  private static Logger logger = Logger.getLogger(IngestionStrategyRepository.class.getName());
+
+  @Autowired
+  @Named("INGESTION_STRATEGY_CONTAINER")
+  private CosmosContainer ingestionStrategyContainer;
+
+  @Override
+  public IngestionStrategy findByWorkflowTypeAndDataTypeAndUserId(WorkflowType workflowType,
+                                                                  String dataType, String userId) {
+    logger.log(Level.INFO, String.format("Requesting dag selection. Workflow type: {%s}, Data type: {%s}, User id: {%s}",
+      workflowType, dataType, userId));
+
+    Optional<IngestionStrategyDoc> document = CosmosFacade.findItem(
+      ingestionStrategyContainer,
+      String.format("%s-%s", workflowType.toString().toLowerCase(), dataType.toLowerCase()),
+      workflowType.toString().toLowerCase(),
+      IngestionStrategyDoc.class);
+
+    IngestionStrategyDoc ingestionStrategyDoc = !document.isPresent() ? null : document.get();
+
+    IngestionStrategy ingestionStrategy = null;
+
+    if(ingestionStrategyDoc != null) {
+      logger.log(Level.INFO, String.format("Found dag: {%s}", ingestionStrategyDoc.dagName));
+      ingestionStrategy = buildIngestionStrategy(ingestionStrategyDoc);
+    }
+
+    return ingestionStrategy;
+  }
+
+  private IngestionStrategy buildIngestionStrategy(IngestionStrategyDoc ingestionStrategyDoc) {
+    logger.log(Level.INFO, String.format("Build ingestion strategy. Ingestion Strategy Doc: {%s}",
+      ingestionStrategyDoc.toString()));
+
+    return IngestionStrategy.builder()
+      .workflowType(WorkflowType.valueOf(ingestionStrategyDoc.workflowType.toUpperCase()))
+      .dataType(ingestionStrategyDoc.dataType)
+      .userId(ingestionStrategyDoc.userId)
+      .dagName(ingestionStrategyDoc.dagName)
+      .build();
+  }
+}

provider/workflow-azure/src/main/java/org/opengroup/osdu/workflow/provider/azure/repository/WorkflowStatusRepository.java

+//  Copyright © Microsoft Corporation
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+package org.opengroup.osdu.workflow.provider.azure.repository;
+
+import java.util.logging.Level;
+import java.util.logging.Logger;
+