Commit c418492b authored by Anuj Gupta's avatar Anuj Gupta
Browse files

Merge branch 'master' into 'ibm-impl-from-og-master'

Synching with Master

See merge request !84
parents 1032d2bf 0ff239b0
Pipeline #26798 passed with stages
in 44 minutes and 19 seconds
variables:
# --- osdu gcp specific variables ---
OSDU_GCP_BUILD_SUBDIR: provider/file-gcp-datastore
# OSDU_GCP_INT_TEST_SUBDIR: testing/file-test-core
OSDU_GCP_SERVICE: workflow
OSDU_GCP_VENDOR: gcp
OSDU_GCP_NAMESPACE: osdu-namespace
OSDU_GCP_APPLICATION_NAME: os-workflow
OSDU_GCP_ENV_VARS: SPRING_CLOUD_GCP_DATASTORE_NAMESPACE=$OSDU_GCP_NAMESPACE,GCP_AIRFLOW_URL=$OSDU_GCP_AIRFLOW_URL,OSDU_ENTITLEMENTS_URL=$OSDU_GCP_ENTITLEMENTS_URL --vpc-connector=$OSDU_GCP_VPC_CONNECTOR
OSDU_GCP_ENVIRONMENT: dev
OSDU_GCP_PORT: 8080
OSDU_GCP_ENV_VARS: OSDU_ENTITLEMENTS_URL=$OSDU_GCP_ENTITLEMENTS_URL,OSDU_ENTITLEMENTS_APPKEY=$OSDU_GCP_ENTITLEMENTS_APPKEY,GCP_AIRFLOW_URL=$OSDU_GCP_AIRFLOW_URL,OSDU_AIRFLOW_URL=$OSDU_GCP_AIRFLOW_URL --vpc-connector=$OSDU_GCP_VPC_CONNECTOR
# --- end of osdu gcp specific variables ---
IBM_BUILD_SUBDIR: provider/workflow-ibm
......@@ -24,10 +19,8 @@ variables:
AZURE_SERVICE: ingestion-workfow
AZURE_BUILD_SUBDIR: provider/workflow-azure
AZURE_TEST_SUBDIR: testing/workflow-test-azure
# End azure specific variables
include:
- project: "osdu/platform/ci-cd-pipelines"
ref: "master"
......@@ -63,7 +56,3 @@ include:
- project: "osdu/platform/ci-cd-pipelines"
ref: "master"
file: "cloud-providers/azure.yml"
osdu-gcp-test:
allow_failure: false
This diff is collapsed.
openapi: 3.0.0
info:
description: |
Workflow Service API is a wrapper API on existing orchestrator services (ariflow) defined in way which suits the domain workflow needs.Operational tasks like starting,stopping, pausing etc will be part of this API. Also creation and maintenance will be responsilbilities of this API.
Workflow Service API is a wrapper API on existing orchestrator services (ariflow) defined in way which suits the domain workflow needs.Operational tasks like starting,stopping, pausing etc will be part of this API. Also creation and maintenance will be responsilbilities of this API.
## Key Points:
......@@ -10,7 +10,7 @@ info:
**Workflow Run** - Workflow Run is single execution of a workflow. For example, a CSV workflow executed to ingest a CSV is CSV workflow run.
In case of Airflow as orchestrator, a workflow run will be equivalent to one dag run.
Each instance of workflow run is represented by unique run id. (`runId`).
Each instance of workflow run is represented by unique run id. (`runId`).
version: 2.0.5
title: Workflow Service API
contact:
......@@ -85,7 +85,7 @@ paths:
description: 'Filter workflow names which start with the full prefix specified.'
required: false
schema:
type: string
type: string
responses:
'200':
content:
......@@ -124,8 +124,8 @@ paths:
content:
application/json:
schema:
$ref: '#/components/schemas/error'
$ref: '#/components/schemas/error'
/v1/workflow/{workflow_name}:
delete:
tags:
......@@ -186,7 +186,7 @@ paths:
required: true
schema:
type: string
responses:
'200':
description: Workflow Details.
......@@ -242,7 +242,7 @@ paths:
content:
application/json:
schema:
$ref: '#/components/schemas/workflowTriggerRequest'
$ref: '#/components/schemas/workflowTriggerRequest'
responses:
'200':
description: Workflow successully triggered.
......@@ -316,13 +316,13 @@ paths:
description: 'The maximum number of workflow runs to create in a single request. Maximum is 500.'
required: false
schema:
type: integer
type: integer
- name: cursor
in: query
description: 'Cursor for subsequent request.'
required: false
schema:
type: string
type: string
- name: partial
in: query
description: 'Whether or not a partial batch can be created. If true, and the number of workflow runs that would be created between the start and end exceeds the limit, no workflow runs will be created.'
......@@ -334,7 +334,7 @@ paths:
description: 'JSON configuration added to the Workflow run conf attribute'
required: false
schema:
type: string
type: string
responses:
'200':
content:
......@@ -374,7 +374,7 @@ paths:
application/json:
schema:
$ref: '#/components/schemas/error'
/v1/workflow/{workflow_name}/workflowRun/{runId}:
get:
tags:
......@@ -450,7 +450,7 @@ paths:
description: Run id for the worfkow.
required: true
schema:
type: string
type: string
requestBody:
content:
application/json:
......@@ -492,9 +492,9 @@ paths:
content:
application/json:
schema:
$ref: '#/components/schemas/error'
$ref: '#/components/schemas/error'
components:
securitySchemes:
bearer:
......@@ -515,8 +515,8 @@ components:
key1:
type: string
example: value1
description: Map to configure workflow speciffic key value pairs.
description: Map to configure workflow speciffic key value pairs.
workflow:
type: object
title: Workflow
......@@ -543,17 +543,17 @@ components:
example: osdu-default
dagContent:
type: string
example:
example:
etc:
type: string
description:
type: string
description: Description of workflow provided by user at time of creation of workflow.
createdby:
createdby:
readOnly: true
type: string
description: System captured user info who created workflow.
creationTimestamp:
creationTimestamp:
readOnly: true
type: number
description: System date of creation of workflow.Epoch tiemstamp.
......@@ -592,12 +592,12 @@ components:
- "FAILED"
- "SUBMITTED"
title: "status"
type: "string"
submittedBy:
type: "string"
submittedBy:
readOnly: true
type: string
description: System captured user details which triggered the run.
error:
title: Error
type: object
......
......@@ -15,21 +15,19 @@
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.2.5.RELEASE</version>
<relativePath/>
<relativePath />
</parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-workflow</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
<packaging>pom</packaging>
<name>os-workflow</name>
......
......@@ -13,20 +13,18 @@
<!-- See the License for the specific language governing permissions and-->
<!-- limitations under the License.-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-workflow</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>workflow-aws</artifactId>
<version>${version.number}</version>
<version>0.6.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>workflow-aws</name>
......@@ -40,7 +38,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>workflow-core</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
</dependency>
<dependency>
......@@ -155,7 +153,7 @@
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>META-INF/spring.schemas</resource>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>org.opengroup.osdu.workflow.WorkflowApplication</mainClass>
</transformer>
......
......@@ -20,7 +20,7 @@
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-workflow</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
......@@ -32,10 +32,10 @@
<properties>
<azure.version>2.1.7</azure.version>
<osdu.corelibazure.version>0.0.56</osdu.corelibazure.version>
<azure.appservice.resourcegroup></azure.appservice.resourcegroup>
<azure.appservice.plan></azure.appservice.plan>
<azure.appservice.appname></azure.appservice.appname>
<azure.appservice.subscription></azure.appservice.subscription>
<azure.appservice.resourcegroup />
<azure.appservice.plan />
<azure.appservice.appname />
<azure.appservice.subscription />
</properties>
<dependencyManagement>
......@@ -93,7 +93,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>workflow-core</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
......
# workflow-gcp
The OSDU R2 Workflow service is designed to start business processes in the system. In the OSDU R2 prototype phase, the service only starts ingestion of OSDU data.
The OSDU R3 Workflow service is designed to start business processes in the system. In the OSDU R3
prototype phase, the service allows you to work with workflow metadata, supporting CRUD operations
and also trigger workflow in airflow, get, delete and change the status of process startup records.
The Workflow service provides a wrapper functionality around the Apache Airflow functions and is designed to carry out preliminary work with files before running the Airflow Directed Acyclic Graphs (DAGs) that will perform actual ingestion of OSDU data.
In OSDU R2, depending on the types of data, workflow, and user, the Workflow service starts the necessary workflow such as well log ingestion or opaque ingestion.
The Workflow service provides a wrapper functionality around the Apache Airflow functions and is
designed to carry out preliminary work with files before running the Airflow Directed Acyclic Graphs
(DAGs) that will perform actual ingestion of OSDU data.
## Running Locally
......
......@@ -15,20 +15,18 @@
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-workflow</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>workflow-gcp</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>workflow-gcp</name>
......@@ -42,7 +40,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>workflow-core</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
</dependency>
<dependency>
......@@ -147,7 +145,7 @@
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>META-INF/spring.schemas</resource>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>org.opengroup.osdu.workflow.WorkflowApplication</mainClass>
</transformer>
......
......@@ -15,20 +15,18 @@
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-workflow</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>workflow-ibm</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>workflow-ibm</name>
......@@ -44,7 +42,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>workflow-core</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
</dependency>
<dependency>
......
......@@ -14,14 +14,12 @@
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.opengroup.osdu</groupId>
<artifactId>workflow-test</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
<description>Workflow Service Integration Test Root Project</description>
<packaging>pom</packaging>
......
......@@ -14,21 +14,19 @@
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>workflow-test</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.opengroup.osdu.workflow</groupId>
<artifactId>workflow-test-aws</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
<packaging>jar</packaging>
......@@ -72,7 +70,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>workflow-test-core</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
......
......@@ -15,21 +15,19 @@
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>workflow-test</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.opengroup.osdu.workflow</groupId>
<artifactId>workflow-test-azure</artifactId>
<version>0.0.2-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
......@@ -46,7 +44,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>workflow-test-core</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>io.jsonwebtoken</groupId>
......
......@@ -14,20 +14,18 @@
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>workflow-test</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>workflow-test-core</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>workflow-test-core</name>
......
......@@ -14,19 +14,17 @@
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>workflow-test</artifactId>
<groupId>org.opengroup.osdu</groupId>
<version>0.0.1-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>workflow-test-gcp</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>workflow-test-gcp-datastore</name>
......@@ -58,7 +56,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>workflow-test-core</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
</dependency>
<dependency>
......
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>workflow-test</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>workflow-test-ibm</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
<packaging>jar</packaging>
......@@ -27,7 +25,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>workflow-test-core</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
</dependency>
<dependency>
......
......@@ -15,26 +15,24 @@
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-workflow</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>workflow-core</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>0.6.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>workflow-core</name>
<description>Core project for the Workflow service</description>
<properties/>
<properties />
<dependencies>
<dependency>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment