Commit 7ba9a622 authored by Igor Filippov (EPAM)'s avatar Igor Filippov (EPAM)
Browse files

GONRG-1301: Finalize refactoring Ingestion

parent d9f27072
Pipeline #19974 failed with stages
in 16 seconds
/*
Copyright 2020 Google LLC
Copyright 2020 EPAM Systems, Inc
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package org.opengroup.osdu.ingest.client.v3;
import static org.opengroup.osdu.core.common.model.http.DpsHeaders.AUTHORIZATION;
import static org.opengroup.osdu.core.common.model.http.DpsHeaders.DATA_PARTITION_ID;
import org.opengroup.osdu.ingest.aspect.CheckClientResponse;
import org.opengroup.osdu.ingest.model.v3.workflow.WorkFlowRunRequest;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestHeader;
import org.springframework.web.bind.annotation.RequestParam;
@FeignClient(url = "${osdu.workflow-service.url}", name = "workflow-service-v3")
public interface WorkflowServiceClient {
@CheckClientResponse
@PostMapping("/v1/workflow/{id}/workflowRun")
feign.Response startWorkflow(@RequestHeader(AUTHORIZATION) String authToken,
@RequestHeader(DATA_PARTITION_ID) String partition,
@PathVariable String id,
@RequestBody WorkFlowRunRequest request);
@CheckClientResponse
@GetMapping("/v1/workflow")
feign.Response getAllWorkflow(@RequestHeader(AUTHORIZATION) String authToken,
@RequestHeader(DATA_PARTITION_ID) String partition,
@RequestParam(required = false) String prefix);
}
/*
Copyright 2020 Google LLC
Copyright 2020 EPAM Systems, Inc
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package org.opengroup.osdu.ingest.exception.v3;
import org.opengroup.osdu.core.common.exception.BadRequestException;
public class IngestionStrategyNotFoundException extends BadRequestException {
public IngestionStrategyNotFoundException(String message) {
super(message);
}
}
......@@ -16,6 +16,7 @@
package org.opengroup.osdu.ingest.model.property;
import javax.validation.constraints.NotNull;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
......@@ -33,6 +34,9 @@ import org.springframework.validation.annotation.Validated;
@Component
public class DataTypeProperties {
@NotNull
String defaultType;
String loadManifestType;
}
/*
Copyright 2020 Google LLC
Copyright 2020 EPAM Systems, Inc
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package org.opengroup.osdu.ingest.model.v3;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.opengroup.osdu.core.common.model.WorkflowType;
@Data
@NoArgsConstructor
@AllArgsConstructor
@Builder
public class IngestionStrategy {
@JsonProperty(Fields.WORKFLOW_TYPE)
WorkflowType workflowType;
@JsonProperty(Fields.DATA_TYPE)
String dataType;
@JsonProperty(Fields.USER_ID)
String userId;
@JsonProperty(Fields.DAG_NAME)
String dagName;
public static final class Fields {
public static final String WORKFLOW_TYPE = "WorkflowType";
public static final String DATA_TYPE = "DataType";
public static final String USER_ID = "UserID";
public static final String DAG_NAME = "DAGName";
private Fields() {
}
}
}
/*
Copyright 2020 Google LLC
Copyright 2020 EPAM Systems, Inc
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package org.opengroup.osdu.ingest.model.v3;
public enum OsduSpecificWorkflowName {
MANIFEST_INGESTION("manifest_ingestion");
private final String name;
OsduSpecificWorkflowName(String name){
this.name = name;
}
public String value() {
return this.name;
}
}
/*
Copyright 2020 Google LLC
Copyright 2020 EPAM Systems, Inc
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package org.opengroup.osdu.ingest.model.v3.workflow;
import java.util.Map;
import lombok.Generated;
public class WorkFlowRunRequest {
private String runId;
private Map<String, Object> executionContext;
@Generated
public static WorkFlowRunRequestBuilder builder() {
return new WorkFlowRunRequestBuilder();
}
@Generated
public String getRunId() {
return this.runId;
}
@Generated
public Map<String, Object> getExecutionContext() {
return this.executionContext;
}
@Generated
public void setRunId(String runId) {
this.runId = runId;
}
@Generated
public void setExecutionContext(Map<String, Object> executionContext) {
this.executionContext = executionContext;
}
@Generated
protected boolean canEqual(Object other) {
return other instanceof WorkFlowRunRequest;
}
@Generated
public String toString() {
return "WorkFlowRunRequest(runId=" + this.getRunId() + ", executionContext=" + this
.getExecutionContext() + ")";
}
@Generated
public WorkFlowRunRequest() {
}
@Generated
public WorkFlowRunRequest(String runId, Map<String, Object> executionContext) {
this.runId = runId;
this.executionContext = executionContext;
}
@Generated
public static class WorkFlowRunRequestBuilder {
@Generated
private String runId;
@Generated
private Map<String, Object> executionContext;
@Generated
WorkFlowRunRequestBuilder() {
}
@Generated
public WorkFlowRunRequestBuilder runId(String runId) {
this.runId = runId;
return this;
}
@Generated
public WorkFlowRunRequestBuilder executionContext(Map<String, Object> executionContext) {
this.executionContext = executionContext;
return this;
}
@Generated
public WorkFlowRunRequest build() {
return new WorkFlowRunRequest(this.runId, this.executionContext);
}
@Generated
public String toString() {
return "WorkFlowRunRequest.WorkFlowRunRequestBuilder(runId=" + this.runId
+ ", executionContext=" + this.executionContext + ")";
}
}
}
/*
Copyright 2020 Google LLC
Copyright 2020 EPAM Systems, Inc
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package org.opengroup.osdu.ingest.model.v3.workflow;
import lombok.Generated;
public class WorkflowRunResponse {
private String workflowId;
private String runId;
private Long startTimestamp;
private Long endTimestamp;
private WorkflowStatusType status;
private String submittedBy;
@Generated
public static WorkflowRunResponseBuilder builder() {
return new WorkflowRunResponseBuilder();
}
@Generated
public String getWorkflowId() {
return this.workflowId;
}
@Generated
public String getRunId() {
return this.runId;
}
@Generated
public Long getStartTimestamp() {
return this.startTimestamp;
}
@Generated
public Long getEndTimestamp() {
return this.endTimestamp;
}
@Generated
public WorkflowStatusType getStatus() {
return this.status;
}
@Generated
public String getSubmittedBy() {
return this.submittedBy;
}
@Generated
public void setWorkflowId(String workflowId) {
this.workflowId = workflowId;
}
@Generated
public void setRunId(String runId) {
this.runId = runId;
}
@Generated
public void setStartTimestamp(Long startTimestamp) {
this.startTimestamp = startTimestamp;
}
@Generated
public void setEndTimestamp(Long endTimestamp) {
this.endTimestamp = endTimestamp;
}
@Generated
public void setStatus(WorkflowStatusType status) {
this.status = status;
}
@Generated
public void setSubmittedBy(String submittedBy) {
this.submittedBy = submittedBy;
}
@Generated
public String toString() {
return "WorkflowRunResponse(workflowId=" + this.getWorkflowId() + ", runId=" + this.getRunId() + ", startTimestamp=" + this.getStartTimestamp() + ", endTimestamp=" + this.getEndTimestamp() + ", status=" + this.getStatus() + ", submittedBy=" + this.getSubmittedBy() + ")";
}
@Generated
public WorkflowRunResponse() {
}
@Generated
public WorkflowRunResponse(String workflowId, String runId, Long startTimestamp, Long endTimestamp, WorkflowStatusType status, String submittedBy) {
this.workflowId = workflowId;
this.runId = runId;
this.startTimestamp = startTimestamp;
this.endTimestamp = endTimestamp;
this.status = status;
this.submittedBy = submittedBy;
}
@Generated
public static class WorkflowRunResponseBuilder {
@Generated
private String workflowId;
@Generated
private String runId;
@Generated
private Long startTimestamp;
@Generated
private Long endTimestamp;
@Generated
private WorkflowStatusType status;
@Generated
private String submittedBy;
@Generated
WorkflowRunResponseBuilder() {
}
@Generated
public WorkflowRunResponseBuilder workflowId(String workflowId) {
this.workflowId = workflowId;
return this;
}
@Generated
public WorkflowRunResponseBuilder runId(String runId) {
this.runId = runId;
return this;
}
@Generated
public WorkflowRunResponseBuilder startTimestamp(Long startTimestamp) {
this.startTimestamp = startTimestamp;
return this;
}
@Generated
public WorkflowRunResponseBuilder endTimestamp(Long endTimestamp) {
this.endTimestamp = endTimestamp;
return this;
}
@Generated
public WorkflowRunResponseBuilder status(WorkflowStatusType status) {
this.status = status;
return this;
}
@Generated
public WorkflowRunResponseBuilder submittedBy(String submittedBy) {
this.submittedBy = submittedBy;
return this;
}
@Generated
public WorkflowRunResponse build() {
return new WorkflowRunResponse(this.workflowId, this.runId, this.startTimestamp, this.endTimestamp, this.status, this.submittedBy);
}
@Generated
public String toString() {
return "WorkflowRunResponse.WorkflowRunResponseBuilder(workflowId=" + this.workflowId + ", runId=" + this.runId + ", startTimestamp=" + this.startTimestamp + ", endTimestamp=" + this.endTimestamp + ", status=" + this.status + ", submittedBy=" + this.submittedBy + ")";
}
}
public static final class Fields {
public static final String WORKFLOW_ID = "WorkflowID";
public static final String RUN_ID = "RunID";
public static final String START_TIME_STAMP = "StartTimeStamp";
public static final String END_TIME_STAMP = "EndTimeStamp";
public static final String STATUS = "Status";
public static final String SUBMITTED_BY = "SubmittedBy";
private Fields() {
}
}
}
/*
* Copyright 2020 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.ingest.model.v3.workflow;
import com.fasterxml.jackson.annotation.JsonProperty;
public enum WorkflowStatusType {
@JsonProperty("submitted")
SUBMITTED,
@JsonProperty("running")
RUNNING,
@JsonProperty("finished")
FINISHED,
@JsonProperty("failed")
FAILED;
}
/*
Copyright 2020 Google LLC
Copyright 2020 EPAM Systems, Inc
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package org.opengroup.osdu.ingest.provider.interfaces.v3;
import org.opengroup.osdu.core.common.model.WorkflowType;
import org.opengroup.osdu.ingest.model.v3.IngestionStrategy;
public interface IIngestionStrategyRepository {
/**
* Finds a dag based on parameters.
*
* @param workflowType type of workflow
* @param dataType data type
* @param userId user id
* @return ingestion strategy
*/
IngestionStrategy findByWorkflowTypeAndDataTypeAndUserId(String workflowType,
String dataType, String userId);
}