Commit dc5ab17a authored by Anuj Gupta's avatar Anuj Gupta
Browse files

IBm Impl , Int test and synched gcp role code chnages

parent 6a6662e5
Pipeline #8538 failed with stages
in 25 minutes and 7 seconds
variables:
# --- osdu gcp specific variables ---
OSDU_GCP_BUILD_SUBDIR: provider/file-gcp-datastore
# OSDU_GCP_INT_TEST_SUBDIR: testing/file-test-core
OSDU_GCP_SERVICE: workflow
......@@ -9,10 +10,16 @@ variables:
OSDU_GCP_ENV_VARS: SPRING_CLOUD_GCP_DATASTORE_NAMESPACE=$OSDU_GCP_NAMESPACE,GCP_AIRFLOW_URL=$OSDU_GCP_AIRFLOW_URL,OSDU_ENTITLEMENTS_URL=$OSDU_GCP_ENTITLEMENTS_URL --vpc-connector=$OSDU_GCP_VPC_CONNECTOR
OSDU_GCP_ENVIRONMENT: dev
OSDU_GCP_PORT: 8080
IBM_BUILD_SUBDIR: provider/workflow-ibm
IBM_INT_TEST_SUBDIR: testing/workflow-test-ibm
# integration tests variables
WORKFLOW_HOST: https://os-workflow-attcrcktoa-uc.a.run.app
FINISHED_WORKFLOW_ID: fad778da-fbc4-4261-8b3e-deb48be44969
# --- end of osdu gcp specific variables ---
include:
- project: 'osdu/platform/ci-cd-pipelines'
ref: 'master'
......
......@@ -254,6 +254,10 @@ development purposes because signing a blob is only available with the service a
Remember to set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable. Follow the [instructions
on the Google developer's portal][application-default-credentials].
**Integration tests**
Instructions for running the GCP integration tests can be found [here](./provider/workflow-gcp-datastore/README.md).
### Persistence layer
The GCP implementation contains two mutually exclusive modules to work with the persistence layer.
......
# workflow-gcp
## Running Locally
### Requirements
In order to run this service locally, you will need the following:
- [Maven 3.6.0+](https://maven.apache.org/download.cgi)
- [AdoptOpenJDK8](https://adoptopenjdk.net/)
- Infrastructure dependencies, deployable through the relevant [infrastructure template](https://community.opengroup.org/osdu/platform/deployment-and-operations/infra-gcp-provisioning)
### Environment Variables
In order to run the service locally, you will need to have the following environment variables defined.
| name | value | description | sensitive? | source |
| --- | --- | --- | --- | --- |
| `LOG_PREFIX` | `workflow` | Logging prefix | no | - |
| `osdu.entitlements.url` | ex `https://entitlements.com/entitlements/v1` | Entitlements API endpoint | no | output of infrastructure deployment |
| `osdu.entitlements.app-key` | ex `test` | Entitlements app key | no | - |
| `gcp.airflow.url` | ex `https://********-tp.appspot.com` | Airflow endpoint | yes | - |
| `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials |
| `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
**Required to run integration tests**
| name | value | description | sensitive? | source |
| --- | --- | --- | --- | --- |
| `DOMAIN` | ex `contoso.com` | OSDU R2 to run tests under | no | - |
| `INTEGRATION_TESTER` | `********` | Service account for API calls. Note: this user must have entitlements configured already | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
| `NO_DATA_ACCESS_TESTER` | `********` | Service account without data access | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
| `LEGAL_TAG` | `********` | Demo legal tag used to pass test| yes | Legal service |
| `WORKFLOW_HOST` | ex `https://os-workflow-dot-opendes.appspot.com` | Endpoint of workflow service | no | - |
| `DEFAULT_DATA_PARTITION_ID_TENANT1`| ex `opendes` | OSDU tenant used for testing | no | - |
| `OTHER_RELEVANT_DATA_COUNTRIES`| `US`| - | no | - |
| `GOOGLE_AUDIENCE` | ex `********.apps.googleusercontent.com`| client application ID | yes | https://console.cloud.google.com/apis/credentials |
| `FINISHED_WORKFLOW_ID` | `********` | Workflow ID with finished status | yes | - |
**Entitlements configuration for integration accounts**
| INTEGRATION_TESTER | NO_DATA_ACCESS_TESTER |
| --- | --- |
| users<br/>service.entitlements.user<br/>service.workflow.admin<br/>service.workflow.creator<br/>service.workflow.viewer<br/>service.legal.admin<br/>service.legal.editor<br/>data.test1<br/>data.integration.test | users |
### Configure Maven
Check that maven is installed:
```bash
$ mvn --version
Apache Maven 3.6.0
Maven home: /usr/share/maven
Java version: 1.8.0_212, vendor: AdoptOpenJDK, runtime: /usr/lib/jvm/jdk8u212-b04/jre
...
```
You may need to configure access to the remote maven repository that holds the OSDU dependencies. This file should live within `~/.mvn/community-maven.settings.xml`:
```bash
$ cat ~/.m2/settings.xml
<?xml version="1.0" encoding="UTF-8"?>
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
<servers>
<server>
<id>community-maven-via-private-token</id>
<!-- Treat this auth token like a password. Do not share it with anyone, including Microsoft support. -->
<!-- The generated token expires on or before 11/14/2019 -->
<configuration>
<httpHeaders>
<property>
<name>Private-Token</name>
<value>${env.COMMUNITY_MAVEN_TOKEN}</value>
</property>
</httpHeaders>
</configuration>
</server>
</servers>
</settings>
```
### Build and run the application
After configuring your environment as specified above, you can follow these steps to build and run the application. These steps should be invoked from the *repository root.*
```bash
cd provider/workflow-gcp/ && mvn spring-boot:run
```
### Test the application
After the service has started it should be accessible via a web browser by visiting [http://localhost:8080/swagger-ui.html](http://localhost:8080/swagger-ui.html). If the request does not fail, you can then run the integration tests.
```bash
# build + install integration test core
$ (cd testing/workflow-test-core/ && mvn clean install)
# build + run GCP integration tests.
#
# Note: this assumes that the environment variables for integration tests as outlined
# above are already exported in your environment.
$ (cd testing/workflow-test-gcp/ && mvn clean test)
```
## License
Copyright 2020 Google LLC
Copyright 2020 EPAM Systems, Inc
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
\ No newline at end of file
......@@ -22,6 +22,7 @@ public class HTTPClientGCP extends HTTPClient {
if (noDataAccessToken == null || noDataAccessToken.isEmpty()) {
noDataAccessToken = new GoogleServiceAccount(getEnvironmentVariableOrDefaultKey(NO_DATA_ACCESS_TESTER))
.getAuthToken(getEnvironmentVariableOrDefaultKey(GOOGLE_AUDIENCE));
}
return "Bearer " + noDataAccessToken;
}
......
......@@ -22,6 +22,7 @@ import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.storage.StorageRole;
import org.opengroup.osdu.core.common.model.workflow.StartWorkflowRequest;
import org.opengroup.osdu.core.common.model.workflow.StartWorkflowResponse;
import org.opengroup.osdu.workflow.model.WorkflowRole;
import org.opengroup.osdu.workflow.provider.interfaces.IWorkflowService;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.validation.annotation.Validated;
......@@ -41,7 +42,7 @@ public class WorkflowApi {
final IWorkflowService workflowService;
@PostMapping("/startWorkflow")
@PreAuthorize("@authorizationFilter.hasPermission('" + StorageRole.CREATOR + "')")
@PreAuthorize("@authorizationFilter.hasPermission('" + WorkflowRole.CREATOR + "')")
public StartWorkflowResponse startWorkflow(@RequestBody StartWorkflowRequest request) {
log.debug("Start Workflow request received : {}", request);
StartWorkflowResponse response = workflowService.startWorkflow(request, headers);
......
......@@ -24,6 +24,7 @@ import org.opengroup.osdu.workflow.model.GetStatusRequest;
import org.opengroup.osdu.workflow.model.GetStatusResponse;
import org.opengroup.osdu.workflow.model.UpdateStatusRequest;
import org.opengroup.osdu.workflow.model.UpdateStatusResponse;
import org.opengroup.osdu.workflow.model.WorkflowRole;
import org.opengroup.osdu.workflow.provider.interfaces.IWorkflowStatusService;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.validation.annotation.Validated;
......@@ -43,7 +44,7 @@ public class WorkflowStatusApi {
final IWorkflowStatusService workflowStatusService;
@PostMapping("/getStatus")
@PreAuthorize("@authorizationFilter.hasPermission('" + StorageRole.CREATOR + "')")
@PreAuthorize("@authorizationFilter.hasPermission('" + WorkflowRole.CREATOR + "')")
public GetStatusResponse getWorkflowStatus(@RequestBody GetStatusRequest request) {
log.debug("Get Workflow Status request received : {}", request);
GetStatusResponse response = workflowStatusService.getWorkflowStatus(request, headers);
......@@ -52,7 +53,7 @@ public class WorkflowStatusApi {
}
@PostMapping("/updateStatus")
@PreAuthorize("@authorizationFilter.hasPermission('" + StorageRole.CREATOR + "')")
@PreAuthorize("@authorizationFilter.hasPermission('" + WorkflowRole.CREATOR + "')")
public UpdateStatusResponse updateWorkflowStatus(@RequestBody UpdateStatusRequest request) {
log.debug("Update Workflow status request received : {}", request);
UpdateStatusResponse response = workflowStatusService.updateWorkflowStatus(request, headers);
......
package org.opengroup.osdu.workflow.model;
public final class WorkflowRole {
public static final String VIEWER = "service.workflow.viewer";
public static final String CREATOR = "service.workflow.creator";
public static final String ADMIN = "service.workflow.admin";
}
......@@ -44,6 +44,7 @@ import org.opengroup.osdu.core.common.model.workflow.StartWorkflowRequest;
import org.opengroup.osdu.core.common.model.workflow.StartWorkflowResponse;
import org.opengroup.osdu.core.common.provider.interfaces.IAuthorizationService;
import org.opengroup.osdu.workflow.model.IngestionStrategy;
import org.opengroup.osdu.workflow.model.WorkflowRole;
import org.opengroup.osdu.workflow.model.WorkflowStatus;
import org.opengroup.osdu.workflow.model.WorkflowStatusType;
import org.opengroup.osdu.workflow.provider.interfaces.IIngestionStrategyRepository;
......@@ -116,7 +117,7 @@ public class WorkflowMvcTest {
given(workflowStatusRepository.saveWorkflowStatus(workflowStatusCaptor.capture()))
.will(returnsFirstArg());
given(authorizationService.authorizeAny(any(), eq("service.storage.creator")))
given(authorizationService.authorizeAny(any(), eq(WorkflowRole.CREATOR)))
.willReturn(AuthorizationResponse.builder()
.user("user@mail.com")
.build());
......@@ -150,7 +151,7 @@ public class WorkflowMvcTest {
// given
HttpHeaders headers = new HttpHeaders();
given(authorizationService.authorizeAny(any(), eq("service.storage.creator")))
given(authorizationService.authorizeAny(any(), eq(WorkflowRole.CREATOR)))
.willReturn(AuthorizationResponse.builder()
.user("user@mail.com")
.build());
......@@ -181,7 +182,7 @@ public class WorkflowMvcTest {
.workflowType(WorkflowType.OSDU)
.build();
given(authorizationService.authorizeAny(any(), eq("service.storage.creator")))
given(authorizationService.authorizeAny(any(), eq(WorkflowRole.CREATOR)))
.willThrow(AppException.createUnauthorized("test: viewer"));
// when
......@@ -197,7 +198,7 @@ public class WorkflowMvcTest {
.andReturn();
// then
verify(authorizationService).authorizeAny(any(), eq("service.storage.creator"));
verify(authorizationService).authorizeAny(any(), eq(WorkflowRole.CREATOR));
}
private HttpHeaders getHttpHeaders() {
......
......@@ -40,6 +40,7 @@ import org.opengroup.osdu.workflow.model.GetStatusRequest;
import org.opengroup.osdu.workflow.model.GetStatusResponse;
import org.opengroup.osdu.workflow.model.UpdateStatusRequest;
import org.opengroup.osdu.workflow.model.UpdateStatusResponse;
import org.opengroup.osdu.workflow.model.WorkflowRole;
import org.opengroup.osdu.workflow.model.WorkflowStatus;
import org.opengroup.osdu.workflow.model.WorkflowStatusType;
import org.opengroup.osdu.workflow.provider.interfaces.IIngestionStrategyRepository;
......@@ -102,7 +103,7 @@ public class WorkflowStatusMvcTest {
given(workflowStatusRepository
.findWorkflowStatus(eq(WORKFLOW_ID))).willReturn(status);
given(authorizationService.authorizeAny(any(), eq("service.storage.creator")))
given(authorizationService.authorizeAny(any(), eq(WorkflowRole.CREATOR)))
.willReturn(AuthorizationResponse.builder()
.user("user@mail.com")
.build());
......@@ -134,7 +135,7 @@ public class WorkflowStatusMvcTest {
given(workflowStatusRepository
.findWorkflowStatus(eq(WORKFLOW_ID))).willReturn(null);
given(authorizationService.authorizeAny(any(), eq("service.storage.creator")))
given(authorizationService.authorizeAny(any(), eq(WorkflowRole.CREATOR)))
.willReturn(AuthorizationResponse.builder()
.user("user@mail.com")
.build());
......@@ -158,7 +159,7 @@ public class WorkflowStatusMvcTest {
// given
HttpHeaders headers = new HttpHeaders();
given(authorizationService.authorizeAny(any(), eq("service.storage.creator")))
given(authorizationService.authorizeAny(any(), eq(WorkflowRole.CREATOR)))
.willReturn(AuthorizationResponse.builder()
.user("user@mail.com")
.build());
......@@ -187,7 +188,7 @@ public class WorkflowStatusMvcTest {
GetStatusRequest request = GetStatusRequest.builder().workflowId(WORKFLOW_ID).build();
given(authorizationService.authorizeAny(any(), eq("service.storage.creator")))
given(authorizationService.authorizeAny(any(), eq(WorkflowRole.CREATOR)))
.willThrow(AppException.createUnauthorized("test: viewer"));
// when
......@@ -203,7 +204,7 @@ public class WorkflowStatusMvcTest {
.andReturn();
// then
verify(authorizationService).authorizeAny(any(), eq("service.storage.creator"));
verify(authorizationService).authorizeAny(any(), eq(WorkflowRole.CREATOR));
}
......@@ -236,7 +237,7 @@ public class WorkflowStatusMvcTest {
.updateWorkflowStatus(eq(status.getWorkflowId()), eq(WorkflowStatusType.RUNNING)))
.willReturn(updatedStatus);
given(authorizationService.authorizeAny(any(), eq("service.storage.creator")))
given(authorizationService.authorizeAny(any(), eq(WorkflowRole.CREATOR)))
.willReturn(AuthorizationResponse.builder()
.user("user@mail.com")
.build());
......@@ -266,7 +267,7 @@ public class WorkflowStatusMvcTest {
// given
HttpHeaders headers = new HttpHeaders();
given(authorizationService.authorizeAny(any(), eq("service.storage.creator")))
given(authorizationService.authorizeAny(any(), eq(WorkflowRole.CREATOR)))
.willReturn(AuthorizationResponse.builder()
.user("user@mail.com")
.build());
......@@ -297,7 +298,7 @@ public class WorkflowStatusMvcTest {
.workflowId(WORKFLOW_ID)
.workflowStatusType(WorkflowStatusType.RUNNING).build();
given(authorizationService.authorizeAny(any(), eq("service.storage.creator")))
given(authorizationService.authorizeAny(any(), eq(WorkflowRole.CREATOR)))
.willThrow(AppException.createUnauthorized("test: viewer"));
// when
......@@ -313,7 +314,7 @@ public class WorkflowStatusMvcTest {
.andReturn();
// then
verify(authorizationService).authorizeAny(any(), eq("service.storage.creator"));
verify(authorizationService).authorizeAny(any(), eq(WorkflowRole.CREATOR));
}
private HttpHeaders getHttpHeaders() {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment