Commit 62961708 authored by Riabokon Stanislav(EPAM)[GCP]'s avatar Riabokon Stanislav(EPAM)[GCP]
Browse files

Merge branch 'feature/GONRG-864-Documentation_&_int_tests' into 'master'

GONRG-864 Documentation and int tests

Closes GONRG-864

See merge request go3-nrg/backup-service!4
parents d394d830 2bb0a2c8
# Backup service
Backup service provide a set of APIs to backup Schedule, list registered Schedules, list available Backups, restore Asset state from Backup.
Backup service provide a set of APIs to schedule backups for different Assets (Datastore, Storage, ElasticSearch, etc depends on which currently implemented ),
list registered Schedules, list available Backups, restore Asset state from Backup.
## Core Interfaces
![Core](./docs/core.png)
## Data Repositories
![Repo](./docs/datarepos.png)
## Sequence diagram
Submit schedule workflow
![Diagram](./docs/sequence.png)
## GCP Implementation
All documentation for the GCP implementation of Backup service lives [here](./provider/backup-gcp/README.md)
......
......@@ -10,5 +10,5 @@ public interface AssetBackupManager {
BackupStamp importBackup(String backupId);
void deleteBackups(List<BackupStamp> currentDate);
void deleteBackups(List<BackupStamp> backupStamps);
}
@startuml
package backup.core <<Rectangle>> {
interface AssetBackupManager{
exportBackup(BackupSchedule backupSchedule);
importBackup(String backupId);
deleteBackups(List<BackupStamp> backupStamps);
}
note right
Provide backup managing access to current Asset,
based on AssetType of schedule
end note
interface SchedulerService{
addTaskToScheduler(BackupSchedule backupSchedule);
cancelScheduledTask(BackupSchedule backupSchedule);
updateTask(BackupSchedule backupSchedule);
}
note right
Holds ThreadPool with scheduled tasks ,
and provide control
end note
interface BackupService{
submitBackupSchedule(BackupSchedule backupSchedule);
updateBackupSchedule(BackupSchedule backupSchedule);
getBackupSchedule(String id);
submitBackupImportRequest(BackupImportRequest importRequest);
listSchedules();
listBackups();
}
class BackupServiceImpl implements BackupService
class SchedulerServiceImpl implements SchedulerService{
ThreadPoolTaskScheduler poolTaskScheduler
}
enum AssetType {
DATASTORE
ELASTIC
etc
}
package backup.gcp <<Rectangle>> {
class DatastoreBackupManager implements AssetBackupManager
class ElasticBackupManager implements AssetBackupManager
DatastoreBackupManager *-- AssetType
ElasticBackupManager *-- AssetType
}
@enduml
\ No newline at end of file
@startuml
package backup.core <<Rectangle>> {
interface BackupStampRepository{
findBackupStampsByTearDownBeforeCurrentDate(LocalDateTime toDate);
submitBackupStamp(BackupStamp backupStamp);
listAvailableBackupStamps();
deleteBackupStamps(List<BackupStamp> currentDate);
findBackupByStampId(String backupStampId);
}
note right
A backup stamp's repository ,
backup stamp is a model of created backup entity
which provide additional info about it : lifetime, location etc :
end note
interface ScheduleRepository{
listBackupSchedules();
submitSchedule(BackupSchedule backupSchedule);
updateSchedule(BackupSchedule backupSchedule);
findByAssetContext(Map<String, String> assetContext);
save(BackupSchedule backupSchedule);
findById(String scheduleId);
}
class BackupStamp{
String stampId;
String backupRepository;
Asset assetType;
LocalDateTime tearDownDate;
Map<String, String> assetContext;
}
class BackupSchedule{
private String scheduleId;
Asset assetType;
int backupPeriod;
boolean active;
int lifetime;
Map<String, String> assetContext;
}
package backup.gcp <<Rectangle>> {
class DatastoreBackupStampRepository implements BackupStampRepository
class DatastoreScheduleRepository implements ScheduleRepository
}
@enduml
\ No newline at end of file
@startuml
actor User
User -> BackupService : submit schedule
activate BackupService
BackupService -> ScheduleRepository : save schedule
database DB
ScheduleRepository -> DB
BackupService -> Scheduler : add task
ScheduleRepository -> BackupService
BackupService -> User : saved schedule(with id)
deactivate BackupService
control ScheduledTask
activate ScheduledTask
Scheduler -> ScheduledTask
ScheduledTask -> ScheduledTask : run by schedule
activate AssetBackupManager
ScheduledTask -> AssetBackupManager : export()
database BackupStorage
AssetBackupManager -> BackupStorage : save backup
AssetBackupManager -> BackupStampRepository : save export stamp
BackupStampRepository -> DB
deactivate AssetBackupManager
deactivate ScheduledTask
User -> BackupService : list backup stamps
@enduml
activate BackupService
activate ScheduleRepository
ScheduleRepository -> BackupSchedule :saved schedule
database DB
ScheduleRepository -> ProviderStorage
BackupService -> ScheduleRepository : save schedule
BackupService -> SchedulerService : add scheduled task
SchedulerService -> ScheduledTask
alt successful case
activate ScheduledTask
ScheduledTask --> ScheduledTask
deactivate ScheduledTask
ScheduledTask -> AssetManager : export backup
AssetManager -> BackupStampsRepository : submit backup stamp
BackupStampsRepository -> ProviderStorage
\ No newline at end of file
## Backup Service
## Introduction <a name="Introduction"></a>
### Submit scheduled backup task
```
POST /backup/v1/submitScheduledTask
POST /backup/v1/schedule
```
```
curl --location --request POST 'localhost:8080/backup/v1/submitScheduledTask' \
--header 'Data-Partition-Id: opendes' \
--header 'Authorization: Bearer <JWT> \
--header 'Content-Type: application/json' \
curl --location --request POST 'localhost:8080/backup/v1/schedule'
--header 'Data-Partition-Id: opendes'
--header 'Authorization: Bearer <JWT>
--header 'Content-Type: application/json'
--data-raw '{
"assetType": "datastore",
"namespace": "opendes",
"kind": "LegalTag",
"backupPeriod": "1",
"active": "true"
"assetType": "DATSTORE",
"backupPeriod": "5",
"active": "true",
"lifetime": 2,
"assetContext": {
"namespace": "opendes",
"kind": "TestKind"
}
}'
```
### Get schedule by id
```
GET /backup/v1/schedule?id=
```
```
curl --location --request GET 'localhost:8080/backup/v1/schedule?id=8de2a9e34bdf489aba20206b8b28ab6e' \
--header 'Data-Partition-Id: opendes'
--header 'Authorization: Bearer <JWT>
```
### Update schedule by id
```
PUT /backup/v1/schedule
```
```
curl --location --request PUT 'localhost:8080/backup/v1/schedule'
--header 'Data-Partition-Id: opendes'
--header 'Content-Type: application/json'
--data-raw '{
"scheduleId":"2049486875f541ad8df6a0e6da777b7f",
"backupPeriod": "5",
"active": "true",
"lifetime": 2,
}'
```
### List backup schedules
```
GET /backup/v1/listSchedules
GET /backup/v1/list_schedules
```
```
curl --location --request GET 'localhost:8080/backup/v1/listSchedules' \
--header 'Data-Partition-Id: opendes' \
curl --location --request GET 'localhost:8080/backup/v1/list_schedules'
--header 'Data-Partition-Id: opendes'
--header 'Authorization: Bearer <JWT>
```
### List available backups
```
GET /backup/v1/listBackups
GET /backup/v1/list_backups
```
```
curl --location --request GET 'localhost:8080/backup/v1/listBackups' \
--header 'Data-Partition-Id: opendes' \
curl --location --request GET 'localhost:8080/backup/v1/list_backups'
--header 'Data-Partition-Id: opendes'
--header 'Authorization: Bearer <JWT>
```
### Import backup
```
POST /backup/v1/submitImport
POST /backup/v1/backup_import
```
```
curl --location --request POST 'localhost:8080/backup/v1/submitImport' \
--header 'Data-Partition-Id: opendes' \
--header 'Authorization: Bearer <JWT> \
--header 'Content-Type: application/json' \
curl --location --request POST 'localhost:8080/backup/v1/backup_import'
--header 'Data-Partition-Id: opendes'
--header 'Authorization: Bearer <JWT>
--header 'Content-Type: application/json'
--data-raw '{
"backupPath": "gs://<backup file path>"
}'
"backupStampId": "4730f8b5dd1145a3b09335c640d455c0",
"asset":"DATASTORE"
}''
```
......@@ -20,11 +20,20 @@ In order to run the service locally or remotely, you will need to have the follo
| name | value | description | sensitive? | source |
| --- | --- | --- | --- | --- |
| `SPRING_CLOUD_GCP_DATASTORE_NAMESPACE` | ex `opendes` | Datastore namespace to backup| no | https://console.cloud.google.com/datastore |
| `OSDU_ENTITLEMENTS_URL` | ex `https://os-entitlements-gcp-jvmvia5dea-uc.a.run.app/entitlements/v1` | Entitlements API endpoint | no | output of infrastructure deployment |
| `GCP_BACKUP_BUCKET` | ex `osdu-cicd-epam-backup-service` | Storage bucket for backups | no | https://console.cloud.google.com/storage |
| `GCP_PROJECT_ID` | ex `osdu-cicd-epam` | GCP project id | no | - |
| `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
Schedulers can be configured with following variables, if not defined default values will be used
| name | value | description | default |
| --- | --- | --- | --- |
| `OSDU_SCHEDULER_TEAR-DOWN-TIME-UNIT` | ex `seconds OR minutes OR hours etc` | Time unit for tear down scheduler | `hours` |
| `OSDU_SCHEDULER_TEAR-DOWN-PERIOD` | ex `24` | The period with which expired backups will be deleted | `24` |
| `OSDU_SCHEDULER_BACKUP-TIME-UNIT` | ex `seconds OR minutes OR hours etc` | Time unit for backup export scheduler (Period will be configured with saved schedule) | `hours` |
| `OSDU_SCHEDULER_INITIAL-DELAY` | ex `1` | Delay before start exporting new added assets, helpful for int tests, to not overwhelm backup storage | `1` |
### Run Locally
Check that maven is installed:
......@@ -95,6 +104,55 @@ After configuring your environment as specified above, you can follow these step
```bash
cd provider/backup-gcp && mvn spring-boot:run
```
## Testing
### Running E2E Tests
This section describes how to run cloud OSDU E2E tests (testing/backup-test-gcp).
You will need to have the following environment variables defined.
| name | value | description | sensitive? | source |
| --- | --- | --- | --- | --- |
| `BACKUP_SERVICE_HOST` | ex`http://localhost:8080/backup/v1` | Service endpoint | no | - |
| `DATA_PARTITION_ID` | `opendes` | OSDU tenant used for testing | no | - |
| `INTEGRATION_TESTER` | ex`/path/to/directory/service-key.json` | Service account .json for API calls. Note: this user must have entitlements configured already | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
| `NO_DATA_ACCESS_TESTER` | ex`/path/to/directory/service-key.json` | Service account .json without data access | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
| `GCP_DEPLOY_FILE` | ex`/path/to/directory/service-key.json` | Service account for test data tear down, must have cloud role configured | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
| `INTEGRATION_TEST_AUDIENCE` | `****` | client application ID | yes | https://console.cloud.google.com/apis/credentials |
| `DATASTORE_NAMESPACE` | ex `opendes` | OSDU tenant used for testing | no | - |
**Entitlements configuration for integration accounts**
| INTEGRATION_TESTER | NO_DATA_ACCESS_TESTER |
| --- | --- |
| users<br/>backup.service| users |
**Cloud roles configuration for integration accounts**
| GCP_DEPLOY_FILE|
| --- |
| Cloud Datastore Owner |
Execute following command to build code and run all the integration tests:
```bash
# Note: this assumes that the environment variables for integration tests as outlined
# above are already exported in your environment.
# build + install integration test core
$ (cd testing/backup-test-core/ && mvn clean install)
```
```bash
# build + run GCP integration tests.
$ (cd testing/backup-test-gcp/ && mvn clean test)
```
## Deployment
* To deploy into Cloud run, please, use this documentation:
https://cloud.google.com/run/docs/quickstarts/build-and-deploy
* To deploy into App Engine, please, use this documentation:
https://cloud.google.com/appengine/docs/flexible/java/quickstart
## License
......
......@@ -24,7 +24,7 @@ public class ElasticBackupManager implements AssetBackupManager {
}
@Override
public void deleteBackups(List<BackupStamp> currentDate) {
public void deleteBackups(List<BackupStamp> backupStamps) {
throw new UnsupportedOperationException();
}
}
server.servlet.contextPath=/backup/v1
server.port=8080
logging.level.org.springframework=INFO
logging.level.org.springframework.boot.autoconfigure=ERROR
log.prefix=backup
spring.cloud.gcp.datastore.namespace=opendes
gcp.backup-bucket=osdu-cicd-epam-backup-service
gcp.project-id=osdu-cicd-epam
#Core
osdu.entitlements.url=https://os-entitlements-gcp-jvmvia5dea-uc.a.run.app/entitlements/v1
osdu.entitlements.app-key=test
osdu.scheduler.tear-down-time-unit=hours
osdu.scheduler.tear-down-period=24
osdu.scheduler.backup-time-unit=minutes
osdu.scheduler.backup-time-unit=hours
osdu.scheduler.initial-delay=1
#GCP
gcp.backup-bucket=osdu-cicd-epam-backup-service
gcp.project-id=osdu-cicd-epam
LOG_PREFIX=${log.prefix}
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>backup-test-core</artifactId>
<version>0.0.1</version>
<name>backup-test-core</name>
<description>Core test project for the backup service</description>
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>backup-test</artifactId>
<version>0.0.1</version>
<relativePath>../pom.xml</relativePath>
</parent>
<properties>
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
</properties>
<dependencies>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>backup-core</artifactId>
<version>0.0.1</version>
</dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-client</artifactId>
<version>1.19.4</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<version>5.5.2</version>
</dependency>
</dependencies>
</project>
package org.opengroup.osdu.backup;
public class Config {
public static final String BACKUP_SERVICE_HOST = "";
public static final String INTEGRATION_TESTER = "";
private static final String NO_DATA_ACCESS_TESTER = "";
public static final String DATA_PARTITION_ID = "";
public static String getBackupServiceHost() {
return getEnvironmentVariableOrDefaultValue("BACKUP_SERVICE_HOST", BACKUP_SERVICE_HOST);
}
public static String getIntegrationTester() {
return getEnvironmentVariableOrDefaultValue("INTEGRATION_TESTER", INTEGRATION_TESTER);
}
public static String getDataPartitionId() {
return getEnvironmentVariableOrDefaultValue("DATA_PARTITION_ID", DATA_PARTITION_ID);
}
public static String getNoAccessTester() {
return getEnvironmentVariableOrDefaultValue("NO_DATA_ACCESS_TESTER", NO_DATA_ACCESS_TESTER);
}
private static String getEnvironmentVariableOrDefaultValue(String key, String defaultValue) {
String environmentVariable = getEnvironmentVariable(key);
if (environmentVariable == null) {
environmentVariable = defaultValue;
}
return environmentVariable;
}
private static String getEnvironmentVariable(String propertyKey) {
return System.getProperty(propertyKey, System.getenv(propertyKey));
}
}
package org.opengroup.osdu.backup;/*
* Copyright 2020 Google LLC
* Copyright 2020 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
import java.io.IOException;
import java.net.URL;
import java.security.SecureRandom;
import java.security.cert.X509Certificate;
import java.util.Map;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import javax.ws.rs.core.MediaType;
public abstract class HttpClient {
protected static String accessToken;
protected static String noDataAccessToken;
public abstract String getAccessToken() throws IOException;
public abstract String getNoDataAccessToken() throws IOException;
public ClientResponse send(String path, String httpMethod, Map<String, String> headers,
String requestBody)
throws Exception {
Client client = this.getClient();
String mergedURL = new URL(Config.getBackupServiceHost() + path).toString();
System.out.println(String.format("calling %s API:%s", httpMethod, mergedURL));
System.out.println(String.format("request body:%s", requestBody));
if (requestBody != null) {
headers.put("Content-Length", Long.toString(requestBody.length()));
} else {
headers.put("Content-Length", "0");
}
WebResource webResource = client.resource(mergedURL);
WebResource.Builder builder = webResource.accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON);
headers.forEach(builder::header);
return builder.method(httpMethod, ClientResponse.class, requestBody);
}
private Client getClient() throws Exception {
TrustManager[] trustAllCerts = new TrustManager[]{new X509TrustManager() {
@Override
public X509Certificate[] getAcceptedIssuers() {
return null;
}
@Override
public void checkClientTrusted(X509Certificate[] certs, String authType) {
}
@Override
public void checkServerTrusted(X509Certificate[] certs, String authType) {
}
}};
try {
SSLContext sc = SSLContext.getInstance("TLS");
sc.init(null, trustAllCerts, new SecureRandom());
HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory());
} catch (Exception e) {
throw new Exception();
}
return Client.create();
}
}
package org.opengroup.osdu.backup;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import org.opengroup.osdu.backup.util.AssetUtil;
public class TestBase {
protected static HttpClient client;
protected static AssetUtil assetUtil;
public static Map<String, String> getCommonHeader() throws IOException {
return getHeaders(Config.getDataPartitionId(), client.getAccessToken());
}
public static Map<String, String> getHeaders(String dataPartition, String token) {
Map<String, String>