Commit 5fd37b5f authored by Oleksandr Kosse (EPAM)'s avatar Oleksandr Kosse (EPAM)
Browse files

Merge branch 'cicd' into 'master'

Configure Community CICD for Backup service

See merge request !1
parents 8b9e07c3 f453058d
Pipeline #26147 failed with stages
in 25 minutes and 25 seconds
# Generated by FOSSA CLI (https://github.com/fossas/fossa-cli)
# Visit https://fossa.com to learn more
version: 2
cli:
server: https://app.fossa.com
fetcher: custom
project: os-backup
analyze:
modules:
- name: backup-service
type: mvn
target: pom.xml
path: .
- name: backup-core
type: mvn
target: backup-core/pom.xml
path: .
- name: backup-gcp
type: mvn
target: provider/backup-gcp/pom.xml
path: .
......@@ -4,6 +4,7 @@ target/
!**/src/main/**
!**/src/test/**
.DS_Store
### STS ###
.apt_generated
......@@ -32,7 +33,6 @@ build/
.vscode/
### Other ###
.mvn
target/*
*/target
/mvn
......
variables:
OSDU_GCP_APPLICATION_NAME: backup
OSDU_GCP_CLOUD_RUN_PARAMETERS: "--max-instances=1"
OSDU_GCP_VENDOR: gcp
OSDU_GCP_SERVICE: backup
OSDU_GCP_BACKUP_BUCKET: nice-etching-277309-backup-service
OSDU_GCP_SPRING_CLOUD_GCP_SQL_DATABASENAME: postgres
OSDU_GCP_SPRING_CLOUD_GCP_SQL_INSTANCECONNECTIONNAME: nice-etching-277309:us-central1:int-pgdb
OSDU_GCP_SQL_BACKUP_KIND: sql#backupRun
OSDU_GCP_SQL_RESTORE_BACKUP_KIND: sql#restoreBackupContext
OSDU_GCP_ENV_VARS: GCP_PROJECT_ID=$OSDU_GCP_PROJECT,OSDU_ENTITLEMENTS_URL=$OSDU_GCP_ENTITLEMENTS_URL,GCP_BACKUP_BUCKET=$OSDU_GCP_BACKUP_BUCKET,SPRING_CLOUD_GCP_SQL_DATABASENAME=$OSDU_GCP_SPRING_CLOUD_GCP_SQL_DATABASENAME,GCP_SQL_RESTORE_BACKUP_KIND=$OSDU_GCP_SQL_RESTORE_BACKUP_KIND
include:
- project: 'osdu/platform/ci-cd-pipelines'
file: 'standard-setup.yml'
- project: 'osdu/platform/ci-cd-pipelines'
file: 'build/maven.yml'
- project: 'osdu/platform/ci-cd-pipelines'
file: 'scanners/fossa-maven.yml'
- project: 'osdu/platform/ci-cd-pipelines'
file: 'scanners/gitlab-ultimate.yml'
- project: 'osdu/platform/ci-cd-pipelines'
file: 'publishing/pages.yml'
- project: "osdu/platform/ci-cd-pipelines"
file: 'cloud-providers/osdu-gcp-cloudrun.yml'
osdu-gcp-test:
allow_failure: true
<!--
~ Copyright 2020 Google LLC
~ Copyright 2020 EPAM Systems, Inc
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ https://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<settings>
<profiles>
<profile>
<!-- This profile uses the CI-Token to authenticate with the server, and is the default case -->
<id>GitLab-Authenticate-With-Job-Token</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<repo.releases.id>community-maven-repo</repo.releases.id>
<publish.snapshots.id>community-maven-via-job-token</publish.snapshots.id>
<publish.releases.id>community-maven-via-job-token</publish.releases.id>
<repo.releases.url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</repo.releases.url>
<publish.snapshots.url>https://community.opengroup.org/api/v4/projects/380/packages/maven</publish.snapshots.url>
<publish.releases.url>https://community.opengroup.org/api/v4/projects/380/packages/maven</publish.releases.url>
</properties>
</profile>
<profile>
<!-- This profile uses a personal token to authenticate, which is useful for local or manual runs. The
presence of the COMMUNITY_MAVEN_TOKEN variable triggers this and overrides the CI Token
based authentication -->
<id>GitLab-Authenticate-With-Private-Token</id>
<activation>
<property>
<name>env.COMMUNITY_MAVEN_TOKEN</name>
</property>
</activation>
<properties>
<repo.releases.id>community-maven-repo</repo.releases.id>
<publish.snapshots.id>community-maven-via-private-token</publish.snapshots.id>
<publish.releases.id>community-maven-via-private-token</publish.releases.id>
<repo.releases.url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</repo.releases.url>
<publish.snapshots.url>https://community.opengroup.org/api/v4/projects/380/packages/maven</publish.snapshots.url>
<publish.releases.url>https://community.opengroup.org/api/v4/projects/380/packages/maven</publish.releases.url>
</properties>
</profile>
</profiles>
<servers>
<server>
<id>community-maven-via-job-token</id>
<configuration>
<httpHeaders>
<property>
<name>Job-Token</name>
<value>${env.CI_JOB_TOKEN}</value>
</property>
</httpHeaders>
</configuration>
</server>
<server>
<id>community-maven-via-private-token</id>
<configuration>
<httpHeaders>
<property>
<name>Private-Token</name>
<value>${env.COMMUNITY_MAVEN_TOKEN}</value>
</property>
</httpHeaders>
</configuration>
</server>
</servers>
</settings>
This diff is collapsed.
# Backup and Restore
# Backup service
There are OSDU R3 requirements to implement and document Backup and Restore procedures for OSDU services components
Backup service provide a set of APIs to schedule backups for different Assets (Datastore, Storage, ElasticSearch, etc depends on which currently implemented ),
list registered Schedules, list available Backups, restore Asset state from Backup.
Recovery Time Objectives (RTO) are different and vary per Data Asset (~ per OSDU Service)
\ No newline at end of file
## Core Interfaces
![Core](./docs/core.png)
## Data Repositories
![Repo](./docs/datarepos.png)
## Sequence diagram
Submit schedule workflow
![Diagram](./docs/sequence.png)
## GCP Implementation
All documentation for the GCP implementation of Backup service lives [here](./provider/backup-gcp/README.md)
## License
Copyright 2020 Google LLC
Copyright 2020 EPAM Systems, Inc
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
\ No newline at end of file
......@@ -2,32 +2,48 @@
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>backup-core</artifactId>
<description>Backup service Core on GCP</description>
<version>0.0.1-SNAPSHOT</version>
<packaging>jar</packaging>
<parent>
<artifactId>backup-service</artifactId>
<groupId>org.opengroup.osdu</groupId>
<version>1.0.0</version>
<version>0.0.1-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>backup-core</artifactId>
<description>Backup service Core on GCP</description>
<version>1.0.0</version>
<properties>
<os-core-common.version>0.3.5</os-core-common.version>
<openapi.version>1.4.8</openapi.version>
</properties>
<dependencies>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
<version>${os-core-common.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-tomcat</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
<version>0.3.6</version>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-security</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.retry</groupId>
<artifactId>spring-retry</artifactId>
<version>1.2.5.RELEASE</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-aspects</artifactId>
<version>5.2.8.RELEASE</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
......@@ -37,9 +53,36 @@
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>1.10.19</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-beans</artifactId>
<artifactId>spring-test</artifactId>
<version>5.2.7.RELEASE</version>
<scope>test</scope>
</dependency>
<!-- OpenAPI -->
<dependency>
<groupId>org.springdoc</groupId>
<artifactId>springdoc-openapi-ui</artifactId>
<version>${openapi.version}</version>
</dependency>
<dependency>
<groupId>org.springdoc</groupId>
<artifactId>springdoc-openapi-data-rest</artifactId>
<version>${openapi.version}</version>
</dependency>
</dependencies>
</project>
\ No newline at end of file
/*
* Copyright 2020 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.backup;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication(scanBasePackages = "org.opengroup")
public class BackupApplication {
public static void main(String[] args) {
SpringApplication.run(BackupApplication.class, args);
}
}
/*
* Copyright 2020 Google LLC
* Copyright 2020 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.backup.api;
import java.util.EnumMap;
import java.util.List;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.opengroup.osdu.backup.model.SubmitRequest;
import org.opengroup.osdu.backup.provider.interfaces.BackupService;
import org.springframework.beans.factory.annotation.Autowired;
import org.opengroup.osdu.backup.locator.Asset;
import org.opengroup.osdu.backup.model.BackupImportRequest;
import org.opengroup.osdu.backup.model.BackupRole;
import org.opengroup.osdu.backup.model.BackupSchedule;
import org.opengroup.osdu.backup.model.BackupStamp;
import org.opengroup.osdu.backup.service.BackupService;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.PutMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.context.annotation.RequestScope;
@Slf4j
@RestController
@RequestScope
@RequiredArgsConstructor
@PreAuthorize("@authorizationFilter.hasPermission('" + BackupRole.BACKUP + "')")
public class BackupApi {
@Autowired
private BackupService backupService;
private final BackupService backupService;
@PostMapping("/schedule")
public ResponseEntity<BackupSchedule> submitBackupSchedule(
@RequestBody BackupSchedule backupSchedule) {
BackupSchedule savedBackupSchedule = backupService.submitBackupSchedule(backupSchedule);
return new ResponseEntity<>(savedBackupSchedule, HttpStatus.CREATED);
}
@PostMapping("/submitTask/")
public ResponseEntity<String> submitTask(@RequestBody SubmitRequest submitRequest) {
backupService.submitTask(submitRequest);
return new ResponseEntity<String>("Backup service is alive", HttpStatus.OK);
}
@PutMapping("/schedule")
public ResponseEntity<BackupSchedule> updateBackupSchedule(
@RequestBody BackupSchedule backupSchedule) {
BackupSchedule savedBackupSchedule = backupService.updateBackupSchedule(backupSchedule);
return new ResponseEntity<>(savedBackupSchedule, HttpStatus.ACCEPTED);
}
@GetMapping("/schedule")
public ResponseEntity<BackupSchedule> getScheduleById(@RequestParam String id) {
return new ResponseEntity<>(backupService.getBackupSchedule(id), HttpStatus.OK);
}
@PostMapping("/backup_import")
public ResponseEntity<BackupStamp> submitImport(@RequestBody BackupImportRequest backupId) {
BackupStamp stamp = backupService.submitBackupImportRequest(backupId);
return new ResponseEntity<>(stamp, HttpStatus.OK);
}
@GetMapping("/list_schedules")
public ResponseEntity<List<BackupSchedule>> listSchedules() {
return new ResponseEntity<>(backupService.listSchedules(), HttpStatus.OK);
}
@GetMapping("/list_backups")
public ResponseEntity<EnumMap<Asset, List<BackupStamp>>> listBackups() {
return new ResponseEntity<>(backupService.listBackups(), HttpStatus.OK);
}
}
/*
* Copyright 2020 Google LLC
* Copyright 2020 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.backup.api;
import java.util.List;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.opengroup.osdu.backup.model.BackupCollection;
import org.opengroup.osdu.backup.model.BackupRole;
import org.opengroup.osdu.backup.model.CollectionState;
import org.opengroup.osdu.backup.service.BackupCollectionService;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.PutMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.context.annotation.RequestScope;
@Slf4j
@RestController
@RequestScope
@RequiredArgsConstructor
@PreAuthorize("@authorizationFilter.hasPermission('" + BackupRole.BACKUP + "')")
public class BackupCollectionApi {
private final BackupCollectionService collectionService;
@PostMapping("/collection")
public ResponseEntity<BackupCollection> createCollection(@RequestBody BackupCollection collection) {
BackupCollection backupCollection = collectionService.saveCollection(collection);
return new ResponseEntity<>(backupCollection, HttpStatus.CREATED);
}
@PutMapping("/collection/{name}")
public ResponseEntity<BackupCollection> updateCollection(@PathVariable String name,
@RequestBody BackupCollection collection) {
BackupCollection backupCollection = collectionService.update(collection);
return new ResponseEntity<>(backupCollection, HttpStatus.CREATED);
}
@GetMapping("/collection/{name}")
public ResponseEntity<List<CollectionState>> getCollectionStates(@PathVariable String name) {
List<CollectionState> collectionStates = collectionService.getCollectionStates(name);
return new ResponseEntity<>(collectionStates, HttpStatus.OK);
}
@GetMapping("/collection/restore/{stateId}")
public ResponseEntity<CollectionState> restoreCollection(@PathVariable String stateId) {
CollectionState collectionState = collectionService.restoreCollection(stateId);
return new ResponseEntity<>(collectionState, HttpStatus.OK);
}
}
/*
* Copyright 2020 Google LLC
* Copyright 2020 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.backup.api;
import javax.annotation.security.PermitAll;
......@@ -11,15 +27,15 @@ import org.springframework.web.bind.annotation.RestController;
@RequestMapping("/health")
public class HealthCheckApi {
@PermitAll
@GetMapping("/liveness_check")
public ResponseEntity<String> livenessCheck() {
return new ResponseEntity<String>("Backup service is alive", HttpStatus.OK);
}
@PermitAll
@GetMapping("/liveness_check")
public ResponseEntity<String> livenessCheck() {
return new ResponseEntity<>("Backup service is alive", HttpStatus.OK);
}
@PermitAll
@GetMapping("/readiness_check")
public ResponseEntity<String> readinessCheck() {
return new ResponseEntity<String>("Backup service is ready", HttpStatus.OK);
}
@PermitAll
@GetMapping("/readiness_check")
public ResponseEntity<String> readinessCheck() {
return new ResponseEntity<>("Backup service is ready", HttpStatus.OK);
}
}
/*
Copyright 2020 Google LLC
Copyright 2020 EPAM Systems, Inc
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package org.opengroup.osdu.backup.config;
import org.opengroup.osdu.backup.locator.AssetManagerFactory;
import org.springframework.beans.factory.FactoryBean;
import org.springframework.beans.factory.config.ServiceLocatorFactoryBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class AssetLocatorConfig {
@Bean
public FactoryBean serviceLocatorFactoryBean() {
ServiceLocatorFactoryBean factoryBean = new ServiceLocatorFactoryBean();
factoryBean.setServiceLocatorInterface(AssetManagerFactory.class);
factoryBean.setServiceLocatorExceptionClass(AssetServiceLocatorException.class);
return factoryBean;
}
}
/*
* Copyright 2020 Google LLC
* Copyright 2020 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.