Commit a97bd09c authored by Artem Dobrynin (EPAM)'s avatar Artem Dobrynin (EPAM)
Browse files

Merge remote-tracking branch 'origin/feature/GONRG-1060-impl-sql-backup' into...

Merge remote-tracking branch 'origin/feature/GONRG-1060-impl-sql-backup' into feature/GONRG-998_-_BackupService_Change_tear-down_time_and_use_created-time_stamp
parents ab189d66 a02440fa
......@@ -4,6 +4,7 @@ public enum Asset {
DATASTORE,
ELASTIC,
STORAGE
STORAGE,
SQL
}
......@@ -2,6 +2,7 @@ package org.opengroup.osdu.backup.repository;
import java.util.List;
import java.util.Map;
import org.opengroup.osdu.backup.locator.Asset;
import org.opengroup.osdu.backup.model.BackupSchedule;
public interface ScheduleRepository {
......@@ -12,7 +13,7 @@ public interface ScheduleRepository {
BackupSchedule updateSchedule(BackupSchedule backupSchedule);
BackupSchedule findByAssetContext(Map<String, String> assetContext);
BackupSchedule findByAssetContext(Asset asset, Map<String, String> assetContext);
BackupSchedule save(BackupSchedule backupSchedule);
......
......@@ -42,7 +42,7 @@ public class BackupServiceImpl implements BackupService {
@Override
public BackupSchedule submitBackupSchedule(BackupSchedule backupSchedule) {
Map<String, String> assetContext = backupSchedule.getAssetContext();
BackupSchedule byAssetContext = scheduleRepository.findByAssetContext(assetContext);
BackupSchedule byAssetContext = scheduleRepository.findByAssetContext(backupSchedule.getAssetType(), assetContext);
if (Objects.nonNull(byAssetContext)) {
throw new AppException(HttpStatus.CONFLICT.value(), "Conflict"
, String.format("Schedule with context %s already exist", assetContext));
......
......@@ -46,7 +46,7 @@ public class BackupServiceImplTest {
@Test
public void testSubmitSchedule() {
when(scheduleRepository.findByAssetContext(any())).thenReturn(null);
when(scheduleRepository.findByAssetContext(any(), any())).thenReturn(null);
when(scheduleRepository.save(any())).thenReturn(schedule);
backupService.submitBackupSchedule(schedule);
verify(schedulerService).addTaskToScheduler(schedule);
......@@ -54,7 +54,7 @@ public class BackupServiceImplTest {
@Test(expected = AppException.class)
public void testSubmitExistingSchedule() {
when(scheduleRepository.findByAssetContext(any())).thenReturn(schedule);
when(scheduleRepository.findByAssetContext(any(), any())).thenReturn(schedule);
backupService.submitBackupSchedule(schedule);
}
......
......@@ -52,6 +52,12 @@
<version>1.2.5.RELEASE</version>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-gcp-starter-sql-postgresql</artifactId>
<version>1.2.6.RELEASE</version>
</dependency>
<dependency>
<groupId>com.google.apis</groupId>
<artifactId>google-api-services-datastore</artifactId>
......
......@@ -13,4 +13,8 @@ public class PropertiesConfiguration {
private String projectId;
private String sqlBackupKind;
private String sqlRestoreBackupKind;
}
/*
Copyright 2020 Google LLC
Copyright 2020 EPAM Systems, Inc
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package org.opengroup.osdu.backup.provider.gcp.manager;
import com.google.api.client.googleapis.util.Utils;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.services.sqladmin.SQLAdmin;
import com.google.api.services.sqladmin.SQLAdminScopes;
import com.google.api.services.sqladmin.model.BackupRun;
import com.google.api.services.sqladmin.model.InstancesRestoreBackupRequest;
import com.google.api.services.sqladmin.model.Operation;
import com.google.api.services.sqladmin.model.RestoreBackupContext;
import com.google.auth.http.HttpCredentialsAdapter;
import com.google.auth.oauth2.GoogleCredentials;
import com.google.common.collect.ImmutableMap;
import java.io.IOException;
import java.time.LocalDateTime;
import java.time.temporal.ChronoUnit;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.Random;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.opengroup.osdu.backup.config.property.SchedulersProperties;
import org.opengroup.osdu.backup.exception.ScheduleException;
import org.opengroup.osdu.backup.manager.AssetBackupManager;
import org.opengroup.osdu.backup.model.BackupSchedule;
import org.opengroup.osdu.backup.model.BackupStamp;
import org.opengroup.osdu.backup.provider.gcp.config.property.PropertiesConfiguration;
import org.opengroup.osdu.backup.provider.gcp.model.constant.Constants;
import org.opengroup.osdu.backup.repository.BackupStampRepository;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Component;
@Slf4j
@Component(value = "SQL")
@RequiredArgsConstructor
public class SQLBackupManager implements AssetBackupManager {
private final BackupStampRepository stampsRepository;
private final SchedulersProperties schedulersProperties;
private final PropertiesConfiguration propertiesConfiguration;
private static final String BACKUP_DESCRIPTION = "Created by backup-service. Id: ";
@Override
public void exportBackup(BackupSchedule backupSchedule) {
String instance = backupSchedule.getAssetContext().get(Constants.CONTEXT_INSTANCE);
Long backupId = new Random().nextLong();
log.info("Starting backup for instance : {} ", instance);
BackupRun backupRun = getBackupRun(instance, backupId);
try {
SQLAdmin sqlAdmin = new SQLAdmin(Utils.getDefaultTransport(), Utils.getDefaultJsonFactory(),
createCredentials());
sqlAdmin.backupRuns()
.insert(propertiesConfiguration.getProjectId(),
instance, backupRun).execute();
BackupStamp stamp = buildBackupStamp(backupSchedule, backupId);
stampsRepository.submitBackupStamp(stamp);
log.info("Export status done. Backup id : {}", "backupId");
} catch (IOException e) {
log.error(e.getMessage());
throw new ScheduleException("SQL export failure", e);
}
}
@Override
public BackupStamp importBackup(String stampId) {
log.info("Starting backup restore for stampId : {}", stampId);
BackupStamp backupByStampId = stampsRepository.findBackupByStampId(stampId);
if (!backupByStampId.isSuccess()) {
throw new AppException(HttpStatus.BAD_REQUEST.value(), "Failed backup",
"Backup not available for import");
}
String instance = backupByStampId.getAssetContext().get(Constants.CONTEXT_INSTANCE);
String idFromStamp = backupByStampId.getAssetContext().get(Constants.CONTEXT_BACKUP_ID);
try {
SQLAdmin sqlAdmin = new SQLAdmin(Utils.getDefaultTransport(), Utils.getDefaultJsonFactory(),
createCredentials());
List<BackupRun> backupRunList = sqlAdmin.backupRuns()
.list(propertiesConfiguration.getProjectId(),
instance).execute().getItems();
Optional<BackupRun> optional = backupRunList.stream()
.filter(c -> c.getDescription().contains(idFromStamp)).findFirst();
if (optional.isPresent()) {
Long backupId = optional.get().getId();
RestoreBackupContext restoreBackupContext = getRestoreBackupContext(backupId, instance);
InstancesRestoreBackupRequest restoreBackupRequest = new InstancesRestoreBackupRequest();
restoreBackupRequest.setRestoreBackupContext(restoreBackupContext);
Operation result = sqlAdmin.instances()
.restoreBackup(propertiesConfiguration.getProjectId(),
instance,
restoreBackupRequest).execute();
log.info("Restore status : {}", result.getStatus());
} else {
throw new AppException(HttpStatus.BAD_REQUEST.value(), "Failed restore backup. Backup not found.",
"Backup not available for restore");
}
} catch (IOException e) {
e.printStackTrace();
throw new ScheduleException("SQL restore failure", e);
}
return backupByStampId;
}
@Override
public void deleteBackups(List<BackupStamp> backupStamps) {
throw new UnsupportedOperationException();
}
private BackupStamp buildBackupStamp(BackupSchedule backupSchedule, Long id) {
String instance = backupSchedule.getAssetContext().get(Constants.CONTEXT_INSTANCE);
return BackupStamp.builder()
.assetType(backupSchedule.getAssetType())
.assetContext(ImmutableMap
.of(Constants.CONTEXT_INSTANCE, instance,
Constants.CONTEXT_BACKUP_ID, String.valueOf(id)))
.tearDownDate(
LocalDateTime.now().plus(
backupSchedule.getLifetime(),
ChronoUnit.valueOf(schedulersProperties.getTearDownTimeUnit().toString())))
.success(true)
.build();
}
private BackupRun getBackupRun(String instance, Long id) {
BackupRun backupRun = new BackupRun();
backupRun.setKind(propertiesConfiguration.getSqlBackupKind());
backupRun.setType("ON_DEMAND");
backupRun.setDescription(BACKUP_DESCRIPTION + id);
backupRun.setInstance(instance);
return backupRun;
}
private HttpRequestInitializer createCredentials() {
GoogleCredentials credentials;
try {
credentials = GoogleCredentials.getApplicationDefault();
} catch (IOException err) {
throw new ScheduleException(
"Unable to obtain credentials to communicate with the Cloud SQL API", err);
}
if (credentials.createScopedRequired()) {
credentials =
credentials.createScoped(Arrays.asList(SQLAdminScopes.SQLSERVICE_ADMIN,
SQLAdminScopes.CLOUD_PLATFORM));
}
return new HttpCredentialsAdapter(credentials);
}
private RestoreBackupContext getRestoreBackupContext(Long backupId, String instance) {
RestoreBackupContext context = new RestoreBackupContext();
context.setBackupRunId(backupId);
context.setProject(propertiesConfiguration.getProjectId());
context.setKind(propertiesConfiguration.getSqlRestoreBackupKind());
context.setInstanceId(instance);
return context;
}
}
......@@ -7,6 +7,8 @@ public class Constants {
public static final String CONTEXT_NAMESPACE = "namespace";
public static final String CONTEXT_KIND = "kind";
public static final String CONTEXT_BACKUP_ID = "backupId";
public static final String CONTEXT_INSTANCE = "instance";
public static final String GCP_STORAGE_PREFIX = "gs://";
public static final String CONTEXT_BACKUP_PATH = "Path";
......
......@@ -46,6 +46,12 @@ public class BackupStampEntity {
@Field(name = "TearDownDate")
private LocalDateTime tearDownDate;
@Field(name = "BackupId")
private String backupId;
@Field(name = "Instance")
private String instance;
public BackupStampEntity(BackupStamp backupStamp) {
// TODO define id generation
if (Objects.isNull(backupStamp.getStampId())) {
......@@ -60,18 +66,29 @@ public class BackupStampEntity {
this.backupPath = backupStamp.getAssetContext().get(Constants.CONTEXT_BACKUP_PATH);
this.tearDownDate = backupStamp.getTearDownDate();
this.success = backupStamp.isSuccess();
this.backupId = backupStamp.getAssetContext().get(Constants.CONTEXT_BACKUP_ID);
this.instance = backupStamp.getAssetContext().get(Constants.CONTEXT_INSTANCE);
}
public BackupStamp getBackupStamp() {
ImmutableMap<String, String> assetCtx;
if (this.assetType.equals(Asset.SQL)) {
assetCtx = ImmutableMap.of(
Constants.CONTEXT_INSTANCE, this.instance,
Constants.CONTEXT_BACKUP_ID, this.backupId);
} else {
assetCtx = ImmutableMap.of(
Constants.CONTEXT_NAMESPACE, this.namespace,
Constants.CONTEXT_KIND, this.kind,
Constants.CONTEXT_BACKUP_PATH, this.backupPath);
}
return BackupStamp.builder()
.stampId(this.getStampId())
.backupRepository(this.getBackupRepo())
.assetType(this.assetType)
.tearDownDate(this.tearDownDate)
.assetContext(ImmutableMap
.of(Constants.CONTEXT_NAMESPACE, this.namespace,
Constants.CONTEXT_KIND, this.kind,
Constants.CONTEXT_BACKUP_PATH, this.backupPath))
.assetContext(assetCtx)
.success(this.success)
.build();
}
......
......@@ -42,6 +42,9 @@ public class DatastoreBackupScheduleEntity {
@Field(name = "Kind")
private String kind;
@Field(name = "Instance")
private String instance;
public DatastoreBackupScheduleEntity(BackupSchedule schedule) {
// TODO define id generation
if (Objects.isNull(schedule.getScheduleId())) {
......@@ -55,18 +58,26 @@ public class DatastoreBackupScheduleEntity {
this.lifetimeInDays = schedule.getLifetime();
this.namespace = schedule.getAssetContext().get(Constants.CONTEXT_NAMESPACE);
this.kind = schedule.getAssetContext().get(Constants.CONTEXT_KIND);
this.instance = schedule.getAssetContext().get(Constants.CONTEXT_INSTANCE);
}
public BackupSchedule toModel() {
ImmutableMap<String, String> assetCtx;
if (this.assetType.equals(Asset.SQL)) {
assetCtx = ImmutableMap.of(
Constants.CONTEXT_INSTANCE, this.instance);
} else {
assetCtx = ImmutableMap.of(
Constants.CONTEXT_NAMESPACE, this.namespace,
Constants.CONTEXT_KIND, this.kind);
}
return BackupSchedule.builder()
.scheduleId(this.scheduleId)
.assetType(this.assetType)
.backupPeriod(this.backupPeriod)
.active(this.active)
.lifetime(this.lifetimeInDays)
.assetContext(ImmutableMap.
of(Constants.CONTEXT_NAMESPACE, this.namespace,
Constants.CONTEXT_KIND, this.kind))
.assetContext(assetCtx)
.build();
}
......
......@@ -6,6 +6,7 @@ import java.util.Objects;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
import lombok.RequiredArgsConstructor;
import org.opengroup.osdu.backup.locator.Asset;
import org.opengroup.osdu.backup.model.BackupSchedule;
import org.opengroup.osdu.backup.provider.gcp.model.constant.Constants;
import org.opengroup.osdu.backup.provider.gcp.model.entity.DatastoreBackupScheduleEntity;
......@@ -49,10 +50,16 @@ public class DatastoreScheduleRepository implements ScheduleRepository {
}
@Override
public BackupSchedule findByAssetContext(Map<String, String> assetContext) {
String kind = assetContext.get(Constants.CONTEXT_KIND);
String namespace = assetContext.get(Constants.CONTEXT_NAMESPACE);
DatastoreBackupScheduleEntity taskExist = repository.findOneByKindAndNamespace(kind, namespace);
public BackupSchedule findByAssetContext(Asset asset, Map<String, String> assetContext) {
DatastoreBackupScheduleEntity taskExist;
if (asset.equals(Asset.SQL)) {
String instance = assetContext.get(Constants.CONTEXT_INSTANCE);
taskExist = repository.findOneByInstance(instance);
} else {
String kind = assetContext.get(Constants.CONTEXT_KIND);
String namespace = assetContext.get(Constants.CONTEXT_NAMESPACE);
taskExist = repository.findOneByKindAndNamespace(kind, namespace);
}
return Objects.isNull(taskExist) ? null : taskExist.toModel();
}
......
......@@ -10,6 +10,7 @@ public interface SchedulesEntityRepository extends DatastoreRepository<Datastore
@Nullable
DatastoreBackupScheduleEntity findOneByKindAndNamespace(String kind, String namespace);
@Nullable
DatastoreBackupScheduleEntity findOneByInstance(String instance);
}
......@@ -18,3 +18,9 @@ gcp.backup-bucket=osdu-cicd-epam-backup-service
gcp.project-id=osdu-cicd-epam
LOG_PREFIX=${log.prefix}
spring.cloud.gcp.sql.databaseName=postgres
spring.cloud.gcp.sql.instanceConnectionName=osdu-cicd-epam:us-central1:test-postgre
gcp.sql-backup-kind=sql#backupRun
gcp.sql-restore-backup-kind=sql#restoreBackupContext
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment