Commit 37dc4eda authored by Riabokon Stanislav(EPAM)[GCP]'s avatar Riabokon Stanislav(EPAM)[GCP]
Browse files

Merge branch 'feature/GONRG-974-Implement_retry_on_fail' into 'master'

Resolve GONRG-974 "Feature/ implement retry on fail"

Closes GONRG-974

See merge request go3-nrg/backup-service!6
parents 62961708 b1ea08aa
......@@ -31,6 +31,16 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-security</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.retry</groupId>
<artifactId>spring-retry</artifactId>
<version>1.2.5.RELEASE</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-aspects</artifactId>
<version>5.2.8.RELEASE</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
......
package org.opengroup.osdu.backup.exception;
public class ScheduleException extends RuntimeException {
public ScheduleException(String message, Throwable cause) {
super(message, cause);
}
}
......@@ -22,5 +22,8 @@ public class BackupStamp {
private LocalDateTime tearDownDate;
private boolean success;
private Map<String, String> assetContext;
}
......@@ -24,7 +24,8 @@ public class TearDownBackupScheduledTask implements Runnable {
EnumMap<Asset, List<BackupStamp>> backupStampsByCurrentDate = backupStampRepository
.findBackupStampsByTearDownBeforeCurrentDate(tearDownDateTime);
backupStampsByCurrentDate.entrySet().stream().filter(assetListEntry -> !assetListEntry.getValue().isEmpty())
backupStampsByCurrentDate.entrySet().stream()
.filter(assetListEntry -> !assetListEntry.getValue().isEmpty())
.forEach(assetListEntry -> assetManagerFactory.getAssetManager(assetListEntry.getKey())
.deleteBackups(assetListEntry.getValue()));
}
......
......@@ -23,25 +23,4 @@ BackupStampRepository -> DB
deactivate AssetBackupManager
deactivate ScheduledTask
User -> BackupService : list backup stamps
@enduml
activate BackupService
activate ScheduleRepository
ScheduleRepository -> BackupSchedule :saved schedule
database DB
ScheduleRepository -> ProviderStorage
BackupService -> ScheduleRepository : save schedule
BackupService -> SchedulerService : add scheduled task
SchedulerService -> ScheduledTask
alt successful case
activate ScheduledTask
ScheduledTask --> ScheduledTask
deactivate ScheduledTask
ScheduledTask -> AssetManager : export backup
AssetManager -> BackupStampsRepository : submit backup stamp
BackupStampsRepository -> ProviderStorage
\ No newline at end of file
......@@ -4,9 +4,11 @@ package org.opengroup.osdu.backup.provider.gcp;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.context.properties.ConfigurationPropertiesScan;
import org.springframework.retry.annotation.EnableRetry;
@ConfigurationPropertiesScan(basePackages = "org.opengroup")
@SpringBootApplication(scanBasePackages = "org.opengroup")
@EnableRetry
public class BackupApplicationGCP {
public static void main(String[] args) {
......
package org.opengroup.osdu.backup.provider.gcp.config;
import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.api.services.datastore.v1.Datastore;
import com.google.api.services.iam.v1.IamScopes;
import com.google.auth.http.HttpCredentialsAdapter;
import com.google.auth.oauth2.GoogleCredentials;
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.StorageOptions;
import java.io.IOException;
import java.security.GeneralSecurityException;
import java.util.ArrayList;
import java.util.List;
import org.opengroup.osdu.backup.provider.gcp.GcpPackageMarker;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.springframework.cloud.gcp.data.datastore.repository.config.EnableDatastoreRepositories;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.http.HttpStatus;
@Configuration
@EnableDatastoreRepositories(basePackageClasses = GcpPackageMarker.class)
......@@ -15,4 +28,28 @@ public class GCPConfiguration {
public Storage googleCloudStorage() {
return StorageOptions.getDefaultInstance().getService();
}
@Bean
public Datastore googleDatastore() {
JsonFactory JSON_FACTORY = new JacksonFactory();
try {
GoogleCredentials credential = GoogleCredentials
.getApplicationDefault();
if (credential.createScopedRequired()) {
List<String> scopes = new ArrayList<>();
scopes.add(IamScopes.CLOUD_PLATFORM);
scopes.add("https://www.googleapis.com/auth/datastore");
credential = credential.createScoped(scopes);
}
credential.refreshAccessToken();
return new Datastore(GoogleNetHttpTransport.newTrustedTransport(), JSON_FACTORY,
new HttpCredentialsAdapter(credential));
} catch (GeneralSecurityException | IOException e) {
throw new AppException(HttpStatus.INTERNAL_SERVER_ERROR.value(), e.getMessage(), e.getMessage());
}
}
}
package org.opengroup.osdu.backup.provider.gcp.manager;
import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.api.gax.paging.Page;
import com.google.api.services.datastore.v1.Datastore;
import com.google.api.services.datastore.v1.Datastore.Projects.Operations;
......@@ -11,9 +8,6 @@ import com.google.api.services.datastore.v1.model.GoogleDatastoreAdminV1EntityFi
import com.google.api.services.datastore.v1.model.GoogleDatastoreAdminV1ExportEntitiesRequest;
import com.google.api.services.datastore.v1.model.GoogleDatastoreAdminV1ImportEntitiesRequest;
import com.google.api.services.datastore.v1.model.GoogleLongrunningOperation;
import com.google.api.services.iam.v1.IamScopes;
import com.google.auth.http.HttpCredentialsAdapter;
import com.google.auth.oauth2.GoogleCredentials;
import com.google.cloud.Timestamp;
import com.google.cloud.storage.Blob;
import com.google.cloud.storage.Storage;
......@@ -21,18 +15,17 @@ import com.google.cloud.storage.Storage.BlobListOption;
import com.google.cloud.storage.StorageBatch;
import com.google.common.collect.ImmutableMap;
import java.io.IOException;
import java.security.GeneralSecurityException;
import java.time.LocalDateTime;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.opengroup.osdu.backup.config.property.SchedulersProperties;
import org.opengroup.osdu.backup.exception.ScheduleException;
import org.opengroup.osdu.backup.manager.AssetBackupManager;
import org.opengroup.osdu.backup.model.BackupSchedule;
import org.opengroup.osdu.backup.model.BackupStamp;
......@@ -42,6 +35,9 @@ import org.opengroup.osdu.backup.provider.gcp.model.entity.BackupStampEntity;
import org.opengroup.osdu.backup.repository.BackupStampRepository;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.springframework.http.HttpStatus;
import org.springframework.retry.annotation.Backoff;
import org.springframework.retry.annotation.Recover;
import org.springframework.retry.annotation.Retryable;
import org.springframework.stereotype.Component;
@Slf4j
......@@ -49,8 +45,6 @@ import org.springframework.stereotype.Component;
@RequiredArgsConstructor
public class DatastoreBackupManager implements AssetBackupManager {
private final JsonFactory JSON_FACTORY = new JacksonFactory();
private final BackupStampRepository stampsRepository;
private final PropertiesConfiguration propertiesConfiguration;
......@@ -59,10 +53,10 @@ public class DatastoreBackupManager implements AssetBackupManager {
private final SchedulersProperties schedulersProperties;
private Datastore datastore;
private final Datastore datastore;
// TODO plan behaviour with long running backup process
@Override
@Retryable(value = ScheduleException.class, maxAttempts = 3, backoff = @Backoff( delay = 60000))
public void exportBackup(BackupSchedule backupSchedule) {
String namespace = backupSchedule.getAssetContext().get(Constants.CONTEXT_NAMESPACE);
......@@ -73,8 +67,6 @@ public class DatastoreBackupManager implements AssetBackupManager {
try {
GoogleDatastoreAdminV1ExportEntitiesRequest context = buildExportEntitiesRequest(backupSchedule);
Datastore datastore = getDatastore();
GoogleLongrunningOperation exportTaskCreationResponse = datastore.projects()
.export(propertiesConfiguration.getProjectId(), context)
.execute();
......@@ -89,29 +81,21 @@ public class DatastoreBackupManager implements AssetBackupManager {
log.info("Export status done : {} backup path : {} export : {}", exportTaskResultResponse.getDone(),
outputUrlPrefix, exportTaskResultResponse);
BackupStamp build = BackupStamp.builder()
.assetType(backupSchedule.getAssetType())
.assetContext(ImmutableMap
.of(Constants.CONTEXT_NAMESPACE, namespace,
Constants.CONTEXT_KIND, kind,
Constants.CONTEXT_BACKUP_PATH, outputUrlPrefix.toString()))
.backupRepository("Storage")
.tearDownDate(
LocalDateTime.now().plus(
backupSchedule.getLifetime(),
ChronoUnit.valueOf(schedulersProperties.getTearDownTimeUnit().toString())))
.build();
stampsRepository.submitBackupStamp(build);
// TODO define retry on fail behaviour
} catch (IOException e) {
BackupStamp stamp = buildBackupStamp(backupSchedule, outputUrlPrefix.toString(), true);
stampsRepository.submitBackupStamp(stamp);
} catch (Exception e) {
log.error(e.getMessage());
throw new ScheduleException("Datastore export failure", e);
}
}
@Override
public BackupStamp importBackup(String backupId) {
BackupStamp backupByStampId = stampsRepository.findBackupByStampId(backupId);
if (!backupByStampId.isSuccess()) {
throw new AppException(HttpStatus.BAD_REQUEST.value(), "Failed backup", "Backup not available for import");
}
String importRequestPath = null;
Page<Blob> list = storage
.list(propertiesConfiguration.getBackupBucket(),
......@@ -127,7 +111,7 @@ public class DatastoreBackupManager implements AssetBackupManager {
try {
GoogleDatastoreAdminV1ImportEntitiesRequest importRequest = new GoogleDatastoreAdminV1ImportEntitiesRequest();
importRequest.setInputUrl(importRequestPath);
getDatastore().projects().datastoreImport(propertiesConfiguration.getProjectId(), importRequest).execute();
datastore.projects().datastoreImport(propertiesConfiguration.getProjectId(), importRequest).execute();
} catch (IOException e) {
throw new AppException(HttpStatus.INTERNAL_SERVER_ERROR.value(), e.getMessage(), e.getMessage());
}
......@@ -142,12 +126,14 @@ public class DatastoreBackupManager implements AssetBackupManager {
ArrayList<Blob> blobs = new ArrayList<>();
for (BackupStampEntity stampEntity : collect) {
Page<Blob> list = storage
.list(propertiesConfiguration.getBackupBucket(), BlobListOption.prefix(stampEntity.getBackupPath()
.replace(Constants.GCP_STORAGE_PREFIX + propertiesConfiguration.getBackupBucket() + "/", "")));
if (stampEntity.isSuccess()) {
Page<Blob> list = storage
.list(propertiesConfiguration.getBackupBucket(), BlobListOption.prefix(stampEntity.getBackupPath()
.replace(Constants.GCP_STORAGE_PREFIX + propertiesConfiguration.getBackupBucket() + "/", "")));
blobs.addAll(StreamSupport.stream(list.getValues().spliterator(), false)
.collect(Collectors.toList()));
blobs.addAll(StreamSupport.stream(list.getValues().spliterator(), false)
.collect(Collectors.toList()));
}
}
StorageBatch storageBatch = storage.batch();
......@@ -160,37 +146,28 @@ public class DatastoreBackupManager implements AssetBackupManager {
log.info("Deleted backups:{}", collect);
}
private Datastore getDatastore() {
if (Objects.isNull(datastore)) {
try {
GoogleCredentials credential = GoogleCredentials
.getApplicationDefault();
if (credential.createScopedRequired()) {
List<String> scopes = new ArrayList<>();
scopes.add(IamScopes.CLOUD_PLATFORM);
scopes.add("https://www.googleapis.com/auth/datastore");
credential = credential.createScoped(scopes);
}
credential.refreshAccessToken();
return datastore = new Datastore(GoogleNetHttpTransport.newTrustedTransport(), JSON_FACTORY,
new HttpCredentialsAdapter(credential));
} catch (GeneralSecurityException | IOException e) {
throw new AppException(HttpStatus.INTERNAL_SERVER_ERROR.value(), e.getMessage(), e.getMessage());
}
}
return datastore;
@Recover
public void saveBackupStampOnFail(ScheduleException e, BackupSchedule backupSchedule) {
BackupStamp build = buildBackupStamp(backupSchedule, "-", false);
stampsRepository.submitBackupStamp(build);
}
private String getOutputUrlPrefix(BackupSchedule backupSchedule) {
return Constants.GCP_STORAGE_PREFIX + propertiesConfiguration.getBackupBucket() + "/datastore/"
+ backupSchedule
.getAssetContext().get(Constants.CONTEXT_NAMESPACE)
+ "-" + backupSchedule
.getAssetContext().get(Constants.CONTEXT_KIND) + "/"
+ Timestamp.now();
private BackupStamp buildBackupStamp(BackupSchedule backupSchedule, String path, boolean successful) {
String namespace = backupSchedule.getAssetContext().get(Constants.CONTEXT_NAMESPACE);
String kind = backupSchedule.getAssetContext().get(Constants.CONTEXT_KIND);
return BackupStamp.builder()
.assetType(backupSchedule.getAssetType())
.assetContext(ImmutableMap
.of(Constants.CONTEXT_NAMESPACE, namespace,
Constants.CONTEXT_KIND, kind,
Constants.CONTEXT_BACKUP_PATH, path))
.backupRepository("Storage")
.tearDownDate(
LocalDateTime.now().plus(
backupSchedule.getLifetime(),
ChronoUnit.valueOf(schedulersProperties.getTearDownTimeUnit().toString())))
.success(successful)
.build();
}
private GoogleDatastoreAdminV1ExportEntitiesRequest buildExportEntitiesRequest(
......@@ -207,4 +184,13 @@ public class DatastoreBackupManager implements AssetBackupManager {
return context;
}
private String getOutputUrlPrefix(BackupSchedule backupSchedule) {
return Constants.GCP_STORAGE_PREFIX + propertiesConfiguration.getBackupBucket() + "/datastore/"
+ backupSchedule
.getAssetContext().get(Constants.CONTEXT_NAMESPACE)
+ "-" + backupSchedule
.getAssetContext().get(Constants.CONTEXT_KIND) + "/"
+ Timestamp.now();
}
}
......@@ -40,6 +40,9 @@ public class BackupStampEntity {
@Field(name = "Path")
private String backupPath;
@Field(name = "Success")
private boolean success;
@Field(name = "TearDownDate")
private LocalDateTime tearDownDate;
......@@ -56,6 +59,7 @@ public class BackupStampEntity {
this.kind = backupStamp.getAssetContext().get(Constants.CONTEXT_KIND);
this.backupPath = backupStamp.getAssetContext().get(Constants.CONTEXT_BACKUP_PATH);
this.tearDownDate = backupStamp.getTearDownDate();
this.success = backupStamp.isSuccess();
}
public BackupStamp getBackupStamp() {
......@@ -68,6 +72,7 @@ public class BackupStampEntity {
.of(Constants.CONTEXT_NAMESPACE, this.namespace,
Constants.CONTEXT_KIND, this.kind,
Constants.CONTEXT_BACKUP_PATH, this.backupPath))
.success(this.success)
.build();
}
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment