Commit 12ecd49d authored by Rustam Lotsmanenko (EPAM)'s avatar Rustam Lotsmanenko (EPAM)
Browse files

GONRG-863,GONRG-835

Changes:
Refactoring
Configurable TimeUnit for schedulers
parent c07136fa
......@@ -6,12 +6,12 @@ import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class AssetRepoConfig {
public class AssetConfig {
@Bean
public FactoryBean serviceLocatorFactoryBean() {
ServiceLocatorFactoryBean factoryBean = new ServiceLocatorFactoryBean();
factoryBean.setServiceLocatorInterface(AssetRepoFactory.class);
factoryBean.setServiceLocatorInterface(AssetManagerFactory.class);
return factoryBean;
}
......
......@@ -3,8 +3,8 @@ package org.opengroup.osdu.backup.config;
import org.opengroup.osdu.backup.locator.Asset;
import org.opengroup.osdu.backup.provider.interfaces.AssetBackupManager;
public interface AssetRepoFactory {
public interface AssetManagerFactory {
AssetBackupManager getAssetRepository(Asset asset);
AssetBackupManager getAssetManager(Asset asset);
}
......@@ -16,23 +16,17 @@
package org.opengroup.osdu.backup.config.property;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
@Data
@NoArgsConstructor
@AllArgsConstructor
@Builder
@Setter
@Getter
@ConfigurationProperties(prefix = "osdu.entitlements")
@Component
public class EntitlementProperties {
String url;
private String url;
String appKey;
private String appKey;
}
package org.opengroup.osdu.backup.config.property;
import java.util.concurrent.TimeUnit;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
@Setter
@Getter
@ConfigurationProperties(prefix = "osdu.scheduler")
public class SchedulersProperties {
private TimeUnit tearDownTimeUnit;
private TimeUnit backupTimeUnit;
private int tearDownPeriod;
}
......@@ -17,6 +17,7 @@
package org.opengroup.osdu.backup.di;
import lombok.RequiredArgsConstructor;
import lombok.extern.java.Log;
import org.opengroup.osdu.backup.config.property.EntitlementProperties;
import org.opengroup.osdu.core.common.entitlements.EntitlementsAPIConfig;
import org.opengroup.osdu.core.common.entitlements.EntitlementsFactory;
......@@ -24,6 +25,7 @@ import org.opengroup.osdu.core.common.entitlements.IEntitlementsFactory;
import org.springframework.beans.factory.config.AbstractFactoryBean;
import org.springframework.stereotype.Component;
@Log
@RequiredArgsConstructor
@Component
public class EntitlementsClientFactory extends AbstractFactoryBean<IEntitlementsFactory> {
......
package org.opengroup.osdu.backup.model;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.util.Map;
import lombok.AllArgsConstructor;
import lombok.Builder;
......@@ -20,7 +20,7 @@ public class BackupStamp {
private Asset assetType;
private LocalDate tearDownDate;
private LocalDateTime tearDownDate;
private Map<String, String> assetContext;
}
package org.opengroup.osdu.backup.provider.interfaces;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.util.EnumMap;
import java.util.List;
import org.opengroup.osdu.backup.locator.Asset;
......@@ -8,7 +8,7 @@ import org.opengroup.osdu.backup.model.BackupStamp;
public interface BackupStampRepository {
EnumMap<Asset, List<BackupStamp>> findBackupStampsByCurrentDate(LocalDate toDate);
EnumMap<Asset, List<BackupStamp>> findBackupStampsByTearDownBeforeCurrentDate(LocalDateTime toDate);
BackupStamp submitBackupStamp(BackupStamp backupStamp);
......
package org.opengroup.osdu.backup.provider.interfaces;
import java.util.List;
import java.util.Map;
import org.opengroup.osdu.backup.model.BackupSchedule;
public interface ScheduleRepository {
......@@ -11,5 +12,9 @@ public interface ScheduleRepository {
BackupSchedule updateSchedule(BackupSchedule backupSchedule);
BackupSchedule findByAssetContext(Map<String, String> assetContext);
BackupSchedule save(BackupSchedule backupSchedule);
BackupSchedule findById(String scheduleId);
}
package org.opengroup.osdu.backup.shedulers;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.util.EnumMap;
import java.util.List;
import lombok.RequiredArgsConstructor;
import org.opengroup.osdu.backup.config.AssetRepoFactory;
import org.opengroup.osdu.backup.config.AssetManagerFactory;
import org.opengroup.osdu.backup.locator.Asset;
import org.opengroup.osdu.backup.model.BackupStamp;
import org.opengroup.osdu.backup.provider.interfaces.BackupStampRepository;
......@@ -14,17 +14,18 @@ public class TearDownBackupScheduledTask implements Runnable {
private final BackupStampRepository backupStampRepository;
private final AssetRepoFactory assetRepoFactory;
private final AssetManagerFactory assetManagerFactory;
private final LocalDate tearDownDate = LocalDate.now();
private LocalDateTime tearDownDateTime;
@Override
public void run() {
tearDownDateTime = LocalDateTime.now();
EnumMap<Asset, List<BackupStamp>> backupStampsByCurrentDate = backupStampRepository
.findBackupStampsByCurrentDate(tearDownDate);
.findBackupStampsByTearDownBeforeCurrentDate(tearDownDateTime);
backupStampsByCurrentDate.entrySet().stream().filter(assetListEntry -> !assetListEntry.getValue().isEmpty())
.forEach(assetListEntry -> assetRepoFactory.getAssetRepository(assetListEntry.getKey())
.forEach(assetListEntry -> assetManagerFactory.getAssetManager(assetListEntry.getKey())
.deleteBackups(assetListEntry.getValue()));
}
......
......@@ -2,11 +2,15 @@ package org.opengroup.osdu.backup.shedulers.impl;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import javax.annotation.PostConstruct;
import lombok.RequiredArgsConstructor;
import org.opengroup.osdu.backup.config.AssetRepoFactory;
import lombok.extern.java.Log;
import org.opengroup.osdu.backup.config.AssetManagerFactory;
import org.opengroup.osdu.backup.config.property.SchedulersProperties;
import org.opengroup.osdu.backup.model.BackupSchedule;
import org.opengroup.osdu.backup.shedulers.BackupScheduledTask;
import org.opengroup.osdu.backup.shedulers.SchedulerService;
......@@ -14,15 +18,18 @@ import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler;
import org.springframework.scheduling.support.PeriodicTrigger;
import org.springframework.stereotype.Service;
@Log
@Service
@RequiredArgsConstructor
public class SchedulerServiceImpl implements SchedulerService {
private final Map<BackupSchedule, ScheduledFuture> taskMap = new HashMap<>();
private final SchedulersProperties schedulersProperties;
private final TimeUnit timeUnit = TimeUnit.MINUTES;
private final AssetRepoFactory repoFactory;
private final AssetManagerFactory repoFactory;
private ThreadPoolTaskScheduler poolTaskScheduler;
......@@ -36,27 +43,36 @@ public class SchedulerServiceImpl implements SchedulerService {
}
@Override
public void addTaskToScheduler(BackupSchedule submitBackupScheduleRequest) {
PeriodicTrigger periodicTrigger = new PeriodicTrigger(submitBackupScheduleRequest.getBackupPeriod(), timeUnit);
public void addTaskToScheduler(BackupSchedule backupSchedule) {
PeriodicTrigger periodicTrigger = new PeriodicTrigger(backupSchedule.getBackupPeriod(),
schedulersProperties.getBackupTimeUnit());
periodicTrigger.setFixedRate(true);
BackupScheduledTask backupScheduledTask = new BackupScheduledTask(
repoFactory.getAssetRepository(submitBackupScheduleRequest.getAssetType()), submitBackupScheduleRequest);
repoFactory.getAssetManager(backupSchedule.getAssetType()), backupSchedule);
ScheduledFuture<?> schedule = poolTaskScheduler.schedule(backupScheduledTask, periodicTrigger);
taskMap.put(submitBackupScheduleRequest, schedule);
taskMap.put(backupSchedule, schedule);
log.log(Level.INFO, "Backup task:{0} added to scheduler", backupSchedule);
}
@Override
public void updateTask(BackupSchedule backupSchedule) {
cancelScheduledTask(backupSchedule);
addTaskToScheduler(backupSchedule);
if (backupSchedule.isActive()) {
addTaskToScheduler(backupSchedule);
}
log.log(Level.INFO, "Backup task:{0} updated", backupSchedule);
}
@Override
public boolean cancelScheduledTask(BackupSchedule backupSchedule) {
boolean cancel = taskMap.get(backupSchedule).cancel(false);
taskMap.remove(backupSchedule);
ScheduledFuture scheduledFuture = taskMap.get(backupSchedule);
boolean cancel = false;
if (Objects.nonNull(scheduledFuture)) {
cancel = scheduledFuture.cancel(false);
taskMap.remove(backupSchedule);
}
return cancel;
}
......
package org.opengroup.osdu.backup.shedulers.impl;
import java.util.concurrent.TimeUnit;
import javax.annotation.PostConstruct;
import lombok.RequiredArgsConstructor;
import org.opengroup.osdu.backup.config.AssetRepoFactory;
import org.opengroup.osdu.backup.config.AssetManagerFactory;
import org.opengroup.osdu.backup.config.property.SchedulersProperties;
import org.opengroup.osdu.backup.provider.interfaces.BackupStampRepository;
import org.opengroup.osdu.backup.shedulers.TearDownBackupScheduledTask;
import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler;
......@@ -14,11 +14,9 @@ import org.springframework.stereotype.Service;
@RequiredArgsConstructor
public class TearDownScheduler {
private final TimeUnit timeUnit = TimeUnit.HOURS;
private final SchedulersProperties schedulersProperties;
private final int tearDownPeriod = 24;
private final AssetRepoFactory repoFactory;
private final AssetManagerFactory repoFactory;
private final BackupStampRepository stampsRepository;
......@@ -32,7 +30,8 @@ public class TearDownScheduler {
poolTaskScheduler.setRemoveOnCancelPolicy(true);
poolTaskScheduler.initialize();
PeriodicTrigger periodicTrigger = new PeriodicTrigger(tearDownPeriod, timeUnit);
PeriodicTrigger periodicTrigger = new PeriodicTrigger(schedulersProperties.getTearDownPeriod(),
schedulersProperties.getTearDownTimeUnit());
periodicTrigger.setFixedRate(true);
TearDownBackupScheduledTask tearDownBackupScheduledTask = new TearDownBackupScheduledTask(stampsRepository,
......
# Backup service
Backup service provide a set of APIs to backup Schedule, list registered Schedules, list available Backups, restore Asset state from Backup.
Backup service provide a set of APIs to back up Schedule, list registered Schedules, list available Backups, restore Asset state from Backup.
In implementation were used datastore admin API : https://cloud.google.com/datastore/docs/reference/admin/rest
......
package org.opengroup.osdu.backup.provider.gcp;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.context.properties.ConfigurationPropertiesScan;
@ConfigurationPropertiesScan(basePackages = "org.opengroup")
@SpringBootApplication(scanBasePackages = "org.opengroup")
public class BackupApplicationGCP {
public static void main(String[] args) {
SpringApplication.run(BackupApplicationGCP.class, args);
}
}
......@@ -3,11 +3,9 @@ package org.opengroup.osdu.backup.provider.gcp.config.property;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
@Setter
@Getter
@Configuration
@ConfigurationProperties(prefix = "gcp")
public class PropertiesConfiguration {
......
......@@ -22,7 +22,7 @@ import com.google.cloud.storage.StorageBatch;
import com.google.common.collect.ImmutableMap;
import java.io.IOException;
import java.security.GeneralSecurityException;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
......@@ -94,7 +94,7 @@ public class DatastoreBackupManager implements AssetBackupManager {
Constants.CONTEXT_KIND, kind,
Constants.CONTEXT_BACKUP_PATH, outputUrlPrefix.toString()))
.backupRepository("Storage")
.tearDownDate(LocalDate.now().plusDays(backupSchedule.getLifetime())).build();
.tearDownDate(LocalDateTime.now().plusDays(backupSchedule.getLifetime())).build();
stampsRepository.submitBackupStamp(build);
......@@ -151,6 +151,7 @@ public class DatastoreBackupManager implements AssetBackupManager {
storageBatch.submit();
stampsRepository.deleteBackupStamps(backupStamps);
log.log(Level.INFO, "Deleted backups:{0}", collect);
}
private Datastore getDatastore() {
......
package org.opengroup.osdu.backup.provider.gcp.model.entity;
import com.google.common.collect.ImmutableMap;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.util.Objects;
import java.util.UUID;
import lombok.AllArgsConstructor;
......@@ -41,7 +41,7 @@ public class BackupStampEntity {
private String backupPath;
@Field(name = "TearDownDate")
private LocalDate tearDownDate;
private LocalDateTime tearDownDate;
public BackupStampEntity(BackupStamp backupStamp) {
// TODO define id generation
......
package org.opengroup.osdu.backup.provider.gcp.repository;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.util.List;
import org.opengroup.osdu.backup.provider.gcp.model.entity.BackupStampEntity;
import org.springframework.cloud.gcp.data.datastore.repository.DatastoreRepository;
import org.springframework.cloud.gcp.data.datastore.repository.query.Query;
import org.springframework.data.repository.query.Param;
import org.springframework.stereotype.Repository;
@Repository
public interface BackupStampsEntityRepository extends DatastoreRepository<BackupStampEntity, String> {
List<BackupStampEntity> findByTearDownDate(LocalDate toDate);
@Query("SELECT * FROM BackupStamp WHERE TearDownDate <= @tear_down_date")
List<BackupStampEntity> findByTearDownDateBeforeCurrent(@Param("tear_down_date") LocalDateTime current);
}
package org.opengroup.osdu.backup.provider.gcp.repository;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.EnumMap;
import java.util.List;
......@@ -22,8 +22,10 @@ public class DatastoreBackupStampRepository implements BackupStampRepository {
private final BackupStampsEntityRepository backupStampsEntityRepository;
@Override
public EnumMap<Asset, List<BackupStamp>> findBackupStampsByCurrentDate(LocalDate toDate) {
List<BackupStampEntity> byTearDownDate = backupStampsEntityRepository.findByTearDownDate(toDate);
public EnumMap<Asset, List<BackupStamp>> findBackupStampsByTearDownBeforeCurrentDate(
LocalDateTime current) {
List<BackupStampEntity> byTearDownDate = backupStampsEntityRepository
.findByTearDownDateBeforeCurrent(current);
return getAssetListEnumMap(byTearDownDate);
}
......
package org.opengroup.osdu.backup.provider.gcp.repository;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
import lombok.RequiredArgsConstructor;
import org.opengroup.osdu.backup.model.BackupSchedule;
import org.opengroup.osdu.backup.provider.gcp.model.constant.Constants;
import org.opengroup.osdu.backup.provider.gcp.model.entity.DatastoreBackupScheduleEntity;
import org.opengroup.osdu.backup.provider.interfaces.ScheduleRepository;
import org.opengroup.osdu.core.common.model.http.AppException;
......@@ -45,5 +47,26 @@ public class DatastoreScheduleRepository implements ScheduleRepository {
return repository.save(scheduleEntity).toModel();
}
@Override
public BackupSchedule findByAssetContext(Map<String, String> assetContext) {
String kind = assetContext.get(Constants.CONTEXT_KIND);
String namespace = assetContext.get(Constants.CONTEXT_NAMESPACE);
DatastoreBackupScheduleEntity taskExist = repository.findOneByKindAndNamespace(kind, namespace);
return taskExist.toModel();
}
@Override
public BackupSchedule save(BackupSchedule backupSchedule) {
return repository.save(new DatastoreBackupScheduleEntity(backupSchedule)).toModel();
}
@Override
public BackupSchedule findById(String scheduleId) {
DatastoreBackupScheduleEntity scheduleById = repository.findById(scheduleId)
.orElseThrow(() -> new AppException(HttpStatus.NOT_FOUND.value(), "Not found",
String.format("Schedule with scheduleId=%s not found", scheduleId)));
return scheduleById.toModel();
}
}
......@@ -3,20 +3,18 @@ package org.opengroup.osdu.backup.provider.gcp.service;
import java.util.EnumMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import javax.annotation.PostConstruct;
import lombok.RequiredArgsConstructor;
import org.opengroup.osdu.backup.config.AssetRepoFactory;
import org.opengroup.osdu.backup.config.AssetManagerFactory;
import org.opengroup.osdu.backup.locator.Asset;
import org.opengroup.osdu.backup.model.BackupImportRequest;
import org.opengroup.osdu.backup.model.BackupSchedule;
import org.opengroup.osdu.backup.model.BackupStamp;
import org.opengroup.osdu.backup.provider.gcp.model.constant.Constants;
import org.opengroup.osdu.backup.provider.gcp.model.entity.DatastoreBackupScheduleEntity;
import org.opengroup.osdu.backup.provider.gcp.repository.DatastoreBackupStampRepository;
import org.opengroup.osdu.backup.provider.gcp.repository.DatastoreScheduleRepository;
import org.opengroup.osdu.backup.provider.gcp.repository.SchedulesEntityRepository;
import org.opengroup.osdu.backup.provider.interfaces.BackupService;
import org.opengroup.osdu.backup.provider.interfaces.BackupStampRepository;
import org.opengroup.osdu.backup.provider.interfaces.ScheduleRepository;
import org.opengroup.osdu.backup.shedulers.SchedulerService;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.springframework.http.HttpStatus;
......@@ -26,34 +24,30 @@ import org.springframework.stereotype.Service;
@RequiredArgsConstructor
public class BackupServiceImpl implements BackupService {
private final SchedulesEntityRepository schedulesEntityRepository;
private final SchedulerService schedulerService;
private final DatastoreBackupStampRepository backupStampsRepositoryDatastore;
private final BackupStampRepository backupStampRepository;
private final DatastoreScheduleRepository schedulesRepositoryDatastore;
private final ScheduleRepository scheduleRepository;
private final AssetRepoFactory factory;
private final AssetManagerFactory factory;
@PostConstruct
public void startUpSchedulers() {
List<BackupSchedule> backupScheduleList = schedulesRepositoryDatastore.listBackupSchedules();
List<BackupSchedule> backupScheduleList = scheduleRepository.listBackupSchedules();
backupScheduleList.stream().filter(BackupSchedule::isActive)
.forEach(schedulerService::addTaskToScheduler);
}
@Override
public BackupSchedule submitBackupSchedule(BackupSchedule backupSchedule) {
String kind = backupSchedule.getAssetContext().get(Constants.CONTEXT_KIND);
String namespace = backupSchedule.getAssetContext().get(Constants.CONTEXT_NAMESPACE);
DatastoreBackupScheduleEntity taskExist = schedulesEntityRepository.findOneByKindAndNamespace(kind, namespace);
if (Objects.nonNull(taskExist)) {
Map<String, String> assetContext = backupSchedule.getAssetContext();
BackupSchedule byAssetContext = scheduleRepository.findByAssetContext(assetContext);
if (Objects.nonNull(byAssetContext)) {
throw new AppException(HttpStatus.CONFLICT.value(), "Conflict"
, String.format("Schedule for %s already exist", namespace + ":" + kind));
, String.format("Schedule with context %s already exist", assetContext));
} else {
BackupSchedule savedSchedule = schedulesEntityRepository
.save(new DatastoreBackupScheduleEntity(backupSchedule)).toModel();
BackupSchedule savedSchedule = scheduleRepository.save(backupSchedule);
schedulerService.addTaskToScheduler(backupSchedule);
return savedSchedule;
}
......@@ -65,33 +59,30 @@ public class BackupServiceImpl implements BackupService {
if (Objects.isNull(backupSchedule.getScheduleId())) {
throw new AppException(HttpStatus.BAD_REQUEST.value(), "Bad request", "Schedule id required");
}
BackupSchedule updatedSchedule = schedulesRepositoryDatastore.updateSchedule(backupSchedule);
BackupSchedule updatedSchedule = scheduleRepository.updateSchedule(backupSchedule);
schedulerService.updateTask(updatedSchedule);
return updatedSchedule;
}
@Override
public BackupSchedule getBackupSchedule(String scheduleId) {
DatastoreBackupScheduleEntity scheduleEntity = schedulesEntityRepository.findById(scheduleId)
.orElseThrow(() -> new AppException(HttpStatus.NOT_FOUND.value(), "Not found",
String.format("Schedule with scheduleId=%s not found", scheduleId)));
return scheduleEntity.toModel();
return scheduleRepository.findById(scheduleId);
}
@Override
public BackupStamp submitBackupImportRequest(BackupImportRequest importRequest) {
return factory.getAssetRepository(Asset.DATASTORE)
return factory.getAssetManager(Asset.DATASTORE)
.importBackup(importRequest.getBackupStampId());
}
@Override