Commit e27db7cc authored by Rustam Lotsmanenko (EPAM)'s avatar Rustam Lotsmanenko (EPAM)
Browse files

GONRG-1374 Backup service support B&R of collection of Data Assets

Changes:
New endpoints for collections;
Move backup configuration to collection(backupPeriod,active,lifetime);
Changed startup to use collection instead single backup schedule;
Added new services and repos for collection;
Refactoring of entities mapping(used mapstruct lib);
parent 4e9adad2
......@@ -46,7 +46,6 @@ public class BackupApi {
private final BackupService backupService;
//TODO Entitlements & roles
@PostMapping("/schedule")
public ResponseEntity<BackupSchedule> submitBackupSchedule(
@RequestBody BackupSchedule backupSchedule) {
......
/*
* Copyright 2020 Google LLC
* Copyright 2020 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.backup.api;
import java.util.List;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.opengroup.osdu.backup.model.BackupCollection;
import org.opengroup.osdu.backup.model.BackupRole;
import org.opengroup.osdu.backup.model.CollectionState;
import org.opengroup.osdu.backup.service.BackupCollectionService;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.PutMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.context.annotation.RequestScope;
@Slf4j
@RestController
@RequestScope
@RequiredArgsConstructor
@PreAuthorize("@authorizationFilter.hasPermission('" + BackupRole.BACKUP + "')")
public class BackupCollectionApi {
private final BackupCollectionService collectionService;
@PostMapping("/collection")
public ResponseEntity<BackupCollection> createCollection(@RequestBody BackupCollection collection) {
BackupCollection backupCollection = collectionService.saveCollection(collection);
return new ResponseEntity<>(backupCollection, HttpStatus.CREATED);
}
@PutMapping("/collection/{name}")
public ResponseEntity<BackupCollection> updateCollection(@PathVariable String name,
@RequestBody BackupCollection collection) {
BackupCollection backupCollection = collectionService.update(collection);
return new ResponseEntity<>(backupCollection, HttpStatus.CREATED);
}
@GetMapping("/collection/{name}")
public ResponseEntity<List<CollectionState>> getCollectionStates(@PathVariable String name) {
List<CollectionState> collectionStates = collectionService.getCollectionStates(name);
return new ResponseEntity<>(collectionStates, HttpStatus.OK);
}
@GetMapping("/collection/restore/{stateId}")
public ResponseEntity<CollectionState> restoreCollection(@PathVariable String stateId) {
CollectionState collectionState = collectionService.restoreCollection(stateId);
return new ResponseEntity<>(collectionState, HttpStatus.OK);
}
}
......@@ -3,8 +3,6 @@ package org.opengroup.osdu.backup.locator;
public enum Asset {
DATASTORE,
ELASTIC,
STORAGE,
SQL
SQL;
}
......@@ -6,6 +6,8 @@ import org.opengroup.osdu.backup.model.BackupStamp;
public interface AssetBackupManager {
BackupStamp exportBackupWithResult(BackupSchedule backupSchedule);
void exportBackup(BackupSchedule backupSchedule);
BackupStamp importBackup(String backupId);
......
/*
* Copyright 2020 Google LLC
* Copyright 2020 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.backup.model;
import java.util.Set;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
@Data
@NoArgsConstructor
@AllArgsConstructor
@Builder
@ToString
public class BackupCollection {
private String collectionId;
private String collectionName;
private boolean active;
private int backupPeriod;
private int lifetime;
private Set<BackupSchedule> scheduleSet;
}
......@@ -4,12 +4,14 @@ import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
import org.opengroup.osdu.backup.locator.Asset;
@Data
@NoArgsConstructor
@AllArgsConstructor
@Builder
@ToString
public class BackupImportRequest {
private String backupStampId;
......
......@@ -6,6 +6,7 @@ import lombok.Builder;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import lombok.ToString;
import org.opengroup.osdu.backup.locator.Asset;
@Data
......@@ -13,18 +14,13 @@ import org.opengroup.osdu.backup.locator.Asset;
@AllArgsConstructor
@Builder
@EqualsAndHashCode(of = {"scheduleId", "assetType", "assetContext"})
@ToString
public class BackupSchedule {
private String scheduleId;
private Asset assetType;
private int backupPeriod;
private boolean active;
private int lifetime;
private Map<String, String> assetContext;
}
package org.opengroup.osdu.backup.model;
import java.time.LocalDateTime;
import java.util.Map;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
import org.opengroup.osdu.backup.locator.Asset;
@Data
@NoArgsConstructor
@AllArgsConstructor
@Builder
@ToString
public class BackupStamp {
private String stampId;
private String backupRepository;
private Asset assetType;
private LocalDateTime tearDownDate;
private boolean success;
private Map<String, String> assetContext;
......
/*
* Copyright 2020 Google LLC
* Copyright 2020 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.backup.model;
import java.time.LocalDateTime;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
@Data
@NoArgsConstructor
@AllArgsConstructor
@Builder
@ToString
public class CollectionState {
private String stateId;
private String collectionId;
private List<BackupStamp> backupStamps;
private LocalDateTime creationDate;
private StateStatus stateStatus;
}
/*
* Copyright 2020 Google LLC
* Copyright 2020 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.backup.model;
public enum StateStatus {
SUCCESS, PARTIAL_SUCCESS, FAILED
}
/*
* Copyright 2020 Google LLC
* Copyright 2020 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.backup.repository;
import java.util.List;
import org.opengroup.osdu.backup.model.BackupCollection;
public interface BackupCollectionRepository {
BackupCollection submitCollection(BackupCollection collection);
BackupCollection updateCollection(BackupCollection collection);
List<BackupCollection> listBackupCollections();
}
package org.opengroup.osdu.backup.repository;
import java.time.LocalDateTime;
import java.util.EnumMap;
import java.util.List;
import org.opengroup.osdu.backup.locator.Asset;
import org.opengroup.osdu.backup.model.BackupStamp;
import org.opengroup.osdu.backup.model.CollectionState;
public interface BackupStampRepository {
EnumMap<Asset, List<BackupStamp>> findBackupStampsByTearDownBeforeCurrentDate(LocalDateTime toDate);
EnumMap<Asset, List<BackupStamp>> findBackupStampsByCollectionState(CollectionState collectionState);
BackupStamp submitBackupStamp(BackupStamp backupStamp);
......
/*
* Copyright 2020 Google LLC
* Copyright 2020 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.backup.repository;
import java.util.List;
import org.opengroup.osdu.backup.model.CollectionState;
public interface CollectionStateRepository {
List<CollectionState> getAllCollectionsStates();
List<CollectionState> getCollectionStates(String name);
List<CollectionState> getCollectionsStatesByCollectionId(String collectionId);
CollectionState getStateById(String stateId);
CollectionState submitCollectionState(CollectionState collectionState);
void deleteCollectionStates(List<CollectionState> collectionsStatesForDelete);
}
/*
* Copyright 2020 Google LLC
* Copyright 2020 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.backup.service;
import java.util.List;
import org.opengroup.osdu.backup.model.BackupCollection;
import org.opengroup.osdu.backup.model.CollectionState;
public interface BackupCollectionService {
BackupCollection saveCollection(BackupCollection collection);
List<CollectionState> getCollectionStates(String name);
CollectionState restoreCollection(String stateId);
BackupCollection update(BackupCollection collection);
}
/*
* Copyright 2020 Google LLC
* Copyright 2020 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.backup.service.impl;
import java.util.List;
import javax.annotation.PostConstruct;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.opengroup.osdu.backup.locator.AssetManagerFactory;
import org.opengroup.osdu.backup.model.BackupCollection;
import org.opengroup.osdu.backup.model.CollectionState;
import org.opengroup.osdu.backup.repository.BackupCollectionRepository;
import org.opengroup.osdu.backup.repository.CollectionStateRepository;
import org.opengroup.osdu.backup.service.BackupCollectionService;
import org.opengroup.osdu.backup.service.BackupService;
import org.opengroup.osdu.backup.shedulers.SchedulerService;
import org.springframework.stereotype.Service;
@Slf4j
@Service
@RequiredArgsConstructor
public class BackupCollectionServiceImpl implements BackupCollectionService {
private final BackupCollectionRepository collectionRepository;
private final CollectionStateRepository collectionStateRepository;
private final BackupService backupService;
private final AssetManagerFactory assetManagerFactory;
private final SchedulerService schedulerService;
@PostConstruct
public void startUpSchedulers() {
List<BackupCollection> backupScheduleList = collectionRepository.listBackupCollections();
backupScheduleList.stream().filter(BackupCollection::isActive)
.forEach(schedulerService::addCollectionToScheduler);
}
@Override
public BackupCollection saveCollection(BackupCollection collection) {
BackupCollection savedCollection = collectionRepository.submitCollection(collection);
schedulerService.addCollectionToScheduler(savedCollection);
return savedCollection;
}
@Override
public List<CollectionState> getCollectionStates(String name) {
return collectionStateRepository.getCollectionStates(name);
}
@Override
public CollectionState restoreCollection(String stateId) {
CollectionState state = collectionStateRepository.getStateById(stateId);
state.getBackupStamps().forEach(backupStamp ->
assetManagerFactory.getAssetManager(backupStamp.getAssetType())
.importBackup(backupStamp.getStampId()));
return state;
}
@Override
public BackupCollection update(BackupCollection collection) {
BackupCollection backupCollection = collectionRepository.updateCollection(collection);
schedulerService.updateCollectionTask(backupCollection);
return backupCollection;
}
}
......@@ -5,7 +5,6 @@ import java.util.EnumMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import javax.annotation.PostConstruct;
import lombok.RequiredArgsConstructor;
import org.opengroup.osdu.backup.locator.Asset;
import org.opengroup.osdu.backup.locator.AssetManagerFactory;
......@@ -31,18 +30,19 @@ public class BackupServiceImpl implements BackupService {
private final ScheduleRepository scheduleRepository;
private final AssetManagerFactory factory;
@PostConstruct
//TODO currently schedules startup changed to BackupCollectionServiceImpl.startUpSchedulers()
// @PostConstruct
public void startUpSchedulers() {
List<BackupSchedule> backupScheduleList = scheduleRepository.listBackupSchedules();
backupScheduleList.stream().filter(BackupSchedule::isActive)
backupScheduleList.stream()/*.filter(BackupSchedule::isActive)*/
.forEach(schedulerService::addTaskToScheduler);
}
@Override
public BackupSchedule submitBackupSchedule(BackupSchedule backupSchedule) {
Map<String, String> assetContext = backupSchedule.getAssetContext();
BackupSchedule byAssetContext = scheduleRepository.findByAssetContext(backupSchedule.getAssetType(), assetContext);
BackupSchedule byAssetContext = scheduleRepository
.findByAssetContext(backupSchedule.getAssetType(), assetContext);
if (Objects.nonNull(byAssetContext)) {
throw new AppException(HttpStatus.CONFLICT.value(), "Conflict"
, String.format("Schedule with context %s already exist", assetContext));
......
/*
* Copyright 2020 Google LLC
* Copyright 2020 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.backup.shedulers;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
import lombok.RequiredArgsConstructor;
import org.opengroup.osdu.backup.locator.AssetManagerFactory;
import org.opengroup.osdu.backup.model.BackupCollection;
import org.opengroup.osdu.backup.model.BackupSchedule;
import org.opengroup.osdu.backup.model.BackupStamp;
import org.opengroup.osdu.backup.model.CollectionState;
import org.opengroup.osdu.backup.model.StateStatus;
import org.opengroup.osdu.backup.repository.CollectionStateRepository;
@RequiredArgsConstructor
public class BackupCollectionScheduledTask implements Runnable {
private final AssetManagerFactory assetManagerFactory;
private final BackupCollection collection;
private final CollectionStateRepository stateRepository;
//TODO redesign to run export tasks for collection in multithreading
@Override