Merge branch 'feature/GONRG-1374-B&R_collection' into 'integration-master'

GONRG-1374 Backup service support B&R of collection of Data Assets

Closes GONRG-1374

See merge request go3-nrg/platform/deployment-and-operations/backup-service!11

backup-core/src/main/java/org/opengroup/osdu/backup/api/BackupApi.java

@@ -46,7 +46,6 @@ public class BackupApi {
 
 	private final BackupService backupService;
 
-	//TODO Entitlements & roles
 	@PostMapping("/schedule")
 	public ResponseEntity<BackupSchedule> submitBackupSchedule(
 		@RequestBody BackupSchedule backupSchedule) {

backup-core/src/main/java/org/opengroup/osdu/backup/api/BackupCollectionApi.java

+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.backup.api;
+
+import java.util.List;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.opengroup.osdu.backup.model.BackupCollection;
+import org.opengroup.osdu.backup.model.BackupRole;
+import org.opengroup.osdu.backup.model.CollectionState;
+import org.opengroup.osdu.backup.service.BackupCollectionService;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.security.access.prepost.PreAuthorize;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.PutMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.context.annotation.RequestScope;
+
+@Slf4j
+@RestController
+@RequestScope
+@RequiredArgsConstructor
+@PreAuthorize("@authorizationFilter.hasPermission('" + BackupRole.BACKUP + "')")
+public class BackupCollectionApi {
+
+	private final BackupCollectionService collectionService;
+
+	@PostMapping("/collection")
+	public ResponseEntity<BackupCollection> createCollection(@RequestBody BackupCollection collection) {
+		BackupCollection backupCollection = collectionService.saveCollection(collection);
+		return new ResponseEntity<>(backupCollection, HttpStatus.CREATED);
+	}
+
+	@PutMapping("/collection/{name}")
+	public ResponseEntity<BackupCollection> updateCollection(@PathVariable String name,
+		@RequestBody BackupCollection collection) {
+		BackupCollection backupCollection = collectionService.update(collection);
+		return new ResponseEntity<>(backupCollection, HttpStatus.CREATED);
+	}
+
+	@GetMapping("/collection/{name}")
+	public ResponseEntity<List<CollectionState>> getCollectionStates(@PathVariable String name) {
+		List<CollectionState> collectionStates = collectionService.getCollectionStates(name);
+		return new ResponseEntity<>(collectionStates, HttpStatus.OK);
+	}
+
+	@GetMapping("/collection/restore/{stateId}")
+	public ResponseEntity<CollectionState> restoreCollection(@PathVariable String stateId) {
+		CollectionState collectionState = collectionService.restoreCollection(stateId);
+		return new ResponseEntity<>(collectionState, HttpStatus.OK);
+	}
+
+
+}

backup-core/src/main/java/org/opengroup/osdu/backup/locator/Asset.java

@@ -3,8 +3,6 @@ package org.opengroup.osdu.backup.locator;
 public enum Asset {
 
 	DATASTORE,
-	ELASTIC,
-	STORAGE,
-	SQL
+	SQL;
 
 }

backup-core/src/main/java/org/opengroup/osdu/backup/manager/AssetBackupManager.java

@@ -6,6 +6,8 @@ import org.opengroup.osdu.backup.model.BackupStamp;
 
 public interface AssetBackupManager {
 
+	BackupStamp exportBackupWithResult(BackupSchedule backupSchedule);
+
 	void exportBackup(BackupSchedule backupSchedule);
 
 	BackupStamp importBackup(String backupId);

backup-core/src/main/java/org/opengroup/osdu/backup/model/BackupCollection.java

+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.backup.model;
+
+import java.util.Set;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.ToString;
+
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+@Builder
+@ToString
+public class BackupCollection {
+
+	private String collectionId;
+
+	private String collectionName;
+
+	private boolean active;
+
+	private int backupPeriod;
+
+	private int lifetime;
+
+	private Set<BackupSchedule> scheduleSet;
+
+}

backup-core/src/main/java/org/opengroup/osdu/backup/model/BackupImportRequest.java

@@ -4,12 +4,14 @@ import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
+import lombok.ToString;
 import org.opengroup.osdu.backup.locator.Asset;
 
 @Data
 @NoArgsConstructor
 @AllArgsConstructor
 @Builder
+@ToString
 public class BackupImportRequest {
 
 	private String backupStampId;

backup-core/src/main/java/org/opengroup/osdu/backup/model/BackupSchedule.java

@@ -6,6 +6,7 @@ import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;
+import lombok.ToString;
 import org.opengroup.osdu.backup.locator.Asset;
 
 @Data
@@ -13,18 +14,13 @@ import org.opengroup.osdu.backup.locator.Asset;
 @AllArgsConstructor
 @Builder
 @EqualsAndHashCode(of = {"scheduleId", "assetType", "assetContext"})
+@ToString
 public class BackupSchedule {
 
 	private String scheduleId;
 
 	private Asset assetType;
 
-	private int backupPeriod;
-
-	private boolean active;
-
-	private int lifetime;
-
 	private Map<String, String> assetContext;
 
 }

backup-core/src/main/java/org/opengroup/osdu/backup/model/BackupStamp.java

 package org.opengroup.osdu.backup.model;
 
-import java.time.LocalDateTime;
 import java.util.Map;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
+import lombok.ToString;
 import org.opengroup.osdu.backup.locator.Asset;
 
 @Data
 @NoArgsConstructor
 @AllArgsConstructor
 @Builder
+@ToString
 public class BackupStamp {
 
 	private String stampId;
 
-	private String backupRepository;
-
 	private Asset assetType;
 
-	private LocalDateTime tearDownDate;
-
 	private boolean success;
 
 	private Map<String, String> assetContext;

backup-core/src/main/java/org/opengroup/osdu/backup/model/CollectionState.java

+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.backup.model;
+
+import java.time.LocalDateTime;
+import java.util.List;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.ToString;
+
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+@Builder
+@ToString
+public class CollectionState {
+
+	private String stateId;
+
+	private String collectionId;
+
+	private List<BackupStamp> backupStamps;
+
+	private LocalDateTime creationDate;
+
+	private StateStatus stateStatus;
+
+}

backup-core/src/main/java/org/opengroup/osdu/backup/model/StateStatus.java

+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.backup.model;
+
+public enum StateStatus {
+	SUCCESS, PARTIAL_SUCCESS, FAILED
+}

backup-core/src/main/java/org/opengroup/osdu/backup/repository/BackupCollectionRepository.java

+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.backup.repository;
+
+import java.util.List;
+import org.opengroup.osdu.backup.model.BackupCollection;
+
+public interface BackupCollectionRepository {
+
+	BackupCollection submitCollection(BackupCollection collection);
+
+	BackupCollection updateCollection(BackupCollection collection);
+
+	List<BackupCollection> listBackupCollections();
+}

backup-core/src/main/java/org/opengroup/osdu/backup/repository/BackupStampRepository.java

 package org.opengroup.osdu.backup.repository;
 
-import java.time.LocalDateTime;
 import java.util.EnumMap;
 import java.util.List;
 import org.opengroup.osdu.backup.locator.Asset;
 import org.opengroup.osdu.backup.model.BackupStamp;
+import org.opengroup.osdu.backup.model.CollectionState;
 
 public interface BackupStampRepository {
 
-	EnumMap<Asset, List<BackupStamp>> findBackupStampsByTearDownBeforeCurrentDate(LocalDateTime toDate);
+	EnumMap<Asset, List<BackupStamp>> findBackupStampsByCollectionState(CollectionState collectionState);
 
 	BackupStamp submitBackupStamp(BackupStamp backupStamp);
 

backup-core/src/main/java/org/opengroup/osdu/backup/repository/CollectionStateRepository.java

+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.backup.repository;
+
+import java.util.List;
+import org.opengroup.osdu.backup.model.CollectionState;
+
+public interface CollectionStateRepository {
+
+	List<CollectionState> getAllCollectionsStates();
+
+	List<CollectionState> getCollectionStates(String name);
+
+	List<CollectionState> getCollectionsStatesByCollectionId(String collectionId);
+
+	CollectionState getStateById(String stateId);
+
+	CollectionState submitCollectionState(CollectionState collectionState);
+
+	void deleteCollectionStates(List<CollectionState> collectionsStatesForDelete);
+}

backup-core/src/main/java/org/opengroup/osdu/backup/service/BackupCollectionService.java

+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.backup.service;
+
+import java.util.List;
+import org.opengroup.osdu.backup.model.BackupCollection;
+import org.opengroup.osdu.backup.model.CollectionState;
+
+public interface BackupCollectionService {
+
+	BackupCollection saveCollection(BackupCollection collection);
+
+	List<CollectionState> getCollectionStates(String name);
+
+	CollectionState restoreCollection(String stateId);
+
+	BackupCollection update(BackupCollection collection);
+}

backup-core/src/main/java/org/opengroup/osdu/backup/service/impl/BackupCollectionServiceImpl.java

+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.backup.service.impl;
+
+import java.util.List;
+import javax.annotation.PostConstruct;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.opengroup.osdu.backup.locator.AssetManagerFactory;
+import org.opengroup.osdu.backup.model.BackupCollection;
+import org.opengroup.osdu.backup.model.CollectionState;
+import org.opengroup.osdu.backup.repository.BackupCollectionRepository;
+import org.opengroup.osdu.backup.repository.CollectionStateRepository;
+import org.opengroup.osdu.backup.service.BackupCollectionService;
+import org.opengroup.osdu.backup.service.BackupService;
+import org.opengroup.osdu.backup.shedulers.SchedulerService;
+import org.springframework.stereotype.Service;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class BackupCollectionServiceImpl implements BackupCollectionService {
+
+	private final BackupCollectionRepository collectionRepository;
+	private final CollectionStateRepository collectionStateRepository;
+	private final BackupService backupService;
+	private final AssetManagerFactory assetManagerFactory;
+	private final SchedulerService schedulerService;
+
+	@PostConstruct
+	public void startUpSchedulers() {
+		List<BackupCollection> backupScheduleList = collectionRepository.listBackupCollections();
+		backupScheduleList.stream().filter(BackupCollection::isActive)
+			.forEach(schedulerService::addCollectionToScheduler);
+	}
+
+	@Override
+	public BackupCollection saveCollection(BackupCollection collection) {
+		BackupCollection savedCollection = collectionRepository.submitCollection(collection);
+		schedulerService.addCollectionToScheduler(savedCollection);
+		return savedCollection;
+	}
+
+	@Override
+	public List<CollectionState> getCollectionStates(String name) {
+		return collectionStateRepository.getCollectionStates(name);
+	}
+
+	@Override
+	public CollectionState restoreCollection(String stateId) {
+		CollectionState state = collectionStateRepository.getStateById(stateId);
+		state.getBackupStamps().forEach(backupStamp ->
+			assetManagerFactory.getAssetManager(backupStamp.getAssetType())
+			.importBackup(backupStamp.getStampId()));
+		return state;
+	}
+
+	@Override
+	public BackupCollection update(BackupCollection collection) {
+		BackupCollection backupCollection = collectionRepository.updateCollection(collection);
+		schedulerService.updateCollectionTask(backupCollection);
+		return backupCollection;
+	}
+}

backup-core/src/main/java/org/opengroup/osdu/backup/service/impl/BackupServiceImpl.java

@@ -5,7 +5,6 @@ import java.util.EnumMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import javax.annotation.PostConstruct;
 import lombok.RequiredArgsConstructor;
 import org.opengroup.osdu.backup.locator.Asset;
 import org.opengroup.osdu.backup.locator.AssetManagerFactory;
@@ -31,18 +30,19 @@ public class BackupServiceImpl implements BackupService {
 	private final ScheduleRepository scheduleRepository;
 
 	private final AssetManagerFactory factory;
-
-	@PostConstruct
+//TODO currently schedules startup changed to BackupCollectionServiceImpl.startUpSchedulers()
+//	@PostConstruct
 	public void startUpSchedulers() {
 		List<BackupSchedule> backupScheduleList = scheduleRepository.listBackupSchedules();
-		backupScheduleList.stream().filter(BackupSchedule::isActive)
+		backupScheduleList.stream()/*.filter(BackupSchedule::isActive)*/
 			.forEach(schedulerService::addTaskToScheduler);
 	}
 
 	@Override
 	public BackupSchedule submitBackupSchedule(BackupSchedule backupSchedule) {
 		Map<String, String> assetContext = backupSchedule.getAssetContext();
-		BackupSchedule byAssetContext = scheduleRepository.findByAssetContext(backupSchedule.getAssetType(), assetContext);
+		BackupSchedule byAssetContext = scheduleRepository
+			.findByAssetContext(backupSchedule.getAssetType(), assetContext);
 		if (Objects.nonNull(byAssetContext)) {
 			throw new AppException(HttpStatus.CONFLICT.value(), "Conflict"
 				, String.format("Schedule with context %s already exist", assetContext));

backup-core/src/main/java/org/opengroup/osdu/backup/shedulers/BackupCollectionScheduledTask.java

+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.backup.shedulers;
+
+import java.time.LocalDateTime;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.stream.Collectors;
+import lombok.RequiredArgsConstructor;
+import org.opengroup.osdu.backup.locator.AssetManagerFactory;
+import org.opengroup.osdu.backup.model.BackupCollection;
+import org.opengroup.osdu.backup.model.BackupSchedule;
+import org.opengroup.osdu.backup.model.BackupStamp;
+import org.opengroup.osdu.backup.model.CollectionState;
+import org.opengroup.osdu.backup.model.StateStatus;
+import org.opengroup.osdu.backup.repository.CollectionStateRepository;
+
+@RequiredArgsConstructor
+public class BackupCollectionScheduledTask implements Runnable {
+
+	private final AssetManagerFactory assetManagerFactory;
+
+	private final BackupCollection collection;
+
+	private final CollectionStateRepository stateRepository;
+
+