Commit 0a0c363a authored by Rustam Lotsmanenko (EPAM)'s avatar Rustam Lotsmanenko (EPAM)
Browse files

GONRG-693 POC

Refactoring
parent bb46c3dd
......@@ -22,20 +22,10 @@
<artifactId>os-core-common</artifactId>
<version>0.3.5</version>
</dependency>
<dependency>
<groupId>org.apache.tomcat.embed</groupId>
<artifactId>tomcat-embed-core</artifactId>
<version>9.0.21</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-tomcat</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
......@@ -46,6 +36,25 @@
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>1.10.19</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-test</artifactId>
<version>5.2.7.RELEASE</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>
\ No newline at end of file
package org.opengroup.osdu.backup.api;
import java.util.List;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.opengroup.osdu.backup.model.BackupImportRequest;
import org.opengroup.osdu.backup.model.BackupsResponse;
import org.opengroup.osdu.backup.model.SubmitTaskRequest;
import org.opengroup.osdu.backup.provider.interfaces.BackupService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;
......@@ -18,10 +18,10 @@ import org.springframework.web.context.annotation.RequestScope;
@Slf4j
@RestController
@RequestScope
@RequiredArgsConstructor
public class BackupApi {
@Autowired
private BackupService backupService;
private final BackupService backupService;
@PostMapping("/submitScheduledTask")
public ResponseEntity<String> submitScheduledTask(@RequestBody SubmitTaskRequest submitTaskRequest) {
......
package org.opengroup.osdu.backup.shedulers;
import lombok.RequiredArgsConstructor;
import org.opengroup.osdu.backup.model.SubmitTaskRequest;
import org.opengroup.osdu.backup.provider.interfaces.BackupRepository;
@RequiredArgsConstructor
public class BackupScheduledTask implements Runnable {
private final BackupRepository backupRepository;
private final SubmitTaskRequest submitTaskRequest;
public BackupScheduledTask(BackupRepository backupRepository,
SubmitTaskRequest submitTaskRequest) {
this.backupRepository = backupRepository;
this.submitTaskRequest = submitTaskRequest;
}
@Override
public void run() {
backupRepository.exportBackup(submitTaskRequest);
......
......@@ -6,7 +6,7 @@ public interface SchedulerService {
void addTaskToScheduler(SubmitTaskRequest submitTaskRequest);
void removeTaskFromScheduler(SubmitTaskRequest submitTaskRequest);
boolean cancelScheduledTask(SubmitTaskRequest submitTaskRequest);
void taskStatusChanged(SubmitTaskRequest submitTaskRequest);
}
......@@ -5,22 +5,24 @@ import java.util.Map;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import javax.annotation.PostConstruct;
import lombok.RequiredArgsConstructor;
import org.opengroup.osdu.backup.model.SubmitTaskRequest;
import org.opengroup.osdu.backup.provider.interfaces.BackupRepository;
import org.opengroup.osdu.backup.shedulers.BackupScheduledTask;
import org.opengroup.osdu.backup.shedulers.SchedulerService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler;
import org.springframework.scheduling.support.PeriodicTrigger;
import org.springframework.stereotype.Service;
@Service
@RequiredArgsConstructor
public class SchedulerServiceImpl implements SchedulerService {
private final Map<SubmitTaskRequest, ScheduledFuture> taskMap = new HashMap<>();
@Autowired
private BackupRepository backupRepository;
private final TimeUnit timeUnit = TimeUnit.MINUTES;
private final BackupRepository backupRepository;
private ThreadPoolTaskScheduler poolTaskScheduler;
......@@ -36,7 +38,7 @@ public class SchedulerServiceImpl implements SchedulerService {
@Override
public void addTaskToScheduler(SubmitTaskRequest submitTaskRequest) {
PeriodicTrigger periodicTrigger
= new PeriodicTrigger(submitTaskRequest.getBackupPeriod(), TimeUnit.MINUTES);
= new PeriodicTrigger(submitTaskRequest.getBackupPeriod(), timeUnit);
periodicTrigger.setFixedRate(true);
BackupScheduledTask backupScheduledTask = new BackupScheduledTask(backupRepository, submitTaskRequest);
ScheduledFuture<?> schedule = poolTaskScheduler.schedule(backupScheduledTask, periodicTrigger);
......@@ -44,8 +46,8 @@ public class SchedulerServiceImpl implements SchedulerService {
}
@Override
public void removeTaskFromScheduler(SubmitTaskRequest submitTaskRequest) {
taskMap.get(submitTaskRequest).cancel(false);
public boolean cancelScheduledTask(SubmitTaskRequest submitTaskRequest) {
return taskMap.get(submitTaskRequest).cancel(false);
}
@Override
......
package org.opengroup.osdu.backup.shedulers.impl;
import static java.lang.Thread.sleep;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import java.util.concurrent.TimeUnit;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
import org.opengroup.osdu.backup.model.SubmitTaskRequest;
import org.opengroup.osdu.backup.provider.interfaces.BackupRepository;
import org.springframework.test.util.ReflectionTestUtils;
@RunWith(MockitoJUnitRunner.class)
public class SchedulerServiceImplTest {
@Mock
private BackupRepository backupRepository;
@InjectMocks
private SchedulerServiceImpl schedulerService;
private SubmitTaskRequest submitTaskRequest;
private SubmitTaskRequest submitTaskRequest1;
private SubmitTaskRequest submitTaskRequest2;
@Before
public void setUp() throws Exception {
submitTaskRequest = SubmitTaskRequest.builder().namespace("test").kind("kind").backupPeriod(2).active(true)
.build();
submitTaskRequest1 = SubmitTaskRequest.builder().namespace("test1").kind("kind1").backupPeriod(2).active(true)
.build();
submitTaskRequest2 = SubmitTaskRequest.builder().namespace("test2").kind("kind2").backupPeriod(2).active(true)
.build();
ReflectionTestUtils.setField(schedulerService, "timeUnit", TimeUnit.SECONDS);
schedulerService.initThreadPool();
}
@Test
public void addTaskToScheduler() throws InterruptedException {
schedulerService.addTaskToScheduler(submitTaskRequest);
schedulerService.addTaskToScheduler(submitTaskRequest1);
schedulerService.addTaskToScheduler(submitTaskRequest2);
sleep(1000);
verify(backupRepository, times(3)).exportBackup(any());
schedulerService.cancelScheduledTask(submitTaskRequest);
schedulerService.cancelScheduledTask(submitTaskRequest1);
schedulerService.cancelScheduledTask(submitTaskRequest2);
verifyNoMoreInteractions(backupRepository);
}
@Test
public void removeTaskFromScheduler() throws InterruptedException {
schedulerService.addTaskToScheduler(submitTaskRequest);
sleep(1000);
verify(backupRepository, times(1)).exportBackup(any());
Assert.assertTrue(schedulerService.cancelScheduledTask(submitTaskRequest));
}
@Test
public void taskStatusChanged() throws InterruptedException {
schedulerService.addTaskToScheduler(submitTaskRequest);
sleep(1000);
verify(backupRepository, times(1)).exportBackup(any());
submitTaskRequest.setActive(false);
schedulerService.taskStatusChanged(submitTaskRequest);
verifyNoMoreInteractions(backupRepository);
}
}
\ No newline at end of file
......@@ -20,7 +20,7 @@
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.2.5.RELEASE</version>
<version>2.3.4.RELEASE</version>
</parent>
<groupId>org.opengroup.osdu</groupId>
......
......@@ -81,6 +81,12 @@
<version>1.10.19</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-test</artifactId>
<version>5.2.7.RELEASE</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
......
package org.opengroup.osdu.backup.provider.gcp.config;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
@Setter
@Getter
@Configuration
@ConfigurationProperties(prefix = "gcp")
public class GcpPropertiesConfiguration {
private String backupBucket;
private String projectId;
}
......@@ -32,34 +32,28 @@ import lombok.extern.java.Log;
import org.opengroup.osdu.backup.model.BackupImportRequest;
import org.opengroup.osdu.backup.model.BackupsResponse;
import org.opengroup.osdu.backup.model.SubmitTaskRequest;
import org.opengroup.osdu.backup.provider.gcp.config.GcpPropertiesConfiguration;
import org.opengroup.osdu.backup.provider.gcp.model.GCPSubmitTask;
import org.opengroup.osdu.backup.provider.interfaces.BackupRepository;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Repository;
@Log
@Repository
@RequiredArgsConstructor
public class BackupRepositoryImpl implements BackupRepository {
public class BackupRepositoryDatastoreImpl implements BackupRepository {
private static final String OUTPUT_URL_PREFIX = "outputUrlPrefix";
private static final String OVERALL_EXPORT_METADATA = ".overall_export_metadata";
private JsonFactory JSON_FACTORY = new JacksonFactory();
private final JsonFactory JSON_FACTORY = new JacksonFactory();
private final Storage storage;
@Value("${gcp.backup.bucket}")
private String backupBucket;
private final GcpPropertiesConfiguration propertiesConfiguration;
@Value("${gcp.project.id}")
private String projectID;
@Autowired
private TaskRepository repository;
private final TaskRepository repository;
private Datastore datastore;
......@@ -74,7 +68,8 @@ public class BackupRepositoryImpl implements BackupRepository {
Datastore datastore = getDatastore();
GoogleLongrunningOperation exportTaskCreationResponse = datastore.projects().export(projectID, context)
GoogleLongrunningOperation exportTaskCreationResponse = datastore.projects()
.export(propertiesConfiguration.getProjectId(), context)
.execute();
Operations dataStoreOperations = datastore.projects().operations();
......@@ -105,7 +100,7 @@ public class BackupRepositoryImpl implements BackupRepository {
try {
GoogleDatastoreAdminV1ImportEntitiesRequest importRequest = new GoogleDatastoreAdminV1ImportEntitiesRequest();
importRequest.setInputUrl(backup.getBackupPath());
getDatastore().projects().datastoreImport(projectID, importRequest).execute();
getDatastore().projects().datastoreImport(propertiesConfiguration.getProjectId(), importRequest).execute();
} catch (IOException e) {
throw new AppException(HttpStatus.INTERNAL_SERVER_ERROR.value(), e.getMessage(), e.getMessage());
}
......@@ -122,7 +117,7 @@ public class BackupRepositoryImpl implements BackupRepository {
@Override
public BackupsResponse listAvailableBackups() {
BackupsResponse backupsResponse = new BackupsResponse();
Page<Blob> list = storage.list(backupBucket);
Page<Blob> list = storage.list(propertiesConfiguration.getBackupBucket());
HashMap<String, HashMap<String, List<String>>> assetsMap = new HashMap<>();
for (Blob blob : list.getValues()) {
......@@ -162,6 +157,7 @@ public class BackupRepositoryImpl implements BackupRepository {
scopes.add("https://www.googleapis.com/auth/datastore");
credential = credential.createScoped(scopes);
}
credential.refreshAccessToken();
return new Datastore(GoogleNetHttpTransport.newTrustedTransport(), JSON_FACTORY,
......@@ -175,13 +171,15 @@ public class BackupRepositoryImpl implements BackupRepository {
}
private String getOutputUrlPrefix(SubmitTaskRequest submitTaskRequest) {
return "gs://" + backupBucket + "/datastore/" + submitTaskRequest.getNamespace() + "-" + submitTaskRequest
return "gs://" + propertiesConfiguration.getBackupBucket() + "/datastore/" + submitTaskRequest.getNamespace()
+ "-" + submitTaskRequest
.getKind() + "/"
+ Timestamp.now();
}
private String getInputUrlPrefixRequest(String[] path) {
return "gs://" + backupBucket + "/" + path[0] + "/" + path[1] + "/" + path[2] + "/" + path[2]
return "gs://" + propertiesConfiguration.getBackupBucket() + "/" + path[0] + "/" + path[1] + "/" + path[2] + "/"
+ path[2]
+ OVERALL_EXPORT_METADATA;
}
......
......@@ -4,6 +4,7 @@ package org.opengroup.osdu.backup.provider.gcp.repository;
import java.util.List;
import java.util.Objects;
import javax.annotation.PostConstruct;
import lombok.RequiredArgsConstructor;
import org.opengroup.osdu.backup.model.BackupImportRequest;
import org.opengroup.osdu.backup.model.BackupsResponse;
import org.opengroup.osdu.backup.model.SubmitTaskRequest;
......@@ -11,26 +12,23 @@ import org.opengroup.osdu.backup.provider.gcp.model.GCPSubmitTask;
import org.opengroup.osdu.backup.provider.interfaces.BackupRepository;
import org.opengroup.osdu.backup.provider.interfaces.BackupService;
import org.opengroup.osdu.backup.shedulers.SchedulerService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@Service
@RequiredArgsConstructor
public class BackupServiceImpl implements BackupService {
@Autowired
private BackupRepository backupRepository;
private final BackupRepository backupRepository;
@Autowired
private TaskRepository taskRepository;
private final TaskRepository taskRepository;
@Autowired
private SchedulerService schedulerService;
private final SchedulerService schedulerService;
@PostConstruct
public void startUpSchedulers() {
List<SubmitTaskRequest> submitTaskRequestList = backupRepository.listBackupSchedules();
submitTaskRequestList.stream().filter(SubmitTaskRequest::isActive)
.forEach(t -> schedulerService.addTaskToScheduler(t));
.forEach(schedulerService::addTaskToScheduler);
}
@Override
......
server.servlet.contextPath=/backup/v1
server.port=8080
logging.level.org.springframework=INFO
logging.level.org.springframework.boot.autoconfigure=ERROR
spring.cloud.gcp.datastore.namespace=opendes
gcp.backup.bucket=osdu-cicd-epam-backup-service
gcp.project.id=osdu-cicd-epam
gcp.backup_bucket=osdu-cicd-epam-backup-service
gcp.project_id=osdu-cicd-epam
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment