Commit d868bec2 authored by Rustam Lotsmanenko (EPAM)'s avatar Rustam Lotsmanenko (EPAM)
Browse files

GONRG-693 POC

Task Scheduling draft
parent 508f7d7b
......@@ -2,45 +2,45 @@
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>backup-service</artifactId>
<groupId>org.opengroup.osdu</groupId>
<version>1.0.0</version>
<relativePath>../pom.xml</relativePath>
</parent>
<parent>
<artifactId>backup-service</artifactId>
<groupId>org.opengroup.osdu</groupId>
<version>1.0.0</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>backup-core</artifactId>
<description>Backup service Core on GCP</description>
<version>1.0.0</version>
<artifactId>backup-core</artifactId>
<description>Backup service Core on GCP</description>
<version>1.0.0</version>
<dependencies>
<dependency>
<groupId>org.apache.tomcat.embed</groupId>
<artifactId>tomcat-embed-core</artifactId>
<version>9.0.21</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-tomcat</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependencies>
<dependency>
<groupId>org.apache.tomcat.embed</groupId>
<artifactId>tomcat-embed-core</artifactId>
<version>9.0.21</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-tomcat</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</dependency>
</dependencies>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</dependency>
</dependencies>
</project>
\ No newline at end of file
......@@ -3,10 +3,10 @@ package org.opengroup.osdu.backup;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.ComponentScan;
@SpringBootApplication(scanBasePackages = "org.opengroup")
public class BackupApplication {
public static void main(String[] args) {
SpringApplication.run(BackupApplication.class, args);
}
......
package org.opengroup.osdu.backup.api;
import lombok.extern.slf4j.Slf4j;
import org.opengroup.osdu.backup.model.ImportRequest;
import org.opengroup.osdu.backup.model.SubmitRequest;
import org.opengroup.osdu.backup.model.SubmitTask;
import org.opengroup.osdu.backup.provider.interfaces.BackupService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
......@@ -16,14 +18,27 @@ import org.springframework.web.context.annotation.RequestScope;
@RequestScope
public class BackupApi {
@Autowired
private BackupService backupService;
@Autowired
private BackupService backupService;
@PostMapping("/submitTask/")
public ResponseEntity<String> submitTask(@RequestBody SubmitRequest submitRequest) {
backupService.submitTask(submitRequest);
return new ResponseEntity<String>("Backup service is alive", HttpStatus.OK);
}
@PostMapping("/submitTask/")
public ResponseEntity<String> submitTask(@RequestBody SubmitRequest submitRequest) {
backupService.submitRequest(submitRequest);
return new ResponseEntity<String>("Backup service is alive", HttpStatus.OK);
}
@PostMapping("/submitTask2/")
public ResponseEntity<String> submitTask2(@RequestBody SubmitTask submitTask) {
backupService.submitTask(submitTask);
return new ResponseEntity<String>("Task submitted", HttpStatus.OK);
}
@PostMapping("/submitImport")
public ResponseEntity<String> submitImport(@RequestBody ImportRequest importRequest) {
backupService.submitImportRequest(importRequest);
return new ResponseEntity<String>("Import request submitted", HttpStatus.OK);
}
}
......@@ -11,15 +11,15 @@ import org.springframework.web.bind.annotation.RestController;
@RequestMapping("/health")
public class HealthCheckApi {
@PermitAll
@GetMapping("/liveness_check")
public ResponseEntity<String> livenessCheck() {
return new ResponseEntity<String>("Backup service is alive", HttpStatus.OK);
}
@PermitAll
@GetMapping("/liveness_check")
public ResponseEntity<String> livenessCheck() {
return new ResponseEntity<String>("Backup service is alive", HttpStatus.OK);
}
@PermitAll
@GetMapping("/readiness_check")
public ResponseEntity<String> readinessCheck() {
return new ResponseEntity<String>("Backup service is ready", HttpStatus.OK);
}
@PermitAll
@GetMapping("/readiness_check")
public ResponseEntity<String> readinessCheck() {
return new ResponseEntity<String>("Backup service is ready", HttpStatus.OK);
}
}
package org.opengroup.osdu.backup.model;
public class ImportRequest {
}
......@@ -13,10 +13,10 @@ import lombok.NoArgsConstructor;
@NoArgsConstructor
public class SubmitRequest {
@JsonProperty("AssetType")
String dataType;
@JsonProperty("AssetType")
String dataType;
@JsonProperty("Context")
Map<String, Object> context;
@JsonProperty("Context")
Map<String, Object> context;
}
package org.opengroup.osdu.backup.model;
import java.util.Date;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@AllArgsConstructor
@Builder
public class SubmitTask {
private String namespace;
private String kind;
private Date date;
private String backupPath;
private int backupPeriod;//1
private int status;
}
......@@ -2,6 +2,7 @@ package org.opengroup.osdu.backup.provider.interfaces;
public interface BackupRepository {
void get(String namespace, String kind);
void exportBackup(String namespace, String kind);
void importBackup(String backupLocationPath);
}
package org.opengroup.osdu.backup.provider.interfaces;
import org.opengroup.osdu.backup.model.ImportRequest;
import org.opengroup.osdu.backup.model.SubmitRequest;
import org.opengroup.osdu.backup.model.SubmitTask;
public interface BackupService {
void submitTask(SubmitRequest submitRequest);
void submitRequest(SubmitRequest submitRequest);
void submitTask(SubmitTask submitTask);
void submitImportRequest(ImportRequest importRequest);
}
package org.opengroup.osdu.backup.shedulers;
import org.opengroup.osdu.backup.model.SubmitTask;
public interface Scheduler {
void addTaskToScheduler(SubmitTask submitTask);
}
......@@ -40,6 +40,11 @@
<version>1.0.0</version>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-gcp-starter-data-datastore</artifactId>
<version>1.2.5.RELEASE</version>
</dependency>
<dependency>
<groupId>com.google.apis</groupId>
......@@ -56,6 +61,11 @@
<artifactId>google-cloud-datastore</artifactId>
<version>1.86.0</version>
</dependency>
<dependency>
<groupId>org.mapstruct</groupId>
<artifactId>mapstruct</artifactId>
<version>1.3.1.Final</version>
</dependency>
<!-- Testing packages -->
<dependency>
......
package org.opengroup.osdu.backup.provider.gcp;/*
* Copyright 2020 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
public interface GcpPackageMarker {
}
package org.opengroup.osdu.backup.provider.gcp.conf;
import org.opengroup.osdu.backup.provider.gcp.GcpPackageMarker;
import org.springframework.cloud.gcp.data.datastore.repository.config.EnableDatastoreRepositories;
import org.springframework.context.annotation.Configuration;
@Configuration
@EnableDatastoreRepositories(basePackageClasses = GcpPackageMarker.class)
public class DatastoreConfiguration {
}
package org.opengroup.osdu.backup.provider.gcp.mapper;
import org.opengroup.osdu.backup.model.SubmitTask;
import org.opengroup.osdu.backup.provider.gcp.model.GCPSubmitTask;
import org.springframework.stereotype.Service;
@Service
public class SubmitTaskMapper {
public GCPSubmitTask toGcpSubmitTask(SubmitTask submitTask) {
return GCPSubmitTask.builder()
.kind(submitTask.getKind())
.namespace(submitTask.getNamespace())
.period(submitTask.getBackupPeriod())
.build();
}
}
package org.opengroup.osdu.backup.provider.gcp.model;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.springframework.cloud.gcp.data.datastore.core.mapping.Entity;
import org.springframework.cloud.gcp.data.datastore.core.mapping.Field;
import org.springframework.data.annotation.Id;
@Entity(name = "BackupTestKind")
@Data
@NoArgsConstructor
@AllArgsConstructor
@Builder
public class GCPSubmitTask {
@Id
Long id;
@Field(name = "Namespace")
private String namespace;
@Field(name = "Kind")
private String kind;
@Field(name = "period")
private int period;
@Field(name = "Path")
private String backupPath;
@Field(name = "Status")
private int status;
}
package org.opengroup.osdu.backup.provider.gcp.repository;
import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.api.services.datastore.v1.Datastore;
import com.google.api.services.datastore.v1.model.GoogleDatastoreAdminV1EntityFilter;
import com.google.api.services.datastore.v1.model.GoogleDatastoreAdminV1ExportEntitiesRequest;
import com.google.api.services.datastore.v1.model.GoogleDatastoreAdminV1ImportEntitiesRequest;
import com.google.api.services.datastore.v1.model.GoogleLongrunningOperation;
import com.google.api.services.iam.v1.IamScopes;
import com.google.auth.http.HttpCredentialsAdapter;
import com.google.auth.oauth2.GoogleCredentials;
......@@ -15,41 +17,94 @@ import java.security.GeneralSecurityException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import lombok.extern.java.Log;
import org.opengroup.osdu.backup.provider.gcp.model.GCPSubmitTask;
import org.opengroup.osdu.backup.provider.interfaces.BackupRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Repository;
@Log
@Repository
public class BackupRepositoryImpl implements BackupRepository {
@Value("${gcp.backup.bucket}")
private String backupBucket;
@Value("${gcp.project.id}")
private String projectID;
private JsonFactory JSON_FACTORY = new JacksonFactory();
@Autowired
private TaskRepository repository;
private Datastore datastore;
@Override
public void exportBackup(String namespace, String kind) {
log.log(Level.INFO, "Starting backup export for namespace:{0} kind:{1} export",
new Object[]{namespace, kind});
try {
GoogleDatastoreAdminV1EntityFilter googleDatastoreAdminV1EntityFilter = new GoogleDatastoreAdminV1EntityFilter();
googleDatastoreAdminV1EntityFilter.setKinds(Collections.singletonList(kind));
googleDatastoreAdminV1EntityFilter.setNamespaceIds(Collections.singletonList(namespace));
GoogleDatastoreAdminV1ExportEntitiesRequest context = new GoogleDatastoreAdminV1ExportEntitiesRequest();
context.setEntityFilter(googleDatastoreAdminV1EntityFilter);
context.setOutputUrlPrefix(backupBucket);
GoogleLongrunningOperation execute = getDatastore().projects().export(projectID, context).execute();
log.log(Level.INFO, "Export status:{0} response:{1} export",
new Object[]{execute.getDone(), execute.getResponse()});
Map<String, Object> response = execute.getResponse();
GCPSubmitTask submittedTask = GCPSubmitTask.builder().kind(kind).namespace(namespace)
.backupPath(response.get("path").toString()).build();
repository.save(submittedTask);
} catch (IOException e) {
e.printStackTrace();
}
}
@Override
public void importBackup(String backupLocationPath) {
try {
GoogleDatastoreAdminV1ImportEntitiesRequest importRequest = new GoogleDatastoreAdminV1ImportEntitiesRequest();
importRequest.setInputUrl(backupLocationPath);
getDatastore().projects().datastoreImport(projectID, importRequest).execute();
} catch (IOException ioException) {
ioException.printStackTrace();
}
}
private Datastore getDatastore() {
if (datastore == null) {
try {
GoogleCredentials credential = GoogleCredentials
.getApplicationDefault();
if (credential.createScopedRequired()) {
List<String> scopes = new ArrayList<>();
scopes.add(IamScopes.CLOUD_PLATFORM);
scopes.add("https://www.googleapis.com/auth/datastore");
credential = credential.createScoped(scopes);
}
credential.refreshAccessToken();
return new Datastore(GoogleNetHttpTransport.newTrustedTransport(), JSON_FACTORY,
new HttpCredentialsAdapter(credential));
@Override
public void get(String namespace, String kind) {
GoogleDatastoreAdminV1EntityFilter googleDatastoreAdminV1EntityFilter = new GoogleDatastoreAdminV1EntityFilter();
googleDatastoreAdminV1EntityFilter.setKinds(Collections.singletonList(kind));
googleDatastoreAdminV1EntityFilter.setNamespaceIds(Collections.singletonList(namespace));
GoogleDatastoreAdminV1ExportEntitiesRequest context = new GoogleDatastoreAdminV1ExportEntitiesRequest();
context.setEntityFilter(googleDatastoreAdminV1EntityFilter);
JsonFactory JSON_FACTORY = new JacksonFactory();
try {
GoogleCredentials credential = GoogleCredentials.getApplicationDefault();
if (credential.createScopedRequired()) {
List<String> scopes = new ArrayList<>();
scopes.add(IamScopes.CLOUD_PLATFORM);
credential = credential.createScoped(scopes);
}
HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport();
com.google.api.services.datastore.v1.Datastore datastore2 =
new com.google.api.services.datastore.v1.Datastore(httpTransport, JSON_FACTORY,
new HttpCredentialsAdapter(credential));
datastore2.projects().export("sss", context);
} catch (GeneralSecurityException e) {
} catch (IOException e) {
e.printStackTrace();
}
//KeyFactory keyFactory = datastore.newKeyFactory().setKind(kind).setNamespace(namespace);
// Query.newEntityQueryBuilder().setKind(kind).setNamespace(namespace);
//datastore.get(keyFactory);
}
} catch (GeneralSecurityException | IOException e) {
e.printStackTrace();
}
}
return datastore;
}
}
package org.opengroup.osdu.backup.provider.gcp.repository;
import org.opengroup.osdu.backup.model.ImportRequest;
import org.opengroup.osdu.backup.model.SubmitRequest;
import org.opengroup.osdu.backup.model.SubmitTask;
import org.opengroup.osdu.backup.provider.gcp.mapper.SubmitTaskMapper;
import org.opengroup.osdu.backup.provider.gcp.model.GCPSubmitTask;
import org.opengroup.osdu.backup.provider.interfaces.BackupRepository;
import org.opengroup.osdu.backup.provider.interfaces.BackupService;
import org.opengroup.osdu.backup.shedulers.Scheduler;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
......@@ -12,8 +18,30 @@ public class BackupServiceImpl implements BackupService {
@Autowired
private BackupRepository backupRepository;
@Autowired
private TaskRepository taskRepository;
@Autowired
private SubmitTaskMapper submitTaskMapper;
@Autowired
private Scheduler scheduler;
@Override
public void submitRequest(SubmitRequest submitRequest) {
backupRepository.exportBackup(
"opendes", "BackupTestKind");
}
@Override
public void submitTask(SubmitTask submitTask) {
scheduler.addTaskToScheduler(submitTask);
GCPSubmitTask gcpSubmitTask = submitTaskMapper.toGcpSubmitTask(submitTask);
taskRepository.save(gcpSubmitTask);
}
@Override
public void submitTask(SubmitRequest submitRequest) {
backupRepository.get("osdu","LegalTag");
public void submitImportRequest(ImportRequest importRequest) {
backupRepository.importBackup("");
}
}
package org.opengroup.osdu.backup.provider.gcp.repository;
import org.opengroup.osdu.backup.provider.gcp.model.GCPSubmitTask;
import org.springframework.cloud.gcp.data.datastore.repository.DatastoreRepository;
import org.springframework.stereotype.Repository;