GONRG-864 int tests

Changes:
Int tests are there

README.md

 # Backup service
 
-Backup service provide a set of APIs  to backup Schedule, list registered Schedules, list available Backups, restore Asset state from Backup.
+Backup service provide a set of APIs  to schedule backups for different Assets (Datastore, Storage, ElasticSearch, etc depends on which currently implemented ), 
+list registered Schedules, list available Backups, restore Asset state from Backup.
 
+## Core Interfaces
+![Core](./docs/core.png)
+## Data Repositories
+![Repo](./docs/datarepos.png)
+## Sequence diagram
+Submit schedule workflow
+
+
+![Diagram](./docs/sequence.png)
 ## GCP Implementation
 
 All documentation for the GCP implementation of Backup service lives [here](./provider/backup-gcp/README.md)

backup-core/src/main/java/org/opengroup/osdu/backup/manager/AssetBackupManager.java

@@ -10,5 +10,5 @@ public interface AssetBackupManager {
 
 	BackupStamp importBackup(String backupId);
 
-	void deleteBackups(List<BackupStamp> currentDate);
+	void deleteBackups(List<BackupStamp> backupStamps);
 }

docs/core.puml

+@startuml
+package backup.core <<Rectangle>> {
+interface AssetBackupManager{
+exportBackup(BackupSchedule backupSchedule);
+importBackup(String backupId);
+deleteBackups(List<BackupStamp> backupStamps);
+}
+note right
+Provide backup managing access to current Asset,
+ based on AssetType of schedule
+end note
+interface SchedulerService{
+addTaskToScheduler(BackupSchedule backupSchedule);
+cancelScheduledTask(BackupSchedule backupSchedule);
+updateTask(BackupSchedule backupSchedule);
+}
+note right
+Holds ThreadPool with scheduled tasks ,
+and provide control
+end note
+
+interface BackupService{
+submitBackupSchedule(BackupSchedule backupSchedule);
+updateBackupSchedule(BackupSchedule backupSchedule);
+getBackupSchedule(String id);
+submitBackupImportRequest(BackupImportRequest importRequest);
+listSchedules();
+listBackups();
+}
+class BackupServiceImpl implements BackupService
+class SchedulerServiceImpl implements SchedulerService{
+ThreadPoolTaskScheduler poolTaskScheduler
+}
+enum AssetType {
+DATASTORE
+ELASTIC
+etc
+}
+package backup.gcp <<Rectangle>> {
+class DatastoreBackupManager implements AssetBackupManager
+class ElasticBackupManager implements AssetBackupManager
+DatastoreBackupManager *-- AssetType
+ElasticBackupManager *-- AssetType
+}
+@enduml
\ No newline at end of file

docs/datarepos.puml

+@startuml
+
+package backup.core <<Rectangle>> {
+interface BackupStampRepository{
+findBackupStampsByTearDownBeforeCurrentDate(LocalDateTime toDate);
+submitBackupStamp(BackupStamp backupStamp);
+listAvailableBackupStamps();
+deleteBackupStamps(List<BackupStamp> currentDate);
+findBackupByStampId(String backupStampId);
+}
+note right
+A backup stamp's repository ,
+backup stamp is a model of created backup entity
+which provide additional info about it : lifetime, location etc :
+end note
+interface ScheduleRepository{
+listBackupSchedules();
+submitSchedule(BackupSchedule backupSchedule);
+updateSchedule(BackupSchedule backupSchedule);
+findByAssetContext(Map<String, String> assetContext);
+save(BackupSchedule backupSchedule);
+findById(String scheduleId);
+}
+class BackupStamp{
+String stampId;
+String backupRepository;
+Asset assetType;
+LocalDateTime tearDownDate;
+Map<String, String> assetContext;
+}
+class BackupSchedule{
+private String scheduleId;
+Asset assetType;
+int backupPeriod;
+boolean active;
+int lifetime;
+Map<String, String> assetContext;
+}
+
+package backup.gcp <<Rectangle>> {
+class DatastoreBackupStampRepository implements BackupStampRepository
+class DatastoreScheduleRepository implements ScheduleRepository
+}
+@enduml
\ No newline at end of file

docs/sequence.puml

+@startuml
+actor User
+
+User -> BackupService : submit schedule
+activate BackupService
+BackupService -> ScheduleRepository : save schedule
+database DB
+ScheduleRepository -> DB
+BackupService -> Scheduler : add task
+ScheduleRepository -> BackupService
+BackupService -> User : saved schedule(with id)
+deactivate BackupService
+control ScheduledTask
+activate ScheduledTask
+Scheduler -> ScheduledTask
+ScheduledTask -> ScheduledTask : run by schedule
+activate AssetBackupManager
+ScheduledTask -> AssetBackupManager : export()
+database BackupStorage
+AssetBackupManager -> BackupStorage : save backup
+AssetBackupManager -> BackupStampRepository : save export stamp
+BackupStampRepository -> DB
+deactivate AssetBackupManager
+deactivate ScheduledTask
+User -> BackupService : list backup stamps
+
+
+
+
+@enduml
+activate BackupService
+activate ScheduleRepository
+ScheduleRepository -> BackupSchedule :saved schedule
+database DB
+ScheduleRepository -> ProviderStorage
+BackupService -> ScheduleRepository : save schedule
+BackupService -> SchedulerService : add scheduled task
+SchedulerService -> ScheduledTask
+alt successful case
+activate ScheduledTask
+ScheduledTask --> ScheduledTask
+deactivate ScheduledTask
+
+
+ScheduledTask -> AssetManager : export backup
+AssetManager -> BackupStampsRepository : submit backup stamp
+BackupStampsRepository -> ProviderStorage
\ No newline at end of file

docs/tutorial/BackUp.md

+## Backup Service
+
+## Introduction <a name="Introduction"></a>
+
 ### Submit scheduled backup task
 
 ```
-POST /backup/v1/submitScheduledTask
+POST /backup/v1/schedule
 ```
 ```
-curl --location --request POST 'localhost:8080/backup/v1/submitScheduledTask' \
---header 'Data-Partition-Id: opendes' \
---header 'Authorization: Bearer <JWT> \
---header 'Content-Type: application/json' \
+curl --location --request POST 'localhost:8080/backup/v1/schedule' 
+--header 'Data-Partition-Id: opendes' 
+--header 'Authorization: Bearer <JWT> 
+--header 'Content-Type: application/json' 
 --data-raw '{
-    "assetType": "datastore",
-    "namespace": "opendes",
-    "kind": "LegalTag",
-    "backupPeriod": "1",
-    "active": "true"
+    "assetType": "DATSTORE",
+    "backupPeriod": "5",
+    "active": "true",
+    "lifetime": 2,
+    "assetContext": {
+        "namespace": "opendes",
+        "kind": "TestKind"
+    }
 }'
 ```
 
+### Get schedule by id
+
+```
+GET /backup/v1/schedule?id=
+```
+```
+curl --location --request GET 'localhost:8080/backup/v1/schedule?id=8de2a9e34bdf489aba20206b8b28ab6e' \
+--header 'Data-Partition-Id: opendes'
+--header 'Authorization: Bearer <JWT> 
+```
+
+### Update schedule by id
+
+```
+PUT /backup/v1/schedule
+```
+```
+curl --location --request PUT 'localhost:8080/backup/v1/schedule' 
+--header 'Data-Partition-Id: opendes' 
+--header 'Content-Type: application/json' 
+--data-raw '{
+    "scheduleId":"2049486875f541ad8df6a0e6da777b7f",
+    "backupPeriod": "5",
+    "active": "true",
+    "lifetime": 2,
+}'
+```
+
+
 ### List backup schedules
 
 ```
-GET /backup/v1/listSchedules
+GET /backup/v1/list_schedules
 ```
 ```
-curl --location --request GET 'localhost:8080/backup/v1/listSchedules' \
---header 'Data-Partition-Id: opendes' \
+curl --location --request GET 'localhost:8080/backup/v1/list_schedules' 
+--header 'Data-Partition-Id: opendes' 
 --header 'Authorization: Bearer <JWT>
 ```
 
 ### List available backups
 
 ```
-GET /backup/v1/listBackups
+GET /backup/v1/list_backups
 ```
 ```
-curl --location --request GET 'localhost:8080/backup/v1/listBackups' \
---header 'Data-Partition-Id: opendes' \
+curl --location --request GET 'localhost:8080/backup/v1/list_backups' 
+--header 'Data-Partition-Id: opendes' 
 --header 'Authorization: Bearer <JWT>
 ```
 
 ### Import backup
 
 ```
-POST /backup/v1/submitImport
+POST /backup/v1/backup_import
 ```
 ```
-curl --location --request POST 'localhost:8080/backup/v1/submitImport' \
---header 'Data-Partition-Id: opendes' \
---header 'Authorization: Bearer <JWT> \
---header 'Content-Type: application/json' \
+curl --location --request POST 'localhost:8080/backup/v1/backup_import' 
+--header 'Data-Partition-Id: opendes' 
+--header 'Authorization: Bearer <JWT>
+--header 'Content-Type: application/json' 
 --data-raw '{
-    "backupPath": "gs://<backup file path>"
-}'
+    "backupStampId": "4730f8b5dd1145a3b09335c640d455c0",
+    "asset":"DATASTORE"
+}''
 ```

provider/backup-gcp/README.md

@@ -20,11 +20,20 @@ In order to run the service locally or remotely, you will need to have the follo
 
 | name | value | description | sensitive? | source |
 | ---  | ---   | ---         | ---        | ---    |
-| `SPRING_CLOUD_GCP_DATASTORE_NAMESPACE` | ex `opendes` | Datastore namespace to backup| no | https://console.cloud.google.com/datastore |
+| `OSDU_ENTITLEMENTS_URL` | ex `https://os-entitlements-gcp-jvmvia5dea-uc.a.run.app/entitlements/v1` | Entitlements API endpoint | no | output of infrastructure deployment |
 | `GCP_BACKUP_BUCKET` | ex `osdu-cicd-epam-backup-service` | Storage bucket for backups | no | https://console.cloud.google.com/storage |
 | `GCP_PROJECT_ID` | ex `osdu-cicd-epam` | GCP project id | no | - |
 | `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
 
+Schedulers can be configured with following variables, if not defined default values will be used
+
+| name | value | description | default |
+| ---  | ---   | ---         |  ---    |
+| `OSDU_SCHEDULER_TEAR-DOWN-TIME-UNIT` | ex `seconds OR minutes OR hours etc` | Time unit for tear down scheduler | `hours`  |
+| `OSDU_SCHEDULER_TEAR-DOWN-PERIOD` | ex `24` | The period with which expired backups will be deleted | `24` |
+| `OSDU_SCHEDULER_BACKUP-TIME-UNIT` | ex `seconds OR minutes OR hours etc` | Time unit for backup export scheduler (Period will be configured with saved schedule) | `hours` |
+| `OSDU_SCHEDULER_INITIAL-DELAY` | ex `1` | Delay before start exporting new added assets, helpful for int tests, to not overwhelm backup storage | `1` |
+
 ### Run Locally
 Check that maven is installed:
 
@@ -95,6 +104,55 @@ After configuring your environment as specified above, you can follow these step
 ```bash
 cd provider/backup-gcp && mvn spring-boot:run
 ```
+## Testing
+ 
+ ### Running E2E Tests 
+ This section describes how to run cloud OSDU E2E tests (testing/backup-test-gcp).
+ 
+ You will need to have the following environment variables defined.
+ 
+ | name | value | description | sensitive? | source |
+ | ---  | ---   | ---         | ---        | ---    |
+ | `BACKUP_SERVICE_HOST` | ex`http://localhost:8080/backup/v1` | Service endpoint | no | - |
+ | `DATA_PARTITION_ID` | `opendes` | OSDU tenant used for testing | no | - |
+ | `INTEGRATION_TESTER` | ex`/path/to/directory/service-key.json` | Service account .json for API calls. Note: this user must have entitlements configured already | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
+ | `NO_DATA_ACCESS_TESTER` | ex`/path/to/directory/service-key.json` | Service account .json without data access | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
+ | `GCP_DEPLOY_FILE` | ex`/path/to/directory/service-key.json` | Service account for test data tear down, must have cloud role configured | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
+ | `INTEGRATION_TEST_AUDIENCE` | `****` | client application ID | yes | https://console.cloud.google.com/apis/credentials |
+ | `DATASTORE_NAMESPACE` | ex `opendes` | OSDU tenant used for testing | no | - |
+ 
+ **Entitlements configuration for integration accounts**
+ 
+ | INTEGRATION_TESTER | NO_DATA_ACCESS_TESTER | 
+ | ---  | ---   |
+ | users<br/>backup.service| users |
+ 
+ **Cloud roles configuration for integration accounts**
+ 
+ | GCP_DEPLOY_FILE|
+ | ---  |
+ | Cloud Datastore Owner |
+ 
+ Execute following command to build code and run all the integration tests:
+ 
+ ```bash
+ # Note: this assumes that the environment variables for integration tests as outlined
+ #       above are already exported in your environment.
+ # build + install integration test core
+ $ (cd testing/backup-test-core/ && mvn clean install)
+ ```
+ ```bash
+ # build + run GCP integration tests.
+ $ (cd testing/backup-test-gcp/ && mvn clean test)
+ ```
+
+## Deployment
+
+* To deploy into Cloud run, please, use this documentation:
+https://cloud.google.com/run/docs/quickstarts/build-and-deploy
+
+* To deploy into App Engine, please, use this documentation:
+https://cloud.google.com/appengine/docs/flexible/java/quickstart
 
 ## License
 

provider/backup-gcp/src/main/java/org/opengroup/osdu/backup/provider/gcp/manager/ElasticBackupManager.java

@@ -24,7 +24,7 @@ public class ElasticBackupManager implements AssetBackupManager {
 	}
 
 	@Override
-	public void deleteBackups(List<BackupStamp> currentDate) {
+	public void deleteBackups(List<BackupStamp> backupStamps) {
 		throw new UnsupportedOperationException();
 	}
 }

provider/backup-gcp/src/main/resources/application.properties

 server.servlet.contextPath=/backup/v1
 server.port=8080
+
 logging.level.org.springframework=INFO
 logging.level.org.springframework.boot.autoconfigure=ERROR
 log.prefix=backup
-spring.cloud.gcp.datastore.namespace=opendes
-gcp.backup-bucket=osdu-cicd-epam-backup-service
-gcp.project-id=osdu-cicd-epam
+
+#Core
 osdu.entitlements.url=https://os-entitlements-gcp-jvmvia5dea-uc.a.run.app/entitlements/v1
 osdu.entitlements.app-key=test
 osdu.scheduler.tear-down-time-unit=hours
 osdu.scheduler.tear-down-period=24
-osdu.scheduler.backup-time-unit=minutes
+osdu.scheduler.backup-time-unit=hours
 osdu.scheduler.initial-delay=1
+
+#GCP
+gcp.backup-bucket=osdu-cicd-epam-backup-service
+gcp.project-id=osdu-cicd-epam
+
 LOG_PREFIX=${log.prefix}

testing/backup-test-core/pom.xml

+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<artifactId>backup-test-core</artifactId>
+	<version>0.0.1</version>
+
+	<name>backup-test-core</name>
+	<description>Core test project for the backup service</description>
+
+	<parent>
+		<groupId>org.opengroup.osdu</groupId>
+		<artifactId>backup-test</artifactId>
+		<version>0.0.1</version>
+		<relativePath>../pom.xml</relativePath>
+	</parent>
+
+	<properties>
+		<maven.compiler.target>1.8</maven.compiler.target>
+		<maven.compiler.source>1.8</maven.compiler.source>
+	</properties>
+
+	<dependencies>
+		<dependency>
+			<groupId>org.opengroup.osdu</groupId>
+			<artifactId>backup-core</artifactId>
+			<version>0.0.1</version>
+		</dependency>
+		<dependency>
+			<groupId>com.sun.jersey</groupId>
+			<artifactId>jersey-client</artifactId>
+			<version>1.19.4</version>
+		</dependency>
+		<dependency>
+			<groupId>junit</groupId>
+			<artifactId>junit</artifactId>
+			<version>4.12</version>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.junit.jupiter</groupId>
+			<artifactId>junit-jupiter</artifactId>
+			<version>5.5.2</version>
+		</dependency>
+	</dependencies>
+</project>

testing/backup-test-core/src/main/java/org/opengroup/osdu/backup/Config.java

+package org.opengroup.osdu.backup;
+
+public class Config {
+
+	public static final String BACKUP_SERVICE_HOST = "";
+	public static final String INTEGRATION_TESTER = "";
+	private static final String NO_DATA_ACCESS_TESTER = "";
+	public static final String DATA_PARTITION_ID = "";
+
+
+	public static String getBackupServiceHost() {
+		return getEnvironmentVariableOrDefaultValue("BACKUP_SERVICE_HOST", BACKUP_SERVICE_HOST);
+	}
+
+	public static String getIntegrationTester() {
+		return getEnvironmentVariableOrDefaultValue("INTEGRATION_TESTER", INTEGRATION_TESTER);
+	}
+
+	public static String getDataPartitionId() {
+		return getEnvironmentVariableOrDefaultValue("DATA_PARTITION_ID", DATA_PARTITION_ID);
+	}
+
+	public static String getNoAccessTester() {
+		return getEnvironmentVariableOrDefaultValue("NO_DATA_ACCESS_TESTER", NO_DATA_ACCESS_TESTER);
+	}
+
+
+	private static String getEnvironmentVariableOrDefaultValue(String key, String defaultValue) {
+		String environmentVariable = getEnvironmentVariable(key);
+		if (environmentVariable == null) {
+			environmentVariable = defaultValue;
+		}
+		return environmentVariable;
+	}
+
+	private static String getEnvironmentVariable(String propertyKey) {
+		return System.getProperty(propertyKey, System.getenv(propertyKey));
+	}
+
+
+}

testing/backup-test-core/src/main/java/org/opengroup/osdu/backup/HttpClient.java

+package org.opengroup.osdu.backup;/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import java.io.IOException;
+import java.net.URL;
+import java.security.SecureRandom;
+import java.security.cert.X509Certificate;
+import java.util.Map;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
+import javax.ws.rs.core.MediaType;
+
+
+public abstract class HttpClient {
+
+	protected static String accessToken;
+	protected static String noDataAccessToken;
+
+	public abstract String getAccessToken() throws IOException;
+
+	public abstract String getNoDataAccessToken() throws IOException;
+
+	public ClientResponse send(String path, String httpMethod, Map<String, String> headers,
+		String requestBody)
+		throws Exception {
+
+		Client client = this.getClient();
+
+		String mergedURL = new URL(Config.getBackupServiceHost() + path).toString();
+		System.out.println(String.format("calling %s API:%s", httpMethod, mergedURL));
+		System.out.println(String.format("request body:%s", requestBody));
+
+		if (requestBody != null) {
+			headers.put("Content-Length", Long.toString(requestBody.length()));
+		} else {
+			headers.put("Content-Length", "0");
+		}
+
+		WebResource webResource = client.resource(mergedURL);
+
+		WebResource.Builder builder = webResource.accept(MediaType.APPLICATION_JSON)
+			.type(MediaType.APPLICATION_JSON);
+		headers.forEach(builder::header);
+
+		return builder.method(httpMethod, ClientResponse.class, requestBody);
+	}
+
+
+	private Client getClient() throws Exception {
+		TrustManager[] trustAllCerts = new TrustManager[]{new X509TrustManager() {
+			@Override
+			public X509Certificate[] getAcceptedIssuers() {
+				return null;
+			}
+
+			@Override
+			public void checkClientTrusted(X509Certificate[] certs, String authType) {
+			}
+
+			@Override
+			public void checkServerTrusted(X509Certificate[] certs, String authType) {
+			}
+		}};
+
+		try {
+			SSLContext sc = SSLContext.getInstance("TLS");
+			sc.init(null, trustAllCerts, new SecureRandom());
+			HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory());
+		} catch (Exception e) {
+			throw new Exception();
+		}
+
+		return Client.create();
+	}
+
+}

testing/backup-test-core/src/main/java/org/opengroup/osdu/backup/TestBase.java

+package org.opengroup.osdu.backup;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import org.opengroup.osdu.backup.util.AssetUtil;
+
+
+public class TestBase {
+
+	protected static HttpClient client;
+
+	protected static AssetUtil assetUtil;
+
+	public static Map<String, String> getCommonHeader() throws IOException {
+		return getHeaders(Config.getDataPartitionId(), client.getAccessToken());
+	}
+
+	public static Map<String, String> getHeaders(String dataPartition, String token) {
+		Map<String, String> headers = new HashMap<>();