Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
Menu
Open sidebar
Open Subsurface Data Universe Software
Platform
Deployment and Operations
Backup Service
Commits
8fa4ea8d
Commit
8fa4ea8d
authored
Feb 03, 2021
by
Rustam Lotsmanenko (EPAM)
Browse files
GONRG-1638 remove obsolete methods
parent
e34026d3
Changes
13
Hide whitespace changes
Inline
Side-by-side
backup-core/src/main/java/org/opengroup/osdu/backup/manager/AssetBackupManager.java
View file @
8fa4ea8d
...
...
@@ -25,8 +25,6 @@ public interface AssetBackupManager {
BackupStamp
exportBackupWithResult
(
BackupSchedule
backupSchedule
);
void
exportBackup
(
BackupSchedule
backupSchedule
);
BackupStamp
importBackup
(
String
backupId
);
void
deleteBackups
(
List
<
BackupStamp
>
backupStamps
);
...
...
backup-core/src/main/java/org/opengroup/osdu/backup/repository/BackupStampRepository.java
View file @
8fa4ea8d
...
...
@@ -31,7 +31,7 @@ public interface BackupStampRepository {
EnumMap
<
Asset
,
List
<
BackupStamp
>>
listAvailableBackupStamps
();
void
deleteBackupStamps
(
List
<
BackupStamp
>
currentDate
);
List
<
BackupStamp
>
deleteBackupStamps
(
List
<
BackupStamp
>
currentDate
);
BackupStamp
findBackupByStampId
(
String
backupStampId
);
}
backup-core/src/main/java/org/opengroup/osdu/backup/shedulers/BackupScheduledTask.java
deleted
100644 → 0
View file @
e34026d3
/*
Copyright 2020 Google LLC
Copyright 2020 EPAM Systems, Inc
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package
org.opengroup.osdu.backup.shedulers
;
import
lombok.RequiredArgsConstructor
;
import
org.opengroup.osdu.backup.manager.AssetBackupManager
;
import
org.opengroup.osdu.backup.model.BackupSchedule
;
@RequiredArgsConstructor
public
class
BackupScheduledTask
implements
Runnable
{
private
final
AssetBackupManager
assetBackupManager
;
private
final
BackupSchedule
backupSchedule
;
@Override
public
void
run
()
{
assetBackupManager
.
exportBackup
(
backupSchedule
);
}
}
backup-core/src/main/java/org/opengroup/osdu/backup/shedulers/impl/SchedulerServiceImpl.java
View file @
8fa4ea8d
...
...
@@ -77,8 +77,13 @@ public class SchedulerServiceImpl implements SchedulerService {
ScheduledFuture
<?>
schedule
=
poolTaskScheduler
.
schedule
(
collectionScheduledTask
,
periodicTrigger
);
collectionTaskMap
.
put
(
collection
,
schedule
);
log
.
info
(
"Task for collection : {} added to scheduler, backup interval : {} , backup lifetime {}"
,
collection
.
getCollectionName
(),
collection
.
getInterval
(),
collection
.
getLifetime
());
log
.
info
(
"Task for collection : {} added to scheduler, backup interval : {} {}, backup lifetime {} {}"
,
collection
.
getCollectionName
(),
collection
.
getInterval
(),
schedulersProperties
.
getBackupTimeUnit
(),
collection
.
getLifetime
(),
schedulersProperties
.
getTearDownTimeUnit
()
);
}
@Override
...
...
provider/backup-gcp/src/main/java/org/opengroup/osdu/backup/provider/gcp/manager/DatastoreBackupManager.java
View file @
8fa4ea8d
...
...
@@ -40,15 +40,16 @@ import java.util.stream.Collectors;
import
java.util.stream.StreamSupport
;
import
lombok.RequiredArgsConstructor
;
import
lombok.extern.slf4j.Slf4j
;
import
org.opengroup.osdu.backup.config.property.SchedulersProperties
;
import
org.opengroup.osdu.backup.exception.ScheduleException
;
import
org.opengroup.osdu.backup.manager.AssetBackupManager
;
import
org.opengroup.osdu.backup.model.BackupSchedule
;
import
org.opengroup.osdu.backup.model.BackupStamp
;
import
org.opengroup.osdu.backup.provider.gcp.config.property.PropertiesConfiguration
;
import
org.opengroup.osdu.backup.provider.gcp.mapper.BackupScheduleMapper
;
import
org.opengroup.osdu.backup.provider.gcp.mapper.BackupStampMapper
;
import
org.opengroup.osdu.backup.provider.gcp.model.constant.Constants
;
import
org.opengroup.osdu.backup.provider.gcp.model.entity.DatastoreTypeBackupStampEntity
;
import
org.opengroup.osdu.backup.provider.gcp.model.entity.DatastoreTypeScheduleEntity
;
import
org.opengroup.osdu.backup.repository.BackupStampRepository
;
import
org.opengroup.osdu.core.common.model.http.AppException
;
import
org.springframework.http.HttpStatus
;
...
...
@@ -68,22 +69,23 @@ public class DatastoreBackupManager implements AssetBackupManager {
private
final
Storage
storage
;
private
final
SchedulersProperties
schedulersProperties
;
private
final
Datastore
datastore
;
private
final
BackupStampMapper
backupStampMapper
;
private
final
BackupScheduleMapper
backupScheduleMapper
;
@Retryable
(
value
=
ScheduleException
.
class
,
maxAttempts
=
3
,
backoff
=
@Backoff
(
delay
=
60000
))
@Override
public
BackupStamp
exportBackupWithResult
(
BackupSchedule
backupSchedule
)
{
String
namespace
=
backupSchedule
.
getAssetContext
().
get
(
Constants
.
CONTEXT_NAMESPACE
);
String
kind
=
backupSchedule
.
getAssetContext
().
get
(
Constants
.
CONTEXT_KIND
);
DatastoreTypeScheduleEntity
scheduleEntity
=
backupScheduleMapper
.
toDatastoreEntity
(
backupSchedule
);
log
.
info
(
"Starting backup export for namespace : {} kind : {}"
,
namespace
,
kind
);
log
.
info
(
"Starting backup export for namespace : {} kind : {}"
,
scheduleEntity
.
getNamespace
(),
scheduleEntity
.
getKind
());
try
{
GoogleDatastoreAdminV1ExportEntitiesRequest
context
=
buildExportEntitiesRequest
(
backupS
chedule
);
GoogleDatastoreAdminV1ExportEntitiesRequest
context
=
buildExportEntitiesRequest
(
s
chedule
Entity
);
GoogleLongrunningOperation
exportTaskCreationResponse
=
datastore
.
projects
()
.
export
(
propertiesConfiguration
.
getProjectId
(),
context
)
...
...
@@ -96,8 +98,8 @@ public class DatastoreBackupManager implements AssetBackupManager {
Object
outputUrlPrefix
=
exportTaskResultResponse
.
getMetadata
().
get
(
Constants
.
DATASTORE_OUTPUT_URL_PREFIX
);
log
.
info
(
"Export
status done
: {} backup path : {}
export : {}
"
,
exportTaskResultResponse
.
getDone
()
,
outputUrlPrefix
,
exportTaskResultResponse
);
log
.
info
(
"Export
result
: {} backup path : {} "
,
exportTaskResultResponse
,
outputUrlPrefix
);
return
buildBackupStamp
(
backupSchedule
,
outputUrlPrefix
.
toString
(),
true
);
...
...
@@ -107,41 +109,6 @@ public class DatastoreBackupManager implements AssetBackupManager {
}
}
@Override
@Retryable
(
value
=
ScheduleException
.
class
,
maxAttempts
=
3
,
backoff
=
@Backoff
(
delay
=
60000
))
public
void
exportBackup
(
BackupSchedule
backupSchedule
)
{
String
namespace
=
backupSchedule
.
getAssetContext
().
get
(
Constants
.
CONTEXT_NAMESPACE
);
String
kind
=
backupSchedule
.
getAssetContext
().
get
(
Constants
.
CONTEXT_KIND
);
log
.
info
(
"Starting backup export for namespace : {} kind : {}"
,
namespace
,
kind
);
try
{
GoogleDatastoreAdminV1ExportEntitiesRequest
context
=
buildExportEntitiesRequest
(
backupSchedule
);
GoogleLongrunningOperation
exportTaskCreationResponse
=
datastore
.
projects
()
.
export
(
propertiesConfiguration
.
getProjectId
(),
context
)
.
execute
();
Operations
dataStoreOperations
=
datastore
.
projects
().
operations
();
GoogleLongrunningOperation
exportTaskResultResponse
=
dataStoreOperations
.
get
(
exportTaskCreationResponse
.
getName
()).
execute
();
Object
outputUrlPrefix
=
exportTaskResultResponse
.
getMetadata
().
get
(
Constants
.
DATASTORE_OUTPUT_URL_PREFIX
);
log
.
info
(
"Export status done : {} backup path : {} export : {}"
,
exportTaskResultResponse
.
getDone
(),
outputUrlPrefix
,
exportTaskResultResponse
);
BackupStamp
stamp
=
buildBackupStamp
(
backupSchedule
,
outputUrlPrefix
.
toString
(),
true
);
stampsRepository
.
submitBackupStamp
(
stamp
);
}
catch
(
Exception
e
)
{
log
.
error
(
e
.
getMessage
());
throw
new
ScheduleException
(
"Datastore export failure"
,
e
);
}
}
@Override
public
BackupStamp
importBackup
(
String
backupId
)
{
BackupStamp
backupByStampId
=
stampsRepository
.
findBackupByStampId
(
backupId
);
...
...
@@ -195,14 +162,8 @@ public class DatastoreBackupManager implements AssetBackupManager {
}
storageBatch
.
submit
();
stampsRepository
.
deleteBackupStamps
(
backupStamps
);
log
.
info
(
"Deleted backups:{}"
,
collect
);
}
@Recover
public
void
saveBackupStampOnFail
(
ScheduleException
e
,
BackupSchedule
backupSchedule
)
{
BackupStamp
build
=
buildBackupStamp
(
backupSchedule
,
"-"
,
false
);
stampsRepository
.
submitBackupStamp
(
build
);
List
<
BackupStamp
>
deletedStamps
=
stampsRepository
.
deleteBackupStamps
(
backupStamps
);
log
.
info
(
"Deleted backups:{}"
,
deletedStamps
);
}
@Recover
...
...
@@ -225,25 +186,25 @@ public class DatastoreBackupManager implements AssetBackupManager {
}
private
GoogleDatastoreAdminV1ExportEntitiesRequest
buildExportEntitiesRequest
(
Backup
Schedule
backupSchedule
)
{
DatastoreType
Schedule
Entity
backupSchedule
)
{
GoogleDatastoreAdminV1EntityFilter
googleDatastoreAdminV1EntityFilter
=
new
GoogleDatastoreAdminV1EntityFilter
();
googleDatastoreAdminV1EntityFilter
.
setKinds
(
Collections
.
singletonList
(
backupSchedule
.
get
AssetContext
().
get
(
Constants
.
CONTEXT_KIND
)));
.
setKinds
(
Collections
.
singletonList
(
backupSchedule
.
get
Kind
(
)));
googleDatastoreAdminV1EntityFilter
.
setNamespaceIds
(
Collections
.
singletonList
(
backupSchedule
.
get
AssetContext
().
get
(
Constants
.
CONTEXT_NAMESPACE
)));
Collections
.
singletonList
(
backupSchedule
.
get
Namespace
(
)));
GoogleDatastoreAdminV1ExportEntitiesRequest
context
=
new
GoogleDatastoreAdminV1ExportEntitiesRequest
();
context
.
setEntityFilter
(
googleDatastoreAdminV1EntityFilter
);
context
.
setOutputUrlPrefix
(
getOutputUrlPrefix
(
backupSchedule
));
return
context
;
}
private
String
getOutputUrlPrefix
(
Backup
Schedule
backupSchedule
)
{
private
String
getOutputUrlPrefix
(
DatastoreType
Schedule
Entity
backupSchedule
)
{
return
Constants
.
GCP_STORAGE_PREFIX
+
propertiesConfiguration
.
getBackupBucket
()
+
"/datastore/"
+
backupSchedule
.
get
AssetContext
().
get
(
Constants
.
CONTEXT_NAMESPACE
)
.
get
Namespace
(
)
+
"-"
+
backupSchedule
.
get
AssetContext
().
get
(
Constants
.
CONTEXT_KIND
)
+
"/"
.
get
Kind
(
)
+
"/"
+
Timestamp
.
now
();
}
...
...
provider/backup-gcp/src/main/java/org/opengroup/osdu/backup/provider/gcp/manager/ElasticBackupManager.java
View file @
8fa4ea8d
...
...
@@ -35,11 +35,6 @@ public class ElasticBackupManager implements AssetBackupManager {
throw
new
UnsupportedOperationException
();
}
@Override
public
void
exportBackup
(
BackupSchedule
backupSchedule
)
{
throw
new
UnsupportedOperationException
();
}
@Override
public
BackupStamp
importBackup
(
String
backupId
)
{
throw
new
UnsupportedOperationException
();
...
...
provider/backup-gcp/src/main/java/org/opengroup/osdu/backup/provider/gcp/manager/SQLBackupManager.java
View file @
8fa4ea8d
...
...
@@ -55,8 +55,6 @@ public class SQLBackupManager implements AssetBackupManager {
private
final
BackupStampRepository
stampsRepository
;
private
final
SchedulersProperties
schedulersProperties
;
private
final
PropertiesConfiguration
propertiesConfiguration
;
private
static
final
String
BACKUP_DESCRIPTION
=
"Created by backup-service. Id: "
;
...
...
@@ -92,29 +90,6 @@ public class SQLBackupManager implements AssetBackupManager {
return
buildBackupStamp
(
backupSchedule
,
0L
,
false
);
}
@Override
public
void
exportBackup
(
BackupSchedule
backupSchedule
)
{
String
instance
=
backupSchedule
.
getAssetContext
().
get
(
Constants
.
CONTEXT_INSTANCE
);
Long
backupId
=
new
Random
().
nextLong
();
log
.
info
(
"Starting backup for instance : {} "
,
instance
);
BackupRun
backupRun
=
getBackupRun
(
instance
,
backupId
);
try
{
SQLAdmin
sqlAdmin
=
new
SQLAdmin
(
Utils
.
getDefaultTransport
(),
Utils
.
getDefaultJsonFactory
(),
createCredentials
());
sqlAdmin
.
backupRuns
()
.
insert
(
propertiesConfiguration
.
getProjectId
(),
instance
,
backupRun
).
execute
();
BackupStamp
stamp
=
buildBackupStamp
(
backupSchedule
,
backupId
,
true
);
stampsRepository
.
submitBackupStamp
(
stamp
);
log
.
info
(
"Export status done. Backup id : {}"
,
"backupId"
);
}
catch
(
IOException
e
)
{
log
.
error
(
e
.
getMessage
());
throw
new
ScheduleException
(
"SQL export failure"
,
e
);
}
}
@Override
public
BackupStamp
importBackup
(
String
stampId
)
{
log
.
info
(
"Starting backup restore for stampId : {}"
,
stampId
);
...
...
provider/backup-gcp/src/main/java/org/opengroup/osdu/backup/provider/gcp/repository/BackupStampsEntityRepository.java
View file @
8fa4ea8d
...
...
@@ -31,5 +31,5 @@ public interface BackupStampsEntityRepository extends DatastoreRepository<Backup
List
<
BackupStampEntity
>
findByCollectionKey
(
CollectionStateEntity
collectionState
);
void
deleteByStampId
(
String
stampId
);
BackupStampEntity
deleteByStampId
(
String
stampId
);
}
provider/backup-gcp/src/main/java/org/opengroup/osdu/backup/provider/gcp/repository/DatastoreBackupStampRepository.java
View file @
8fa4ea8d
...
...
@@ -67,11 +67,13 @@ public class DatastoreBackupStampRepository implements BackupStampRepository {
}
@Override
public
void
deleteBackupStamps
(
List
<
BackupStamp
>
stampList
)
{
List
<
BackupStampEntity
>
stampEntities
=
stampList
.
stream
().
map
(
backupStampMapper:
:
toEntity
)
.
collect
(
Collectors
.
toList
());
backupStampsEntityRepository
.
deleteAll
(
stampEntities
);
public
List
<
BackupStamp
>
deleteBackupStamps
(
List
<
BackupStamp
>
stampList
)
{
return
backupStampsEntityRepository
.
performTransaction
(
transactionalRequest
->
stampList
.
stream
().
map
(
backupStampMapper:
:
toEntity
)
.
collect
(
Collectors
.
toList
())
.
stream
()
.
map
(
backupStampEntity
->
backupStampsEntityRepository
.
deleteByStampId
(
backupStampEntity
.
getStampId
()))
.
map
(
backupStampMapper:
:
toBackupStamp
)
.
collect
(
Collectors
.
toList
()));
}
@Override
...
...
provider/backup-gcp/src/test/java/org/opengroup/osdu/backup/provider/gcp/manager/DatastoreBackupManagerTest.java
deleted
100644 → 0
View file @
e34026d3
/*
* Copyright 2020 Google LLC
* Copyright 2020 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package
org.opengroup.osdu.backup.provider.gcp.manager
;
import
static
org
.
junit
.
Assert
.*;
import
org.junit.Test
;
public
class
DatastoreBackupManagerTest
{
@Test
public
void
exportBackupWithResult
()
{
}
@Test
public
void
exportBackup
()
{
}
@Test
public
void
importBackup
()
{
}
@Test
public
void
deleteBackups
()
{
}
@Test
public
void
saveBackupStampOnFail
()
{
}
@Test
public
void
returnBackupStampOnFail
()
{
}
}
\ No newline at end of file
provider/backup-gcp/src/test/java/org/opengroup/osdu/backup/provider/gcp/manager/SQLBackupManagerTest.java
deleted
100644 → 0
View file @
e34026d3
/*
* Copyright 2020 Google LLC
* Copyright 2020 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package
org.opengroup.osdu.backup.provider.gcp.manager
;
import
static
org
.
junit
.
Assert
.*;
import
org.junit.Test
;
public
class
SQLBackupManagerTest
{
@Test
public
void
exportBackupWithResult
()
{
}
@Test
public
void
returnBackupStampOnFail
()
{
}
@Test
public
void
exportBackup
()
{
}
@Test
public
void
importBackup
()
{
}
@Test
public
void
deleteBackups
()
{
}
}
\ No newline at end of file
provider/backup-gcp/src/test/java/org/opengroup/osdu/backup/provider/gcp/repository/DatastoreBackupStampRepositoryTest.java
deleted
100644 → 0
View file @
e34026d3
/*
* Copyright 2020 Google LLC
* Copyright 2020 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package
org.opengroup.osdu.backup.provider.gcp.repository
;
import
org.junit.Test
;
import
org.junit.runner.RunWith
;
import
org.opengroup.osdu.backup.provider.gcp.mapper.BackupStampMapper
;
import
org.opengroup.osdu.backup.provider.gcp.mapper.BackupStampMapperImpl
;
import
org.opengroup.osdu.backup.provider.gcp.mapper.CollectionStateMapper
;
import
org.opengroup.osdu.backup.provider.gcp.mapper.CollectionStateMapperImpl
;
import
org.opengroup.osdu.backup.provider.gcp.mapper.util.MapUtil
;
import
org.opengroup.osdu.backup.repository.BackupStampRepository
;
import
org.springframework.beans.factory.annotation.Autowired
;
import
org.springframework.boot.test.context.SpringBootTest
;
import
org.springframework.boot.test.mock.mockito.MockBean
;
import
org.springframework.test.context.junit4.SpringRunner
;
@RunWith
(
SpringRunner
.
class
)
@SpringBootTest
(
classes
=
{
BackupStampMapperImpl
.
class
,
CollectionStateMapperImpl
.
class
,
MapUtil
.
class
,
DatastoreBackupStampRepository
.
class
})
public
class
DatastoreBackupStampRepositoryTest
{
@MockBean
BackupStampsEntityRepository
backupStampsEntityRepository
;
@Autowired
BackupStampMapper
backupStampMapper
;
@Autowired
CollectionStateMapper
collectionStateMapper
;
@Autowired
BackupStampRepository
backupStampRepository
;
@Test
public
void
findBackupStampsByCollectionState
()
{
}
@Test
public
void
submitBackupStamp
()
{
}
@Test
public
void
listAvailableBackupStamps
()
{
}
@Test
public
void
deleteBackupStamps
()
{
}
@Test
public
void
findBackupByStampId
()
{
}
}
\ No newline at end of file
provider/backup-gcp/src/test/java/org/opengroup/osdu/backup/provider/gcp/repository/DatastoreCollectionStateRepositoryTest.java
deleted
100644 → 0
View file @
e34026d3
/*
* Copyright 2020 Google LLC
* Copyright 2020 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package
org.opengroup.osdu.backup.provider.gcp.repository
;
import
static
org
.
junit
.
Assert
.*;
import
org.junit.Test
;
public
class
DatastoreCollectionStateRepositoryTest
{
@Test
public
void
getAllCollectionsStates
()
{
}
@Test
public
void
getCollectionStates
()
{
}
@Test
public
void
getCollectionsStatesByCollectionId
()
{
}
@Test
public
void
getStateById
()
{
}
@Test
public
void
submitCollectionState
()
{
}
@Test
public
void
deleteCollectionStates
()
{
}
}
\ No newline at end of file
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment