Commit e04b8189 authored by snehal jagtap's avatar snehal jagtap
Browse files

initial commit

parent 64976ee6
Pipeline #62035 failed with stages
in 3 minutes and 7 seconds
package org.opengroup.osdu.production.restore
import akka.actor.ActorSystem
import akka.stream.ActorMaterializer
import com.google.cloud.bigtable.data.v2.models.Mutation
import com.google.cloud.pubsub.v1.SubscriptionAdminClient
import com.google.protobuf.ByteString
import com.google.pubsub.v1.PubsubMessage
import org.scalactic.ErrorMessage
import java.util.concurrent.Executors
import scala.concurrent.ExecutionContext
object GlobalVariables {
lazy val checkpointTime = System.currentTimeMillis()
lazy val backupFolderName = "backup"
lazy val restoreFolderName = "restore"
lazy val googleProjectName = "GOOGLE_PROJECT_NAME"
lazy val globalGoogleProjectName = "GLOBAL_GOOGLE_PROJECT_NAME"
lazy val restoreEnvName = "RESTORE_TENANT_NAME"
lazy val backupEnvName = "BACKUP_TENANT_NAME"
lazy implicit val actorSystem = ActorSystem("backup-actor-system")
lazy implicit val mat = ActorMaterializer()
implicit lazy val ec = ExecutionContext.fromExecutor(Executors.newFixedThreadPool(2))
implicit lazy val pubsubClient = SubscriptionAdminClient.create()
type RowKeyAndMutation = (RowKeyString, Mutation)
type RowKeyString = ByteString
type TableWithBlobId = (String, String)
type ErrorWithAckId = (AckId, ErrorMessage)
type AckId = String
type MessageWithAckId = (AckId, PubsubMessage)
type BigTableBackupMetadataWithAckId = (AckId, TableWithBlobId)
}
package org.opengroup.osdu.production.restore
import com.google.cloud.bigtable.grpc.BigtableSession
import org.opengroup.osdu.production.config.BigTableConfiguration
import org.opengroup.osdu.production.domain.generated.DataPointsAddedMessage.StreamDatum
import org.opengroup.osdu.production.restore.GlobalVariables._
import org.opengroup.osdu.production.restore.config.RestoreServiceConfiguration
import org.opengroup.osdu.production.restore.restore.dao.BigTableSettings
import org.opengroup.osdu.production.restore.restore.jobs.RestoreData
import org.opengroup.osdu.production.storage.BigTableMigration
import com.typesafe.config.{ Config, ConfigFactory, ConfigRenderOptions }
import com.typesafe.scalalogging.LazyLogging
import org.opengroup.osdu.production.restore.restore.ProtocolConverters
import java.io.File
import scala.util.{ Failure, Success, Try }
object RestoreDataJob extends App with LazyLogging {
val conf = replaceEnvironmentVariablesInConf(ConfigFactory.parseFile(new File(args(0))))
implicit val sc = RestoreServiceConfiguration(conf)
implicit val btSettings = bigTableSettings(sc.bigTableConfiguration, ProtocolConverters.dataPointGenerator(_, sc.bigTableConfiguration), ProtocolConverters.indexGenerator(_, sc.bigTableConfiguration))
BigTableMigration.runMigration(conf)(btSettings.session)
Try(RestoreData(conf).start()) match {
case Success(_) => logger.info("RestoreData Job Completed")
case Failure(x) => logger.error(s"exception in restoration", x)
}
GlobalVariables.actorSystem.terminate().foreach(_ => System.exit(0))
def bigTableSettings(configWrapper: BigTableConfiguration, dataGen: StreamDatum => RowKeyAndMutation, indexGen: StreamDatum => RowKeyAndMutation): BigTableSettings[Seq[StreamDatum]] = {
BigTableSettings[Seq[StreamDatum]](new BigtableSession(configWrapper.opts), configWrapper, liftConverter(dataGen), liftConverter(indexGen))
}
def liftConverter(converter: StreamDatum => RowKeyAndMutation): Seq[StreamDatum] => Seq[RowKeyAndMutation] = _.map(converter)
def replaceEnvironmentVariablesInConf(conf: Config) = {
ConfigFactory.parseString(conf.root().render(ConfigRenderOptions.concise()).replaceAll("ENV-TAG", System.getenv(restoreEnvName))
.replaceAll("GOOGLE-PROJECT-NAME", System.getenv(googleProjectName)))
}
}
package org.opengroup.osdu.production.restore
import com.google.pubsub.v1.PushConfig
import org.opengroup.osdu.production.restore.GlobalVariables._
import org.opengroup.osdu.production.restore.config.RestoreServiceConfiguration
import org.opengroup.osdu.production.restore.restore.jobs.RestoreMetadataPublish
import com.typesafe.config.{ Config, ConfigFactory, ConfigRenderOptions }
import com.typesafe.scalalogging.LazyLogging
import java.io.File
import java.util.concurrent.TimeUnit
import scala.concurrent.duration.Duration
import scala.concurrent.{ Await, Future }
import scala.util.{ Failure, Success, Try }
object RestoreMetadataPublishJob extends App with LazyLogging {
val conf = replaceEnvironmentVariablesInConf(ConfigFactory.parseFile(new File(args(0))))
implicit val sc = RestoreServiceConfiguration(conf)
Try(pubsubClient.getSubscription(sc.subscription)).toOption.getOrElse(
pubsubClient.createSubscription(sc.subscription, sc.topic, PushConfig.newBuilder().build(), 600))
val jobtrigger: Future[Seq[String]] = RestoreMetadataPublish(conf).start(sc.backupProjectBucketName, sc.backupTable)
Try(Await.ready(jobtrigger, Duration(1, TimeUnit.HOURS))) match {
case Success(_) =>
logger.info("RestoreMetadataPublish Job Completed Successfully")
System.exit(0)
case Failure(e) => {
logger.error("Error in RestoreData", e)
System.exit(-1)
}
}
def replaceEnvironmentVariablesInConf(conf: Config) = {
ConfigFactory.parseString(conf.root().render(ConfigRenderOptions.concise()).replaceAll("ENV-TAG", System.getenv(restoreEnvName))
.replaceAll("GOOGLE-PROJECT-NAME", System.getenv(googleProjectName)))
}
}
package org.opengroup.osdu.production.restore.config
import com.google.cloud.bigtable.data.v2.{ BigtableDataClient, BigtableDataSettings }
import com.google.cloud.pubsub.v1.Publisher
import com.google.pubsub.v1.{ ProjectSubscriptionName, ProjectTopicName, TopicName }
import org.opengroup.osdu.production.config.BigTableConfiguration
import org.opengroup.osdu.production.restore.GlobalVariables._
import com.typesafe.config.Config
case class RestoreServiceConfiguration(bigTableConfiguration: BigTableConfiguration, googleProjectName: String, globalGoogleProjectName: String, topic: ProjectTopicName, subscription: ProjectSubscriptionName, publisher: Publisher, bigTableDataClient: BigtableDataClient, backupProjectBucketName: String, restoreProjectBucketName: String, backupTable: String, restoreTable: String)
object RestoreServiceConfiguration {
def apply(conf: Config): RestoreServiceConfiguration = RestoreServiceConfiguration(
BigTableConfiguration(conf),
System.getenv(googleProjectName),
System.getenv(globalGoogleProjectName),
ProjectTopicName.of(System.getenv(globalGoogleProjectName), conf.getString("cloud.pubsub.topic")),
ProjectSubscriptionName.of(System.getenv(globalGoogleProjectName), conf.getString("cloud.pubsub.subscription")),
PublisherCache(System.getenv(globalGoogleProjectName), conf.getString("cloud.pubsub.topic")).topicPublisher,
BigtableDataClient.
create(BigtableDataSettings.newBuilder()
.setProjectId(System.getenv(googleProjectName))
.setInstanceId(conf.getString("cloud.bigtable.instance.name"))
.setAppProfileId("restore")
.build()),
s"${System.getenv(backupEnvName)}-prodops-backup-bucket",
s"${System.getenv(restoreEnvName)}-prodops-backup-bucket",
s"ts-${System.getenv(backupEnvName)}-v2",
s"ts-${System.getenv(restoreEnvName)}-v2")
case class PublisherCache(projectId: String, topicId: String) {
lazy val topicPublisher = publisher(ProjectTopicName.of(projectId, topicId))
def publisher: (TopicName) => Publisher = topicName => Publisher.newBuilder(topicName).build()
}
}
package org.opengroup.osdu.production.restore.restore
import com.google.cloud.bigtable.data.v2.BigtableDataClient
import com.google.cloud.bigtable.data.v2.models.BulkMutation
import org.opengroup.osdu.production.domain.generated.DataPointsAddedMessage.StreamDatum
import org.opengroup.osdu.production.restore.GlobalVariables.RowKeyAndMutation
import org.opengroup.osdu.production.restore.config.RestoreServiceConfiguration
import org.opengroup.osdu.production.restore.restore.dao.{ BigTableSettings, DataAndIndexRowGeneratorsWithTimeSeriesBackupRecords }
import com.typesafe.scalalogging.LazyLogging
object BigTableHandler extends LazyLogging {
def createDataAndIndexRowGenerators(timeSeriesRecords: Seq[StreamDatum], settings: BigTableSettings[Seq[StreamDatum]])(implicit sc: RestoreServiceConfiguration): DataAndIndexRowGeneratorsWithTimeSeriesBackupRecords = {
val dataRowGenerator = settings.dataRowGenerator(timeSeriesRecords)
val indexRowGenerator = settings.indexRowGenerator(timeSeriesRecords)
if (dataRowGenerator.size != indexRowGenerator.size) throw new RuntimeException("Row data and Row Index data point length do not match")
DataAndIndexRowGeneratorsWithTimeSeriesBackupRecords(dataRowGenerator, indexRowGenerator)
}
def executeDataAndIndexBatches(dataAndIndexRowGeneratorsWithTimeSeriesBackupRecords: DataAndIndexRowGeneratorsWithTimeSeriesBackupRecords, settings: BigTableSettings[Seq[StreamDatum]])(implicit sc: RestoreServiceConfiguration): Unit = {
val client = sc.bigTableDataClient
mutateRows(dataAndIndexRowGeneratorsWithTimeSeriesBackupRecords.dataRows, settings.config.dataTableName, client)
mutateRows(dataAndIndexRowGeneratorsWithTimeSeriesBackupRecords.indexRows, settings.config.indexTableName, client)
logger.trace(s"Successfully inserted Data & Index for ${dataAndIndexRowGeneratorsWithTimeSeriesBackupRecords.dataRows} datapoints")
}
private def mutateRows(rowBatch: Seq[RowKeyAndMutation], tableName: String, client: BigtableDataClient) = {
rowBatch.grouped(99999).foreach { rows =>
val bulkDataMutation = BulkMutation.create(tableName)
rows.foreach(pair => bulkDataMutation.add(pair._1, pair._2))
client.bulkMutateRows(bulkDataMutation)
}
}
}
package org.opengroup.osdu.production.restore.restore
import com.google.cloud.bigtable.data.v2.models.{ Mutation => NewMutation }
import com.google.common.primitives.{ Ints, Longs }
import com.google.protobuf.ByteString
import org.opengroup.osdu.production.config.BigTableConfiguration
import org.opengroup.osdu.production.domain.generated.DataPointsAddedMessage.{ BigTableRecord, StreamDatum }
import org.opengroup.osdu.production.restore.GlobalVariables.RowKeyAndMutation
import org.opengroup.osdu.production.{ dataOffset, indexOffset }
import com.typesafe.scalalogging.LazyLogging
object ProtocolConverters extends LazyLogging {
def dataPointGenerator(v: StreamDatum, config: BigTableConfiguration): RowKeyAndMutation = {
(
ByteString.copyFrom(Longs.toByteArray(v.getStreamId) ++ Longs.toByteArray(v.getPhysicalTime + dataOffset)),
NewMutation.create().setCell(config.columnFamilies.head, ByteString.copyFrom(Longs.toByteArray(dataOffset - v.getVersionTime)), ByteString.copyFrom(convertValueToBytes(v.getRecord))))
}
def indexGenerator(v: StreamDatum, config: BigTableConfiguration): RowKeyAndMutation = {
(
ByteString.copyFrom(Longs.toByteArray(v.getStreamId) ++ Longs.toByteArray(indexOffset - v.getPhysicalTime)),
NewMutation.create().setCell(config.columnFamilies.head, ByteString.copyFrom(Longs.toByteArray(dataOffset - v.getVersionTime)), ByteString.copyFrom(Ints.toByteArray(v.getRecord.getStatusCode))))
}
def convertValueToBytes(value: BigTableRecord): Array[Byte] = {
value.toByteArray
}
}
package org.opengroup.osdu.production.restore.restore
import com.google.cloud.ReadChannel
import com.google.cloud.storage.Storage.BlobListOption
import com.google.cloud.storage.{ BlobId, BlobInfo, StorageOptions }
import com.google.common.primitives.Ints
import org.opengroup.osdu.production.domain.generated.DataPointsAddedMessage.StreamDatum
import org.opengroup.osdu.production.restore.GlobalVariables
import org.opengroup.osdu.production.restore.GlobalVariables.{ BigTableBackupMetadataWithAckId, _ }
import org.opengroup.osdu.production.restore.config.RestoreServiceConfiguration
import org.opengroup.osdu.production.restore.restore.dao.BigTableSettings
import com.typesafe.scalalogging.LazyLogging
import org.scalactic.{ Bad, Good, Or }
import java.io.IOException
import java.nio.ByteBuffer
import scala.collection.JavaConverters._
import scala.concurrent.Future
import scala.{ Option => ScalaOption }
object StorageHandler extends LazyLogging {
val storageService = StorageOptions.getDefaultInstance.getService
val intByteSize = 4
def readBlobAndWriteToBigTable(blobId: BlobId, reader: ReadChannel, bigTableBackupMetadataWithAckId: BigTableBackupMetadataWithAckId)(implicit
settings: BigTableSettings[Seq[StreamDatum]],
sc: RestoreServiceConfiguration): Future[Or[BigTableBackupMetadataWithAckId, GlobalVariables.AckId]] = {
Future {
writeToBigTable(blobId, reader, bigTableBackupMetadataWithAckId)
}
}
def writeToBigTable(blobId: BlobId, reader: ReadChannel, bigTableBackupMetadataWithAckId: BigTableBackupMetadataWithAckId)(implicit
settings: BigTableSettings[Seq[StreamDatum]],
sc: RestoreServiceConfiguration) = {
try {
val blobSize = storageService.get(blobId).getSize
var localReader = reader
var bufferPosition = 0L
var chunkStart = 0l
val chunkSize = 1000000l
var pendingBytes = Array.emptyByteArray
localReader.setChunkSize(chunkSize.toInt)
while (bufferPosition <= blobSize - 1) {
if (blobSize - bufferPosition < 1000000L) {
logger.info(s"last million bytes!!!! for ${blobId.getName}")
}
val chunkEnd = chunkStart + Math.min(blobSize - bufferPosition, chunkSize)
val (readerState, contentByteArray) = readContentRange(chunkStart, chunkEnd, reader)
pendingBytes = writeToBigTableRecordsAndReturnPendingByteArray(pendingBytes ++ contentByteArray)
chunkStart = chunkEnd
bufferPosition = bufferPosition + contentByteArray.size
localReader = readerState.restore()
}
logger.info(s"finished blob ${bigTableBackupMetadataWithAckId._2._2}");
} catch {
case e: Throwable => logger.error(s"bigtable write failed, exiting the process", e); System.exit(-1); Bad("")
} finally {
if (reader != null) reader.close()
}
updateRestoredFiles(sc.restoreProjectBucketName, sc.restoreTable, blobId)
Good(bigTableBackupMetadataWithAckId)
}
private def writeToBigTableRecordsAndReturnPendingByteArray(contentByteArray: Array[Byte])(implicit settings: BigTableSettings[Seq[StreamDatum]], sc: RestoreServiceConfiguration): (Array[Byte]) = {
var bigTableRecords = Vector[StreamDatum]()
var currentPosition = 0
while (contentByteArray.size - (currentPosition + 1) >= intByteSize && Ints.fromByteArray(contentByteArray.slice(currentPosition, currentPosition + 4)) <= ((contentByteArray.size - 1) - (currentPosition + 4))) {
val nextContentLength = Ints.fromByteArray(contentByteArray.slice(currentPosition, currentPosition + 4))
currentPosition += 4
//logger.info(s"next content length is ${nextContentLength} position is ${currentPosition} and totalsize is ${contentByteArray.size}")
val nextContentArray = contentByteArray.slice(currentPosition, currentPosition + nextContentLength)
currentPosition += nextContentLength
bigTableRecords = bigTableRecords :+ StreamDatum.newBuilder().mergeFrom(nextContentArray).build()
}
if (bigTableRecords.nonEmpty) {
val bigtableRecordsWithDataAndIndexRowGenerators = BigTableHandler.createDataAndIndexRowGenerators(bigTableRecords, settings)
BigTableHandler.executeDataAndIndexBatches(bigtableRecordsWithDataAndIndexRowGenerators, settings)
}
contentByteArray.slice(currentPosition, contentByteArray.size)
}
@throws[IOException]
private def readContentRange(start: Long, end: Long, reader: ReadChannel) = { // [START readContentRange]
assert(end - start <= Int.MaxValue)
val bytes = ByteBuffer.allocate((end - start).asInstanceOf[Int])
reader.read(bytes)
(reader.capture(), bytes.array)
}
def getFilesToRestore(backupProjectBucketName: String, tableName: String)(implicit sc: RestoreServiceConfiguration) = {
val blobFromBackup = storageService.list(backupProjectBucketName, BlobListOption.prefix(s"$backupFolderName/${tableName}/data/"))
val blobAlreadyRestored = storageService.list(sc.restoreProjectBucketName, BlobListOption.prefix(s"$restoreFolderName/${sc.restoreTable}/completed/"))
val blobNamesOfBackUps = blobFromBackup.iterateAll().asScala.map(blobId => (blobId.getName.slice(blobId.getName.lastIndexOf('/') + 1, blobId.getName.size))).toSeq
val blobNamesOfCompletedRestores = blobAlreadyRestored.iterateAll().asScala.map(blobId => (blobId.getName.slice(blobId.getName.lastIndexOf('/') + 1, blobId.getName.size))).toSeq
val blobs = blobNamesOfBackUps.diff(blobNamesOfCompletedRestores)
logger.info(blobs.toString)
blobs
}
def updateRestoredFiles(bucketName: String, table: String, blobId: BlobId)(implicit sc: RestoreServiceConfiguration) = {
val blobToMarkAsRestored = BlobId.of(bucketName, s"$restoreFolderName/${table}/completed/${blobId.getName.slice(blobId.getName.lastIndexOf('/') + 1, blobId.getName.size)}")
val blobInfoToMarkAsRestored = BlobInfo.newBuilder(blobToMarkAsRestored).build
if (!ScalaOption(storageService.get(blobToMarkAsRestored)).isDefined) {
storageService.create(blobInfoToMarkAsRestored)
}
}
case class PendingBytesWithBigTableRecords(pendingBytes: Array[Byte], bigTableRecords: Seq[StreamDatum])
}
package org.opengroup.osdu.production.restore.restore.dao
import com.google.cloud.bigtable.grpc.BigtableSession
import org.opengroup.osdu.production.config.BigTableConfiguration
import org.opengroup.osdu.production.restore.GlobalVariables.RowKeyAndMutation
import org.opengroup.osdu.production.config.BigTableConfiguration
case class BigTableSettings[T](
session: BigtableSession,
config: BigTableConfiguration,
dataRowGenerator: T => Seq[RowKeyAndMutation],
indexRowGenerator: T => Seq[RowKeyAndMutation],
maxMessageSize: Int = 50000)
package org.opengroup.osdu.production.restore.restore.dao
import org.opengroup.osdu.production.restore.GlobalVariables.RowKeyAndMutation
case class DataAndIndexRowGeneratorsWithTimeSeriesBackupRecords(dataRows: Seq[RowKeyAndMutation], indexRows: Seq[RowKeyAndMutation])
package org.opengroup.osdu.production.restore.restore.jobs
import com.google.api.gax.grpc.GrpcCallContext
import com.google.cloud.ReadChannel
import com.google.cloud.storage.BlobId
import com.google.pubsub.v1.{ AcknowledgeRequest, ProjectSubscriptionName, PubsubMessage, PullRequest }
import org.opengroup.osdu.production.domain.generated.DataPointsAddedMessage.StreamDatum
import org.opengroup.osdu.production.restore.GlobalVariables.{ AckId, _ }
import org.opengroup.osdu.production.restore.config.RestoreServiceConfiguration
import org.opengroup.osdu.production.restore.restore.StorageHandler
import org.opengroup.osdu.production.restore.restore.StorageHandler.storageService
import org.opengroup.osdu.production.restore.restore.dao.BigTableSettings
import org.opengroup.osdu.production.restore.restore.pipeline.steps.{ MessageTranslatorStep, ValidateAlreadyRestoredBlobStep }
import com.typesafe.config.Config
import com.typesafe.scalalogging.LazyLogging
import io.grpc.CallOptions
import java.util.concurrent.TimeUnit
import scala.collection.JavaConverters._
import scala.concurrent.duration.Duration
import scala.concurrent.{ Await, ExecutionContext, Future }
import scala.util.Try
case class RestoreData(conf: Config) extends LazyLogging {
val parallelPulls: Int = Try(System.getenv("PARALLELISM-LEVEL").toInt).toOption.getOrElse(4)
def start()(implicit settings: BigTableSettings[Seq[StreamDatum]], sc: RestoreServiceConfiguration, ec: ExecutionContext) = {
restoreBigTableData()
}
def restoreBigTableData()(implicit settings: BigTableSettings[Seq[StreamDatum]], sc: RestoreServiceConfiguration, ec: ExecutionContext) = {
var pubsubMessages = getPubsubMessages()
while (pubsubMessages.nonEmpty) {
val blobsToProcess = processPubsubMessages(pubsubMessages).map(verifyBlobExistenceAndPermissions(_))
.filter { doesStorageFileExists => doesStorageFileExists.isDefined }.map(_.get)
Await.result(Future.sequence(blobsToProcess.map { case (blobId, reader, pair) => blobRestoreProcess(blobId, reader, pair) }), Duration.apply(6, TimeUnit.HOURS))
pubsubMessages = getPubsubMessages()
}
}
def verifyBlobExistenceAndPermissions(pair: BigTableBackupMetadataWithAckId)(implicit settings: BigTableSettings[Seq[StreamDatum]], sc: RestoreServiceConfiguration, ec: ExecutionContext) = {
val tableName = pair._2._1
val blobName = pair._2._2
val blobId = BlobId.of(sc.backupProjectBucketName, s"$backupFolderName/$tableName/data/$blobName")
logger.info(s"starting blob $blobName");
Option(storageService.get(blobId)) map { _ => (blobId, storageService.reader(blobId), pair) }
}
def processPubsubMessages(messages: Seq[(PubsubMessage, String)])(implicit sc: RestoreServiceConfiguration) = {
val partitionedMessages = messages.map(message => MessageTranslatorStep.protoDeserialize(message))
.partition(message => message.isGood)
partitionedMessages._2.map(message => acknowledgeMessage(message.get._1))
val alreadyProcessedMessagesPartition = partitionedMessages._1.map(message => ValidateAlreadyRestoredBlobStep.isBlobAlreadyRestored(message.get)).partition(message => message.isGood)
alreadyProcessedMessagesPartition._1.map(message => acknowledgeMessage(message.get))
val processedMessages: Seq[(AckId, (String, String))] = alreadyProcessedMessagesPartition._2.map(message => message.swap.get)
processedMessages
}
def blobRestoreProcess(blobId: BlobId, reader: ReadChannel, pair: BigTableBackupMetadataWithAckId)(implicit settings: BigTableSettings[Seq[StreamDatum]], sc: RestoreServiceConfiguration, ec: ExecutionContext) = {
acknowledgeMessage(pair._1)
StorageHandler.readBlobAndWriteToBigTable(blobId: BlobId, reader: ReadChannel, pair: BigTableBackupMetadataWithAckId)
}
def getPubsubMessages()(implicit sc: RestoreServiceConfiguration) = {
val subscriptionName = ProjectSubscriptionName.format(sc.globalGoogleProjectName, sc.subscription.getSubscription)
val pullRequest = PullRequest.newBuilder.setMaxMessages(1).setReturnImmediately(true).setSubscription(subscriptionName) // return immediately if messages are not available
.build
val pullResponse = pubsubClient.getStub.pullCallable().call(pullRequest, GrpcCallContext.createDefault().withCallOptions(CallOptions.DEFAULT))
pullResponse.getReceivedMessagesList.asScala.map(message => (message.getMessage, message.getAckId))
}
def acknowledgeMessage(ackId: AckId)(implicit sc: RestoreServiceConfiguration) = {
logger.info(s"Acknowledged message $ackId")
val acknowledgeRequest = AcknowledgeRequest.newBuilder.setSubscription(ProjectSubscriptionName.format(sc.globalGoogleProjectName, sc.subscription.getSubscription)).addAckIds(ackId).build
pubsubClient.getStub.acknowledgeCallable.call(acknowledgeRequest)
ackId
}
}
package org.opengroup.osdu.production.restore.restore.jobs
import com.google.pubsub.v1.PubsubMessage
import org.opengroup.osdu.production.domain.generated.BigTableBackupMetadata.BigTableBackupBlobDetails
import org.opengroup.osdu.production.restore.FutureConverter._
import org.opengroup.osdu.production.restore.GlobalVariables.{ TableWithBlobId, _ }
import org.opengroup.osdu.production.restore.config.RestoreServiceConfiguration
import org.opengroup.osdu.production.restore.restore.StorageHandler
import com.typesafe.config.Config
import com.typesafe.scalalogging.LazyLogging
import org.opengroup.osdu.production.domain.generated.BigTableBackupMetadata.BigTableBackupBlobDetails
import org.opengroup.osdu.production.restore.config.RestoreServiceConfiguration
import org.opengroup.osdu.production.restore.restore.StorageHandler
import scala.concurrent.Future
case class RestoreMetadataPublish(conf: Config) extends LazyLogging {
def start(backupProjectBucketName: String, backupTableName: String)(implicit sc: RestoreServiceConfiguration): Future[Seq[String]] = {
publishMetadataForRestore(backupProjectBucketName, backupTableName)
}
def publishMetadataForRestore(backupProjectBucketName: String, backupTableName: String)(implicit sc: RestoreServiceConfiguration): Future[Seq[String]] = {
val results = StorageHandler.getFilesToRestore(backupProjectBucketName, backupTableName)
.map((backupTableName, _))
.map(tableWithBlob => publishMessages(tableWithBlob))
Future.sequence(results)
}
//toScalaFuture
def publishMessages(tableWithBlobId: TableWithBlobId)(implicit sc: RestoreServiceConfiguration): Future[String] = {
sc.publisher.publish(createPubSubMessage(tableWithBlobId)).toScalaFuture()
}
def createPubSubMessage(tableWithBlobId: TableWithBlobId): PubsubMessage = {
PubsubMessage.newBuilder().setData(BigTableBackupBlobDetails.newBuilder().setBlobId(tableWithBlobId._2).setTableName(tableWithBlobId._1).build().toByteString).build()
}
}
package org.opengroup.osdu.production.restore.restore.pipeline.steps
import org.opengroup.osdu.production.common.LogF
import org.opengroup.osdu.production.common.entities.{ EventTypes, PipelineLog }
import org.opengroup.osdu.production.domain.generated.BigTableBackupMetadata.BigTableBackupBlobDetails
import org.opengroup.osdu.production.restore.GlobalVariables.BigTableBackupMetadataWithAckId
import org.opengroup.osdu.production.{ ErrorWithAckId, MessageWithAckId }
import org.opengroup.osdu.production.domain.generated.BigTableBackupMetadata.BigTableBackupBlobDetails
import org.scalactic.{ Bad, Good, Or }
import scala.util.{ Failure, Success, Try }
object MessageTranslatorStep extends LogF {
def protoDeserialize(messageWithAckId: MessageWithAckId): Or[BigTableBackupMetadataWithAckId, ErrorWithAckId] = {
loggerF.logTrace(_ => PipelineLog(s"Received pub sub message $messageWithAckId ${messageWithAckId._1}", EventTypes.DataProcess, "", messageWithAckId._2, None))
Try(BigTableBackupBlobDetails.parseFrom(messageWithAckId._1.getData)) match {
case Success(bigTableBackupBlobDetails) => {
loggerF.logInfo(_ => PipelineLog(s"Received blobId ${bigTableBackupBlobDetails.getBlobId} for table ${bigTableBackupBlobDetails.getTableName}", EventTypes.DataProcess, "", messageWithAckId._2, None))
Good((messageWithAckId._2, (bigTableBackupBlobDetails.getTableName, bigTableBackupBlobDetails.getBlobId)))
}
case Failure(ex) => loggerF.logError(_ => PipelineLog(s"Pub sub message can not be translated, ${ex.getMessage}", EventTypes.DataProcess, "", messageWithAckId._2, None)); Bad((messageWithAckId._2, ex.getMessage))