Commit 338d1865 authored by Komal Makkar's avatar Komal Makkar
Browse files

Merge branch 'users/nikhil/MigrationToServiceBus' into 'master'

Migration Script to Service Bus

See merge request !104
parents b0d1d49a f9f8602f
Pipeline #64119 passed with stages
in 22 minutes and 8 seconds
# Source: istio/templates/notification.yaml
# Copyright © Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: security.istio.io/v1beta1
kind: AuthorizationPolicy
metadata:
name: notification-sb-jwt-authz
namespace: osdu
spec:
selector:
matchLabels:
app: notification-sb
action: DENY
rules:
- from:
- source:
notRequestPrincipals: ["*"]
to:
- operation:
notPaths: ["/","*/swagger-resources","*/swagger",
"/api/notification/v1/swagger-resources/*","*/swagger-ui.html","*/actuator/health"]
\ No newline at end of file
# Copyright © Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ .Release.Name }}-sb
namespace: osdu
spec:
replicas: {{ .Values.global.replicaCount }}
selector:
matchLabels:
app: {{ .Chart.Name }}-sb
template:
metadata:
labels:
app: {{ .Chart.Name }}-sb
aadpodidbinding: osdu-identity
spec:
volumes:
- name: azure-keyvault
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: azure-keyvault
containers:
- name: {{ .Chart.Name }}-sb
image: {{ .Values.image.repository }}/{{ .Chart.Name }}-{{ .Values.image.branch }}:{{ .Values.image.tag | default .Chart.AppVersion }}
imagePullPolicy: Always
ports:
- containerPort: 81
readinessProbe:
httpGet:
path: /api/notification/v1/swagger-ui.html
port: 81
volumeMounts:
- name: azure-keyvault
mountPath: "/mnt/azure-keyvault"
readOnly: true
env:
- name: spring_application_name
value: notification-sb-azure
- name: LOG_PREFIX
value: "notification-sb"
- name: server.servlet.contextPath
value: /api/notification/v1
- name: server_port
value: "81"
- name: notification_spring_logging_level
value: INFO
- name: KEYVAULT_URI
valueFrom:
configMapKeyRef:
name: osdu-svc-properties
key: ENV_KEYVAULT
- name: aad_client_id
valueFrom:
secretKeyRef:
name: active-directory
key: application-appid
- name: appinsights_key
valueFrom:
secretKeyRef:
name: central-logging
key: appinsights
- name: cosmosdb_database
value: osdu-db
- name: entitlements_service_endpoint
value: http://entitlements/api/entitlements/v2
- name: entitlements_service_api_key
value: "OBSOLETE"
- name: registeration_service_endpoint
value: http://register/api/register/v1
- name: partition_service_endpoint
value: http://partition/api/partition/v1
- name: maxCacheSize
value: "20"
- name: max_concurrent_calls
value: "3"
- name: executor_n_threads
value: "32"
- name: max_lock_renew_duration_seconds
value: "500"
- name: initial_subscription_manager_delay_seconds
value: "0"
- name: consecutive_subscription_manager_delay_seconds
value: "600"
- name: service_bus_enabled
value: "true"
- name: event_grid_to_service_bus_enabled
value: "false"
- name: event_grid_enabled
value: "false"
\ No newline at end of file
# Copyright © Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: v1
kind: Service
metadata:
name: {{ .Chart.Name }}-sb
namespace: osdu
spec:
type: ClusterIP
ports:
- protocol: TCP
port: 81
targetPort: 81
selector:
app: {{ .Chart.Name }}-sb
\ No newline at end of file
## Introduction
The document talks about the plan to move from Event Grid to Service Bus. The major consumer of the same iss
Notification Service. As Notification Service is consumed by external customers, so a clean migration path is needed.
## Goals
The Migration must happen respecting the following
1. No re-registration
2. Zero downtime.
3. No notification loss.
#### Prerequisites
1. Please verify that the topic you want to use exists. If not,
follow [this](https://community.opengroup.org/osdu/platform/system/notification/-/blob/master/provider/notification-azure/PLAYBOOK_FOR_TOPIC_CREATION.md)
guide to create one.
2. Install
the [latest version](https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell?view=powershell-7.1)
of PowerShell available for your operating system.
## How to do the migration
Migration is something which must be performed by SRE. The flags must be switched in a sequence.The flags are controlled
by deployment.yaml for the service. Following are the steps needed for successful migration.
1. Have 2 deployment of Notification Service running before migration:
1. Deployment 1,having event_grid_enabled.
2. Deployment 2,having service_bus_enabled.The consecutive_subscription_manager_delay_seconds should not be set much
higher as it will pile up alot of SB messages because of the longer time of listening new subscribers on the fly.
**The number of pods for both the deployments should be managed as per the load during migration**
2. Get list of subscriptions per partition and prepare migrationConfig.json [Manual-from azure portal]:
3. Complete powershell Az module installation and Azure login by running script
- [migrationSetup.ps1](https://community.opengroup.org/osdu/platform/system/notification/-/blob/master/provider/notification-azure/src/main/resources/migrationSetup.ps1)
.
**Note:The Azure Az PowerShell module works with PowerShell 7.x and later on all platforms.To check your PowerShell
version, run the following command from within a PowerShell session:$PSVersionTable.PSVersion**
4. Create the subscribers on Service Bus with the same notification id as in CosmosDB by providing migrationConfig.json
to the
script-[migrationToServiceBus.ps1](https://community.opengroup.org/osdu/platform/system/notification/-/blob/master/provider/notification-azure/src/main/resources/migrationToServiceBus.ps1)
. All the subscriptions from one Event grid topic will be moved to service bus topic for one execution of above
mentioned script.We need to rerun script for multiple topic across multiple partitions.
5. Turn on **Service bus flags** and turn off **Event Grid flags** in Producer Services. For example Storage service
publishes to Service bus and Event grid both as of now. During migration,while we are moving to Service bus
completely we have to stop publishing to Event Grid as a part of migration, so we can eventually disable it and move
to Service Bus completely.Same applies to other producer services like -Legal, Schema
etc. [Manual- Configuring deployment.yaml for Legal, Schema, Storage Services]
6. Wait for Event Grid subscribers to drain.[Manual-from azure portal]
**Note: If there are delivery failures,then retry can occur after a longer duration,EG should be kept enabled for
sufficiently long time**
7. Turn on **Service bus flags** in Register Service.[Manual]
8. Event Grid Subscriber clean up.[Manual-from azure portal]
The 3rd and 4th steps would be performed by the scripts in the order given below :
** Powershell must be elevated to admin.**
1. [migrationSetup.ps1](https://community.opengroup.org/osdu/platform/system/notification/-/blob/master/provider/notification-azure/src/main/resources/migrationSetup.ps1)
2. [migrationToServiceBus.ps1](https://community.opengroup.org/osdu/platform/system/notification/-/blob/master/provider/notification-azure/src/main/resources/migrationToServiceBus.ps1)
## List of Feature Flags to be used from deployment.yaml of respective services
1. Notification Service
1. service_bus_enabled
2. event_grid_enabled
2. Register Service
1. azure_serviceBus_enabled
2. azure_eventGrid_enabled
3. Producer Services
1. Storage Service : azure_publishToEventGrid
2. Legal Service : azure.publishToEventGrid
3. Schema Service : event_grid_enabled
## Post Migration Plan
1. Redeploy Notification Service to have only 1 deployment with service_bus_enabled set to true and event_grid_enabled
set to false.The number of pods can be managed as per the load
## Note
There would be duplication of notifications that accounts for the time when producer services will be publishing to both
Service bus and Event grid at any time during migration.For example Storage service by default publish to both.We have
to use Publish flags carefully to avoid duplication
\ No newline at end of file
[
{
"ResourceGroupIdCosmos": "This is the subscription ID corresponding to a resource group having Cosmos DB",
"PartitionKeyCosmos": "This is used by Cosmos DB to distribute data among multiple partitions.Example -opendes",
"ReadWriteMasterKeyCosmosDb": "Cosmos read write keys are very sensitive ones and provide access to the administrative resources",
"CosmosDBEndPoint": "Cosmos endpoint URL",
"DatabaseName": "Cosmos Database name. Example - osdu-db",
"ContainerName": "Cosmos Container name. Example -RegisterSubscription",
"ResourceGroupNameServiceBus": "This is the resource group name of a resource group having Service bus",
"ResourceGroupIdServiceBus": "This is the subscription ID corresponding to a resource group having Service Bus.This can be same as ResourceGroupIdCosmos mentioned above",
"NamespaceNameServiceBus": "Service bus namespace name(not host name) under ResourceGroupNameServiceBus",
"ServiceBusTopicName": "Service bus topic name under ResourceGroupNameServiceBus",
"SubscriptionIdsCosmos": [
"Subs_Id1",
"Subs_Id2"
]
}
]
\ No newline at end of file
# This script provides the pre migration setup for subscriptions from event grid to service bus.
param([Parameter(Mandatory = $true)][Boolean]$InstallAzModule)
try
{
# Set execution policy
Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser
# Installation of modules. This Can be ignored by $InstallAzModule if already installed.
if ($InstallAzModule)
{
Install-Module -Name Az -Scope CurrentUser -Repository PSGallery -Force -ErrorAction Stop
}
# Azure login
Connect-AzAccount -ErrorAction Stop
Write-Output "Migration Setup Successful."
}
catch
{
Write-Output "Migration Setup Failed" $_
}
\ No newline at end of file
<# This script performs migration of event grid subscriptions to service bus subscriptions for a given configuration file.
migrationSetup.ps1 is the prerequistes for successful execution of this script.
#>
Param([Parameter(Mandatory = $true)][String]$migrationConfigFilePath)
Add-Type -AssemblyName System.Web
# Generates auth key for cosmos API
Function GenerateAuthorizationSignature
{
[CmdletBinding()]
Param
(
[Parameter(Mandatory = $true)][String]$method,
[Parameter(Mandatory = $true)][String]$resourceLink,
[Parameter(Mandatory = $true)][String]$resourceType,
[Parameter(Mandatory = $true)][String]$key,
[Parameter(Mandatory = $true)][String]$keyType,
[Parameter(Mandatory = $true)][String]$tokenVersion,
[Parameter(Mandatory = $true)][String]$dateTime
)
$hmacSha = New-Object System.Security.Cryptography.HMACSHA256
$hmacSha.Key = [System.Convert]::FromBase64String($key)
$payLoad = "$($method.ToLowerInvariant() )`n$($resourceType.ToLowerInvariant() )`n$resourceLink`n$($dateTime.ToLowerInvariant() )`n`n"
$hashPayLoad = $hmacSha.ComputeHash([System.Text.Encoding]::UTF8.GetBytes($payLoad))
$signature = [System.Convert]::ToBase64String($hashPayLoad);
[System.Web.HttpUtility]::UrlEncode("type=$keyType&ver=$tokenVersion&sig=$signature")
}
# Fetch the subscription from cosmos db
Function GetCosmosSubscription
{
[CmdletBinding()]
Param(
[Parameter(Mandatory = $true)][String]$Partition,
[Parameter(Mandatory = $true)][String]$SubscriptionId,
[Parameter(Mandatory = $true)][String]$ReadWriteMasterKeyCosmosDb,
[Parameter(Mandatory = $true)][String]$CosmosDBEndPoint,
[Parameter(Mandatory = $true)][String]$DatabaseName,
[Parameter(Mandatory = $true)][String]$ContainerName
)
$resourceType = "docs";
$resourceLink = "dbs/$DatabaseName/colls/$ContainerName/docs/$SubscriptionId"
$partitionkey = "[""$( $Partition )""]"
$cosmosURI = "$CosmosDBEndPoint$resourceLink"
$dateTime = [DateTime]::UtcNow.ToString("r")
$authToken = GenerateAuthorizationSignature -method "GET" -resourceLink $resourceLink -resourceType $resourceType -key $ReadWriteMasterKeyCosmosDb -keyType "master" -tokenVersion "1.0" -dateTime $dateTime
$headers = @{ authorization = $authToken; "x-ms-version" = "2017-02-22"; "x-ms-documentdb-partitionkey" = $partitionkey; "x-ms-date" = $dateTime }
# Call cosmos API to get subscription
Invoke-RestMethod -Method "GET" -Uri $cosmosURI -headers $headers -Verbose
}
# Update the cosmos subscription
Function UpdateCosmosSubscription
{
[CmdletBinding()]
Param(
[Parameter(Mandatory = $true)][String]$JsonSubscription,
[Parameter(Mandatory = $true)][String]$Partition,
[Parameter(Mandatory = $true)][String]$SubscriptionId,
[Parameter(Mandatory = $true)][String]$ReadWriteMasterKeyCosmosDb,
[Parameter(Mandatory = $true)][String]$CosmosDBEndPoint,
[Parameter(Mandatory = $true)][String]$DatabaseName,
[Parameter(Mandatory = $true)][String]$ContainerName
)
$resourceType = "docs";
$resourceLink = "dbs/$DatabaseName/colls/$ContainerName/docs/$SubscriptionId"
$partitionkey = "[""$( $Partition )""]"
$contentType = "application/json"
$cosmosURI = "$CosmosDBEndPoint$resourceLink"
$dateTime = [DateTime]::UtcNow.ToString("r")
$authToken = GenerateAuthorizationSignature -method "PUT" -resourceLink $ResourceLink -resourceType $ResourceType -key $ReadWriteMasterKeyCosmosDb -keyType "master" -tokenVersion "1.0" -dateTime $dateTime
$headers = @{ authorization = $authToken; "x-ms-version" = "2017-02-22"; "x-ms-documentdb-partitionkey" = $partitionkey; "x-ms-date" = $dateTime }
# Call cosmos API to update subscription
Invoke-RestMethod -Method "PUT" -ContentType $contentType -Uri $cosmosURI -headers $headers -Body $JsonSubscription -Verbose
}
# Migration Execution
try
{
$configurations = Get-Content -Raw -Path $migrationConfigFilePath| ConvertFrom-Json
foreach ($config in $configurations)
{
try
{
$ResourceGroupIdCosmos = $config.ResourceGroupIdCosmos
$PartitionKeyCosmos = $config.PartitionKeyCosmos
$ReadWriteMasterKeyCosmosDb = $config.ReadWriteMasterKeyCosmosDb
$CosmosDBEndPoint = $config.CosmosDBEndPoint
$DatabaseName = $config.DatabaseName
$ContainerName = $config.ContainerName
$ResourceGroupNameServiceBus = $config.ResourceGroupNameServiceBus
$ResourceGroupIdServiceBus = $config.ResourceGroupIdServiceBus
$NamespaceNameServiceBus = $config.NamespaceNameServiceBus
$ServiceBusTopicName = $config.ServiceBusTopicName
$SubscriptionIds = $config.SubscriptionIdsCosmos
foreach ($SubscriptionId in $SubscriptionIds)
{
$Subscription = $null
$ServiceBusSubscription = $null
try
{
# Set resource group susbscription for cosmos
Select-AzSubscription -SubscriptionName $ResourceGroupIdCosmos -Verbose -ErrorAction Stop
$Subscription = GetCosmosSubscription -Partition $PartitionKeyCosmos -SubscriptionId $SubscriptionId -ReadWriteMasterKeyCosmosDb $ReadWriteMasterKeyCosmosDb -CosmosDBEndPoint $CosmosDBEndPoint -DatabaseName $DatabaseName -ContainerName $ContainerName
if ((!$Subscription) -or (!$Subscription.notificationId) -or (!$Subscription.topic))
{
throw "Subscription is not present/null/empty or NotificationId/Topic is not present/null/empty in subscription with ID - $SubscriptionId"
}
$NotificationId = $Subscription.notificationId
# Set resource group susbscription for service bus
Select-AzSubscription -SubscriptionName $ResourceGroupIdServiceBus -Verbose -ErrorAction Stop
# Create service bus topic subscription
$ServiceBusSubscription = New-AzServiceBusSubscription -ResourceGroupName $ResourceGroupNameServiceBus -Namespace $NamespaceNameServiceBus -Topic $ServiceBusTopicName -Name $NotificationId -Verbose
if (!$ServiceBusSubscription)
{
throw "Unable to create service bus subscription with Id - $NotificationId and topic - $ServiceBusTopicName "
}
# Update subscription topic name
$Subscription.topic = $ServiceBusTopicName
$JsonSubscription = $Subscription| ConvertTo-Json
# Update cosmos subscription
$UpdatedSubscription = UpdateCosmosSubscription -JsonSubscription $JsonSubscription -Partition $PartitionKeyCosmos -SubscriptionId $SubscriptionId -ReadWriteMasterKeyCosmosDb $ReadWriteMasterKeyCosmosDb -CosmosDBEndPoint $CosmosDBEndPoint -DatabaseName $DatabaseName -ContainerName $ContainerName
Write-Output "Migration to Service Bus is Successful for Subscription - $SubscriptionId `n"
}
catch
{
Write-Output "Migration Failed for Subscription - $SubscriptionId." $_
}
}
}
catch
{
Write-Output "Unable to process configuration -$config.ResourceGroupName." $_
}
}
}
catch
{
Write-Output "Unable to process configurations" $_
}
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment