Migration Script to Service Bus

devops/azure/chart/templates/authSB.yaml

+# Source: istio/templates/notification.yaml
+#  Copyright © Microsoft Corporation
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+apiVersion: security.istio.io/v1beta1
+kind: AuthorizationPolicy
+metadata:
+  name: notification-sb-jwt-authz
+  namespace: osdu
+spec:
+  selector:
+    matchLabels:
+      app: notification-sb
+  action: DENY
+  rules:
+    - from:
+        - source:
+            notRequestPrincipals: ["*"]
+      to:
+        - operation:
+            notPaths: ["/","*/swagger-resources","*/swagger",
+                       "/api/notification/v1/swagger-resources/*","*/swagger-ui.html","*/actuator/health"]
\ No newline at end of file

devops/azure/chart/templates/deploymentSB.yaml

+#  Copyright © Microsoft Corporation
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Release.Name }}-sb
+  namespace: osdu
+spec:
+  replicas: {{ .Values.global.replicaCount }}
+  selector:
+    matchLabels:
+      app: {{ .Chart.Name }}-sb
+  template:
+    metadata:
+      labels:
+        app: {{ .Chart.Name }}-sb
+        aadpodidbinding: osdu-identity
+    spec:
+      volumes:
+        - name: azure-keyvault
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: azure-keyvault
+      containers:
+        - name: {{ .Chart.Name }}-sb
+          image: {{ .Values.image.repository }}/{{ .Chart.Name }}-{{ .Values.image.branch }}:{{ .Values.image.tag | default .Chart.AppVersion }}
+          imagePullPolicy: Always
+          ports:
+            - containerPort: 81
+          readinessProbe:
+            httpGet:
+              path: /api/notification/v1/swagger-ui.html
+              port: 81
+          volumeMounts:
+            - name: azure-keyvault
+              mountPath: "/mnt/azure-keyvault"
+              readOnly: true
+          env:
+            - name: spring_application_name
+              value: notification-sb-azure
+            - name: LOG_PREFIX
+              value: "notification-sb"
+            - name: server.servlet.contextPath
+              value: /api/notification/v1
+            - name: server_port
+              value: "81"
+            - name: notification_spring_logging_level
+              value: INFO
+            - name: KEYVAULT_URI
+              valueFrom:
+                configMapKeyRef:
+                  name: osdu-svc-properties
+                  key: ENV_KEYVAULT
+            - name: aad_client_id
+              valueFrom:
+                secretKeyRef:
+                  name: active-directory
+                  key: application-appid
+            - name: appinsights_key
+              valueFrom:
+                secretKeyRef:
+                  name: central-logging
+                  key: appinsights
+            - name: cosmosdb_database
+              value: osdu-db
+            - name: entitlements_service_endpoint
+              value: http://entitlements/api/entitlements/v2
+            - name: entitlements_service_api_key
+              value: "OBSOLETE"
+            - name: registeration_service_endpoint
+              value: http://register/api/register/v1
+            - name: partition_service_endpoint
+              value: http://partition/api/partition/v1
+            - name: maxCacheSize
+              value:  "20"
+            - name: max_concurrent_calls
+              value: "3"
+            - name: executor_n_threads
+              value: "32"
+            - name: max_lock_renew_duration_seconds
+              value: "500"
+            - name: initial_subscription_manager_delay_seconds
+              value: "0"
+            - name: consecutive_subscription_manager_delay_seconds
+              value: "600"
+            - name: service_bus_enabled
+              value: "true"
+            - name: event_grid_to_service_bus_enabled
+              value: "false"
+            - name: event_grid_enabled
+              value: "false"
\ No newline at end of file

devops/azure/chart/templates/serviceSB.yaml

+#  Copyright © Microsoft Corporation
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Chart.Name }}-sb
+  namespace: osdu
+spec:
+  type: ClusterIP
+  ports:
+    - protocol: TCP
+      port: 81
+      targetPort: 81
+  selector:
+    app: {{ .Chart.Name }}-sb
\ No newline at end of file

provider/notification-azure/docs/MIGRATION.md

+## Introduction
+
+The document talks about the plan to move from Event Grid to Service Bus. The major consumer of the same iss
+Notification Service. As Notification Service is consumed by external customers, so a clean migration path is needed.
+
+## Goals
+
+The Migration must happen respecting the following
+
+1. No re-registration
+2. Zero downtime.
+3. No notification loss.
+
+#### Prerequisites
+
+1. Please verify that the topic you want to use exists. If not,
+   follow [this](https://community.opengroup.org/osdu/platform/system/notification/-/blob/master/provider/notification-azure/PLAYBOOK_FOR_TOPIC_CREATION.md)
+   guide to create one.
+2. Install
+   the [latest version](https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell?view=powershell-7.1)
+   of PowerShell available for your operating system.
+
+## How to do the migration
+
+Migration is something which must be performed by SRE. The flags must be switched in a sequence.The flags are controlled
+by deployment.yaml for the service. Following are the steps needed for successful migration.
+
+1. Have 2 deployment of Notification Service running before migration:
+    1. Deployment 1,having event_grid_enabled.
+    2. Deployment 2,having service_bus_enabled.The consecutive_subscription_manager_delay_seconds should not be set much
+       higher as it will pile up alot of SB messages because of the longer time of listening new subscribers on the fly.
+       **The number of pods for both the deployments should be managed as per the load during migration**
+2. Get list of subscriptions per partition and prepare migrationConfig.json [Manual-from azure portal]:
+3. Complete powershell Az module installation and Azure login by running script
+   - [migrationSetup.ps1](https://community.opengroup.org/osdu/platform/system/notification/-/blob/master/provider/notification-azure/src/main/resources/migrationSetup.ps1)
+   .
+
+**Note:The Azure Az PowerShell module works with PowerShell 7.x and later on all platforms.To check your PowerShell
+version, run the following command from within a PowerShell session:$PSVersionTable.PSVersion**
+
+4. Create the subscribers on Service Bus with the same notification id as in CosmosDB by providing migrationConfig.json
+   to the
+   script-[migrationToServiceBus.ps1](https://community.opengroup.org/osdu/platform/system/notification/-/blob/master/provider/notification-azure/src/main/resources/migrationToServiceBus.ps1)
+   . All the subscriptions from one Event grid topic will be moved to service bus topic for one execution of above
+   mentioned script.We need to rerun script for multiple topic across multiple partitions.
+5. Turn on **Service bus flags** and turn off **Event Grid flags** in Producer Services. For example Storage service
+   publishes to Service bus and Event grid both as of now. During migration,while we are moving to Service bus
+   completely we have to stop publishing to Event Grid as a part of migration, so we can eventually disable it and move
+   to Service Bus completely.Same applies to other producer services like -Legal, Schema
+   etc. [Manual- Configuring deployment.yaml for Legal, Schema, Storage Services]
+6. Wait for Event Grid subscribers to drain.[Manual-from azure portal]
+   **Note: If there are delivery failures,then retry can occur after a longer duration,EG should be kept enabled for
+   sufficiently long time**
+7. Turn on **Service bus flags** in Register Service.[Manual]
+8. Event Grid Subscriber clean up.[Manual-from azure portal]
+
+The 3rd and 4th steps would be performed by the scripts in the order given below :
+** Powershell must be elevated to admin.**
+
+1. [migrationSetup.ps1](https://community.opengroup.org/osdu/platform/system/notification/-/blob/master/provider/notification-azure/src/main/resources/migrationSetup.ps1)
+2. [migrationToServiceBus.ps1](https://community.opengroup.org/osdu/platform/system/notification/-/blob/master/provider/notification-azure/src/main/resources/migrationToServiceBus.ps1)
+
+## List of Feature Flags to be used from deployment.yaml of respective services
+
+1. Notification Service
+    1. service_bus_enabled
+    2. event_grid_enabled
+2. Register Service
+    1. azure_serviceBus_enabled
+    2. azure_eventGrid_enabled
+3. Producer Services
+    1. Storage Service : azure_publishToEventGrid
+    2. Legal Service : azure.publishToEventGrid
+    3. Schema Service : event_grid_enabled
+
+## Post Migration Plan
+
+1. Redeploy Notification Service to have only 1 deployment with service_bus_enabled set to true and event_grid_enabled
+   set to false.The number of pods can be managed as per the load
+
+## Note
+
+There would be duplication of notifications that accounts for the time when producer services will be publishing to both
+Service bus and Event grid at any time during migration.For example Storage service by default publish to both.We have
+to use Publish flags carefully to avoid duplication  
\ No newline at end of file

provider/notification-azure/src/main/resources/migrationConfig.json

+[
+  {
+    "ResourceGroupIdCosmos": "This is the subscription ID corresponding to a resource group having Cosmos DB",
+    "PartitionKeyCosmos": "This is used by Cosmos DB to distribute data among multiple partitions.Example -opendes",
+    "ReadWriteMasterKeyCosmosDb": "Cosmos read write keys are very sensitive ones and provide access to the administrative resources",
+    "CosmosDBEndPoint": "Cosmos endpoint URL",
+    "DatabaseName": "Cosmos Database name. Example - osdu-db",
+    "ContainerName": "Cosmos Container name. Example -RegisterSubscription",
+    "ResourceGroupNameServiceBus": "This is the resource group name of a resource group having Service bus",
+    "ResourceGroupIdServiceBus": "This is the subscription ID corresponding to a resource group having Service Bus.This can be same as ResourceGroupIdCosmos mentioned above",
+    "NamespaceNameServiceBus": "Service bus namespace name(not host name) under ResourceGroupNameServiceBus",
+    "ServiceBusTopicName": "Service bus topic name under ResourceGroupNameServiceBus",
+    "SubscriptionIdsCosmos": [
+      "Subs_Id1",
+      "Subs_Id2"
+    ]
+  }
+]
\ No newline at end of file

provider/notification-azure/src/main/resources/migrationSetup.ps1

+# This script provides the pre migration setup for subscriptions from event grid to service bus.
+
+param([Parameter(Mandatory = $true)][Boolean]$InstallAzModule)
+
+try
+{
+    # Set execution policy
+    Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser
+
+    # Installation of modules. This Can be ignored by $InstallAzModule if already installed.
+    if ($InstallAzModule)
+    {
+        Install-Module -Name Az -Scope CurrentUser -Repository PSGallery -Force -ErrorAction Stop
+    }
+
+    # Azure login
+    Connect-AzAccount -ErrorAction Stop
+
+    Write-Output "Migration Setup Successful."
+}
+catch
+{
+    Write-Output "Migration Setup Failed" $_
+}
\ No newline at end of file

provider/notification-azure/src/main/resources/migrationToServiceBus.ps1

+<# This script performs migration of event grid subscriptions to service bus subscriptions for a given configuration file.
+   migrationSetup.ps1 is the prerequistes for successful execution of this script.
+#>
+Param([Parameter(Mandatory = $true)][String]$migrationConfigFilePath)
+
+Add-Type -AssemblyName System.Web
+
+# Generates auth key for cosmos API
+Function GenerateAuthorizationSignature
+{
+    [CmdletBinding()]
+    Param
+    (
+        [Parameter(Mandatory = $true)][String]$method,
+        [Parameter(Mandatory = $true)][String]$resourceLink,
+        [Parameter(Mandatory = $true)][String]$resourceType,
+        [Parameter(Mandatory = $true)][String]$key,
+        [Parameter(Mandatory = $true)][String]$keyType,
+        [Parameter(Mandatory = $true)][String]$tokenVersion,
+        [Parameter(Mandatory = $true)][String]$dateTime
+    )
+    $hmacSha = New-Object System.Security.Cryptography.HMACSHA256
+    $hmacSha.Key = [System.Convert]::FromBase64String($key)
+
+    $payLoad = "$($method.ToLowerInvariant() )`n$($resourceType.ToLowerInvariant() )`n$resourceLink`n$($dateTime.ToLowerInvariant() )`n`n"
+    $hashPayLoad = $hmacSha.ComputeHash([System.Text.Encoding]::UTF8.GetBytes($payLoad))
+    $signature = [System.Convert]::ToBase64String($hashPayLoad);
+
+    [System.Web.HttpUtility]::UrlEncode("type=$keyType&ver=$tokenVersion&sig=$signature")
+}
+
+# Fetch the subscription from cosmos db
+Function GetCosmosSubscription
+{
+    [CmdletBinding()]
+    Param(
+        [Parameter(Mandatory = $true)][String]$Partition,
+        [Parameter(Mandatory = $true)][String]$SubscriptionId,
+        [Parameter(Mandatory = $true)][String]$ReadWriteMasterKeyCosmosDb,
+        [Parameter(Mandatory = $true)][String]$CosmosDBEndPoint,
+        [Parameter(Mandatory = $true)][String]$DatabaseName,
+        [Parameter(Mandatory = $true)][String]$ContainerName
+    )
+
+    $resourceType = "docs";
+    $resourceLink = "dbs/$DatabaseName/colls/$ContainerName/docs/$SubscriptionId"
+    $partitionkey = "[""$( $Partition )""]"
+    $cosmosURI = "$CosmosDBEndPoint$resourceLink"
+    $dateTime = [DateTime]::UtcNow.ToString("r")
+
+    $authToken = GenerateAuthorizationSignature -method "GET" -resourceLink $resourceLink -resourceType $resourceType -key $ReadWriteMasterKeyCosmosDb -keyType "master" -tokenVersion "1.0" -dateTime $dateTime
+
+    $headers = @{ authorization = $authToken; "x-ms-version" = "2017-02-22"; "x-ms-documentdb-partitionkey" = $partitionkey; "x-ms-date" = $dateTime }
+
+    # Call cosmos API to get subscription
+    Invoke-RestMethod -Method "GET" -Uri $cosmosURI -headers $headers -Verbose
+
+}
+
+# Update the cosmos subscription
+Function UpdateCosmosSubscription
+{
+    [CmdletBinding()]
+    Param(
+        [Parameter(Mandatory = $true)][String]$JsonSubscription,
+        [Parameter(Mandatory = $true)][String]$Partition,
+        [Parameter(Mandatory = $true)][String]$SubscriptionId,
+        [Parameter(Mandatory = $true)][String]$ReadWriteMasterKeyCosmosDb,
+        [Parameter(Mandatory = $true)][String]$CosmosDBEndPoint,
+        [Parameter(Mandatory = $true)][String]$DatabaseName,
+        [Parameter(Mandatory = $true)][String]$ContainerName
+    )
+
+    $resourceType = "docs";
+    $resourceLink = "dbs/$DatabaseName/colls/$ContainerName/docs/$SubscriptionId"
+    $partitionkey = "[""$( $Partition )""]"
+    $contentType = "application/json"
+    $cosmosURI = "$CosmosDBEndPoint$resourceLink"
+    $dateTime = [DateTime]::UtcNow.ToString("r")
+
+    $authToken = GenerateAuthorizationSignature -method "PUT" -resourceLink $ResourceLink -resourceType $ResourceType -key $ReadWriteMasterKeyCosmosDb -keyType "master" -tokenVersion "1.0" -dateTime $dateTime
+
+    $headers = @{ authorization = $authToken; "x-ms-version" = "2017-02-22"; "x-ms-documentdb-partitionkey" = $partitionkey; "x-ms-date" = $dateTime }
+
+    # Call cosmos API to update subscription
+    Invoke-RestMethod -Method "PUT" -ContentType $contentType -Uri $cosmosURI -headers $headers -Body $JsonSubscription -Verbose
+
+}
+
+# Migration Execution
+try
+{
+    $configurations = Get-Content -Raw -Path $migrationConfigFilePath| ConvertFrom-Json
+
+    foreach ($config in $configurations)
+    {
+        try
+        {
+            $ResourceGroupIdCosmos = $config.ResourceGroupIdCosmos
+            $PartitionKeyCosmos = $config.PartitionKeyCosmos
+            $ReadWriteMasterKeyCosmosDb = $config.ReadWriteMasterKeyCosmosDb
+            $CosmosDBEndPoint = $config.CosmosDBEndPoint
+            $DatabaseName = $config.DatabaseName
+            $ContainerName = $config.ContainerName
+            $ResourceGroupNameServiceBus = $config.ResourceGroupNameServiceBus
+            $ResourceGroupIdServiceBus = $config.ResourceGroupIdServiceBus
+            $NamespaceNameServiceBus = $config.NamespaceNameServiceBus
+            $ServiceBusTopicName = $config.ServiceBusTopicName
+            $SubscriptionIds = $config.SubscriptionIdsCosmos
+
+            foreach ($SubscriptionId in $SubscriptionIds)
+            {
+                $Subscription = $null
+                $ServiceBusSubscription = $null
+                try
+                {
+                    # Set resource group susbscription for cosmos
+                    Select-AzSubscription -SubscriptionName $ResourceGroupIdCosmos -Verbose -ErrorAction Stop
+
+                    $Subscription = GetCosmosSubscription -Partition $PartitionKeyCosmos -SubscriptionId $SubscriptionId -ReadWriteMasterKeyCosmosDb $ReadWriteMasterKeyCosmosDb -CosmosDBEndPoint $CosmosDBEndPoint -DatabaseName $DatabaseName -ContainerName $ContainerName
+                    if ((!$Subscription) -or (!$Subscription.notificationId) -or (!$Subscription.topic))
+                    {
+                        throw "Subscription is not present/null/empty or NotificationId/Topic is not present/null/empty in subscription with ID - $SubscriptionId"
+                    }
+                    $NotificationId = $Subscription.notificationId
+
+                    # Set resource group susbscription for service bus
+                    Select-AzSubscription -SubscriptionName $ResourceGroupIdServiceBus -Verbose -ErrorAction Stop
+
+                    # Create service bus topic subscription
+                    $ServiceBusSubscription = New-AzServiceBusSubscription -ResourceGroupName $ResourceGroupNameServiceBus -Namespace $NamespaceNameServiceBus -Topic $ServiceBusTopicName -Name $NotificationId -Verbose
+
+                    if (!$ServiceBusSubscription)
+                    {
+                        throw "Unable to create service bus subscription with Id - $NotificationId and topic - $ServiceBusTopicName "
+                    }
+                    # Update subscription topic name
+                    $Subscription.topic = $ServiceBusTopicName
+                    $JsonSubscription = $Subscription| ConvertTo-Json
+
+                    # Update cosmos subscription
+                    $UpdatedSubscription = UpdateCosmosSubscription -JsonSubscription $JsonSubscription -Partition $PartitionKeyCosmos -SubscriptionId $SubscriptionId -ReadWriteMasterKeyCosmosDb $ReadWriteMasterKeyCosmosDb -CosmosDBEndPoint $CosmosDBEndPoint -DatabaseName $DatabaseName -ContainerName $ContainerName
+
+                    Write-Output "Migration to Service Bus is Successful for Subscription - $SubscriptionId `n"
+                }
+                catch
+                {
+                    Write-Output "Migration Failed for Subscription - $SubscriptionId." $_
+                }
+            }
+        }
+        catch
+        {
+            Write-Output "Unable to process configuration -$config.ResourceGroupName." $_
+        }
+    }
+
+}
+catch
+{
+    Write-Output "Unable to process configurations" $_
+}
\ No newline at end of file