Merge branch 'GONRG-6509-pubsub-update' into 'master'

GONRG-6509 - new pub/sub structure

See merge request !911

modules/osdu/pubsub-legal.tf

+/**
+*  Copyright 2023 Google LLC
+*  Copyright 2023 EPAM
+*
+*  Licensed under the Apache License, Version 2.0 (the "License");
+*  you may not use this file except in compliance with the License.
+*  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+*  Unless required by applicable law or agreed to in writing, software
+*  distributed under the License is distributed on an "AS IS" BASIS,
+*  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*  See the License for the specific language governing permissions and
+*  limitations under the License.
+*/
+
+# Legal topics & subscriptions
+resource "google_pubsub_topic" "legal_topic" {
+  depends_on = [module.pubsub_sa]
+  name       = "legaltags_changed"
+  project    = var.service_google_project
+}
+
+resource "google_pubsub_topic" "legal_dead_letter_topic" {
+  depends_on = [module.pubsub_sa]
+  name       = "legaltags_changed-dead-letter"
+  project    = var.service_google_project
+}
+
+resource "google_pubsub_subscription" "legal_subscription" {
+  for_each = local.data_partition_list # create sub for each partition due to old storage logic. Will be changed soon
+
+  topic                      = google_pubsub_topic.legal_topic.id
+  name                       = "storage-oqm-legaltags-changed-${each.value}"
+  project                    = var.service_google_project
+  message_retention_duration = "604800s"
+
+  expiration_policy {
+    ttl = ""
+  }
+
+  dead_letter_policy {
+    dead_letter_topic     = google_pubsub_topic.legal_dead_letter_topic.id
+    max_delivery_attempts = 10
+  }
+
+  retry_policy {
+    minimum_backoff = "10s"
+    maximum_backoff = "600s"
+  }
+}
+
+resource "google_pubsub_subscription" "notification_service_legal_subscription" {
+  # for_each = local.data_partition_list - multipartition is not implemented yet here
+
+  topic                      = google_pubsub_topic.legal_topic.id
+  name                       = "notification-legaltags_changed-service"
+  project                    = var.service_google_project
+  message_retention_duration = "604800s"
+
+  expiration_policy {
+    ttl = ""
+  }
+
+  dead_letter_policy {
+    dead_letter_topic     = google_pubsub_topic.notification_dead_letter_topic.id
+    max_delivery_attempts = 10
+  }
+
+  retry_policy {
+    minimum_backoff = "10s"
+    maximum_backoff = "600s"
+  }
+}
+
+resource "google_pubsub_topic" "legal_publish_topic" {
+  depends_on = [module.pubsub_sa]
+  name       = "legaltags_changed-publish"
+  project    = var.service_google_project
+}
+
+resource "google_pubsub_subscription" "notification_publish_legal_subscription" {
+  # for_each = local.data_partition_list - multipartition is not implemented yet here
+
+  topic                      = google_pubsub_topic.legal_publish_topic.id
+  name                       = "notification-legaltags_changed-publish"
+  project                    = var.service_google_project
+  message_retention_duration = "604800s"
+
+  expiration_policy {
+    ttl = ""
+  }
+
+  dead_letter_policy {
+    dead_letter_topic     = google_pubsub_topic.notification_dead_letter_topic.id
+    max_delivery_attempts = 10
+  }
+
+  retry_policy {
+    minimum_backoff = "10s"
+    maximum_backoff = "600s"
+  }
+}

modules/osdu/pubsub.tf → modules/osdu/pubsub-main.tf

@@ -57,134 +57,39 @@ resource "google_pubsub_topic" "indexer_topic" {
   project    = var.service_google_project
 }
 
-resource "google_pubsub_topic" "legal_topic" {
+# Control topic & subscriptions
+resource "google_pubsub_topic" "control_topic" {
   depends_on = [module.pubsub_sa]
-  name       = "legaltags-changed"
+  name       = "register-subscriber-control"
   project    = var.service_google_project
 }
 
-resource "google_pubsub_topic" "records_topic" {
+resource "google_pubsub_topic" "control_dead_letter_topic" {
   depends_on = [module.pubsub_sa]
-  name       = "records-changed"
+  name       = "register-subscriber-control-dead-letter"
   project    = var.service_google_project
 }
 
-resource "google_pubsub_topic" "records_dead_letter_topic" {
+resource "google_pubsub_topic" "notification_dead_letter_topic" {
   depends_on = [module.pubsub_sa]
-  name       = "records-changed-dead-letter"
+  name       = "notification-dead-letter"
   project    = var.service_google_project
 }
 
-resource "google_pubsub_topic" "reprocess_topic" {
-  depends_on = [module.pubsub_sa]
-  name       = "reprocess"
-  project    = var.service_google_project
-}
-
-resource "google_pubsub_topic" "reprocess_dead_letter_topic" {
-  depends_on = [module.pubsub_sa]
-  name       = "reprocess-dead-letter"
-  project    = var.service_google_project
-}
-resource "google_pubsub_topic" "schema_topic" {
-  depends_on = [module.pubsub_sa]
-  name       = "schema-changed"
-  project    = var.service_google_project
-}
-
-resource "google_pubsub_topic" "schema_dead_letter_topic" {
-  depends_on = [module.pubsub_sa]
-  name       = "schema-changed-dead-letter"
-  project    = var.service_google_project
-}
-
-resource "google_pubsub_topic" "status_topic" {
-  depends_on = [module.pubsub_sa]
-  name       = "status-changed"
-  project    = var.service_google_project
-}
-
-resource "google_pubsub_subscription" "records_subscription" {
-  depends_on = [google_pubsub_topic.records_topic]
-  for_each   = local.data_partition_list
-
-  topic                      = google_pubsub_topic.records_topic.id
-  name                       = "indexer-records-changed-${each.value}"
-  project                    = var.service_google_project
-  message_retention_duration = "604800s"
-
-  expiration_policy {
-    ttl = ""
-  }
-
-  dead_letter_policy {
-    dead_letter_topic     = google_pubsub_topic.records_dead_letter_topic.id
-    max_delivery_attempts = 5
-  }
-
-  retry_policy {
-    minimum_backoff = "10s"
-    maximum_backoff = "600s"
-  }
-}
-
-resource "google_pubsub_subscription" "records_dead_letter_subscription" {
-  depends_on = [google_pubsub_topic.schema_dead_letter_topic]
-  for_each   = local.data_partition_list
-
-  topic                      = google_pubsub_topic.records_dead_letter_topic.id
-  name                       = "indexer-records-changed-dead-letter-${each.value}"
-  project                    = var.service_google_project
-  message_retention_duration = "604800s"
-}
-
-resource "google_pubsub_subscription" "reprocess_subscription" {
-  depends_on = [google_pubsub_topic.reprocess_topic]
-  for_each   = local.data_partition_list
-
-  topic   = google_pubsub_topic.reprocess_topic.id
-  name    = "indexer-reprocess-${each.value}"
-  project = var.service_google_project
-
-  expiration_policy {
-    ttl = ""
-  }
-
-  dead_letter_policy {
-    dead_letter_topic     = google_pubsub_topic.reprocess_dead_letter_topic.id
-    max_delivery_attempts = 5
-  }
+resource "google_pubsub_subscription" "notification_control_subscription" {
+  # for_each = local.data_partition_list - multipartition is not implemented yet here
 
-  retry_policy {
-    minimum_backoff = "10s"
-    maximum_backoff = "600s"
-  }
-}
-
-resource "google_pubsub_subscription" "reprocess_dead_letter_subscription" {
-  depends_on = [google_pubsub_topic.reprocess_dead_letter_topic]
-  for_each   = local.data_partition_list
-
-  topic                      = google_pubsub_topic.reprocess_dead_letter_topic.id
-  name                       = "indexer-reprocess-dead-letter-${each.value}"
+  topic                      = google_pubsub_topic.control_topic.id
+  name                       = "notification-control-topic"
   project                    = var.service_google_project
   message_retention_duration = "604800s"
-}
-
-resource "google_pubsub_subscription" "schema_subscription" {
-  depends_on = [google_pubsub_topic.schema_topic]
-  for_each   = local.data_partition_list
-
-  topic   = google_pubsub_topic.schema_topic.id
-  name    = "indexer-schema-changed-${each.value}"
-  project = var.service_google_project
 
   expiration_policy {
     ttl = ""
   }
 
   dead_letter_policy {
-    dead_letter_topic     = google_pubsub_topic.schema_dead_letter_topic.id
+    dead_letter_topic     = google_pubsub_topic.control_dead_letter_topic.id
     max_delivery_attempts = 5
   }
 
@@ -193,13 +98,3 @@ resource "google_pubsub_subscription" "schema_subscription" {
     maximum_backoff = "600s"
   }
 }
-
-resource "google_pubsub_subscription" "schema_dead_letter_subscription" {
-  depends_on = [google_pubsub_topic.schema_dead_letter_topic]
-  for_each   = local.data_partition_list
-
-  topic                      = google_pubsub_topic.schema_dead_letter_topic.id
-  name                       = "indexer-schema-changed-dead-letter-${each.value}"
-  project                    = var.service_google_project
-  message_retention_duration = "604800s"
-}

modules/osdu/pubsub-records.tf

+/**
+*  Copyright 2023 Google LLC
+*  Copyright 2023 EPAM
+*
+*  Licensed under the Apache License, Version 2.0 (the "License");
+*  you may not use this file except in compliance with the License.
+*  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+*  Unless required by applicable law or agreed to in writing, software
+*  distributed under the License is distributed on an "AS IS" BASIS,
+*  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*  See the License for the specific language governing permissions and
+*  limitations under the License.
+*/
+
+# Records topics & subscriptions
+resource "google_pubsub_topic" "records_topic" {
+  depends_on = [module.pubsub_sa]
+  name       = "records-changed"
+  project    = var.service_google_project
+}
+
+resource "google_pubsub_topic" "records_dead_letter_topic" {
+  depends_on = [module.pubsub_sa]
+  name       = "records-changed-dead-letter"
+  project    = var.service_google_project
+}
+
+resource "google_pubsub_subscription" "records_subscription" {
+  # for_each = local.data_partition_list - multipartition is not implemented yet here
+
+  topic                      = google_pubsub_topic.records_topic.id
+  name                       = "indexer-records-changed"
+  project                    = var.service_google_project
+  message_retention_duration = "604800s"
+
+  expiration_policy {
+    ttl = ""
+  }
+
+  dead_letter_policy {
+    dead_letter_topic     = google_pubsub_topic.records_dead_letter_topic.id
+    max_delivery_attempts = 10
+  }
+
+  retry_policy {
+    minimum_backoff = "0s"
+    maximum_backoff = "30s"
+  }
+}
+
+resource "google_pubsub_subscription" "notification_service_records_subscription" {
+  # for_each = local.data_partition_list - multipartition is not implemented yet here
+
+  topic                      = google_pubsub_topic.records_topic.id
+  name                       = "notification-records-changed-service"
+  project                    = var.service_google_project
+  message_retention_duration = "604800s"
+
+  expiration_policy {
+    ttl = ""
+  }
+
+  dead_letter_policy {
+    dead_letter_topic     = google_pubsub_topic.notification_dead_letter_topic.id
+    max_delivery_attempts = 10
+  }
+
+  retry_policy {
+    minimum_backoff = "10s"
+    maximum_backoff = "600s"
+  }
+}
+
+resource "google_pubsub_topic" "records_publish_topic" {
+  depends_on = [module.pubsub_sa]
+  name       = "records-changed-publish"
+  project    = var.service_google_project
+}
+
+resource "google_pubsub_topic" "records_publish_dead_letter_topic" {
+  depends_on = [module.pubsub_sa]
+  name       = "records-changed-publish-dead-letter"
+  project    = var.service_google_project
+}
+
+resource "google_pubsub_subscription" "notification_publish_records_subscription" {
+  # for_each = local.data_partition_list - multipartition is not implemented yet here
+
+  topic                      = google_pubsub_topic.records_publish_topic.id
+  name                       = "notification-records-changed-publish"
+  project                    = var.service_google_project
+  message_retention_duration = "604800s"
+
+  expiration_policy {
+    ttl = ""
+  }
+
+  dead_letter_policy {
+    dead_letter_topic     = google_pubsub_topic.notification_dead_letter_topic.id
+    max_delivery_attempts = 10
+  }
+
+  retry_policy {
+    minimum_backoff = "10s"
+    maximum_backoff = "600s"
+  }
+}

modules/osdu/pubsub-reprocess.tf

+/**
+*  Copyright 2023 Google LLC
+*  Copyright 2023 EPAM
+*
+*  Licensed under the Apache License, Version 2.0 (the "License");
+*  you may not use this file except in compliance with the License.
+*  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+*  Unless required by applicable law or agreed to in writing, software
+*  distributed under the License is distributed on an "AS IS" BASIS,
+*  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*  See the License for the specific language governing permissions and
+*  limitations under the License.
+*/
+
+# Reprocess topics & subscriptions
+resource "google_pubsub_topic" "reprocess_topic" {
+  depends_on = [module.pubsub_sa]
+  name       = "reprocess"
+  project    = var.service_google_project
+}
+
+resource "google_pubsub_topic" "reprocess_dead_letter_topic" {
+  depends_on = [module.pubsub_sa]
+  name       = "reprocess-dead-letter"
+  project    = var.service_google_project
+}
+
+resource "google_pubsub_subscription" "reprocess_subscription" {
+  # for_each = local.data_partition_list - multipartition is not implemented yet here - multipartition is not implemented yet here
+
+  topic                      = google_pubsub_topic.reprocess_topic.id
+  name                       = "indexer-reprocess"
+  project                    = var.service_google_project
+  message_retention_duration = "604800s"
+
+  expiration_policy {
+    ttl = ""
+  }
+
+  dead_letter_policy {
+    dead_letter_topic     = google_pubsub_topic.reprocess_dead_letter_topic.id
+    max_delivery_attempts = 5
+  }
+
+  retry_policy {
+    minimum_backoff = "10s"
+    maximum_backoff = "600s"
+  }
+}

modules/osdu/pubsub-schema.tf

+/**
+*  Copyright 2023 Google LLC
+*  Copyright 2023 EPAM
+*
+*  Licensed under the Apache License, Version 2.0 (the "License");
+*  you may not use this file except in compliance with the License.
+*  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+*  Unless required by applicable law or agreed to in writing, software
+*  distributed under the License is distributed on an "AS IS" BASIS,
+*  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*  See the License for the specific language governing permissions and
+*  limitations under the License.
+*/
+
+# Schema topics & subscriptions
+resource "google_pubsub_topic" "schema_topic" {
+  depends_on = [module.pubsub_sa]
+  name       = "schema-changed"
+  project    = var.service_google_project
+}
+
+resource "google_pubsub_topic" "schema_dead_letter_topic" {
+  depends_on = [module.pubsub_sa]
+  name       = "schema-changed-dead-letter"
+  project    = var.service_google_project
+}
+
+resource "google_pubsub_subscription" "schema_subscription" {
+  # for_each = local.data_partition_list - multipartition is not implemented yet here
+
+  topic                      = google_pubsub_topic.schema_topic.id
+  name                       = "indexer-schema-changed"
+  project                    = var.service_google_project
+  message_retention_duration = "604800s"
+
+  expiration_policy {
+    ttl = ""
+  }
+
+  dead_letter_policy {
+    dead_letter_topic     = google_pubsub_topic.schema_dead_letter_topic.id
+    max_delivery_attempts = 5
+  }
+
+  retry_policy {
+    minimum_backoff = "10s"
+    maximum_backoff = "600s"
+  }
+}
+
+resource "google_pubsub_subscription" "notification_service_schema_subscription" {
+  # for_each = local.data_partition_list - multipartition is not implemented yet here
+
+  topic                      = google_pubsub_topic.schema_topic.id
+  name                       = "notification-schema-changed-service"
+  project                    = var.service_google_project
+  message_retention_duration = "604800s"
+
+  expiration_policy {
+    ttl = ""
+  }
+
+  dead_letter_policy {
+    dead_letter_topic     = google_pubsub_topic.notification_dead_letter_topic.id
+    max_delivery_attempts = 5
+  }
+
+  retry_policy {
+    minimum_backoff = "10s"
+    maximum_backoff = "600s"
+  }
+}
+
+resource "google_pubsub_topic" "schema_publish_topic" {
+  depends_on = [module.pubsub_sa]
+  name       = "schema-changed-publsih"
+  project    = var.service_google_project
+}
+
+resource "google_pubsub_topic" "schema_publish_dead_letter_topic" {
+  depends_on = [module.pubsub_sa]
+  name       = "schema-changed-publish-dead-letter"
+  project    = var.service_google_project
+}
+
+resource "google_pubsub_subscription" "notification_publish_schema_subscription" {
+  # for_each = local.data_partition_list - multipartition is not implemented yet here
+
+  topic                      = google_pubsub_topic.schema_publish_topic.id
+  name                       = "notification-schema-changed-publish"
+  project                    = var.service_google_project
+  message_retention_duration = "604800s"
+
+  expiration_policy {
+    ttl = ""
+  }
+
+  dead_letter_policy {
+    dead_letter_topic     = google_pubsub_topic.notification_dead_letter_topic.id
+    max_delivery_attempts = 5
+  }
+
+  retry_policy {
+    minimum_backoff = "10s"
+    maximum_backoff = "600s"
+  }
+}

modules/osdu/pubsub-status.tf

+/**
+*  Copyright 2023 Google LLC
+*  Copyright 2023 EPAM
+*
+*  Licensed under the Apache License, Version 2.0 (the "License");
+*  you may not use this file except in compliance with the License.
+*  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+*  Unless required by applicable law or agreed to in writing, software
+*  distributed under the License is distributed on an "AS IS" BASIS,
+*  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*  See the License for the specific language governing permissions and
+*  limitations under the License.
+*/
+
+# Status topics & subscriptions
+resource "google_pubsub_topic" "status_topic" {
+  depends_on = [module.pubsub_sa]
+  name       = "status-changed"
+  project    = var.service_google_project
+}
+
+resource "google_pubsub_topic" "status_dead_letter_topic" {
+  depends_on = [module.pubsub_sa]
+  name       = "status-changed-dead-letter"
+  project    = var.service_google_project
+}
+
+resource "google_pubsub_subscription" "notification_service_status_subscription" {
+  # for_each = local.data_partition_list - multipartition is not implemented yet here
+
+  topic                      = google_pubsub_topic.status_topic.id
+  name                       = "notification-status-changed-service"
+  project                    = var.service_google_project
+  message_retention_duration = "604800s"
+
+  expiration_policy {
+    ttl = ""
+  }
+
+  dead_letter_policy {
+    dead_letter_topic     = google_pubsub_topic.status_dead_letter_topic.id
+    max_delivery_attempts = 10
+  }
+
+  retry_policy {
+    minimum_backoff = "10s"
+    maximum_backoff = "600s"
+  }
+}
+
+resource "google_pubsub_topic" "status_publish_topic" {
+  depends_on = [module.pubsub_sa]
+  name       = "status-changed-publish"
+  project    = var.service_google_project
+}
+
+resource "google_pubsub_subscription" "notification_publish_status_subscription" {
+  # for_each = local.data_partition_list - multipartition is not implemented yet here
+
+  topic                      = google_pubsub_topic.status_topic.id
+  name                       = "notification-status-changed-publish"
+  project                    = var.service_google_project
+  message_retention_duration = "604800s"
+
+  expiration_policy {
+    ttl = ""
+  }
+
+  dead_letter_policy {
+    dead_letter_topic     = google_pubsub_topic.notification_dead_letter_topic.id
+    max_delivery_attempts = 10
+  }
+
+  retry_policy {
+    minimum_backoff = "10s"
+    maximum_backoff = "600s"
+  }
+}