Commit 0ae25775 authored by Daniel Scholl's avatar Daniel Scholl
Browse files

Add Cosmos Graph Database to Central Resources

parent c9794c2f
......@@ -6,6 +6,7 @@ __Infra Changes__
- [Issue 75 - Upgrade Infrastructure tools and software dependencies](https://community.opengroup.org/osdu/platform/deployment-and-operations/infra-azure-provisioning/-/issues/75) - _* Manual Intervention Required_
- [Issue 76 - Add Terraform Service Resource Template Feature Flags](https://community.opengroup.org/osdu/platform/deployment-and-operations/infra-azure-provisioning/-/issues/76)
- [Issue 80 - Feature Change - Data Partition - Enable CORS configuration for Blob Containers on Storage Accounts](https://community.opengroup.org/osdu/platform/deployment-and-operations/infra-azure-provisioning/-/issues/80)
- [Issue 77 - Architecture Change - Central Resources - Add Graph Database](https://community.opengroup.org/osdu/platform/deployment-and-operations/infra-azure-provisioning/-/issues/77)
......
docs/images/architecture.png

144 KB | W: | H:

docs/images/architecture.png

143 KB | W: | H:

docs/images/architecture.png
docs/images/architecture.png
docs/images/architecture.png
docs/images/architecture.png
  • 2-up
  • Swipe
  • Onion skin
......@@ -26,6 +26,13 @@ resource "azurerm_cosmosdb_account" "cosmosdb" {
enable_automatic_failover = var.automatic_failover
dynamic "capabilities" {
for_each = var.graph_databases == null ? [] : [1]
content {
name = "EnableGremlin"
}
}
consistency_policy {
consistency_level = var.consistency_level
}
......@@ -77,3 +84,51 @@ resource "azurerm_cosmosdb_sql_container" "cosmos_collections" {
]
}
}
resource "azurerm_cosmosdb_gremlin_database" "cosmos_dbs" {
depends_on = [azurerm_cosmosdb_account.cosmosdb]
count = var.graph_databases == null ? 0 : length(var.graph_databases)
name = var.graph_databases[count.index].name
account_name = var.name
resource_group_name = data.azurerm_resource_group.cosmosdb.name
throughput = null
autoscale_settings {
max_throughput = var.graph_databases[count.index].throughput
}
lifecycle {
ignore_changes = [
autoscale_settings,
throughput
]
}
}
resource "azurerm_cosmosdb_gremlin_graph" "cosmos_graphs" {
depends_on = [azurerm_cosmosdb_gremlin_database.cosmos_dbs]
count = length(var.graphs)
name = var.graphs[count.index].name
account_name = var.name
database_name = var.graphs[count.index].database_name
resource_group_name = data.azurerm_resource_group.cosmosdb.name
partition_key_path = var.graphs[count.index].partition_key_path
index_policy {
automatic = true
indexing_mode = "Consistent"
included_paths = ["/*"]
}
conflict_resolution_policy {
mode = "LastWriterWins"
conflict_resolution_path = "/_ts"
}
lifecycle {
ignore_changes = [
autoscale_settings,
throughput
]
}
}
......@@ -10,11 +10,11 @@ module "resource_group" {
}
module "cosmosdb_autoscale" {
module "cosmosdb_sql" {
source = "../"
depends_on = [module.resource_group]
name = "osdu-module-db2-${module.resource_group.random}"
name = "osdu-module-sql-${module.resource_group.random}"
resource_group_name = module.resource_group.name
kind = "GlobalDocumentDB"
......@@ -46,3 +46,40 @@ module "cosmosdb_autoscale" {
source = "terraform",
}
}
module "cosmosdb_graph" {
source = "../"
depends_on = [module.resource_group]
name = "osdu-module-graph-${module.resource_group.random}"
resource_group_name = module.resource_group.name
kind = "GlobalDocumentDB"
automatic_failover = true
consistency_level = "Session"
primary_replica_location = module.resource_group.location
graph_databases = [
{
name = "osdu-module-database"
throughput = 4000 # This is max throughput Minimum level is 4000
}
]
graphs = [
{
name = "osdu-module-graph1"
database_name = "osdu-module-database"
partition_key_path = "/mypartition"
},
{
name = "osdu-module-graph2"
database_name = "osdu-module-database"
partition_key_path = "/mypartition"
}
]
resource_tags = {
source = "terraform",
}
}
......@@ -11,7 +11,7 @@ import (
var name = "cosmosdb-"
var location = "eastus"
var count = 7
var count = 12
var tfOptions = &terraform.Options{
TerraformDir: "./",
......@@ -28,7 +28,7 @@ func asMap(t *testing.T, jsonString string) map[string]interface{} {
func TestTemplate(t *testing.T) {
expectedAccountResult := asMap(t, `{
expectedSqlAccountResult := asMap(t, `{
"kind": "GlobalDocumentDB",
"enable_automatic_failover": true,
"enable_multiple_write_locations": false,
......@@ -36,6 +36,20 @@ func TestTemplate(t *testing.T) {
"offer_type": "Standard",
"consistency_policy": [{
"consistency_level": "Session"
}]
}`)
expectedGraphAccountResult := asMap(t, `{
"kind": "GlobalDocumentDB",
"enable_automatic_failover": true,
"enable_multiple_write_locations": false,
"is_virtual_network_filter_enabled": false,
"offer_type": "Standard",
"consistency_policy": [{
"consistency_level": "Session"
}],
"capabilities": [{
"name": "EnableGremlin"
}]
}`)
......@@ -50,6 +64,12 @@ func TestTemplate(t *testing.T) {
"database_name": "osdu-module-database",
"name": "osdu-module-container1",
"partition_key_path": "/id"
}`)
expectedGraphResult := asMap(t, `{
"database_name": "osdu-module-database",
"name": "osdu-module-graph1",
"partition_key_path": "/mypartition"
}`)
testFixture := infratests.UnitTestFixture{
......@@ -59,9 +79,12 @@ func TestTemplate(t *testing.T) {
PlanAssertions: nil,
ExpectedResourceCount: count,
ExpectedResourceAttributeValues: infratests.ResourceDescription{
"module.cosmosdb_autoscale.azurerm_cosmosdb_account.cosmosdb": expectedAccountResult,
"module.cosmosdb_autoscale.azurerm_cosmosdb_sql_database.cosmos_dbs[0]": expectedDatabaseResult,
"module.cosmosdb_autoscale.azurerm_cosmosdb_sql_container.cosmos_collections[0]": expectedContainerResult,
"module.cosmosdb_sql.azurerm_cosmosdb_account.cosmosdb": expectedSqlAccountResult,
"module.cosmosdb_graph.azurerm_cosmosdb_account.cosmosdb": expectedGraphAccountResult,
"module.cosmosdb_sql.azurerm_cosmosdb_sql_database.cosmos_dbs[0]": expectedDatabaseResult,
"module.cosmosdb_graph.azurerm_cosmosdb_gremlin_database.cosmos_dbs[0]": expectedDatabaseResult,
"module.cosmosdb_graph.azurerm_cosmosdb_gremlin_graph.cosmos_graphs[0]": expectedGraphResult,
"module.cosmosdb_sql.azurerm_cosmosdb_sql_container.cosmos_collections[0]": expectedContainerResult,
},
}
......
......@@ -50,7 +50,6 @@ variable "sql_collections" {
name = string
database_name = string
partition_key_path = string
# throughput = number
}))
default = []
}
......@@ -64,6 +63,25 @@ variable "databases" {
default = []
}
variable "graph_databases" {
description = "The list of Cosmos DB Graph Databases."
type = list(object({
name = string
throughput = number
}))
default = null
}
variable "graphs" {
description = "The list of cosmos graphs to create. Names must be unique per cosmos instance."
type = list(object({
name = string
database_name = string
partition_key_path = string
}))
default = []
}
variable "resource_tags" {
description = "Map of tags to apply to taggable resources in this module. By default the taggable resources are tagged with the name defined above and this map is merged in"
type = map(string)
......
......@@ -83,4 +83,102 @@ resource "azurerm_monitor_diagnostic_setting" "acr_diagnostics" {
enabled = local.retention_policy
}
}
}
\ No newline at end of file
}
#-------------------------------
# GraphDB
#-------------------------------
resource "azurerm_monitor_diagnostic_setting" "graph_diagnostics" {
name = "graph_diagnostics"
target_resource_id = module.graph_account.account_id
log_analytics_workspace_id = module.log_analytics.id
// This one always off.
log {
category = "CassandraRequests"
enabled = false
retention_policy {
days = 0
enabled = false
}
}
log {
category = "ControlPlaneRequests"
retention_policy {
days = var.log_retention_days
enabled = local.retention_policy
}
}
log {
category = "DataPlaneRequests"
enabled = true
retention_policy {
days = var.log_retention_days
enabled = local.retention_policy
}
}
// This one always off.
log {
category = "GremlinRequests"
enabled = true
retention_policy {
days = var.log_retention_days
enabled = local.retention_policy
}
}
// This one always off.
log {
category = "MongoRequests"
enabled = false
retention_policy {
days = 0
enabled = false
}
}
log {
category = "PartitionKeyRUConsumption"
retention_policy {
days = var.log_retention_days
enabled = local.retention_policy
}
}
log {
category = "PartitionKeyStatistics"
retention_policy {
days = var.log_retention_days
enabled = local.retention_policy
}
}
log {
category = "QueryRuntimeStatistics"
enabled = true
retention_policy {
days = var.log_retention_days
enabled = local.retention_policy
}
}
metric {
category = "Requests"
retention_policy {
days = var.log_retention_days
enabled = local.retention_policy
}
}
}
......@@ -84,6 +84,7 @@ locals {
kv_name = "${local.base_name_21}-kv"
storage_name = "${replace(local.base_name_21, "-", "")}tbl"
graphdb_name = "${local.base_name}-graph"
container_registry_name = "${replace(local.base_name_21, "-", "")}cr"
osdupod_identity_name = "${local.base_name}-osdu-identity"
ai_name = "${local.base_name}-ai"
......@@ -196,6 +197,32 @@ resource "azurerm_role_assignment" "storage_access" {
scope = module.storage_account.id
}
#-------------------------------
# CosmosDB
#-------------------------------
module "graph_account" {
source = "../../../modules/providers/azure/cosmosdb"
name = local.graphdb_name
resource_group_name = azurerm_resource_group.main.name
primary_replica_location = var.cosmosdb_replica_location
automatic_failover = var.cosmosdb_automatic_failover
consistency_level = var.cosmosdb_consistency_level
graph_databases = var.cosmos_graph_databases
graphs = var.cosmos_graphs
resource_tags = var.resource_tags
}
// Add Access Control to Principal
resource "azurerm_role_assignment" "graph_access" {
count = length(local.rbac_principals)
role_definition_name = "Contributor"
principal_id = local.rbac_principals[count.index]
scope = module.graph_account.account_id
}
#-------------------------------
# Container Registry
......@@ -203,7 +230,6 @@ resource "azurerm_role_assignment" "storage_access" {
module "container_registry" {
source = "../../../modules/providers/azure/container-registry"
container_registry_name = local.container_registry_name
resource_group_name = azurerm_resource_group.main.name
......@@ -339,3 +365,10 @@ resource "azurerm_management_lock" "acr_lock" {
scope = module.container_registry.container_registry_id
lock_level = "CanNotDelete"
}
// Lock the GraphDB
resource "azurerm_management_lock" "graph_lock" {
name = "osdu_graph_db_lock"
scope = module.graph_account.account_id
lock_level = "CanNotDelete"
}
......@@ -28,6 +28,10 @@ locals {
storage_account_name = format("tbl-storage")
storage_key_name = format("%s-key", local.storage_account_name)
graph_connection = format("graph-db-connection")
graph_endpoint = format("graph-db-endpoint")
graph_primary_key = format("graph-db-primary-key")
logs_id_name = "log-workspace-id"
logs_key_name = "log-workspace-key"
}
......@@ -81,6 +85,27 @@ resource "azurerm_key_vault_secret" "storage_key" {
}
#-------------------------------
# GraphDB
#-------------------------------
resource "azurerm_key_vault_secret" "graph_connection" {
name = local.graph_connection
value = module.graph_account.properties.cosmosdb.connection_strings[0]
key_vault_id = module.keyvault.keyvault_id
}
resource "azurerm_key_vault_secret" "graph_endpoint" {
name = local.graph_endpoint
value = module.graph_account.properties.cosmosdb.endpoint
key_vault_id = module.keyvault.keyvault_id
}
resource "azurerm_key_vault_secret" "graph_key" {
name = local.graph_primary_key
value = module.graph_account.properties.cosmosdb.primary_master_key
key_vault_id = module.keyvault.keyvault_id
}
#-------------------------------
# Application Insights
......@@ -147,4 +172,4 @@ resource "azurerm_key_vault_secret" "identity_id" {
name = "osdu-identity-id"
value = azurerm_user_assigned_identity.osduidentity.client_id
key_vault_id = module.keyvault.keyvault_id
}
\ No newline at end of file
}
......@@ -27,3 +27,20 @@ resource_tags = {
# Storage Settings
storage_replication_type = "LRS"
# Database Settings
cosmosdb_consistency_level = "Session"
cosmos_graph_databases = [
{
name = "osdu-graph"
throughput = 4000
}
]
cosmos_graphs = [
{
name = "users"
database_name = "osdu-graph"
partition_key_path = "/oid"
}
]
......@@ -49,7 +49,7 @@ func TestTemplate(t *testing.T) {
TfOptions: tfOptions,
Workspace: workspace,
PlanAssertions: nil,
ExpectedResourceCount: 48,
ExpectedResourceCount: 59,
ExpectedResourceAttributeValues: resourceDescription,
}
......
......@@ -82,3 +82,40 @@ variable "storage_replication_type" {
type = string
default = "LRS"
}
variable "cosmosdb_replica_location" {
description = "The name of the Azure region to host replicated data. i.e. 'East US' 'East US 2'. More locations can be found at https://azure.microsoft.com/en-us/global-infrastructure/locations/"
type = string
default = null
}
variable "cosmosdb_consistency_level" {
description = "The level of consistency backed by SLAs for Cosmos database. Developers can chose from five well-defined consistency levels on the consistency spectrum."
type = string
default = "Session"
}
variable "cosmosdb_automatic_failover" {
description = "Determines if automatic failover is enabled for CosmosDB."
type = bool
default = true
}
variable "cosmos_graph_databases" {
description = "The list of Cosmos DB Graph Databases."
type = list(object({
name = string
throughput = number
}))
default = null
}
variable "cosmos_graphs" {
description = "The list of cosmos graphs to create. Names must be unique per cosmos instance."
type = list(object({
name = string
database_name = string
partition_key_path = string
}))
default = []
}
......@@ -29,7 +29,7 @@ variable "prefix" {
}
variable "feature_flag" {
description = "(Optional) A toggle for experimental features"
description = "(Optional) A toggle for incubator features"
type = object({
osdu_namespace = bool
flux = bool
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment