Commit a6ebb775 authored by Daniel Scholl's avatar Daniel Scholl
Browse files

Merge branch 'infra_changes_for_ingestion_workflow' into 'master'

Infra changes for ingestion workflow

See merge request osdu/platform/deployment-and-operations/infra-azure-provisioning!192
parents 03dd6b54 d90f7d7a
......@@ -65,13 +65,14 @@ resource "azurerm_cosmosdb_sql_database" "cosmos_dbs" {
resource "azurerm_cosmosdb_sql_container" "cosmos_collections" {
depends_on = [azurerm_cosmosdb_sql_database.cosmos_dbs]
count = length(var.sql_collections)
name = var.sql_collections[count.index].name
account_name = var.name
database_name = var.sql_collections[count.index].database_name
resource_group_name = data.azurerm_resource_group.cosmosdb.name
partition_key_path = var.sql_collections[count.index].partition_key_path
depends_on = [azurerm_cosmosdb_sql_database.cosmos_dbs]
count = length(var.sql_collections)
name = var.sql_collections[count.index].name
account_name = var.name
database_name = var.sql_collections[count.index].database_name
resource_group_name = data.azurerm_resource_group.cosmosdb.name
partition_key_path = var.sql_collections[count.index].partition_key_path
partition_key_version = var.sql_collections[count.index].partition_key_version
# autoscale_settings {
# max_throughput = var.sql_collections[count.index].throughput
......
......@@ -47,9 +47,10 @@ variable "consistency_level" {
variable "sql_collections" {
description = "The list of cosmos collection names to create. Names must be unique per cosmos instance."
type = list(object({
name = string
database_name = string
partition_key_path = string
name = string
database_name = string
partition_key_path = string
partition_key_version = number
}))
default = []
}
......
......@@ -56,110 +56,149 @@ cosmos_databases = [
]
cosmos_sql_collections = [
{
name = "LegalTag"
database_name = "osdu-db"
partition_key_path = "/id"
name = "LegalTag"
database_name = "osdu-db"
partition_key_path = "/id"
partition_key_version = null
},
{
name = "StorageRecord"
database_name = "osdu-db"
partition_key_path = "/id"
name = "StorageRecord"
database_name = "osdu-db"
partition_key_path = "/id"
partition_key_version = null
},
{
name = "StorageSchema"
database_name = "osdu-db"
partition_key_path = "/kind"
name = "StorageSchema"
database_name = "osdu-db"
partition_key_path = "/kind"
partition_key_version = null
},
{
name = "TenantInfo"
database_name = "osdu-db"
partition_key_path = "/id"
name = "TenantInfo"
database_name = "osdu-db"
partition_key_path = "/id"
partition_key_version = null
},
{
name = "UserInfo"
database_name = "osdu-db"
partition_key_path = "/id"
name = "UserInfo"
database_name = "osdu-db"
partition_key_path = "/id"
partition_key_version = null
},
{
name = "Authority"
database_name = "osdu-db"
partition_key_path = "/id"
name = "Authority"
database_name = "osdu-db"
partition_key_path = "/id"
partition_key_version = null
},
{
name = "EntityType"
database_name = "osdu-db"
partition_key_path = "/id"
name = "EntityType"
database_name = "osdu-db"
partition_key_path = "/id"
partition_key_version = null
},
{
name = "SchemaInfo"
database_name = "osdu-db"
partition_key_path = "/partitionId"
name = "SchemaInfo"
database_name = "osdu-db"
partition_key_path = "/partitionId"
partition_key_version = null
},
{
name = "Source"
database_name = "osdu-db"
partition_key_path = "/id"
name = "Source"
database_name = "osdu-db"
partition_key_path = "/id"
partition_key_version = null
},
{
name = "RegisterAction"
database_name = "osdu-db"
partition_key_path = "/dataPartitionId"
name = "RegisterAction"
database_name = "osdu-db"
partition_key_path = "/dataPartitionId"
partition_key_version = null
},
{
name = "RegisterDdms"
database_name = "osdu-db"
partition_key_path = "/dataPartitionId"
name = "RegisterDdms"
database_name = "osdu-db"
partition_key_path = "/dataPartitionId"
partition_key_version = null
},
{
name = "RegisterSubscription"
database_name = "osdu-db"
partition_key_path = "/dataPartitionId"
name = "RegisterSubscription"
database_name = "osdu-db"
partition_key_path = "/dataPartitionId"
partition_key_version = null
},
{
name = "IngestionStrategy"
database_name = "osdu-db"
partition_key_path = "/workflowType"
name = "IngestionStrategy"
database_name = "osdu-db"
partition_key_path = "/workflowType"
partition_key_version = null
},
{
name = "WorkflowStatus"
database_name = "osdu-db"
partition_key_path = "/workflowId"
name = "WorkflowStatus"
database_name = "osdu-db"
partition_key_path = "/workflowId"
partition_key_version = null
},
{
name = "Workflow"
database_name = "osdu-db"
partition_key_path = "/workflowId"
name = "Workflow"
database_name = "osdu-db"
partition_key_path = "/workflowId"
partition_key_version = null
},
{
name = "WorkflowRun"
database_name = "osdu-db"
partition_key_path = "/workflowId"
name = "WorkflowRun"
database_name = "osdu-db"
partition_key_path = "/workflowId"
partition_key_version = null
},
{
name = "RelationshipStatus"
database_name = "osdu-db"
partition_key_path = "/id"
name = "RelationshipStatus"
database_name = "osdu-db"
partition_key_path = "/id"
partition_key_version = null
},
{
name = "MappingInfo"
database_name = "osdu-db"
partition_key_path = "/sourceSchemaKind"
name = "MappingInfo"
database_name = "osdu-db"
partition_key_path = "/sourceSchemaKind"
partition_key_version = null
},
{
name = "WorkflowTasksSharingInfo"
database_name = "osdu-db"
partition_key_path = "/workflowId"
name = "WorkflowTasksSharingInfo"
database_name = "osdu-db"
partition_key_path = "/workflowId"
partition_key_version = null
},
{
name = "FileLocationEntity"
database_name = "osdu-db"
partition_key_path = "/id"
name = "FileLocationEntity"
database_name = "osdu-db"
partition_key_path = "/id"
partition_key_version = null
},
{
name = "WorkflowCustomOperatorInfo"
database_name = "osdu-db"
partition_key_path = "/operatorId"
name = "WorkflowCustomOperatorInfo"
database_name = "osdu-db"
partition_key_path = "/operatorId"
partition_key_version = null
},
{
name = "WorkflowV2"
database_name = "osdu-db"
partition_key_path = "/partitionKey"
partition_key_version = 2
},
{
name = "WorkflowRunV2"
database_name = "osdu-db"
partition_key_path = "/partitionKey"
partition_key_version = 2
},
{
name = "WorkflowCustomOperatorV2"
database_name = "osdu-db"
partition_key_path = "/partitionKey"
partition_key_version = 2
}
]
......
......@@ -49,7 +49,7 @@ func TestTemplate(t *testing.T) {
TfOptions: tfOptions,
Workspace: workspace,
PlanAssertions: nil,
ExpectedResourceCount: 102,
ExpectedResourceCount: 105,
ExpectedResourceAttributeValues: resourceDescription,
}
......
......@@ -126,9 +126,10 @@ variable "cosmos_databases" {
variable "cosmos_sql_collections" {
description = "The list of cosmos collection names to create. Names must be unique per cosmos instance."
type = list(object({
name = string
database_name = string
partition_key_path = string
name = string
database_name = string
partition_key_path = string
partition_key_version = number
}))
default = []
}
......
......@@ -93,6 +93,20 @@ resource "azurerm_storage_share_directory" "operators" {
depends_on = [azurerm_storage_share_directory.plugins]
}
resource "azurerm_storage_share_directory" "hooks" {
name = "plugins/hooks"
share_name = azurerm_storage_share.airflow_share.name
storage_account_name = module.storage_account.name
depends_on = [azurerm_storage_share_directory.plugins]
}
resource "azurerm_storage_share_directory" "sensors" {
name = "plugins/sensors"
share_name = azurerm_storage_share.airflow_share.name
storage_account_name = module.storage_account.name
depends_on = [azurerm_storage_share_directory.plugins]
}
// Airflow log container
resource "azurerm_storage_container" "main" {
name = "airflow-logs"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment