Commit 66fc70f9 authored by harshit aggarwal's avatar harshit aggarwal Committed by MANISH KUMAR
Browse files

Adding Airflow Multipartition Partition Changes

parent 017cd947
......@@ -24,6 +24,12 @@ data "azurerm_resource_group" "main" {
name = var.resource_group_name
}
data "azurerm_public_ip" "aks_egress_ip" {
// Splits the Resource Id for the Egress IP to get the name
name = split("/", tolist(azurerm_kubernetes_cluster.main.network_profile[0].load_balancer_profile[0].effective_outbound_ips)[0])[8]
resource_group_name = azurerm_kubernetes_cluster.main.node_resource_group
}
data "azurerm_subscription" "current" {}
resource "random_id" "main" {
......
......@@ -57,4 +57,8 @@ output "kubelet_client_id" {
output "node_resource_group" {
value = azurerm_kubernetes_cluster.main.node_resource_group
}
output "aks_egress_ip_address" {
value = data.azurerm_public_ip.aks_egress_ip.ip_address
}
\ No newline at end of file
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#-------------------------------
# Kubernetes Config Map
#-------------------------------
locals {
osdu_ns = "osdu"
}
resource "kubernetes_namespace" "osdu" {
count = var.feature_flag.osdu_namespace ? 1 : 0
metadata {
name = local.osdu_ns
labels = {
"istio-injection" = "enabled"
}
}
}
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#-------------------------------
# Certificate Manager
#-------------------------------
locals {
helm_certs_name = "jetstack"
helm_certs_ns = "cert-manager"
helm_certs_repo = "https://charts.jetstack.io"
helm_certs_version = "v1.1.0"
}
resource "kubernetes_namespace" "certs" {
metadata {
name = local.helm_certs_ns
labels = {
"cert-manager.io/disable-validation" = "true"
}
}
}
resource "helm_release" "certmgr" {
name = local.helm_certs_name
repository = local.helm_certs_repo
chart = "cert-manager"
version = local.helm_certs_version
namespace = local.helm_certs_ns
depends_on = [kubernetes_namespace.certs]
set {
name = "installCRDs"
value = true
}
}
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#-------------------------------
# Keda
#-------------------------------
locals {
helm_keda_name = "keda"
helm_keda_ns = "keda"
helm_keda_repo = "https://kedacore.github.io/charts"
helm_keda_version = "2.1.0"
}
resource "kubernetes_namespace" "keda" {
metadata {
name = local.helm_keda_ns
}
}
resource "helm_release" "keda" {
name = local.helm_keda_name
repository = local.helm_keda_repo
chart = "keda"
version = local.helm_keda_version
namespace = local.helm_keda_ns
depends_on = [kubernetes_namespace.keda]
}
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#-------------------------------
# KeyVault Secret Driver
#-------------------------------
locals {
helm_kv_csi_name = "kvsecrets"
helm_kv_csi_ns = "kvsecrets"
helm_kv_csi_repo = "https://raw.githubusercontent.com/Azure/secrets-store-csi-driver-provider-azure/master/charts"
helm_kv_csi_version = "0.0.15"
}
resource "kubernetes_namespace" "kvsecrets" {
metadata {
name = local.helm_kv_csi_ns
}
}
resource "helm_release" "kvsecrets" {
name = local.helm_kv_csi_name
repository = local.helm_kv_csi_repo
chart = "csi-secrets-store-provider-azure"
version = local.helm_kv_csi_version
namespace = local.helm_kv_csi_ns
set {
name = "secrets-store-csi-driver.linux.metricsAddr"
value = ":8081"
}
depends_on = [kubernetes_namespace.kvsecrets]
}
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#-------------------------------
# Pod Identity
#-------------------------------
locals {
pod_identity_name = "${var.aks_cluster_name}-pod-identity"
helm_pod_identity_name = "aad-pod-identity"
helm_pod_identity_ns = "podidentity"
helm_pod_identity_repo = "https://raw.githubusercontent.com/Azure/aad-pod-identity/master/charts"
helm_pod_identity_version = "3.0.0"
}
resource "kubernetes_namespace" "pod_identity" {
metadata {
name = local.helm_pod_identity_ns
}
}
resource "helm_release" "aad_pod_id" {
name = local.helm_pod_identity_name
repository = local.helm_pod_identity_repo
chart = "aad-pod-identity"
version = local.helm_pod_identity_version
namespace = kubernetes_namespace.pod_identity.metadata.0.name
depends_on = [kubernetes_namespace.pod_identity]
}
variable "feature_flag" {
description = "(Optional) A toggle for incubator features"
type = object({
osdu_namespace = bool
})
default = {
osdu_namespace = true
}
}
variable "aks_cluster_name" {
type = string
}
\ No newline at end of file
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/*
.Synopsis
Terraform Diagnostics Control
.DESCRIPTION
This file holds diagnostics settings.
*/
locals {
retention_policy = var.log_retention_days == 0 ? false : true
}
#-------------------------------
# Network
#-------------------------------
resource "azurerm_monitor_diagnostic_setting" "vnet_diagnostics" {
name = "vnet_diagnostics"
target_resource_id = module.network.id
log_analytics_workspace_id = var.log_analytics_id
log {
category = "VMProtectionAlerts"
enabled = false
retention_policy {
days = 0
enabled = false
}
}
metric {
category = "AllMetrics"
retention_policy {
days = var.log_retention_days
enabled = local.retention_policy
}
}
}
#-------------------------------
# Azure AKS
#-------------------------------
resource "azurerm_monitor_diagnostic_setting" "aks_diagnostics" {
name = "aks_diagnostics"
target_resource_id = module.aks.id
log_analytics_workspace_id = var.log_analytics_id
log {
category = "cluster-autoscaler"
retention_policy {
days = var.log_retention_days
enabled = local.retention_policy
}
}
log {
category = "guard"
enabled = false
retention_policy {
days = 0
enabled = false
}
}
log {
category = "kube-apiserver"
retention_policy {
days = var.log_retention_days
enabled = local.retention_policy
}
}
log {
category = "kube-audit"
retention_policy {
days = var.log_retention_days
enabled = local.retention_policy
}
}
log {
category = "kube-audit-admin"
retention_policy {
days = var.log_retention_days
enabled = local.retention_policy
}
}
log {
category = "kube-controller-manager"
retention_policy {
days = var.log_retention_days
enabled = local.retention_policy
}
}
log {
category = "kube-scheduler"
retention_policy {
days = var.log_retention_days
enabled = local.retention_policy
}
}
metric {
category = "AllMetrics"
retention_policy {
days = var.log_retention_days
enabled = local.retention_policy
}
}
}
#-------------------------------
# Network
#-------------------------------
module "network" {
source = "../network"
name = var.vnet_name
resource_group_name = var.resource_group_name
address_space = var.address_space
subnet_prefixes = [var.subnet_fe_prefix, var.subnet_aks_prefix]
subnet_names = [var.fe_subnet_name, var.aks_subnet_name]
subnet_service_endpoints = {
(var.aks_subnet_name) = ["Microsoft.Storage",
"Microsoft.Sql",
"Microsoft.KeyVault",
"Microsoft.EventHub"]
}
resource_tags = var.resource_tags
}
#-------------------------------
# Azure AKS
#-------------------------------
module "aks" {
source = "../aks"
name = var.aks_cluster_name
resource_group_name = var.resource_group_name
dns_prefix = var.aks_dns_prefix
agent_vm_count = var.aks_agent_vm_count
agent_vm_size = var.aks_agent_vm_size
agent_vm_disk = var.aks_agent_vm_disk
max_node_count = var.aks_agent_vm_maxcount
vnet_subnet_id = module.network.subnets.1
ssh_public_key = file(var.ssh_public_key_file)
kubernetes_version = var.kubernetes_version
log_analytics_id = var.log_analytics_id
msi_enabled = true
oms_agent_enabled = true
auto_scaling_default_node = true
kubeconfig_to_disk = false
enable_kube_dashboard = false
resource_tags = var.resource_tags
}
data "azurerm_resource_group" "aks_node_resource_group" {
name = module.aks.node_resource_group
}
// Give AKS Access rights to Operate the Node Resource Group
resource "azurerm_role_assignment" "all_mi_operator" {
principal_id = module.aks.kubelet_object_id
scope = data.azurerm_resource_group.aks_node_resource_group.id
role_definition_name = "Managed Identity Operator"
}
// Give AKS Access to Create and Remove VM's in Node Resource Group
resource "azurerm_role_assignment" "vm_contributor" {
principal_id = module.aks.kubelet_object_id
scope = data.azurerm_resource_group.aks_node_resource_group.id
role_definition_name = "Virtual Machine Contributor"
}
// Give AKS Access to Pull from Central ACR
resource "azurerm_role_assignment" "acr_reader" {
principal_id = module.aks.kubelet_object_id
scope = var.container_registry_id_central
role_definition_name = "AcrPull"
}
// Give AKS Access to Pull from Data partition ACR
resource "azurerm_role_assignment" "acr_reader_dp" {
principal_id = module.aks.kubelet_object_id
scope = var.container_registry_id_data_partition
role_definition_name = "AcrPull"
}
// Give AKS Access Rights to operate the OSDU Identity
resource "azurerm_role_assignment" "osdu_identity_mi_operator" {
principal_id = module.aks.kubelet_object_id
scope = var.osdu_identity_id
role_definition_name = "Managed Identity Operator"
}
resource "azurerm_network_security_group" "aks-nsg" {
name = "${var.base_name}-aks-nsg"
location = var.resource_group_location
resource_group_name = var.resource_group_name
}
resource "azurerm_network_security_rule" "aks-nsg-security-rule" {
name = "nsg-rule"
priority = 100
direction = "Inbound"
access = "Allow"
protocol = "Tcp"
source_port_range = "*"
destination_port_range = "*"
source_address_prefix = var.sr_aks_egress_ip_address
destination_address_prefix = "*"
resource_group_name = var.resource_group_name
network_security_group_name = azurerm_network_security_group.aks-nsg.name
}
resource "azurerm_subnet_network_security_group_association" "nsg-association" {
subnet_id = module.network.subnets.1
network_security_group_id = azurerm_network_security_group.aks-nsg.id
}
\ No newline at end of file
output "kube_config" {
sensitive = true
value = module.aks.kube_config
}
output "kube_config_block" {
sensitive = true
value = module.aks.kube_config_block
}
\ No newline at end of file
variable "resource_group_name" {
description = "The address space that is used by the virtual network."
type = string
}
variable "resource_group_location" {
description = "The address space that is used by the virtual network."
type = string
}
variable "resource_tags" {
description = "Map of tags to apply to this template."
type = map(string)
}
variable "vnet_name" {
description = "The address space that is used by the virtual network."
type = string
}
variable "fe_subnet_name" {
description = "The address space that is used by the virtual network."
type = string
}
variable "aks_subnet_name" {
description = "The address space that is used by the virtual network."
type = string
}
variable "address_space" {
description = "The address space that is used by the virtual network."
type = string
}
variable "subnet_fe_prefix" {
description = "The address prefix to use for the frontend subnet."
type = string
}
variable "subnet_aks_prefix" {
description = "The address prefix to use for the aks subnet."
type = string
}
variable "aks_cluster_name" {
description = "The address space that is used by the virtual network."
type = string
}
variable "aks_dns_prefix" {
description = "The address space that is used by the virtual network."
type = string
}
variable "log_analytics_id" {
description = "The address space that is used by the virtual network."
type = string
}
variable "log_retention_days" {
description = "Number of days to retain logs."
type = number
}
variable "aks_agent_vm_count" {
description = "The initial number of agent pools / nodes allocated to the AKS cluster"
type = string
}
variable "aks_agent_vm_maxcount" {
<