Skip to content
GitLab
Commit ff24578a authored by harshit aggarwal's avatar harshit aggarwal Committed by MANISH KUMAR
Browse files

[Airflow Multipartition] Removing keyvault added in CR for dp resources

parent 835ab312
Hide whitespace changes
Inline Side-by-side
charts/airflow/helm-config-dp.yaml
View file @ ff24578a
......@@ -2,27 +2,19 @@
# Specify the azure environment specific values
#
azure:
dp:
tenant: #{data-partition-tenant-id}#
subscription: #{data-partition-subscription-id}#
resourcegroup: #{base-name-dp}#-rg
identity: #{base-name-dp}#-osdu-identity
identity_id: #{management-identity-id}#
keyvault: #{base-name-dp}#-kv
cr:
tenant: #{tenant-id}#
subscription: #{subscription-id}#
resourcegroup: #{base-name-cr}#-rg
keyvault: #{base-name-cr}#-dpkv
tenant: #{data-partition-tenant-id}#
subscription: #{data-partition-subscription-id}#
resourcegroup: #{base-name-dp}#-rg
identity: #{base-name-dp}#-osdu-identity
identity_id: #{management-identity-id}#
keyvault: #{base-name-dp}#-kv
################################################################################
# App insights configuration
#
appinsightstatsd:
aadpodidbinding: "osdu-identity"
key: #{appinsights-key}#
#################################################################################
# Specify log analytics configuration
......@@ -284,10 +276,7 @@ airflow:
- name: AIRFLOW_VAR_AAD_CLIENT_ID
value: "#{AAD_CLIENT_ID}#"
- name: AIRFLOW_VAR_APPINSIGHTS_KEY
valueFrom:
secretKeyRef:
name: central-logging
key: appinsights
value: "#{appinsights-key}#"
extraConfigmapMounts:
- name: remote-log-config
mountPath: /opt/airflow/config
......
charts/airflow/templates/appinsight-statsd-deployment.yaml
View file @ ff24578a
{{- $isEnabled := .Values.airflow.isDataPartitionDeployment | default false -}}
{{- if $isEnabled }}
apiVersion: v1
kind: Secret
metadata:
name: central-logging
type: Opaque
stringData:
appinsights: {{ .Values.appinsightstatsd.key }}
---
{{- end }}
apiVersion: apps/v1
kind: Deployment
metadata:
......@@ -25,11 +35,6 @@ spec:
volumeMounts:
- name: config-volume
mountPath: /usr/src/app/statsd/backends/config
{{- if $isEnabled }}
- name: azure-cr-keyvault
mountPath: "/mnt/azure-cr-keyvault"
readOnly: true
{{- end }}
env:
- name: APPLICATION_INSIGHTS_INSTRUMENTATION_KEY
valueFrom:
......@@ -50,12 +55,4 @@ spec:
- name: config-volume
configMap:
name: airflow-appinsight-statsd-config
{{- if $isEnabled }}
- name: azure-cr-keyvault
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: azure-cr-keyvault
{{- end }}
\ No newline at end of file
charts/airflow/templates/identity.yaml
View file @ ff24578a
......@@ -6,8 +6,8 @@ metadata:
name: osdu-identity
spec:
type: 0
resourceID: "/subscriptions/{{ .Values.azure.dp.subscription }}/resourcegroups/{{ .Values.azure.dp.resourcegroup }}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{{ .Values.azure.dp.identity }}"
clientID: "{{ .Values.azure.dp.identity_id }}"
resourceID: "/subscriptions/{{ .Values.azure.subscription }}/resourcegroups/{{ .Values.azure.resourcegroup }}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{{ .Values.azure.identity }}"
clientID: "{{ .Values.azure.identity_id }}"
---
apiVersion: aadpodidentity.k8s.io/v1
kind: AzureIdentityBinding
......
charts/airflow/templates/ingress/ssl/byoc-certificate.yaml
View file @ ff24578a
......@@ -19,10 +19,10 @@ spec:
usePodIdentity: "true"
useVMManagedIdentity: "false"
userAssignedIdentityID: ""
resourceGroup: "{{ .Values.azure.dp.resourcegroup }}"
keyvaultName: "{{ .Values.azure.dp.keyvault }}"
subscriptionId: "{{ .Values.azure.dp.subscription }}"
tenantId: "{{ .Values.azure.dp.tenant }}"
resourceGroup: "{{ .Values.azure.resourcegroup }}"
keyvaultName: "{{ .Values.azure.keyvault }}"
subscriptionId: "{{ .Values.azure.subscription }}"
tenantId: "{{ .Values.azure.tenant }}"
objects: |
array:
- |
......
charts/airflow/templates/kv-secrets.yaml
View file @ ff24578a
......@@ -47,10 +47,10 @@ spec:
usePodIdentity: "true"
useVMManagedIdentity: "false"
userAssignedIdentityID: ""
resourceGroup: "{{ .Values.azure.dp.resourcegroup }}"
keyvaultName: "{{ .Values.azure.dp.keyvault }}"
subscriptionId: "{{ .Values.azure.dp.subscription }}"
tenantId: "{{ .Values.azure.dp.tenant }}"
resourceGroup: "{{ .Values.azure.resourcegroup }}"
keyvaultName: "{{ .Values.azure.keyvault }}"
subscriptionId: "{{ .Values.azure.subscription }}"
tenantId: "{{ .Values.azure.tenant }}"
objects: |
array:
- |
......@@ -83,30 +83,4 @@ spec:
- |
objectName: log-workspace-key
objectType: secret
---
apiVersion: secrets-store.csi.x-k8s.io/v1alpha1
kind: SecretProviderClass
metadata:
name: azure-cr-keyvault
spec:
provider: azure
secretObjects:
- secretName: central-logging
type: Opaque
data:
- objectName: "appinsights-key"
key: appinsights
parameters:
usePodIdentity: "true"
useVMManagedIdentity: "false"
userAssignedIdentityID: ""
resourceGroup: "{{ .Values.azure.cr.resourcegroup }}"
keyvaultName: "{{ .Values.azure.cr.keyvault }}"
subscriptionId: "{{ .Values.azure.cr.subscription }}"
tenantId: "{{ .Values.azure.cr.tenant }}"
objects: |
array:
- |
objectName: appinsights-key
objectType: secret
{{ end }}
infra/templates/osdu-r3-mvp/central_resources/main.tf
View file @ ff24578a
......@@ -83,7 +83,6 @@ locals {
retention_policy = var.log_retention_days == 0 ? false : true
kv_name = "${local.base_name_21}-kv"
kv_name_dp = length(local.base_name_21) < 19 ? "${local.base_name_21}-dpkv" : "${substr(local.base_name_21, 0, 19)}-dpkv"
storage_name = "${replace(local.base_name_21, "-", "")}tbl"
graphdb_name = "${local.base_name}-graph"
container_registry_name = "${replace(local.base_name_21, "-", "")}cr"
......@@ -151,20 +150,6 @@ module "keyvault" {
resource_tags = var.resource_tags
}
#-------------------------------
# Key Vault for Storing App insights Instrumentation Key
# required by Data Partition resource
#-------------------------------
module "keyvaultdp" {
source = "../../../modules/providers/azure/keyvault"
count = var.feature_flag.deploy_dp_airflow ? 1 : 0
keyvault_name = local.kv_name_dp
resource_group_name = azurerm_resource_group.main.name
resource_tags = var.resource_tags
}
module "keyvault_policy" {
source = "../../../modules/providers/azure/keyvault-policy"
......
infra/templates/osdu-r3-mvp/central_resources/output.tf
View file @ ff24578a
......@@ -74,9 +74,4 @@ output "principal_objectId" {
output "app_insights_name" {
description = "The name of the appinsights resource"
value = module.app_insights.app_insights_name
}
output "keyvault_dp_id" {
description = "URI for keyvault for Storing App insights Instrumentation Key required by Data Partition resource"
value = var.feature_flag.deploy_dp_airflow ? module.keyvaultdp.0.keyvault_id : ""
}
}
\ No newline at end of file
infra/templates/osdu-r3-mvp/central_resources/secrets.tf
View file @ ff24578a
......@@ -116,8 +116,6 @@ resource "azurerm_key_vault_secret" "insights" {
key_vault_id = module.keyvault.keyvault_id
}
#-------------------------------
# Log Analytics
#-------------------------------
......@@ -133,7 +131,6 @@ resource "azurerm_key_vault_secret" "workspace_key" {
key_vault_id = module.keyvault.keyvault_id
}
#-------------------------------
# AD Principal and Applications
#-------------------------------
......@@ -171,11 +168,4 @@ resource "azurerm_key_vault_secret" "identity_id" {
name = "osdu-identity-id"
value = azurerm_user_assigned_identity.osduidentity.client_id
key_vault_id = module.keyvault.keyvault_id
}
resource "azurerm_key_vault_secret" "insights_dp" {
count = var.feature_flag.deploy_dp_airflow ? 1 : 0
name = "appinsights-key"
value = module.app_insights.app_insights_instrumentation_key
key_vault_id = module.keyvaultdp.0.keyvault_id
}
\ No newline at end of file
infra/templates/osdu-r3-mvp/central_resources/variables.tf
View file @ ff24578a
......@@ -31,14 +31,12 @@ variable "prefix" {
variable "feature_flag" {
description = "(Optional) A toggle for incubator features"
type = object({
kv_lock = bool
acr_lock = bool
deploy_dp_airflow = bool
kv_lock = bool
acr_lock = bool
})
default = {
kv_lock = true
acr_lock = true
deploy_dp_airflow = false
kv_lock = true
acr_lock = true
}
}
......
infra/templates/osdu-r3-mvp/data_partition/airflow/airflow_main.tf
View file @ ff24578a
......@@ -208,28 +208,6 @@ resource "azurerm_role_assignment" "kv_roles" {
scope = module.keyvault.keyvault_id
}
// Policies for Keyvault in Central resources
module "keyvault_cr_dp_policy" {
source = "../../../../modules/providers/azure/keyvault-policy"
vault_id = data.terraform_remote_state.central_resources.outputs.keyvault_dp_id
tenant_id = data.azurerm_client_config.current.tenant_id
object_ids = [
azurerm_user_assigned_identity.osduidentity.principal_id
]
key_permissions = ["get", "encrypt", "decrypt"]
certificate_permissions = ["get"]
secret_permissions = ["get"]
}
resource "azurerm_role_assignment" "kv_cr_dp_roles" {
count = length(local.rbac_principals_airflow)
role_definition_name = "Reader"
principal_id = local.rbac_principals_airflow[count.index]
scope = data.terraform_remote_state.central_resources.outputs.keyvault_dp_id
}
#-------------------------------
# OSDU Identity
#-------------------------------
......
infra/templates/osdu-r3-mvp/data_partition/airflow/airflow_secrets.tf
View file @ ff24578a
......@@ -141,4 +141,58 @@ resource "azurerm_key_vault_secret" "data_partition_name" {
name = "data-partition-name"
value = var.data_partition_name
key_vault_id = module.keyvault.keyvault_id
}
// data source to output the value of secret
data "azurerm_key_vault_secret" "data_principal_id" {
name = "app-dev-sp-username"
key_vault_id = var.cr_keyvault_id
}
// data source to output the value of secret
data "azurerm_key_vault_secret" "data_principal_secret" {
name = "app-dev-sp-password"
key_vault_id = var.cr_keyvault_id
}
// data source to output the value of secret
data "azurerm_key_vault_secret" "data_application_id" {
name = "aad-client-id"
key_vault_id = var.cr_keyvault_id
}
// data source to output the value of secret
data "azurerm_key_vault_secret" "data_insights" {
name = "appinsights-key"
key_vault_id = var.cr_keyvault_id
}
resource "azurerm_key_vault_secret" "principal_id" {
name = "app-dev-sp-username"
value = data.azurerm_key_vault_secret.data_principal_id.value
key_vault_id = module.keyvault.keyvault_id
}
resource "azurerm_key_vault_secret" "principal_secret" {
name = "app-dev-sp-password"
value = data.azurerm_key_vault_secret.data_principal_secret.value
key_vault_id = module.keyvault.keyvault_id
}
resource "azurerm_key_vault_secret" "application_id" {
name = "aad-client-id"
value = data.azurerm_key_vault_secret.data_application_id.value
key_vault_id = module.keyvault.keyvault_id
}
resource "azurerm_key_vault_secret" "azure_tenant_id" {
name = "app-dev-sp-tenant-id"
value = data.azurerm_client_config.current.tenant_id
key_vault_id = module.keyvault.keyvault_id
}
resource "azurerm_key_vault_secret" "insights" {
name = "appinsights-key"
value = data.azurerm_key_vault_secret.data_insights.value
key_vault_id = module.keyvault.keyvault_id
}
\ No newline at end of file
infra/templates/osdu-r3-mvp/data_partition/airflow/variables.tf
View file @ ff24578a
......@@ -244,4 +244,9 @@ variable "redis_queue_zones" {
description = "A list of a one or more Availability Zones, where the Redis Cache should be allocated."
type = list(number)
default = [1, 2]
}
variable "cr_keyvault_id" {
description = "Id for Keyvault in Central Resources"
type = string
}
\ No newline at end of file
infra/templates/osdu-r3-mvp/data_partition/main.tf
View file @ ff24578a
......@@ -489,6 +489,7 @@ module "airflow" {
ssh_public_key_file = var.ssh_public_key_file
sr_aks_egress_ip_address = data.terraform_remote_state.service_resources.outputs.aks_egress_ip_address
cr_keyvault_id = data.terraform_remote_state.central_resources.outputs.keyvault_id
}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment