Merge branch 'fix_ssl_dp_airflow' into 'master'

Added code to add new security rule to fix SSL validation

See merge request !422

charts/airflow/templates/ingress/ssl/byoc-certificate-downloader.yaml

 {{- $isEnabled := .Values.airflow.isDataPartitionDeployment | default false -}}
 {{ if and $isEnabled .Values.airflow.ingress.web.tls.BYOCEnabled }}
-apiVersion: batch/v1
-kind: Job
+apiVersion: apps/v1
+kind: Deployment
 metadata:
-  # This Job is needed to download the certificate from keyvault via csi provider
-  name: "byoc-certificate-downloader-job"
+  # This Deployment is needed to download the certificate from keyvault via csi provider
+  name: byoc-certificate-downloader
   namespace: istio-system
   labels:
-    app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
-    app.kubernetes.io/instance: {{ .Release.Name | quote }}
-    app.kubernetes.io/version: {{ .Chart.AppVersion }}
+    app: byoc-certificate-downloader
 spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: byoc-certificate-downloader
   template:
     metadata:
-      name: "{{ .Release.Name }}"
       labels:
-        app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
-        app.kubernetes.io/instance: {{ .Release.Name | quote }}
-        helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+        app: byoc-certificate-downloader
         aadpodidbinding: "osdu-identity"
     spec:
-      restartPolicy: Never
       containers:
-      - name: certificate-downloader
-        image: python:3.6.12-slim-buster
-        command: ['sh','-c','python -c "import time"']
+      - name: byoc-certificate-downloader
+        image: mcr.microsoft.com/azuredocs/aci-helloworld
+        imagePullPolicy: IfNotPresent
         volumeMounts:
         - name: airflow-tls
           mountPath: "/mnt/airflow-tls"
@@ -41,5 +39,5 @@ spec:
           driver: secrets-store.csi.k8s.io
           readOnly: true
           volumeAttributes:
-            secretProviderClass: airflow-tls       
+            secretProviderClass: airflow-tls  
 {{ end }}
\ No newline at end of file

devops/tasks/tests-unit.yml

@@ -44,6 +44,8 @@ steps:
       TF_VAR_gitops_path: $(TF_VAR_gitops_path)
       TF_VAR_istio_int_load_balancer_ip: $(TF_VAR_istio_int_load_balancer_ip)
       TF_VAR_aks_dns_host: $(TF_VAR_aks_dns_host)
+      ${{ if ne(variables['TF_VAR_ssl_challenge_required'], '') }}:
+        TF_VAR_ssl_challenge_required: $(TF_VAR_ssl_challenge_required)
 
     condition: not(coalesce(variables.SKIP_TESTS, ${{ parameters.skip }}))
     inputs:

devops/tasks/tf-plan.yml

@@ -59,6 +59,8 @@ steps:
       TF_VAR_deploy_dp_airflow: $(TF_VAR_deploy_dp_airflow)
       TF_VAR_istio_int_load_balancer_ip: $(TF_VAR_istio_int_load_balancer_ip)
       TF_VAR_aks_dns_host: $(TF_VAR_aks_dns_host)
+      ${{ if ne(variables['TF_VAR_ssl_challenge_required'], '') }}:
+        TF_VAR_ssl_challenge_required: $(TF_VAR_ssl_challenge_required)
 
     inputs:
       azureSubscription: '$(SERVICE_CONNECTION_NAME)'

docs/infra-automation.md

@@ -119,6 +119,7 @@ To enable airflow multi partition turn on the feature flag by following the belo
   | Variable | Value |
   |----------|-------|
   | TF_VAR_deploy_dp_airflow | true |
+  | TF_VAR_ssl_challenge_required | true (if not using BYOC) <br> false (if using BYOC) |
 
 
 __Setup and Configure the ADO Library `Infrastructure Pipeline Secrets - demo`__

infra/modules/providers/azure/aks_deployment_resources/osdu_main.tf

@@ -92,10 +92,24 @@ resource "azurerm_network_security_group" "aks-nsg" {
   resource_group_name = var.resource_group_name
 }
 
+resource "azurerm_network_security_rule" "aks-nsg-http-allow-rule" {
+  count                       = var.ssl_challenge_required ? 1 : 0
+  name                        = "http-allow-rule"
+  priority                    = 100
+  direction                   = "Inbound"
+  access                      = "Allow"
+  protocol                    = "Tcp"
+  source_port_range           = "*"
+  destination_port_range      = "80"
+  source_address_prefix       = "*"
+  destination_address_prefix  = "*"
+  resource_group_name         = var.resource_group_name
+  network_security_group_name = azurerm_network_security_group.aks-nsg.name
+}
 
 resource "azurerm_network_security_rule" "aks-nsg-security-rule" {
   name                        = "nsg-rule"
-  priority                    = 100
+  priority                    = 110
   direction                   = "Inbound"
   access                      = "Allow"
   protocol                    = "Tcp"

infra/modules/providers/azure/aks_deployment_resources/variables.tf

@@ -114,3 +114,8 @@ variable "sr_aks_egress_ip_address" {
 variable "base_name" {
   description = "Resource Base name"
 }
+
+variable "ssl_challenge_required" {
+  description = "Flag to indicate whether http01 ssl challenge is required"
+  type        = bool
+}

infra/templates/osdu-r3-mvp/data_partition/airflow/airflow_main.tf

@@ -352,6 +352,7 @@ module "aks_deployment_resources" {
   osdu_identity_id                     = azurerm_user_assigned_identity.osduidentity.id
   base_name                            = var.base_name
   sr_aks_egress_ip_address             = var.sr_aks_egress_ip_address
+  ssl_challenge_required               = var.ssl_challenge_required
 }
 
 #-------------------------------

infra/templates/osdu-r3-mvp/data_partition/airflow/variables.tf

@@ -249,4 +249,9 @@ variable "redis_queue_zones" {
 variable "cr_keyvault_id" {
   description = "Id for Keyvault in Central Resources"
   type        = string
-}
\ No newline at end of file
+}
+
+variable "ssl_challenge_required" {
+  description = "Flag to indicate whether http01 ssl challenge is required"
+  type        = bool
+}

infra/templates/osdu-r3-mvp/data_partition/main.tf

@@ -488,6 +488,7 @@ module "airflow" {
   base_name_60 = local.base_name_60
 
   ssh_public_key_file      = var.ssh_public_key_file
+  ssl_challenge_required   = var.ssl_challenge_required
   sr_aks_egress_ip_address = data.terraform_remote_state.service_resources.outputs.aks_egress_ip_address
   cr_keyvault_id           = data.terraform_remote_state.central_resources.outputs.keyvault_id
 }

infra/templates/osdu-r3-mvp/data_partition/variables.tf

@@ -223,3 +223,9 @@ variable "sa_retention_days" {
   type        = number
   default     = 30
 }
+
+variable "ssl_challenge_required" {
+  description = "Flag to indicate whether http01 ssl challenge is required"
+  type        = bool
+  default     = true
+}