Newer
Older
![Daniel Scholl (MS]'s avatar](https://secure.gravatar.com/avatar/4b98f2e25f83d067f827cf9d9e878f0c54687e00c7e0647530d5c916467c0ec4?s=144&d=identicon "Daniel Scholl (MS]")
################################################################################
# Specify the azure environment specific values
#
aadpodidbinding: "osdu-identity"
################################################################################
# Specify any optional override values
#
image:
repository: #{container-registry}#.azurecr.io
branch: #{ENVIRONMENT_NAME}#
tag: #{Build.SourceVersion}#
################################################################################
# Specify the airflow configuration
#
###################################
# Kubernetes - Ingress Configs
###################################
ingress:
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
web:
annotations:
kubernetes.io/ingress.class: azure/application-gateway
appgw.ingress.kubernetes.io/request-timeout: "300"
appgw.ingress.kubernetes.io/connection-draining: "true"
appgw.ingress.kubernetes.io/connection-draining-timeout: "30"
cert-manager.io/cluster-issuer: letsencrypt
cert-manager.io/acme-challenge-type: http01
path: "/airflow"
host: #{DNS_HOST}#
livenessPath: "/airflow/health"
tls:
enabled: true
secretName: osdu-certificate
precedingPaths:
- path: "/airflow/*"
serviceName: airflow-web
servicePort: 8080
###################################
# Database - External Database
###################################
postgresql:
enabled: false
externalDatabase:
type: postgres
host: #{base-name-sr}#-pg.postgres.database.azure.com #<-- Azure PostgreSQL Database host
user: osdu_admin@#{base-name-sr}#-pg #<-- Azure PostgreSQL Database username, formatted as {username}@{hostname}
passwordSecret: "postgres"
passwordSecretKey: "postgres-password"
port: 5432
properties: "?sslmode=require"
database: airflow
###################################
# Database - External Redis
###################################
redis:
enabled: false
externalRedis:
host: #{base-name-sr}#-cache.redis.cache.windows.net #<-- Azure Redis Cache host
port: 6380
passwordSecret: "redis"
passwordSecretKey: "redis-password"
###################################
# Airflow - DAGs Configs
###################################
dags:
installRequirements: true
persistence:
enabled: true
existingClaim: airflowdagpvc
subPath: "dags"
###################################
# Airflow - WebUI Configs
###################################
web:
podLabels:
aadpodidbinding: "osdu-identity"
baseUrl: "http://localhost/airflow"
###################################
# Airflow - Worker Configs
###################################
workers:
podLabels:
aadpodidbinding: "osdu-identity"
autoscaling:
enabled: true
## minReplicas is picked from Values.workers.replicas and default value is 1
maxReplicas: 3
metrics:
- type: Resource
resource:
name: memory
target:
type: Utilization
averageUtilization: 60
resources:
requests:
memory: "512Mi"
###################################
# Airflow - Flower Configs
###################################
flower:
enabled: false
###################################
# Airflow - Scheduler Configs
###################################
scheduler:
podLabels:
aadpodidbinding: "osdu-identity"
variables: |
{}
###################################
# Airflow - Common Configs
###################################
airflow:
image:
repository: apache/airflow
tag: 1.10.12-python3.6
pullPolicy: IfNotPresent
pullSecret: ""
config:
AIRFLOW__SCHEDULER__STATSD_ON: "True"
AIRFLOW__SCHEDULER__STATSD_HOST: "appinsights-statsd"
AIRFLOW__SCHEDULER__STATSD_PORT: 8125
AIRFLOW__SCHEDULER__STATSD_PREFIX: "osdu_airflow"
AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "False"
## Enable for Debug purpose
AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "False"
AIRFLOW__WEBSERVER__AUTHENTICATE: "True"
AIRFLOW__WEBSERVER__AUTH_BACKEND: "airflow.contrib.auth.backends.password_auth"
AIRFLOW__API__AUTH_BACKEND: "airflow.contrib.auth.backends.password_auth"
AIRFLOW__CORE__REMOTE_LOGGING: "True"
AIRFLOW__CORE__REMOTE_LOG_CONN_ID: "az_log"
AIRFLOW__CORE__REMOTE_BASE_LOG_FOLDER: "wasb-airflowlog"
AIRFLOW__CORE__LOGGING_CONFIG_CLASS: "log_config.DEFAULT_LOGGING_CONFIG"
AIRFLOW__CORE__LOG_FILENAME_TEMPLATE: "{{ run_id }}/{{ ti.dag_id }}/{{ ti.task_id }}/{{ ts }}/{% if 'correlationId' in dag_run.conf %}{{ dag_run.conf['correlationId'] }}{% else %}None{% endif %}/{{ try_number }}.log"
AIRFLOW__CELERY__SSL_ACTIVE: "True"
AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX: "True"
AIRFLOW__CORE__PLUGINS_FOLDER: "/opt/airflow/plugins"
AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: 60
AIRFLOW__CORE__LOGGING_LEVEL: DEBUG
extraEnv:
- name: AIRFLOW__CORE__FERNET_KEY
valueFrom:
secretKeyRef:
name: airflow
- name: AIRFLOW_CONN_AZ_LOG
valueFrom:
secretKeyRef:
name: airflow
- name: CLOUD_PROVIDER
value: "azure"
- name: KEYVAULT_URI
valueFrom:
configMapKeyRef:
name: osdu-svc-properties
key: ENV_KEYVAULT
extraConfigmapMounts:
- name: remote-log-config
mountPath: /opt/airflow/config
configMap: airflow-remote-log-config
readOnly: true
extraPipPackages: [
"flask-bcrypt",
"apache-airflow[statsd]",
"apache-airflow[kubernetes]",
"apache-airflow-backport-providers-microsoft-azure",
"dataclasses",
"google-cloud-storage",
"azure-identity",
"azure-keyvault-secrets",
"msal",
"https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_api-0.0.4.tar.gz"
]
extraVolumeMounts:
- name: azure-keyvault
mountPath: "/mnt/azure-keyvault"
readOnly: true
- name: dags-data
mountPath: /opt/airflow/plugins
subPath: plugins
extraVolumes:
- name: azure-keyvault
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: azure-keyvault