helm-config.yaml 8.72 KB
Newer Older
1
2
3
################################################################################
# Specify the azure environment specific values
#
Daniel Scholl's avatar
Daniel Scholl committed
4
appinsightstatsd:
5
6
7
8
9
10
11
12
13
14
  aadpodidbinding: "osdu-identity"

################################################################################
# Specify any optional override values
#
image:
  repository: #{container-registry}#.azurecr.io
  branch: #{ENVIRONMENT_NAME}#
  tag: #{Build.SourceVersion}#

Daniel Scholl's avatar
Daniel Scholl committed
15
16
airflowLogin:
  name: admin
17

18
19
20
21
22
23
24
################################################################################
# Specify any custom configs/environment values
#
customConfig:
  rbac:
    createUser: "True"

25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
################################################################################
# Specify pgbouncer configuration
#
pgbouncer:
  enabled: true
  port: 6543
  max_client_connections: 3000
  airflowdb:
    name: airflow
    host: #{base-name-sr}#-pg.postgres.database.azure.com
    port: 5432
    pool_size: 100
    user:  osdu_admin@#{base-name-sr}#-pg
    passwordSecret: "postgres"
    passwordSecretKey: "postgres-password"


42
43
44
################################################################################
# Specify the airflow configuration
#
Daniel Scholl's avatar
Daniel Scholl committed
45
airflow:
46
47
48
49
50

  ###################################
  # Kubernetes - Ingress Configs
  ###################################
  ingress:
Daniel Scholl's avatar
Daniel Scholl committed
51
    enabled: true
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
    web:
      annotations:
        kubernetes.io/ingress.class: azure/application-gateway
        appgw.ingress.kubernetes.io/request-timeout: "300"
        appgw.ingress.kubernetes.io/connection-draining: "true"
        appgw.ingress.kubernetes.io/connection-draining-timeout: "30"
        cert-manager.io/cluster-issuer: letsencrypt
        cert-manager.io/acme-challenge-type: http01
      path: "/airflow"
      host: #{DNS_HOST}#
      livenessPath: "/airflow/health"
      tls:
        enabled: true
        secretName: osdu-certificate
      precedingPaths:
        - path: "/airflow/*"
          serviceName: airflow-web
          servicePort: 8080

  ###################################
  # Database - External Database
  ###################################
  postgresql:
    enabled: false
  externalDatabase:
    type: postgres
78
    host: airflow-pgbouncer.osdu.svc.cluster.local           #<-- Azure PostgreSQL Database host or pgbouncer host (if pgbouncer is enabled)
79
    user: osdu_admin@#{base-name-sr}#-pg                     #<-- Azure PostgreSQL Database username, formatted as {username}@{hostname}
80
81
    passwordSecret: "postgres"
    passwordSecretKey: "postgres-password"
82
    port: 6543
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
    database: airflow

  ###################################
  # Database - External Redis
  ###################################
  redis:
    enabled: false
  externalRedis:
    host: #{base-name-sr}#-cache.redis.cache.windows.net    #<-- Azure Redis Cache host
    port: 6380
    passwordSecret: "redis"
    passwordSecretKey: "redis-password"

  ###################################
  # Airflow - DAGs Configs
  ###################################
  dags:
    installRequirements: true
    persistence:
      enabled: true
      existingClaim: airflowdagpvc
      subPath: "dags"

  ###################################
  # Airflow - WebUI Configs
  ###################################
  web:
    podLabels:
      aadpodidbinding: "osdu-identity"
    baseUrl: "http://localhost/airflow"

  ###################################
  # Airflow - Worker Configs
  ###################################
  workers:
    podLabels:
      aadpodidbinding: "osdu-identity"
    autoscaling:
121
      enabled: false
122
123
124
125
126
127
128
129
130
      ## minReplicas is picked from Values.workers.replicas and default value is 1
      maxReplicas: 3
      metrics:
      - type: Resource
        resource:
          name: memory
          target:
            type: Utilization
            averageUtilization: 60
131
132
133
    labels:
      # DO NOT DELETE THIS LABEL. SET IT TO FALSE WHEN AUTOSCALING IS DISABLED, SET IT TO TRUE WHEN AUTOSCALING IS ENABLED
      autoscalingEnabled: false
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
    resources:
      requests:
        memory: "512Mi"

  ###################################
  # Airflow - Flower Configs
  ###################################
  flower:
    enabled: false

  ###################################
  # Airflow - Scheduler Configs
  ###################################
  scheduler:
    podLabels:
      aadpodidbinding: "osdu-identity"
    variables: |
      {}

  ###################################
  # Airflow - Common Configs
  ###################################
Daniel Scholl's avatar
Daniel Scholl committed
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
  airflow:
    image:
      repository: apache/airflow
      tag: 1.10.12-python3.6
      pullPolicy: IfNotPresent
      pullSecret: ""
    config:
      AIRFLOW__SCHEDULER__STATSD_ON: "True"
      AIRFLOW__SCHEDULER__STATSD_HOST: "appinsights-statsd"
      AIRFLOW__SCHEDULER__STATSD_PORT: 8125
      AIRFLOW__SCHEDULER__STATSD_PREFIX: "osdu_airflow"
      AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "False"
      ## Enable for Debug purpose
      AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "False"
      AIRFLOW__WEBSERVER__AUTHENTICATE: "True"
      AIRFLOW__WEBSERVER__AUTH_BACKEND: "airflow.contrib.auth.backends.password_auth"
172
      AIRFLOW__WEBSERVER__RBAC: "True"
Daniel Scholl's avatar
Daniel Scholl committed
173
174
175
176
177
      AIRFLOW__API__AUTH_BACKEND: "airflow.contrib.auth.backends.password_auth"
      AIRFLOW__CORE__REMOTE_LOGGING: "True"
      AIRFLOW__CORE__REMOTE_LOG_CONN_ID: "az_log"
      AIRFLOW__CORE__REMOTE_BASE_LOG_FOLDER: "wasb-airflowlog"
      AIRFLOW__CORE__LOGGING_CONFIG_CLASS: "log_config.DEFAULT_LOGGING_CONFIG"
178
      AIRFLOW__CORE__LOG_FILENAME_TEMPLATE: "{{ run_id }}/{{ ti.dag_id }}/{{ ti.task_id }}/{{ ts }}/{% if 'correlation_id' in dag_run.conf %}{{ dag_run.conf['correlation_id'] }}{% else %}None{% endif %}/{{ try_number }}.log"
Daniel Scholl's avatar
Daniel Scholl committed
179
180
      AIRFLOW__CELERY__SSL_ACTIVE: "True"
      AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX: "True"
181
      AIRFLOW__CORE__PLUGINS_FOLDER: "/opt/airflow/plugins"
182
      AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: 60
183
      AIRFLOW__CORE__LOGGING_LEVEL: DEBUG
184
185
      AIRFLOW_VAR_CORE__CONFIG__DATALOAD_CONFIG_PATH: "/opt/airflow/dags/configs/dataload.ini"
      AIRFLOW_VAR_CORE__SERVICE__SCHEMA__URL: "http://schema-service.osdu.svc.cluster.local/api/schema-service/v1/schema"
Kishore Battula's avatar
Kishore Battula committed
186
      AIRFLOW_VAR_CORE__SERVICE__SEARCH__URL: "http://search-service.osdu.svc.cluster.local/api/search/v2/query"
187
188
189
      AIRFLOW_VAR_CORE__SERVICE__STORAGE__URL: "http://storage.osdu.svc.cluster.local/api/storage/v2/records"
      AIRFLOW_VAR_CORE__SERVICE__FILE__HOST: "http://file.osdu.svc.cluster.local/api/file/v2"
      AIRFLOW_VAR_CORE__SERVICE__WORKFLOW__HOST: "http://ingestion-workflow.osdu.svc.cluster.local/api/workflow"
Daniel Scholl's avatar
Daniel Scholl committed
190
    extraEnv:
Daniel Scholl's avatar
Daniel Scholl committed
191
192
193
194
195
196
197
    - name: CLOUD_PROVIDER
      value: "azure"
    - name: AIRFLOW_VAR_KEYVAULT_URI
      valueFrom:
        configMapKeyRef:
          name: osdu-svc-properties
          key: ENV_KEYVAULT
Daniel Scholl's avatar
Daniel Scholl committed
198
199
200
201
    - name: AIRFLOW__CORE__FERNET_KEY
      valueFrom:
        secretKeyRef:
          name: airflow
202
          key: fernet-key
Daniel Scholl's avatar
Daniel Scholl committed
203
204
205
206
    - name: AIRFLOW_CONN_AZ_LOG
      valueFrom:
        secretKeyRef:
          name: airflow
207
          key: remote-log-connection
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
    - name: AIRFLOW_VAR_AZURE_TENANT_ID
      valueFrom:
        secretKeyRef:
          name: active-directory
          key: tenantid
    - name: AIRFLOW_VAR_AZURE_CLIENT_ID
      valueFrom:
        secretKeyRef:
          name: active-directory
          key: principal-clientid
    - name: AIRFLOW_VAR_AZURE_CLIENT_SECRET
      valueFrom:
        secretKeyRef:
          name: active-directory
          key: principal-clientpassword
    - name: AIRFLOW_VAR_AAD_CLIENT_ID
      valueFrom:
        secretKeyRef:
          name: active-directory
          key: application-appid
    - name: AIRFLOW_VAR_APPINSIGHTS_KEY
      valueFrom:
230
        secretKeyRef:
231
          name: central-logging
232
          key: appinsights
Daniel Scholl's avatar
Daniel Scholl committed
233
234
235
236
237
238
    extraConfigmapMounts:
        - name: remote-log-config
          mountPath: /opt/airflow/config
          configMap: airflow-remote-log-config
          readOnly: true
    extraPipPackages: [
239
        "flask-bcrypt==0.7.1",
Daniel Scholl's avatar
Daniel Scholl committed
240
241
        "apache-airflow[statsd]",
        "apache-airflow[kubernetes]",
242
243
        "apache-airflow-backport-providers-microsoft-azure==2021.2.5",
        "dataclasses==0.8",
244
        "google-cloud-storage",
245
246
247
248
        "python-keycloak==0.24.0",
        "msal==1.9.0",
        "azure-identity==1.5.0",
        "azure-keyvault-secrets==4.2.0",
249
        "azure-storage-blob",
250
251
        "azure-servicebus==7.0.1",
        "toposort==1.6",
Daniel Scholl's avatar
Daniel Scholl committed
252
        "https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_api-0.0.4.tar.gz"
Daniel Scholl's avatar
Daniel Scholl committed
253
254
255
256
257
    ]
    extraVolumeMounts:
        - name: azure-keyvault
          mountPath: "/mnt/azure-keyvault"
          readOnly: true
258
259
260
        - name: dags-data
          mountPath: /opt/airflow/plugins
          subPath: plugins
Daniel Scholl's avatar
Daniel Scholl committed
261
262
263
264
265
266
    extraVolumes:
        - name: azure-keyvault
          csi:
            driver: secrets-store.csi.k8s.io
            readOnly: true
            volumeAttributes:
267
              secretProviderClass: azure-keyvault