helm-config.yaml 12.1 KB
Newer Older
1
2
3
################################################################################
# Specify the azure environment specific values
#
Daniel Scholl's avatar
Daniel Scholl committed
4
appinsightstatsd:
5
6
  aadpodidbinding: "osdu-identity"

7
8
9
10
11
12
13
14
15
16
17
#################################################################################
# Specify log analytics configuration
#
logAnalytics:
  workspaceId:
    secretName: "central-logging"
    secretKey: "workspace-id"
  workspaceKey:
    secretName: "central-logging"
    secretKey: "workspace-key"

18
19
20
21
22
23
24
25
################################################################################
# Specify any optional override values
#
image:
  repository: #{container-registry}#.azurecr.io
  branch: #{ENVIRONMENT_NAME}#
  tag: #{Build.SourceVersion}#

Daniel Scholl's avatar
Daniel Scholl committed
26
27
airflowLogin:
  name: admin
28

harshit aggarwal's avatar
harshit aggarwal committed
29

harshit aggarwal's avatar
init    
harshit aggarwal committed
30
31
32
33
34
airflowAuthentication:
  username: admin
  keyvaultMountPath: /mnt/azure-keyvault/
  passwordKey: airflow-admin-password

35
36
37
38
39
40
41
################################################################################
# Specify any custom configs/environment values
#
customConfig:
  rbac:
    createUser: "True"

42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
################################################################################
# Specify pgbouncer configuration
#
pgbouncer:
  enabled: true
  port: 6543
  max_client_connections: 3000
  airflowdb:
    name: airflow
    host: #{base-name-sr}#-pg.postgres.database.azure.com
    port: 5432
    pool_size: 100
    user:  osdu_admin@#{base-name-sr}#-pg
    passwordSecret: "postgres"
    passwordSecretKey: "postgres-password"

58
59
60
61
62
63
################################################################################
# Specify KEDA configuration
#
keda:
  version_2_enabled: false

64

65
66
67
################################################################################
# Specify the airflow configuration
#
Daniel Scholl's avatar
Daniel Scholl committed
68
airflow:
69

70
71
72
73
74
75
76
77
78
  ##################################
  # Kubernetes Pod Operator config
  ##################################
  kubernetesPodOperator:
    namespace: airflow
  
  serviceAccount:
    name: airflow

79
80
81
82
  ###################################
  # Kubernetes - Ingress Configs
  ###################################
  ingress:
Daniel Scholl's avatar
Daniel Scholl committed
83
    enabled: true
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
    web:
      annotations:
        kubernetes.io/ingress.class: azure/application-gateway
        appgw.ingress.kubernetes.io/request-timeout: "300"
        appgw.ingress.kubernetes.io/connection-draining: "true"
        appgw.ingress.kubernetes.io/connection-draining-timeout: "30"
        cert-manager.io/cluster-issuer: letsencrypt
        cert-manager.io/acme-challenge-type: http01
      path: "/airflow"
      host: #{DNS_HOST}#
      livenessPath: "/airflow/health"
      tls:
        enabled: true
        secretName: osdu-certificate
      precedingPaths:
        - path: "/airflow/*"
          serviceName: airflow-web
          servicePort: 8080

  ###################################
  # Database - External Database
  ###################################
  postgresql:
    enabled: false
  externalDatabase:
    type: postgres
110
    host: airflow-pgbouncer.osdu.svc.cluster.local           #<-- Azure PostgreSQL Database host or pgbouncer host (if pgbouncer is enabled)
111
    user: osdu_admin@#{base-name-sr}#-pg                     #<-- Azure PostgreSQL Database username, formatted as {username}@{hostname}
112
113
    passwordSecret: "postgres"
    passwordSecretKey: "postgres-password"
114
    port: 6543
115
116
117
118
119
120
121
122
    database: airflow

  ###################################
  # Database - External Redis
  ###################################
  redis:
    enabled: false
  externalRedis:
123
    host: #{base-name-sr}#-queue.redis.cache.windows.net    #<-- Azure Redis Cache host
124
125
    port: 6380
    passwordSecret: "redis"
126
    passwordSecretKey: "redis-queue-password"
127
    databaseNumber: 1  #<-- Adding redis database number according to the Redis config map https://community.opengroup.org/osdu/platform/deployment-and-operations/infra-azure-provisioning/-/blob/master/charts/osdu-common/templates/redis-map.yaml#L7
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142

  ###################################
  # Airflow - DAGs Configs
  ###################################
  dags:
    installRequirements: true
    persistence:
      enabled: true
      existingClaim: airflowdagpvc
      subPath: "dags"

  ###################################
  # Airflow - WebUI Configs
  ###################################
  web:
harshit aggarwal's avatar
init    
harshit aggarwal committed
143
144
145
146
147
    replicas: 1
    livenessProbe:
      timeoutSeconds: 60
    resources:
      requests:
148
149
        cpu: "1000m"
        memory: "4Gi"
harshit aggarwal's avatar
init    
harshit aggarwal committed
150
      limits:
151
152
        cpu: "1000m"
        memory: "4Gi"
153
154
    podLabels:
      aadpodidbinding: "osdu-identity"
155
156
157
158
159
160
161
162
163
    autoscale:
      enabled: false
      minReplicas: 2
      maxReplicas: 20
      scaleDown:
        coolDownPeriod: 60
    labels:
      # DO NOT DELETE THIS LABEL. SET IT TO "false" WHEN AUTOSCALING IS DISABLED, SET IT TO "true" WHEN AUTOSCALING IS ENABLED
      autoscalingEnabled: "false"
harshit aggarwal's avatar
init    
harshit aggarwal committed
164
165
    podAnnotations:
      sidecar.istio.io/userVolumeMount: '[{"name": "azure-keyvault", "mountPath": "/mnt/azure-keyvault", "readonly": true}]'
166
167
168
169
170
171
    baseUrl: "http://localhost/airflow"

  ###################################
  # Airflow - Worker Configs
  ###################################
  workers:
172
173
174
175
176
177
178
    resources:
      requests:
        cpu: "1200m"
        memory: "5Gi"
      limits:
        cpu: "1200m"
        memory: "5Gi"
179
180
    podLabels:
      aadpodidbinding: "osdu-identity"
181
182
    podAnnotations:
      sidecar.istio.io/inject: "false"
183
    autoscale:
184
      enabled: false
185
186
187
188
      minReplicas: 2
      maxReplicas: 20
      scaleDown:
        coolDownPeriod: 300
189
190
191
    celery:
      gracefullTermination: true
      gracefullTerminationPeriod: 600
192
    labels:
193
194
      # DO NOT DELETE THIS LABEL. SET IT TO "false" WHEN AUTOSCALING IS DISABLED, SET IT TO "true" WHEN AUTOSCALING IS ENABLED
      autoscalingEnabled: "false"
harshit aggarwal's avatar
harshit aggarwal committed
195

196
197
198
199
200
201
202
203
204
205
  ###################################
  # Airflow - Flower Configs
  ###################################
  flower:
    enabled: false

  ###################################
  # Airflow - Scheduler Configs
  ###################################
  scheduler:
206
207
208
209
210
211
212
    resources:
      requests:
        cpu: "3000m"
        memory: "1Gi"
      limits:
        cpu: "3000m"
        memory: "1Gi"
213
214
    podLabels:
      aadpodidbinding: "osdu-identity"
215
216
    podAnnotations:
      sidecar.istio.io/inject: "false"
217
218
219
220
221
222
    variables: |
      {}

  ###################################
  # Airflow - Common Configs
  ###################################
Daniel Scholl's avatar
Daniel Scholl committed
223
224
  airflow:
    image:
225
226
      repository: community.opengroup.org:5555/osdu/platform/deployment-and-operations/base-containers-azure/airflow-docker-image/master
      tag: v0.9
Daniel Scholl's avatar
Daniel Scholl committed
227
228
229
230
231
232
233
234
235
236
237
238
      pullPolicy: IfNotPresent
      pullSecret: ""
    config:
      AIRFLOW__SCHEDULER__STATSD_ON: "True"
      AIRFLOW__SCHEDULER__STATSD_HOST: "appinsights-statsd"
      AIRFLOW__SCHEDULER__STATSD_PORT: 8125
      AIRFLOW__SCHEDULER__STATSD_PREFIX: "osdu_airflow"
      AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "False"
      ## Enable for Debug purpose
      AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "False"
      AIRFLOW__WEBSERVER__AUTHENTICATE: "True"
      AIRFLOW__WEBSERVER__AUTH_BACKEND: "airflow.contrib.auth.backends.password_auth"
239
      AIRFLOW__WEBSERVER__RBAC: "True"
harshit aggarwal's avatar
init    
harshit aggarwal committed
240
      AIRFLOW__API__AUTH_BACKEND: "airflow.api.auth.backend.default"
Daniel Scholl's avatar
Daniel Scholl committed
241
242
243
244
      AIRFLOW__CORE__REMOTE_LOGGING: "True"
      AIRFLOW__CORE__REMOTE_LOG_CONN_ID: "az_log"
      AIRFLOW__CORE__REMOTE_BASE_LOG_FOLDER: "wasb-airflowlog"
      AIRFLOW__CORE__LOGGING_CONFIG_CLASS: "log_config.DEFAULT_LOGGING_CONFIG"
245
      AIRFLOW__CORE__LOG_FILENAME_TEMPLATE: "{{ run_id }}/{{ ti.dag_id }}/{{ ti.task_id }}/{{ ts }}/{% if dag_run.conf is not none and 'correlation_id' in dag_run.conf %}{{ dag_run.conf['correlation_id'] }}{% else %}None{% endif %}/{{ try_number }}.log"
Daniel Scholl's avatar
Daniel Scholl committed
246
247
      AIRFLOW__CELERY__SSL_ACTIVE: "True"
      AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX: "True"
248
      AIRFLOW__CORE__PLUGINS_FOLDER: "/opt/airflow/plugins"
249
      AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: 60
250
      AIRFLOW__CORE__LOGGING_LEVEL: DEBUG
251
252
      AIRFLOW_VAR_CORE__CONFIG__DATALOAD_CONFIG_PATH: "/opt/airflow/dags/configs/dataload.ini"
      AIRFLOW_VAR_CORE__SERVICE__SCHEMA__URL: "http://schema-service.osdu.svc.cluster.local/api/schema-service/v1/schema"
Kishore Battula's avatar
Kishore Battula committed
253
      AIRFLOW_VAR_CORE__SERVICE__SEARCH__URL: "http://search-service.osdu.svc.cluster.local/api/search/v2/query"
254
255
256
      AIRFLOW_VAR_CORE__SERVICE__STORAGE__URL: "http://storage.osdu.svc.cluster.local/api/storage/v2/records"
      AIRFLOW_VAR_CORE__SERVICE__FILE__HOST: "http://file.osdu.svc.cluster.local/api/file/v2"
      AIRFLOW_VAR_CORE__SERVICE__WORKFLOW__HOST: "http://ingestion-workflow.osdu.svc.cluster.local/api/workflow"
Vivek Ojha's avatar
Vivek Ojha committed
257
      AIRFLOW_VAR_CORE__SERVICE__DATASET__HOST: "http://dataset.osdu.svc.cluster.local/api/dataset/v1"
harshit aggarwal's avatar
harshit aggarwal committed
258
      AIRFLOW_VAR_CORE__SERVICE__SEARCH_WITH_CURSOR__URL: "http://search-service.osdu.svc.cluster.local/api/search/v2/query_with_cursor"
259
      AIRFLOW__WEBSERVER__WORKERS: 8
harshit aggarwal's avatar
init    
harshit aggarwal committed
260
261
262
263
      AIRFLOW__WEBSERVER__WORKER_REFRESH_BATCH_SIZE: 0
      AIRFLOW__CORE__STORE_SERIALIZED_DAGS: True #This flag decides whether to serialise DAGs and persist them in DB
      AIRFLOW__CORE__STORE_DAG_CODE: True #This flag decides whether to persist DAG files code in DB
      AIRFLOW__WEBSERVER__WORKER_CLASS: gevent
264
      AIRFLOW__CELERY__WORKER_CONCURRENCY: 16 # Do not remove this config as it is used for autoscaling as well
Daniel Scholl's avatar
Daniel Scholl committed
265
    extraEnv:
harshit aggarwal's avatar
harshit aggarwal committed
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
    - name: CLOUD_PROVIDER
      value: "azure"
    - name: AIRFLOW_VAR_KEYVAULT_URI
      valueFrom:
        configMapKeyRef:
          name: osdu-svc-properties
          key: ENV_KEYVAULT
    - name: AIRFLOW__CORE__FERNET_KEY
      valueFrom:
        secretKeyRef:
          name: airflow
          key: fernet-key
    - name: AIRFLOW_CONN_AZ_LOG
      valueFrom:
        secretKeyRef:
          name: airflow
          key: remote-log-connection
    - name: AIRFLOW_VAR_AZURE_TENANT_ID
      valueFrom:
        secretKeyRef:
          name: active-directory
          key: tenantid
    - name: AIRFLOW_VAR_AZURE_CLIENT_ID
      valueFrom:
        secretKeyRef:
          name: active-directory
          key: principal-clientid
    - name: AIRFLOW_VAR_AZURE_CLIENT_SECRET
      valueFrom:
        secretKeyRef:
          name: active-directory
          key: principal-clientpassword
    - name: AIRFLOW_VAR_AAD_CLIENT_ID
      valueFrom:
        secretKeyRef:
          name: active-directory
          key: application-appid
    - name: AIRFLOW_VAR_APPINSIGHTS_KEY
      valueFrom:
        secretKeyRef:
          name: central-logging
          key: appinsights
308
309
310
311
    - name: AIRFLOW_VAR_AZURE_DNS_HOST
      value: #{DNS_HOST}#
    - name: AIRFLOW_VAR_AZURE_ENABLE_MSI
      value: "false"
312
313
    - name: PYTHONPATH
      value: "/opt/celery"
Kishore Battula's avatar
Kishore Battula committed
314
315
    # Needed for installing python osdu python sdk. In future this will be changed
    - name: CI_COMMIT_TAG
harshit aggarwal's avatar
harshit aggarwal committed
316
317
318
      value: "v0.12.0"
    - name: BUILD_TAG
      value: "v0.12.0"
Daniel Scholl's avatar
Daniel Scholl committed
319
    extraConfigmapMounts:
harshit aggarwal's avatar
harshit aggarwal committed
320
321
322
323
        - name: remote-log-config
          mountPath: /opt/airflow/config
          configMap: airflow-remote-log-config
          readOnly: true
324
325
326
327
        - name: celery-config
          mountPath: /opt/celery
          configMap: celery-config
          readOnly: true
Daniel Scholl's avatar
Daniel Scholl committed
328
    extraPipPackages: [
329
        "flask-bcrypt==0.7.1",
Daniel Scholl's avatar
Daniel Scholl committed
330
331
        "apache-airflow[statsd]",
        "apache-airflow[kubernetes]",
332
333
        "apache-airflow-backport-providers-microsoft-azure==2021.2.5",
        "dataclasses==0.8",
334
        "google-cloud-storage",
335
336
337
338
        "python-keycloak==0.24.0",
        "msal==1.9.0",
        "azure-identity==1.5.0",
        "azure-keyvault-secrets==4.2.0",
339
        "azure-storage-blob",
340
341
        "azure-servicebus==7.0.1",
        "toposort==1.6",
342
        "strict-rfc3339==0.7",
Kishore Battula's avatar
Kishore Battula committed
343
344
345
        "jsonschema==3.2.0",
        "pyyaml==5.4.1",
        "requests==2.25.1",
346
        "tenacity==8.0.1",
harshit aggarwal's avatar
harshit aggarwal committed
347
348
        "https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_api-0.10.1.dev151+503e364a.tar.gz",
        "https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_airflow-0.0.1.dev32+ea39f8bd.tar.gz"
Daniel Scholl's avatar
Daniel Scholl committed
349
350
    ]
    extraVolumeMounts:
harshit aggarwal's avatar
harshit aggarwal committed
351
352
        - name: azure-keyvault
          mountPath: "/mnt/azure-keyvault"
harshit aggarwal's avatar
init    
harshit aggarwal committed
353
          readOnly: true
harshit aggarwal's avatar
harshit aggarwal committed
354
355
356
357
358
359
360
361
362
363
        - name: dags-data
          mountPath: /opt/airflow/plugins
          subPath: plugins
    extraVolumes:
        - name: azure-keyvault
          csi:
            driver: secrets-store.csi.k8s.io
            readOnly: true
            volumeAttributes:
              secretProviderClass: azure-keyvault