helm-config.yaml 11.6 KB
Newer Older
1
2
3
################################################################################
# Specify the azure environment specific values
#
Daniel Scholl's avatar
Daniel Scholl committed
4
appinsightstatsd:
5
6
  aadpodidbinding: "osdu-identity"

7
8
9
10
11
12
13
14
15
16
17
#################################################################################
# Specify log analytics configuration
#
logAnalytics:
  workspaceId:
    secretName: "central-logging"
    secretKey: "workspace-id"
  workspaceKey:
    secretName: "central-logging"
    secretKey: "workspace-key"

18
19
20
21
22
23
24
25
################################################################################
# Specify any optional override values
#
image:
  repository: #{container-registry}#.azurecr.io
  branch: #{ENVIRONMENT_NAME}#
  tag: #{Build.SourceVersion}#

Daniel Scholl's avatar
Daniel Scholl committed
26
27
airflowLogin:
  name: admin
28

harshit aggarwal's avatar
harshit aggarwal committed
29

harshit aggarwal's avatar
init    
harshit aggarwal committed
30
31
32
33
34
airflowAuthentication:
  username: admin
  keyvaultMountPath: /mnt/azure-keyvault/
  passwordKey: airflow-admin-password

35
36
37
38
39
40
41
################################################################################
# Specify any custom configs/environment values
#
customConfig:
  rbac:
    createUser: "True"

42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
################################################################################
# Specify pgbouncer configuration
#
pgbouncer:
  enabled: true
  port: 6543
  max_client_connections: 3000
  airflowdb:
    name: airflow
    host: #{base-name-sr}#-pg.postgres.database.azure.com
    port: 5432
    pool_size: 100
    user:  osdu_admin@#{base-name-sr}#-pg
    passwordSecret: "postgres"
    passwordSecretKey: "postgres-password"

58
59
60
61
62
63
################################################################################
# Specify KEDA configuration
#
keda:
  version_2_enabled: false

64

65
66
67
################################################################################
# Specify the airflow configuration
#
Daniel Scholl's avatar
Daniel Scholl committed
68
airflow:
69

70
71
72
73
74
75
76
77
78
  ##################################
  # Kubernetes Pod Operator config
  ##################################
  kubernetesPodOperator:
    namespace: airflow
  
  serviceAccount:
    name: airflow

79
80
81
82
  ###################################
  # Kubernetes - Ingress Configs
  ###################################
  ingress:
Daniel Scholl's avatar
Daniel Scholl committed
83
    enabled: true
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
    web:
      annotations:
        kubernetes.io/ingress.class: azure/application-gateway
        appgw.ingress.kubernetes.io/request-timeout: "300"
        appgw.ingress.kubernetes.io/connection-draining: "true"
        appgw.ingress.kubernetes.io/connection-draining-timeout: "30"
        cert-manager.io/cluster-issuer: letsencrypt
        cert-manager.io/acme-challenge-type: http01
      path: "/airflow"
      host: #{DNS_HOST}#
      livenessPath: "/airflow/health"
      tls:
        enabled: true
        secretName: osdu-certificate
      precedingPaths:
        - path: "/airflow/*"
          serviceName: airflow-web
          servicePort: 8080

  ###################################
  # Database - External Database
  ###################################
  postgresql:
    enabled: false
  externalDatabase:
    type: postgres
110
    host: airflow-pgbouncer.osdu.svc.cluster.local           #<-- Azure PostgreSQL Database host or pgbouncer host (if pgbouncer is enabled)
111
    user: osdu_admin@#{base-name-sr}#-pg                     #<-- Azure PostgreSQL Database username, formatted as {username}@{hostname}
112
113
    passwordSecret: "postgres"
    passwordSecretKey: "postgres-password"
114
    port: 6543
115
116
117
118
119
120
121
122
    database: airflow

  ###################################
  # Database - External Redis
  ###################################
  redis:
    enabled: false
  externalRedis:
123
    host: #{base-name-sr}#-queue.redis.cache.windows.net    #<-- Azure Redis Cache host
124
125
    port: 6380
    passwordSecret: "redis"
126
    passwordSecretKey: "redis-queue-password"
127
    databaseNumber: 1  #<-- Adding redis database number according to the Redis config map https://community.opengroup.org/osdu/platform/deployment-and-operations/infra-azure-provisioning/-/blob/master/charts/osdu-common/templates/redis-map.yaml#L7
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142

  ###################################
  # Airflow - DAGs Configs
  ###################################
  dags:
    installRequirements: true
    persistence:
      enabled: true
      existingClaim: airflowdagpvc
      subPath: "dags"

  ###################################
  # Airflow - WebUI Configs
  ###################################
  web:
harshit aggarwal's avatar
init    
harshit aggarwal committed
143
144
145
146
147
    replicas: 1
    livenessProbe:
      timeoutSeconds: 60
    resources:
      requests:
148
149
        cpu: "1000m"
        memory: "4Gi"
harshit aggarwal's avatar
init    
harshit aggarwal committed
150
      limits:
151
152
        cpu: "1000m"
        memory: "4Gi"
153
154
    podLabels:
      aadpodidbinding: "osdu-identity"
155
156
157
158
159
160
161
162
163
    autoscale:
      enabled: false
      minReplicas: 2
      maxReplicas: 20
      scaleDown:
        coolDownPeriod: 60
    labels:
      # DO NOT DELETE THIS LABEL. SET IT TO "false" WHEN AUTOSCALING IS DISABLED, SET IT TO "true" WHEN AUTOSCALING IS ENABLED
      autoscalingEnabled: "false"
harshit aggarwal's avatar
init    
harshit aggarwal committed
164
165
    podAnnotations:
      sidecar.istio.io/userVolumeMount: '[{"name": "azure-keyvault", "mountPath": "/mnt/azure-keyvault", "readonly": true}]'
166
167
168
169
170
171
    baseUrl: "http://localhost/airflow"

  ###################################
  # Airflow - Worker Configs
  ###################################
  workers:
172
173
174
175
176
177
178
    resources:
      requests:
        cpu: "1200m"
        memory: "5Gi"
      limits:
        cpu: "1200m"
        memory: "5Gi"
179
180
    podLabels:
      aadpodidbinding: "osdu-identity"
181
    autoscale:
182
      enabled: false
183
184
185
186
      minReplicas: 2
      maxReplicas: 20
      scaleDown:
        coolDownPeriod: 300
187
188
189
    celery:
      gracefullTermination: true
      gracefullTerminationPeriod: 600
190
    labels:
191
192
      # DO NOT DELETE THIS LABEL. SET IT TO "false" WHEN AUTOSCALING IS DISABLED, SET IT TO "true" WHEN AUTOSCALING IS ENABLED
      autoscalingEnabled: "false"
harshit aggarwal's avatar
harshit aggarwal committed
193

194
195
196
197
198
199
200
201
202
203
  ###################################
  # Airflow - Flower Configs
  ###################################
  flower:
    enabled: false

  ###################################
  # Airflow - Scheduler Configs
  ###################################
  scheduler:
204
205
206
207
208
209
210
    resources:
      requests:
        cpu: "3000m"
        memory: "1Gi"
      limits:
        cpu: "3000m"
        memory: "1Gi"
211
212
213
214
215
216
217
218
    podLabels:
      aadpodidbinding: "osdu-identity"
    variables: |
      {}

  ###################################
  # Airflow - Common Configs
  ###################################
Daniel Scholl's avatar
Daniel Scholl committed
219
220
  airflow:
    image:
221
222
      repository: community.opengroup.org:5555/osdu/platform/deployment-and-operations/base-containers-azure/airflow-docker-image/master
      tag: v0.9
Daniel Scholl's avatar
Daniel Scholl committed
223
224
225
226
227
228
229
230
231
232
233
234
      pullPolicy: IfNotPresent
      pullSecret: ""
    config:
      AIRFLOW__SCHEDULER__STATSD_ON: "True"
      AIRFLOW__SCHEDULER__STATSD_HOST: "appinsights-statsd"
      AIRFLOW__SCHEDULER__STATSD_PORT: 8125
      AIRFLOW__SCHEDULER__STATSD_PREFIX: "osdu_airflow"
      AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "False"
      ## Enable for Debug purpose
      AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "False"
      AIRFLOW__WEBSERVER__AUTHENTICATE: "True"
      AIRFLOW__WEBSERVER__AUTH_BACKEND: "airflow.contrib.auth.backends.password_auth"
235
      AIRFLOW__WEBSERVER__RBAC: "True"
harshit aggarwal's avatar
init    
harshit aggarwal committed
236
      AIRFLOW__API__AUTH_BACKEND: "airflow.api.auth.backend.default"
Daniel Scholl's avatar
Daniel Scholl committed
237
238
239
240
      AIRFLOW__CORE__REMOTE_LOGGING: "True"
      AIRFLOW__CORE__REMOTE_LOG_CONN_ID: "az_log"
      AIRFLOW__CORE__REMOTE_BASE_LOG_FOLDER: "wasb-airflowlog"
      AIRFLOW__CORE__LOGGING_CONFIG_CLASS: "log_config.DEFAULT_LOGGING_CONFIG"
241
      AIRFLOW__CORE__LOG_FILENAME_TEMPLATE: "{{ run_id }}/{{ ti.dag_id }}/{{ ti.task_id }}/{{ ts }}/{% if dag_run.conf is not none and 'correlation_id' in dag_run.conf %}{{ dag_run.conf['correlation_id'] }}{% else %}None{% endif %}/{{ try_number }}.log"
Daniel Scholl's avatar
Daniel Scholl committed
242
243
      AIRFLOW__CELERY__SSL_ACTIVE: "True"
      AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX: "True"
244
      AIRFLOW__CORE__PLUGINS_FOLDER: "/opt/airflow/plugins"
245
      AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: 60
246
      AIRFLOW__CORE__LOGGING_LEVEL: DEBUG
247
248
      AIRFLOW_VAR_CORE__CONFIG__DATALOAD_CONFIG_PATH: "/opt/airflow/dags/configs/dataload.ini"
      AIRFLOW_VAR_CORE__SERVICE__SCHEMA__URL: "http://schema-service.osdu.svc.cluster.local/api/schema-service/v1/schema"
Kishore Battula's avatar
Kishore Battula committed
249
      AIRFLOW_VAR_CORE__SERVICE__SEARCH__URL: "http://search-service.osdu.svc.cluster.local/api/search/v2/query"
250
251
252
      AIRFLOW_VAR_CORE__SERVICE__STORAGE__URL: "http://storage.osdu.svc.cluster.local/api/storage/v2/records"
      AIRFLOW_VAR_CORE__SERVICE__FILE__HOST: "http://file.osdu.svc.cluster.local/api/file/v2"
      AIRFLOW_VAR_CORE__SERVICE__WORKFLOW__HOST: "http://ingestion-workflow.osdu.svc.cluster.local/api/workflow"
harshit aggarwal's avatar
harshit aggarwal committed
253
      AIRFLOW_VAR_CORE__SERVICE__SEARCH_WITH_CURSOR__URL: "http://search-service.osdu.svc.cluster.local/api/search/v2/query_with_cursor"
254
      AIRFLOW__WEBSERVER__WORKERS: 8
harshit aggarwal's avatar
init    
harshit aggarwal committed
255
256
257
258
      AIRFLOW__WEBSERVER__WORKER_REFRESH_BATCH_SIZE: 0
      AIRFLOW__CORE__STORE_SERIALIZED_DAGS: True #This flag decides whether to serialise DAGs and persist them in DB
      AIRFLOW__CORE__STORE_DAG_CODE: True #This flag decides whether to persist DAG files code in DB
      AIRFLOW__WEBSERVER__WORKER_CLASS: gevent
259
      AIRFLOW__CELERY__WORKER_CONCURRENCY: 16 # Do not remove this config as it is used for autoscaling as well
Daniel Scholl's avatar
Daniel Scholl committed
260
    extraEnv:
harshit aggarwal's avatar
harshit aggarwal committed
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
    - name: CLOUD_PROVIDER
      value: "azure"
    - name: AIRFLOW_VAR_KEYVAULT_URI
      valueFrom:
        configMapKeyRef:
          name: osdu-svc-properties
          key: ENV_KEYVAULT
    - name: AIRFLOW__CORE__FERNET_KEY
      valueFrom:
        secretKeyRef:
          name: airflow
          key: fernet-key
    - name: AIRFLOW_CONN_AZ_LOG
      valueFrom:
        secretKeyRef:
          name: airflow
          key: remote-log-connection
    - name: AIRFLOW_VAR_AZURE_TENANT_ID
      valueFrom:
        secretKeyRef:
          name: active-directory
          key: tenantid
    - name: AIRFLOW_VAR_AZURE_CLIENT_ID
      valueFrom:
        secretKeyRef:
          name: active-directory
          key: principal-clientid
    - name: AIRFLOW_VAR_AZURE_CLIENT_SECRET
      valueFrom:
        secretKeyRef:
          name: active-directory
          key: principal-clientpassword
    - name: AIRFLOW_VAR_AAD_CLIENT_ID
      valueFrom:
        secretKeyRef:
          name: active-directory
          key: application-appid
    - name: AIRFLOW_VAR_APPINSIGHTS_KEY
      valueFrom:
        secretKeyRef:
          name: central-logging
          key: appinsights
303
304
    - name: PYTHONPATH
      value: "/opt/celery"
Kishore Battula's avatar
Kishore Battula committed
305
306
307
    # Needed for installing python osdu python sdk. In future this will be changed
    - name: CI_COMMIT_TAG
      value: "v0.10.0"
Daniel Scholl's avatar
Daniel Scholl committed
308
    extraConfigmapMounts:
harshit aggarwal's avatar
harshit aggarwal committed
309
310
311
312
        - name: remote-log-config
          mountPath: /opt/airflow/config
          configMap: airflow-remote-log-config
          readOnly: true
313
314
315
316
        - name: celery-config
          mountPath: /opt/celery
          configMap: celery-config
          readOnly: true
Daniel Scholl's avatar
Daniel Scholl committed
317
    extraPipPackages: [
318
        "flask-bcrypt==0.7.1",
Daniel Scholl's avatar
Daniel Scholl committed
319
320
        "apache-airflow[statsd]",
        "apache-airflow[kubernetes]",
321
322
        "apache-airflow-backport-providers-microsoft-azure==2021.2.5",
        "dataclasses==0.8",
323
        "google-cloud-storage",
324
325
326
327
        "python-keycloak==0.24.0",
        "msal==1.9.0",
        "azure-identity==1.5.0",
        "azure-keyvault-secrets==4.2.0",
328
        "azure-storage-blob",
329
330
        "azure-servicebus==7.0.1",
        "toposort==1.6",
331
        "strict-rfc3339==0.7",
Kishore Battula's avatar
Kishore Battula committed
332
333
334
        "jsonschema==3.2.0",
        "pyyaml==5.4.1",
        "requests==2.25.1",
335
        "tenacity==8.0.1",
Kishore Battula's avatar
Kishore Battula committed
336
        "https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_api-0.10.0.tar.gz"
Daniel Scholl's avatar
Daniel Scholl committed
337
338
    ]
    extraVolumeMounts:
harshit aggarwal's avatar
harshit aggarwal committed
339
340
        - name: azure-keyvault
          mountPath: "/mnt/azure-keyvault"
harshit aggarwal's avatar
init    
harshit aggarwal committed
341
          readOnly: true
harshit aggarwal's avatar
harshit aggarwal committed
342
343
344
345
346
347
348
349
350
351
        - name: dags-data
          mountPath: /opt/airflow/plugins
          subPath: plugins
    extraVolumes:
        - name: azure-keyvault
          csi:
            driver: secrets-store.csi.k8s.io
            readOnly: true
            volumeAttributes:
              secretProviderClass: azure-keyvault