helm-config.yaml 10.7 KB
Newer Older
1
2
3
################################################################################
# Specify the azure environment specific values
#
Daniel Scholl's avatar
Daniel Scholl committed
4
appinsightstatsd:
5
6
  aadpodidbinding: "osdu-identity"

7
8
9
10
11
12
13
14
15
16
17
#################################################################################
# Specify log analytics configuration
#
logAnalytics:
  workspaceId:
    secretName: "central-logging"
    secretKey: "workspace-id"
  workspaceKey:
    secretName: "central-logging"
    secretKey: "workspace-key"

18
19
20
21
22
23
24
25
################################################################################
# Specify any optional override values
#
image:
  repository: #{container-registry}#.azurecr.io
  branch: #{ENVIRONMENT_NAME}#
  tag: #{Build.SourceVersion}#

Daniel Scholl's avatar
Daniel Scholl committed
26
27
airflowLogin:
  name: admin
28

harshit aggarwal's avatar
harshit aggarwal committed
29

harshit aggarwal's avatar
init    
harshit aggarwal committed
30
31
32
33
34
airflowAuthentication:
  username: admin
  keyvaultMountPath: /mnt/azure-keyvault/
  passwordKey: airflow-admin-password

35
36
37
38
39
40
41
################################################################################
# Specify any custom configs/environment values
#
customConfig:
  rbac:
    createUser: "True"

42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
################################################################################
# Specify pgbouncer configuration
#
pgbouncer:
  enabled: true
  port: 6543
  max_client_connections: 3000
  airflowdb:
    name: airflow
    host: #{base-name-sr}#-pg.postgres.database.azure.com
    port: 5432
    pool_size: 100
    user:  osdu_admin@#{base-name-sr}#-pg
    passwordSecret: "postgres"
    passwordSecretKey: "postgres-password"


59
60
61
################################################################################
# Specify the airflow configuration
#
Daniel Scholl's avatar
Daniel Scholl committed
62
airflow:
63

64
65
66
67
68
69
70
71
72
  ##################################
  # Kubernetes Pod Operator config
  ##################################
  kubernetesPodOperator:
    namespace: airflow
  
  serviceAccount:
    name: airflow

73
74
75
76
  ###################################
  # Kubernetes - Ingress Configs
  ###################################
  ingress:
Daniel Scholl's avatar
Daniel Scholl committed
77
    enabled: true
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
    web:
      annotations:
        kubernetes.io/ingress.class: azure/application-gateway
        appgw.ingress.kubernetes.io/request-timeout: "300"
        appgw.ingress.kubernetes.io/connection-draining: "true"
        appgw.ingress.kubernetes.io/connection-draining-timeout: "30"
        cert-manager.io/cluster-issuer: letsencrypt
        cert-manager.io/acme-challenge-type: http01
      path: "/airflow"
      host: #{DNS_HOST}#
      livenessPath: "/airflow/health"
      tls:
        enabled: true
        secretName: osdu-certificate
      precedingPaths:
        - path: "/airflow/*"
          serviceName: airflow-web
          servicePort: 8080

  ###################################
  # Database - External Database
  ###################################
  postgresql:
    enabled: false
  externalDatabase:
    type: postgres
104
    host: airflow-pgbouncer.osdu.svc.cluster.local           #<-- Azure PostgreSQL Database host or pgbouncer host (if pgbouncer is enabled)
105
    user: osdu_admin@#{base-name-sr}#-pg                     #<-- Azure PostgreSQL Database username, formatted as {username}@{hostname}
106
107
    passwordSecret: "postgres"
    passwordSecretKey: "postgres-password"
108
    port: 6543
109
110
111
112
113
114
115
116
117
118
119
120
    database: airflow

  ###################################
  # Database - External Redis
  ###################################
  redis:
    enabled: false
  externalRedis:
    host: #{base-name-sr}#-cache.redis.cache.windows.net    #<-- Azure Redis Cache host
    port: 6380
    passwordSecret: "redis"
    passwordSecretKey: "redis-password"
121
    databaseNumber: 1  #<-- Adding redis database number according to the Redis config map https://community.opengroup.org/osdu/platform/deployment-and-operations/infra-azure-provisioning/-/blob/master/charts/osdu-common/templates/redis-map.yaml#L7
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136

  ###################################
  # Airflow - DAGs Configs
  ###################################
  dags:
    installRequirements: true
    persistence:
      enabled: true
      existingClaim: airflowdagpvc
      subPath: "dags"

  ###################################
  # Airflow - WebUI Configs
  ###################################
  web:
harshit aggarwal's avatar
init    
harshit aggarwal committed
137
138
139
140
141
142
143
144
145
146
    replicas: 1
    livenessProbe:
      timeoutSeconds: 60
    resources:
      requests:
        cpu: "2000m"
        memory: "2Gi"
      limits:
        cpu: "3000m"
        memory: "2Gi"
147
148
    podLabels:
      aadpodidbinding: "osdu-identity"
harshit aggarwal's avatar
init    
harshit aggarwal committed
149
150
    podAnnotations:
      sidecar.istio.io/userVolumeMount: '[{"name": "azure-keyvault", "mountPath": "/mnt/azure-keyvault", "readonly": true}]'
151
152
153
154
155
156
    baseUrl: "http://localhost/airflow"

  ###################################
  # Airflow - Worker Configs
  ###################################
  workers:
157
158
159
160
161
162
163
    resources:
      requests:
        cpu: "1200m"
        memory: "5Gi"
      limits:
        cpu: "1200m"
        memory: "5Gi"
164
165
166
    podLabels:
      aadpodidbinding: "osdu-identity"
    autoscaling:
167
      enabled: false
168
169
170
      ## minReplicas is picked from Values.workers.replicas and default value is 1
      maxReplicas: 3
      metrics:
harshit aggarwal's avatar
harshit aggarwal committed
171
172
173
174
175
176
      - type: Resource
        resource:
          name: memory
          target:
            type: Utilization
            averageUtilization: 60
177
    labels:
178
179
      # DO NOT DELETE THIS LABEL. SET IT TO "false" WHEN AUTOSCALING IS DISABLED, SET IT TO "true" WHEN AUTOSCALING IS ENABLED
      autoscalingEnabled: "false"
harshit aggarwal's avatar
harshit aggarwal committed
180

181
182
183
184
185
186
187
188
189
190
  ###################################
  # Airflow - Flower Configs
  ###################################
  flower:
    enabled: false

  ###################################
  # Airflow - Scheduler Configs
  ###################################
  scheduler:
191
192
193
194
195
196
197
    resources:
      requests:
        cpu: "3000m"
        memory: "1Gi"
      limits:
        cpu: "3000m"
        memory: "1Gi"
198
199
200
201
202
203
204
205
    podLabels:
      aadpodidbinding: "osdu-identity"
    variables: |
      {}

  ###################################
  # Airflow - Common Configs
  ###################################
Daniel Scholl's avatar
Daniel Scholl committed
206
207
  airflow:
    image:
208
209
      repository: community.opengroup.org:5555/osdu/platform/deployment-and-operations/base-containers-azure/airflow-docker-image/master
      tag: v0.9
Daniel Scholl's avatar
Daniel Scholl committed
210
211
212
213
214
215
216
217
218
219
220
221
      pullPolicy: IfNotPresent
      pullSecret: ""
    config:
      AIRFLOW__SCHEDULER__STATSD_ON: "True"
      AIRFLOW__SCHEDULER__STATSD_HOST: "appinsights-statsd"
      AIRFLOW__SCHEDULER__STATSD_PORT: 8125
      AIRFLOW__SCHEDULER__STATSD_PREFIX: "osdu_airflow"
      AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "False"
      ## Enable for Debug purpose
      AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "False"
      AIRFLOW__WEBSERVER__AUTHENTICATE: "True"
      AIRFLOW__WEBSERVER__AUTH_BACKEND: "airflow.contrib.auth.backends.password_auth"
222
      AIRFLOW__WEBSERVER__RBAC: "True"
harshit aggarwal's avatar
init    
harshit aggarwal committed
223
      AIRFLOW__API__AUTH_BACKEND: "airflow.api.auth.backend.default"
Daniel Scholl's avatar
Daniel Scholl committed
224
225
226
227
      AIRFLOW__CORE__REMOTE_LOGGING: "True"
      AIRFLOW__CORE__REMOTE_LOG_CONN_ID: "az_log"
      AIRFLOW__CORE__REMOTE_BASE_LOG_FOLDER: "wasb-airflowlog"
      AIRFLOW__CORE__LOGGING_CONFIG_CLASS: "log_config.DEFAULT_LOGGING_CONFIG"
228
      AIRFLOW__CORE__LOG_FILENAME_TEMPLATE: "{{ run_id }}/{{ ti.dag_id }}/{{ ti.task_id }}/{{ ts }}/{% if dag_run.conf is not none and 'correlation_id' in dag_run.conf %}{{ dag_run.conf['correlation_id'] }}{% else %}None{% endif %}/{{ try_number }}.log"
Daniel Scholl's avatar
Daniel Scholl committed
229
230
      AIRFLOW__CELERY__SSL_ACTIVE: "True"
      AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX: "True"
231
      AIRFLOW__CORE__PLUGINS_FOLDER: "/opt/airflow/plugins"
232
      AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: 60
233
      AIRFLOW__CORE__LOGGING_LEVEL: DEBUG
234
235
      AIRFLOW_VAR_CORE__CONFIG__DATALOAD_CONFIG_PATH: "/opt/airflow/dags/configs/dataload.ini"
      AIRFLOW_VAR_CORE__SERVICE__SCHEMA__URL: "http://schema-service.osdu.svc.cluster.local/api/schema-service/v1/schema"
Kishore Battula's avatar
Kishore Battula committed
236
      AIRFLOW_VAR_CORE__SERVICE__SEARCH__URL: "http://search-service.osdu.svc.cluster.local/api/search/v2/query"
237
238
239
      AIRFLOW_VAR_CORE__SERVICE__STORAGE__URL: "http://storage.osdu.svc.cluster.local/api/storage/v2/records"
      AIRFLOW_VAR_CORE__SERVICE__FILE__HOST: "http://file.osdu.svc.cluster.local/api/file/v2"
      AIRFLOW_VAR_CORE__SERVICE__WORKFLOW__HOST: "http://ingestion-workflow.osdu.svc.cluster.local/api/workflow"
harshit aggarwal's avatar
harshit aggarwal committed
240
      AIRFLOW_VAR_CORE__SERVICE__SEARCH_WITH_CURSOR__URL: "http://search-service.osdu.svc.cluster.local/api/search/v2/query_with_cursor"
harshit aggarwal's avatar
init    
harshit aggarwal committed
241
242
243
244
245
      AIRFLOW__WEBSERVER__WORKERS: 15
      AIRFLOW__WEBSERVER__WORKER_REFRESH_BATCH_SIZE: 0
      AIRFLOW__CORE__STORE_SERIALIZED_DAGS: True #This flag decides whether to serialise DAGs and persist them in DB
      AIRFLOW__CORE__STORE_DAG_CODE: True #This flag decides whether to persist DAG files code in DB
      AIRFLOW__WEBSERVER__WORKER_CLASS: gevent
Daniel Scholl's avatar
Daniel Scholl committed
246
    extraEnv:
harshit aggarwal's avatar
harshit aggarwal committed
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
    - name: CLOUD_PROVIDER
      value: "azure"
    - name: AIRFLOW_VAR_KEYVAULT_URI
      valueFrom:
        configMapKeyRef:
          name: osdu-svc-properties
          key: ENV_KEYVAULT
    - name: AIRFLOW__CORE__FERNET_KEY
      valueFrom:
        secretKeyRef:
          name: airflow
          key: fernet-key
    - name: AIRFLOW_CONN_AZ_LOG
      valueFrom:
        secretKeyRef:
          name: airflow
          key: remote-log-connection
    - name: AIRFLOW_VAR_AZURE_TENANT_ID
      valueFrom:
        secretKeyRef:
          name: active-directory
          key: tenantid
    - name: AIRFLOW_VAR_AZURE_CLIENT_ID
      valueFrom:
        secretKeyRef:
          name: active-directory
          key: principal-clientid
    - name: AIRFLOW_VAR_AZURE_CLIENT_SECRET
      valueFrom:
        secretKeyRef:
          name: active-directory
          key: principal-clientpassword
    - name: AIRFLOW_VAR_AAD_CLIENT_ID
      valueFrom:
        secretKeyRef:
          name: active-directory
          key: application-appid
    - name: AIRFLOW_VAR_APPINSIGHTS_KEY
      valueFrom:
        secretKeyRef:
          name: central-logging
          key: appinsights
Daniel Scholl's avatar
Daniel Scholl committed
289
    extraConfigmapMounts:
harshit aggarwal's avatar
harshit aggarwal committed
290
291
292
293
        - name: remote-log-config
          mountPath: /opt/airflow/config
          configMap: airflow-remote-log-config
          readOnly: true
Daniel Scholl's avatar
Daniel Scholl committed
294
    extraPipPackages: [
295
        "flask-bcrypt==0.7.1",
Daniel Scholl's avatar
Daniel Scholl committed
296
297
        "apache-airflow[statsd]",
        "apache-airflow[kubernetes]",
298
299
        "apache-airflow-backport-providers-microsoft-azure==2021.2.5",
        "dataclasses==0.8",
300
        "google-cloud-storage",
301
302
303
304
        "python-keycloak==0.24.0",
        "msal==1.9.0",
        "azure-identity==1.5.0",
        "azure-keyvault-secrets==4.2.0",
305
        "azure-storage-blob",
306
307
        "azure-servicebus==7.0.1",
        "toposort==1.6",
308
        "strict-rfc3339==0.7",
Daniel Scholl's avatar
Daniel Scholl committed
309
        "https://azglobalosdutestlake.blob.core.windows.net/pythonsdk/osdu_api-0.0.4.tar.gz"
Daniel Scholl's avatar
Daniel Scholl committed
310
311
    ]
    extraVolumeMounts:
harshit aggarwal's avatar
harshit aggarwal committed
312
313
        - name: azure-keyvault
          mountPath: "/mnt/azure-keyvault"
harshit aggarwal's avatar
init    
harshit aggarwal committed
314
          readOnly: true
harshit aggarwal's avatar
harshit aggarwal committed
315
316
317
318
319
320
321
322
323
324
        - name: dags-data
          mountPath: /opt/airflow/plugins
          subPath: plugins
    extraVolumes:
        - name: azure-keyvault
          csi:
            driver: secrets-store.csi.k8s.io
            readOnly: true
            volumeAttributes:
              secretProviderClass: azure-keyvault