airflow-log-processor-deployment.yaml 2.25 KB
Newer Older
1
{{- $isEnabled := .Values.airflow.isDataPartitionDeployment | default false -}}
Daniel Scholl's avatar
Daniel Scholl committed
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
apiVersion: apps/v1
kind: Deployment
metadata:
  name: airflow-log-processor
  labels:
    app: airflow-log-processor
spec:
  selector:
    matchLabels:
      app: airflow-log-processor
  template:
    metadata:
      labels:
        app: airflow-log-processor
        aadpodidbinding: "osdu-identity"
    spec:
      containers:
      - name: airflow-log-processor
20
        image: {{ .Values.image.repository }}/airflow-function-{{ .Values.image.branch }}:{{ .Values.image.tag | default .Chart.AppVersion }}
Daniel Scholl's avatar
Daniel Scholl committed
21
22
23
24
25
26
27
28
29
30
31
        imagePullPolicy: Always
        env:
          - name: AzureFunctionsJobHost__functions__0
            value: queueTrigger
          - name: FUNCTIONS_WORKER_RUNTIME
            value: dotnet
          - name: AzureLogWorkspaceLogName
            value: airflow_dag
          - name: AzureLogWorkspaceCustomerId
            valueFrom:
              secretKeyRef:
32
33
                name: {{ .Values.logAnalytics.workspaceId.secretName }}
                key: {{ .Values.logAnalytics.workspaceId.secretKey }}
Daniel Scholl's avatar
Daniel Scholl committed
34
35
36
          - name: AzureLogWorkspaceSharedKey
            valueFrom:
              secretKeyRef:
37
38
                name: {{ .Values.logAnalytics.workspaceKey.secretName }}
                key: {{ .Values.logAnalytics.workspaceKey.secretKey }}
Daniel Scholl's avatar
Daniel Scholl committed
39
40
41
42
          - name: AzureWebJobsStorage
            valueFrom:
              secretKeyRef:
                name: airflow
43
                key: storage-connection
Daniel Scholl's avatar
Daniel Scholl committed
44
45
        
---
46
47
48
{{- if $isEnabled }}
apiVersion: keda.sh/v1alpha1
{{- else }}
Daniel Scholl's avatar
Daniel Scholl committed
49
apiVersion: keda.k8s.io/v1alpha1
50
{{- end }}
Daniel Scholl's avatar
Daniel Scholl committed
51
52
53
54
55
56
57
58
59
60
61
kind: TriggerAuthentication
metadata:
  name: azure-queue-auth
spec:
  podIdentity:
    provider: azure

---
kind: ScaledObject
metadata:
  name: azure-queue-scaledobject
62
63
64
65
66
67
68
{{- if $isEnabled }}
apiVersion: keda.sh/v1alpha1
spec:
  scaleTargetRef:
    name: airflow-log-processor
{{- else }}
apiVersion: keda.k8s.io/v1alpha1
Daniel Scholl's avatar
Daniel Scholl committed
69
70
spec:
  scaleTargetRef:
71
72
    deploymentName: airflow-log-processor
{{- end }}
Daniel Scholl's avatar
Daniel Scholl committed
73
74
75
76
77
78
79
80
81
  triggers:
  - type: azure-queue
    metadata:
      # Required
      queueName: airflowlogqueue
      # Optional
      queueLength: "5" # default 5
    authenticationRef:
        name: azure-queue-auth # authenticationRef would need either podIdentity or define a connection parameter