airflow-log-processor-deployment.yaml 1.8 KB
Newer Older
Daniel Scholl's avatar
Daniel Scholl committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
apiVersion: apps/v1
kind: Deployment
metadata:
  name: airflow-log-processor
  namespace: osdu
  labels:
    app: airflow-log-processor
spec:
  selector:
    matchLabels:
      app: airflow-log-processor
  template:
    metadata:
      labels:
        app: airflow-log-processor
        aadpodidbinding: "osdu-identity"
    spec:
      containers:
      - name: airflow-log-processor
        image: binroon/airflow-logs-to-workspace
        imagePullPolicy: Always
        env:
          - name: AzureFunctionsJobHost__functions__0
            value: queueTrigger
          - name: FUNCTIONS_WORKER_RUNTIME
            value: dotnet
          - name: AzureLogWorkspaceLogName
            value: airflow_dag
          - name: AzureLogWorkspaceCustomerId
            valueFrom:
              secretKeyRef:
                name: central-logging
                key: workspace-id
          - name: AzureLogWorkspaceSharedKey
            valueFrom:
              secretKeyRef:
                name: central-logging
                key: workspace-key
          - name: AzureWebJobsStorage
            valueFrom:
              secretKeyRef:
                name: airflow
                key: airflow-storage-connection
        
---
apiVersion: keda.k8s.io/v1alpha1
kind: TriggerAuthentication
metadata:
  name: azure-queue-auth
  namespace: osdu
spec:
  podIdentity:
    provider: azure

---
apiVersion: keda.k8s.io/v1alpha1
kind: ScaledObject
metadata:
  name: azure-queue-scaledobject
  namespace: osdu
spec:
  scaleTargetRef:
    deploymentName: azurequeue-function
  triggers:
  - type: azure-queue
    metadata:
      # Required
      queueName: airflowlogqueue
      # Optional
      queueLength: "5" # default 5
    authenticationRef:
        name: azure-queue-auth # authenticationRef would need either podIdentity or define a connection parameter