Commit 51267b12 authored by Vineeth Guna [Microsoft]'s avatar Vineeth Guna [Microsoft]
Browse files

Reorganized airflow charts according to functionality

parent 3ebce81d
Pipeline #69871 passed with stages
in 1 minute and 29 seconds
# This file contains the essential configs for the osdu airflow on azure helm chart
################################################################################
# Specify the azure environment specific values
# Specify appinsight statsd config
#
appinsightstatsd:
aadpodidbinding: "osdu-identity"
......@@ -107,7 +108,7 @@ airflow:
enabled: false
externalDatabase:
type: postgres
host: airflow-pgbouncer.osdu.svc.cluster.local #<-- Azure PostgreSQL Database host or pgbouncer host (if pgbouncer is enabled)
host: airflow-pgbouncer #<-- Azure PostgreSQL Database host or pgbouncer host (if pgbouncer is enabled)
user: osdu_admin@#{base-name-sr}#-pg #<-- Azure PostgreSQL Database username, formatted as {username}@{hostname}
passwordSecret: "postgres"
passwordSecretKey: "postgres-password"
......@@ -115,7 +116,7 @@ airflow:
database: airflow
###################################
# Database - External Redis
# Database - Redis Chart
###################################
redis:
enabled: false
......@@ -248,14 +249,6 @@ airflow:
AIRFLOW__CORE__PLUGINS_FOLDER: "/opt/airflow/plugins"
AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: 60
AIRFLOW__CORE__LOGGING_LEVEL: DEBUG
AIRFLOW_VAR_CORE__CONFIG__DATALOAD_CONFIG_PATH: "/opt/airflow/dags/configs/dataload.ini"
AIRFLOW_VAR_CORE__SERVICE__SCHEMA__URL: "http://schema-service.osdu.svc.cluster.local/api/schema-service/v1/schema"
AIRFLOW_VAR_CORE__SERVICE__SEARCH__URL: "http://search-service.osdu.svc.cluster.local/api/search/v2/query"
AIRFLOW_VAR_CORE__SERVICE__STORAGE__URL: "http://storage.osdu.svc.cluster.local/api/storage/v2/records"
AIRFLOW_VAR_CORE__SERVICE__FILE__HOST: "http://file.osdu.svc.cluster.local/api/file/v2"
AIRFLOW_VAR_CORE__SERVICE__WORKFLOW__HOST: "http://ingestion-workflow.osdu.svc.cluster.local/api/workflow"
AIRFLOW_VAR_CORE__SERVICE__DATASET__HOST: "http://dataset.osdu.svc.cluster.local/api/dataset/v1"
AIRFLOW_VAR_CORE__SERVICE__SEARCH_WITH_CURSOR__URL: "http://search-service.osdu.svc.cluster.local/api/search/v2/query_with_cursor"
AIRFLOW__WEBSERVER__WORKERS: 8
AIRFLOW__WEBSERVER__WORKER_REFRESH_BATCH_SIZE: 0
AIRFLOW__CORE__STORE_SERIALIZED_DAGS: True #This flag decides whether to serialise DAGs and persist them in DB
......@@ -315,7 +308,28 @@ airflow:
- name: CI_COMMIT_TAG
value: "v0.12.0"
- name: BUILD_TAG
value: "v0.12.0"
value: "v0.12.0"
## Begin -- Ingest Manifest DAG variables
- name: AIRFLOW_VAR_ENTITLEMENTS_MODULE_NAME
value: "entitlements_client"
- name: AIRFLOW_VAR_CORE__CONFIG__DATALOAD_CONFIG_PATH
value: "/opt/airflow/dags/configs/dataload.ini"
- name: AIRFLOW_VAR_CORE__SERVICE__SCHEMA__URL
value: "http://schema-service.osdu.svc.cluster.local/api/schema-service/v1/schema"
- name: AIRFLOW_VAR_CORE__SERVICE__SEARCH__URL
value: "http://search-service.osdu.svc.cluster.local/api/search/v2/query"
- name: AIRFLOW_VAR_CORE__SERVICE__STORAGE__URL
value: "http://storage.osdu.svc.cluster.local/api/storage/v2/records"
- name: AIRFLOW_VAR_CORE__SERVICE__FILE__HOST
value: "http://file.osdu.svc.cluster.local/api/file/v2"
- name: AIRFLOW_VAR_CORE__SERVICE__WORKFLOW__HOST
value: "http://ingestion-workflow.osdu.svc.cluster.local/api/workflow"
- name: AIRFLOW_VAR_CORE__SERVICE__DATASET__HOST
value: "http://dataset.osdu.svc.cluster.local/api/dataset/v1"
- name: AIRFLOW_VAR_CORE__SERVICE__SEARCH_WITH_CURSOR__URL
value: "http://search-service.osdu.svc.cluster.local/api/search/v2/query_with_cursor"
## End -- Ingest Manifest DAG variables
extraConfigmapMounts:
- name: remote-log-config
mountPath: /opt/airflow/config
......
......@@ -25,6 +25,8 @@ spec:
labels:
app: appinsights-statsd
aadpodidbinding: "osdu-identity"
annotations:
sidecar.istio.io/inject: "false"
spec:
containers:
- name: appinsights-statsd
......
......@@ -14,6 +14,8 @@ spec:
labels:
app: airflow-log-processor
aadpodidbinding: "osdu-identity"
annotations:
sidecar.istio.io/inject: "false"
spec:
containers:
- name: airflow-log-processor
......@@ -41,28 +43,3 @@ spec:
secretKeyRef:
name: airflow
key: storage-connection
---
kind: ScaledObject
metadata:
name: azure-queue-scaledobject
{{- if $isEnabled }}
apiVersion: keda.sh/v1alpha1
spec:
scaleTargetRef:
name: airflow-log-processor
{{- else }}
apiVersion: keda.k8s.io/v1alpha1
spec:
scaleTargetRef:
deploymentName: airflow-log-processor
{{- end }}
triggers:
- type: azure-queue
metadata:
queueName: airflowlogqueue
queueLength: "5" # default 5
{{- if $isEnabled }}
connectionFromEnv: AzureWebJobsStorage
{{- else }}
connection: AzureWebJobsStorage
{{- end }}
\ No newline at end of file
{{- $isEnabled := .Values.keda.version_2_enabled | default false -}}
kind: ScaledObject
metadata:
name: azure-queue-scaledobject
{{- if $isEnabled }}
apiVersion: keda.sh/v1alpha1
spec:
scaleTargetRef:
name: airflow-log-processor
{{- else }}
apiVersion: keda.k8s.io/v1alpha1
spec:
scaleTargetRef:
deploymentName: airflow-log-processor
{{- end }}
triggers:
- type: azure-queue
metadata:
queueName: airflowlogqueue
queueLength: "5" # default 5
{{- if $isEnabled }}
connectionFromEnv: AzureWebJobsStorage
{{- else }}
connection: AzureWebJobsStorage
{{- end }}
\ No newline at end of file
......@@ -24,6 +24,8 @@ spec:
tier: airflow
component: pgbouncer
release: {{ .Release.Name }}
annotations:
sidecar.istio.io/inject: "false"
spec:
restartPolicy: Always
containers:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment