Commit 716f8b07 authored by Kishore Battula's avatar Kishore Battula
Browse files

Merge branch 'airflow-webserver-autoscaling' into 'master'

Airflow webserver autoscaling configuration

See merge request !414
parents 138f956a a591f338
Pipeline #54546 passed with stages
in 1 minute and 1 second
......@@ -144,13 +144,22 @@ airflow:
timeoutSeconds: 60
resources:
requests:
cpu: "2000m"
memory: "2Gi"
cpu: "1000m"
memory: "4Gi"
limits:
cpu: "3000m"
memory: "2Gi"
cpu: "1000m"
memory: "4Gi"
podLabels:
aadpodidbinding: "osdu-identity"
autoscale:
enabled: false
minReplicas: 2
maxReplicas: 20
scaleDown:
coolDownPeriod: 60
labels:
# DO NOT DELETE THIS LABEL. SET IT TO "false" WHEN AUTOSCALING IS DISABLED, SET IT TO "true" WHEN AUTOSCALING IS ENABLED
autoscalingEnabled: "false"
podAnnotations:
sidecar.istio.io/userVolumeMount: '[{"name": "azure-keyvault", "mountPath": "/mnt/azure-keyvault", "readonly": true}]'
baseUrl: "http://localhost/airflow"
......@@ -247,7 +256,7 @@ airflow:
AIRFLOW_VAR_CORE__SERVICE__FILE__HOST: "https://#{OSDU_SVC_ENDPOINT}#/api/file/v2"
AIRFLOW_VAR_CORE__SERVICE__WORKFLOW__HOST: "https://#{OSDU_SVC_ENDPOINT}#/api/workflow"
AIRFLOW_VAR_CORE__SERVICE__SEARCH_WITH_CURSOR__URL: "https://#{OSDU_SVC_ENDPOINT}#/api/search/v2/query_with_cursor"
AIRFLOW__WEBSERVER__WORKERS: 15
AIRFLOW__WEBSERVER__WORKERS: 8
AIRFLOW__WEBSERVER__WORKER_REFRESH_BATCH_SIZE: 0
AIRFLOW__CORE__STORE_SERIALIZED_DAGS: True #This flag decides whether to serialise DAGs and persist them in DB
AIRFLOW__CORE__STORE_DAG_CODE: True #This flag decides whether to persist DAG files code in DB
......
......@@ -139,13 +139,22 @@ airflow:
timeoutSeconds: 60
resources:
requests:
cpu: "2000m"
memory: "2Gi"
cpu: "1000m"
memory: "4Gi"
limits:
cpu: "3000m"
memory: "2Gi"
cpu: "1000m"
memory: "4Gi"
podLabels:
aadpodidbinding: "osdu-identity"
autoscale:
enabled: false
minReplicas: 2
maxReplicas: 20
scaleDown:
coolDownPeriod: 60
labels:
# DO NOT DELETE THIS LABEL. SET IT TO "false" WHEN AUTOSCALING IS DISABLED, SET IT TO "true" WHEN AUTOSCALING IS ENABLED
autoscalingEnabled: "false"
podAnnotations:
sidecar.istio.io/userVolumeMount: '[{"name": "azure-keyvault", "mountPath": "/mnt/azure-keyvault", "readonly": true}]'
baseUrl: "http://localhost/airflow"
......@@ -238,7 +247,7 @@ airflow:
AIRFLOW_VAR_CORE__SERVICE__FILE__HOST: "http://file.osdu.svc.cluster.local/api/file/v2"
AIRFLOW_VAR_CORE__SERVICE__WORKFLOW__HOST: "http://ingestion-workflow.osdu.svc.cluster.local/api/workflow"
AIRFLOW_VAR_CORE__SERVICE__SEARCH_WITH_CURSOR__URL: "http://search-service.osdu.svc.cluster.local/api/search/v2/query_with_cursor"
AIRFLOW__WEBSERVER__WORKERS: 15
AIRFLOW__WEBSERVER__WORKERS: 8
AIRFLOW__WEBSERVER__WORKER_REFRESH_BATCH_SIZE: 0
AIRFLOW__CORE__STORE_SERIALIZED_DAGS: True #This flag decides whether to serialise DAGs and persist them in DB
AIRFLOW__CORE__STORE_DAG_CODE: True #This flag decides whether to persist DAG files code in DB
......
......@@ -8,6 +8,7 @@ def addNamespace(namespace, manifest):
manifest['metadata']['namespace'] = namespace
if 'subjects' in manifest:
manifest['subjects'][0]['namespace'] = namespace
def removeReplicasFromWorkerStatefulSet(manifest):
# This function removes replicas property if autoscaling is enabled.
# This is done to make sure kubernetes does not reset the pod count when autoscaling is enabled
......@@ -16,12 +17,22 @@ def removeReplicasFromWorkerStatefulSet(manifest):
if manifest['metadata']['labels']['autoscalingEnabled'] == "true":
del manifest['spec']['replicas']
def removeReplicasFromWebserverDeployment(manifest):
# This function removes replicas property if autoscaling is enabled.
# This is done to make sure kubernetes does not reset the pod count when autoscaling is enabled
# Check the related issue for more information - https://community.opengroup.org/osdu/platform/deployment-and-operations/infra-azure-provisioning/-/issues/88
if manifest['kind'] == 'Deployment' and manifest['metadata']['name'] == 'airflow-web':
if manifest['metadata']['labels']['autoscalingEnabled'] == "true":
del manifest['spec']['replicas']
def addingNamespace(namespace):
for manifest in yaml.load_all(sys.stdin, Loader=yaml.FullLoader):
if manifest:
addNamespace(namespace, manifest)
removeReplicasFromWorkerStatefulSet(manifest)
removeReplicasFromWebserverDeployment(manifest)
print ('---')
print (yaml.dump(manifest, default_flow_style=False, sort_keys=False))
......
{{- if .Values.airflow.web.autoscale.enabled -}}
apiVersion: autoscaling/v2beta2
kind: HorizontalPodAutoscaler
metadata:
name: {{ printf "%s" .Release.Name | trunc 63 | trimSuffix "-" }}-web
labels:
component: web
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: {{ printf "%s" .Release.Name | trunc 63 | trimSuffix "-" }}-web
minReplicas: {{ .Values.airflow.web.autoscale.minReplicas }}
maxReplicas: {{ .Values.airflow.web.autoscale.maxReplicas }}
metrics:
- type: Resource
resource:
name: cpu
target:
averageUtilization: 70
type: Utilization
behavior:
scaleUp:
policies:
- type: Percent
value: 200
periodSeconds: 30
scaleDown:
policies:
- type: Percent
value: 30
periodSeconds: {{ .Values.airflow.web.autoscale.scaleDown.coolDownPeriod }}
{{- end }}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment