Commit de33abbc authored by Hema Vishnu Pola [Microsoft]'s avatar Hema Vishnu Pola [Microsoft]
Browse files

Merge branch 'airflow_pgbouncer_changes' into 'master'

Airflow pgbouncer changes

See merge request osdu/platform/deployment-and-operations/infra-azure-provisioning!254
parents fbe35b4a 275f12a0
......@@ -116,6 +116,22 @@ customConfig:
rbac:
createUser: "True"
################################################################################
# Specify pgbouncer configuration
#
pgbouncer:
enabled: true
port: 6543
max_client_connections: 3000
airflowdb:
name: airflow
host: $(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/base-name-sr --query value -otsv)-pg.postgres.database.azure.com
port: 5432
pool_size: 100
user: osdu_admin@$(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/base-name-sr --query value -otsv)-pg
passwordSecret: "postgres"
passwordSecretKey: "postgres-password"
################################################################################
# Specify the airflow configuration
#
......@@ -152,14 +168,14 @@ airflow:
enabled: false
externalDatabase:
type: postgres
## Azure PostgreSQL Database host or pgbouncer host (if pgbouncer is enabled)
host: airflow-pgbouncer.osdu.svc.cluster.local
## Azure PostgreSQL Database username, formatted as {username}@{hostname}
user: osdu_admin@$(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/base-name-sr --query value -otsv)-pg
user: osdu_admin@$(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/base-name-sr --query value -otsv)-pg
passwordSecret: "postgres"
passwordSecretKey: "postgres-password"
## Azure PostgreSQL Database host
host: $(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/base-name-sr --query value -otsv)-pg.postgres.database.azure.com
port: 5432
properties: "?sslmode=require"
port: 6543
database: airflow
database: airflow
###################################
......
......@@ -22,6 +22,23 @@ customConfig:
rbac:
createUser: "True"
################################################################################
# Specify pgbouncer configuration
#
pgbouncer:
enabled: true
port: 6543
max_client_connections: 3000
airflowdb:
name: airflow
host: #{base-name-sr}#-pg.postgres.database.azure.com
port: 5432
pool_size: 100
user: osdu_admin@#{base-name-sr}#-pg
passwordSecret: "postgres"
passwordSecretKey: "postgres-password"
################################################################################
# Specify the airflow configuration
#
......@@ -58,12 +75,11 @@ airflow:
enabled: false
externalDatabase:
type: postgres
host: #{base-name-sr}#-pg.postgres.database.azure.com #<-- Azure PostgreSQL Database host
user: osdu_admin@#{base-name-sr}#-pg #<-- Azure PostgreSQL Database username, formatted as {username}@{hostname}
host: airflow-pgbouncer.osdu.svc.cluster.local #<-- Azure PostgreSQL Database host or pgbouncer host (if pgbouncer is enabled)
user: osdu_admin@#{base-name-sr}#-pg #<-- Azure PostgreSQL Database username, formatted as {username}@{hostname}
passwordSecret: "postgres"
passwordSecretKey: "postgres-password"
port: 5432
properties: "?sslmode=require"
port: 6543
database: airflow
###################################
......
{{- if .Values.pgbouncer.enabled }}
kind: ConfigMap
apiVersion: v1
metadata:
name: airflow-pgbouncer-config
labels:
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}
data:
pgbouncer.ini: |
[databases]
{{ .Values.pgbouncer.airflowdb.name }} = host={{ .Values.pgbouncer.airflowdb.host }} dbname={{ .Values.pgbouncer.airflowdb.name }} port={{ .Values.pgbouncer.airflowdb.port }} pool_size={{ .Values.pgbouncer.airflowdb.pool_size }}
[pgbouncer]
pool_mode = transaction
listen_port = {{ .Values.pgbouncer.port }}
server_tls_sslmode = require
listen_addr = 0.0.0.0
auth_type = trust
auth_file = /etc/pgbouncer/users.txt
ignore_startup_parameters = extra_float_digits
max_client_conn = {{ .Values.pgbouncer.max_client_connections }}
verbose = 0
log_disconnections = 0
log_connections = 0
user = postgres
admin_users = postgres
users.txt.template: |
"${AIRFLOW_DATABASE_USERNAME}" "${AIRFLOW_DATABASE_PASSWORD}"
{{- end }}
\ No newline at end of file
{{- if .Values.pgbouncer.enabled }}
kind: Deployment
apiVersion: apps/v1
metadata:
name: airflow-pgbouncer
labels:
tier: airflow
component: pgbouncer
release: {{ .Release.Name }}
spec:
replicas: 2
strategy:
rollingUpdate:
maxSurge: 1
maxUnavailable: 0
selector:
matchLabels:
tier: airflow
component: pgbouncer
release: {{ .Release.Name }}
template:
metadata:
labels:
tier: airflow
component: pgbouncer
release: {{ .Release.Name }}
spec:
restartPolicy: Always
containers:
- name: pgbouncer
image: mcr.microsoft.com/azure-oss-db-tools/pgbouncer-sidecar:pgbouncerv1.13
imagePullPolicy: IfNotPresent
command:
- "/bin/sh"
args:
- "-c"
- >
( echo "cat <<EOF" ; cat /etc/pgbouncer/users.txt.template ; echo EOF ) | sh > /etc/pgbouncer/users.txt \
&& exec pgbouncer /etc/pgbouncer/pgbouncer.ini
ports:
- name: pgbouncer
containerPort: {{ .Values.pgbouncer.port }}
livenessProbe:
tcpSocket:
port: {{ .Values.pgbouncer.port }}
readinessProbe:
tcpSocket:
port: {{ .Values.pgbouncer.port }}
env:
- name: AIRFLOW_DATABASE_USERNAME
value: {{ .Values.pgbouncer.airflowdb.user }}
- name: AIRFLOW_DATABASE_PASSWORD
valueFrom:
secretKeyRef:
name: {{ .Values.pgbouncer.airflowdb.passwordSecret }}
key: {{ .Values.pgbouncer.airflowdb.passwordSecretKey }}
volumeMounts:
- name: pgbouncer-config
subPath: pgbouncer.ini
mountPath: /etc/pgbouncer/pgbouncer.ini
readOnly: true
- name: pgbouncer-config
subPath: users.txt.template
mountPath: /etc/pgbouncer/users.txt.template
readOnly: true
lifecycle:
preStop:
exec:
# Allow existing queries clients to complete within 120 seconds
command: ["/bin/sh", "-c", "killall -INT pgbouncer && sleep 120"]
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ['all']
volumes:
- name: pgbouncer-config
configMap:
name: airflow-pgbouncer-config
{{- end }}
\ No newline at end of file
{{- if .Values.pgbouncer.enabled }}
apiVersion: v1
kind: Service
metadata:
name: airflow-pgbouncer
labels:
tier: airflow
component: pgbouncer
release: {{ .Release.Name }}
spec:
type: ClusterIP
selector:
tier: airflow
component: pgbouncer
release: {{ .Release.Name }}
ports:
- name: pgbouncer
protocol: TCP
port: {{ .Values.pgbouncer.port }}
targetPort: {{ .Values.pgbouncer.port }}
{{- end }}
\ No newline at end of file
......@@ -33,16 +33,33 @@ spec:
readOnly: true
env:
- name: DATABASE_USER
{{- if .Values.pgbouncer.enabled }}
value: {{ .Values.pgbouncer.airflowdb.user }}
{{- else }}
value: {{ .Values.airflow.externalDatabase.user }}
{{- end }}
- name: DATABASE_HOST
{{- if .Values.pgbouncer.enabled }}
value: {{ .Values.pgbouncer.airflowdb.host }}
{{- else }}
value: {{ .Values.airflow.externalDatabase.host }}
{{- end }}
- name: DATABASE_DB
{{- if .Values.pgbouncer.enabled }}
value: {{ .Values.pgbouncer.airflowdb.name }}
{{- else }}
value: {{ .Values.airflow.externalDatabase.database }}
{{- end }}
- name: DATABASE_USER_PASS
valueFrom:
secretKeyRef:
name: postgres
key: postgres-password
{{- if .Values.pgbouncer.enabled }}
name: {{ .Values.pgbouncer.airflowdb.passwordSecret }}
key: {{ .Values.pgbouncer.airflowdb.passwordSecretKey }}
{{- else }}
name: {{ .Values.airflow.externalDatabase.passwordSecret }}
key: {{ .Values.airflow.externalDatabase.passwordSecretKey }}
{{- end }}
- name: AIRFLOW_ADMIN
value: {{ .Values.airflowLogin.name }}
- name: AIRFLOW_ADMIN_PASS
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment