Merge branch 'users/kiveerap/UpdateReadMeDoc' into 'master'

updating readme doc for airflow manual deployment

See merge request osdu/platform/deployment-and-operations/infra-azure-provisioning!73

charts/README.md

@@ -71,6 +71,129 @@ global:
     password: $(az keyvault secret show --id https://${COMMON_VAULT}.vault.azure.net/secrets/istio-password --query value -otsv)
 EOF
 ```
+Create the helm chart values file necessary to install airflow charts.
+
+- Download [helm-config.yaml](https://community.opengroup.org/osdu/platform/deployment-and-operations/infra-azure-provisioning/-/raw/master/charts/airflow/helm-config.yaml), which will configure OSDU on Azure.
+
+  `wget https://community.opengroup.org/osdu/platform/deployment-and-operations/infra-azure-provisioning/-/raw/master/charts/airflow/helm-config.yaml -O config_airflow.yaml`
+
+- Edit the newly downloaded [config_airflow.yaml](https://community.opengroup.org/osdu/platform/deployment-and-operations/infra-azure-provisioning/-/raw/master/charts/airflow/helm-config.yaml) and fill out the required sections `externalDatabase` and `externalRedis`.
+
+
+
+```bash
+# Setup Variables
+GROUP=$(az group list --query "[?contains(name, 'cr${UNIQUE}')].name" -otsv)
+ENV_VAULT=$(az keyvault list --resource-group $GROUP --query [].name -otsv)
+
+# Translate Values File
+cat > config_airflow.yaml << EOF
+# This file contains the essential configs for the osdu airflow on azure helm chart
+appinsightstatsd:
+  aadpodidbinding: "osdu-identity"
+airflowLogin:
+  name: admin
+airflow:
+  airflow:
+    image:
+      repository: apache/airflow
+      tag: 1.10.12-python3.6
+      pullPolicy: IfNotPresent
+      pullSecret: ""
+    config:
+      AIRFLOW__SCHEDULER__STATSD_ON: "True"
+      AIRFLOW__SCHEDULER__STATSD_HOST: "appinsights-statsd"
+      AIRFLOW__SCHEDULER__STATSD_PORT: 8125
+      AIRFLOW__SCHEDULER__STATSD_PREFIX: "osdu_airflow"
+      AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "False"
+      ## Enable for Debug purpose
+      AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "False"
+      AIRFLOW__WEBSERVER__AUTHENTICATE: "True"
+      AIRFLOW__WEBSERVER__AUTH_BACKEND: "airflow.contrib.auth.backends.password_auth"
+      AIRFLOW__API__AUTH_BACKEND: "airflow.contrib.auth.backends.password_auth"
+      AIRFLOW__CORE__REMOTE_LOGGING: "True"
+      AIRFLOW__CORE__REMOTE_LOG_CONN_ID: "az_log"
+      AIRFLOW__CORE__REMOTE_BASE_LOG_FOLDER: "wasb-airflowlog"
+      AIRFLOW__CORE__LOGGING_CONFIG_CLASS: "log_config.DEFAULT_LOGGING_CONFIG"
+      AIRFLOW__CORE__LOG_FILENAME_TEMPLATE: "{{ run_id }}/{{ ti.dag_id }}/{{ ti.task_id }}/{{ ts }}/{{ try_number }}.log"
+      AIRFLOW__CELERY__SSL_ACTIVE: "True"
+      AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX: "True"
+    extraEnv:
+      - name: AIRFLOW__CORE__FERNET_KEY
+        valueFrom:
+          secretKeyRef:
+            name: airflow
+            key: fernet-key
+      - name: AIRFLOW_CONN_AZ_LOG
+        valueFrom:
+          secretKeyRef:
+            name: airflow
+            key: remote-log-connection
+    extraConfigmapMounts:
+      - name: remote-log-config
+        mountPath: /opt/airflow/config
+        configMap: airflow-remote-log-config
+        readOnly: true
+    extraPipPackages: [
+        "flask-bcrypt",
+        "apache-airflow[statsd]",
+        "apache-airflow[kubernetes]",
+        "apache-airflow-backport-providers-microsoft-azure"
+    ]
+    extraVolumeMounts:
+      - name: azure-keyvault
+        mountPath: "/mnt/azure-keyvault"
+        readOnly: true
+    extraVolumes:
+      - name: azure-keyvault
+        csi:
+          driver: secrets-store.csi.k8s.io
+          readOnly: true
+          volumeAttributes:
+            secretProviderClass: azure-keyvault
+  dags:
+    installRequirements: true
+    persistence:
+      enabled: true
+      existingClaim: airflowdagpvc
+  scheduler:
+    podLabels:
+      aadpodidbinding: "osdu-identity"
+    variables: |
+      {}
+  web:
+    podLabels:
+      aadpodidbinding: "osdu-identity"
+    baseUrl: "http://localhost/airflow"
+  workers:
+    podLabels:
+      aadpodidbinding: "osdu-identity"
+  flower:
+    enabled: false
+  postgresql:
+    enabled: false
+  externalDatabase:
+    type: postgres
+    ## Azure PostgreSQL Database username, formatted as {username}@{hostname}
+    user:  osdu_admin@$(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/base-name-sr --query value -otsv)-pg
+    passwordSecret: "postgres"
+    passwordSecretKey: "postgres-password"
+    ## Azure PostgreSQL Database host
+    host: $(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/base-name-sr --query value -otsv)-pg.postgres.database.azure.com
+    port: 5432
+    properties: "?sslmode=require"
+    database: airflow
+  redis:
+    enabled: false
+  externalRedis:
+    ## Azure Redis Cache host
+    host: $(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/base-name-sr --query value -otsv)-cache.redis.cache.windows.net
+    port: 6380
+    passwordSecret: "redis"
+    passwordSecretKey: "redis-password"
+
+EOF
+```
 
 __Clone Service Repositories__
 
@@ -139,6 +262,18 @@ helm template osdu-flux ${INFRA_SRC}/charts/osdu-istio-auth -f ${INFRA_SRC}/char
   && git commit -m "Initialize Istio Auth Chart" \
   && git push origin $UNIQUE)
 
+#Installing PyYaml required for airflow
+pip3 install -U PyYAML
+# Extract manifests from the airflow charts.
+helm template airflow ${INFRA_SRC}/charts/airflow -f ${INFRA_SRC}/charts/config_airflow.yaml | python3 ${INFRA_SRC}/charts/airflow/scripts/add-namespace.py > ${FLUX_SRC}/providers/azure/hld-registry/airflow.yaml
+
+# Commit and Checkin to Deploy
+(cd $FLUX_SRC \
+  && git switch $UNIQUE \
+  && git add ${FLUX_SRC}/providers/azure/hld-registry/airflow.yaml \
+  && git commit -m "Initialize Airflow Chart" \
+  && git push origin $UNIQUE)
+
 
 # Extract manifests from each service chart.
 for SERVICE in partition entitlements-azure legal storage indexer-queue indexer-service search-service;

charts/airflow/values.yaml → charts/airflow/helm-config.yaml

@@ -50,21 +50,10 @@ airflow:
         "apache-airflow-backport-providers-microsoft-azure"
     ]
     extraVolumeMounts:
-        ## mount the kube config from volume into default kube config file for Airflow Kubernetes Operator 
-        # - name: airflow-kubernetes-config
-        #   mountPath: "/home/airflow/.kube"
-        #   readOnly: true
         - name: azure-keyvault
           mountPath: "/mnt/azure-keyvault"
           readOnly: true
     extraVolumes:
-        ## mount Kube config from secret 
-        # - name: airflow-kubernetes-config
-        #   secret:
-        #     secretName: airflow-kubernetes-config
-        #     items:
-        #     - key: airflow-kubernetes-config
-        #       path: config
         - name: azure-keyvault
           csi:
             driver: secrets-store.csi.k8s.io
@@ -85,24 +74,6 @@ airflow:
     podLabels:
       aadpodidbinding: "osdu-identity"
     baseUrl: "http://localhost/airflow"
-  ingress:
-    enabled: true
-    web:
-      annotations:
-        kubernetes.io/ingress.class: azure/application-gateway
-        cert-manager.io/cluster-issuer: letsencrypt
-        cert-manager.io/acme-challenge-type: http01
-      path: "/airflow"
-      ## DNS name mapping to Application Gateway Public IP
-      host: #{DNS_HOST}#
-      livenessPath: "/airflow/health"
-      tls:
-        enabled: true
-        secretName: osdu-certificate
-      precedingPaths:
-        - path: "/airflow/*"
-          serviceName: airflow-web
-          servicePort: 8080
   workers:
     podLabels:
       aadpodidbinding: "osdu-identity"             

charts/airflow/pipeline.yml

@@ -51,7 +51,7 @@ variables:
   - name: chartPath
     value: "charts/airflow"
   - name: valuesFile
-    value: "charts/airflow/values.yaml"
+    value: "charts/airflow/helm-config.yaml"
   - name: 'MANIFEST_REPO'
     value: $[ resources.repositories['FluxRepo'].name ]