diff --git a/charts/README.md b/charts/README.md index cd560b8a37caf76e9df5507132dcb63e9886e223..647da4e888d6fd4d7d533025e28d65b4588fde09 100644 --- a/charts/README.md +++ b/charts/README.md @@ -139,6 +139,16 @@ helm template osdu-flux ${INFRA_SRC}/charts/osdu-istio-auth -f ${INFRA_SRC}/char && git commit -m "Initialize Istio Auth Chart" \ && git push origin $UNIQUE) +# Extract manifests from the airflow charts. +helm template airflow ${INFRA_SRC}/charts/airflow -f ${INFRA_SRC}/charts/config.yaml | ${INFRA_SRC}/charts/airflow/add-namespace.py > ${FLUX_SRC}/providers/azure/hld-registry/airflow.yaml + +# Commit and Checkin to Deploy +(cd $FLUX_SRC \ + && git switch $UNIQUE \ + && git add ${FLUX_SRC}/providers/azure/hld-registry/airflow.yaml \ + && git commit -m "Initialize Airflow Chart" \ + && git push origin $UNIQUE) + # Extract manifests from each service chart. for SERVICE in partition entitlements-azure legal storage indexer-queue indexer-service search-service; diff --git a/charts/osdu-airflow/Chart.yaml b/charts/airflow/Chart.yaml similarity index 97% rename from charts/osdu-airflow/Chart.yaml rename to charts/airflow/Chart.yaml index 6c07dddc50c0a2f79da0898b792a275f9dd174d3..0587c61c023e6dad6f9cf14d694cca0caf3e6e4e 100644 --- a/charts/osdu-airflow/Chart.yaml +++ b/charts/airflow/Chart.yaml @@ -13,7 +13,7 @@ # limitations under the License. apiVersion: v2 -name: osdu-airflow +name: airflow appVersion: "latest" description: Installs the airflow and required components for osdu on Azure" version: 0.1.0 diff --git a/charts/osdu-airflow/charts/airflow-7.5.0.tgz b/charts/airflow/charts/airflow-7.5.0.tgz similarity index 100% rename from charts/osdu-airflow/charts/airflow-7.5.0.tgz rename to charts/airflow/charts/airflow-7.5.0.tgz diff --git a/charts/airflow/pipeline.yml b/charts/airflow/pipeline.yml new file mode 100644 index 0000000000000000000000000000000000000000..43b7754e8454376df12f132ba1eb116ef15773f7 --- /dev/null +++ b/charts/airflow/pipeline.yml @@ -0,0 +1,71 @@ +# Copyright © Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +trigger: + batch: true + branches: + include: + - master + paths: + include: + - /charts/airflow/* + exclude: + - /**/*.md + +pr: + autoCancel: false + branches: + include: + - "*" + exclude: + - master + paths: + include: + - /charts/airflow/* + exclude: + - /**/*.md + +resources: + repositories: + - repository: FluxRepo + type: git + name: k8-gitops-manifests + +variables: + - group: 'Azure - OSDU' + - group: 'Azure - OSDU secrets' + + - name: serviceName + value: "airflow" + - name: chartPath + value: "charts/airflow" + - name: valuesFile + value: "charts/airflow/values.yaml" + - name: 'MANIFEST_REPO' + value: $[ resources.repositories['FluxRepo'].name ] + +stages: + - template: /devops/chart-stages.yml + parameters: + serviceName: ${{ variables.serviceName }} + chartPath: ${{ variables.chartPath }} + valuesFile: ${{ variables.valuesFile }} + skipDeploy: ${{ variables.SKIP_DEPLOY }} + skipCheck: true + chartModificationScript: "scripts/add-namespace.py" + # Add multiple chart folder paths with pipe(|) seperated + extractedChartFolder: "templates|charts/airflow/templates" + providers: + - name: Azure + environments: ["dev"] diff --git a/charts/airflow/scripts/add-namespace.py b/charts/airflow/scripts/add-namespace.py new file mode 100644 index 0000000000000000000000000000000000000000..20485e81e0244f14b24b0d6dd985a8553e36ead7 --- /dev/null +++ b/charts/airflow/scripts/add-namespace.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python + +import yaml +import sys + +def addingNamespace(namespace): + for manifest in yaml.load_all(sys.stdin, Loader=yaml.FullLoader): + if manifest: + if 'metadata' in manifest and 'namespace' not in manifest['metadata'] and 'Namespace' not in manifest['kind']: + manifest['metadata']['namespace'] = namespace + if 'subjects' in manifest: + manifest['subjects'][0]['namespace'] = namespace + print ('---') + print (yaml.dump(manifest, default_flow_style=False, sort_keys=False)) + +namespace="osdu" +addingNamespace(namespace) diff --git a/charts/osdu-airflow/templates/airflow-log-processor-deployment.yaml b/charts/airflow/templates/airflow-log-processor-deployment.yaml similarity index 100% rename from charts/osdu-airflow/templates/airflow-log-processor-deployment.yaml rename to charts/airflow/templates/airflow-log-processor-deployment.yaml diff --git a/charts/osdu-airflow/templates/appinsight-statsd-configmap.yaml b/charts/airflow/templates/appinsight-statsd-configmap.yaml similarity index 100% rename from charts/osdu-airflow/templates/appinsight-statsd-configmap.yaml rename to charts/airflow/templates/appinsight-statsd-configmap.yaml diff --git a/charts/osdu-airflow/templates/appinsight-statsd-deployment.yaml b/charts/airflow/templates/appinsight-statsd-deployment.yaml similarity index 100% rename from charts/osdu-airflow/templates/appinsight-statsd-deployment.yaml rename to charts/airflow/templates/appinsight-statsd-deployment.yaml diff --git a/charts/osdu-airflow/templates/appinsight-statsd-service.yaml b/charts/airflow/templates/appinsight-statsd-service.yaml similarity index 100% rename from charts/osdu-airflow/templates/appinsight-statsd-service.yaml rename to charts/airflow/templates/appinsight-statsd-service.yaml diff --git a/charts/osdu-airflow/templates/dag-pvc.yaml b/charts/airflow/templates/dag-pvc.yaml similarity index 100% rename from charts/osdu-airflow/templates/dag-pvc.yaml rename to charts/airflow/templates/dag-pvc.yaml diff --git a/charts/osdu-airflow/templates/namespace.yaml b/charts/airflow/templates/namespace.yaml similarity index 100% rename from charts/osdu-airflow/templates/namespace.yaml rename to charts/airflow/templates/namespace.yaml diff --git a/charts/osdu-airflow/templates/post-install-job-configmap.yaml b/charts/airflow/templates/post-install-job-configmap.yaml similarity index 100% rename from charts/osdu-airflow/templates/post-install-job-configmap.yaml rename to charts/airflow/templates/post-install-job-configmap.yaml diff --git a/charts/osdu-airflow/templates/post-install-job.yaml b/charts/airflow/templates/post-install-job.yaml similarity index 100% rename from charts/osdu-airflow/templates/post-install-job.yaml rename to charts/airflow/templates/post-install-job.yaml diff --git a/charts/osdu-airflow/templates/remote-log-configmap.yaml b/charts/airflow/templates/remote-log-configmap.yaml similarity index 100% rename from charts/osdu-airflow/templates/remote-log-configmap.yaml rename to charts/airflow/templates/remote-log-configmap.yaml diff --git a/charts/osdu-airflow/templates/roles.yaml b/charts/airflow/templates/roles.yaml similarity index 100% rename from charts/osdu-airflow/templates/roles.yaml rename to charts/airflow/templates/roles.yaml diff --git a/charts/osdu-airflow/values.yaml b/charts/airflow/values.yaml similarity index 95% rename from charts/osdu-airflow/values.yaml rename to charts/airflow/values.yaml index 888e1972f78689c1cc52ef3f088b57097ea8462a..6c3c43fbcaa248b04a4e28c28cbeb647c46e9096 100644 --- a/charts/osdu-airflow/values.yaml +++ b/charts/airflow/values.yaml @@ -94,7 +94,7 @@ airflow: cert-manager.io/acme-challenge-type: http01 path: "/airflow" ## DNS name mapping to Application Gateway Public IP - host: appgatewayhostfqdn + host: #{DNS_HOST}# livenessPath: "/airflow/health" tls: enabled: true @@ -113,11 +113,11 @@ airflow: externalDatabase: type: postgres ## Azure PostgreSQL Database username, formatted as {username}@{hostname} - user: dbadmin@dbhost + user: osdu_admin@#{base-name-sr}#-pg passwordSecret: "postgres" passwordSecretKey: "postgres-password" ## Azure PostgreSQL Database host - host: dbhostfqdn + host: #{base-name-sr}#-pg.postgres.database.azure.com port: 5432 properties: "?sslmode=require" database: airflow @@ -125,7 +125,7 @@ airflow: enabled: false externalRedis: ## Azure Redis Cache host - host: redishost + host: #{base-name-sr}#-cache.redis.cache.windows.net port: 6380 passwordSecret: "redis" passwordSecretKey: "redis-password" diff --git a/charts/osdu-airflow/config.yaml b/charts/osdu-airflow/config.yaml deleted file mode 100644 index cd0ff03442d11c83177ff96016ce05fdbad4aba2..0000000000000000000000000000000000000000 --- a/charts/osdu-airflow/config.yaml +++ /dev/null @@ -1,125 +0,0 @@ -appinsightstatsd: - aadpodidbinding: "osdu-identity" -airflowLogin: - name: admin -airflow: - airflow: - image: - repository: apache/airflow - tag: 1.10.12-python3.6 - pullPolicy: IfNotPresent - pullSecret: "" - config: - AIRFLOW__SCHEDULER__STATSD_ON: "True" - AIRFLOW__SCHEDULER__STATSD_HOST: "appinsights-statsd" - AIRFLOW__SCHEDULER__STATSD_PORT: 8125 - AIRFLOW__SCHEDULER__STATSD_PREFIX: "osdu_airflow" - AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "False" - ## Enable for Debug purpose - AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "False" - AIRFLOW__WEBSERVER__AUTHENTICATE: "True" - AIRFLOW__WEBSERVER__AUTH_BACKEND: "airflow.contrib.auth.backends.password_auth" - AIRFLOW__API__AUTH_BACKEND: "airflow.contrib.auth.backends.password_auth" - AIRFLOW__CORE__REMOTE_LOGGING: "True" - AIRFLOW__CORE__REMOTE_LOG_CONN_ID: "az_log" - AIRFLOW__CORE__REMOTE_BASE_LOG_FOLDER: "wasb-airflowlog" - AIRFLOW__CORE__LOGGING_CONFIG_CLASS: "log_config.DEFAULT_LOGGING_CONFIG" - AIRFLOW__CORE__LOG_FILENAME_TEMPLATE: "{{ run_id }}/{{ ti.dag_id }}/{{ ti.task_id }}/{{ ts }}/{{ try_number }}.log" - AIRFLOW__CELERY__SSL_ACTIVE: "True" - AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX: "True" - extraEnv: - - name: AIRFLOW__CORE__FERNET_KEY - valueFrom: - secretKeyRef: - name: airflow - key: airflow-fernet-key - - name: AIRFLOW_CONN_AZ_LOG - valueFrom: - secretKeyRef: - name: airflow - key: airflow-remote-log-connection - extraConfigmapMounts: - - name: remote-log-config - mountPath: /opt/airflow/config - configMap: airflow-remote-log-config - readOnly: true - extraPipPackages: [ - "flask-bcrypt", - "apache-airflow[statsd]", - "apache-airflow[kubernetes]", - "apache-airflow-backport-providers-microsoft-azure" - ] - extraVolumeMounts: - # - name: airflow-kubernetes-config - # mountPath: "/home/airflow/.kube" - # readOnly: true - - name: azure-keyvault - mountPath: "/mnt/azure-keyvault" - readOnly: true - extraVolumes: - # - name: airflow-kubernetes-config - # secret: - # secretName: airflow-kubernetes-config - # items: - # - key: airflow-kubernetes-config - # path: config - - name: azure-keyvault - csi: - driver: secrets-store.csi.k8s.io - readOnly: true - volumeAttributes: - secretProviderClass: azure-keyvault - dags: - installRequirements: true - persistence: - enabled: true - existingClaim: airflowdagpvc - scheduler: - podLabels: - aadpodidbinding: "osdu-identity" - variables: | - {} - web: - podLabels: - aadpodidbinding: "osdu-identity" - baseUrl: "http://localhost/airflow" - ingress: - enabled: true - web: - annotations: - kubernetes.io/ingress.class: azure/application-gateway - cert-manager.io/cluster-issuer: letsencrypt - cert-manager.io/acme-challenge-type: http01 - path: "/airflow" - host: osdu-weisun.msft-osdu-test.org - livenessPath: "/airflow/health" - tls: - enabled: true - secretName: osdu-certificate - precedingPaths: - - path: "/airflow/*" - serviceName: airflow-web - servicePort: 8080 - workers: - podLabels: - aadpodidbinding: "osdu-identity" - flower: - enabled: false - postgresql: - enabled: false - externalDatabase: - type: postgres - user: osdu_admin@osdu-mvp-weisr-7heu-pg - passwordSecret: "postgres" - passwordSecretKey: "postgres-password" - host: "osdu-mvp-weisr-7heu-pg.postgres.database.azure.com" - port: 5432 - properties: "?sslmode=require" - database: airflow - redis: - enabled: false - externalRedis: - host: "osdu-mvp-weisr-7heu-cache.redis.cache.windows.net" - port: 6380 - passwordSecret: "redis" - passwordSecretKey: "redis-password" diff --git a/devops/chart-stages.yml b/devops/chart-stages.yml index f9da464032132268d1dc78ada9557150c4109625..f1bd5098c576a3dcca648e61021dce05b32838d3 100644 --- a/devops/chart-stages.yml +++ b/devops/chart-stages.yml @@ -19,6 +19,7 @@ parameters: skipDeploy: false hldRegPath: "providers/azure/hld-registry" checkoutRepo: self + extractedChartFolder: "templates" stages: - ${{ each provider in parameters.providers }}: @@ -51,6 +52,8 @@ stages: skipDeploy: ${{ parameters.skipDeploy }} hldRegPath: ${{ parameters.hldRegPath }} checkoutRepo: ${{ parameters.checkoutRepo }} + chartModificationScript: ${{ parameters.chartModificationScript }} + extractedChartFolder: ${{ parameters.extractedChartFolder }} - template: tasks/flux-chart-wait.yml parameters: diff --git a/devops/tasks/aks-deployment-steps.yml b/devops/tasks/aks-deployment-steps.yml index 66102158e8d5e2ae59f22b9946a6e14bd80edf11..7b78bb0086d909369283adb01847aba0f2659c18 100644 --- a/devops/tasks/aks-deployment-steps.yml +++ b/devops/tasks/aks-deployment-steps.yml @@ -22,6 +22,8 @@ parameters: hldRegPath: "" generationPath: "generated" checkoutRepo: self + chartModificationScript: "" + extractedChartFolder: "" steps: - checkout: FluxRepo @@ -50,6 +52,7 @@ steps: chartPath: ${{ parameters.chartPath }} valuesFile: ${{ parameters.valuesFile }} generationPath: ${{parameters.generationPath}} + chartModificationScript: ${{parameters.chartModificationScript}} - template: gitops.yml parameters: serviceName: ${{parameters.serviceName}} @@ -58,3 +61,4 @@ steps: branchName: ${{parameters.environment}} hldRegPath: ${{parameters.hldRegPath}} skipDeploy: ${{parameters.skipDeploy}} + extractedChartFolder: ${{parameters.extractedChartFolder}} diff --git a/devops/tasks/gitops.yml b/devops/tasks/gitops.yml index b0a718d4d1fc6823c764904ff1408600803447d5..3825548d8f2ac6cf343f93d2d33a8e55ffe6ce9a 100644 --- a/devops/tasks/gitops.yml +++ b/devops/tasks/gitops.yml @@ -26,6 +26,8 @@ steps: SERVICE_NAME: ${{parameters.serviceName}} HLD_REG_PATH: ${{parameters.hldRegPath}} GENERATION_PATH: ${{parameters.generationPath}} + CHART_FOLDERS: ${{parameters.extractedChartFolder}} + inputs: targetType: "inline" script: | @@ -72,9 +74,14 @@ steps: echo "COPYING YAML FILES TO $HLD_REG_PATH/$SERVICE_NAME" mkdir -p ./$HLD_REG_PATH/$SERVICE_NAME rm -rf ./$HLD_REG_PATH/$SERVICE_NAME/* - cp -rf $GITOPS_MANIFEST_DIRECTORY/$SERVICE_NAME/templates/* ./$HLD_REG_PATH/$SERVICE_NAME + + IFS="|"; for folder in $CHART_FOLDERS; do + echo "COPYING YAML FILES OF EXTRACTED CHARTS FOLDER $folder" + cp -rf $GITOPS_MANIFEST_DIRECTORY/$SERVICE_NAME/$folder/* ./$HLD_REG_PATH/$SERVICE_NAME + done fi + git add -A if [[ $(git status --porcelain) ]]; then diff --git a/devops/tasks/helm-template.yml b/devops/tasks/helm-template.yml index 09cbfb81190d307286d449ace18cd41b1310c5ed..ca32265179b532a927fcdf4cd0b5deaca6627a33 100644 --- a/devops/tasks/helm-template.yml +++ b/devops/tasks/helm-template.yml @@ -46,3 +46,20 @@ steps: echo "Extracting Manifest" helm template $SERVICE_NAME $CHART_PATH -f $(Build.SourcesDirectory)/$HELM_SOURCE_NAME/${{parameters.chartPath}}/values.yaml --output-dir $CHART_PATH/$GENERATION_PATH + + if [[ "${{parameters.chartModificationScript}}" != "" ]] + then + echo "Updating generated charts with the script ${{parameters.chartModificationScript}}" + chmod -fR 755 $(Build.SourcesDirectory)/$HELM_SOURCE_NAME/${{parameters.chartPath}}/${{parameters.chartModificationScript}} + + pip install -U PyYAML + + touch tempFile + + for file in `find $CHART_PATH/$GENERATION_PATH -name '*.yaml'`; do + cat $file | $(Build.SourcesDirectory)/$HELM_SOURCE_NAME/${{parameters.chartPath}}/${{parameters.chartModificationScript}} > tempFile + cat tempFile > $file + done + + rm tempFile + fi \ No newline at end of file