Merge branch 'users/kiveerap/airflowlogprocessorpipeline' into 'master'

adding custom stages for publishing images for airflow log processor

See merge request osdu/platform/deployment-and-operations/infra-azure-provisioning!69

charts/README.md

@@ -83,6 +83,9 @@ Create the helm chart values file necessary to install airflow charts.
 
 ```bash
 # Setup Variables
+BRANCH="master"
+TAG="latest"
+
 GROUP=$(az group list --query "[?contains(name, 'cr${UNIQUE}')].name" -otsv)
 ENV_VAULT=$(az keyvault list --resource-group $GROUP --query [].name -otsv)
 
@@ -192,6 +195,10 @@ airflow:
     passwordSecret: "redis"
     passwordSecretKey: "redis-password"
 
+image:
+  repository: $(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/container-registry --query value -otsv).azurecr.io
+  branch: $BRANCH
+  tag: $TAG
 EOF
 ```
 
@@ -248,7 +255,6 @@ helm template osdu-flux ${INFRA_SRC}/charts/osdu-common -f ${INFRA_SRC}/charts/c
   && git commit -m "Initialize Common Chart" \
   && git push origin $UNIQUE)
 
-
 # Extract manifests from the istio charts.
 helm template osdu-flux ${INFRA_SRC}/charts/osdu-istio -f ${INFRA_SRC}/charts/config.yaml > ${FLUX_SRC}/providers/azure/hld-registry/osdu-istio.yaml
 
@@ -262,8 +268,20 @@ helm template osdu-flux ${INFRA_SRC}/charts/osdu-istio-auth -f ${INFRA_SRC}/char
   && git commit -m "Initialize Istio Auth Chart" \
   && git push origin $UNIQUE)
 
-#Installing PyYaml required for airflow
+# Publish Docker Images for airflow components
+CONTAINER_REGISTRY=$(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/container-registry --query value -otsv)
+az acr login -n $CONTAINER_REGISTRY
+for SERVICE in airflow-function;
+do
+  cd ${INFRA_SRC}/source/$SERVICE
+  IMAGE=$CONTAINER_REGISTRY.azurecr.io/$SERVICE-$BRANCH:$(TAG)
+  docker build -t $IMAGE .
+  docker push $IMAGE
+done
+
+# Installing PyYaml required for airflow
 pip3 install -U PyYAML
+
 # Extract manifests from the airflow charts.
 helm template airflow ${INFRA_SRC}/charts/airflow -f ${INFRA_SRC}/charts/config_airflow.yaml | python3 ${INFRA_SRC}/charts/airflow/scripts/add-namespace.py > ${FLUX_SRC}/providers/azure/hld-registry/airflow.yaml
 
@@ -274,7 +292,6 @@ helm template airflow ${INFRA_SRC}/charts/airflow -f ${INFRA_SRC}/charts/config_
   && git commit -m "Initialize Airflow Chart" \
   && git push origin $UNIQUE)
 
-
 # Extract manifests from each service chart.
 for SERVICE in partition entitlements-azure legal storage indexer-queue indexer-service search-service;
 do

charts/airflow/helm-config.yaml

@@ -100,3 +100,8 @@ airflow:
     port: 6380
     passwordSecret: "redis"
     passwordSecretKey: "redis-password"
+    
+image:
+  repository: #{container-registry}#.azurecr.io
+  branch: #{ENVIRONMENT_NAME}#
+  tag: #{Build.SourceVersion}#
\ No newline at end of file

charts/airflow/pipeline-stages/acr-publish.yml

+#  Copyright � Microsoft Corporation
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+# This template is responsible for using an artifact docker image and pushing it to an ACR.
+parameters:
+  environment: ''
+  providerName: ''
+  sourceFolder: ''
+  dockerFilePaths: ''
+
+steps:
+  - checkout: ${{ parameters.checkoutRepo }}
+    persistCredentials: true
+  - task: AzureCLI@1
+    displayName: 'Docker Build + ACR Push'
+    condition: and(succeeded(), eq('${{ parameters.providerName }}', 'Azure'))
+    env:
+      SOURCE: ${{ parameters.sourceFolder }}
+      DOCKER_FILE_PATHS: ${{ parameters.dockerFilePaths }}
+
+    inputs:
+      azureSubscription: '$(SERVICE_CONNECTION_NAME)'
+      addSpnToEnvironment: true
+      scriptLocation: inlineScript
+      inlineScript: |
+        #!/usr/bin/env bash
+        set -euo pipefail
+
+        curl -L https://aka.ms/acr/installaad/bash | /bin/bash
+        echo "Logging in to the ACR Registry"
+        echo "------------------------------------"
+        az acr login -n $(CONTAINER_REGISTRY_NAME)
+
+        IFS="|"; for dockerFile in $DOCKER_FILE_PATHS; do
+          cd $(Build.SourcesDirectory)/$SOURCE/$dockerFile
+          IMAGE=$(CONTAINER_REGISTRY_NAME).azurecr.io/$dockerFile-${{ parameters.environment }}:$(Build.SourceVersion)
+          echo "Tagging and Pushing $IMAGE to ACR "
+          echo "------------------------------------"
+          docker build -t $IMAGE .
+
+          docker push $IMAGE
+          echo "Show recent tags of repository $dockerFile"
+          az acr repository show-tags -n $(CONTAINER_REGISTRY_NAME) --repository $dockerFile-${{ parameters.environment }} --top 10 --orderby time_desc
+        done
\ No newline at end of file

charts/airflow/pipeline-stages/airflow-custom-stages.yml

+#  Copyright © Microsoft Corporation
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+
+parameters:
+  providers: []
+  checkoutRepo: self
+
+stages:
+  - ${{ each provider in parameters.providers }}:
+    - ${{ each environment in provider.environments }}:
+      - stage: 'Publish_Images_${{ provider.name }}_${{ environment }}'
+
+        variables:
+          - group: '${{ provider.name }} Target Env Secrets - ${{ environment }}'
+          - group: '${{ provider.name }} Target Env - ${{ environment }}'
+
+
+        jobs:
+        - deployment: PublishImages
+          pool:
+            name: $(AGENT_POOL)
+          environment: ${{ environment }}
+          strategy:
+            runOnce:
+              deploy:
+                steps:
+                - download: none
+
+                - template: acr-publish.yml
+                  parameters:
+                    providerName: ${{ provider.name }}
+                    environment: ${{ environment }}
+                    sourceFolder: ${{ parameters.sourceFolder }}
+                    dockerFilePaths: ${{ parameters.dockerFilePaths }}
+                    checkoutRepo: ${{ parameters.checkoutRepo }}

charts/airflow/pipeline.yml

@@ -20,6 +20,7 @@ trigger:
   paths:
     include:
       - /charts/airflow/*
+      - /source/airflow-function/*
     exclude:
       - /**/*.md
 
@@ -33,6 +34,7 @@ pr:
   paths:
     include:
       - /charts/airflow/*
+      - /source/airflow-function/*
     exclude:
       - /**/*.md
 
@@ -56,6 +58,15 @@ variables:
     value: $[ resources.repositories['FluxRepo'].name ]
 
 stages:
+  - template: pipeline-stages/airflow-custom-stages.yml
+    parameters:
+      sourceFolder: "source"
+      #multiple file paths to be provided with pipe(|) seperated
+      dockerFilePaths: "airflow-function"
+      providers:
+        - name: Azure
+          environments: ["dev", "glab"]
+
   - template: /devops/chart-stages.yml
     parameters:
       serviceName: ${{ variables.serviceName }}
@@ -68,4 +79,4 @@ stages:
       extractedChartFolder: "templates|charts/airflow/templates"
       providers:
         - name: Azure
-          environments: ["dev"]
+          environments: ["dev", "glab"]

charts/airflow/templates/airflow-log-processor-deployment.yaml

@@ -17,7 +17,7 @@ spec:
     spec:
       containers:
       - name: airflow-log-processor
-        image: binroon/airflow-logs-to-workspace
+        image: {{ .Values.image.repository }}/airflow-function-{{ .Values.image.branch }}:{{ .Values.image.tag | default .Chart.AppVersion }}
         imagePullPolicy: Always
         env:
           - name: AzureFunctionsJobHost__functions__0

devops/pipelines/chart-airflow.yml

+#  Copyright © Microsoft Corporation
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+trigger:
+  batch: true
+  branches:
+    include:
+      - master
+  paths:
+    include:
+      - /charts/airflow/*
+    exclude:
+      - /**/*.md
+
+resources:
+  repositories:
+  - repository: FluxRepo
+    type: git
+    name: k8-gitops-manifests
+
+variables:
+  - group: 'Azure - OSDU'
+  - group: 'Azure - OSDU secrets'
+
+  - name: serviceName
+    value: "airflow"
+  - name: chartPath
+    value: "charts/airflow"
+  - name: valuesFile
+    value: "charts/airflow/helm-config.yaml"
+  - name: 'MANIFEST_REPO'
+    value: $[ resources.repositories['FluxRepo'].name ]
+
+stages:
+  - template: /devops/chart-stages.yml
+    parameters:
+      serviceName: ${{ variables.serviceName }}
+      chartPath: ${{ variables.chartPath }}
+      valuesFile: ${{ variables.valuesFile }}
+      skipDeploy: ${{ variables.SKIP_DEPLOY }}
+      skipCheck: true
+      chartModificationScript: "scripts/add-namespace.py"
+      # Add multiple chart folder paths with pipe(|) seperated
+      extractedChartFolder: "templates|charts/airflow/templates"
+      providers:
+        - name: Azure
+          environments: ["demo"]

docs/service-automation.md

@@ -356,6 +356,22 @@ az pipelines create \
   -ojson
 ```
 
+4. Add a Pipeline for __chart-osdu-airflow__  to deploy Istio Authorization Policies.
+
+    _Repo:_ `infra-azure-provisioning`
+    _Path:_ `/devops/pipelines/chart-airflow.yml`
+    _Validate:_ Airflow Pods are running except for airflow-setup-default-user which is a job pod.
+
+```bash
+az pipelines create \
+  --name 'chart-airflow'  \
+  --repository infra-azure-provisioning  \
+  --branch master  \
+  --repository-type tfsgit  \
+  --yaml-path /devops/pipelines/chart-airflow.yml  \
+  -ojson
+```
+
 __Create the Service Pipelines__
 
 Create the pipelines and run things in this exact order.