Commit e80b386d authored by Kiran Veerapaneni's avatar Kiran Veerapaneni Committed by Daniel Scholl
Browse files

adding custom stages for publishing images for airflow log processor

parent f7f12059
......@@ -83,6 +83,9 @@ Create the helm chart values file necessary to install airflow charts.
```bash
# Setup Variables
BRANCH="master"
TAG="latest"
GROUP=$(az group list --query "[?contains(name, 'cr${UNIQUE}')].name" -otsv)
ENV_VAULT=$(az keyvault list --resource-group $GROUP --query [].name -otsv)
......@@ -192,6 +195,10 @@ airflow:
passwordSecret: "redis"
passwordSecretKey: "redis-password"
image:
repository: $(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/container-registry --query value -otsv).azurecr.io
branch: $BRANCH
tag: $TAG
EOF
```
......@@ -248,7 +255,6 @@ helm template osdu-flux ${INFRA_SRC}/charts/osdu-common -f ${INFRA_SRC}/charts/c
&& git commit -m "Initialize Common Chart" \
&& git push origin $UNIQUE)
# Extract manifests from the istio charts.
helm template osdu-flux ${INFRA_SRC}/charts/osdu-istio -f ${INFRA_SRC}/charts/config.yaml > ${FLUX_SRC}/providers/azure/hld-registry/osdu-istio.yaml
......@@ -262,8 +268,20 @@ helm template osdu-flux ${INFRA_SRC}/charts/osdu-istio-auth -f ${INFRA_SRC}/char
&& git commit -m "Initialize Istio Auth Chart" \
&& git push origin $UNIQUE)
#Installing PyYaml required for airflow
# Publish Docker Images for airflow components
CONTAINER_REGISTRY=$(az keyvault secret show --id https://${ENV_VAULT}.vault.azure.net/secrets/container-registry --query value -otsv)
az acr login -n $CONTAINER_REGISTRY
for SERVICE in airflow-function;
do
cd ${INFRA_SRC}/source/$SERVICE
IMAGE=$CONTAINER_REGISTRY.azurecr.io/$SERVICE-$BRANCH:$(TAG)
docker build -t $IMAGE .
docker push $IMAGE
done
# Installing PyYaml required for airflow
pip3 install -U PyYAML
# Extract manifests from the airflow charts.
helm template airflow ${INFRA_SRC}/charts/airflow -f ${INFRA_SRC}/charts/config_airflow.yaml | python3 ${INFRA_SRC}/charts/airflow/scripts/add-namespace.py > ${FLUX_SRC}/providers/azure/hld-registry/airflow.yaml
......@@ -274,7 +292,6 @@ helm template airflow ${INFRA_SRC}/charts/airflow -f ${INFRA_SRC}/charts/config_
&& git commit -m "Initialize Airflow Chart" \
&& git push origin $UNIQUE)
# Extract manifests from each service chart.
for SERVICE in partition entitlements-azure legal storage indexer-queue indexer-service search-service;
do
......
......@@ -100,3 +100,8 @@ airflow:
port: 6380
passwordSecret: "redis"
passwordSecretKey: "redis-password"
image:
repository: #{container-registry}#.azurecr.io
branch: #{ENVIRONMENT_NAME}#
tag: #{Build.SourceVersion}#
\ No newline at end of file
# Copyright � Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This template is responsible for using an artifact docker image and pushing it to an ACR.
parameters:
environment: ''
providerName: ''
sourceFolder: ''
dockerFilePaths: ''
steps:
- checkout: ${{ parameters.checkoutRepo }}
persistCredentials: true
- task: AzureCLI@1
displayName: 'Docker Build + ACR Push'
condition: and(succeeded(), eq('${{ parameters.providerName }}', 'Azure'))
env:
SOURCE: ${{ parameters.sourceFolder }}
DOCKER_FILE_PATHS: ${{ parameters.dockerFilePaths }}
inputs:
azureSubscription: '$(SERVICE_CONNECTION_NAME)'
addSpnToEnvironment: true
scriptLocation: inlineScript
inlineScript: |
#!/usr/bin/env bash
set -euo pipefail
curl -L https://aka.ms/acr/installaad/bash | /bin/bash
echo "Logging in to the ACR Registry"
echo "------------------------------------"
az acr login -n $(CONTAINER_REGISTRY_NAME)
IFS="|"; for dockerFile in $DOCKER_FILE_PATHS; do
cd $(Build.SourcesDirectory)/$SOURCE/$dockerFile
IMAGE=$(CONTAINER_REGISTRY_NAME).azurecr.io/$dockerFile-${{ parameters.environment }}:$(Build.SourceVersion)
echo "Tagging and Pushing $IMAGE to ACR "
echo "------------------------------------"
docker build -t $IMAGE .
docker push $IMAGE
echo "Show recent tags of repository $dockerFile"
az acr repository show-tags -n $(CONTAINER_REGISTRY_NAME) --repository $dockerFile-${{ parameters.environment }} --top 10 --orderby time_desc
done
\ No newline at end of file
# Copyright © Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
parameters:
providers: []
checkoutRepo: self
stages:
- ${{ each provider in parameters.providers }}:
- ${{ each environment in provider.environments }}:
- stage: 'Publish_Images_${{ provider.name }}_${{ environment }}'
variables:
- group: '${{ provider.name }} Target Env Secrets - ${{ environment }}'
- group: '${{ provider.name }} Target Env - ${{ environment }}'
jobs:
- deployment: PublishImages
pool:
name: $(AGENT_POOL)
environment: ${{ environment }}
strategy:
runOnce:
deploy:
steps:
- download: none
- template: acr-publish.yml
parameters:
providerName: ${{ provider.name }}
environment: ${{ environment }}
sourceFolder: ${{ parameters.sourceFolder }}
dockerFilePaths: ${{ parameters.dockerFilePaths }}
checkoutRepo: ${{ parameters.checkoutRepo }}
......@@ -20,6 +20,7 @@ trigger:
paths:
include:
- /charts/airflow/*
- /source/airflow-function/*
exclude:
- /**/*.md
......@@ -33,6 +34,7 @@ pr:
paths:
include:
- /charts/airflow/*
- /source/airflow-function/*
exclude:
- /**/*.md
......@@ -56,6 +58,15 @@ variables:
value: $[ resources.repositories['FluxRepo'].name ]
stages:
- template: pipeline-stages/airflow-custom-stages.yml
parameters:
sourceFolder: "source"
#multiple file paths to be provided with pipe(|) seperated
dockerFilePaths: "airflow-function"
providers:
- name: Azure
environments: ["dev", "glab"]
- template: /devops/chart-stages.yml
parameters:
serviceName: ${{ variables.serviceName }}
......@@ -68,4 +79,4 @@ stages:
extractedChartFolder: "templates|charts/airflow/templates"
providers:
- name: Azure
environments: ["dev"]
environments: ["dev", "glab"]
......@@ -17,7 +17,7 @@ spec:
spec:
containers:
- name: airflow-log-processor
image: binroon/airflow-logs-to-workspace
image: {{ .Values.image.repository }}/airflow-function-{{ .Values.image.branch }}:{{ .Values.image.tag | default .Chart.AppVersion }}
imagePullPolicy: Always
env:
- name: AzureFunctionsJobHost__functions__0
......
# Copyright © Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
trigger:
batch: true
branches:
include:
- master
paths:
include:
- /charts/airflow/*
exclude:
- /**/*.md
resources:
repositories:
- repository: FluxRepo
type: git
name: k8-gitops-manifests
variables:
- group: 'Azure - OSDU'
- group: 'Azure - OSDU secrets'
- name: serviceName
value: "airflow"
- name: chartPath
value: "charts/airflow"
- name: valuesFile
value: "charts/airflow/helm-config.yaml"
- name: 'MANIFEST_REPO'
value: $[ resources.repositories['FluxRepo'].name ]
stages:
- template: /devops/chart-stages.yml
parameters:
serviceName: ${{ variables.serviceName }}
chartPath: ${{ variables.chartPath }}
valuesFile: ${{ variables.valuesFile }}
skipDeploy: ${{ variables.SKIP_DEPLOY }}
skipCheck: true
chartModificationScript: "scripts/add-namespace.py"
# Add multiple chart folder paths with pipe(|) seperated
extractedChartFolder: "templates|charts/airflow/templates"
providers:
- name: Azure
environments: ["demo"]
......@@ -356,6 +356,22 @@ az pipelines create \
-ojson
```
4. Add a Pipeline for __chart-osdu-airflow__ to deploy Istio Authorization Policies.
_Repo:_ `infra-azure-provisioning`
_Path:_ `/devops/pipelines/chart-airflow.yml`
_Validate:_ Airflow Pods are running except for airflow-setup-default-user which is a job pod.
```bash
az pipelines create \
--name 'chart-airflow' \
--repository infra-azure-provisioning \
--branch master \
--repository-type tfsgit \
--yaml-path /devops/pipelines/chart-airflow.yml \
-ojson
```
__Create the Service Pipelines__
Create the pipelines and run things in this exact order.
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment