Skip to content
Snippets Groups Projects
Commit 904451f7 authored by David Diederich's avatar David Diederich
Browse files

Merge branch 'cherry-pick-for-137' into 'release/0.17'

Cherry-pick 'Merge ibm helm to master' into release/0.17

See merge request !138
parents e931bb61 892bdb38
No related branches found
No related tags found
1 merge request!138Cherry-pick 'Merge ibm helm to master' into release/0.17
Pipeline #146008 failed
......@@ -4,6 +4,9 @@ variables:
AZURE_CLIENT_ID: ${AZURE_PRINCIPAL_ID}
AZURE_CLIENT_SECRET: ${AZURE_PRINCIPAL_SECRET}
AZURE_AD_APP_RESOURCE_ID: ${AZURE_APP_ID}
#IBM
IBM_HELM_DAG_PATH: devops/ibm/ibm-zgy-dag
#AWS
AWS_DEPLOY_TARGET: EKS
......
ibm-helm-charts-master:
stage: build
tags: ["osdu-small"]
image:
name: alpine/helm:3.6.3
entrypoint: [""]
before_script:
- helm plugin install https://github.com/chartmuseum/helm-push.git
script:
- helm cm-push ${IBM_HELM_DAG_PATH} ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/helm/stable --username gitlab-ci-token --password $CI_JOB_TOKEN
rules:
- if: '$CI_COMMIT_BRANCH == "master"'
ibm_deploy:
image: openshift/origin-cli
tags: ['osdu-medium']
......
apiVersion: v2
name: ibm-zgy-dag
description: A Helm chart for Kubernetes
# A chart can be either an 'application' or a 'library' chart.
#
# Application charts are a collection of templates that can be packaged into versioned archives
# to be deployed.
#
# Library charts provide useful utilities or functions for the chart developer. They're included as
# a dependency of application charts to inject those utilities and functions into the rendering
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.1.0
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: "1.16.0"
#!/bin/bash
ZGY_DAG="/opt/airflow/dags/segy_to_zgy_ingestion_dag.py"
chown -R 50000:root /opt/airflow/dags
chmod 775 /opt/airflow/dags
df -h
pwd
ls -ltr
echo "Printing Source Dags"
ls -ltr /tmp/scripts
cp /tmp/scripts/segy_to_zgy_ingestion_dag.py /opt/airflow/dags/.
chown -R 50000:root /opt/airflow/dags
chmod 775 /opt/airflow/dags
echo "printing target dags"
ls -ltr /opt/airflow/dags
#!/bin/bash
echo "Changing Source Dags"
sed -i '/from datetime import timedelta/ifrom json import dumps, loads' /opt/airflow/dags/segy_to_zgy_ingestion_dag.py
sed -i '/from datetime import timedelta/ifrom airflow.models import Variable' /opt/airflow/dags/segy_to_zgy_ingestion_dag.py
sed -i "/DAG_NAME = \"{| DAG_NAME |}\"/iaccess_token = \"{{ dag_run.conf['execution_context']['access_token'] }}\"" /opt/airflow/dags/segy_to_zgy_ingestion_dag.py
sed -i 's/"{| DAG_NAME |}"/"sgy-to-zgy"/g' /opt/airflow/dags/segy_to_zgy_ingestion_dag.py
sed -i 's/"{| DOCKER_IMAGE |}"/Variable.get("SGY_TO_ZGY_DOCKER_IMAGE")/g' /opt/airflow/dags/segy_to_zgy_ingestion_dag.py
sed -i 's/"{| NAMESPACE |}"/Variable.get("SGY_TO_ZGY_NAMESPACE")/g' /opt/airflow/dags/segy_to_zgy_ingestion_dag.py
sed -i 's/{| K8S_POD_OPERATOR_KWARGS |}/{}/g' /opt/airflow/dags/segy_to_zgy_ingestion_dag.py
sed -i 's/^env_vars = {/env_vars = loads(env_dict)/g' /opt/airflow/dags/segy_to_zgy_ingestion_dag.py
sed -i '/^ "STORAGE_SVC_URL/d' /opt/airflow/dags/segy_to_zgy_ingestion_dag.py
sed -i '/^ "SD_SVC_URL"/d' /opt/airflow/dags/segy_to_zgy_ingestion_dag.py
#sed -i '/^ "SD_SVC_TOKEN"/d' /opt/airflow/dags/segy_to_zgy_ingestion_dag.py
sed -i '/^ "STORAGE_SVC_API_KEY"/d' /opt/airflow/dags/segy_to_zgy_ingestion_dag.py
sed -i '/^ "SD_SVC_API_KEY"/d' /opt/airflow/dags/segy_to_zgy_ingestion_dag.py
sed -i '/^ "OSDU_DATAPARTITIONID"/d' /opt/airflow/dags/segy_to_zgy_ingestion_dag.py
sed -i '/^ "SD_READ_CACHE_PAGE_SIZE"/d' /opt/airflow/dags/segy_to_zgy_ingestion_dag.py
sed -i '/^ "SD_READ_CACHE_MAX_PAGES"/d' /opt/airflow/dags/segy_to_zgy_ingestion_dag.py
sed -i '/^ "SEGYTOZGY_VERBOSITY"/d' /opt/airflow/dags/segy_to_zgy_ingestion_dag.py
sed -i '/^ "STORAGE_SVC_TOKEN"/d' /opt/airflow/dags/segy_to_zgy_ingestion_dag.py
sed -i '/^ "SEGYTOZGY_GENERATE_INDEX"/,+1d' /opt/airflow/dags/segy_to_zgy_ingestion_dag.py
sed -i '/**K8S_POD_KWARGS/d' /opt/airflow/dags/segy_to_zgy_ingestion_dag.py
sed -i '/env_vars.update/d' /opt/airflow/dags/segy_to_zgy_ingestion_dag.py
sed -i 's/^ image=DOCKER_IMAGE,/ image=DOCKER_IMAGE/g' /opt/airflow/dags/segy_to_zgy_ingestion_dag.py
sed -i '/env_vars = loads(env_dict)/ienv_dict = Variable.get("sgy_to_zgy_env_var")' /opt/airflow/dags/segy_to_zgy_ingestion_dag.py
echo "ZGY DAGS Changed"
ls -ltr /opt/airflow/dags
\ No newline at end of file
from datetime import timedelta
from osdu_airflow.backward_compatibility.default_args import \
update_default_args
import airflow
from airflow import DAG
from airflow.contrib.operators.kubernetes_pod_operator import \
KubernetesPodOperator
from osdu_airflow.operators.update_status import UpdateStatusOperator
# default args for airflow
default_args = {
'owner': 'airflow',
'depends_on_past': False,
'start_date': airflow.utils.dates.days_ago(0),
'retries': 0,
'retry_delay': timedelta(minutes=5),
'execution_timeout': timedelta(hours=24),
'dagrun_timeout': timedelta(hours=24),
}
default_args = update_default_args(default_args)
# Get values from dag run configuration
authorization = "{{ dag_run.conf['authToken'] }}"
sd_svc_token = "{{ dag_run.conf['execution_context']['id_token'] or dag_run.conf['authToken'] }}"
dataPartitionId = "{{ dag_run.conf['execution_context']['data_partition_id'] }}"
run_id = "{{ dag_run.conf['runId'] }}"
sd_svc_api_key = "{{ dag_run.conf['execution_context']['sd_svc_api_key'] }}"
storage_svc_api_key = "{{ dag_run.conf['execution_context']['storage_svc_api_key'] }}"
filecollection_segy_id = "{{ dag_run.conf['execution_context']['filecollection_segy_id'] }}"
work_product_id = "{{ dag_run.conf['execution_context']['work_product_id'] }}"
svc_token = "{{ dag_run.conf['execution_context']['id_token'] or dag_run.conf['authToken'] }}"
# Constants
DAG_NAME = "{| DAG_NAME |}" # Replace it by the workflowName passed on workflow service API.
DOCKER_IMAGE = "{| DOCKER_IMAGE |}" # Replace it by the segy-to-zgy container
SEGY_CONVERTER = "segy-to-zgy"
DAG_INIT = "dag-init"
NAMESPACE = "{| NAMESPACE |}"
K8S_POD_KWARGS = {| K8S_POD_OPERATOR_KWARGS |}
if not K8S_POD_KWARGS:
K8S_POD_KWARGS = {}
# Values to pass to csv parser
params = ["--osdu", filecollection_segy_id, work_product_id ]
# Get environment variables
env_vars = {
"STORAGE_SVC_URL": "{| STORAGE_SVC_URL |}",
"SD_SVC_URL": "{| SD_SVC_URL |}",
"STORAGE_SVC_TOKEN": authorization,
"STORAGE_SVC_API_KEY": storage_svc_api_key,
"SD_SVC_API_KEY": sd_svc_api_key,
"OSDU_DATAPARTITIONID": dataPartitionId,
"SD_READ_CACHE_PAGE_SIZE": "4195024",
"SD_READ_CACHE_MAX_PAGES": "256",
"SEGYTOZGY_VERBOSITY": "3",
"SEGYTOZGY_GENERATE_INDEX": "1"
}
env_vars.update({| EXTRA_ENV_VARS|default('{}') |})
env_vars['SD_SVC_TOKEN'] = env_vars['STORAGE_SVC_TOKEN'] = 'Bearer ' + access_token
with DAG(
DAG_NAME,
default_args=default_args,
schedule_interval=None
) as dag:
update_status_running = UpdateStatusOperator(
task_id="update_status_running",
)
segytozgy_converter = KubernetesPodOperator(
namespace=NAMESPACE,
task_id=SEGY_CONVERTER,
name=SEGY_CONVERTER,
env_vars=env_vars,
arguments=params,
is_delete_operator_pod=True,
image=DOCKER_IMAGE,
**K8S_POD_KWARGS
)
update_status_finished = UpdateStatusOperator(
task_id="update_status_finished",
trigger_rule="all_done"
)
update_status_running >> segytozgy_converter >> update_status_finished # pylint: disable=pointless-statement
---
apiVersion: v1
kind: ConfigMap
metadata:
name: osdu-zgy-dag-config
data:
{{ (.Files.Glob "files/*").AsConfig | indent 2 }}
kind: Job
apiVersion: batch/v1
metadata:
name: osdu-zgy-dag-setup-job
labels:
app: osdu-zgy-dag-setup-job
job-name: osdu-zgy-dag-setup-job
spec:
activeDeadlineSeconds: 900
parallelism: 1
completions: 1
backoffLimit: 6
selector:
matchLabels:
template:
metadata:
name: osdu-zgy-dag-setup-job
creationTimestamp: null
labels:
app: osdu-zgy-dag-setup-job
job-name: osdu-zgy-dag-setup-job
spec:
restartPolicy: OnFailure
schedulerName: default-scheduler
terminationGracePeriodSeconds: 30
securityContext:
runAsUser: 0
containers:
- name: deploy-dags
image: >-
alpine/k8s:1.21.12
command:
- /bin/sh
- '-c'
- |
# Execute scripts "chown -R 50000:root /opt/airflow/dags; chmod 775 /opt/airflow/dags"
/tmp/scripts/copy-dags.sh
/tmp/scripts/replace-variable.sh
resources:
requests:
cpu: 5m
memory: 5Mi
volumeMounts:
- name: scripts-pvc
mountPath: /tmp/scripts
- name: dags-pvc
mountPath: /opt/airflow/dags
subPath: dags
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
imagePullPolicy: IfNotPresent
volumes:
- name: scripts-pvc
projected:
defaultMode: 0755
sources:
- configMap:
name: osdu-zgy-dag-config
- name: dags-pvc
persistentVolumeClaim:
claimName: dags-pvc
dnsPolicy: ClusterFirst
# Default values for ibm-csv-dag.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.
replicaCount: 1
image:
repository: nginx
pullPolicy: IfNotPresent
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment