Merge ibm helm to master

.gitlab-ci.yml

@@ -15,6 +15,7 @@
 #  limitations under the License.
 
 variables:
+  IBM_HELM_DAG_PATH: devops/ibm/ibm-vds-dag
   AZURE_SERVICE: segy-to-vds-conversion
   AZURE_DEPLOYMENTS_SUBDIR: deployments/scripts/azure
   AZURE_CLIENT_ID: ${AZURE_PRINCIPAL_ID}

devops/ibm/ibm-stages.yml

@@ -17,6 +17,19 @@ ibm_deploy:
     - oc cp src/dags/segy_to_vds_ssdms_conversion_dag_ibm.py $airflowWebPod:/opt/airflow/dags/repo -c airflow-web
 
 
+ibm-helm-charts-master:
+  stage: build
+  tags: ["osdu-small"]
+  image:
+    name: alpine/helm:3.6.3
+    entrypoint: [""]
+  before_script:
+    - cp src/dags/segy_to_vds_ssdms_conversion_dag.py devops/ibm/ibm-vds-dag/files/.
+    - helm plugin install https://github.com/chartmuseum/helm-push.git
+  script:
+    - helm cm-push ${IBM_HELM_DAG_PATH} ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/helm/stable --username gitlab-ci-token --password $CI_JOB_TOKEN  
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "master"'
 
 ibm_bootstrap:
   image: openshift/origin-cli

devops/ibm/ibm-vds-dag/Chart.yaml

+apiVersion: v2
+name: ibm-vds-dag
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.16.0"
+
+

devops/ibm/ibm-vds-dag/files/copy-dags.sh

+#!/bin/bash
+VDS_DAG="/opt/airflow/dags/segy_to_vds_ssdms_conversion_dag.py"
+chown -R 50000:root /opt/airflow/dags
+chmod 775 /opt/airflow/dags
+df -h
+pwd
+ls -ltr
+echo "Printing Source Dags"
+ls -ltr /tmp/scripts
+cp /tmp/scripts/segy_to_vds_ssdms_conversion_dag.py /opt/airflow/dags/.
+chown -R 50000:root /opt/airflow/dags
+chmod 775 /opt/airflow/dags
+echo "printing target dags"
+ls -ltr /opt/airflow/dags

devops/ibm/ibm-vds-dag/files/replace-values.sh

+#!/bin/bash
+echo "Changing Source Dags"
+sed -i '/from datetime import timedelta/ifrom json import dumps, loads' /opt/airflow/dags/segy_to_vds_ssdms_conversion_dag.py
+sed -i '/from datetime import timedelta/ifrom airflow.models import Variable' /opt/airflow/dags/segy_to_vds_ssdms_conversion_dag.py
+sed -i 's/"{| DAG_NAME |}"/"openvds_import"/g' /opt/airflow/dags/segy_to_vds_ssdms_conversion_dag.py
+sed -i 's/"{| DOCKER_IMAGE |}"/Variable.get("SGY_TO_VDS_DOCKER_IMAGE")/g' /opt/airflow/dags/segy_to_vds_ssdms_conversion_dag.py
+sed -i 's/"{| K8S_NAMESPACE |}"/Variable.get("SGY_TO_VDS_NAMESPACE")/g' /opt/airflow/dags/segy_to_vds_ssdms_conversion_dag.py
+sed -i 's/{| K8S_POD_KWARGS |}/{}/g' /opt/airflow/dags/segy_to_vds_ssdms_conversion_dag.py
+sed -i '/seismic_store_url/d' /opt/airflow/dags/segy_to_vds_ssdms_conversion_dag.py  
+sed -i 's/^env_vars = {/env_vars = loads(env_dict)/g' /opt/airflow/dags/segy_to_vds_ssdms_conversion_dag.py
+sed -i '/env_vars = loads(env_dict)/ienv_dict = Variable.get("sgy_to_vds_env_var")' /opt/airflow/dags/segy_to_vds_ssdms_conversion_dag.py
+sed -i '/^    "SD_SVC_API_KEY": "test",/,+1d' /opt/airflow/dags/segy_to_vds_ssdms_conversion_dag.py
+sed -i '/env_vars.update/d' /opt/airflow/dags/segy_to_vds_ssdms_conversion_dag.py  
+sed  -i 's/^        trigger_rule="none_failed_or_skipped",/        trigger_rule="none_failed_or_skipped"/g' /opt/airflow/dags/segy_to_vds_ssdms_conversion_dag.py
+sed -i '/**K8S_POD_KWARGS/d' /opt/airflow/dags/segy_to_vds_ssdms_conversion_dag.py  
+echo "VDS DAGS Changed"
+ls -ltr /opt/airflow/dags

devops/ibm/ibm-vds-dag/templates/dag-config.yaml

+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: osdu-vds-dag-config
+data:
+{{ (.Files.Glob "files/*").AsConfig | indent 2 }}

devops/ibm/ibm-vds-dag/templates/dag-deploy-job.yaml

+kind: Job
+apiVersion: batch/v1
+metadata:
+  name: osdu-vds-dag-setup-job
+  labels:
+    app: osdu-vds-dag-setup-job
+    job-name: osdu-vds-dag-setup-job
+spec:
+  activeDeadlineSeconds: 900
+  parallelism: 1
+  completions: 1
+  backoffLimit: 6
+  selector:
+    matchLabels:
+  template:
+    metadata:
+      name: osdu-vds-dag-setup-job
+      creationTimestamp: null
+      labels:
+        app: osdu-vds-dag-setup-job
+        job-name: osdu-vds-dag-setup-job
+    spec:
+      restartPolicy: OnFailure
+      schedulerName: default-scheduler
+      terminationGracePeriodSeconds: 30
+      securityContext:
+        runAsUser: 0
+      containers:
+        - name: deploy-dags
+          image: >-
+            alpine/k8s:1.21.12
+          command:
+            - /bin/sh
+            - '-c'
+            - |
+              # Execute scripts "chown -R 50000:root /opt/airflow/dags; chmod 775 /opt/airflow/dags"
+              /tmp/scripts/copy-dags.sh
+              /tmp/scripts/replace-values.sh
+          resources:
+            requests:
+              cpu: 5m
+              memory: 5Mi
+          volumeMounts:
+            - name: scripts-pvc
+              mountPath: /tmp/scripts
+            - name: dags-pvc
+              mountPath: /opt/airflow/dags
+              subPath: dags
+          terminationMessagePath: /dev/termination-log
+          terminationMessagePolicy: File
+          imagePullPolicy: IfNotPresent
+      volumes:
+        - name: scripts-pvc
+          projected:
+            defaultMode: 0755
+            sources:
+              - configMap:
+                  name: osdu-vds-dag-config
+        - name: dags-pvc
+          persistentVolumeClaim:
+            claimName: dags-pvc
+      dnsPolicy: ClusterFirst

devops/ibm/ibm-vds-dag/values.yaml

+# Default values for ibm-vds-dag.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+replicaCount: 1
+
+image:
+  repository: nginx
+  pullPolicy: IfNotPresent