Merge branch 'GONRG-5944-merge-all-helms-for-schema-service' into 'master'

Merge GC helms for schema service

See merge request !424

.gitlab-ci.yml

@@ -7,6 +7,8 @@ variables:
   GCP_TENANT_NAME: opendesevt
   GCP_DEPLOY_ENV: p4d
   GCP_DOMAIN: cloud.slb-ds.com
+  # FIXME remove when all services are migrated to a single helm
+  OSDU_GCP_ENABLE_HELM_CONFIG: "false"
 
   IBM_BUILD_SUBDIR: provider/schema-ibm
   IBM_INT_TEST_SUBDIR: testing/schema-test-core

devops/gcp/configmap/.helmignore

-# Patterns to ignore when building packages.
-# This supports shell glob matching, relative path matching, and
-# negation (prefixed with !). Only one pattern per line.
-.DS_Store
-# Common VCS dirs
-.git/
-.gitignore
-.bzr/
-.bzrignore
-.hg/
-.hgignore
-.svn/
-# Common backup files
-*.swp
-*.bak
-*.tmp
-*.orig
-*~
-# Various IDEs
-.project
-.idea/
-*.tmproj
-.vscode/

devops/gcp/configmap/Chart.yaml

-apiVersion: v2
-name: gcp-schema-configmap
-description: A Helm chart for Kubernetes
-
-# A chart can be either an 'application' or a 'library' chart.
-#
-# Application charts are a collection of templates that can be packaged into versioned archives
-# to be deployed.
-#
-# Library charts provide useful utilities or functions for the chart developer. They're included as
-# a dependency of application charts to inject those utilities and functions into the rendering
-# pipeline. Library charts do not define any templates and therefore cannot be deployed.
-type: application
-
-# This is the chart version. This version number should be incremented each time you make changes
-# to the chart and its templates, including the app version.
-# Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.0
-
-# This is the version number of the application being deployed. This version number should be
-# incremented each time you make changes to the application. Versions are not expected to
-# follow Semantic Versioning. They should reflect the version the application is using.
-# It is recommended to use it with quotes.
-appVersion: "1.16.0"

devops/gcp/configmap/README.md

-<!--- Configmap --->
-
-# Configmap helm chart
-
-This chart installs a configmap deployment on a [Kubernetes](https://kubernetes.io) cluster using [Helm](https://helm.sh) package manager.
-
-## Prerequisites
-
-The code was tested on **Kubernetes cluster** (v1.21.11) with **Istio** (1.12.6)
-> It is possible to use other versions, but it hasn't been tested
-
-### Operation system
-
-The code works in Debian-based Linux (Debian 10 and Ubuntu 20.04) and Windows WSL 2. Also, it works but is not guaranteed in Google Cloud Shell. All other operating systems, including macOS, are not verified and supported.
-
-### Packages
-
-Packages are only needed for installation from a local computer.
-
-* **HELM** (version: v3.7.1 or higher) [helm](https://helm.sh/docs/intro/install/)
-* **Kubectl** (version: v1.21.0 or higher) [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl)
-
-## Installation
-
-First you need to set variables in **values.yaml** file using any code editor. Some of the values are prefilled, but you need to specify some values as well. You can find more information about them below.
-
-### Common variables for GCP and Anthos implementation
-
-| Name | Description | Type | Default |Required |
-|------|-------------|------|---------|---------|
-**dataPartitionId** | data partition id | string | - | yes
-**entitlementsHost** | entitlements host | string | "http://entitlements" | yes
-**javaOptions** | java options | string | "-Xms512M -Xmx1024M -XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45" | yes
-**logLevel** | logging level | string | INFO | yes
-**partitionHost** | partition host | string | "http://partition" | yes
-**schemaTopicName** | topic for schema changes events | string | "schema-changed" | yes
-**springProfilesActive** | active spring profile | string | gcp | yes
-
-### GCP specific variables
-
-| Name | Description | Type | Default |Required |
-|------|-------------|------|---------|---------|
-**googleAudiences** | your GCP client ID | string | - | yes
-
-### Datastore cleanup and bootstrap schemas variables
-
-> Datastore cleanup is used for cleaning Datastore Schema Entities if they are not present in Schema bucket
-
-| Name | Description | Type | Default |Required |
-|------|-------------|------|---------|---------|
-**datastoreKind** | Datastore Kind for Schema | string | "system_schema_osm" | yes
-**datastoreNamespace** | Datastore Namespace for Schema | string | "dataecosystem" | yes
-**enableCleanup** | whether cleanup is enabled | boolean | false | yes
-**schemaBucket** | name of the bucket with schemas | string | - | yes
-**schemaHost** | schema host | string | "http://schema" | yes
-
-### Config variables
-
-| Name | Description | Type | Default |Required |
-|------|-------------|------|---------|---------|
-**appName** | name of the app | string | schema | yes
-**configmap** | configmap to be used | string | schema-config | yes
-**onPremEnabled** | whether on-prem is enabled | boolean | false | yes
-
-### Install the helm chart
-
-Run this command from within this directory:
-
-```console
-helm install gcp-schema-configmap .
-```
-
-## Uninstalling the chart
-
-To uninstall the helm deployment:
-
-```console
-helm uninstall gcp-schema-configmap
-```
-
-[Move-to-Top](#configmap-helm-chart)

devops/gcp/configmap/templates/_helpers.tpl

-{{/*
-Expand the name of the chart.
-*/}}
-{{- define "configmap.name" -}}
-{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
-{{- end }}
-
-{{/*
-Create a default fully qualified app name.
-We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
-If release name contains chart name it will be used as a full name.
-*/}}
-{{- define "configmap.fullname" -}}
-{{- if .Values.fullnameOverride }}
-{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
-{{- else }}
-{{- $name := default .Chart.Name .Values.nameOverride }}
-{{- if contains $name .Release.Name }}
-{{- .Release.Name | trunc 63 | trimSuffix "-" }}
-{{- else }}
-{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
-{{- end }}
-{{- end }}
-{{- end }}
-
-{{/*
-Create chart name and version as used by the chart label.
-*/}}
-{{- define "configmap.chart" -}}
-{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
-{{- end }}
-
-{{/*
-Common labels
-*/}}
-{{- define "configmap.labels" -}}
-helm.sh/chart: {{ include "configmap.chart" . }}
-{{ include "configmap.selectorLabels" . }}
-{{- if .Chart.AppVersion }}
-app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
-{{- end }}
-app.kubernetes.io/managed-by: {{ .Release.Service }}
-{{- end }}
-
-{{/*
-Selector labels
-*/}}
-{{- define "configmap.selectorLabels" -}}
-app.kubernetes.io/name: {{ include "configmap.name" . }}
-app.kubernetes.io/instance: {{ .Release.Name }}
-{{- end }}
-
-{{/*
-Create the name of the service account to use
-*/}}
-{{- define "configmap.serviceAccountName" -}}
-{{- if .Values.serviceAccount.create }}
-{{- default (include "configmap.fullname" .) .Values.serviceAccount.name }}
-{{- else }}
-{{- default "default" .Values.serviceAccount.name }}
-{{- end }}
-{{- end }}

devops/gcp/configmap/templates/configmap-bootstrap.yaml

-apiVersion: v1
-kind: ConfigMap
-metadata:
-  labels:
-    app: "{{ .Values.conf.appName }}-bootstrap"
-  name: "{{ .Values.conf.configmap }}-bootstrap"
-  namespace: "{{ .Release.Namespace }}"
-data:
-  DATA_PARTITION: "{{ .Values.data.dataPartitionId }}"
-  ONPREM_ENABLED: "{{ .Values.conf.onPremEnabled }}"
-  ENABLE_CLEANUP: "{{ .Values.data.enableCleanup }}"
-  SCHEMA_URL: "{{ .Values.data.schemaHost }}"
-  ENTITLEMENTS_HOST: "{{ .Values.data.entitlementsHost }}"
-  {{- if not .Values.conf.onPremEnabled }}
-  AUDIENCES: "{{ .Values.data.googleAudiences }}"
-  {{- end }}  
-  {{- if .Values.data.enableCleanup }}
-  SCHEMA_BUCKET: "{{ .Values.data.schemaBucket }}"
-  DATASTORE_NAMESPACE: "{{ .Values.data.datastoreNamespace }}"
-  DATASTORE_KIND: "{{ .Values.data.datastoreKind }}"
-  {{- end }}  

devops/gcp/configmap/templates/configmap.yaml

-apiVersion: v1
-kind: ConfigMap
-metadata:
-  labels:
-    app: "{{ .Values.conf.appName }}"
-  name: "{{ .Values.conf.configmap }}"
-  namespace: "{{ .Release.Namespace }}"
-data:
-  LOG_LEVEL: "{{ .Values.data.logLevel }}"
-  ENTITLEMENTS_HOST: "{{ .Values.data.entitlementsHost }}"
-  GCP_SCHEMA_CHANGED_TOPIC_NAME: "{{ .Values.data.schemaTopicName }}"
-  PARTITION_HOST: "{{ .Values.data.partitionHost }}"
-  SHARED_TENANT_NAME: "{{ .Values.data.dataPartitionId }}"
-  SPRING_PROFILES_ACTIVE: "{{ .Values.data.springProfilesActive }}"
-  _JAVA_OPTIONS: "{{ .Values.data.javaOptions }}"
-  {{- if not .Values.conf.onPremEnabled }}
-  GOOGLE_AUDIENCES: "{{ .Values.data.googleAudiences }}"
-  {{- end }}

devops/gcp/configmap/values.yaml

-data:
-  # common
-  dataPartitionId: ""
-  entitlementsHost: "http://entitlements"
-  javaOptions: "-Xms512M -Xmx1024M -XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45"
-  logLevel: "ERROR"
-  partitionHost: "http://partition"
-  schemaTopicName: "schema-changed"
-  springProfilesActive: "gcp"
-  # gcp
-  googleAudiences: ""
-  # Datastore cleanup and bootstrap schemas
-  datastoreKind: "system_schema_osm"
-  datastoreNamespace: "dataecosystem"
-  enableCleanup: false
-  schemaBucket: ""
-  schemaHost: "http://schema"
-conf:
-  appName: "schema"
-  configmap: "schema-config"
-  onPremEnabled: false

devops/gcp/deploy/README.md

@@ -24,10 +24,22 @@ Packages are only needed for installation from a local computer.
 
 ## Installation
 
-Before installing deploy Helm chart you need to install [configmap Helm chart](../configmap).
 First you need to set variables in **values.yaml** file using any code editor. Some of the values are prefilled, but you need to specify some values as well. You can find more information about them below.
 
-### Common variables
+### Configmap variables
+
+| Name | Description | Type | Default |Required |
+|------|-------------|------|---------|---------|
+**dataPartitionId** | data partition id | string | - | yes
+**entitlementsHost** | entitlements host | string | "http://entitlements" | yes
+**javaOptions** | java options | string | "-Xms512M -Xmx1024M -XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45" | yes
+**logLevel** | logging level | string | INFO | yes
+**partitionHost** | partition host | string | "http://partition" | yes
+**schemaTopicName** | topic for schema changes events | string | "schema-changed" | yes
+**springProfilesActive** | active spring profile | string | gcp | yes
+**googleAudiences** | your GCP client ID | string | - | yes
+
+### Deployment variables
 
 | Name | Description | Type | Default |Required |
 |------|-------------|------|---------|---------|
@@ -35,30 +47,36 @@ First you need to set variables in **values.yaml** file using any code editor. S
 **requestsMemory** | amount of requested memory| string | 1G | yes
 **limitsCpu** | CPU limit | string | 1 | yes
 **limitsMemory** | memory limit | string | 1.5G | yes
+**bootstrapImage** | bootstrap image | string | - | yes
+**bootstrapServiceAccountName** | bootstrap service account name | string | - | yes
 **image** | service image | string | - | yes
 **imagePullPolicy** | when to pull image | string | IfNotPresent | yes
 **serviceAccountName** | name of your service account | string | schema | yes
 
-### Bootstrap variables
-
-| Name | Description | Type | Default |Required |
-|------|-------------|------|---------|---------|
-**bootstrapImage** | bootstrap image | string | - | yes
-**bootstrapServiceAccountName** | bootstrap service account name | string | - | yes
-
-### Config variables
+### Configuration variables
 
 | Name | Description | Type | Default |Required |
 |------|-------------|------|---------|---------|
 **appName** | name of the app | string | `schema` | yes
-**configmap** | configmap to be used | string | `schema-config` | yes
 **bootstrapSecretName** | secret for bootstrap | string | `datafier-secret` | yes
+**configmap** | configmap to be used | string | `schema-config` | yes
 **domain** | your domain | string | - | yes
 **minioSecretName** | secret for minio | string | `schema-minio-secret` | yes
 **onPremEnabled** | whether on-prem is enabled | boolean | false | yes
 **postgresSecretName** | secret for postgres | string | `schema-postgres-secret` | yes
 **rabbitmqSecretName** | secret for rabbitmq | string | `rabbitmq-secret` | yes
 
+### Datastore cleanup and bootstrap schemas variables
+> Datastore cleanup is used for cleaning Datastore Schema Entities if they are not present in Schema bucket
+
+| Name | Description | Type | Default |Required |
+|------|-------------|------|---------|---------|
+**datastoreKind** | Datastore Kind for Schema | string | "system_schema_osm" | yes
+**datastoreNamespace** | Datastore Namespace for Schema | string | "dataecosystem" | yes
+**enableCleanup** | whether cleanup is enabled | boolean | false | yes
+**schemaBucket** | name of the bucket with schemas | string | - | yes
+**schemaHost** | schema host | string | "http://schema" | yes
+
 ### Install the helm chart
 
 Run this command from within this directory:

devops/gcp/deploy/templates/configmap-bootstrap.yaml

+apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    app: {{ printf "%s-bootstrap" .Values.conf.appName | quote }}
+  name: {{ printf "%s-bootstrap" .Values.conf.configmap | quote }}
+  namespace: {{ .Release.Namespace | quote }}
+data:
+  DATA_PARTITION: {{ .Values.data.dataPartitionId | quote }}
+  ONPREM_ENABLED: {{ .Values.conf.onPremEnabled | quote }}
+  ENABLE_CLEANUP: {{ .Values.data.enableCleanup | quote }}
+  SCHEMA_URL: {{ .Values.data.schemaHost | quote }}
+  ENTITLEMENTS_HOST: {{ .Values.data.entitlementsHost | quote }}
+  {{- if not .Values.conf.onPremEnabled }}
+  AUDIENCES: {{ .Values.data.googleAudiences | quote }}
+  {{- end }}  
+  {{- if .Values.data.enableCleanup }}
+  SCHEMA_BUCKET: {{ .Values.data.schemaBucket | quote }}
+  DATASTORE_NAMESPACE: {{ .Values.data.datastoreNamespace | quote }}
+  DATASTORE_KIND: {{ .Values.data.datastoreKind | quote }}
+  {{- end }}  

devops/gcp/deploy/templates/configmap.yaml

+apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    app: {{ .Values.conf.appName | quote }}
+  name: {{ .Values.conf.configmap | quote }}
+  namespace: {{ .Release.Namespace | quote }}
+data:
+  LOG_LEVEL: {{ .Values.data.logLevel | quote }}
+  ENTITLEMENTS_HOST: {{ .Values.data.entitlementsHost | quote }}
+  GCP_SCHEMA_CHANGED_TOPIC_NAME: {{ .Values.data.schemaTopicName | quote }}
+  PARTITION_HOST: {{ .Values.data.partitionHost | quote }}
+  SHARED_TENANT_NAME: {{ .Values.data.dataPartitionId | quote }}
+  SPRING_PROFILES_ACTIVE: {{ .Values.data.springProfilesActive | quote }}
+  _JAVA_OPTIONS: {{ .Values.data.javaOptions | quote }}
+  {{- if not .Values.conf.onPremEnabled }}
+  GOOGLE_AUDIENCES: {{ .Values.data.googleAudiences | quote }}
+  {{- end }}

devops/gcp/deploy/templates/deployment-bootstrap.yaml

@@ -2,35 +2,35 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   labels:
-    app: "{{ .Values.conf.appName }}-bootstrap"
-  name: "{{ .Values.conf.appName }}-bootstrap"
-  namespace: "{{ .Release.Namespace }}"
+    app: {{ printf "%s-bootstrap" .Values.conf.appName | quote }}
+  name: {{ printf "%s-bootstrap" .Values.conf.appName | quote }}
+  namespace: {{ .Release.Namespace | quote }}
 spec:
   replicas: 1
   selector:
     matchLabels:
-      app: "{{ .Values.conf.appName }}-bootstrap"
+      app: {{ printf "%s-bootstrap" .Values.conf.appName | quote }}
   template:
     metadata:
       labels:
-        app: "{{ .Values.conf.appName }}-bootstrap"
+        app: {{ printf "%s-bootstrap" .Values.conf.appName | quote }}
       annotations:
         rollme: {{ randAlphaNum 5 | quote }}
     spec:
       containers:
-        - name: "{{ .Values.conf.appName }}-bootstrap"
-          image: "{{ .Values.data.bootstrapImage }}"
+        - name: {{ printf "%s-bootstrap" .Values.conf.appName | quote }}
+          image: {{ .Values.data.bootstrapImage | quote }}
           readinessProbe:
             exec:
               command:
               - cat
               - /tmp/bootstrap_ready
-          imagePullPolicy: "{{ .Values.data.imagePullPolicy }}"
+          imagePullPolicy: {{ .Values.data.imagePullPolicy | quote }}
           envFrom:
           - configMapRef:
-              name: "{{ .Values.conf.configmap }}-bootstrap"
+              name: {{ printf "%s-bootstrap" .Values.conf.configmap | quote }}
           {{- if .Values.conf.onPremEnabled }}
           - secretRef:
-              name: "{{ .Values.conf.bootstrapSecretName }}"
+              name: {{ .Values.conf.bootstrapSecretName | quote }}
           {{- end }}
-      serviceAccountName: "{{ .Values.data.bootstrapServiceAccountName }}"
+      serviceAccountName: {{ .Values.data.bootstrapServiceAccountName | quote }}

devops/gcp/deploy/templates/deployment.yaml

@@ -2,35 +2,35 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   labels:
-    app: "{{ .Values.conf.appName }}"
-  name: "{{ .Values.conf.appName }}"
-  namespace: "{{ .Release.Namespace }}"
+    app: {{ .Values.conf.appName | quote }}
+  name: {{ .Values.conf.appName | quote }}
+  namespace: {{ .Release.Namespace | quote }}
 spec:
   selector:
     matchLabels:
-      app: "{{ .Values.conf.appName }}"
+      app: {{ .Values.conf.appName | quote }}
   replicas: 1
   template:
     metadata:
       labels:
-        app: "{{ .Values.conf.appName }}"
+        app: {{ .Values.conf.appName | quote }}
       annotations:
         rollme: {{ randAlphaNum 5 | quote }}
     spec:
       containers:
-        - name: "{{ .Values.conf.appName }}"
-          image: "{{ .Values.data.image }}"
-          imagePullPolicy: "{{ .Values.data.imagePullPolicy }}"
+        - name: {{ .Values.conf.appName | quote }}
+          image: {{ .Values.data.image | quote }}
+          imagePullPolicy: {{ .Values.data.imagePullPolicy | quote }}
           envFrom:
           - configMapRef:
-              name: "{{ .Values.conf.configmap }}"
+              name: {{ .Values.conf.configmap | quote }}
           {{- if .Values.conf.onPremEnabled }}
           - secretRef:
-              name: "{{ .Values.conf.minioSecretName }}"
+              name: {{ .Values.conf.minioSecretName | quote }}
           - secretRef:
-              name: "{{ .Values.conf.postgresSecretName }}"
+              name: {{ .Values.conf.postgresSecretName | quote }}
           - secretRef:
-              name: "{{ .Values.conf.rabbitmqSecretName }}"
+              name: {{ .Values.conf.rabbitmqSecretName | quote }}
           {{- end }}
           securityContext:
             allowPrivilegeEscalation: false
@@ -39,9 +39,9 @@ spec:
             - containerPort: 8080
           resources:
             requests:
-              cpu: "{{ .Values.data.requestsCpu }}"
-              memory: "{{ .Values.data.requestsMemory }}"
+              cpu: {{ .Values.data.requestsCpu | quote }}
+              memory: {{ .Values.data.requestsMemory | quote }}
             limits:
-              cpu: "{{ .Values.data.limitsCpu }}"
-              memory: "{{ .Values.data.limitsMemory }}"
-      serviceAccountName: "{{ .Values.data.serviceAccountName }}"
+              cpu: {{ .Values.data.limitsCpu | quote }}
+              memory: {{ .Values.data.limitsMemory | quote }}
+      serviceAccountName: {{ .Values.data.serviceAccountName | quote }}

devops/gcp/deploy/templates/service-account.yaml

@@ -2,6 +2,6 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: "{{ .Values.data.serviceAccountName }}"
-  namespace: "{{ .Release.Namespace }}"
+  name: {{ .Values.data.serviceAccountName | quote }}
+  namespace: {{ .Release.Namespace | quote }}
 {{- end }}

devops/gcp/deploy/templates/service.yaml

 apiVersion: v1
 kind: Service
 metadata:
-  name: "{{ .Values.conf.appName }}"
+  name: {{ .Values.conf.appName | quote }}
   annotations:
     {{- if not .Values.conf.onPremEnabled }}
     cloud.google.com/neg: '{"ingress": true}'
     {{- end }}
-  namespace: "{{ .Release.Namespace }}"
+  namespace: {{ .Release.Namespace | quote }}
   labels:
-    app: "{{ .Values.conf.appName }}"
+    app: {{ .Values.conf.appName | quote }}
 spec:
   ports:
     - protocol: TCP
@@ -16,4 +16,4 @@ spec:
       targetPort: 8080
       name: http
   selector:
-    app: "{{ .Values.conf.appName }}"
+    app: {{ .Values.conf.appName | quote }}

devops/gcp/deploy/templates/virtual-service.yaml

 apiVersion: networking.istio.io/v1alpha3
 kind: VirtualService
 metadata:
-  name: "{{ .Values.conf.appName }}"
-  namespace: "{{ .Release.Namespace }}"
+  name: {{ .Values.conf.appName | quote }}
+  namespace: {{ .Release.Namespace | quote }}
 spec:
   hosts:
     {{- if and .Values.conf.domain .Values.conf.onPremEnabled }}

devops/gcp/deploy/values.yaml

 data:
+  #Configmaps
+  dataPartitionId: ""
+  entitlementsHost: "http://entitlements"
+  javaOptions: "-Xms512M -Xmx1024M -XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45"
+  logLevel: "ERROR"
+  partitionHost: "http://partition"
+  schemaTopicName: "schema-changed"
+  springProfilesActive: "gcp"
+  googleAudiences: ""
+  datastoreKind: "system_schema_osm"
+  datastoreNamespace: "dataecosystem"
+  enableCleanup: false
+  schemaBucket: ""
+  schemaHost: "http://schema"
+  #Deployments
   requestsCpu: "0.1"
   requestsMemory: "1G"
   limitsCpu: "1"