diff --git a/devops/gcp/bootstrap-osdu-module/bootstrap_schema.sh b/devops/gcp/bootstrap-osdu-module/bootstrap_schema.sh index 9d26dcc3f2b47bb8dd566cd3c693103d7d887ff6..005670bed772b22c4b4322705c4da8719bcb5d84 100644 --- a/devops/gcp/bootstrap-osdu-module/bootstrap_schema.sh +++ b/devops/gcp/bootstrap-osdu-module/bootstrap_schema.sh @@ -14,6 +14,11 @@ # - OPENID_PROVIDER_URL # - OPENID_PROVIDER_CLIENT_ID # - OPENID_PROVIDER_CLIENT_SECRET +# (with datastore cleanup) +# - SCHEMA_BUCKET +# - DATASTORE_NAMESPACE +# - DATASTORE_KIND +# - ENABLE_CLEANUP # set -e @@ -31,20 +36,21 @@ bootstrap_schema_gettoken_onprem() { --data-urlencode "scope=openid" \ --data-urlencode "client_id=${OPENID_PROVIDER_CLIENT_ID}" \ --data-urlencode "client_secret=${OPENID_PROVIDER_CLIENT_SECRET}" | jq -r ".id_token")" + export BEARER_TOKEN="Bearer ${ID_TOKEN}" } bootstrap_schema_gettoken_gcp() { BEARER_TOKEN=$(gcloud auth print-identity-token --audiences="${AUDIENCES}") - export BEARER_TOKEN + export BEARER_TOKEN } bootstrap_schema_prechek_env() { - status_code=$(curl --retry 1 --location -globoff --request GET \ - "${ENTITLEMENTS_HOST}/api/entitlements/v2/groups" \ - --write-out "%{http_code}" --silent --output "/dev/null"\ + + status_code=$(curl --retry 1 --location -globoff --request GET "${ENTITLEMENTS_HOST}/api/entitlements/v2/groups" \ + --write-out "%{http_code}" --silent --output "/dev/null" \ --header 'Content-Type: application/json' \ --header "data-partition-id: ${DATA_PARTITION}" \ --header "Authorization: ${BEARER_TOKEN}") @@ -62,7 +68,6 @@ bootstrap_schema_deploy_shared_schemas() { python3 ./scripts/DeploySharedSchemas.py -u "${SCHEMA_URL}"/api/schema-service/v1/schemas/system } - if [ "${ONPREM_ENABLED}" == "true" ] then source ./validate-env.sh "OPENID_PROVIDER_URL" @@ -91,7 +96,6 @@ else fi # Precheck entitlements - bootstrap_schema_prechek_env # Deploy shared schemas diff --git a/devops/gcp/configmap/README.md b/devops/gcp/configmap/README.md new file mode 100644 index 0000000000000000000000000000000000000000..c46a24c1cf979b38f2f7f988e76248b057f74d4f --- /dev/null +++ b/devops/gcp/configmap/README.md @@ -0,0 +1,81 @@ +<!--- Configmap ---> + +# Configmap helm chart + +This chart installs a configmap deployment on a [Kubernetes](https://kubernetes.io) cluster using [Helm](https://helm.sh) package manager. + +## Prerequisites + +The code was tested on **Kubernetes cluster** (v1.21.11) with **Istio** (1.12.6) +> It is possible to use other versions, but it hasn't been tested + +### Operation system + +The code works in Debian-based Linux (Debian 10 and Ubuntu 20.04) and Windows WSL 2. Also, it works but is not guaranteed in Google Cloud Shell. All other operating systems, including macOS, are not verified and supported. + +### Packages + +Packages are only needed for installation from a local computer. + +* **HELM** (version: v3.7.1 or higher) [helm](https://helm.sh/docs/intro/install/) +* **Kubectl** (version: v1.21.0 or higher) [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) + +## Installation + +First you need to set variables in **values.yaml** file using any code editor. Some of the values are prefilled, but you need to specify some values as well. You can find more information about them below. + +### Common variables for GCP and Anthos implementation + +| Name | Description | Type | Default |Required | +|------|-------------|------|---------|---------| +**dataPartitionId** | data partition id | string | - | yes +**entitlementsHost** | entitlements host | string | "http://entitlements" | yes +**javaOptions** | java options | string | "-Xms512M -Xmx1024M -XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45" | yes +**logLevel** | logging level | string | INFO | yes +**partitionHost** | partition host | string | "http://partition" | yes +**schemaTopicName** | topic for schema changes events | string | "schema-changed" | yes +**springProfilesActive** | active spring profile | string | gcp | yes + +### GCP specific variables + +| Name | Description | Type | Default |Required | +|------|-------------|------|---------|---------| +**googleAudiences** | your GCP client ID | string | - | yes + +### Datastore cleanup and bootstrap schemas variables + +> Datastore cleanup is used for cleaning Datastore Schema Entities if they are not present in Schema bucket + +| Name | Description | Type | Default |Required | +|------|-------------|------|---------|---------| +**datastoreKind** | Datastore Kind for Schema | string | "system_schema_osm" | yes +**datastoreNamespace** | Datastore Namespace for Schema | string | "dataecosystem" | yes +**enableCleanup** | whether cleanup is enabled | boolean | false | yes +**schemaBucket** | name of the bucket with schemas | string | - | yes +**schemaHost** | schema host | string | "http://schema" | yes + +### Config variables + +| Name | Description | Type | Default |Required | +|------|-------------|------|---------|---------| +**appName** | name of the app | string | schema | yes +**configmap** | configmap to be used | string | schema-config | yes +**onPremEnabled** | whether on-prem is enabled | boolean | false | yes + +### Install the helm chart + +Run this command from within this directory: + +```console +helm install gcp-schema-configmap . +``` + +## Uninstalling the chart + +To uninstall the helm deployment: + +```console +helm uninstall gcp-schema-configmap +``` + +[Move-to-Top](#configmap-helm-chart) diff --git a/devops/gcp/configmap/templates/configmap-bootstrap.yaml b/devops/gcp/configmap/templates/configmap-bootstrap.yaml index 30197741eba921c0be87fe2918df17f962d57e43..487e00c33e48ee9a4e7fc3c4b849d78e7177c88a 100644 --- a/devops/gcp/configmap/templates/configmap-bootstrap.yaml +++ b/devops/gcp/configmap/templates/configmap-bootstrap.yaml @@ -2,20 +2,20 @@ apiVersion: v1 kind: ConfigMap metadata: labels: - app: "{{ .Values.conf.app_name }}-bootstrap" + app: "{{ .Values.conf.appName }}-bootstrap" name: "{{ .Values.conf.configmap }}-bootstrap" namespace: "{{ .Release.Namespace }}" data: - DATA_PARTITION: "{{ .Values.data.data_partition_id }}" - ONPREM_ENABLED: "{{ .Values.conf.on_prem_enabled }}" - ENABLE_CLEANUP: "{{ .Values.data.enable_cleanup }}" - SCHEMA_URL: "{{ .Values.data.schema_host }}" - ENTITLEMENTS_HOST: "{{ .Values.data.entitlements_host }}" - {{- if not .Values.conf.on_prem_enabled }} - AUDIENCES: "{{ .Values.data.google_audiences }}" + DATA_PARTITION: "{{ .Values.data.dataPartitionId }}" + ONPREM_ENABLED: "{{ .Values.conf.onPremEnabled }}" + ENABLE_CLEANUP: "{{ .Values.data.enableCleanup }}" + SCHEMA_URL: "{{ .Values.data.schemaHost }}" + ENTITLEMENTS_HOST: "{{ .Values.data.entitlementsHost }}" + {{- if not .Values.conf.onPremEnabled }} + AUDIENCES: "{{ .Values.data.googleAudiences }}" {{- end }} - {{- if .Values.data.enable_cleanup }} - SCHEMA_BUCKET: "{{ .Values.data.schema_bucket }}" - DATASTORE_NAMESPACE: "{{ .Values.data.datastore_namespace }}" - DATASTORE_KIND: "{{ .Values.data.datastore_kind }}" + {{- if .Values.data.enableCleanup }} + SCHEMA_BUCKET: "{{ .Values.data.schemaBucket }}" + DATASTORE_NAMESPACE: "{{ .Values.data.datastoreNamespace }}" + DATASTORE_KIND: "{{ .Values.data.datastoreKind }}" {{- end }} diff --git a/devops/gcp/configmap/templates/configmap.yaml b/devops/gcp/configmap/templates/configmap.yaml index bd9295bfe87a7f3b931183698887285603e51258..0eed468521669f7cae425e643b6b2f2ca55abce8 100644 --- a/devops/gcp/configmap/templates/configmap.yaml +++ b/devops/gcp/configmap/templates/configmap.yaml @@ -2,17 +2,17 @@ apiVersion: v1 kind: ConfigMap metadata: labels: - app: "{{ .Values.conf.app_name }}" + app: "{{ .Values.conf.appName }}" name: "{{ .Values.conf.configmap }}" namespace: "{{ .Release.Namespace }}" data: - LOG_LEVEL: "{{ .Values.data.log_level }}" - ENTITLEMENTS_HOST: "{{ .Values.data.entitlements_host }}" - GCP_SCHEMA_CHANGED_TOPIC_NAME: "{{ .Values.data.gcp_schema_changed_topic_name }}" - PARTITION_HOST: "{{ .Values.data.partition_host }}" - SHARED_TENANT_NAME: "{{ .Values.data.shared_tenant_name }}" - SPRING_PROFILES_ACTIVE: "{{ .Values.data.spring_profiles_active }}" - _JAVA_OPTIONS: "{{ .Values.data.java_options }}" - {{- if not .Values.conf.on_prem_enabled }} - GOOGLE_AUDIENCES: "{{ .Values.data.google_audiences }}" + LOG_LEVEL: "{{ .Values.data.logLevel }}" + ENTITLEMENTS_HOST: "{{ .Values.data.entitlementsHost }}" + GCP_SCHEMA_CHANGED_TOPIC_NAME: "{{ .Values.data.schemaTopicName }}" + PARTITION_HOST: "{{ .Values.data.partitionHost }}" + SHARED_TENANT_NAME: "{{ .Values.data.dataPartitionId }}" + SPRING_PROFILES_ACTIVE: "{{ .Values.data.springProfilesActive }}" + _JAVA_OPTIONS: "{{ .Values.data.javaOptions }}" + {{- if not .Values.conf.onPremEnabled }} + GOOGLE_AUDIENCES: "{{ .Values.data.googleAudiences }}" {{- end }} diff --git a/devops/gcp/configmap/values.yaml b/devops/gcp/configmap/values.yaml index e4720561d2d760cf03236037bac3b14a7907eee1..a6aee707dc6fd1858ea4278f6a5d0302dd7f9f5d 100644 --- a/devops/gcp/configmap/values.yaml +++ b/devops/gcp/configmap/values.yaml @@ -1,23 +1,21 @@ data: # common - entitlements_host: "http://entitlements" - gcp_schema_changed_topic_name: "schema-changed" - log_level: "INFO" - partition_host: "http://partition" - shared_tenant_name: "" - spring_profiles_active: "gcp" + dataPartitionId: "" + entitlementsHost: "http://entitlements" + javaOptions: "-Xms512M -Xmx1024M -XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45" + logLevel: "INFO" + partitionHost: "http://partition" + schemaTopicName: "schema-changed" + springProfilesActive: "gcp" # gcp - google_audiences: "" - # bootstrap - enable_cleanup: false - data_partition_id: "" - schema_host: "http://schema" - # cleanup - schema_bucket: "" - datastore_namespace: "dataecosystem" - datastore_kind: "system_schema_osm" - java_options: "-Xms512M -Xmx1024M -XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45" + googleAudiences: "" + # Datastore cleanup and bootstrap schemas + datastoreKind: "system_schema_osm" + datastoreNamespace: "dataecosystem" + enableCleanup: false + schemaBucket: "" + schemaHost: "http://schema" conf: - app_name: "schema" + appName: "schema" configmap: "schema-config" - on_prem_enabled: false + onPremEnabled: false diff --git a/devops/gcp/deploy/README.md b/devops/gcp/deploy/README.md new file mode 100644 index 0000000000000000000000000000000000000000..5c378ba1ece957eeb79c4a00a2bbc5e8913658fa --- /dev/null +++ b/devops/gcp/deploy/README.md @@ -0,0 +1,80 @@ +<!--- Deploy ---> + +# Deploy helm chart + +## Introduction + +This chart installs a deployment on a [Kubernetes](https://kubernetes.io) cluster using [Helm](https://helm.sh) package manager. + +## Prerequisites + +The code was tested on **Kubernetes cluster** (v1.21.11) with **Istio** (1.12.6) +> It is possible to use other versions, but it hasn't been tested + +### Operation system + +The code works in Debian-based Linux (Debian 10 and Ubuntu 20.04) and Windows WSL 2. Also, it works but is not guaranteed in Google Cloud Shell. All other operating systems, including macOS, are not verified and supported. + +### Packages + +Packages are only needed for installation from a local computer. + +* **HELM** (version: v3.7.1 or higher) [helm](https://helm.sh/docs/intro/install/) +* **Kubectl** (version: v1.21.0 or higher) [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) + +## Installation + +Before installing deploy Helm chart you need to install [configmap Helm chart](../configmap). +First you need to set variables in **values.yaml** file using any code editor. Some of the values are prefilled, but you need to specify some values as well. You can find more information about them below. + +### Common variables + +| Name | Description | Type | Default |Required | +|------|-------------|------|---------|---------| +**requestsCpu** | amount of requested CPU | string | 0.1 | yes +**requestsMemory** | amount of requested memory| string | 1G | yes +**limitsCpu** | CPU limit | string | 1 | yes +**limitsMemory** | memory limit | string | 1.5G | yes +**image** | service image | string | - | yes +**imagePullPolicy** | when to pull image | string | IfNotPresent | yes +**serviceAccountName** | name of your service account | string | schema | yes + +### Bootstrap variables + +| Name | Description | Type | Default |Required | +|------|-------------|------|---------|---------| +**bootstrapLimitsCpu** | CPU limit | string | 0.2 | yes +**bootstrapLimitsMemory** | memory limit | string | 200M | yes +**bootstrapImage** | bootstrap image | string | - | yes +**bootstrapServiceAccountName** | bootstrap service account name | string | - | yes + +### Config variables + +| Name | Description | Type | Default |Required | +|------|-------------|------|---------|---------| +**appName** | name of the app | string | `schema` | yes +**configmap** | configmap to be used | string | `schema-config` | yes +**bootstrapSecretName** | secret for bootstrap | string | `datafier-secret` | yes +**domain** | your domain | string | - | yes +**minioSecretName** | secret for minio | string | `schema-minio-secret` | yes +**onPremEnabled** | whether on-prem is enabled | boolean | false | yes +**postgresSecretName** | secret for postgres | string | `schema-postgres-secret` | yes +**rabbitmqSecretName** | secret for rabbitmq | string | `rabbitmq-secret` | yes + +### Install the helm chart + +Run this command from within this directory: + +```console +helm install gcp-schema-deploy . +``` + +## Uninstalling the Chart + +To uninstall the helm deployment: + +```console +helm uninstall gcp-schema-deploy +``` + +[Move-to-Top](#deploy-helm-chart) diff --git a/devops/gcp/deploy/templates/deployment-bootstrap.yaml b/devops/gcp/deploy/templates/deployment-bootstrap.yaml index e95a443d8d1f84d056c493284e98769698382ea2..a9fbbb6e64d96a39bcda7135c1fcd3e702bf2e60 100644 --- a/devops/gcp/deploy/templates/deployment-bootstrap.yaml +++ b/devops/gcp/deploy/templates/deployment-bootstrap.yaml @@ -2,24 +2,24 @@ apiVersion: apps/v1 kind: Deployment metadata: labels: - app: "{{ .Values.conf.app_name }}-bootstrap" - name: "{{ .Values.conf.app_name }}-bootstrap" + app: "{{ .Values.conf.appName }}-bootstrap" + name: "{{ .Values.conf.appName }}-bootstrap" namespace: "{{ .Release.Namespace }}" spec: replicas: 1 selector: matchLabels: - app: "{{ .Values.conf.app_name }}-bootstrap" + app: "{{ .Values.conf.appName }}-bootstrap" template: metadata: labels: - app: "{{ .Values.conf.app_name }}-bootstrap" + app: "{{ .Values.conf.appName }}-bootstrap" annotations: rollme: {{ randAlphaNum 5 | quote }} spec: containers: - - name: "{{ .Values.conf.app_name }}-bootstrap" - image: "{{ .Values.data.bootstrap_image }}" + - name: "{{ .Values.conf.appName }}-bootstrap" + image: "{{ .Values.data.bootstrapImage }}" readinessProbe: exec: command: @@ -29,12 +29,12 @@ spec: envFrom: - configMapRef: name: "{{ .Values.conf.configmap }}-bootstrap" - {{- if .Values.conf.on_prem_enabled }} + {{- if .Values.conf.onPremEnabled }} - secretRef: - name: "{{ .Values.conf.bootstrap_secret_name }}" + name: "{{ .Values.conf.bootstrapSecretName }}" {{- end }} resources: limits: - cpu: "{{ .Values.data.bootstrap_limits_cpu}}" - memory: "{{ .Values.data.bootstrap_limits_memory }}" + cpu: "{{ .Values.data.bootstrapLimitsCpu}}" + memory: "{{ .Values.data.bootstrapLimitsMemory }}" serviceAccountName: "{{ .Values.data.bootstrapServiceAccountName }}" diff --git a/devops/gcp/deploy/templates/deployment.yaml b/devops/gcp/deploy/templates/deployment.yaml index 0f4b50bcafdda1c6aba4453bbc6e765187c8f19a..e1c9ba03c51e4e52942bb36dd266bda45d9b081b 100644 --- a/devops/gcp/deploy/templates/deployment.yaml +++ b/devops/gcp/deploy/templates/deployment.yaml @@ -2,35 +2,35 @@ apiVersion: apps/v1 kind: Deployment metadata: labels: - app: "{{ .Values.conf.app_name }}" - name: "{{ .Values.conf.app_name }}" + app: "{{ .Values.conf.appName }}" + name: "{{ .Values.conf.appName }}" namespace: "{{ .Release.Namespace }}" spec: selector: matchLabels: - app: "{{ .Values.conf.app_name }}" + app: "{{ .Values.conf.appName }}" replicas: 1 template: metadata: labels: - app: "{{ .Values.conf.app_name }}" + app: "{{ .Values.conf.appName }}" annotations: rollme: {{ randAlphaNum 5 | quote }} spec: containers: - - name: "{{ .Values.conf.app_name }}" + - name: "{{ .Values.conf.appName }}" image: "{{ .Values.data.image }}" imagePullPolicy: "{{ .Values.data.imagePullPolicy }}" envFrom: - configMapRef: name: "{{ .Values.conf.configmap }}" - {{- if .Values.conf.on_prem_enabled }} + {{- if .Values.conf.onPremEnabled }} - secretRef: - name: "{{ .Values.conf.minio_secret_name }}" + name: "{{ .Values.conf.minioSecretName }}" - secretRef: - name: "{{ .Values.conf.postgres_secret_name }}" + name: "{{ .Values.conf.postgresSecretName }}" - secretRef: - name: "{{ .Values.conf.rabbitmq_secret_name }}" + name: "{{ .Values.conf.rabbitmqSecretName }}" {{- end }} securityContext: allowPrivilegeEscalation: false @@ -39,9 +39,9 @@ spec: - containerPort: 8080 resources: requests: - cpu: "{{ .Values.data.requests_cpu }}" - memory: "{{ .Values.data.requests_memory }}" + cpu: "{{ .Values.data.requestsCpu }}" + memory: "{{ .Values.data.requestsMemory }}" limits: - cpu: "{{ .Values.data.limits_cpu }}" - memory: "{{ .Values.data.limits_memory }}" + cpu: "{{ .Values.data.limitsCpu }}" + memory: "{{ .Values.data.limitsMemory }}" serviceAccountName: "{{ .Values.data.serviceAccountName }}" diff --git a/devops/gcp/deploy/templates/service-account.yaml b/devops/gcp/deploy/templates/service-account.yaml index 3fede170cb014a4cfdf8cc99d745a32eb66aeed0..25fd74efbfca059f541af82d98b676cfcb930d64 100644 --- a/devops/gcp/deploy/templates/service-account.yaml +++ b/devops/gcp/deploy/templates/service-account.yaml @@ -1,4 +1,4 @@ -{{- if .Values.conf.on_prem_enabled }} +{{- if .Values.conf.onPremEnabled }} apiVersion: v1 kind: ServiceAccount metadata: diff --git a/devops/gcp/deploy/templates/service.yaml b/devops/gcp/deploy/templates/service.yaml index 205b134595675a3df5e72627d681f8bf58defa68..8245eacfe6bc540c616b603d3bae00cc0e4af019 100644 --- a/devops/gcp/deploy/templates/service.yaml +++ b/devops/gcp/deploy/templates/service.yaml @@ -1,14 +1,14 @@ apiVersion: v1 kind: Service metadata: - name: "{{ .Values.conf.app_name }}" + name: "{{ .Values.conf.appName }}" annotations: - {{- if not .Values.conf.on_prem_enabled }} + {{- if not .Values.conf.onPremEnabled }} cloud.google.com/neg: '{"ingress": true}' {{- end }} namespace: "{{ .Release.Namespace }}" labels: - app: "{{ .Values.conf.app_name }}" + app: "{{ .Values.conf.appName }}" spec: ports: - protocol: TCP @@ -16,4 +16,4 @@ spec: targetPort: 8080 name: http selector: - app: "{{ .Values.conf.app_name }}" + app: "{{ .Values.conf.appName }}" diff --git a/devops/gcp/deploy/templates/virtual-service.yaml b/devops/gcp/deploy/templates/virtual-service.yaml index f160679776882dbe046438934da6fcb7be2fa794..7eea258fd5df13e2e4e92e2f5165381455e53242 100644 --- a/devops/gcp/deploy/templates/virtual-service.yaml +++ b/devops/gcp/deploy/templates/virtual-service.yaml @@ -1,12 +1,14 @@ apiVersion: networking.istio.io/v1alpha3 kind: VirtualService metadata: - name: "{{ .Values.conf.app_name }}" + name: "{{ .Values.conf.appName }}" namespace: "{{ .Release.Namespace }}" spec: hosts: - {{- if .Values.conf.domain }} + {{- if and .Values.conf.domain .Values.conf.onPremEnabled }} - {{ printf "osdu.%s" .Values.conf.domain | quote }} + {{- else if .Values.conf.domain }} + - {{ .Values.conf.domain | quote }} {{- else }} - "*" {{- end }} @@ -20,4 +22,4 @@ spec: - destination: port: number: 80 - host: "{{ .Values.conf.app_name }}.{{ .Release.Namespace }}.svc.cluster.local" + host: "{{ .Values.conf.appName }}.{{ .Release.Namespace }}.svc.cluster.local" diff --git a/devops/gcp/deploy/values.yaml b/devops/gcp/deploy/values.yaml index 79a59e639b071746893b95bfbe3a58b6218b86b4..21c44710e21707e16f704fe9397185b5f28ea1f0 100644 --- a/devops/gcp/deploy/values.yaml +++ b/devops/gcp/deploy/values.yaml @@ -1,21 +1,21 @@ data: - requests_cpu: "0.1" - requests_memory: "1G" - limits_cpu: "1" - limits_memory: "1.5G" - bootstrap_limits_cpu: "0.2" - bootstrap_limits_memory: "200M" - bootstrap_image: "" + requestsCpu: "0.1" + requestsMemory: "1G" + limitsCpu: "1" + limitsMemory: "1.5G" + bootstrapLimitsCpu: "0.2" + bootstrapLimitsMemory: "200M" + bootstrapImage: "" bootstrapServiceAccountName: "" image: "" imagePullPolicy: "IfNotPresent" serviceAccountName: "schema" conf: - app_name: "schema" - bootstrap_secret_name: "datafier-secret" + appName: "schema" + bootstrapSecretName: "datafier-secret" configmap: "schema-config" - minio_secret_name: "schema-minio-secret" - on_prem_enabled: false - postgres_secret_name: "schema-postgres-secret" - rabbitmq_secret_name: "rabbitmq-secret" domain: "" + minioSecretName: "schema-minio-secret" + onPremEnabled: false + postgresSecretName: "schema-postgres-secret" + rabbitmqSecretName: "rabbitmq-secret"