From 72214abd7dcb318482ebe9bbc59d9cb61cf8b416 Mon Sep 17 00:00:00 2001 From: Danylo Vanin Date: Wed, 13 Apr 2022 17:05:53 +0300 Subject: [PATCH 01/15] [GONRG-4576] Add wks helm, fix minio bucket --- bootstrap_infra_on_prem/minio/minio.sh | 2 +- helm/osdu-infra-onprem/Chart.yaml | 16 +++++++--- .../templates/minio-bootstrap-deployment.yaml | 4 +-- .../postgres-bootstrap-deployment.yaml | 4 +-- .../templates/wks-secret.yaml | 13 ++++++-- helm/osdu-infra-onprem/values.yaml | 32 ++++++++++++++++++- 6 files changed, 57 insertions(+), 14 deletions(-) diff --git a/bootstrap_infra_on_prem/minio/minio.sh b/bootstrap_infra_on_prem/minio/minio.sh index 74505cfb..9aa8749b 100644 --- a/bootstrap_infra_on_prem/minio/minio.sh +++ b/bootstrap_infra_on_prem/minio/minio.sh @@ -54,7 +54,7 @@ mc mb minio/"${MINIO_PROJECT_ID}-persistent-area" -p mc mb minio/"${MINIO_PROJECT_ID}-legal-service-configuration" -p mc mb minio/"${MINIO_PROJECT_ID}-${MINIO_PARTITION_NAME}-legal-service-configuration" -p mc mb minio/"${MINIO_PROJECT_ID}-schema" -p -mc mb minio/"${MINIO_PROJECT_ID}-${MINIO_WKS_GCP_STORAGE_BUCKET_NAME}" -p +mc mb minio/"${MINIO_WKS_GCP_STORAGE_BUCKET_NAME}" -p #fixme after dev-s bucket refactor for wks mc mb minio/"ss-${MINIO_APP_ENVIRONMENT_IDENTIFIER}" -p # FIXME create policies afre re-run script diff --git a/helm/osdu-infra-onprem/Chart.yaml b/helm/osdu-infra-onprem/Chart.yaml index abd8266c..6bf67a34 100644 --- a/helm/osdu-infra-onprem/Chart.yaml +++ b/helm/osdu-infra-onprem/Chart.yaml @@ -199,11 +199,6 @@ dependencies: repository: https://community.opengroup.org/api/v4/projects/395/packages/helm/stable condition: gcp-seismic-store-sdms-configmap.enabled alias: gcp_seismic_store_sdms_configmap - - name: gcp-wks-configmap - version: 0.1.0 - repository: https://community.opengroup.org/api/v4/projects/191/packages/helm/stable - condition: gcp-wks-configmap.enabled - alias: gcp_wks_configmap ## Schema Service - name: gcp-schema-configmap version: 0.1.0 @@ -218,6 +213,17 @@ dependencies: version: 0.1.0 repository: https://community.opengroup.org/api/v4/projects/26/packages/helm/stable condition: osdu-schema-bootstrap.enabled + ## Wks Service + - name: gcp-wks-configmap + version: 0.1.0 + repository: https://community.opengroup.org/api/v4/projects/191/packages/helm/stable + condition: gcp_wks_configmap.enabled + alias: gcp_wks_configmap + - name: gcp-wks-deploy + version: 0.1.0 + repository: https://community.opengroup.org/api/v4/projects/191/packages/helm/stable + condition: gcp_wks_deploy.enabled + alias: gcp_wks_deploy ## Workflow Service - name: gcp-workflow-configmap version: 0.1.0 diff --git a/helm/osdu-infra-onprem/templates/minio-bootstrap-deployment.yaml b/helm/osdu-infra-onprem/templates/minio-bootstrap-deployment.yaml index 0187aadd..961334b0 100644 --- a/helm/osdu-infra-onprem/templates/minio-bootstrap-deployment.yaml +++ b/helm/osdu-infra-onprem/templates/minio-bootstrap-deployment.yaml @@ -99,10 +99,10 @@ spec: - name: MINIO_WKS_USERNAME valueFrom: secretKeyRef: - name: minio-wks-secret #FIXME change to dynamic name from service values, when service is brought + name: "{{ .Values.gcp_wks_deploy.conf.minio_secret_name }}" key: MINIO_ACCESS_KEY - name: MINIO_WKS_PASSWORD valueFrom: secretKeyRef: - name: minio-wks-secret #FIXME change to dynamic name from service values, when service is brought + name: "{{ .Values.gcp_wks_deploy.conf.minio_secret_name }}" key: MINIO_SECRET_KEY diff --git a/helm/osdu-infra-onprem/templates/postgres-bootstrap-deployment.yaml b/helm/osdu-infra-onprem/templates/postgres-bootstrap-deployment.yaml index fd0f7cef..e023dc52 100644 --- a/helm/osdu-infra-onprem/templates/postgres-bootstrap-deployment.yaml +++ b/helm/osdu-infra-onprem/templates/postgres-bootstrap-deployment.yaml @@ -141,12 +141,12 @@ spec: - name: WKS_POSTGRESQL_USERNAME valueFrom: secretKeyRef: - name: wks-postgres-secret #FIXME change to dynamic name from service values, when service is brought + name: {{ .Values.gcp_wks_deploy.conf.wks_postgres_secret_name | quote }} key: POSTGRES_DB_USERNAME_{{ upper .Values.bootstrap.infra.secret.dataPartitionId }} - name: WKS_POSTGRESQL_PASSWORD valueFrom: secretKeyRef: - name: wks-postgres-secret #FIXME change to dynamic name from service values, when service is brought + name: {{ .Values.gcp_wks_deploy.conf.wks_postgres_secret_name | quote }} key: POSTGRES_DB_PASSWORD_{{ upper .Values.bootstrap.infra.secret.dataPartitionId }} # Workflow - name: WORKFLOW_POSTGRESQL_USERNAME diff --git a/helm/osdu-infra-onprem/templates/wks-secret.yaml b/helm/osdu-infra-onprem/templates/wks-secret.yaml index 102e00a7..11be5084 100644 --- a/helm/osdu-infra-onprem/templates/wks-secret.yaml +++ b/helm/osdu-infra-onprem/templates/wks-secret.yaml @@ -1,22 +1,29 @@ apiVersion: v1 kind: Secret metadata: - name: minio-wks-secret + name: {{ .Values.gcp_wks_deploy.conf.minio_secret_name | quote }} annotations: "helm.sh/resource-policy": no-upgrade-existing type: Opaque data: MINIO_ACCESS_KEY: {{ print "wksUser" | b64enc | quote }} MINIO_SECRET_KEY: {{ randAlphaNum 16 | b64enc | quote }} + MINIO_ENDPOINT_URL: {{ printf "http://%s:9000" .Values.minio.fullnameOverride | b64enc | quote }} --- +{{- $wksDatabasePassword := (randAlphaNum 16)}} apiVersion: v1 kind: Secret metadata: - name: wks-postgres-secret #FIXME change to dynamic name from service values, when service is brought + name: {{ .Values.gcp_wks_deploy.conf.wks_postgres_secret_name | quote }} annotations: "helm.sh/resource-policy": no-upgrade-existing type: Opaque data: POSTGRES_DATASOURCE_URL_{{ upper .Values.bootstrap.infra.secret.dataPartitionId }}: {{ print "jdbc:postgresql://postgresql-db:5432/wks" | b64enc | quote }} POSTGRES_DB_USERNAME_{{ upper .Values.bootstrap.infra.secret.dataPartitionId }}: {{ print "wks_owner" | b64enc | quote }} - POSTGRES_DB_PASSWORD_{{ upper .Values.bootstrap.infra.secret.dataPartitionId }}: {{ randAlphaNum 16 | b64enc | quote }} + POSTGRES_DB_PASSWORD_{{ upper .Values.bootstrap.infra.secret.dataPartitionId }}: {{ $wksDatabasePassword | b64enc | quote }} + POSTGRES_USER: {{ print "wks_owner" | b64enc | quote }} + POSTGRES_PASSWORD: {{ $wksDatabasePassword | b64enc | quote }} + POSTGRES_SERVER_HOST: {{ print "http://postgresql-db" | b64enc | quote}} + POSTGRES_SERVER_PORT: {{ print "5432" | b64enc | quote }} + POSTGRES_DATABASE_NAME: {{ print "wks" | b64enc | quote }} diff --git a/helm/osdu-infra-onprem/values.yaml b/helm/osdu-infra-onprem/values.yaml index fd86aeaa..08509d31 100644 --- a/helm/osdu-infra-onprem/values.yaml +++ b/helm/osdu-infra-onprem/values.yaml @@ -749,10 +749,40 @@ gcp_seismic_store_sdms_configmap: data: app_environment_identifier: "dev" +# --- Wks values --- # gcp_wks_configmap: enabled: true data: - wks_gcp_storage_bucket_name: "wks-mapping-definitions" + spring_profiles_active: "anthos" + wks_gcp_storage_bucket_name: "wks-mapping-definitions" #fixme must be unique with project id, fix after dev-s bucket refactor for wks + wks_gcp_redis_host: "redis-search-master" + wks_gcp_tenant_name: "osdu" + wks_gcp_records_topic: "records-changed" # has default in helm, mb need to add to rabbitmq config + conf: + configmap: "wks-config" + app_name: "wks" + on_prem_enabled: true + bootstrap: + bootstrap_configmap: "wks-bootstrap-config" + wks_namespace: "osdu" # data partition id + +gcp_wks_deploy: + enabled: true + data: + image: "community.opengroup.org:5555/osdu/platform/data-flow/enrichment/wks/osdu-gcp" + conf: + configmap: "wks-config" + app_name: "wks" + openid_secret_name: "wks-openid-secret" + minio_secret_name: "wks-minio-secret" + rabbitmq_secret_name: "rabbitmq-secret" + wks_postgres_secret_name: "wks-postgres-secret" + on_prem_enabled: true + bootstrap: + bootstrap_image: "community.opengroup.org:5555/osdu/platform/data-flow/enrichment/wks/osdu-gcp-bootstrap" + bootstrap_configmap: "wks-bootstrap-config" + +# --- End of Wks values --- # # --- Schema values --- # gcp-schema-configmap: -- GitLab From af8fb01d965afb818e1ddaeb7f936c5e881f5250 Mon Sep 17 00:00:00 2001 From: Danylo Vanin Date: Thu, 14 Apr 2022 20:46:20 +0300 Subject: [PATCH 02/15] [GONRG-4576] Change helm values --- helm/osdu-infra-onprem/templates/wks-secret.yaml | 2 +- helm/osdu-infra-onprem/values.yaml | 12 +++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/helm/osdu-infra-onprem/templates/wks-secret.yaml b/helm/osdu-infra-onprem/templates/wks-secret.yaml index 11be5084..7898ae57 100644 --- a/helm/osdu-infra-onprem/templates/wks-secret.yaml +++ b/helm/osdu-infra-onprem/templates/wks-secret.yaml @@ -24,6 +24,6 @@ data: POSTGRES_DB_PASSWORD_{{ upper .Values.bootstrap.infra.secret.dataPartitionId }}: {{ $wksDatabasePassword | b64enc | quote }} POSTGRES_USER: {{ print "wks_owner" | b64enc | quote }} POSTGRES_PASSWORD: {{ $wksDatabasePassword | b64enc | quote }} - POSTGRES_SERVER_HOST: {{ print "http://postgresql-db" | b64enc | quote}} + POSTGRES_SERVER_HOST: {{ print "http://postgresql-db.default.svc.cluster.local" | b64enc | quote}} POSTGRES_SERVER_PORT: {{ print "5432" | b64enc | quote }} POSTGRES_DATABASE_NAME: {{ print "wks" | b64enc | quote }} diff --git a/helm/osdu-infra-onprem/values.yaml b/helm/osdu-infra-onprem/values.yaml index ee12e20e..c69c8691 100644 --- a/helm/osdu-infra-onprem/values.yaml +++ b/helm/osdu-infra-onprem/values.yaml @@ -749,21 +749,21 @@ gcp_wks_configmap: data: spring_profiles_active: "anthos" wks_gcp_storage_bucket_name: "wks-mapping-definitions" #fixme must be unique with project id, fix after dev-s bucket refactor for wks - wks_gcp_redis_host: "redis-search-master" + wks_gcp_redis_host: "redis-search-master.default.svc.cluster.local" wks_gcp_tenant_name: "osdu" wks_gcp_records_topic: "records-changed" # has default in helm, mb need to add to rabbitmq config + wks_namespace: "osdu" # data partition id conf: configmap: "wks-config" app_name: "wks" - on_prem_enabled: true - bootstrap: bootstrap_configmap: "wks-bootstrap-config" - wks_namespace: "osdu" # data partition id + on_prem_enabled: true gcp_wks_deploy: enabled: true data: image: "community.opengroup.org:5555/osdu/platform/data-flow/enrichment/wks/osdu-gcp" + bootstrap_image: "community.opengroup.org:5555/osdu/platform/data-flow/enrichment/wks/osdu-gcp-bootstrap" conf: configmap: "wks-config" app_name: "wks" @@ -771,10 +771,8 @@ gcp_wks_deploy: minio_secret_name: "wks-minio-secret" rabbitmq_secret_name: "rabbitmq-secret" wks_postgres_secret_name: "wks-postgres-secret" - on_prem_enabled: true - bootstrap: - bootstrap_image: "community.opengroup.org:5555/osdu/platform/data-flow/enrichment/wks/osdu-gcp-bootstrap" bootstrap_configmap: "wks-bootstrap-config" + on_prem_enabled: true # --- End of Wks values --- # -- GitLab From 88009c8fad46ad0be8854dd7a067af1281385614 Mon Sep 17 00:00:00 2001 From: Danylo Vanin Date: Thu, 14 Apr 2022 21:21:41 +0300 Subject: [PATCH 03/15] [GONRG-4576] Add terraform values --- modules/osdu/bootstrap_wks.tf | 64 ----------------------------------- modules/osdu/helm-wks.tf | 40 +++++++++++++++++----- 2 files changed, 31 insertions(+), 73 deletions(-) delete mode 100644 modules/osdu/bootstrap_wks.tf diff --git a/modules/osdu/bootstrap_wks.tf b/modules/osdu/bootstrap_wks.tf deleted file mode 100644 index 2d341f76..00000000 --- a/modules/osdu/bootstrap_wks.tf +++ /dev/null @@ -1,64 +0,0 @@ -locals { - wks_bootstrap_configmap = "wks-bootstrap-configmap" - wks_bootstrap_name = "wks-bootstrap" - wks_bootstrap_image = "community.opengroup.org:5555/osdu/platform/data-flow/enrichment/wks/osdu-gcp-bootstrap-wks" -} - -resource "kubernetes_config_map" "wks-bootstrap-configmap" { - depends_on = [ - helm_release.ent-deploy, - helm_release.ent-configmap, - kubernetes_namespace.config - ] - metadata { - name = local.wks_bootstrap_configmap - namespace = kubernetes_namespace.config.metadata[0].name - labels = { - app = local.wks_bootstrap_name - } - } - data = { - PROJECT_ID = var.service_google_project - WKS_KIND = var.service_wks_mapping - WKS_TENANT = var.data_partition_id - WKS_BUCKET = "${var.service_google_project}-wks-mapping-definitions" - } -} - -resource "kubernetes_job" "wks-bootstrap-job" { - depends_on = [ - kubernetes_config_map.wks-bootstrap-configmap, - kubernetes_job.ent-bootstrap-job, - helm_release.wks-config, - helm_release.wks-deploy, - google_service_account_iam_member.bootstrap-iam-gke-binding - ] - metadata { - name = local.wks_bootstrap_name - namespace = kubernetes_namespace.config.metadata[0].name - } - spec { - template { - metadata {} - spec { - service_account_name = kubernetes_service_account.gke-bootstrap-sa.metadata[0].name - container { - name = local.wks_bootstrap_name - image = join(":", [local.wks_bootstrap_image, local.wks_tag]) - command = ["/bin/bash", "-c", "source /opt/bootstrap.sh"] - env_from { - config_map_ref { - name = local.wks_bootstrap_configmap - } - } - } - } - } - backoff_limit = 2 - } - wait_for_completion = true - timeouts { - create = "10m" - update = "10m" - } -} diff --git a/modules/osdu/helm-wks.tf b/modules/osdu/helm-wks.tf index 07435cc0..02b15bee 100644 --- a/modules/osdu/helm-wks.tf +++ b/modules/osdu/helm-wks.tf @@ -1,6 +1,9 @@ locals { wks_name = "wks" wks_image = "community.opengroup.org:5555/osdu/platform/data-flow/enrichment/wks/osdu-gcp" + wks_bootstrap_configmap = "wks-bootstrap-configmap" + wks_bootstrap_name = "wks-bootstrap" + wks_bootstrap_image = "community.opengroup.org:5555/osdu/platform/data-flow/enrichment/wks/osdu-gcp-bootstrap-wks" wks_roles_name = [ "roles/datastore.owner", "roles/storage.objectViewer", @@ -29,19 +32,10 @@ resource "helm_release" "wks-config" { version = local.wks_tag == "latest" ? "0.1.0" : trimprefix(local.wks_tag, "v") namespace = "default" recreate_pods = true - - set { - name = "data.project_id" - value = var.service_google_project - } set { name = "data.wks_gcp_audiences" value = var.audiences } - set { - name = "data.google_cloud_project" - value = var.service_google_project - } set { name = "data.wks_gcp_tenant_name" value = var.data_partition_id @@ -54,6 +48,18 @@ resource "helm_release" "wks-config" { name = "data.wks_gcp_redis_host" value = "${kubernetes_service.redis-cache-search.metadata[0].name}.${kubernetes_service.redis-cache-search.metadata[0].namespace}.svc.cluster.local" } + set { + name = "data.wks_namespace" + value = var.data_partition_id + } + set { + name = "data.wks_gcp_datastore_mapping_info_kind" + value = var.service_wks_mapping + } + set { + name = "conf.bootstrap_configmap" + value = local.wks_bwks_bootstrap_configmapootstrap_name + } } # Apply deploy from helm chart @@ -75,12 +81,28 @@ resource "helm_release" "wks-deploy" { name = "data.image" value = join(":", [local.wks_image, local.wks_tag]) } + set { + name = "data.bootstrap_image" + value = join(":", [local.wks_bootstrap_image, local.wks_tag]) + } set { name = "data.serviceAccountName" value = module.wks_workload_identity.service_account } + set { + name = "data.bootstrapServiceAccountName" + value = kubernetes_service_account.gke-bootstrap-sa.metadata[0].name + } set { name = "conf.app_name" value = local.wks_name } + set { + name = "conf.bootstrap_name" + value = local.wks_bootstrap_name + } + set { + name = "conf.bootstrap_configmap" + value = local.wks_bwks_bootstrap_configmapootstrap_name + } } -- GitLab From 168a24c616b0b4b7f06e0d2c8cf96e7283d927d7 Mon Sep 17 00:00:00 2001 From: Danylo Vanin Date: Thu, 14 Apr 2022 21:22:28 +0300 Subject: [PATCH 04/15] [GONRG-4576] terraform fmt --- modules/osdu/helm-wks.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/osdu/helm-wks.tf b/modules/osdu/helm-wks.tf index 02b15bee..c351edbf 100644 --- a/modules/osdu/helm-wks.tf +++ b/modules/osdu/helm-wks.tf @@ -1,6 +1,6 @@ locals { - wks_name = "wks" - wks_image = "community.opengroup.org:5555/osdu/platform/data-flow/enrichment/wks/osdu-gcp" + wks_name = "wks" + wks_image = "community.opengroup.org:5555/osdu/platform/data-flow/enrichment/wks/osdu-gcp" wks_bootstrap_configmap = "wks-bootstrap-configmap" wks_bootstrap_name = "wks-bootstrap" wks_bootstrap_image = "community.opengroup.org:5555/osdu/platform/data-flow/enrichment/wks/osdu-gcp-bootstrap-wks" -- GitLab From b8b7e5d71ec374776ebead6ca2727d45155cc6f1 Mon Sep 17 00:00:00 2001 From: Danylo Vanin Date: Thu, 14 Apr 2022 21:35:31 +0300 Subject: [PATCH 05/15] [GONRG-4576] rabbitmq definition --- .../templates/rabbitmq-secrets.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/helm/osdu-infra-onprem/templates/rabbitmq-secrets.yaml b/helm/osdu-infra-onprem/templates/rabbitmq-secrets.yaml index 27d05892..f6b709bf 100644 --- a/helm/osdu-infra-onprem/templates/rabbitmq-secrets.yaml +++ b/helm/osdu-infra-onprem/templates/rabbitmq-secrets.yaml @@ -48,6 +48,15 @@ stringData: "arguments": { "x-queue-type": "classic" } + }, + { + "name": "{{ printf "records-changed-sub-%s-wks" .Values.bootstrap.infra.secret.dataPartitionId}}", + "vhost": "/", + "durable": true, + "auto_delete": false, + "arguments": { + "x-queue-type": "classic" + } } ], "exchanges": [ @@ -68,6 +77,15 @@ stringData: "auto_delete": false, "internal": false, "arguments": {} + }, + { + "name": "records-changed", + "vhost": "/", + "type": "fanout", + "durable": true, + "auto_delete": false, + "internal": false, + "arguments": {} } ], "bindings": [ -- GitLab From 16ae2594c17c588448d766343f1a2502e3749cb1 Mon Sep 17 00:00:00 2001 From: Danylo Vanin Date: Fri, 15 Apr 2022 09:51:23 +0300 Subject: [PATCH 06/15] [GONRG-4576] rename table postgres --- bootstrap_infra_on_prem/postgresql/scripts/wks.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bootstrap_infra_on_prem/postgresql/scripts/wks.sql b/bootstrap_infra_on_prem/postgresql/scripts/wks.sql index 7959f90d..4a75cb48 100644 --- a/bootstrap_infra_on_prem/postgresql/scripts/wks.sql +++ b/bootstrap_infra_on_prem/postgresql/scripts/wks.sql @@ -4,7 +4,7 @@ CREATE SCHEMA IF NOT EXISTS "" AUTHORIZATION ; -CREATE TABLE IF NOT EXISTS ."wksMapping"( +CREATE TABLE IF NOT EXISTS ."WksMapping"( id text COLLATE pg_catalog."default" NOT NULL, pk bigint NOT NULL GENERATED ALWAYS AS IDENTITY PRIMARY KEY, data jsonb NOT NULL, @@ -12,9 +12,9 @@ CREATE TABLE IF NOT EXISTS ."wksMapping"( ); ALTER TABLE - ."wksMapping" OWNER to ; + ."WksMapping" OWNER to ; -CREATE INDEX IF NOT EXISTS wksMapping_datagin ON ."wksMapping" USING GIN (data); +CREATE INDEX IF NOT EXISTS wksMapping_datagin ON ."WksMapping" USING GIN (data); CREATE TABLE IF NOT EXISTS ."wksStatus"( id text COLLATE pg_catalog."default" NOT NULL, -- GitLab From 56aea821f16a7ba8384bdacece159a4fa37e97fc Mon Sep 17 00:00:00 2001 From: Danylo Vanin Date: Fri, 15 Apr 2022 18:05:44 +0300 Subject: [PATCH 07/15] [GONRG-4576] Fix terraform --- modules/osdu/helm-wks.tf | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/modules/osdu/helm-wks.tf b/modules/osdu/helm-wks.tf index c351edbf..3dac1cbe 100644 --- a/modules/osdu/helm-wks.tf +++ b/modules/osdu/helm-wks.tf @@ -58,7 +58,11 @@ resource "helm_release" "wks-config" { } set { name = "conf.bootstrap_configmap" - value = local.wks_bwks_bootstrap_configmapootstrap_name + value = local.wks_bootstrap_configmap + } + set { + name = "conf.bootstrap_namespace" + value = kubernetes_namespace.config.metadata[0].name } } @@ -101,8 +105,12 @@ resource "helm_release" "wks-deploy" { name = "conf.bootstrap_name" value = local.wks_bootstrap_name } + set { + name = "conf.bootstrap_namespace" + value = kubernetes_namespace.config.metadata[0].name + } set { name = "conf.bootstrap_configmap" - value = local.wks_bwks_bootstrap_configmapootstrap_name + value = local.wks_bootstrap_configmap } } -- GitLab From 7cd57c765e4486f5ed946004fd9a9157e9756e0b Mon Sep 17 00:00:00 2001 From: Danylo Vanin Date: Mon, 18 Apr 2022 10:01:16 +0300 Subject: [PATCH 08/15] [GONRG-4576] Removed duplicated value --- helm/osdu-infra-onprem/values.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/helm/osdu-infra-onprem/values.yaml b/helm/osdu-infra-onprem/values.yaml index 92829e1a..0f58ed4d 100644 --- a/helm/osdu-infra-onprem/values.yaml +++ b/helm/osdu-infra-onprem/values.yaml @@ -743,7 +743,6 @@ gcp_wks_configmap: wks_gcp_redis_host: "redis-search-master.default.svc.cluster.local" wks_gcp_tenant_name: "osdu" wks_gcp_records_topic: "records-changed" # has default in helm, mb need to add to rabbitmq config - wks_namespace: "osdu" # data partition id conf: configmap: "wks-config" app_name: "wks" -- GitLab From ce7510a017b14ae2a217863506a4341f3781bb0d Mon Sep 17 00:00:00 2001 From: Danylo Vanin Date: Mon, 18 Apr 2022 15:48:07 +0300 Subject: [PATCH 09/15] [GONRG-4576] Wks keycloak added --- bootstrap_infra_on_prem/bootstrap_infra.yml | 1 + bootstrap_infra_on_prem/keycloak/keycloak.py | 1 + .../templates/keycloak-bootstrap-deployment.yaml | 5 +++++ helm/osdu-infra-onprem/templates/wks-secret.yaml | 13 +++++++++++++ 4 files changed, 20 insertions(+) diff --git a/bootstrap_infra_on_prem/bootstrap_infra.yml b/bootstrap_infra_on_prem/bootstrap_infra.yml index ab05afda..14e114d5 100644 --- a/bootstrap_infra_on_prem/bootstrap_infra.yml +++ b/bootstrap_infra_on_prem/bootstrap_infra.yml @@ -77,4 +77,5 @@ osdu-gcp-containerize-bootstrap-keycloak-on-prem: - master - /.*bootstrap.*/ - tags + - GONRG-4576-Wks-Helm - release* diff --git a/bootstrap_infra_on_prem/keycloak/keycloak.py b/bootstrap_infra_on_prem/keycloak/keycloak.py index 800cac5b..b0cab682 100644 --- a/bootstrap_infra_on_prem/keycloak/keycloak.py +++ b/bootstrap_infra_on_prem/keycloak/keycloak.py @@ -18,6 +18,7 @@ SERVICES = { "notification": os.getenv('KEYCLOAK_NOTIFICATION_CLIENT_SECRET'), "register": os.getenv('KEYCLOAK_REGISTER_CLIENT_SECRET'), "file": os.getenv('KEYCLOAK_FILE_CLIENT_SECRET'), + "wks": os.getenv('KEYCLOAK_WKS_CLIENT_SECRET'), "datafier": os.getenv('KEYCLOAK_DATAFIER_CLIENT_SECRET') } diff --git a/helm/osdu-infra-onprem/templates/keycloak-bootstrap-deployment.yaml b/helm/osdu-infra-onprem/templates/keycloak-bootstrap-deployment.yaml index b1e81f00..2fe15d2e 100644 --- a/helm/osdu-infra-onprem/templates/keycloak-bootstrap-deployment.yaml +++ b/helm/osdu-infra-onprem/templates/keycloak-bootstrap-deployment.yaml @@ -96,3 +96,8 @@ spec: secretKeyRef: name: "{{ .Values.indexer_queue_deploy.conf.keycloak_secret_name }}" key: OPENID_PROVIDER_CLIENT_SECRET + - name: KEYCLOAK_WKS_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: "{{ .Values.gcp_wks_deploy.conf.openid_secret_name }}" + key: OPENID_PROVIDER_CLIENT_SECRET diff --git a/helm/osdu-infra-onprem/templates/wks-secret.yaml b/helm/osdu-infra-onprem/templates/wks-secret.yaml index 7898ae57..7a336561 100644 --- a/helm/osdu-infra-onprem/templates/wks-secret.yaml +++ b/helm/osdu-infra-onprem/templates/wks-secret.yaml @@ -1,5 +1,18 @@ apiVersion: v1 kind: Secret +metadata: + name: "{{ .Values.gcp_wks_deploy.conf.openid_secret_name }}" + annotations: + "helm.sh/resource-policy": no-upgrade-existing +type: Opaque +data: + OPENID_PROVIDER_CLIENT_ID: {{ print "wks" | b64enc | quote }} + OPENID_PROVIDER_CLIENT_SECRET: {{ randAlphaNum 16 | b64enc | quote }} + OPENID_PROVIDER_URL: {{ printf "http://keycloak/auth/realms/%s" .Values.bootstrap.keycloak.secret.keycloakRealmName | b64enc | quote }} + +--- +apiVersion: v1 +kind: Secret metadata: name: {{ .Values.gcp_wks_deploy.conf.minio_secret_name | quote }} annotations: -- GitLab From 4fcc95bcbe1ee783c843013c930c9eca127242ce Mon Sep 17 00:00:00 2001 From: Danylo Vanin Date: Mon, 18 Apr 2022 15:51:32 +0300 Subject: [PATCH 10/15] [GONRG-4576] tf fmt --- modules/osdu/helm-wks.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/osdu/helm-wks.tf b/modules/osdu/helm-wks.tf index 3dac1cbe..15f86596 100644 --- a/modules/osdu/helm-wks.tf +++ b/modules/osdu/helm-wks.tf @@ -60,7 +60,7 @@ resource "helm_release" "wks-config" { name = "conf.bootstrap_configmap" value = local.wks_bootstrap_configmap } - set { + set { name = "conf.bootstrap_namespace" value = kubernetes_namespace.config.metadata[0].name } -- GitLab From c375f6dc6c17c0bdf8bddce0569859d1bef26a7f Mon Sep 17 00:00:00 2001 From: Danylo Vanin Date: Mon, 18 Apr 2022 18:54:47 +0300 Subject: [PATCH 11/15] [GONRG-4576] bootstrap image and temp pipeline --- bootstrap_infra_on_prem/bootstrap_infra.yml | 1 + helm/osdu-infra-onprem/values.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/bootstrap_infra_on_prem/bootstrap_infra.yml b/bootstrap_infra_on_prem/bootstrap_infra.yml index 14e114d5..18103972 100644 --- a/bootstrap_infra_on_prem/bootstrap_infra.yml +++ b/bootstrap_infra_on_prem/bootstrap_infra.yml @@ -57,6 +57,7 @@ osdu-gcp-containerize-bootstrap-minio-on-prem: - master - /.*bootstrap.*/ - tags + - GONRG-4576-Wks-Helm - release* osdu-gcp-containerize-bootstrap-keycloak-on-prem: diff --git a/helm/osdu-infra-onprem/values.yaml b/helm/osdu-infra-onprem/values.yaml index 0f58ed4d..11df22ff 100644 --- a/helm/osdu-infra-onprem/values.yaml +++ b/helm/osdu-infra-onprem/values.yaml @@ -753,7 +753,7 @@ gcp_wks_deploy: enabled: true data: image: "community.opengroup.org:5555/osdu/platform/data-flow/enrichment/wks/osdu-gcp" - bootstrap_image: "community.opengroup.org:5555/osdu/platform/data-flow/enrichment/wks/osdu-gcp-bootstrap" + bootstrap_image: "community.opengroup.org:5555/osdu/platform/data-flow/enrichment/wks/osdu-gcp-bootstrap-wks" conf: configmap: "wks-config" app_name: "wks" -- GitLab From 395b061e14f0475125e316025e7a9b96cbb9e8dd Mon Sep 17 00:00:00 2001 From: Danylo Vanin Date: Mon, 18 Apr 2022 20:35:15 +0300 Subject: [PATCH 12/15] [GONRG-4576] fix minio and postgres bootstrap --- bootstrap_infra_on_prem/minio/minio_policy/wks.json | 2 +- helm/osdu-infra-onprem/templates/wks-secret.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bootstrap_infra_on_prem/minio/minio_policy/wks.json b/bootstrap_infra_on_prem/minio/minio_policy/wks.json index 024f287a..af4f4f89 100644 --- a/bootstrap_infra_on_prem/minio/minio_policy/wks.json +++ b/bootstrap_infra_on_prem/minio/minio_policy/wks.json @@ -4,7 +4,7 @@ { "Effect" : "Allow", "Action" : "s3:*", - "Resource" : "arn:aws:s3:::" + "Resource" : "arn:aws:s3:::/*" } ] } diff --git a/helm/osdu-infra-onprem/templates/wks-secret.yaml b/helm/osdu-infra-onprem/templates/wks-secret.yaml index 7a336561..8a0d2e0e 100644 --- a/helm/osdu-infra-onprem/templates/wks-secret.yaml +++ b/helm/osdu-infra-onprem/templates/wks-secret.yaml @@ -37,6 +37,6 @@ data: POSTGRES_DB_PASSWORD_{{ upper .Values.bootstrap.infra.secret.dataPartitionId }}: {{ $wksDatabasePassword | b64enc | quote }} POSTGRES_USER: {{ print "wks_owner" | b64enc | quote }} POSTGRES_PASSWORD: {{ $wksDatabasePassword | b64enc | quote }} - POSTGRES_SERVER_HOST: {{ print "http://postgresql-db.default.svc.cluster.local" | b64enc | quote}} + POSTGRES_SERVER_HOST: {{ print "postgresql-db.default.svc.cluster.local" | b64enc | quote}} POSTGRES_SERVER_PORT: {{ print "5432" | b64enc | quote }} POSTGRES_DATABASE_NAME: {{ print "wks" | b64enc | quote }} -- GitLab From 37b2f89b0d6c3efe5e67beb6ea2a85e9608546cc Mon Sep 17 00:00:00 2001 From: Danylo Vanin Date: Mon, 18 Apr 2022 21:16:53 +0300 Subject: [PATCH 13/15] [GONRG-4576] Remove temp --- bootstrap_infra_on_prem/bootstrap_infra.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/bootstrap_infra_on_prem/bootstrap_infra.yml b/bootstrap_infra_on_prem/bootstrap_infra.yml index 18103972..ab05afda 100644 --- a/bootstrap_infra_on_prem/bootstrap_infra.yml +++ b/bootstrap_infra_on_prem/bootstrap_infra.yml @@ -57,7 +57,6 @@ osdu-gcp-containerize-bootstrap-minio-on-prem: - master - /.*bootstrap.*/ - tags - - GONRG-4576-Wks-Helm - release* osdu-gcp-containerize-bootstrap-keycloak-on-prem: @@ -78,5 +77,4 @@ osdu-gcp-containerize-bootstrap-keycloak-on-prem: - master - /.*bootstrap.*/ - tags - - GONRG-4576-Wks-Helm - release* -- GitLab From 5346eec0844ce91785b6d178988ec6e34f9093d0 Mon Sep 17 00:00:00 2001 From: Danylo Vanin Date: Tue, 19 Apr 2022 13:13:02 +0300 Subject: [PATCH 14/15] [GONRG-4576] Remove unused database --- bootstrap_infra_on_prem/postgresql/scripts/wks.sql | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/bootstrap_infra_on_prem/postgresql/scripts/wks.sql b/bootstrap_infra_on_prem/postgresql/scripts/wks.sql index 4a75cb48..3b778ed5 100644 --- a/bootstrap_infra_on_prem/postgresql/scripts/wks.sql +++ b/bootstrap_infra_on_prem/postgresql/scripts/wks.sql @@ -15,15 +15,3 @@ ALTER TABLE ."WksMapping" OWNER to ; CREATE INDEX IF NOT EXISTS wksMapping_datagin ON ."WksMapping" USING GIN (data); - -CREATE TABLE IF NOT EXISTS ."wksStatus"( - id text COLLATE pg_catalog."default" NOT NULL, - pk bigint NOT NULL GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - data jsonb NOT NULL, - CONSTRAINT wksStatus_id UNIQUE (id) -); - -ALTER TABLE - ."wksStatus" OWNER to ; - -CREATE INDEX IF NOT EXISTS wksStatus_datagin ON ."wksStatus" USING GIN (data); -- GitLab From 29dc5a745093f5aefaf310787863b17c7b714575 Mon Sep 17 00:00:00 2001 From: Danylo Vanin Date: Tue, 19 Apr 2022 13:50:43 +0300 Subject: [PATCH 15/15] [GONRG-4576] add fixme --- helm/osdu-infra-onprem/templates/wks-secret.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/helm/osdu-infra-onprem/templates/wks-secret.yaml b/helm/osdu-infra-onprem/templates/wks-secret.yaml index 8a0d2e0e..1d0035e1 100644 --- a/helm/osdu-infra-onprem/templates/wks-secret.yaml +++ b/helm/osdu-infra-onprem/templates/wks-secret.yaml @@ -35,6 +35,7 @@ data: POSTGRES_DATASOURCE_URL_{{ upper .Values.bootstrap.infra.secret.dataPartitionId }}: {{ print "jdbc:postgresql://postgresql-db:5432/wks" | b64enc | quote }} POSTGRES_DB_USERNAME_{{ upper .Values.bootstrap.infra.secret.dataPartitionId }}: {{ print "wks_owner" | b64enc | quote }} POSTGRES_DB_PASSWORD_{{ upper .Values.bootstrap.infra.secret.dataPartitionId }}: {{ $wksDatabasePassword | b64enc | quote }} + # fixme find a way to pass bootstrap reused secret values POSTGRES_USER: {{ print "wks_owner" | b64enc | quote }} POSTGRES_PASSWORD: {{ $wksDatabasePassword | b64enc | quote }} POSTGRES_SERVER_HOST: {{ print "postgresql-db.default.svc.cluster.local" | b64enc | quote}} -- GitLab