diff --git a/bootstrap_infra_on_prem/keycloak/keycloak.py b/bootstrap_infra_on_prem/keycloak/keycloak.py index 800cac5bd39c22abf6faa1c654cb7a5a2997b007..b0cab682e5762380ad5b040435237e85053d3374 100644 --- a/bootstrap_infra_on_prem/keycloak/keycloak.py +++ b/bootstrap_infra_on_prem/keycloak/keycloak.py @@ -18,6 +18,7 @@ SERVICES = { "notification": os.getenv('KEYCLOAK_NOTIFICATION_CLIENT_SECRET'), "register": os.getenv('KEYCLOAK_REGISTER_CLIENT_SECRET'), "file": os.getenv('KEYCLOAK_FILE_CLIENT_SECRET'), + "wks": os.getenv('KEYCLOAK_WKS_CLIENT_SECRET'), "datafier": os.getenv('KEYCLOAK_DATAFIER_CLIENT_SECRET') } diff --git a/bootstrap_infra_on_prem/minio/minio.sh b/bootstrap_infra_on_prem/minio/minio.sh index b821707f5e0bce97ba103e4f04eee63924496df5..9918d9d6ea465f8a02ccfc5e14a788fe80ec9b47 100644 --- a/bootstrap_infra_on_prem/minio/minio.sh +++ b/bootstrap_infra_on_prem/minio/minio.sh @@ -54,7 +54,7 @@ mc mb minio/"${MINIO_PROJECT_ID}-persistent-area" -p mc mb minio/"${MINIO_PARTITION_NAME}-legal-service-configuration" -p mc mb minio/"${MINIO_PROJECT_ID}-${MINIO_PARTITION_NAME}-legal-service-configuration" -p mc mb minio/"${MINIO_PROJECT_ID}-schema" -p -mc mb minio/"${MINIO_PROJECT_ID}-${MINIO_WKS_GCP_STORAGE_BUCKET_NAME}" -p +mc mb minio/"${MINIO_WKS_GCP_STORAGE_BUCKET_NAME}" -p #fixme after dev-s bucket refactor for wks mc mb minio/"ss-${MINIO_APP_ENVIRONMENT_IDENTIFIER}" -p # FIXME create policies afre re-run script diff --git a/bootstrap_infra_on_prem/postgresql/scripts/wks.sql b/bootstrap_infra_on_prem/postgresql/scripts/wks.sql index 7959f90d338b23a1ddb78c93fa98f8a0048173aa..3b778ed5654b604a6cbf7cd5404b1fe4a97e2951 100644 --- a/bootstrap_infra_on_prem/postgresql/scripts/wks.sql +++ b/bootstrap_infra_on_prem/postgresql/scripts/wks.sql @@ -4,7 +4,7 @@ CREATE SCHEMA IF NOT EXISTS "" AUTHORIZATION ; -CREATE TABLE IF NOT EXISTS ."wksMapping"( +CREATE TABLE IF NOT EXISTS ."WksMapping"( id text COLLATE pg_catalog."default" NOT NULL, pk bigint NOT NULL GENERATED ALWAYS AS IDENTITY PRIMARY KEY, data jsonb NOT NULL, @@ -12,18 +12,6 @@ CREATE TABLE IF NOT EXISTS ."wksMapping"( ); ALTER TABLE - ."wksMapping" OWNER to ; + ."WksMapping" OWNER to ; -CREATE INDEX IF NOT EXISTS wksMapping_datagin ON ."wksMapping" USING GIN (data); - -CREATE TABLE IF NOT EXISTS ."wksStatus"( - id text COLLATE pg_catalog."default" NOT NULL, - pk bigint NOT NULL GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - data jsonb NOT NULL, - CONSTRAINT wksStatus_id UNIQUE (id) -); - -ALTER TABLE - ."wksStatus" OWNER to ; - -CREATE INDEX IF NOT EXISTS wksStatus_datagin ON ."wksStatus" USING GIN (data); +CREATE INDEX IF NOT EXISTS wksMapping_datagin ON ."WksMapping" USING GIN (data); diff --git a/helm/osdu-infra-onprem/Chart.yaml b/helm/osdu-infra-onprem/Chart.yaml index f645d11a6adea2c1b1544868cd7a04d629dc13ff..73eb2ccdc66e8e0b2a77b0d05abdc269f549fa11 100644 --- a/helm/osdu-infra-onprem/Chart.yaml +++ b/helm/osdu-infra-onprem/Chart.yaml @@ -216,12 +216,7 @@ dependencies: repository: https://community.opengroup.org/api/v4/projects/395/packages/helm/stable condition: gcp-seismic-store-sdms-configmap.enabled alias: gcp_seismic_store_sdms_configmap - - name: gcp-wks-configmap - version: 0.1.0 - repository: https://community.opengroup.org/api/v4/projects/191/packages/helm/stable - condition: gcp-wks-configmap.enabled - alias: gcp_wks_configmap - ## Schema + ## Schema Service - name: gcp-schema-configmap version: 0.1.0 repository: https://community.opengroup.org/api/v4/projects/26/packages/helm/stable @@ -235,7 +230,18 @@ dependencies: version: 0.1.0 repository: https://community.opengroup.org/api/v4/projects/26/packages/helm/stable condition: osdu-schema-bootstrap.enabled - ## Workflow + ## Wks Service + - name: gcp-wks-configmap + version: 0.1.0 + repository: https://community.opengroup.org/api/v4/projects/191/packages/helm/stable + condition: gcp_wks_configmap.enabled + alias: gcp_wks_configmap + - name: gcp-wks-deploy + version: 0.1.0 + repository: https://community.opengroup.org/api/v4/projects/191/packages/helm/stable + condition: gcp_wks_deploy.enabled + alias: gcp_wks_deploy + ## Workflow Service - name: gcp-workflow-configmap version: 0.1.0 repository: https://community.opengroup.org/api/v4/projects/146/packages/helm/stable diff --git a/helm/osdu-infra-onprem/templates/keycloak-bootstrap-deployment.yaml b/helm/osdu-infra-onprem/templates/keycloak-bootstrap-deployment.yaml index aae58d99686857df365b16c88eac62ae7adf49ac..34b89dd669b4f1acfe94effb6db83e8963b3f583 100644 --- a/helm/osdu-infra-onprem/templates/keycloak-bootstrap-deployment.yaml +++ b/helm/osdu-infra-onprem/templates/keycloak-bootstrap-deployment.yaml @@ -96,3 +96,8 @@ spec: secretKeyRef: name: "{{ .Values.indexer_queue_deploy.conf.keycloak_secret_name }}" key: OPENID_PROVIDER_CLIENT_SECRET + - name: KEYCLOAK_WKS_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: "{{ .Values.gcp_wks_deploy.conf.openid_secret_name }}" + key: OPENID_PROVIDER_CLIENT_SECRET diff --git a/helm/osdu-infra-onprem/templates/minio-bootstrap-deployment.yaml b/helm/osdu-infra-onprem/templates/minio-bootstrap-deployment.yaml index e792ff60defdaacabbccd413f9220cc7f2c05f53..cc33aa72c1068c16f1e11a40bc8135adabe1c0ca 100644 --- a/helm/osdu-infra-onprem/templates/minio-bootstrap-deployment.yaml +++ b/helm/osdu-infra-onprem/templates/minio-bootstrap-deployment.yaml @@ -99,10 +99,10 @@ spec: - name: MINIO_WKS_USERNAME valueFrom: secretKeyRef: - name: minio-wks-secret #FIXME change to dynamic name from service values, when service is brought + name: "{{ .Values.gcp_wks_deploy.conf.minio_secret_name }}" key: MINIO_ACCESS_KEY - name: MINIO_WKS_PASSWORD valueFrom: secretKeyRef: - name: minio-wks-secret #FIXME change to dynamic name from service values, when service is brought + name: "{{ .Values.gcp_wks_deploy.conf.minio_secret_name }}" key: MINIO_SECRET_KEY diff --git a/helm/osdu-infra-onprem/templates/postgres-bootstrap-deployment.yaml b/helm/osdu-infra-onprem/templates/postgres-bootstrap-deployment.yaml index 34b71f1845869a328a3332c3305ee7742c184d52..65be2739116bfd01984d88877375b436014d2616 100644 --- a/helm/osdu-infra-onprem/templates/postgres-bootstrap-deployment.yaml +++ b/helm/osdu-infra-onprem/templates/postgres-bootstrap-deployment.yaml @@ -141,12 +141,12 @@ spec: - name: WKS_POSTGRESQL_USERNAME valueFrom: secretKeyRef: - name: wks-postgres-secret #FIXME change to dynamic name from service values, when service is brought + name: {{ .Values.gcp_wks_deploy.conf.wks_postgres_secret_name | quote }} key: POSTGRES_DB_USERNAME_{{ upper .Values.bootstrap.infra.secret.dataPartitionId }} - name: WKS_POSTGRESQL_PASSWORD valueFrom: secretKeyRef: - name: wks-postgres-secret #FIXME change to dynamic name from service values, when service is brought + name: {{ .Values.gcp_wks_deploy.conf.wks_postgres_secret_name | quote }} key: POSTGRES_DB_PASSWORD_{{ upper .Values.bootstrap.infra.secret.dataPartitionId }} # Workflow - name: WORKFLOW_POSTGRESQL_USERNAME diff --git a/helm/osdu-infra-onprem/templates/rabbitmq-secrets.yaml b/helm/osdu-infra-onprem/templates/rabbitmq-secrets.yaml index 27d05892da1f1d5fd2ab76f76e71313758123ef1..f6b709bf91c5d8c146c777519722aef196d6b505 100644 --- a/helm/osdu-infra-onprem/templates/rabbitmq-secrets.yaml +++ b/helm/osdu-infra-onprem/templates/rabbitmq-secrets.yaml @@ -48,6 +48,15 @@ stringData: "arguments": { "x-queue-type": "classic" } + }, + { + "name": "{{ printf "records-changed-sub-%s-wks" .Values.bootstrap.infra.secret.dataPartitionId}}", + "vhost": "/", + "durable": true, + "auto_delete": false, + "arguments": { + "x-queue-type": "classic" + } } ], "exchanges": [ @@ -68,6 +77,15 @@ stringData: "auto_delete": false, "internal": false, "arguments": {} + }, + { + "name": "records-changed", + "vhost": "/", + "type": "fanout", + "durable": true, + "auto_delete": false, + "internal": false, + "arguments": {} } ], "bindings": [ diff --git a/helm/osdu-infra-onprem/templates/wks-secret.yaml b/helm/osdu-infra-onprem/templates/wks-secret.yaml index 102e00a7cf576b5f5cfadaed68c55484baca8b0b..1d0035e11fe1ffe144ac3d07011f2edddaab86cb 100644 --- a/helm/osdu-infra-onprem/templates/wks-secret.yaml +++ b/helm/osdu-infra-onprem/templates/wks-secret.yaml @@ -1,22 +1,43 @@ apiVersion: v1 kind: Secret metadata: - name: minio-wks-secret + name: "{{ .Values.gcp_wks_deploy.conf.openid_secret_name }}" + annotations: + "helm.sh/resource-policy": no-upgrade-existing +type: Opaque +data: + OPENID_PROVIDER_CLIENT_ID: {{ print "wks" | b64enc | quote }} + OPENID_PROVIDER_CLIENT_SECRET: {{ randAlphaNum 16 | b64enc | quote }} + OPENID_PROVIDER_URL: {{ printf "http://keycloak/auth/realms/%s" .Values.bootstrap.keycloak.secret.keycloakRealmName | b64enc | quote }} + +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.gcp_wks_deploy.conf.minio_secret_name | quote }} annotations: "helm.sh/resource-policy": no-upgrade-existing type: Opaque data: MINIO_ACCESS_KEY: {{ print "wksUser" | b64enc | quote }} MINIO_SECRET_KEY: {{ randAlphaNum 16 | b64enc | quote }} + MINIO_ENDPOINT_URL: {{ printf "http://%s:9000" .Values.minio.fullnameOverride | b64enc | quote }} --- +{{- $wksDatabasePassword := (randAlphaNum 16)}} apiVersion: v1 kind: Secret metadata: - name: wks-postgres-secret #FIXME change to dynamic name from service values, when service is brought + name: {{ .Values.gcp_wks_deploy.conf.wks_postgres_secret_name | quote }} annotations: "helm.sh/resource-policy": no-upgrade-existing type: Opaque data: POSTGRES_DATASOURCE_URL_{{ upper .Values.bootstrap.infra.secret.dataPartitionId }}: {{ print "jdbc:postgresql://postgresql-db:5432/wks" | b64enc | quote }} POSTGRES_DB_USERNAME_{{ upper .Values.bootstrap.infra.secret.dataPartitionId }}: {{ print "wks_owner" | b64enc | quote }} - POSTGRES_DB_PASSWORD_{{ upper .Values.bootstrap.infra.secret.dataPartitionId }}: {{ randAlphaNum 16 | b64enc | quote }} + POSTGRES_DB_PASSWORD_{{ upper .Values.bootstrap.infra.secret.dataPartitionId }}: {{ $wksDatabasePassword | b64enc | quote }} + # fixme find a way to pass bootstrap reused secret values + POSTGRES_USER: {{ print "wks_owner" | b64enc | quote }} + POSTGRES_PASSWORD: {{ $wksDatabasePassword | b64enc | quote }} + POSTGRES_SERVER_HOST: {{ print "postgresql-db.default.svc.cluster.local" | b64enc | quote}} + POSTGRES_SERVER_PORT: {{ print "5432" | b64enc | quote }} + POSTGRES_DATABASE_NAME: {{ print "wks" | b64enc | quote }} diff --git a/helm/osdu-infra-onprem/values.yaml b/helm/osdu-infra-onprem/values.yaml index 9db2bef233c4769434808074743de26130b0eff8..61f4c8705054690cc815fe792276f0818aba40d0 100644 --- a/helm/osdu-infra-onprem/values.yaml +++ b/helm/osdu-infra-onprem/values.yaml @@ -734,10 +734,37 @@ gcp_seismic_store_sdms_configmap: data: app_environment_identifier: "dev" +# --- Wks values --- # gcp_wks_configmap: enabled: true data: - wks_gcp_storage_bucket_name: "wks-mapping-definitions" + spring_profiles_active: "anthos" + wks_gcp_storage_bucket_name: "wks-mapping-definitions" #fixme must be unique with project id, fix after dev-s bucket refactor for wks + wks_gcp_redis_host: "redis-search-master.default.svc.cluster.local" + wks_gcp_tenant_name: "osdu" + wks_gcp_records_topic: "records-changed" # has default in helm, mb need to add to rabbitmq config + conf: + configmap: "wks-config" + app_name: "wks" + bootstrap_configmap: "wks-bootstrap-config" + on_prem_enabled: true + +gcp_wks_deploy: + enabled: true + data: + image: "community.opengroup.org:5555/osdu/platform/data-flow/enrichment/wks/osdu-gcp" + bootstrap_image: "community.opengroup.org:5555/osdu/platform/data-flow/enrichment/wks/osdu-gcp-bootstrap-wks" + conf: + configmap: "wks-config" + app_name: "wks" + openid_secret_name: "wks-openid-secret" + minio_secret_name: "wks-minio-secret" + rabbitmq_secret_name: "rabbitmq-secret" + wks_postgres_secret_name: "wks-postgres-secret" + bootstrap_configmap: "wks-bootstrap-config" + on_prem_enabled: true + +# --- End of Wks values --- # # --- Schema values --- # gcp-schema-configmap: diff --git a/modules/osdu/bootstrap_wks.tf b/modules/osdu/bootstrap_wks.tf deleted file mode 100644 index 488cfc9265210fcacf2c4421c7a60292994b72a7..0000000000000000000000000000000000000000 --- a/modules/osdu/bootstrap_wks.tf +++ /dev/null @@ -1,64 +0,0 @@ -locals { - wks_bootstrap_configmap = "wks-bootstrap-configmap" - wks_bootstrap_name = "wks-bootstrap" - wks_bootstrap_image = "community.opengroup.org:5555/osdu/platform/data-flow/enrichment/wks/osdu-gcp-bootstrap-wks" -} - -resource "kubernetes_config_map" "wks-bootstrap-configmap" { - depends_on = [ - helm_release.ent-deploy, - helm_release.ent-configmap, - kubernetes_namespace.config - ] - metadata { - name = local.wks_bootstrap_configmap - namespace = kubernetes_namespace.config.metadata[0].name - labels = { - app = local.wks_bootstrap_name - } - } - data = { - PROJECT_ID = var.service_google_project - WKS_KIND = var.service_wks_mapping - WKS_TENANT = var.data_partition_id - WKS_BUCKET = "${var.service_google_project}-wks-mapping-definitions" - } -} - -resource "kubernetes_job" "wks-bootstrap-job" { - depends_on = [ - kubernetes_config_map.wks-bootstrap-configmap, - helm_release.ent-deploy, - helm_release.wks-config, - helm_release.wks-deploy, - google_service_account_iam_member.bootstrap-iam-gke-binding - ] - metadata { - name = local.wks_bootstrap_name - namespace = kubernetes_namespace.config.metadata[0].name - } - spec { - template { - metadata {} - spec { - service_account_name = kubernetes_service_account.gke-bootstrap-sa.metadata[0].name - container { - name = local.wks_bootstrap_name - image = join(":", [local.wks_bootstrap_image, local.wks_tag]) - command = ["/bin/bash", "-c", "source /opt/bootstrap.sh"] - env_from { - config_map_ref { - name = local.wks_bootstrap_configmap - } - } - } - } - } - backoff_limit = 2 - } - wait_for_completion = true - timeouts { - create = "10m" - update = "10m" - } -} diff --git a/modules/osdu/helm-wks.tf b/modules/osdu/helm-wks.tf index 07435cc09d72ab9ca4096b0eac25ffbe5b1bb0d2..15f865962ce12afc057a84328b8cb4681a4a51c3 100644 --- a/modules/osdu/helm-wks.tf +++ b/modules/osdu/helm-wks.tf @@ -1,6 +1,9 @@ locals { - wks_name = "wks" - wks_image = "community.opengroup.org:5555/osdu/platform/data-flow/enrichment/wks/osdu-gcp" + wks_name = "wks" + wks_image = "community.opengroup.org:5555/osdu/platform/data-flow/enrichment/wks/osdu-gcp" + wks_bootstrap_configmap = "wks-bootstrap-configmap" + wks_bootstrap_name = "wks-bootstrap" + wks_bootstrap_image = "community.opengroup.org:5555/osdu/platform/data-flow/enrichment/wks/osdu-gcp-bootstrap-wks" wks_roles_name = [ "roles/datastore.owner", "roles/storage.objectViewer", @@ -29,19 +32,10 @@ resource "helm_release" "wks-config" { version = local.wks_tag == "latest" ? "0.1.0" : trimprefix(local.wks_tag, "v") namespace = "default" recreate_pods = true - - set { - name = "data.project_id" - value = var.service_google_project - } set { name = "data.wks_gcp_audiences" value = var.audiences } - set { - name = "data.google_cloud_project" - value = var.service_google_project - } set { name = "data.wks_gcp_tenant_name" value = var.data_partition_id @@ -54,6 +48,22 @@ resource "helm_release" "wks-config" { name = "data.wks_gcp_redis_host" value = "${kubernetes_service.redis-cache-search.metadata[0].name}.${kubernetes_service.redis-cache-search.metadata[0].namespace}.svc.cluster.local" } + set { + name = "data.wks_namespace" + value = var.data_partition_id + } + set { + name = "data.wks_gcp_datastore_mapping_info_kind" + value = var.service_wks_mapping + } + set { + name = "conf.bootstrap_configmap" + value = local.wks_bootstrap_configmap + } + set { + name = "conf.bootstrap_namespace" + value = kubernetes_namespace.config.metadata[0].name + } } # Apply deploy from helm chart @@ -75,12 +85,32 @@ resource "helm_release" "wks-deploy" { name = "data.image" value = join(":", [local.wks_image, local.wks_tag]) } + set { + name = "data.bootstrap_image" + value = join(":", [local.wks_bootstrap_image, local.wks_tag]) + } set { name = "data.serviceAccountName" value = module.wks_workload_identity.service_account } + set { + name = "data.bootstrapServiceAccountName" + value = kubernetes_service_account.gke-bootstrap-sa.metadata[0].name + } set { name = "conf.app_name" value = local.wks_name } + set { + name = "conf.bootstrap_name" + value = local.wks_bootstrap_name + } + set { + name = "conf.bootstrap_namespace" + value = kubernetes_namespace.config.metadata[0].name + } + set { + name = "conf.bootstrap_configmap" + value = local.wks_bootstrap_configmap + } }