Merge branch 'gcp-helm-anthos-on-prem' into 'master'

Helms for Anthos on-prem [GONRG-4427]

See merge request !304

.gitlab-ci.yml

@@ -18,10 +18,19 @@ variables:
 
   OSDU_GCP_SERVICE: indexer
   OSDU_GCP_VENDOR: gcp
-  OSDU_GCP_QUEUE_SA_EMAIL: pub-sub-indexer-queue@nice-etching-277309.iam.gserviceaccount.com
-  OSDU_GCP_HELM_CONFIG_SERVICE_VARS: "--set data.log_level=INFO --set data.schema_host=$OSDU_GCP_SCHEMA_API/schema  --set data.google_cloud_project=$OSDU_GCP_PROJECT --set data.redis_search_host=$REDIS_SEARCH_HOST --set data.redis_group_host=$REDIS_GROUP_HOST --set data.indexer_queue_host=$OSDU_GCP_INDEXER_QUEUE_URL --set data.crs_api=$OSDU_GCP_CRS_API --set data.partition_api=$OSDU_GCP_PARTITION_API --set data.google_audiences=$GOOGLE_AUDIENCE --set data.indexer_que_service_mail=$OSDU_GCP_QUEUE_SA_EMAIL"
-  OSDU_GCP_HELM_CONFIG_SERVICE_VARS_DEV2: "--set data.key_ring=$OSDU_GCP_PARTITION_KEY_RING --set data.log_level=INFO --set data.schema_host=$OSDU_GCP_SCHEMA_API/schema  --set data.google_cloud_project=$OSDU_GCP_PROJECT --set data.redis_search_host=$REDIS_SEARCH_HOST --set data.redis_group_host=$REDIS_GROUP_HOST --set data.indexer_queue_host=$OSDU_GCP_INDEXER_QUEUE_URL --set data.crs_api=$OSDU_GCP_CRS_API --set data.partition_api=$OSDU_GCP_PARTITION_API --set data.google_audiences=$GOOGLE_AUDIENCE --set data.indexer_que_service_mail=$OSDU_GCP_QUEUE_SA_EMAIL"
-  OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS: "--set data.image=$CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA --set data.serviceAccountName=$OSDU_GCP_SERVICE-k8s"
+  OSDU_GCP_HELM_CONFIG_SERVICE_VARS: >
+    --set data.log_level=INFO
+    --set data.redis_search_host=$REDIS_SEARCH_HOST
+    --set data.redis_group_host=$REDIS_GROUP_HOST
+    --set data.google_audiences=$GOOGLE_AUDIENCE
+  OSDU_GCP_HELM_CONFIG_SERVICE_VARS_DEV2: >
+    --set data.log_level=INFO
+    --set data.redis_search_host=$REDIS_SEARCH_HOST
+    --set data.redis_group_host=$REDIS_GROUP_HOST
+    --set data.google_audiences=$GOOGLE_AUDIENCE
+  OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS: > 
+    --set data.image=$CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA
+    --set data.serviceAccountName=$OSDU_GCP_SERVICE-k8s
   OSDU_GCP_HELM_CONFIG_SERVICE: indexer-config
   OSDU_GCP_HELM_DEPLOYMENT_SERVICE: indexer-deploy
   OSDU_GCP_HELM_PACKAGE_CHARTS: "devops/gcp/deploy devops/gcp/configmap"

devops/gcp/configmap/templates/indexer-variables.yml → devops/gcp/configmap/templates/variables.yaml

@@ -6,18 +6,16 @@ metadata:
   name: "{{ .Values.conf.configmap }}"
   namespace: "{{ .Release.Namespace }}"
 data:
+  ENTITLEMENTS_HOST: "{{ .Values.data.entitlements_host }}"
+  INDEXER_QUEUE_BASE_HOST: "{{ .Values.data.indexer_queue_base_host }}"
   LOG_LEVEL: "{{ .Values.data.log_level }}"
-  SCHEMA_HOST: "{{ .Values.data.schema_host }}"
-  AUTHORIZE_API: "{{ .Values.data.authorize_api }}"
-  REDIS_SEARCH_HOST: "{{ .Values.data.redis_search_host }}"
+  PARTITION_HOST: "{{ .Values.data.partition_host }}"
   REDIS_GROUP_HOST: "{{ .Values.data.redis_group_host }}"
+  REDIS_SEARCH_HOST: "{{ .Values.data.redis_search_host }}"
+  SCHEMA_BASE_HOST: "{{ .Values.data.schema_base_host }}"
   SECURITY_HTTPS_CERTIFICATE_TRUST: "{{ .Values.data.security_https_certificate_trust }}"
-  STORAGE_QUERY_RECORD_HOST: "{{ .Values.data.storage_query_record_host }}"
-  STORAGE_SCHEMA_HOST: "{{ .Values.data.storage_schema_host }}"
-  STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST: "{{ .Values.data.storage_query_record_for_conversion_host }}"
-  STORAGE_RECORDS_BATCH_SIZE: "{{ .Values.data.storage_records_batch_size }}"
-  STORAGE_QUERY_KINDS_HOST: "{{ .Values.data.storage_query_kinds_host }}"
-  INDEXER_QUEUE_HOST: "{{ .Values.data.indexer_queue_host }}"
-  PARTITION_API: "{{ .Values.data.partition_api }}"
-  GOOGLE_AUDIENCES: "{{ .Values.data.google_audiences }}"
   SPRING_PROFILES_ACTIVE: "{{ .Values.data.spring_profiles_active }}"
+  STORAGE_HOST: "{{ .Values.data.storage_host }}"
+  {{- if not .Values.conf.on_prem_enabled }}
+  GOOGLE_AUDIENCES: "{{ .Values.data.google_audiences }}"
+  {{- end }}

devops/gcp/configmap/values.yaml

@@ -3,23 +3,20 @@
 # Declare variables to be passed into your templates.
 
 data:
+  # common
+  entitlements_host: "http://entitlements"
+  indexer_queue_base_host: "http://indexer-queue"
   log_level: "INFO"
-  schema_host: ""
-  authorize_api: "http://entitlements/api/entitlements/v2/"
-  redis_search_host: ""
-  redis_group_host: ""
+  partition_host: "http://partition"
+  redis_group_host: "redis-group-master"
+  redis_search_host: "redis-search-master"
+  schema_base_host: "http://schema"
   security_https_certificate_trust: "true"
-  storage_query_record_host: "http://storage/api/storage/v2/query/records"
-  storage_schema_host: "http://storage/api/storage/v2/schemas"
-  storage_query_record_for_conversion_host: "http://storage/api/storage/v2/query/records:batch"
-  storage_records_batch_size: "20"
-  storage_query_kinds_host: "http://storage/api/storage/v2/query/kinds"
-  indexer_queue_host: ""
-  partition_api: "http://partition/api/partition/v1/"
-  google_audiences: ""
   spring_profiles_active: "gcp"
-
-
+  storage_host: "http://storage"
+  # gcp
+  google_audiences: ""
 conf:
-  configmap: "indexer-config"
   app_name: "indexer"
+  configmap: "indexer-config"
+  on_prem_enabled: false

devops/gcp/deploy/templates/indexer-deploy.yml → devops/gcp/deploy/templates/deployment.yaml

@@ -26,6 +26,12 @@ spec:
             name: "{{ .Values.conf.configmap }}"
         - secretRef:
             name: "{{ .Values.conf.elastic_secret_name }}"
+        {{- if .Values.conf.on_prem_enabled }}
+        - secretRef:
+            name: "{{ .Values.conf.keycloak_secret_name }}"
+        - secretRef:
+            name: "{{ .Values.conf.rabbitmq_secret_name }}"
+        {{- end }}
         securityContext:
           allowPrivilegeEscalation: false
           runAsUser: 0

devops/gcp/deploy/templates/service-account.yaml

+{{- if .Values.conf.on_prem_enabled }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: "{{ .Values.data.serviceAccountName }}"
+  namespace: "{{ .Release.Namespace }}"
+{{- end }}

devops/gcp/deploy/templates/indexer-service.yml → devops/gcp/deploy/templates/service.yaml

 apiVersion: v1
 kind: Service
 metadata:
-  name: "{{ .Values.conf.app_name }}"
+  {{- if not .Values.conf.on_prem_enabled }}
   annotations:
     cloud.google.com/neg: '{"ingress": true}'
+  {{- end }}
+  name: "{{ .Values.conf.app_name }}"
   namespace: "{{ .Release.Namespace }}"
-  labels:
-    app: "{{ .Values.conf.app_name }}"
-    service: "{{ .Values.conf.app_name }}"
 spec:
   ports:
     - protocol: TCP

devops/gcp/deploy/values.yaml

@@ -7,11 +7,13 @@ data:
   requests_memory: "640M"
   limits_cpu: "1"
   limits_memory: "1G"
-  serviceAccountName: ""
-  imagePullPolicy: "IfNotPresent"
   image: ""
-
+  imagePullPolicy: "IfNotPresent"
+  serviceAccountName: "indexer"
 conf:
+  app_name: "indexer"
   configmap: "indexer-config"
   elastic_secret_name: "elastic-secret"
-  app_name: "indexer"
+  keycloak_secret_name: "indexer-keycloak-secret"
+  on_prem_enabled: false
+  rabbitmq_secret_name: "rabbitmq-secret"

provider/indexer-gcp/src/main/resources/application.properties

@@ -56,4 +56,6 @@ SCHEMA_BASE_HOST=http://schema
 SCHEMA_PATH=/api/schema-service/v1/schema
 SCHEMA_HOST=${SCHEMA_BASE_HOST}${SCHEMA_PATH}
 
-INDEXER_QUEUE_HOST=http://indexer-queue
+INDEXER_QUEUE_BASE_HOST=http://indexer-queue
\ No newline at end of file
+INDEXER_QUEUE_PATH=/api/indexer-queue/v1/_dps/task-handlers/enqueue
+INDEXER_QUEUE_HOST=${INDEXER_QUEUE_BASE_HOST}${INDEXER_QUEUE_PATH}