Merge branch 'master' into schema-events

.gitlab-ci.yml

@@ -18,10 +18,19 @@ variables:
 
   OSDU_GCP_SERVICE: indexer
   OSDU_GCP_VENDOR: gcp
-  OSDU_GCP_QUEUE_SA_EMAIL: pub-sub-indexer-queue@nice-etching-277309.iam.gserviceaccount.com
-  OSDU_GCP_HELM_CONFIG_SERVICE_VARS: "--set data.log_level=INFO --set data.schema_host=$OSDU_GCP_SCHEMA_API/schema  --set data.google_cloud_project=$OSDU_GCP_PROJECT --set data.redis_search_host=$REDIS_SEARCH_HOST --set data.redis_group_host=$REDIS_GROUP_HOST --set data.indexer_queue_host=$OSDU_GCP_INDEXER_QUEUE_URL --set data.crs_api=$OSDU_GCP_CRS_API --set data.partition_api=$OSDU_GCP_PARTITION_API --set data.google_audiences=$GOOGLE_AUDIENCE --set data.indexer_que_service_mail=$OSDU_GCP_QUEUE_SA_EMAIL"
-  OSDU_GCP_HELM_CONFIG_SERVICE_VARS_DEV2: "--set data.key_ring=$OSDU_GCP_PARTITION_KEY_RING --set data.log_level=INFO --set data.schema_host=$OSDU_GCP_SCHEMA_API/schema  --set data.google_cloud_project=$OSDU_GCP_PROJECT --set data.redis_search_host=$REDIS_SEARCH_HOST --set data.redis_group_host=$REDIS_GROUP_HOST --set data.indexer_queue_host=$OSDU_GCP_INDEXER_QUEUE_URL --set data.crs_api=$OSDU_GCP_CRS_API --set data.partition_api=$OSDU_GCP_PARTITION_API --set data.google_audiences=$GOOGLE_AUDIENCE --set data.indexer_que_service_mail=$OSDU_GCP_QUEUE_SA_EMAIL"
-  OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS: "--set data.image=$CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA --set data.serviceAccountName=$OSDU_GCP_SERVICE-k8s"
+  OSDU_GCP_HELM_CONFIG_SERVICE_VARS: >
+    --set data.log_level=INFO
+    --set data.redis_search_host=$REDIS_SEARCH_HOST
+    --set data.redis_group_host=$REDIS_GROUP_HOST
+    --set data.google_audiences=$GOOGLE_AUDIENCE
+  OSDU_GCP_HELM_CONFIG_SERVICE_VARS_DEV2: >
+    --set data.log_level=INFO
+    --set data.redis_search_host=$REDIS_SEARCH_HOST
+    --set data.redis_group_host=$REDIS_GROUP_HOST
+    --set data.google_audiences=$GOOGLE_AUDIENCE
+  OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS: > 
+    --set data.image=$CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA
+    --set data.serviceAccountName=$OSDU_GCP_SERVICE-k8s
   OSDU_GCP_HELM_CONFIG_SERVICE: indexer-config
   OSDU_GCP_HELM_DEPLOYMENT_SERVICE: indexer-deploy
   OSDU_GCP_HELM_PACKAGE_CHARTS: "devops/gcp/deploy devops/gcp/configmap"

NOTICE

@@ -13,8 +13,6 @@ The following software have components provided under the terms of this license:
 ========================================================================
 Apache-1.1
 ========================================================================
-The following software have components provided under the terms of this license:
-
 - Apache Commons CLI (from https://commons.apache.org/proper/commons-cli/, https://repo1.maven.org/maven2/commons-cli/commons-cli)
 - Default Plexus Container (from https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-container-default)
 - oro (from https://repo1.maven.org/maven2/oro/oro)
@@ -30,7 +28,7 @@ The following software have components provided under the terms of this license:
 - AWS SDK for Java - Models (from https://aws.amazon.com/sdkforjava)
 - Apache Ant + JUnit (from http://ant.apache.org/, https://ant.apache.org/)
 - Apache Ant Core
-- Apache Ant Launcher (from http://ant.apache.org/)
+- Apache Ant Launcher (from http://ant.apache.org/, https://ant.apache.org/)
 - Apache Commons BeanUtils (from http://commons.apache.org/proper/commons-beanutils/, https://repo1.maven.org/maven2/commons-beanutils/commons-beanutils)
 - Apache Commons CLI (from https://commons.apache.org/proper/commons-cli/, https://repo1.maven.org/maven2/commons-cli/commons-cli)
 - Apache Commons Codec (from http://commons.apache.org/proper/commons-codec/, https://commons.apache.org/proper/commons-codec/)
@@ -201,23 +199,17 @@ The following software have components provided under the terms of this license:
 ========================================================================
 CC-BY-2.5
 ========================================================================
-The following software have components provided under the terms of this license:
-
 - FindBugs-jsr305 (from http://findbugs.sourceforge.net/)
 - MongoDB Java Driver (from http://mongodb.org/, http://www.mongodb.org)
 
 ========================================================================
 CC0-1.0
 ========================================================================
-The following software have components provided under the terms of this license:
-
 - reactive-streams (from http://www.reactive-streams.org/)
 
 ========================================================================
 CDDL-1.0
 ========================================================================
-The following software have components provided under the terms of this license:
-
 - Jakarta Activation API (from https://github.com/eclipse-ee4j/jaf, https://repo1.maven.org/maven2/jakarta/activation/jakarta.activation-api)
 - Servlet Specification 2.5 API (from http://jetty.mortbay.org, https://repo1.maven.org/maven2/org/mortbay/jetty/servlet-api-2.5)
 
@@ -273,8 +265,6 @@ The following software have components provided under the terms of this license:
 ========================================================================
 EPL-2.0
 ========================================================================
-The following software have components provided under the terms of this license:
-
 - HK2 Implementation Utilities (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-utils)
 - JUnit Jupiter (Aggregator) (from https://junit.org/junit5/)
 - JUnit Jupiter API (from http://junit.org/junit5/, https://junit.org/junit5/)
@@ -305,8 +295,6 @@ The following software have components provided under the terms of this license:
 ========================================================================
 GPL-2.0-or-later
 ========================================================================
-The following software have components provided under the terms of this license:
-
 - SnakeYAML (from http://code.google.com/p/snakeyaml/, http://www.snakeyaml.org)
 
 ========================================================================
@@ -331,8 +319,6 @@ The following software have components provided under the terms of this license:
 ========================================================================
 GPL-3.0-only
 ========================================================================
-The following software have components provided under the terms of this license:
-
 - Jakarta Annotations API (from https://projects.eclipse.org/projects/ee4j.ca)
 - OSGi resource locator (from https://repo1.maven.org/maven2/org/glassfish/hk2/osgi-resource-locator)
 - Project Lombok (from http://projectlombok.org, https://projectlombok.org)
@@ -341,8 +327,6 @@ The following software have components provided under the terms of this license:
 ========================================================================
 JSON
 ========================================================================
-The following software have components provided under the terms of this license:
-
 - JSON in Java (from https://github.com/douglascrockford/JSON-java)
 
 ========================================================================
@@ -363,8 +347,6 @@ The following software have components provided under the terms of this license:
 ========================================================================
 LGPL-2.1-or-later
 ========================================================================
-The following software have components provided under the terms of this license:
-
 - Javassist (from http://www.javassist.org/)
 - SnakeYAML (from http://code.google.com/p/snakeyaml/, http://www.snakeyaml.org)
 
@@ -396,16 +378,12 @@ The following software have components provided under the terms of this license:
 ========================================================================
 MPL-1.1
 ========================================================================
-The following software have components provided under the terms of this license:
-
 - Javassist (from http://www.javassist.org/)
 - RabbitMQ Java Client (from http://www.rabbitmq.com, https://www.rabbitmq.com)
 
 ========================================================================
 MPL-2.0
 ========================================================================
-The following software have components provided under the terms of this license:
-
 - Javassist (from http://www.javassist.org/)
 - OkHttp (from https://repo1.maven.org/maven2/com/squareup/okhttp3/okhttp, https://square.github.io/okhttp/)
 - RabbitMQ Java Client (from http://www.rabbitmq.com, https://www.rabbitmq.com)
@@ -413,8 +391,6 @@ The following software have components provided under the terms of this license:
 ========================================================================
 PHP-3.01
 ========================================================================
-The following software have components provided under the terms of this license:
-
 - Jakarta Activation API (from https://github.com/eclipse-ee4j/jaf, https://repo1.maven.org/maven2/jakarta/activation/jakarta.activation-api)
 - Jakarta XML Binding API (from https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api, https://repo1.maven.org/maven2/org/jboss/spec/javax/xml/bind/jboss-jaxb-api_2.3_spec)
 
@@ -428,8 +404,6 @@ The following software have components provided under the terms of this license:
 ========================================================================
 SPL-1.0
 ========================================================================
-The following software have components provided under the terms of this license:
-
 - Servlet Specification 2.5 API (from http://jetty.mortbay.org, https://repo1.maven.org/maven2/org/mortbay/jetty/servlet-api-2.5)
 
 ========================================================================
@@ -453,8 +427,6 @@ The following software have components provided under the terms of this license:
 ========================================================================
 WTFPL
 ========================================================================
-The following software have components provided under the terms of this license:
-
 - Reflections (from http://code.google.com/p/reflections/, http://github.com/ronmamo/reflections)
 
 ========================================================================
@@ -495,4 +467,4 @@ The following software have components provided under the terms of this license:
 - Jakarta Activation API (from https://github.com/eclipse-ee4j/jaf, https://repo1.maven.org/maven2/jakarta/activation/jakarta.activation-api)
 - Jakarta XML Binding API (from https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api, https://repo1.maven.org/maven2/org/jboss/spec/javax/xml/bind/jboss-jaxb-api_2.3_spec)
 - jaxen (from http://jaxen.codehaus.org/, https://repo1.maven.org/maven2/jaxen/jaxen)
-- xml-apis
\ No newline at end of file
+- xml-apis

devops/gcp/configmap/templates/indexer-variables.yml → devops/gcp/configmap/templates/variables.yaml

@@ -6,18 +6,16 @@ metadata:
   name: "{{ .Values.conf.configmap }}"
   namespace: "{{ .Release.Namespace }}"
 data:
+  ENTITLEMENTS_HOST: "{{ .Values.data.entitlements_host }}"
+  INDEXER_QUEUE_BASE_HOST: "{{ .Values.data.indexer_queue_base_host }}"
   LOG_LEVEL: "{{ .Values.data.log_level }}"
-  SCHEMA_HOST: "{{ .Values.data.schema_host }}"
-  AUTHORIZE_API: "{{ .Values.data.authorize_api }}"
-  REDIS_SEARCH_HOST: "{{ .Values.data.redis_search_host }}"
+  PARTITION_HOST: "{{ .Values.data.partition_host }}"
   REDIS_GROUP_HOST: "{{ .Values.data.redis_group_host }}"
+  REDIS_SEARCH_HOST: "{{ .Values.data.redis_search_host }}"
+  SCHEMA_BASE_HOST: "{{ .Values.data.schema_base_host }}"
   SECURITY_HTTPS_CERTIFICATE_TRUST: "{{ .Values.data.security_https_certificate_trust }}"
-  STORAGE_QUERY_RECORD_HOST: "{{ .Values.data.storage_query_record_host }}"
-  STORAGE_SCHEMA_HOST: "{{ .Values.data.storage_schema_host }}"
-  STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST: "{{ .Values.data.storage_query_record_for_conversion_host }}"
-  STORAGE_RECORDS_BATCH_SIZE: "{{ .Values.data.storage_records_batch_size }}"
-  STORAGE_QUERY_KINDS_HOST: "{{ .Values.data.storage_query_kinds_host }}"
-  INDEXER_QUEUE_HOST: "{{ .Values.data.indexer_queue_host }}"
-  PARTITION_API: "{{ .Values.data.partition_api }}"
-  GOOGLE_AUDIENCES: "{{ .Values.data.google_audiences }}"
   SPRING_PROFILES_ACTIVE: "{{ .Values.data.spring_profiles_active }}"
+  STORAGE_HOST: "{{ .Values.data.storage_host }}"
+  {{- if not .Values.conf.on_prem_enabled }}
+  GOOGLE_AUDIENCES: "{{ .Values.data.google_audiences }}"
+  {{- end }}

devops/gcp/configmap/values.yaml

@@ -3,23 +3,20 @@
 # Declare variables to be passed into your templates.
 
 data:
+  # common
+  entitlements_host: "http://entitlements"
+  indexer_queue_base_host: "http://indexer-queue"
   log_level: "INFO"
-  schema_host: ""
-  authorize_api: "http://entitlements/api/entitlements/v2/"
-  redis_search_host: ""
-  redis_group_host: ""
+  partition_host: "http://partition"
+  redis_group_host: "redis-group-master"
+  redis_search_host: "redis-search-master"
+  schema_base_host: "http://schema"
   security_https_certificate_trust: "true"
-  storage_query_record_host: "http://storage/api/storage/v2/query/records"
-  storage_schema_host: "http://storage/api/storage/v2/schemas"
-  storage_query_record_for_conversion_host: "http://storage/api/storage/v2/query/records:batch"
-  storage_records_batch_size: "20"
-  storage_query_kinds_host: "http://storage/api/storage/v2/query/kinds"
-  indexer_queue_host: ""
-  partition_api: "http://partition/api/partition/v1/"
-  google_audiences: ""
   spring_profiles_active: "gcp"
-
-
+  storage_host: "http://storage"
+  # gcp
+  google_audiences: ""
 conf:
-  configmap: "indexer-config"
   app_name: "indexer"
+  configmap: "indexer-config"
+  on_prem_enabled: false

devops/gcp/deploy/templates/indexer-deploy.yml → devops/gcp/deploy/templates/deployment.yaml

@@ -26,6 +26,12 @@ spec:
             name: "{{ .Values.conf.configmap }}"
         - secretRef:
             name: "{{ .Values.conf.elastic_secret_name }}"
+        {{- if .Values.conf.on_prem_enabled }}
+        - secretRef:
+            name: "{{ .Values.conf.keycloak_secret_name }}"
+        - secretRef:
+            name: "{{ .Values.conf.rabbitmq_secret_name }}"
+        {{- end }}
         securityContext:
           allowPrivilegeEscalation: false
           runAsUser: 0

devops/gcp/deploy/templates/service-account.yaml

+{{- if .Values.conf.on_prem_enabled }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: "{{ .Values.data.serviceAccountName }}"
+  namespace: "{{ .Release.Namespace }}"
+{{- end }}

devops/gcp/deploy/templates/indexer-service.yml → devops/gcp/deploy/templates/service.yaml

 apiVersion: v1
 kind: Service
 metadata:
-  name: "{{ .Values.conf.app_name }}"
+  {{- if not .Values.conf.on_prem_enabled }}
   annotations:
     cloud.google.com/neg: '{"ingress": true}'
+  {{- end }}
+  name: "{{ .Values.conf.app_name }}"
   namespace: "{{ .Release.Namespace }}"
-  labels:
-    app: "{{ .Values.conf.app_name }}"
-    service: "{{ .Values.conf.app_name }}"
 spec:
   ports:
     - protocol: TCP

devops/gcp/deploy/values.yaml

@@ -7,11 +7,13 @@ data:
   requests_memory: "640M"
   limits_cpu: "1"
   limits_memory: "1G"
-  serviceAccountName: ""
-  imagePullPolicy: "IfNotPresent"
   image: ""
-
+  imagePullPolicy: "IfNotPresent"
+  serviceAccountName: "indexer"
 conf:
+  app_name: "indexer"
   configmap: "indexer-config"
   elastic_secret_name: "elastic-secret"
-  app_name: "indexer"
+  keycloak_secret_name: "indexer-keycloak-secret"
+  on_prem_enabled: false
+  rabbitmq_secret_name: "rabbitmq-secret"

provider/indexer-aws/pom.xml

@@ -69,7 +69,7 @@
     <dependency>
         <groupId>org.opengroup.osdu.core.aws</groupId>
         <artifactId>os-core-lib-aws</artifactId>
-        <version>0.13.0</version>
+        <version>0.14.0-rc2</version>
     </dependency>
 
     <!-- AWS managed packages -->

provider/indexer-gcp/src/main/resources/application.properties

@@ -56,4 +56,6 @@ SCHEMA_BASE_HOST=http://schema
 SCHEMA_PATH=/api/schema-service/v1/schema
 SCHEMA_HOST=${SCHEMA_BASE_HOST}${SCHEMA_PATH}
 
-INDEXER_QUEUE_HOST=http://indexer-queue
\ No newline at end of file
+INDEXER_QUEUE_BASE_HOST=http://indexer-queue
+INDEXER_QUEUE_PATH=/api/indexer-queue/v1/_dps/task-handlers/enqueue
+INDEXER_QUEUE_HOST=${INDEXER_QUEUE_BASE_HOST}${INDEXER_QUEUE_PATH}

testing/indexer-test-aws/pom.xml

@@ -49,7 +49,7 @@
         <dependency>
             <groupId>org.opengroup.osdu.core.aws</groupId>
             <artifactId>os-core-lib-aws</artifactId>
-            <version>0.3.16</version>
+            <version>0.14.0-rc2</version>
         </dependency>
 
         <!-- Testing -->