diff --git a/.editorconfig b/.editorconfig
new file mode 100644
index 0000000000000000000000000000000000000000..5b462cccdadf92874f22e8ef07043baace8eebf9
--- /dev/null
+++ b/.editorconfig
@@ -0,0 +1,4 @@
+root = true
+
+[*]
+insert_final_newline = true
diff --git a/NOTICE b/NOTICE
index 11f8606cf3e44d0113fb66196f2aae4e1d2a567f..a2af8a5f0244b1cad89d0c288e791f112f5b764f 100644
--- a/NOTICE
+++ b/NOTICE
@@ -376,10 +376,10 @@ The following software have components provided under the terms of this license:
- Byte Buddy Java agent (from https://repo1.maven.org/maven2/net/bytebuddy/byte-buddy-agent)
- ClassMate (from http://github.com/cowtowncoder/java-classmate)
- Cloud Key Management Service (KMS) API v1-rev20220407-1.32.1 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-cloudkms)
-- Cloud Key Management Service (KMS) API v1-rev20220701-1.32.1 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-cloudkms)
+- Cloud Key Management Service (KMS) API v1-rev20220617-1.32.1 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-cloudkms)
- Cloud Storage JSON API v1-rev20220604-1.32.1 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-storage)
- Cloud Storage JSON API v1-rev20220608-1.32.1 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-storage)
-- Cloud Storage JSON API v1-rev20220705-2.0.0 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-storage)
+- Cloud Storage JSON API v1-rev20220705-1.32.1 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-storage)
- CloudWatch Metrics for AWS Java SDK (from https://aws.amazon.com/sdkforjava)
- Cobertura (from http://cobertura.sourceforge.net)
- Cobertura Limited Runtime (from http://cobertura.sourceforge.net)
@@ -429,7 +429,7 @@ The following software have components provided under the terms of this license:
- IBM COS SDK For Java (from https://github.com/ibm/ibm-cos-sdk-java)
- IBM COS SDK for Java - Core (from https://github.com/ibm/ibm-cos-sdk-java)
- Identity and Access Management (IAM) API v2beta-rev20220421-1.32.1 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-iam)
-- Identity and Access Management (IAM) API v2beta-rev20220713-1.32.1 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-iam)
+- Identity and Access Management (IAM) API v2beta-rev20220526-1.32.1 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-iam)
- J2ObjC Annotations (from https://github.com/google/j2objc/)
- JBoss Logging 3 (from http://www.jboss.org)
- JBoss Marshalling API (from https://repo1.maven.org/maven2/org/jboss/marshalling/jboss-marshalling)
diff --git a/devops/azure/chart/templates/deployment.yaml b/devops/azure/chart/templates/deployment.yaml
index a4ddf5438887a7e338bbcd857d3cdd46cffdc0da..400579dc3d4032f1683cf0565cea9f315e1c8294 100644
--- a/devops/azure/chart/templates/deployment.yaml
+++ b/devops/azure/chart/templates/deployment.yaml
@@ -43,6 +43,10 @@ spec:
- name: {{ .Chart.Name }}
image: {{ .Values.image.repository }}/{{ .Chart.Name }}-{{ .Values.image.branch }}:{{ .Values.image.tag | default .Chart.AppVersion }}
imagePullPolicy: Always
+ lifecycle:
+ preStop:
+ exec:
+ command: ["sleep","90"]
ports:
- containerPort: 80
resources:
@@ -120,4 +124,5 @@ spec:
- name: azure_activedirectory_AppIdUri
value: "api://$(aad_client_id)"
- name: SPRING_CONFIG_NAME
- value: "common,application"
\ No newline at end of file
+ value: "common,application"
+ terminationGracePeriodSeconds: 101
\ No newline at end of file
diff --git a/devops/gcp/configmap/README.md b/devops/gcp/configmap/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..6ecc6fe4fdc4803bc1a8bd97ec528daf7226e2f3
--- /dev/null
+++ b/devops/gcp/configmap/README.md
@@ -0,0 +1,74 @@
+<!--- Configmap -->
+
+# Configmap helm chart
+
+## Introduction
+
+This chart bootstraps a configmap deployment on a [Kubernetes](https://kubernetes.io) cluster using [Helm](https://helm.sh) package manager.
+
+## Prerequisites
+
+The code was tested on **Kubernetes cluster** (v1.21.11) with **Istio** (1.12.6)
+> It is possible to use other versions, but it hasn't been tested
+
+### Operation system
+
+The code works in Debian-based Linux (Debian 10 and Ubuntu 20.04) and Windows WSL 2. Also, it works but is not guaranteed in Google Cloud Shell. All other operating systems, including macOS, are not verified and supported.
+
+### Packages
+
+Packages are only needed for installation from a local computer.
+
+* **HELM** (version: v3.7.1 or higher) [helm](https://helm.sh/docs/intro/install/)
+* **Kubectl** (version: v1.21.0 or higher) [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl)
+
+## Installation
+
+First you need to set variables in **values.yaml** file using any code editor. Some of the values are prefilled, but you need to specify some values as well. You can find more information about them below.
+
+### Common variables
+
+| Name | Description | Type | Default |Required |
+|------|-------------|------|---------|---------|
+**entitlementsHost** | entitlements host | string | "http://entitlements" | yes
+**indexerQueueHost** | indexer-queue host | string | "http://indexer-queue" | yes
+**logLevel** | logging level | string | INFO | yes
+**partitionHost** | partition host | string | "http://partition" | yes
+**redisGroupHost** | redis group host | string | redis-group-master | yes
+**redisSearchHost** | redis search host | string | redis-search-master | yes
+**schemaHost** | schema host | string | "http://schema" | yes
+**securityHttpsCertificateTrust** | whether https is enabled | boolean | true | yes
+**springProfilesActive** | active spring profile | string | gcp | yes
+**storageHost** | storage host | string | "http://storage" | yes
+
+### GCP variables
+
+| Name | Description | Type | Default |Required |
+|------|-------------|------|---------|---------|
+**googleAudiences** | your GCP client ID | string | - | yes
+
+### Config variables
+
+| Name | Description | Type | Default |Required |
+|------|-------------|------|---------|---------|
+**configmap** | configmap to be used | string | indexer-config | yes
+**appName** | name of the app | string | indexer | yes
+**onPremEnabled** | whether on-prem is enabled | boolean | false | yes
+
+### Install the helm chart
+
+Run this command from within this directory:
+
+```console
+helm install gcp-indexer-configmap .
+```
+
+## Uninstalling the Chart
+
+To uninstall the helm deployment:
+
+```console
+helm uninstall gcp-indexer-configmap
+```
+
+[Move-to-Top](#configmap-helm-chart)
diff --git a/devops/gcp/configmap/templates/variables.yaml b/devops/gcp/configmap/templates/variables.yaml
index a429f4f988ec20540ac4c220ea0cdf785b07393e..0988de1435130fd0de537eb094eaf1518513b3d9 100644
--- a/devops/gcp/configmap/templates/variables.yaml
+++ b/devops/gcp/configmap/templates/variables.yaml
@@ -2,20 +2,20 @@ apiVersion: v1
kind: ConfigMap
metadata:
labels:
- app: "{{ .Values.conf.app_name }}"
+ app: "{{ .Values.conf.appName }}"
name: "{{ .Values.conf.configmap }}"
namespace: "{{ .Release.Namespace }}"
data:
- ENTITLEMENTS_HOST: "{{ .Values.data.entitlements_host }}"
- INDEXER_QUEUE_BASE_HOST: "{{ .Values.data.indexer_queue_base_host }}"
- LOG_LEVEL: "{{ .Values.data.log_level }}"
- PARTITION_HOST: "{{ .Values.data.partition_host }}"
- REDIS_GROUP_HOST: "{{ .Values.data.redis_group_host }}"
- REDIS_SEARCH_HOST: "{{ .Values.data.redis_search_host }}"
- SCHEMA_BASE_HOST: "{{ .Values.data.schema_base_host }}"
- SECURITY_HTTPS_CERTIFICATE_TRUST: "{{ .Values.data.security_https_certificate_trust }}"
- SPRING_PROFILES_ACTIVE: "{{ .Values.data.spring_profiles_active }}"
- STORAGE_HOST: "{{ .Values.data.storage_host }}"
- {{- if not .Values.conf.on_prem_enabled }}
- GOOGLE_AUDIENCES: "{{ .Values.data.google_audiences }}"
+ ENTITLEMENTS_HOST: "{{ .Values.data.entitlementsHost }}"
+ INDEXER_QUEUE_BASE_HOST: "{{ .Values.data.indexerQueueHost }}"
+ LOG_LEVEL: "{{ .Values.data.logLevel }}"
+ PARTITION_HOST: "{{ .Values.data.partitionHost }}"
+ REDIS_GROUP_HOST: "{{ .Values.data.redisGroupHost }}"
+ REDIS_SEARCH_HOST: "{{ .Values.data.redisSearchHost }}"
+ SCHEMA_BASE_HOST: "{{ .Values.data.schemaHost }}"
+ SECURITY_HTTPS_CERTIFICATE_TRUST: "{{ .Values.data.securityHttpsCertificateTrust }}"
+ SPRING_PROFILES_ACTIVE: "{{ .Values.data.springProfilesActive }}"
+ STORAGE_HOST: "{{ .Values.data.storageHost }}"
+ {{- if not .Values.conf.onPremEnabled }}
+ GOOGLE_AUDIENCES: "{{ .Values.data.googleAudiences }}"
{{- end }}
diff --git a/devops/gcp/configmap/values.yaml b/devops/gcp/configmap/values.yaml
index 340c112780c74c6bab88c1ec493bfde5c7c1a34f..84a3f7939a26e7df7c23ad4ada9d020cc49811cf 100644
--- a/devops/gcp/configmap/values.yaml
+++ b/devops/gcp/configmap/values.yaml
@@ -4,19 +4,19 @@
data:
# common
- entitlements_host: "http://entitlements"
- indexer_queue_base_host: "http://indexer-queue"
- log_level: "INFO"
- partition_host: "http://partition"
- redis_group_host: "redis-group-master"
- redis_search_host: "redis-search-master"
- schema_base_host: "http://schema"
- security_https_certificate_trust: "true"
- spring_profiles_active: "gcp"
- storage_host: "http://storage"
+ entitlementsHost: "http://entitlements"
+ indexerQueueHost: "http://indexer-queue"
+ logLevel: "INFO"
+ partitionHost: "http://partition"
+ redisGroupHost: "redis-group-master"
+ redisSearchHost: "redis-search-master"
+ schemaHost: "http://schema"
+ securityHttpsCertificateTrust: "true"
+ springProfilesActive: "gcp"
+ storageHost: "http://storage"
# gcp
- google_audiences: ""
+ googleAudiences: ""
conf:
- app_name: "indexer"
+ appName: "indexer"
configmap: "indexer-config"
- on_prem_enabled: false
+ onPremEnabled: false
diff --git a/devops/gcp/deploy/README.md b/devops/gcp/deploy/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..91af5795453aa4fce107ed48e2bd7a692dc92f0b
--- /dev/null
+++ b/devops/gcp/deploy/README.md
@@ -0,0 +1,70 @@
+<!--- Deploy -->
+
+# Deploy helm chart
+
+## Introduction
+
+This chart bootstraps a deployment on a [Kubernetes](https://kubernetes.io) cluster using [Helm](https://helm.sh) package manager.
+
+## Prerequisites
+
+The code was tested on **Kubernetes cluster** (v1.21.11) with **Istio** (1.12.6)
+> It is possible to use other versions, but it hasn't been tested
+
+### Operation system
+
+The code works in Debian-based Linux (Debian 10 and Ubuntu 20.04) and Windows WSL 2. Also, it works but is not guaranteed in Google Cloud Shell. All other operating systems, including macOS, are not verified and supported.
+
+### Packages
+
+Packages are only needed for installation from a local computer.
+
+* **HELM** (version: v3.7.1 or higher) [helm](https://helm.sh/docs/intro/install/)
+* **Kubectl** (version: v1.21.0 or higher) [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl)
+
+## Installation
+
+Before installing deploy Helm chart you need to install [configmap Helm chart](../configmap).
+First you need to set variables in **values.yaml** file using any code editor. Some of the values are prefilled, but you need to specify some values as well. You can find more information about them below.
+
+### Common variables
+
+| Name | Description | Type | Default |Required |
+|------|-------------|------|---------|---------|
+**requestsCpu** | amount of requested CPU | string | 0.1 | yes
+**requestsMemory** | amount of requested memory| string | 640M | yes
+**limitsCpu** | CPU limit | string | 1 | yes
+**limitsMemory** | memory limit | string | 1G | yes
+**image** | service image | string | - | yes
+**imagePullPolicy** | when to pull image | string | IfNotPresent | yes
+**serviceAccountName** | name of your service account | string | indexer | yes
+
+### Config variables
+
+| Name | Description | Type | Default |Required |
+|------|-------------|------|---------|---------|
+**appName** | name of the app | string | `indexer` | yes
+**configmap** | configmap to be used | string | `indexer-config` | yes
+**elasticSecretName** | secret for elastic | string | `indexer-elastic-secret` | yes
+**keycloakSecretName** | secret for keycloak | string | `indexer-keycloak-secret` | yes
+**rabbitmqSecretName** | secret for rabbitmq | string | `rabbitmq-secret` | yes
+**onPremEnabled** | whether on-prem is enabled | boolean | false | yes
+**domain** | your domain | string | - | yes
+
+### Install the helm chart
+
+Run this command from within this directory:
+
+```console
+helm install gcp-indexer-deploy .
+```
+
+## Uninstalling the Chart
+
+To uninstall the helm deployment:
+
+```console
+helm uninstall gcp-indexer-deploy
+```
+
+[Move-to-Top](#deploy-helm-chart)
diff --git a/devops/gcp/deploy/templates/deployment.yaml b/devops/gcp/deploy/templates/deployment.yaml
index 149c9f097b7c9abbd7d555246aaf7d55f5157cda..8c883af78718775e0258b30d0bfef0521f5a4fe2 100644
--- a/devops/gcp/deploy/templates/deployment.yaml
+++ b/devops/gcp/deploy/templates/deployment.yaml
@@ -2,35 +2,35 @@ apiVersion: apps/v1
kind: Deployment
metadata:
labels:
- app: "{{ .Values.conf.app_name }}"
- name: "{{ .Values.conf.app_name }}"
+ app: "{{ .Values.conf.appName }}"
+ name: "{{ .Values.conf.appName }}"
namespace: "{{ .Release.Namespace }}"
spec:
selector:
matchLabels:
- app: "{{ .Values.conf.app_name }}"
+ app: "{{ .Values.conf.appName }}"
replicas: 1
template:
metadata:
labels:
- app: "{{ .Values.conf.app_name }}"
+ app: "{{ .Values.conf.appName }}"
annotations:
rollme: {{ randAlphaNum 5 | quote }}
spec:
containers:
- - name: "{{ .Values.conf.app_name }}"
+ - name: "{{ .Values.conf.appName }}"
image: "{{ .Values.data.image }}"
imagePullPolicy: "{{ .Values.data.imagePullPolicy }}"
envFrom:
- configMapRef:
name: "{{ .Values.conf.configmap }}"
- secretRef:
- name: "{{ .Values.conf.elastic_secret_name }}"
- {{- if .Values.conf.on_prem_enabled }}
+ name: "{{ .Values.conf.elasticSecretName }}"
+ {{- if .Values.conf.onPremEnabled }}
- secretRef:
- name: "{{ .Values.conf.keycloak_secret_name }}"
+ name: "{{ .Values.conf.keycloakSecretName }}"
- secretRef:
- name: "{{ .Values.conf.rabbitmq_secret_name }}"
+ name: "{{ .Values.conf.rabbitmqSecretName }}"
{{- end }}
securityContext:
allowPrivilegeEscalation: false
@@ -39,9 +39,9 @@ spec:
- containerPort: 8080
resources:
requests:
- cpu: "{{ .Values.data.requests_cpu }}"
- memory: "{{ .Values.data.requests_memory }}"
+ cpu: "{{ .Values.data.requestsCpu }}"
+ memory: "{{ .Values.data.requestsMemory }}"
limits:
- cpu: "{{ .Values.data.limits_cpu }}"
- memory: "{{ .Values.data.limits_memory }}"
+ cpu: "{{ .Values.data.limitsCpu }}"
+ memory: "{{ .Values.data.limitsMemory }}"
serviceAccountName: "{{ .Values.data.serviceAccountName }}"
diff --git a/devops/gcp/deploy/templates/service-account.yaml b/devops/gcp/deploy/templates/service-account.yaml
index 3fede170cb014a4cfdf8cc99d745a32eb66aeed0..25fd74efbfca059f541af82d98b676cfcb930d64 100644
--- a/devops/gcp/deploy/templates/service-account.yaml
+++ b/devops/gcp/deploy/templates/service-account.yaml
@@ -1,4 +1,4 @@
-{{- if .Values.conf.on_prem_enabled }}
+{{- if .Values.conf.onPremEnabled }}
apiVersion: v1
kind: ServiceAccount
metadata:
diff --git a/devops/gcp/deploy/templates/service.yaml b/devops/gcp/deploy/templates/service.yaml
index 86a62664207433e99140a462d942c0dc80f5a68f..b7e02b2ed551ad16a96b9894dad08399ddc4d02e 100644
--- a/devops/gcp/deploy/templates/service.yaml
+++ b/devops/gcp/deploy/templates/service.yaml
@@ -1,11 +1,11 @@
apiVersion: v1
kind: Service
metadata:
- {{- if not .Values.conf.on_prem_enabled }}
+ {{- if not .Values.conf.onPremEnabled }}
annotations:
cloud.google.com/neg: '{"ingress": true}'
{{- end }}
- name: "{{ .Values.conf.app_name }}"
+ name: "{{ .Values.conf.appName }}"
namespace: "{{ .Release.Namespace }}"
spec:
ports:
@@ -14,4 +14,4 @@ spec:
targetPort: 8080
name: http
selector:
- app: "{{ .Values.conf.app_name }}"
+ app: "{{ .Values.conf.appName }}"
diff --git a/devops/gcp/deploy/templates/virtual-service.yaml b/devops/gcp/deploy/templates/virtual-service.yaml
index 321945b4a9cb67789462a280436258fdcb5eac49..30ebb98ae58802b79ad8145f12d195ba05ac392c 100644
--- a/devops/gcp/deploy/templates/virtual-service.yaml
+++ b/devops/gcp/deploy/templates/virtual-service.yaml
@@ -1,12 +1,14 @@
apiVersion: networking.istio.io/v1alpha3
kind: VirtualService
metadata:
- name: "{{ .Values.conf.app_name }}"
+ name: "{{ .Values.conf.appName }}"
namespace: "{{ .Release.Namespace }}"
spec:
hosts:
- {{- if .Values.conf.domain }}
+ {{- if and .Values.conf.domain .Values.conf.onPremEnabled }}
- {{ printf "osdu.%s" .Values.conf.domain | quote }}
+ {{- else if .Values.conf.domain }}
+ - {{ .Values.conf.domain | quote }}
{{- else }}
- "*"
{{- end }}
@@ -20,4 +22,4 @@ spec:
- destination:
port:
number: 80
- host: "{{ .Values.conf.app_name }}.{{ .Release.Namespace }}.svc.cluster.local"
+ host: "{{ .Values.conf.appName }}.{{ .Release.Namespace }}.svc.cluster.local"
diff --git a/devops/gcp/deploy/values.yaml b/devops/gcp/deploy/values.yaml
index fd467615a4a44eec72f87ca9ca3b65e403057173..2ef4cb8867cc264cf10b3f298f7629042eddb692 100644
--- a/devops/gcp/deploy/values.yaml
+++ b/devops/gcp/deploy/values.yaml
@@ -3,18 +3,18 @@
# Declare variables to be passed into your templates.
data:
- requests_cpu: "0.1"
- requests_memory: "640M"
- limits_cpu: "1"
- limits_memory: "1G"
+ requestsCpu: "0.1"
+ requestsMemory: "640M"
+ limitsCpu: "1"
+ limitsMemory: "1G"
image: ""
imagePullPolicy: "IfNotPresent"
serviceAccountName: "indexer"
conf:
- app_name: "indexer"
+ appName: "indexer"
configmap: "indexer-config"
- elastic_secret_name: "indexer-elastic-secret"
- keycloak_secret_name: "indexer-keycloak-secret"
- on_prem_enabled: false
- rabbitmq_secret_name: "rabbitmq-secret"
+ elasticSecretName: "indexer-elastic-secret"
+ keycloakSecretName: "indexer-keycloak-secret"
+ rabbitmqSecretName: "rabbitmq-secret"
+ onPremEnabled: false
domain: ""
diff --git a/indexer-core/pom.xml b/indexer-core/pom.xml
index fee139eb8a5ae914d5d71e8d95bec105a9604dde..1610d9e90a746b0bc994bbc8afd5b77334c6edd4 100644
--- a/indexer-core/pom.xml
+++ b/indexer-core/pom.xml
@@ -232,21 +232,27 @@
<version>2.7</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.codehaus.plexus</groupId>
+ <artifactId>plexus-utils</artifactId>
+ <version>3.0.16</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
- <build>
- <defaultGoal>test</defaultGoal>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <version>2.4.2</version>
- <configuration>
- <useSystemClassLoader>false</useSystemClassLoader>
- <threadCount>1</threadCount>
- </configuration>
- </plugin>
- </plugins>
- </build>
+ <build>
+ <defaultGoal>test</defaultGoal>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.4.2</version>
+ <configuration>
+ <useSystemClassLoader>false</useSystemClassLoader>
+ <threadCount>1</threadCount>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
</project>
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/geojson/Position.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/geojson/Position.java
index d578de28d0a6d9e8dbb13f30bd4dfaf1a8aa60de..89762041a6b50de0b9344dd8e51317da5782d201 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/geojson/Position.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/geojson/Position.java
@@ -18,9 +18,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
-import lombok.AllArgsConstructor;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
+import lombok.*;
import org.opengroup.osdu.indexer.model.geojson.jackson.PositionDeserializer;
import org.opengroup.osdu.indexer.model.geojson.jackson.PositionSerializer;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/geojson/jackson/FeatureCollectionDeserializer.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/geojson/jackson/FeatureCollectionDeserializer.java
index dc9504be28c85d915aca6ce40af798aedfe480c6..390229859f06d35bbe3f4f5d27950521f7f614c6 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/geojson/jackson/FeatureCollectionDeserializer.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/geojson/jackson/FeatureCollectionDeserializer.java
@@ -14,6 +14,7 @@
package org.opengroup.osdu.indexer.model.geojson.jackson;
+import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.ObjectCodec;
import com.fasterxml.jackson.databind.DeserializationContext;
@@ -39,6 +40,11 @@ public class FeatureCollectionDeserializer extends JsonDeserializer<FeatureColle
ObjectCodec codec = jsonParser.getCodec();
JsonNode featureCollection = codec.readTree(jsonParser);
JsonNode features = featureCollection.get("features");
+
+ if(features == null){
+ throw new JsonParseException(jsonParser, "Missing feature field in the ");
+ }
+
final List<Feature> result = new ArrayList<>();
for (JsonNode node : features) {
result.add(codec.treeToValue(node, Feature.class));
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
index 63ea7062d7da8701d788aa231758386960fee115..d04a2c9d449d7b58387443df241ee3a51a221755 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
@@ -66,7 +66,7 @@ public class IndexerServiceImpl implements IndexerService {
private static final TimeValue BULK_REQUEST_TIMEOUT = TimeValue.timeValueMinutes(1);
- private static final List<RestStatus> RETRY_ELASTIC_EXCEPTION = new ArrayList<>(Arrays.asList(RestStatus.TOO_MANY_REQUESTS, RestStatus.BAD_GATEWAY, RestStatus.SERVICE_UNAVAILABLE));
+ private static final List<RestStatus> RETRY_ELASTIC_EXCEPTION = new ArrayList<>(Arrays.asList(RestStatus.TOO_MANY_REQUESTS, RestStatus.BAD_GATEWAY, RestStatus.SERVICE_UNAVAILABLE, RestStatus.FORBIDDEN));
private final Gson gson = new GsonBuilder().serializeNulls().create();
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java
index 25f5945654edc51b92cd8a53099298dca6d59d5f..db5a2f37a19857fd609da3fc3abdc953c5889fd8 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java
@@ -33,6 +33,7 @@ import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.CreateIndexRequest;
import org.elasticsearch.client.indices.CreateIndexResponse;
import org.elasticsearch.client.indices.GetIndexRequest;
+import org.elasticsearch.client.indices.GetIndexResponse;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentType;
@@ -50,6 +51,7 @@ import org.springframework.web.context.annotation.RequestScope;
import java.io.IOException;
import java.lang.reflect.Type;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -194,7 +196,11 @@ public class IndicesServiceImpl implements IndicesService {
* @param index Index name
*/
public boolean deleteIndex(RestHighLevelClient client, String index) throws ElasticsearchException, IOException, AppException {
- boolean responseStatus = removeIndexInElasticsearch(client, index);
+ List<String> indices = this.resolveIndex(client, index);
+ boolean responseStatus = true;
+ for (String idx: indices) {
+ responseStatus &= removeIndexInElasticsearch(client, idx);
+ }
if (responseStatus) {
this.clearCacheOnIndexDeletion(index);
}
@@ -275,4 +281,26 @@ public class IndicesServiceImpl implements IndicesService {
this.indexCache.delete(index);
this.indexCache.delete(syncCacheKey);
}
+
+ public List<String> resolveIndex(RestHighLevelClient client, String index) throws IOException {
+
+ Preconditions.checkArgument(client, Objects::nonNull, "client cannot be null");
+ Preconditions.checkArgument(index, Objects::nonNull, "index cannot be null");
+
+ try {
+ GetIndexRequest request = new GetIndexRequest(index);
+ request.setTimeout(REQUEST_TIMEOUT);
+ GetIndexResponse getIndexResponse = client.indices().get(request, RequestOptions.DEFAULT);
+ String[] indices = getIndexResponse.getIndices();
+ if (indices != null && indices.length != 0) {
+ return Arrays.asList(indices);
+ }
+ throw new AppException(HttpStatus.SC_NOT_FOUND, "Index deletion error", notFoundErrorMessage(index));
+ } catch (ElasticsearchException exception) {
+ if (exception.status() == RestStatus.NOT_FOUND) {
+ throw new AppException(HttpStatus.SC_NOT_FOUND, "Index deletion error", notFoundErrorMessage(index), exception);
+ }
+ throw exception;
+ }
+ }
}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/MappingServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/MappingServiceImpl.java
index 566de1d5bc7aff30d3c166bf69fa529834e48403..5e58b482108f9919011d68fcd76199789084f4ed 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/MappingServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/MappingServiceImpl.java
@@ -21,6 +21,7 @@ import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.GetMappingsRequest;
import org.elasticsearch.client.indices.GetMappingsResponse;
+import org.elasticsearch.cluster.metadata.MappingMetadata;
import org.elasticsearch.common.unit.TimeValue;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
@@ -34,6 +35,7 @@ import java.io.IOException;
import java.lang.reflect.Type;
import java.util.Map;
import java.util.Objects;
+import java.util.Optional;
@Service
@RequestScope
@@ -87,8 +89,12 @@ public abstract class MappingServiceImpl implements IMappingService {
request.indices(index);
request.setTimeout(REQUEST_TIMEOUT);
GetMappingsResponse response = client.indices().getMapping(request, RequestOptions.DEFAULT);
+ if (response.mappings().isEmpty()) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Unknown error", String.format("Error retrieving mapping for kind %s", this.elasticIndexNameResolver.getKindFromIndexName(index)));
+ }
Type type = new TypeToken<Map<String, Object>>() {}.getType();
- return new Gson().toJson(response.mappings().get(index).getSourceAsMap(), type);
+ Optional<MappingMetadata> mapping = response.mappings().values().stream().findFirst();
+ return new Gson().toJson(mapping.get().getSourceAsMap(), type);
} catch (IOException e) {
throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Unknown error", String.format("Error retrieving mapping for kind %s", this.elasticIndexNameResolver.getKindFromIndexName(index)), e);
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/parser/GeoShapeParser.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/parser/GeoShapeParser.java
index 8bd6ec834dd72dc399c9ea7b65a4772a771d959c..3674ca7950005cfe73963d84e0f86f5985103469 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/parser/GeoShapeParser.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/parser/GeoShapeParser.java
@@ -14,6 +14,7 @@
package org.opengroup.osdu.indexer.util.parser;
+import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -42,6 +43,8 @@ public class GeoShapeParser {
});
} catch (InvalidTypeIdException e) {
throw new IllegalArgumentException("must be a valid FeatureCollection");
+ } catch (JsonParseException e){
+ throw new IllegalArgumentException(e.getMessage());
} catch (JsonProcessingException e) {
throw new IllegalArgumentException("unable to parse FeatureCollection");
}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndicesServiceTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndicesServiceTest.java
index 17bce5783d76f793e848268d9f6a68de7f105a6a..8fb08c6e6ac0be3e3116cec674108af1a6f8231d 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndicesServiceTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndicesServiceTest.java
@@ -27,6 +27,7 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.*;
import org.elasticsearch.client.indices.GetIndexRequest;
+import org.elasticsearch.client.indices.GetIndexResponse;
import org.elasticsearch.rest.RestStatus;
import org.junit.Before;
import org.junit.Test;
@@ -89,10 +90,14 @@ public class IndicesServiceTest {
@Test
public void delete_existingElasticIndex() throws Exception {
AcknowledgedResponse indexResponse = new AcknowledgedResponse(true);
+ GetIndexResponse getIndexResponse = PowerMockito.mock(GetIndexResponse.class);
+ String[] indices = {"anyIndex"};
when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
doReturn(indicesClient).when(restHighLevelClient).indices();
doReturn(indexResponse).when(indicesClient).delete(any(), any(RequestOptions.class));
+ doReturn(getIndexResponse).when(indicesClient).get(any(GetIndexRequest.class), any(RequestOptions.class));
+ doReturn(indices).when(getIndexResponse).getIndices();
boolean response = this.sut.deleteIndex("anyIndex");
assertTrue(response);
}
@@ -100,10 +105,14 @@ public class IndicesServiceTest {
@Test
public void delete_existingElasticIndex_usingSameClient() throws Exception {
AcknowledgedResponse indexResponse = new AcknowledgedResponse(true);
+ GetIndexResponse getIndexResponse = PowerMockito.mock(GetIndexResponse.class);
+ String[] indices = {"anyIndex"};
when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
doReturn(indicesClient).when(restHighLevelClient).indices();
doReturn(indexResponse).when(indicesClient).delete(any(), any(RequestOptions.class));
+ doReturn(getIndexResponse).when(indicesClient).get(any(GetIndexRequest.class), any(RequestOptions.class));
+ doReturn(indices).when(getIndexResponse).getIndices();
boolean response = this.sut.deleteIndex(restHighLevelClient, "anyIndex");
assertTrue(response);
}
@@ -111,9 +120,13 @@ public class IndicesServiceTest {
@Test
public void should_throw_internalServerException_delete_isNotAcknowledged() throws Exception {
AcknowledgedResponse indexResponse = new AcknowledgedResponse(false);
+ GetIndexResponse getIndexResponse = PowerMockito.mock(GetIndexResponse.class);
+ String[] indices = {"anyIndex"};
when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
doReturn(indicesClient).when(restHighLevelClient).indices();
doReturn(indexResponse).when(indicesClient).delete(any(), any(RequestOptions.class));
+ doReturn(getIndexResponse).when(indicesClient).get(any(GetIndexRequest.class), any(RequestOptions.class));
+ doReturn(indices).when(getIndexResponse).getIndices();
try {
this.sut.deleteIndex("anyIndex");
@@ -131,9 +144,13 @@ public class IndicesServiceTest {
public void should_throwAppException_when_delete_existingElasticIndex_and_backupIsRunning() throws Exception {
ElasticsearchStatusException exception = new ElasticsearchStatusException(
"Cannot delete indices that are being snapshotted: [[anyIndex/8IXuPeFnTJGEu_LjjXrHwA]]. Try again after snapshot finishes or cancel the currently running snapshot.", RestStatus.BAD_REQUEST);
+ GetIndexResponse getIndexResponse = PowerMockito.mock(GetIndexResponse.class);
+ String[] indices = {"anyIndex"};
when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
doReturn(indicesClient).when(restHighLevelClient).indices();
doThrow(exception).when(indicesClient).delete(any(), any(RequestOptions.class));
+ doReturn(getIndexResponse).when(indicesClient).get(any(GetIndexRequest.class), any(RequestOptions.class));
+ doReturn(indices).when(getIndexResponse).getIndices();
try {
this.sut.deleteIndex("anyIndex");
@@ -152,7 +169,7 @@ public class IndicesServiceTest {
ElasticsearchStatusException exception = new ElasticsearchStatusException("no such index", RestStatus.NOT_FOUND);
when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
doReturn(indicesClient).when(restHighLevelClient).indices();
- doThrow(exception).when(indicesClient).delete(any(), any(RequestOptions.class));
+ doThrow(exception).when(indicesClient).get(any(GetIndexRequest.class), any(RequestOptions.class));
try {
this.sut.deleteIndex("anyIndex");
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/parser/GeoShapeParserTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/parser/GeoShapeParserTest.java
index 2b47f5ab3a525cdceccd5b65d7f055f51c7468cb..fdec7478428205923da22ffc9d7d53040845830f 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/parser/GeoShapeParserTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/parser/GeoShapeParserTest.java
@@ -34,6 +34,7 @@ import java.util.function.Function;
import static org.hamcrest.CoreMatchers.containsString;
import static org.junit.Assert.*;
+import static org.junit.jupiter.api.Assertions.assertThrows;
@RunWith(SpringRunner.class)
public class GeoShapeParserTest {
@@ -178,6 +179,16 @@ public class GeoShapeParserTest {
this.validateInput(this.sut::parseGeoJson, shapeJson, expectedParsedShape, Strings.EMPTY);
}
+ @Test
+ public void should_throwException_parseInvalidFeatureCollection() {
+ String shapeJson = getGeoShapeFromFile("input/invalid_feature_collection.json");
+
+ Type type = new TypeToken<Map<String, Object>>() {}.getType();
+ Map<String, Object> shapeObj = new Gson().fromJson(shapeJson, type);
+
+ assertThrows(IllegalArgumentException.class, () -> this.sut.parseGeoJson(shapeObj));
+ }
+
private void validateInput(Function<Map<String, Object>, Map<String, Object>> parser, String shapeJson, String expectedParsedShape, String errorMessage) {
try {
Type type = new TypeToken<Map<String, Object>>() {}.getType();
diff --git a/indexer-core/src/test/resources/geojson/parsing/input/invalid_feature_collection.json b/indexer-core/src/test/resources/geojson/parsing/input/invalid_feature_collection.json
new file mode 100644
index 0000000000000000000000000000000000000000..5a91d57a7aa494faac5ab1f04bca66ca6873ded7
--- /dev/null
+++ b/indexer-core/src/test/resources/geojson/parsing/input/invalid_feature_collection.json
@@ -0,0 +1,15 @@
+{
+ "type": "FeatureCollection",
+ "featur3es": [
+ {
+ "type": "Feature",
+ "geometry": {
+ "type": "Point",
+ "coordinates": [
+ -105.01621,
+ 39.57422
+ ]
+ }
+ }
+ ]
+}
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index 03564d40d08e5c8afd1a713982accb6a04d13b79..3907b9ac819906e948e7252025559435a8bd112f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -184,7 +184,6 @@
<module>provider/indexer-azure</module>
<module>provider/indexer-gcp</module>
<module>provider/indexer-ibm</module>
- <module>provider/indexer-reference</module>
</modules>
<repositories>
diff --git a/provider/indexer-azure/pom.xml b/provider/indexer-azure/pom.xml
index 00611c1627bfb3fff639819b9a45d390cc63a796..bd531416054177f557b65de1560435d1ed9dd98a 100644
--- a/provider/indexer-azure/pom.xml
+++ b/provider/indexer-azure/pom.xml
@@ -51,7 +51,6 @@
<spring-boot.version>2.1.18.RELEASE</spring-boot.version>
<reactor-netty.version>1.0.7</reactor-netty.version>
<reactor-core.version>3.4.6</reactor-core.version>
- <jackson-dataformat-xml.version>2.11.4</jackson-dataformat-xml.version>
<oauth2-oidc-sdk.version>6.5</oauth2-oidc-sdk.version>
<jackson-databind.version>2.13.2.2</jackson-databind.version>
<jackson.version>2.13.2</jackson.version>
@@ -82,16 +81,6 @@
<artifactId>hibernate-validator</artifactId>
</dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
-
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
@@ -202,11 +191,6 @@
</exclusion>
</exclusions>
</dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.dataformat</groupId>
- <artifactId>jackson-dataformat-cbor</artifactId>
- <version>2.11.4</version>
- </dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
@@ -216,7 +200,6 @@
<artifactId>elasticsearch-rest-high-level-client</artifactId>
</dependency>
- <!-- Jackson explicit Dependencies-->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
@@ -232,6 +215,11 @@
<artifactId>jackson-annotations</artifactId>
<version>${jackson.version}</version>
</dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.dataformat</groupId>
+ <artifactId>jackson-dataformat-cbor</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
<!-- Resilience4j Dependencies-->
<dependency>
@@ -316,6 +304,12 @@
<version>${cobertura-maven-plugin.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.codehaus.plexus</groupId>
+ <artifactId>plexus-utils</artifactId>
+ <version>3.0.16</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>io.projectreactor.netty</groupId>
<artifactId>reactor-netty</artifactId>
@@ -329,7 +323,7 @@
<dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-xml</artifactId>
- <version>${jackson-dataformat-xml.version}</version>
+ <version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>com.nimbusds</groupId>
diff --git a/provider/indexer-gcp/docker/Dockerfile b/provider/indexer-gcp/docker/Dockerfile
deleted file mode 100644
index 926302723f4248e1991f8ea9d2d550367903347f..0000000000000000000000000000000000000000
--- a/provider/indexer-gcp/docker/Dockerfile
+++ /dev/null
@@ -1,6 +0,0 @@
-FROM openjdk:8-slim
-VOLUME /tmp
-ARG JAR_FILE
-COPY ${JAR_FILE} app.jar
-ENTRYPOINT ["java","-Djava.security.egd=file:/dev/./urandom","-jar","-Dspring.profiles.active=kuber","/app.jar"]
-EXPOSE 8080
diff --git a/provider/indexer-gcp/docker/docker-compose.yml b/provider/indexer-gcp/docker/docker-compose.yml
deleted file mode 100644
index 4d9b01efd895b740252b7b42ccd666264c841909..0000000000000000000000000000000000000000
--- a/provider/indexer-gcp/docker/docker-compose.yml
+++ /dev/null
@@ -1,12 +0,0 @@
-version: "3"
-services:
- os-indexer-app:
- build:
- args:
- JAR_FILE: target/indexer-gcp-1.0.4-SNAPSHOT-spring-boot.jar
- context: ..
- dockerfile: docker/Dockerfile
- image: gcr.io/opendes/os-indexer-app
- ports:
- - "8080:8080"
-
diff --git a/provider/indexer-gcp/pom.xml b/provider/indexer-gcp/pom.xml
index e1d0699e5975fc470e97d525349060e2161a6cc3..011a618f99bba5b42595777657594c1d33d9fa26 100644
--- a/provider/indexer-gcp/pom.xml
+++ b/provider/indexer-gcp/pom.xml
@@ -96,6 +96,12 @@
<version>2.7</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.codehaus.plexus</groupId>
+ <artifactId>plexus-utils</artifactId>
+ <version>3.0.16</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>com.auth0</groupId>
diff --git a/provider/indexer-gcp/src/main/resources/application-anthos.properties b/provider/indexer-gcp/src/main/resources/application-anthos.properties
index 9e553661404fae2ae946a221d506b5b56d16f8a0..67e63d39f7862bed24f541b7594661665b2214b1 100644
--- a/provider/indexer-gcp/src/main/resources/application-anthos.properties
+++ b/provider/indexer-gcp/src/main/resources/application-anthos.properties
@@ -3,4 +3,4 @@ service.token.provider=OPENID
partition-auth-enabled=false
openid.provider-url=
openid.provider-client-id=
-openid.provider-client-secret=
\ No newline at end of file
+openid.provider-client-secret=
diff --git a/provider/indexer-gcp/src/main/resources/application-gcp.properties b/provider/indexer-gcp/src/main/resources/application-gcp.properties
index fd738d7f484bf44fc3b2f2043191259dd0b3a1b8..8ac15c3f94a0bb7addf3ffcd4fead8bcbd2ffa5e 100644
--- a/provider/indexer-gcp/src/main/resources/application-gcp.properties
+++ b/provider/indexer-gcp/src/main/resources/application-gcp.properties
@@ -1,3 +1,3 @@
oqmDriver=pubsub
service.token.provider=GCP
-partition-auth-enabled=true
\ No newline at end of file
+partition-auth-enabled=true
diff --git a/provider/indexer-ibm/pom.xml b/provider/indexer-ibm/pom.xml
index 1c1c36c04209c955a64520d1154d6a1bd845ce3d..0d3326b4e8af00268aa567b7600e9249a4323ee0 100644
--- a/provider/indexer-ibm/pom.xml
+++ b/provider/indexer-ibm/pom.xml
@@ -31,7 +31,8 @@
<packaging>jar</packaging>
<properties>
- <os-core-lib-ibm.version>0.15.2</os-core-lib-ibm.version>
+ <tomcat.embed.core.version>9.0.54</tomcat.embed.core.version>
+ <os-core-lib-ibm.version>0.16.0-rc1</os-core-lib-ibm.version>
</properties>
<profiles>
@@ -59,8 +60,27 @@
<groupId>io.netty</groupId>
<artifactId>netty-codec</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.tomcat.embed</groupId>
+ <artifactId>tomcat-embed-core</artifactId>
+ </exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.apache.tomcat.embed</groupId>
+ <artifactId>tomcat-embed-core</artifactId>
+ <version>${tomcat.embed.core.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-web</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-tomcat</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec</artifactId>
diff --git a/provider/indexer-reference/Dockerfile b/provider/indexer-reference/Dockerfile
deleted file mode 100644
index 1e6901f365e2e9c7049c5019b4037ade931b65f0..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/Dockerfile
+++ /dev/null
@@ -1,5 +0,0 @@
-FROM openjdk:8-slim
-WORKDIR /app
-COPY target/indexer-reference-0.0.1-SNAPSHOT-spring-boot.jar indexer-reference.jar
-# Run the web service on container startup.
-CMD java -Djava.security.egd=file:/dev/./urandom -Dserver.port=8080 -jar /app/indexer-reference.jar
diff --git a/provider/indexer-reference/README.md b/provider/indexer-reference/README.md
deleted file mode 100644
index 13529b214dccc308fa01f9439efeb5bc07274047..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/README.md
+++ /dev/null
@@ -1,219 +0,0 @@
-# Indexer Service
-os-indexer-reference is a [Spring Boot](https://spring.io/projects/spring-boot) service that is responsible for indexing Records that enable the `os-search` service to execute OSDU hybrid cloud searches against Elasticsearch.
-
-## Getting Started
-These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system.
-
-### Prerequisites
-Pre-requisites
-
-* GCloud SDK with java (latest version)
-* JDK 8
-* Lombok 1.16 or later
-* Maven
-
-### Installation
-In order to run the service locally or remotely, you will need to have the following environment variables defined.
-
-| name | value | description | sensitive? | source |
-| --- | --- | --- | --- | --- |
-| `LOG_PREFIX` | `service` | Logging prefix | no | - |
-| `SERVER_SERVLET_CONTEXPATH` | `/api/indexer/v2` | Servlet context path | no | - |
-| `AUTHORIZE_API` | ex `https://entitlements.com/entitlements/v1` | Entitlements API endpoint | no | output of infrastructure deployment |
-| `LEGALTAG_API` | ex `https://legal.com/api/legal/v1` | Legal API endpoint | no | output of infrastructure deployment |
-| `INDEXER_QUEUE_HOST` | ex `https://os-indexer-queue-dot-opendes.appspot.com/_dps/task-handlers/enqueue` | Indexer-Queue API endpoint | no | output of infrastructure deployment |
-| `CRS_API` | ex `https://crs-converter-gae-dot-opendes.appspot.com/api/crs/v1` | CRS API endpoint | no | https://console.cloud.google.com/memorystore/redis/instances |
-| `STORAGE_HOSTNAME` | ex `os-storage-dot-opendes.appspot.com` | Storage Host | no | output of infrastructure deployment |
-| `STORAGE_SCHEMA_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/schemas` | Storage API endpoint 'schemas' | no | https://console.cloud.google.com/apis/credentials |
-| `STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records:batch` | Storage API endpoint 'records' | no | https://console.cloud.google.com/iam-admin/serviceaccounts |
-| `STORAGE_QUERY_RECORD_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records` | Storage API endpoint 'query/records' | no | https://console.cloud.google.com/iam-admin/serviceaccounts |
-| `STORAGE_RECORDS_BATCH_SIZE` | 20 | Batch size for storage API `POST {endpoint}/query/records:batch` | no | - |
-| `STORAGE_RECORDS_BY_KIND_BATCH_SIZE` | - | Batch size for storage API `GET {endpoint}/query/records`. If this is not present, defaults to value of `STORAGE_RECORDS_BATCH_SIZE` | no | - |
-| `REDIS_SEARCH_HOST` | ex `127.0.0.1` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances |
-| `REDIS_GROUP_HOST` | ex `127.0.0.1` | Redis host for groups | no | https://console.cloud.google.com/memorystore/redis/instances |
-| `REDIS_SEARCH_PORT` | ex `6379` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances |
-| `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials |
-| `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
-| `security.https.certificate.trust` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment |
-| `indexer.que.service.mail` | ex `default@iam.gserviceaccount.com` | Indexer Que environment service account mail, required if Indexer Que deployed in cloud task mode, to validate token from it | yes | - |
-| `SCHEMA_HOST` | ex `https://os-schema-dot-opendes.appspot.com/api/schema-service/v1/schema` | Schema API endpoint | no | output of infrastructure deployment |
-| `PARTITION_API` | ex `https://localhost:8081/api/partition/v1` | Partition API endpoint | no | output of infrastructure deployment |
-| `MONGO_DB_URL` | ex `mongodb://localhost:27017` | Mongo DB Url| yes | output of infrastructure deployment |
-| `MONGO_DB_USER` | ex `mongouser` | Mongo DB userName| yes | output of infrastructure deployment |
-| `MONGO_DB_PASSWORD` | ex `mongopassword` | Mongo DB userPassword| yes | output of infrastructure deployment |
-| `MONGO_DB_NAME` | ex `mongoDBName` | Mongo DB DbName| yes | output of infrastructure deployment |
-| `MB_RABBITMQ_URI` | ex `amqp://guest:guest@127.0.0.1:5672` | MessageBroker RabbitMQ URI | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
-
-### Run Locally
-Check that maven is installed:
-
-```bash
-$ mvn --version
-Apache Maven 3.6.0
-Maven home: /usr/share/maven
-Java version: 1.8.0_212, vendor: AdoptOpenJDK, runtime: /usr/lib/jvm/jdk8u212-b04/jre
-...
-```
-
-You may need to configure access to the remote maven repository that holds the OSDU dependencies. This file should live within `~/.mvn/community-maven.settings.xml`:
-
-```bash
-$ cat ~/.m2/settings.xml
-<?xml version="1.0" encoding="UTF-8"?>
-<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
- <servers>
- <server>
- <id>community-maven-via-private-token</id>
- <!-- Treat this auth token like a password. Do not share it with anyone, including Microsoft support. -->
- <!-- The generated token expires on or before 11/14/2019 -->
- <configuration>
- <httpHeaders>
- <property>
- <name>Private-Token</name>
- <value>${env.COMMUNITY_MAVEN_TOKEN}</value>
- </property>
- </httpHeaders>
- </configuration>
- </server>
- </servers>
-</settings>
-```
-
-* Update the Google cloud SDK to the latest version:
-
-```bash
-gcloud components update
-```
-* Set Google Project Id:
-
-```bash
-gcloud config set project <YOUR-PROJECT-ID>
-```
-
-* Perform a basic authentication in the selected project:
-
-```bash
-gcloud auth application-default login
-```
-
-* Navigate to indexer service's root folder and run:
-
-```bash
-mvn jetty:run
-## Testing
-* Navigate to indexer service's root folder and run:
-
-```bash
-mvn clean install
-```
-
-* If you wish to see the coverage report then go to testing/target/site/jacoco-aggregate and open index.html
-
-* If you wish to build the project without running tests
-
-```bash
-mvn clean install -DskipTests
-```
-
-After configuring your environment as specified above, you can follow these steps to build and run the application. These steps should be invoked from the *repository root.*
-
-```bash
-cd provider/indexer-reference/ && mvn spring-boot:run
-```
-
-## Testing
-Navigate to indexer service's root folder and run all the tests:
-
-```bash
-# build + install integration test core
-$ (cd testing/indexer-test-core/ && mvn clean install)
-```
-
-### Running E2E Tests
-This section describes how to run cloud OSDU E2E tests (testing/integration-tests/indexer-test-gcp).
-
-You will need to have the following environment variables defined.
-
-| name | value | description | sensitive? | source |
-| --- | --- | --- | --- | --- |
-| `ELASTIC_PASSWORD` | `********` | Password for Elasticsearch | yes | output of infrastructure deployment |
-| `ELASTIC_USER_NAME` | `********` | User name for Elasticsearch | yes | output of infrastructure deployment |
-| `ELASTIC_HOST` | ex `elastic.domain.com` | Host Elasticsearch | yes | output of infrastructure deployment |
-| `ELASTIC_PORT` | ex `9243` | Port Elasticsearch | yes | output of infrastructure deployment |
-| `GCLOUD_PROJECT` | ex `opendes` | Google Cloud Project Id| no | output of infrastructure deployment |
-| `INDEXER_HOST` | ex `https://os-indexer-dot-opendes.appspot.com/api/indexer/v2/` | Indexer API endpoint | no | output of infrastructure deployment |
-| `ENTITLEMENTS_DOMAIN` | ex `opendes-gcp.projects.com` | OSDU R2 to run tests under | no | - |
-| `INTEGRATION_TEST_AUDIENCE` | `********` | client application ID | yes | https://console.cloud.google.com/apis/credentials |
-| `OTHER_RELEVANT_DATA_COUNTRIES` | ex `US` | valid legal tag with a other relevant data countries | no | - |
-| `LEGAL_TAG` | ex `opendes-demo-legaltag` | valid legal tag with a other relevant data countries from `DEFAULT_OTHER_RELEVANT_DATA_COUNTRIES` | no | - |
-| `DEFAULT_DATA_PARTITION_ID_TENANT1` | ex `opendes` | HTTP Header 'Data-Partition-ID' | no | - |
-| `SEARCH_INTEGRATION_TESTER` | `********` | Service account for API calls. Note: this user must have entitlements configured already | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
-| `SEARCH_HOST` | ex `http://localhost:8080/api/search/v2/` | Endpoint of search service | no | - |
-| `STORAGE_HOST` | ex `http://os-storage-dot-opendes.appspot.com/api/storage/v2/` | Storage API endpoint | no | output of infrastructure deployment |
-| `SECURITY_HTTPS_CERTIFICATE_TRUST` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment |
-
-**Entitlements configuration for integration accounts**
-
-| INTEGRATION_TESTER | NO_DATA_ACCESS_TESTER |
-| --- | --- |
-| users<br/>service.entitlements.user<br/>service.search.user<br/>service.search.admin<br/>data.test1<br/>data.integration.test<br/>users@{tenant1}@{domain}.com |
-
-Execute following command to build code and run all the integration tests:
-
-```bash
-# Note: this assumes that the environment variables for integration tests as outlined
-# above are already exported in your environment.
-$ (cd testing/indexer-test-gcp/ && mvn clean test)
-```
-
-## Deployment
-GKE Google Documentation: https://cloud.google.com/build/docs/deploying-builds/deploy-gke
-Anthos Google Documentation: https://cloud.google.com/anthos/multicluster-management/gateway/tutorials/cloud-build-integration
-
-#### Cloud KMS Setup
-
-Enable cloud KMS on master project
-
-Create king ring and key in the ***master project***
-
-```bash
- gcloud services enable cloudkms.googleapis.com
- export KEYRING_NAME="csqp"
- export CRYPTOKEY_NAME="searchService"
- gcloud kms keyrings create $KEYRING_NAME --location global
- gcloud kms keys create $CRYPTOKEY_NAME --location global \
- --keyring $KEYRING_NAME \
- --purpose encryption
-```
-
-Add **Cloud KMS CryptoKey Encrypter/Decrypter** role to the **default service account** of the ***master project*** through IAM - Role tab
-
-Add **Cloud KMS Encrypt/Decrypt** role to the **default service account** of ***master project*** through IAM - Role tab
-
-#### Memory Store (Redis Instance) Setup
-
-Create a new Standard tier Redis instance on the ***service project***
-
-The Redis instance must be created under the same region with the App Engine application which needs to access it.
-
-```bash
- gcloud beta redis instances create redis-cache-search --size=10 --region=<service-deployment-region> --zone=<service-deployment-zone> --tier=STANDARD
-```
-
-## Licence
-Copyright © Google LLC
-Copyright © EPAM Systems
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-[http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0)
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
\ No newline at end of file
diff --git a/provider/indexer-reference/docker-compose.yml b/provider/indexer-reference/docker-compose.yml
deleted file mode 100644
index a6838447de241e658325e275434293b2d4e2be09..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/docker-compose.yml
+++ /dev/null
@@ -1,12 +0,0 @@
-version: "3"
-services:
- os-storage-app:
- build:
- args:
- JAR_FILE: target/indexer-reference-0.0.1-SNAPSHOT-spring-boot.jar
- context: ""
- dockerfile: ../Dockerfile
- image: us.gcr.io/osdu-anthos-02/os-indexer/anthos-indexer-reference
- ports:
- - "8080:8080"
-
diff --git a/provider/indexer-reference/kubernetes/deployments/deployment-os-indexer-service.yml b/provider/indexer-reference/kubernetes/deployments/deployment-os-indexer-service.yml
deleted file mode 100644
index 0157d9875389754f1c640f0d186e02b2ea0020d8..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/kubernetes/deployments/deployment-os-indexer-service.yml
+++ /dev/null
@@ -1,179 +0,0 @@
-apiVersion: v1
-data:
- AUTHORIZE_API: ${AUTHORIZE_API}
- MONGO_DB_URL: ${MONGO_DB_URL}
- MONGO_DB_USER: ${MONGO_DB_USER}
- MONGO_DB_NAME: ${MONGO_DB_NAME}
- REGION: ${REGION}
- LEGALTAG_API: ${LEGALTAG_API}
- INDEXER_HOST: ${INDEXER_HOST}
- GOOGLE_CLOUD_PROJECT: ${GOOGLE_CLOUD_PROJECT}
- STORAGE_HOSTNAME: ${STORAGE_HOSTNAME}
- STORAGE_SCHEMA_HOST: ${STORAGE_SCHEMA_HOST}
- STORAGE_QUERY_RECORD_HOST: ${STORAGE_QUERY_RECORD_HOST}
- STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST: ${STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST}
- INDEXER_QUEUE_HOST: ${INDEXER_QUEUE_HOST}
- CRS_API: ${CRS_API}
- REDIS_GROUP_HOST: ${REDIS_GROUP_HOST}
- REDIS_SEARCH_HOST: ${REDIS_SEARCH_HOST}
- GOOGLE_AUDIENCES: ${GOOGLE_AUDIENCES}
- KEY_RING: ${KEY_RING}
- KMS_KEY: ${KMS_KEY}
-kind: ConfigMap
-metadata:
- labels:
- app: indexer-reference
- name: indexer-config
- namespace: default
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- generateName: indexer-reference-anthos
- labels:
- app: indexer-reference
- name: indexer-reference
- namespace: default
-spec:
- selector:
- matchLabels:
- app: indexer-reference
- replicas: 1
- template:
- metadata:
- labels:
- app: indexer-reference
- spec:
- containers:
- - env:
- - name: AUTHORIZE_API
- valueFrom:
- configMapKeyRef:
- key: AUTHORIZE_API
- name: indexer-config
- - name: MONGO_DB_URL
- valueFrom:
- configMapKeyRef:
- key: MONGO_DB_URL
- name: indexer-config
- - name: MONGO_DB_USER
- valueFrom:
- configMapKeyRef:
- key: MONGO_DB_USER
- name: indexer-config
- - name: MONGO_DB_PASSWORD
- valueFrom:
- secretKeyRef:
- name: indexer-secret
- key: mongo.db.password
- - name: MONGO_DB_NAME
- valueFrom:
- configMapKeyRef:
- key: MONGO_DB_NAME
- name: indexer-config
- - name: MB_RABBITMQ_URI
- valueFrom:
- secretKeyRef:
- name: indexer-secret
- key: mb.rabbitmq.uri
- - name: REGION
- valueFrom:
- configMapKeyRef:
- key: REGION
- name: indexer-config
- - name: LEGALTAG_API
- valueFrom:
- configMapKeyRef:
- key: LEGALTAG_API
- name: indexer-config
- - name: INDEXER_HOST
- valueFrom:
- configMapKeyRef:
- key: INDEXER_HOST
- name: indexer-config
- - name: GOOGLE_CLOUD_PROJECT
- valueFrom:
- configMapKeyRef:
- key: GOOGLE_CLOUD_PROJECT
- name: indexer-config
- - name: STORAGE_HOSTNAME
- valueFrom:
- configMapKeyRef:
- key: STORAGE_HOSTNAME
- name: indexer-config
- - name: STORAGE_SCHEMA_HOST
- valueFrom:
- configMapKeyRef:
- key: STORAGE_SCHEMA_HOST
- name: indexer-config
- - name: STORAGE_QUERY_RECORD_HOST
- valueFrom:
- configMapKeyRef:
- key: STORAGE_QUERY_RECORD_HOST
- name: indexer-config
- - name: STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST
- valueFrom:
- configMapKeyRef:
- key: STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST
- name: indexer-config
- - name: INDEXER_QUEUE_HOST
- valueFrom:
- configMapKeyRef:
- key: INDEXER_QUEUE_HOST
- name: indexer-config
- - name: CRS_API
- valueFrom:
- configMapKeyRef:
- key: CRS_API
- name: indexer-config
- - name: REDIS_GROUP_HOST
- valueFrom:
- configMapKeyRef:
- key: REDIS_GROUP_HOST
- name: indexer-config
- - name: REDIS_SEARCH_HOST
- valueFrom:
- configMapKeyRef:
- key: REDIS_SEARCH_HOST
- name: indexer-config
- - name: GOOGLE_AUDIENCES
- valueFrom:
- configMapKeyRef:
- key: GOOGLE_AUDIENCES
- name: indexer-config
- - name: KEY_RING
- valueFrom:
- configMapKeyRef:
- key: KEY_RING
- name: indexer-config
- - name: KMS_KEY
- valueFrom:
- configMapKeyRef:
- key: KMS_KEY
- name: indexer-config
- image: us.gcr.io/osdu-anthos-02/os-indexer/anthos-indexer-reference:9966597-dirty
- name: indexer-reference
----
-apiVersion: v1
-kind: Service
-metadata:
- name: indexer-reference
- namespace: default
-spec:
- ports:
- - protocol: TCP
- port: 80
- targetPort: 8080
- selector:
- app: indexer-reference
- type: LoadBalancer
----
-apiVersion: v1
-data:
- mongo.db.password: ${mongo.db.password}
- mb.rabbitmq.uri: ${mb.rabbitmq.uri}
-kind: Secret
-metadata:
- name: indexer-secret
- namespace: default
-type: Opaque
\ No newline at end of file
diff --git a/provider/indexer-reference/pom.xml b/provider/indexer-reference/pom.xml
deleted file mode 100644
index 7f4acbe48e6c3508d162b8b8be8b4eb770dba9b8..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/pom.xml
+++ /dev/null
@@ -1,191 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Copyright 2020 Google LLC
- ~ Copyright 2020 EPAM Systems, Inc
- ~
- ~ Licensed under the Apache License, Version 2.0 (the "License");
- ~ you may not use this file except in compliance with the License.
- ~ You may obtain a copy of the License at
- ~
- ~ https://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.opengroup.osdu.indexer</groupId>
- <artifactId>indexer-service</artifactId>
- <version>0.16.0-SNAPSHOT</version>
- <relativePath>../../pom.xml</relativePath>
- </parent>
-
- <artifactId>indexer-reference</artifactId>
- <version>0.16.0-SNAPSHOT</version>
- <name>indexer-reference</name>
- <description>Indexer Service GCP Anthos</description>
- <packaging>jar</packaging>
-
- <dependencies>
- <dependency>
- <groupId>org.opengroup.osdu.indexer</groupId>
- <artifactId>indexer-core</artifactId>
- <version>0.16.0-SNAPSHOT</version>
- </dependency>
-
- <dependency>
- <groupId>org.opengroup.osdu</groupId>
- <artifactId>core-lib-gcp</artifactId>
- <version>0.10.0</version>
- </dependency>
-
- <dependency>
- <groupId>org.opengroup.osdu</groupId>
- <artifactId>os-core-common</artifactId>
- <version>${os-core-common.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-starter-data-mongodb</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.google.cloud</groupId>
- <artifactId>google-cloud-logging</artifactId>
- <version>1.72.0</version>
- </dependency>
- <dependency>
- <groupId>com.google.apis</groupId>
- <artifactId>google-api-services-storage</artifactId>
- <version>v1-rev150-1.25.0</version>
- <exclusions>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava-jdk5</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>com.google.apis</groupId>
- <artifactId>google-api-services-cloudkms</artifactId>
- <version>v1-rev81-1.25.0</version>
- </dependency>
-
- <dependency>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-starter-amqp</artifactId>
- </dependency>
- <dependency>
- <groupId>org.elasticsearch</groupId>
- <artifactId>elasticsearch</artifactId>
- </dependency>
- <dependency>
- <groupId>org.elasticsearch.client</groupId>
- <artifactId>elasticsearch-rest-client</artifactId>
- </dependency>
- <dependency>
- <groupId>org.elasticsearch.client</groupId>
- <artifactId>elasticsearch-rest-high-level-client</artifactId>
- </dependency>
-
- <!-- Test Dependencies -->
- <dependency>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-starter-test</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.springframework.security</groupId>
- <artifactId>spring-security-test</artifactId>
- <scope>test</scope>
- </dependency>
-
- <!-- Test Dependencies -->
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.12</version>
- <scope>test</scope>
- </dependency>
- <!-- https://mvnrepository.com/artifact/org.powermock/powermock-api-mockito2 -->
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-api-mockito2</artifactId>
- <version>2.0.2</version>
- <scope>test</scope>
- </dependency>
- <!-- https://mvnrepository.com/artifact/org.powermock/powermock-module-junit4 -->
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-module-junit4</artifactId>
- <version>2.0.2</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-core</artifactId>
- <version>3.12.0</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>cobertura-maven-plugin</artifactId>
- <version>2.7</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>com.auth0</groupId>
- <artifactId>java-jwt</artifactId>
- <version>3.8.1</version>
- </dependency>
-
- <!-- https://mvnrepository.com/artifact/com.github.stefanbirkner/system-rules -->
- <dependency>
- <groupId>com.github.stefanbirkner</groupId>
- <artifactId>system-rules</artifactId>
- <version>1.2.0</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>com.google.api.grpc</groupId>
- <artifactId>proto-google-iam-v1</artifactId>
- <version>0.12.0</version>
- </dependency>
- <dependency>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-actuator-autoconfigure</artifactId>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-maven-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>repackage</goal>
- </goals>
- <configuration>
- <classifier>spring-boot</classifier>
- <mainClass>
- org.opengroup.osdu.indexer.IndexerAnthosApplication
- </mainClass>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
-
-</project>
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/IndexerAnthosApplication.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/IndexerAnthosApplication.java
deleted file mode 100644
index be11d6bbb0f311e1b9bef0e1e20adf96761d3595..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/IndexerAnthosApplication.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer;
-
-import org.springframework.boot.SpringApplication;
-import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
-import org.springframework.boot.autoconfigure.SpringBootApplication;
-import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration;
-import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
-import org.springframework.context.annotation.ComponentScan;
-import org.springframework.context.annotation.Configuration;
-
-@SpringBootApplication(exclude = {MongoAutoConfiguration.class, SecurityAutoConfiguration.class,
- ManagementWebSecurityAutoConfiguration.class})
-@Configuration
-@ComponentScan(value = {"org.opengroup.osdu"})
-public class IndexerAnthosApplication {
-
- public static void main(String[] args) {
- SpringApplication.run(IndexerAnthosApplication.class, args);
- }
-
-}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/ServletInitializer.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/ServletInitializer.java
deleted file mode 100644
index ec6996fe06ba02a1675486c9c22d341389736d32..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/ServletInitializer.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer;
-
-import org.springframework.boot.builder.SpringApplicationBuilder;
-import org.springframework.boot.web.servlet.support.SpringBootServletInitializer;
-
-public class ServletInitializer extends SpringBootServletInitializer {
-
- @Override
- protected SpringApplicationBuilder configure(SpringApplicationBuilder application) {
- return application.sources(IndexerAnthosApplication.class);
- }
-}
-
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/AttributesCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/AttributesCache.java
deleted file mode 100644
index b60b4d14405fbadaaa0e25977b81e08fb66fe905..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/AttributesCache.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.cache;
-
-import java.util.Set;
-import org.opengroup.osdu.core.common.cache.RedisCache;
-import org.opengroup.osdu.core.common.provider.interfaces.IAttributesCache;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-@Component
-public class AttributesCache implements IAttributesCache<String, Set>, AutoCloseable {
-
- private RedisCache<String, Set> cache;
-
- @Autowired
- public AttributesCache(IndexerConfigurationProperties indexerConfigurationProperties) {
- cache = new RedisCache(indexerConfigurationProperties.getRedisSearchHost(),
- Integer.parseInt(indexerConfigurationProperties.getRedisSearchPort()),
- indexerConfigurationProperties.getIndexCacheExpiration() * 60,
- String.class,
- Boolean.class);
- }
-
- @Override
- public void put(String key, Set value) {
- this.cache.put(key, value);
- }
-
- @Override
- public Set get(String key) {
- return this.cache.get(key);
- }
-
- @Override
- public void delete(String key) {
- this.cache.delete(key);
- }
-
- @Override
- public void clearAll() {
- this.cache.clearAll();
- }
-
- @Override
- public void close() {
- this.cache.close();
- }
-}
\ No newline at end of file
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java
deleted file mode 100644
index e1c79251f4bc404ab8c822b430ce864e8b563a0b..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.cache;
-
-import com.google.auth.oauth2.AccessToken;
-import org.opengroup.osdu.core.common.cache.RedisCache;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-@Component
-public class DatastoreCredentialCache extends RedisCache<String, AccessToken> {
-
- @Autowired
- public DatastoreCredentialCache(final IndexerConfigurationProperties configurationProperties) {
- super(configurationProperties.getRedisSearchHost(), Integer.parseInt(configurationProperties.getRedisSearchPort()), 58 * 60, String.class, AccessToken.class);
- }
-}
\ No newline at end of file
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java
deleted file mode 100644
index 24804bc9000a8f10298b5e30a033ed99886c90de..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.cache;
-
-import org.opengroup.osdu.core.common.cache.RedisCache;
-import org.opengroup.osdu.core.common.model.search.ClusterSettings;
-import org.opengroup.osdu.core.common.provider.interfaces.IElasticCredentialsCache;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-@Component
-public class ElasticCredentialsCache implements IElasticCredentialsCache<String, ClusterSettings>,
- AutoCloseable {
-
- private RedisCache<String, ClusterSettings> cache;
-
- @Autowired
- public ElasticCredentialsCache(IndexerConfigurationProperties indexerConfigurationProperties) {
- cache = new RedisCache<>(indexerConfigurationProperties.getRedisSearchHost(),
- Integer.parseInt(indexerConfigurationProperties.getRedisSearchPort()),
- indexerConfigurationProperties.getIndexCacheExpiration() * 60,
- String.class,
- ClusterSettings.class);
- }
-
- @Override
- public void close() throws Exception {
- this.cache.close();
- }
-
- @Override
- public void put(String s, ClusterSettings o) {
- this.cache.put(s, o);
- }
-
- @Override
- public ClusterSettings get(String s) {
- return this.cache.get(s);
- }
-
- @Override
- public void delete(String s) {
- this.cache.delete(s);
- }
-
- @Override
- public void clearAll() {
- this.cache.clearAll();
- }
-}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/IndexCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/IndexCache.java
deleted file mode 100644
index de447717623d63ded8d1d1c46efc42817d618183..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/IndexCache.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.cache;
-
-import org.opengroup.osdu.core.common.cache.RedisCache;
-import org.opengroup.osdu.core.common.provider.interfaces.IIndexCache;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-@Component
-public class IndexCache implements IIndexCache<String, Boolean>, AutoCloseable {
-
- private RedisCache<String, Boolean> cache;
-
- @Autowired
- public IndexCache(IndexerConfigurationProperties indexerConfigurationProperties) {
- cache = new RedisCache<>(indexerConfigurationProperties.getRedisSearchHost(),
- Integer.parseInt(indexerConfigurationProperties.getRedisSearchPort()),
- indexerConfigurationProperties.getIndexCacheExpiration() * 60,
- String.class,
- Boolean.class);
- }
-
- @Override
- public void close() throws Exception {
- this.cache.close();
- }
-
- @Override
- public void put(String s, Boolean o) {
- this.cache.put(s, o);
- }
-
- @Override
- public Boolean get(String s) {
- return this.cache.get(s);
- }
-
- @Override
- public void delete(String s) {
- this.cache.delete(s);
- }
-
- @Override
- public void clearAll() {
- this.cache.clearAll();
- }
-}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/JwtCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/JwtCache.java
deleted file mode 100644
index fc00eb6ed72d5d82924a95286b33ab39a935165a..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/JwtCache.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.cache;
-
-import org.opengroup.osdu.core.common.cache.RedisCache;
-import org.opengroup.osdu.core.common.model.search.IdToken;
-import org.opengroup.osdu.core.common.provider.interfaces.IJwtCache;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.springframework.stereotype.Component;
-
-@Component
-public class JwtCache implements IJwtCache<String, IdToken>, AutoCloseable {
-
- RedisCache<String, IdToken> cache;
-
- // google service account id_token can be requested only for 1 hr
- private static final int EXPIRED_AFTER = 59;
-
- public JwtCache(IndexerConfigurationProperties indexerConfigurationProperties) {
- cache = new RedisCache<>(indexerConfigurationProperties.getRedisSearchHost(),
- Integer.parseInt(indexerConfigurationProperties.getRedisSearchPort()),
- EXPIRED_AFTER * 60,
- String.class,
- IdToken.class);
- }
-
- @Override
- public void close() throws Exception {
- this.cache.close();
- }
-
- @Override
- public void put(String s, IdToken o) {
- this.cache.put(s, o);
- }
-
- @Override
- public IdToken get(String s) {
- return this.cache.get(s);
- }
-
- @Override
- public void delete(String s) {
- this.cache.delete(s);
- }
-
- @Override
- public void clearAll() {
- this.cache.clearAll();
- }
-}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/KindsCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/KindsCache.java
deleted file mode 100644
index afc2a8c39299158c3a50e1e9f5d1f5b67e5a6095..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/KindsCache.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.cache;
-
-import java.util.Set;
-import org.opengroup.osdu.core.common.cache.RedisCache;
-import org.opengroup.osdu.core.common.provider.interfaces.IKindsCache;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-@Component
-public class KindsCache implements IKindsCache<String, Set>, AutoCloseable {
-
- private RedisCache<String, Set> cache;
-
- @Autowired
- public KindsCache(IndexerConfigurationProperties indexerConfigurationProperties) {
- cache = new RedisCache<>(indexerConfigurationProperties.getRedisSearchHost(),
- Integer.parseInt(indexerConfigurationProperties.getRedisSearchPort()),
- indexerConfigurationProperties.getKindsCacheExpiration() * 60,
- indexerConfigurationProperties.getKindsRedisDatabase(),
- String.class,
- Set.class);
- }
-
- @Override
- public void close() throws Exception {
- this.cache.close();
- }
-
- @Override
- public void put(String s, Set o) {
- this.cache.put(s, o);
- }
-
- @Override
- public Set get(String s) {
- return this.cache.get(s);
- }
-
- @Override
- public void delete(String s) {
- this.cache.delete(s);
- }
-
- @Override
- public void clearAll() {
- this.cache.clearAll();
- }
-}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/SchemaCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/SchemaCache.java
deleted file mode 100644
index 3d84e22959501decedf8dd08564bc0869bee8d50..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/SchemaCache.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.cache;
-
-import org.opengroup.osdu.core.common.cache.RedisCache;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-@Component
-public class SchemaCache implements ISchemaCache<String, String>, AutoCloseable {
-
- private RedisCache<String, String> cache;
-
- @Autowired
- public SchemaCache(IndexerConfigurationProperties indexerConfigurationProperties) {
- cache = new RedisCache<>(indexerConfigurationProperties.getRedisSearchHost(),
- Integer.parseInt(indexerConfigurationProperties.getRedisSearchPort()),
- indexerConfigurationProperties.getSchemaCacheExpiration() * 60,
- String.class,
- String.class);
- }
-
- @Override
- public void close() throws Exception {
- this.cache.close();
- }
-
- @Override
- public void put(String s, String o) {
- this.cache.put(s, o);
- }
-
- @Override
- public String get(String s) {
- return this.cache.get(s);
- }
-
- @Override
- public void delete(String s) {
- this.cache.delete(s);
- }
-
- @Override
- public void clearAll() {
- this.cache.clearAll();
- }
-}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/EntitlementsConfigProperties.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/EntitlementsConfigProperties.java
deleted file mode 100644
index 9a1cea7a8111613ca1aff589f596b39c5e63f3cb..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/EntitlementsConfigProperties.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.config;
-
-import lombok.Getter;
-import lombok.Setter;
-import org.springframework.boot.context.properties.ConfigurationProperties;
-import org.springframework.context.annotation.Configuration;
-
-@Configuration
-@ConfigurationProperties
-@Getter
-@Setter
-public class EntitlementsConfigProperties {
-
- private String authorizeApi;
-}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/IndexerReferenceConfigProperties.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/IndexerReferenceConfigProperties.java
deleted file mode 100644
index 4256d3726a174111ec9454fdf23e862066befed4..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/IndexerReferenceConfigProperties.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.config;
-
-import lombok.Getter;
-import lombok.Setter;
-import org.springframework.boot.context.properties.ConfigurationProperties;
-import org.springframework.context.annotation.Configuration;
-
-@Configuration
-@ConfigurationProperties
-@Getter
-@Setter
-public class IndexerReferenceConfigProperties {
-
- private String elasticSettingsId;
-
-}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/MongoDBConfigProperties.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/MongoDBConfigProperties.java
deleted file mode 100644
index 080f76dfc25465b8539e8c0789e81941bd026e07..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/MongoDBConfigProperties.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.config;
-
-import lombok.Getter;
-import lombok.Setter;
-import org.springframework.boot.context.properties.ConfigurationProperties;
-import org.springframework.context.annotation.Configuration;
-
-@Configuration
-@ConfigurationProperties
-@Getter
-@Setter
-public class MongoDBConfigProperties {
-
- private String mongoDbUrl;
- private String mongoDbUser;
- private String mongoDbPassword;
- private String mongoDbName;
-}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/RabbitMqConfigProperties.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/RabbitMqConfigProperties.java
deleted file mode 100644
index 55efbf1a75763e976d71a840335e60b14c8a5ad4..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/RabbitMqConfigProperties.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.config;
-
-import lombok.Getter;
-import lombok.Setter;
-import org.springframework.boot.context.properties.ConfigurationProperties;
-import org.springframework.context.annotation.Configuration;
-
-@Configuration
-@ConfigurationProperties
-@Getter
-@Setter
-public class RabbitMqConfigProperties {
-
- private String mbRabbitMqUri;
-}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/DatastoreCredentialsCacheFactory.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/DatastoreCredentialsCacheFactory.java
deleted file mode 100644
index 6f4d9b7505504f259ceca3d3b8e1834dc53f91ba..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/DatastoreCredentialsCacheFactory.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- Copyright 2020 Google LLC
- Copyright 2020 EPAM Systems, Inc
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.di;
-
-import org.opengroup.osdu.core.common.cache.ICache;
-import org.opengroup.osdu.core.common.cache.VmCache;
-import org.opengroup.osdu.core.gcp.multitenancy.credentials.DatastoreCredential;
-import org.springframework.beans.factory.config.AbstractFactoryBean;
-import org.springframework.stereotype.Component;
-
-@Component
-public class DatastoreCredentialsCacheFactory extends
- AbstractFactoryBean<ICache<String, DatastoreCredential>> {
-
- @Override
- public Class<?> getObjectType() {
- return ICache.class;
- }
-
- @Override
- protected ICache<String, DatastoreCredential> createInstance() throws Exception {
- return new VmCache<>(5 * 60, 20);
- }
-}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/EntitlementsClientFactory.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/EntitlementsClientFactory.java
deleted file mode 100644
index 2872c5c53405e12550c901064da117764208dc4f..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/EntitlementsClientFactory.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.di;
-
-import org.opengroup.osdu.core.common.entitlements.EntitlementsAPIConfig;
-import org.opengroup.osdu.core.common.entitlements.EntitlementsFactory;
-import org.opengroup.osdu.core.common.entitlements.IEntitlementsFactory;
-import org.opengroup.osdu.core.common.http.json.HttpResponseBodyMapper;
-import org.opengroup.osdu.indexer.config.EntitlementsConfigProperties;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.config.AbstractFactoryBean;
-import org.springframework.context.annotation.Lazy;
-import org.springframework.stereotype.Component;
-import org.springframework.web.context.annotation.RequestScope;
-
-@Component
-@RequestScope
-@Lazy
-public class EntitlementsClientFactory extends AbstractFactoryBean<IEntitlementsFactory> {
-
- private EntitlementsConfigProperties entitlementsConfigProperties;
-
- @Autowired
- private HttpResponseBodyMapper mapper;
-
- @Override
- protected IEntitlementsFactory createInstance() throws Exception {
- return new EntitlementsFactory(EntitlementsAPIConfig
- .builder()
- .rootUrl(entitlementsConfigProperties.getAuthorizeApi())
- .build(), mapper);
- }
-
- @Override
- public Class<?> getObjectType() {
- return IEntitlementsFactory.class;
- }
-
- @Autowired
- public void setEntitlementsConfigProperties(
- EntitlementsConfigProperties entitlementsConfigProperties) {
- this.entitlementsConfigProperties = entitlementsConfigProperties;
- }
-}
\ No newline at end of file
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/RabbitMQFactoryImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/RabbitMQFactoryImpl.java
deleted file mode 100644
index 3103acc8e88eca93f51397fc921499de1fc11c35..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/RabbitMQFactoryImpl.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Copyright 2020 Google LLC
- * Copyright 2020 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.di;
-
-import com.rabbitmq.client.Channel;
-import com.rabbitmq.client.Connection;
-import com.rabbitmq.client.ConnectionFactory;
-import java.io.IOException;
-import java.net.URISyntaxException;
-import java.security.KeyManagementException;
-import java.security.NoSuchAlgorithmException;
-import java.util.Arrays;
-import java.util.concurrent.TimeoutException;
-import javax.annotation.PostConstruct;
-import org.opengroup.osdu.indexer.config.RabbitMqConfigProperties;
-import org.opengroup.osdu.indexer.messagebus.IMessageFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-@Component
-public class RabbitMQFactoryImpl implements IMessageFactory {
-
- private static final Logger LOG = LoggerFactory.getLogger(RabbitMQFactoryImpl.class);
- private final RabbitMqConfigProperties rabbitMqConfigProperties;
-
- private Channel channel;
-
- @Autowired
- public RabbitMQFactoryImpl(RabbitMqConfigProperties rabbitMqConfigProperties) {
- this.rabbitMqConfigProperties = rabbitMqConfigProperties;
- }
-
- @PostConstruct
- private void init() {
- ConnectionFactory factory = new ConnectionFactory();
- try {
- String uri = rabbitMqConfigProperties.getMbRabbitMqUri();
- LOG.debug(String.format("RabbitMQ Channel = %s", uri));
- factory.setUri(uri);
- factory.setAutomaticRecoveryEnabled(true);
- Connection conn = factory.newConnection();
- this.channel = conn.createChannel();
- LOG.debug("RabbitMQ Channel was created.");
- for (String queue : Arrays.asList(DEFAULT_QUEUE_NAME, INDEXER_QUEUE_NAME, LEGAL_QUEUE_NAME)) {
- channel.queueDeclare(queue, true, false, false, null);
- LOG.debug(String.format("Queue [%s] was declared.", queue));
- }
- } catch (KeyManagementException | NoSuchAlgorithmException | URISyntaxException | IOException | TimeoutException e) {
- LOG.error(e.getMessage(), e);
- }
-
- }
-
- @Override
- public void sendMessage(String msg) {
- this.sendMessage("records", msg);
- }
-
- @Override
- public void sendMessage(String queueName, String msg) {
- String queueNameWithPrefix = queueName;
- try {
- channel.basicPublish("", queueNameWithPrefix, null, msg.getBytes());
- LOG.info(String.format("[x] Sent '%s' to queue [%s]", msg, queueNameWithPrefix));
- } catch (IOException e) {
- LOG.error(String.format("Unable to publish message to [%s]", queueNameWithPrefix));
- LOG.error(e.getMessage(), e);
- }
- }
-}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryService.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryService.java
deleted file mode 100644
index 6ed88c4f209ec55fcc1043d019c67d498c5d36c9..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryService.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.di;
-
-import lombok.extern.java.Log;
-import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
-import org.opengroup.osdu.core.gcp.multitenancy.TenantFactory;
-import org.springframework.beans.factory.config.AbstractFactoryBean;
-import org.springframework.stereotype.Component;
-import org.springframework.web.context.annotation.RequestScope;
-
-@Log
-@Component
-@RequestScope
-public class TenantFactoryService extends AbstractFactoryBean<ITenantFactory> {
-
- @Override
- protected ITenantFactory createInstance() throws Exception {
- return new TenantFactory();
- }
-
- @Override
- public Class<?> getObjectType() {
- return ITenantFactory.class;
- }
-}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantInfoService.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantInfoService.java
deleted file mode 100644
index 46c1b707826a938d951bdf8f4628b04674e399b5..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantInfoService.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.di;
-
-import java.util.ArrayList;
-import java.util.List;
-import javax.inject.Inject;
-
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
-import org.opengroup.osdu.core.common.multitenancy.ITenantInfoService;
-import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
-import org.springframework.stereotype.Component;
-import org.springframework.web.context.annotation.RequestScope;
-
-@RequestScope
-@Component
-public class TenantInfoService implements ITenantInfoService {
-
- @Inject
- private ITenantFactory tenantFactory;
-
- @Inject
- private DpsHeaders headers;
-
- @Override
- public TenantInfo getTenantInfo() {
- return tenantFactory.getTenantInfo(headers.getPartitionId());
- }
-
- @Override
- public List<TenantInfo> getAllTenantInfos() {
- return new ArrayList<>(tenantFactory.listTenantInfo());
- }
-}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/messagebus/IMessageFactory.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/messagebus/IMessageFactory.java
deleted file mode 100644
index 21316cc68c9d5fa64615a8616f3f6a0e533f8ed4..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/messagebus/IMessageFactory.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.messagebus;
-
-public interface IMessageFactory {
- String DEFAULT_QUEUE_NAME = "records";
- String LEGAL_QUEUE_NAME = "legal";
- String INDEXER_QUEUE_NAME = "indexer";
-
- void sendMessage(String msg);
-
- void sendMessage(String queueName, String msg);
-}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/middleware/IndexFilter.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/middleware/IndexFilter.java
deleted file mode 100644
index 97244e67ce9a4cd9130807cd0a437c77d4064c82..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/middleware/IndexFilter.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.middleware;
-
-import com.google.common.base.Strings;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import javax.servlet.Filter;
-import javax.servlet.FilterChain;
-import javax.servlet.FilterConfig;
-import javax.servlet.ServletException;
-import javax.servlet.ServletRequest;
-import javax.servlet.ServletResponse;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import lombok.extern.java.Log;
-import org.apache.http.HttpStatus;
-import org.opengroup.osdu.core.common.http.ResponseHeaders;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.search.DeploymentEnvironment;
-import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.http.HttpMethod;
-import org.springframework.stereotype.Component;
-
-@Log
-@Component
-public class IndexFilter implements Filter {
-
-
- private final IndexerConfigurationProperties indexerConfigurationProperties;
- private final DpsHeaders dpsHeaders;
- private final IRequestInfo requestInfo;
-
- private FilterConfig filterConfig;
-
- private static final String PATH_SWAGGER = "/swagger.json";
- private static final String PATH_TASK_HANDLERS = "task-handlers";
- private static final String PATH_CRON_HANDLERS = "cron-handlers";
-
- @Autowired
- public IndexFilter(IndexerConfigurationProperties indexerConfigurationProperties,
- DpsHeaders dpsHeaders, IRequestInfo requestInfo) {
- this.indexerConfigurationProperties = indexerConfigurationProperties;
- this.dpsHeaders = dpsHeaders;
- this.requestInfo = requestInfo;
- }
-
- @Override
- public void init(FilterConfig filterConfig) throws ServletException {
- }
-
- @Override
- public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse,
- FilterChain filterChain)
- throws IOException, ServletException {
- HttpServletRequest httpRequest = (HttpServletRequest) servletRequest;
- String uri = httpRequest.getRequestURI().toLowerCase();
-
- if (httpRequest.getMethod().equalsIgnoreCase(HttpMethod.POST.name()) && uri
- .contains(PATH_TASK_HANDLERS)) {
- if (DeploymentEnvironment.valueOf(indexerConfigurationProperties.getEnvironment())
- != DeploymentEnvironment.LOCAL) {
- checkWorkerApiAccess(requestInfo);
- }
- }
-
- if (httpRequest.getMethod().equalsIgnoreCase(HttpMethod.GET.name()) && uri
- .contains(PATH_CRON_HANDLERS)) {
- checkWorkerApiAccess(requestInfo);
- }
-
- filterChain.doFilter(servletRequest, servletResponse);
-
- HttpServletResponse httpResponse = (HttpServletResponse) servletResponse;
- Map<String, List<Object>> standardHeaders = ResponseHeaders.STANDARD_RESPONSE_HEADERS;
- for (Map.Entry<String, List<Object>> header : standardHeaders.entrySet()) {
- httpResponse.addHeader(header.getKey(), header.getValue().toString());
- }
- if (httpResponse.getHeader(DpsHeaders.CORRELATION_ID) == null) {
- httpResponse.addHeader(DpsHeaders.CORRELATION_ID, dpsHeaders.getCorrelationId());
- }
-
- }
-
- @Override
- public void destroy() {
- }
-
- private void checkWorkerApiAccess(IRequestInfo requestInfo) {
- }
-
- private List<String> validateAccountId(DpsHeaders requestHeaders) {
- String accountHeader = requestHeaders.getPartitionIdWithFallbackToAccountId();
- String debuggingInfo = String.format("%s:%s", DpsHeaders.DATA_PARTITION_ID, accountHeader);
-
- if (Strings.isNullOrEmpty(accountHeader)) {
- throw new AppException(HttpStatus.SC_BAD_REQUEST, "Bad request",
- "invalid or empty data partition", debuggingInfo);
- }
-
- List<String> dataPartitions = Arrays.asList(accountHeader.trim().split("\\s*,\\s*"));
- if (dataPartitions.isEmpty() || dataPartitions.size() > 1) {
- throw new AppException(HttpStatus.SC_BAD_REQUEST, "Bad request",
- "invalid or empty data partition", debuggingInfo);
- }
- return dataPartitions;
- }
-}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/model/ElasticSettingSchema.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/model/ElasticSettingSchema.java
deleted file mode 100644
index b18abdb494c8fd18118e44b2ce8f674496ac9ede..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/model/ElasticSettingSchema.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.model;
-
-import javax.validation.constraints.NotEmpty;
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@AllArgsConstructor
-@NoArgsConstructor
-public class ElasticSettingSchema {
-
- @NotEmpty
- private String host;
-
- @NotEmpty
- private String port;
-
- @NotEmpty
- private String usernameAndPassword;
-
- @NotEmpty
- private boolean isHttps;
-
-}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/model/ElasticSettingsDoc.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/model/ElasticSettingsDoc.java
deleted file mode 100644
index 153f5513e912b09d1b2770a3d661ef362452407b..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/model/ElasticSettingsDoc.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-import org.springframework.data.annotation.Id;
-
-@Data
-@AllArgsConstructor
-@NoArgsConstructor
-public class ElasticSettingsDoc {
-
- @Id
- private String id;
- private ElasticSettingSchema settingSchema;
-
- public void setId(String id) {
- this.id = id;
- }
-
-}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryMongoDB.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryMongoDB.java
deleted file mode 100644
index ea6b2da9de66747da3fa72b57d00ea424dac4669..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryMongoDB.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.persistence;
-
-import static com.mongodb.client.model.Filters.eq;
-import static com.mongodb.BasicDBObject.parse;
-
-import com.google.gson.Gson;
-import com.mongodb.client.MongoCollection;
-import java.util.Objects;
-import lombok.RequiredArgsConstructor;
-import org.apache.http.HttpStatus;
-import org.bson.Document;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.search.ClusterSettings;
-import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
-import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository;
-import org.opengroup.osdu.core.common.search.Preconditions;
-import org.opengroup.osdu.indexer.config.IndexerReferenceConfigProperties;
-import org.opengroup.osdu.indexer.model.ElasticSettingsDoc;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.stereotype.Component;
-
-@Component
-@RequiredArgsConstructor
-public class ElasticRepositoryMongoDB implements IElasticRepository {
-
- private static final Logger LOG = LoggerFactory.getLogger(ElasticRepositoryMongoDB.class);
-
- private static final String ELASTIC_SETTINGS_DATABASE = "local";
- private static final String ELASTIC_SETTINGS_COLLECTION = "SearchSettings";
-
- private static final String MISSING_TENANT_INFO_REASON = "TenantInfo is null";
- private static final String MISSING_TENANT_INFO_MESSAGE = "TenantInfo is missing.";
- private static final String SETTINGS_FETCH_ERROR_REASON = "Cluster setting fetch error";
- private static final String SETTINGS_FETCH_ERROR_MESSAGE = "An error has occurred fetching cluster settings from the database.";
-
- private final MongoDdmsClient mongoClient;
- private final IndexerReferenceConfigProperties configProperties;
-
- @Override
- public ClusterSettings getElasticClusterSettings(TenantInfo tenantInfo) {
-
- if (Objects.isNull(tenantInfo)) {
- throw new AppException(HttpStatus.SC_NOT_FOUND, MISSING_TENANT_INFO_REASON,
- MISSING_TENANT_INFO_MESSAGE);
- }
-
- String settingId = tenantInfo.getName().concat("-")
- .concat(this.configProperties.getElasticSettingsId());
-
- MongoCollection<Document> collection = this.mongoClient.getMongoCollection(
- ELASTIC_SETTINGS_DATABASE, ELASTIC_SETTINGS_COLLECTION);
- Document record = collection.find(eq("_id", settingId)).first();
-
- if (Objects.isNull(record)) {
- LOG.error(String.format("\'%s\' credentials not found in database.", settingId));
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, SETTINGS_FETCH_ERROR_REASON,
- SETTINGS_FETCH_ERROR_MESSAGE);
- }
-
- ElasticSettingsDoc elasticSettingsDoc = new Gson().fromJson(String.valueOf(parse(record.toString())),
- ElasticSettingsDoc.class);
-
- try {
- String encryptedUsernameAndPassword = elasticSettingsDoc.getSettingSchema()
- .getUsernameAndPassword();
- String encryptedHost = elasticSettingsDoc.getSettingSchema().getHost();
- String encryptedPortString = elasticSettingsDoc.getSettingSchema().getPort();
-
- String usernameAndPassword = encryptedUsernameAndPassword; //this.kmsClient.decryptString(encryptedHost);
- String host = encryptedHost; //this.kmsClient.decryptString(encryptedHost);
- String portString = encryptedPortString; //this.kmsClient.decryptString(encryptedHost);
-
- Preconditions.checkNotNullOrEmpty(usernameAndPassword, "configuration cannot be null");
- Preconditions.checkNotNullOrEmpty(host, "host cannot be null");
- Preconditions.checkNotNullOrEmpty(portString, "port cannot be null");
-
- int port = Integer.parseInt(portString);
- boolean isHttps = elasticSettingsDoc.getSettingSchema().isHttps();
-
- return new ClusterSettings(host, port, usernameAndPassword, isHttps, isHttps);
- } catch (Exception e) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, SETTINGS_FETCH_ERROR_REASON,
- SETTINGS_FETCH_ERROR_MESSAGE, e);
- }
- }
-
-}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/MongoDdmsClient.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/MongoDdmsClient.java
deleted file mode 100644
index 8caba4bfdc77db033a245c07fba031ae095d7fbb..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/MongoDdmsClient.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.persistence;
-
-import com.mongodb.client.MongoCollection;
-import org.bson.Document;
-import org.opengroup.osdu.indexer.util.MongoClientHandler;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-@Component
-public class MongoDdmsClient {
-
- private final MongoClientHandler mongoClientHandler;
-
- @Autowired
- public MongoDdmsClient(MongoClientHandler mongoClientHandler) {
- this.mongoClientHandler = mongoClientHandler;
- }
-
- public MongoCollection<Document> getMongoCollection(String dbName, String collectionName) {
- return mongoClientHandler.getMongoClient().getDatabase(dbName)
- .getCollection(collectionName);
- }
-}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java
deleted file mode 100644
index a9a82d8cffb4ea3b86d56d66e210f00575f0a034..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.publish;
-
-
-import com.google.common.reflect.TypeToken;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import com.google.gson.JsonElement;
-import java.lang.reflect.Type;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import org.elasticsearch.common.Strings;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.indexer.JobStatus;
-import org.opengroup.osdu.core.common.model.indexer.RecordStatus;
-import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
-import org.opengroup.osdu.indexer.messagebus.IMessageFactory;
-import org.opengroup.osdu.indexer.provider.interfaces.IPublisher;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-import org.springframework.web.context.annotation.RequestScope;
-
-@Component
-@RequestScope
-public class PublisherImpl implements IPublisher {
-
- private static final Logger LOG = LoggerFactory.getLogger(PublisherImpl.class);
- private final IMessageFactory mq;
-
- @Autowired
- public PublisherImpl(IMessageFactory mq) {
- this.mq = mq;
- }
-
- @Override
- public void publishStatusChangedTagsToTopic(DpsHeaders headers, JobStatus indexerBatchStatus) {
-
- String tenant = headers.getPartitionId();
- if (Strings.isNullOrEmpty(tenant)) {
- tenant = headers.getAccountId();
- }
-
- Map<String, String> message = new HashMap<>();
- message.put(tenant, headers.getPartitionIdWithFallbackToAccountId());
- headers.addCorrelationIdIfMissing();
- message.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
-
- RecordChangedMessages recordChangedMessages = getRecordChangedMessage(headers,
- indexerBatchStatus);
- message.put("data", recordChangedMessages.toString());
-
- try {
- LOG.info("Indexer publishes message " + headers.getCorrelationId());
- mq.sendMessage(IMessageFactory.INDEXER_QUEUE_NAME, message.toString());
- } catch (Exception e) {
- LOG.error(e.getMessage(), e);
- }
- }
-
- private RecordChangedMessages getRecordChangedMessage(DpsHeaders headers,
- JobStatus indexerBatchStatus) {
-
- Gson gson = new GsonBuilder().create();
- Map<String, String> attributesMap = new HashMap<>();
- Type listType = new TypeToken<List<RecordStatus>>() {
- }.getType();
-
- JsonElement statusChangedTagsJson = gson
- .toJsonTree(indexerBatchStatus.getStatusesList(), listType);
- String statusChangedTagsData = (statusChangedTagsJson.toString());
-
- String tenant = headers.getPartitionId();
- // This code it to provide backward compatibility to slb-account-id
- if (!Strings.isNullOrEmpty(tenant)) {
- attributesMap
- .put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
- } else {
- attributesMap.put(DpsHeaders.ACCOUNT_ID, headers.getPartitionIdWithFallbackToAccountId());
- }
- headers.addCorrelationIdIfMissing();
- attributesMap.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
-
- RecordChangedMessages recordChangedMessages = new RecordChangedMessages();
- // statusChangedTagsData is not ByteString but String
- recordChangedMessages.setData(statusChangedTagsData);
- recordChangedMessages.setAttributes(attributesMap);
-
- return recordChangedMessages;
- }
-}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/security/SecurityConfig.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/security/SecurityConfig.java
deleted file mode 100644
index 9096cbd6eddbbbee6cd7e63c12506bcf06f40dca..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/security/SecurityConfig.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.security;
-
-import org.springframework.context.annotation.Configuration;
-import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity;
-import org.springframework.security.config.annotation.web.builders.HttpSecurity;
-import org.springframework.security.config.annotation.web.builders.WebSecurity;
-import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
-import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter;
-
-@Configuration
-@EnableWebSecurity
-@EnableGlobalMethodSecurity(prePostEnabled = true)
-public class SecurityConfig extends WebSecurityConfigurerAdapter {
-
- @Override
- protected void configure(HttpSecurity http) throws Exception {
- http.httpBasic().disable()
- .csrf().disable();
- }
-
- @Override
- public void configure(WebSecurity web) throws Exception {
- web.ignoring().antMatchers("/api-docs")
- .antMatchers("/info")
- .antMatchers("/swagger");
- }
-
-}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
deleted file mode 100644
index 748d130fe4482ebc7086de100d7d4a8a8b3323b9..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
+++ /dev/null
@@ -1,643 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.service;
-
-import com.google.common.base.Strings;
-import com.google.gson.Gson;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.function.Consumer;
-import java.util.stream.Collectors;
-import javax.inject.Inject;
-import org.apache.http.HttpStatus;
-import org.elasticsearch.ElasticsearchStatusException;
-import org.elasticsearch.action.bulk.BulkItemResponse;
-import org.elasticsearch.action.bulk.BulkRequest;
-import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.delete.DeleteRequest;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.update.UpdateRequest;
-import org.elasticsearch.client.RequestOptions;
-import org.elasticsearch.client.RestHighLevelClient;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.rest.RestStatus;
-import org.opengroup.osdu.core.common.Constants;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.entitlements.Acl;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.http.RequestStatus;
-import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
-import org.opengroup.osdu.core.common.model.indexer.IndexingStatus;
-import org.opengroup.osdu.core.common.model.indexer.JobStatus;
-import org.opengroup.osdu.core.common.model.indexer.OperationType;
-import org.opengroup.osdu.core.common.model.indexer.RecordIndexerPayload;
-import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
-import org.opengroup.osdu.core.common.model.indexer.RecordStatus;
-import org.opengroup.osdu.core.common.model.indexer.Records;
-import org.opengroup.osdu.core.common.model.search.IndexInfo;
-import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
-import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
-import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
-import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
-import org.opengroup.osdu.indexer.logging.AuditLogger;
-import org.opengroup.osdu.indexer.provider.interfaces.IPublisher;
-import org.opengroup.osdu.indexer.util.ElasticClientHandler;
-import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
-import org.springframework.stereotype.Service;
-
-@Service
-public class IndexerServiceImpl implements IndexerService {
-
- private static final TimeValue BULK_REQUEST_TIMEOUT = TimeValue.timeValueMinutes(1);
-
- private static final List<RestStatus> RETRY_ELASTIC_EXCEPTION = new ArrayList<>(Arrays
- .asList(RestStatus.TOO_MANY_REQUESTS, RestStatus.BAD_GATEWAY,
- RestStatus.SERVICE_UNAVAILABLE));
-
- public static final String ELASTIC_ERROR_REASON = "Elastic error";
-
- private final Gson gson = new Gson();
-
- @Inject
- private JaxRsDpsLog jaxRsDpsLog;
- @Inject
- private AuditLogger auditLogger;
- @Inject
- private StorageService storageService;
- @Inject
- private IndexSchemaService schemaService;
- @Inject
- private IndicesService indicesService;
- @Inject
- private IMappingService mappingService;
- @Inject
- private IPublisher progressPublisher;
- @Inject
- private ElasticClientHandler elasticClientHandler;
- @Inject
- private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
- @Inject
- private ElasticIndexNameResolver elasticIndexNameResolver;
- @Inject
- private StorageIndexerPayloadMapper storageIndexerPayloadMapper;
- @Inject
- private IRequestInfo requestInfo;
- @Inject
- private JobStatus jobStatus;
-
- private DpsHeaders headers;
-
- @Override
- public JobStatus processRecordChangedMessages(RecordChangedMessages message,
- List<RecordInfo> recordInfos) throws Exception {
-
- // this should not happen
- if (recordInfos.isEmpty()) {
- return null;
- }
-
- String errorMessage = "";
- List<String> retryRecordIds = new LinkedList<>();
-
- // get auth header with service account Authorization
- this.headers = this.requestInfo.getHeaders();
- String authorization = message.getAttributes().get("authorization");
- this.headers.put("authorization", authorization);
- // initialize status for all messages.
- this.jobStatus.initialize(recordInfos);
-
- try {
- // get upsert records
- Map<String, Map<String, OperationType>> upsertRecordMap = RecordInfo
- .getUpsertRecordIds(recordInfos);
- if (upsertRecordMap != null && !upsertRecordMap.isEmpty()) {
- List<String> upsertFailureRecordIds = processUpsertRecords(upsertRecordMap);
- retryRecordIds.addAll(upsertFailureRecordIds);
- }
-
- // get delete records
- Map<String, List<String>> deleteRecordMap = RecordInfo.getDeleteRecordIds(recordInfos);
- if (deleteRecordMap != null && !deleteRecordMap.isEmpty()) {
- List<String> deleteFailureRecordIds = processDeleteRecords(deleteRecordMap);
- retryRecordIds.addAll(deleteFailureRecordIds);
- }
-
- auditLogger.indexStarted(recordInfos.stream()
- .map(RecordInfo::getKind)
- .collect(Collectors.toList()));
-
- // process schema change messages
- Map<String, OperationType> schemaMsgs = RecordInfo.getSchemaMsgs(recordInfos);
- if (schemaMsgs != null && !schemaMsgs.isEmpty()) {
- this.schemaService.processSchemaMessages(schemaMsgs);
- }
-
- // process failed records
- if (!retryRecordIds.isEmpty()) {
- retryAndEnqueueFailedRecords(recordInfos, retryRecordIds, message);
- }
- } catch (IOException e) {
- errorMessage = e.getMessage();
- throw new AppException(HttpStatus.SC_GATEWAY_TIMEOUT, "Internal communication failure",
- errorMessage, e);
- } catch (AppException e) {
- errorMessage = e.getMessage();
- throw e;
- } catch (Exception e) {
- errorMessage = "error indexing records";
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Unknown error",
- "An unknown error has occurred.", e);
- } finally {
- this.jobStatus.finalizeRecordStatus(errorMessage);
- this.updateAuditLog();
- this.progressPublisher.publishStatusChangedTagsToTopic(this.headers, this.jobStatus);
- }
-
- return jobStatus;
- }
-
- @Override
- public void processSchemaMessages(List<RecordInfo> recordInfos) throws IOException {
- Map<String, OperationType> schemaMsgs = RecordInfo.getSchemaMsgs(recordInfos);
- if (schemaMsgs != null && !schemaMsgs.isEmpty()) {
- try (RestHighLevelClient restClient = elasticClientHandler.createRestClient()) {
- schemaMsgs.entrySet().forEach(msg -> {
- try {
- processSchemaEvents(restClient, msg);
- } catch (IOException | ElasticsearchStatusException e) {
- throw new AppException(
- org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR.value(),
- "unable to process schema delete", e.getMessage());
- }
- });
- }
- }
- }
-
- private void processSchemaEvents(RestHighLevelClient restClient,
- Map.Entry<String, OperationType> msg) throws IOException, ElasticsearchStatusException {
- String kind = msg.getKey();
- String index = elasticIndexNameResolver.getIndexNameFromKind(kind);
-
- boolean indexExist = indicesService.isIndexExist(restClient, index);
- if (indexExist && msg.getValue() == OperationType.purge_schema) {
- indicesService.deleteIndex(restClient, index);
- }
- }
-
- private List<String> processUpsertRecords(Map<String, Map<String, OperationType>> upsertRecordMap)
- throws Exception {
- // get schema for kind
- Map<String, IndexSchema> schemas = this.getSchema(upsertRecordMap);
-
- if (schemas.isEmpty()) {
- return new LinkedList<>();
- }
-
- // get recordIds with valid upsert index-status
- List<String> recordIds = this.jobStatus.getIdsByValidUpsertIndexingStatus();
-
- if (recordIds.isEmpty()) {
- return new LinkedList<>();
- }
-
- // get records via storage api
- Records storageRecords = this.storageService.getStorageRecords(recordIds);
- List<String> failedOrRetryRecordIds = new LinkedList<>(storageRecords.getMissingRetryRecords());
-
- // map storage records to indexer payload
- RecordIndexerPayload recordIndexerPayload = this
- .getIndexerPayload(upsertRecordMap, schemas, storageRecords);
-
- jaxRsDpsLog.info(String.format(
- "records change messages received : %s | valid storage bulk records: %s | valid index payload: %s",
- recordIds.size(), storageRecords.getRecords().size(),
- recordIndexerPayload.getRecords().size()));
-
- // index records
- failedOrRetryRecordIds.addAll(processElasticMappingAndUpsertRecords(recordIndexerPayload));
-
- return failedOrRetryRecordIds;
- }
-
- private Map<String, IndexSchema> getSchema(
- Map<String, Map<String, OperationType>> upsertRecordMap) {
-
- Map<String, IndexSchema> schemas = new HashMap<>();
-
- try {
- for (Map.Entry<String, Map<String, OperationType>> entry : upsertRecordMap.entrySet()) {
-
- String kind = entry.getKey();
- List<String> errors = new ArrayList<>();
- IndexSchema schemaObj = this.schemaService.getIndexerInputSchema(kind, errors);
- if (!errors.isEmpty()) {
- this.jobStatus.addOrUpdateRecordStatus(entry.getValue().keySet(), IndexingStatus.WARN,
- HttpStatus.SC_BAD_REQUEST, String.join("|", errors),
- String.format("error | kind: %s", kind));
- } else if (schemaObj.isDataSchemaMissing()) {
- this.jobStatus.addOrUpdateRecordStatus(entry.getValue().keySet(), IndexingStatus.WARN,
- HttpStatus.SC_NOT_FOUND, "schema not found",
- String.format("schema not found | kind: %s", kind));
- }
-
- schemas.put(kind, schemaObj);
- }
- } catch (AppException e) {
- throw e;
- } catch (Exception e) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Get schema error",
- "An error has occurred while getting schema", e);
- }
-
- return schemas;
- }
-
- private RecordIndexerPayload getIndexerPayload(
- Map<String, Map<String, OperationType>> upsertRecordMap,
- Map<String, IndexSchema> kindSchemaMap, Records records) {
- List<Records.Entity> storageValidRecords = records.getRecords();
- List<RecordIndexerPayload.Record> indexerPayload = new ArrayList<>();
- List<IndexSchema> schemas = new ArrayList<>();
-
- for (Records.Entity storageRecord : storageValidRecords) {
-
- Map<String, OperationType> idOperationMap = upsertRecordMap.get(storageRecord.getKind());
-
- // skip if storage returned record with same id but different kind
- if (idOperationMap == null) {
- String message = String.format(
- "storage service returned incorrect record | requested kind: %s | received kind: %s",
- this.jobStatus.getRecordKindById(storageRecord.getId()), storageRecord.getKind());
- this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.SKIP,
- RequestStatus.STORAGE_CONFLICT, message,
- String.format("%s | record-id: %s", message, storageRecord.getId()));
- continue;
- }
-
- IndexSchema schema = kindSchemaMap.get(storageRecord.getKind());
- schemas.add(schema);
-
- // skip indexing of records if data block is empty
- RecordIndexerPayload.Record document = prepareIndexerPayload(schema, storageRecord,
- idOperationMap);
- if (document != null) {
- indexerPayload.add(document);
- }
- }
-
- // this should only happen if storage service returned WRONG records with kind for all the records in the messages
- if (indexerPayload.isEmpty()) {
- throw new AppException(RequestStatus.STORAGE_CONFLICT, "Indexer error",
- "upsert record failed, storage service returned incorrect records");
- }
-
- return RecordIndexerPayload.builder().records(indexerPayload).schemas(schemas).build();
- }
-
- private RecordIndexerPayload.Record prepareIndexerPayload(IndexSchema schemaObj,
- Records.Entity storageRecord, Map<String, OperationType> idToOperationMap) {
-
- RecordIndexerPayload.Record document = null;
-
- try {
- Map<String, Object> storageRecordData = storageRecord.getData();
- document = new RecordIndexerPayload.Record();
- if (storageRecordData == null || storageRecordData.isEmpty()) {
- String message = "empty or null data block found in the storage record";
- this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.WARN,
- HttpStatus.SC_NOT_FOUND, message,
- String.format("record-id: %s | %s", storageRecord.getId(), message));
- } else if (schemaObj.isDataSchemaMissing()) {
- document.setSchemaMissing(true);
- } else {
- Map<String, Object> dataMap = this.storageIndexerPayloadMapper
- .mapDataPayload(schemaObj, storageRecordData, storageRecord.getId());
- if (dataMap.isEmpty()) {
- document.setMappingMismatch(true);
- String message = String.format(
- "complete schema mismatch: none of the data attribute can be mapped | data: %s",
- storageRecordData);
- this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.WARN,
- HttpStatus.SC_NOT_FOUND, message,
- String.format("record-id: %s | %s", storageRecord.getId(), message));
- }
- document.setData(dataMap);
- }
- } catch (AppException e) {
- this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL,
- HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getMessage());
- jaxRsDpsLog
- .warning(String.format("record-id: %s | %s", storageRecord.getId(), e.getMessage()), e);
- } catch (Exception e) {
- this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL,
- HttpStatus.SC_INTERNAL_SERVER_ERROR,
- String.format("error parsing records against schema, error-message: %s", e.getMessage()));
- jaxRsDpsLog.error(String
- .format("record-id: %s | error parsing records against schema, error-message: %s",
- storageRecord.getId(), e.getMessage()), e);
- }
-
- try {
- // index individual parts of kind
- String[] kindParts = storageRecord.getKind().split(":");
-
- document.setKind(storageRecord.getKind());
- document.setNamespace(kindParts[0] + ":" + kindParts[1]);
- document.setType(kindParts[2]);
- document.setId(storageRecord.getId());
- document.setVersion(storageRecord.getVersion());
- document.setAcl(storageRecord.getAcl());
- document.setLegal(storageRecord.getLegal());
- if (storageRecord.getTags() != null) {
- document.setTags(storageRecord.getTags());
- }
- RecordStatus recordStatus = this.jobStatus.getJobStatusByRecordId(storageRecord.getId());
- if (recordStatus.getIndexProgress().getStatusCode() == 0) {
- recordStatus.getIndexProgress().setStatusCode(HttpStatus.SC_OK);
- }
- document.setCreateUser(storageRecord.getCreateUser());
- document.setCreateTime(storageRecord.getCreateTime());
- if (!Strings.isNullOrEmpty(storageRecord.getModifyUser())) {
- document.setModifyUser(storageRecord.getModifyUser());
- }
- if (!Strings.isNullOrEmpty(storageRecord.getModifyTime())) {
- document.setModifyTime(storageRecord.getModifyTime());
- }
- document.setIndexProgress(recordStatus.getIndexProgress());
- if (storageRecord.getAncestry() != null) {
- document.setAncestry(storageRecord.getAncestry());
- }
- document.setOperationType(idToOperationMap.get(storageRecord.getId()));
- } catch (Exception e) {
- this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL,
- HttpStatus.SC_INTERNAL_SERVER_ERROR,
- String.format("error parsing meta data, error-message: %s", e.getMessage()));
- jaxRsDpsLog.error(String.format("record-id: %s | error parsing meta data, error-message: %s",
- storageRecord.getId(), e.getMessage()), e);
- }
- return document;
- }
-
- private List<String> processElasticMappingAndUpsertRecords(
- RecordIndexerPayload recordIndexerPayload) throws Exception {
-
- try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
- List<IndexSchema> schemas = recordIndexerPayload.getSchemas();
- if (schemas == null || schemas.isEmpty()) {
- return new LinkedList<>();
- }
-
- // process the schema
- this.cacheOrCreateElasticMapping(schemas, restClient);
-
- // process the records
- return this.upsertRecords(recordIndexerPayload.getRecords(), restClient);
- }
- }
-
- private void cacheOrCreateElasticMapping(List<IndexSchema> schemas,
- RestHighLevelClient restClient) throws Exception {
-
- for (IndexSchema schema : schemas) {
- String index = this.elasticIndexNameResolver.getIndexNameFromKind(schema.getKind());
-
- // check if index exist and sync meta attribute schema if required
- if (this.indicesService.isIndexExist(restClient, index)) {
- this.mappingService.syncIndexMappingIfRequired(restClient, schema);
- continue;
- }
-
- // create index
- Map<String, Object> mapping = this.mappingService.getIndexMappingFromRecordSchema(schema);
- if (!this.indicesService.createIndex(restClient, index, null, schema.getType(), mapping)) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, ELASTIC_ERROR_REASON,
- "Error creating index.",
- String.format("Failed to get confirmation from elastic server for index: %s", index));
- }
- }
- }
-
- private List<String> upsertRecords(List<RecordIndexerPayload.Record> records,
- RestHighLevelClient restClient) throws AppException {
- if (records == null || records.isEmpty()) {
- return new LinkedList<>();
- }
-
- BulkRequest bulkRequest = new BulkRequest();
- bulkRequest.timeout(BULK_REQUEST_TIMEOUT);
-
- for (RecordIndexerPayload.Record record : records) {
- if ((record.getData() == null || record.getData().isEmpty()) && !record
- .skippedDataIndexing()) {
- // it will come here when schema is missing
- // TODO: rollback once we know what is causing the problem
- jaxRsDpsLog.warning(String.format("data not found for record: %s", record));
- }
-
- OperationType operation = record.getOperationType();
- Map<String, Object> sourceMap = getSourceMap(record);
- String index = this.elasticIndexNameResolver.getIndexNameFromKind(record.getKind());
-
- if (operation == OperationType.create) {
- IndexRequest indexRequest = new IndexRequest(index).id(record.getId())
- .source(this.gson.toJson(sourceMap), XContentType.JSON);
- bulkRequest.add(indexRequest);
- } else if (operation == OperationType.update) {
- UpdateRequest updateRequest = new UpdateRequest(index, record.getId())
- .upsert(this.gson.toJson(sourceMap), XContentType.JSON);
- bulkRequest.add(updateRequest);
- }
- }
-
- return processBulkRequest(restClient, bulkRequest);
- }
-
- private List<String> processDeleteRecords(Map<String, List<String>> deleteRecordMap)
- throws Exception {
- BulkRequest bulkRequest = new BulkRequest();
- bulkRequest.timeout(BULK_REQUEST_TIMEOUT);
-
- for (Map.Entry<String, List<String>> record : deleteRecordMap.entrySet()) {
- String index = this.elasticIndexNameResolver.getIndexNameFromKind(record.getKey());
-
- for (String id : record.getValue()) {
- DeleteRequest deleteRequest = new DeleteRequest(index, id);
- bulkRequest.add(deleteRequest);
- }
- }
-
- try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
- return processBulkRequest(restClient, bulkRequest);
- }
- }
-
- private List<String> processBulkRequest(RestHighLevelClient restClient, BulkRequest bulkRequest)
- throws AppException {
-
- List<String> failureRecordIds = new LinkedList<>();
- if (bulkRequest.numberOfActions() == 0) {
- return failureRecordIds;
- }
-
- try {
- BulkResponse bulkResponse = restClient.bulk(bulkRequest, RequestOptions.DEFAULT);
-
- // log failed bulk requests
- ArrayList<String> bulkFailures = new ArrayList<>();
- int succeededResponses = 0;
- int failedResponses = 0;
-
- for (BulkItemResponse bulkItemResponse : bulkResponse.getItems()) {
- if (bulkItemResponse.isFailed()) {
- BulkItemResponse.Failure failure = bulkItemResponse.getFailure();
- bulkFailures.add(String.format("elasticsearch bulk service status: %s id: %s message: %s",
- failure.getStatus(), failure.getId(), failure.getMessage()));
- this.jobStatus.addOrUpdateRecordStatus(bulkItemResponse.getId(), IndexingStatus.FAIL,
- failure.getStatus().getStatus(), bulkItemResponse.getFailureMessage());
- if (RETRY_ELASTIC_EXCEPTION.contains(bulkItemResponse.status())) {
- failureRecordIds.add(bulkItemResponse.getId());
- }
- failedResponses++;
- } else {
- succeededResponses++;
- this.jobStatus.addOrUpdateRecordStatus(bulkItemResponse.getId(), IndexingStatus.SUCCESS,
- HttpStatus.SC_OK, "Indexed Successfully");
- }
- }
- if (!bulkFailures.isEmpty()) {
- this.jaxRsDpsLog.warning(bulkFailures);
- }
-
- jaxRsDpsLog.info(String
- .format("records in elasticsearch service bulk request: %s | successful: %s | failed: %s",
- bulkRequest.numberOfActions(), succeededResponses, failedResponses));
- } catch (IOException e) {
- // throw explicit 504 for IOException
- throw new AppException(HttpStatus.SC_GATEWAY_TIMEOUT, ELASTIC_ERROR_REASON,
- "Request cannot be completed in specified time.", e);
- } catch (ElasticsearchStatusException e) {
- throw new AppException(e.status().getStatus(), ELASTIC_ERROR_REASON, e.getMessage(), e);
- } catch (Exception e) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, ELASTIC_ERROR_REASON,
- "Error indexing records.", e);
- }
- return failureRecordIds;
- }
-
- private Map<String, Object> getSourceMap(RecordIndexerPayload.Record record) {
-
- Map<String, Object> indexerPayload = new HashMap<>();
-
- // get the key and get the corresponding object from the individualRecord object
- if (record.getData() != null) {
- Map<String, Object> data = new HashMap<>();
- for (Map.Entry<String, Object> entry : record.getData().entrySet()) {
- data.put(entry.getKey(), entry.getValue());
- }
- indexerPayload.put(Constants.DATA, data);
- }
-
- indexerPayload.put(RecordMetaAttribute.ID.getValue(), record.getId());
- indexerPayload.put(RecordMetaAttribute.KIND.getValue(), record.getKind());
- indexerPayload.put(RecordMetaAttribute.AUTHORITY.getValue(), record.getAuthority());
- indexerPayload.put(RecordMetaAttribute.SOURCE.getValue(), record.getSource());
- indexerPayload.put(RecordMetaAttribute.NAMESPACE.getValue(), record.getNamespace());
- indexerPayload.put(RecordMetaAttribute.TYPE.getValue(), record.getType());
- indexerPayload.put(RecordMetaAttribute.VERSION.getValue(), record.getVersion());
- indexerPayload.put(RecordMetaAttribute.ACL.getValue(), record.getAcl());
- indexerPayload.put(RecordMetaAttribute.TAGS.getValue(), record.getTags());
- indexerPayload.put(RecordMetaAttribute.X_ACL.getValue(), Acl.flattenAcl(record.getAcl()));
- indexerPayload.put(RecordMetaAttribute.LEGAL.getValue(), record.getLegal());
- indexerPayload.put(RecordMetaAttribute.INDEX_STATUS.getValue(), record.getIndexProgress());
- if (record.getAncestry() != null) {
- indexerPayload.put(RecordMetaAttribute.ANCESTRY.getValue(), record.getAncestry());
- }
- indexerPayload.put(RecordMetaAttribute.CREATE_USER.getValue(), record.getCreateUser());
- indexerPayload.put(RecordMetaAttribute.CREATE_TIME.getValue(), record.getCreateTime());
- if (!Strings.isNullOrEmpty(record.getModifyUser())) {
- indexerPayload.put(RecordMetaAttribute.MODIFY_USER.getValue(), record.getModifyUser());
- }
- if (!Strings.isNullOrEmpty(record.getModifyTime())) {
- indexerPayload.put(RecordMetaAttribute.MODIFY_TIME.getValue(), record.getModifyTime());
- }
- return indexerPayload;
- }
-
- private void retryAndEnqueueFailedRecords(List<RecordInfo> recordInfos,
- List<String> failuresRecordIds, RecordChangedMessages message) throws IOException {
-
- jaxRsDpsLog.info(String.format(
- "queuing bulk failed records back to task-queue for retry | count: %s | records: %s",
- failuresRecordIds.size(), failuresRecordIds));
- List<RecordInfo> retryRecordInfos = new LinkedList<>();
- for (String recordId : failuresRecordIds) {
- for (RecordInfo origMessage : recordInfos) {
- if (origMessage.getId().equalsIgnoreCase(recordId)) {
- retryRecordInfos.add(origMessage);
- }
- }
- }
-
- RecordChangedMessages newMessage = RecordChangedMessages.builder()
- .messageId(message.getMessageId())
- .publishTime(message.getPublishTime())
- .data(this.gson.toJson(retryRecordInfos))
- .attributes(message.getAttributes()).build();
-
- String payLoad = this.gson.toJson(newMessage);
- this.indexerQueueTaskBuilder.createWorkerTask(payLoad, this.headers);
- }
-
- private void updateAuditLog() {
- logAuditEvents(OperationType.create, this.auditLogger::indexCreateRecordSuccess,
- this.auditLogger::indexCreateRecordFail);
- logAuditEvents(OperationType.update, this.auditLogger::indexUpdateRecordSuccess,
- this.auditLogger::indexUpdateRecordFail);
- logAuditEvents(OperationType.purge, this.auditLogger::indexPurgeRecordSuccess,
- this.auditLogger::indexPurgeRecordFail);
- logAuditEvents(OperationType.delete, this.auditLogger::indexDeleteRecordSuccess,
- this.auditLogger::indexDeleteRecordFail);
- }
-
- private void logAuditEvents(OperationType operationType, Consumer<List<String>> successEvent,
- Consumer<List<String>> failedEvent) {
- List<RecordStatus> succeededRecords = this.jobStatus
- .getRecordStatuses(IndexingStatus.SUCCESS, operationType);
- if (!succeededRecords.isEmpty()) {
- successEvent.accept(succeededRecords.stream().map(RecordStatus::succeededAuditLogMessage)
- .collect(Collectors.toList()));
- }
- List<RecordStatus> skippedRecords = this.jobStatus
- .getRecordStatuses(IndexingStatus.SKIP, operationType);
- List<RecordStatus> failedRecords = this.jobStatus
- .getRecordStatuses(IndexingStatus.FAIL, operationType);
- failedRecords.addAll(skippedRecords);
- if (!failedRecords.isEmpty()) {
- failedEvent.accept(failedRecords.stream().map(RecordStatus::failedAuditLogMessage)
- .collect(Collectors.toList()));
- }
- }
-}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/DpsHeaderFactoryGcp.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/DpsHeaderFactoryGcp.java
deleted file mode 100644
index 226f74229fce34469f84068c3e468f4d59e3b58a..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/DpsHeaderFactoryGcp.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.util;
-
-import com.google.common.base.Strings;
-import java.util.Collections;
-import java.util.Map;
-import java.util.stream.Collectors;
-import javax.inject.Inject;
-import javax.servlet.http.HttpServletRequest;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.gcp.model.CloudTaskHeaders;
-import org.opengroup.osdu.core.gcp.util.TraceIdExtractor;
-import org.springframework.context.annotation.Primary;
-import org.springframework.stereotype.Component;
-import org.springframework.web.context.annotation.RequestScope;
-
-@Component
-@RequestScope
-@Primary
-public class DpsHeaderFactoryGcp extends DpsHeaders {
-
- @Inject
- public DpsHeaderFactoryGcp(HttpServletRequest request) {
-
- Map<String, String> headers = Collections
- .list(request.getHeaderNames())
- .stream()
- .collect(Collectors.toMap(h -> h, request::getHeader));
-
- String traceContext = headers.get(CloudTaskHeaders.CLOUD_TRACE_CONTEXT);
-
- if (!Strings.isNullOrEmpty(traceContext)) {
- headers.put(CloudTaskHeaders.TRACE_ID, TraceIdExtractor.getTraceId(traceContext));
- }
-
- this.addFromMap(headers);
-
- // Add Correlation ID if missing
- this.addCorrelationIdIfMissing();
- }
-}
\ No newline at end of file
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/MongoClientHandler.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/MongoClientHandler.java
deleted file mode 100644
index 8e53b593ec171e2962ca3b286d328e89a1a7716c..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/MongoClientHandler.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Copyright 2020 Google LLC
- * Copyright 2020 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.util;
-
-import com.mongodb.ConnectionString;
-import com.mongodb.MongoClientSettings;
-import com.mongodb.client.MongoClient;
-import com.mongodb.client.MongoClients;
-import org.apache.http.HttpStatus;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.indexer.config.MongoDBConfigProperties;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-@Component
-public class MongoClientHandler {
-
- private static final Logger LOG = LoggerFactory.getLogger(
- org.opengroup.osdu.indexer.util.MongoClientHandler.class);
- private static final String MONGO_PREFIX = "mongodb://";
- private static final String MONGO_OPTIONS = "retryWrites=true&w=majority&maxIdleTimeMS=10000";
-
- private com.mongodb.client.MongoClient mongoClient = null;
- private MongoDBConfigProperties mongoDBConfigProperties;
-
- private MongoClient getOrInitMongoClient() throws RuntimeException {
- if (mongoClient != null) {
- return mongoClient;
- }
-
- final String connectionString = String.format("%s%s:%s@%s/?%s",
- MONGO_PREFIX,
- mongoDBConfigProperties.getMongoDbUser(),
- mongoDBConfigProperties.getMongoDbPassword(),
- mongoDBConfigProperties.getMongoDbUrl(),
- MONGO_OPTIONS);
- ConnectionString connString = new ConnectionString(connectionString);
- MongoClientSettings settings = MongoClientSettings.builder()
- .applyConnectionString(connString)
- .retryWrites(true)
- .build();
- try {
- mongoClient = MongoClients.create(settings);
- } catch (Exception ex) {
- LOG.error("Error connecting MongoDB", ex);
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Error connecting MongoDB",
- ex.getMessage(), ex);
- }
- return mongoClient;
- }
-
- public MongoClient getMongoClient() {
- if (mongoClient == null) {
- getOrInitMongoClient();
- }
- return mongoClient;
- }
-
- @Autowired
- public void setMongoDBConfigProperties(MongoDBConfigProperties mongoDBConfigProperties) {
- this.mongoDBConfigProperties = mongoDBConfigProperties;
- }
-}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java
deleted file mode 100644
index b86b91ce440748e52c559da0152b5d3c4060d0a4..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
- Copyright 2021 Google LLC
- Copyright 2021 EPAM Systems, Inc
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.util;
-
-import com.google.api.client.googleapis.auth.oauth2.GoogleIdToken;
-import com.google.api.client.googleapis.auth.oauth2.GoogleIdTokenVerifier;
-import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
-import com.google.api.client.json.jackson2.JacksonFactory;
-import com.google.common.base.Strings;
-import java.io.IOException;
-import java.security.GeneralSecurityException;
-import java.util.Arrays;
-import java.util.logging.Level;
-import lombok.extern.java.Log;
-import org.apache.http.HttpStatus;
-import org.opengroup.osdu.core.common.Constants;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.search.DeploymentEnvironment;
-import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient;
-import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.opengroup.osdu.core.gcp.model.CloudTaskHeaders;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Component;
-import org.springframework.web.context.annotation.RequestScope;
-
-import javax.inject.Inject;
-import java.util.Map;
-
-import static org.opengroup.osdu.core.common.model.http.DpsHeaders.AUTHORIZATION;
-
-
-@Log
-@Component
-@RequestScope
-public class RequestInfoImpl implements IRequestInfo {
-
- @Inject
- private DpsHeaders dpsHeaders;
-
- @Inject
- private IServiceAccountJwtClient serviceAccountJwtClient;
-
- @Inject
- private TenantInfo tenantInfo;
-
- @Inject
- private IndexerConfigurationProperties properties;
-
- @Value("${indexer.que.service.mail}")
- private String indexerQueServiceMail;
-
- private static final String EXPECTED_CRON_HEADER_VALUE = "true";
-
- @Override
- public DpsHeaders getHeaders() {
-
- return this.dpsHeaders;
- }
-
- @Override
- public String getPartitionId() {
- return this.dpsHeaders.getPartitionId();
- }
-
- @Override
- public Map<String, String> getHeadersMap() {
- return this.dpsHeaders.getHeaders();
- }
-
- @Override
- public Map<String, String> getHeadersMapWithDwdAuthZ() {
- return getHeadersWithDwdAuthZ().getHeaders();
- }
-
- @Override
- public DpsHeaders getHeadersWithDwdAuthZ() {
- // Update DpsHeaders so that service account creds are passed down
- this.dpsHeaders.put(AUTHORIZATION, this.checkOrGetAuthorizationHeader());
- return this.dpsHeaders;
- }
-
- @Override
- public boolean isCronRequest() {
- String appEngineCronHeader = this.dpsHeaders.getHeaders().getOrDefault(CloudTaskHeaders.CLOUD_CRON_SERVICE, null);
- return EXPECTED_CRON_HEADER_VALUE.equalsIgnoreCase(appEngineCronHeader);
- }
-
- @Override
- public boolean isTaskQueueRequest() {
- if(this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.CLOUD_TASK_QUEUE_NAME)){
- log.log(Level.INFO,"Request acknowledged as Cloud task, proceeding token validation");
- return isCloudTaskRequest();
- }
- if(this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.APPENGINE_TASK_QUEUE_NAME)){
- log.log(Level.INFO,"Request acknowledged as AppEngine task, proceeding headers validation");
- return isAppEngineTaskRequest();
- }
- return false;
- }
-
- private boolean isCloudTaskRequest() {
- log.log(Level.INFO,dpsHeaders.getHeaders().toString());
- try {
- GoogleIdTokenVerifier verifier = new GoogleIdTokenVerifier.Builder(
- GoogleNetHttpTransport.newTrustedTransport(), JacksonFactory.getDefaultInstance())
- .setIssuers(Arrays.asList(
- "accounts.google.com", "https://accounts.google.com",
- "googleapis.com", "https://www.googleapis.com/auth/userinfo.profile"
- )
- ).build();
- String authorization = dpsHeaders.getAuthorization().replace("Bearer ", "");
-
- GoogleIdToken googleIdToken = verifier.verify(authorization);
- if(googleIdToken.getPayload().getEmail().equals(indexerQueServiceMail)){
- return true;
- }
- log.log(Level.WARNING,"Token email doesn't match with variable \"indexer.que.service.mail\"");
- return false;
-
- } catch (GeneralSecurityException | IOException e) {
- log.log(Level.WARNING,"Not valid or expired cloud task token provided");
- return false;
- }
- }
-
- private boolean isAppEngineTaskRequest(){
- if (!this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.APPENGINE_TASK_QUEUE_NAME)) {
- return false;
- }
- String queueId = this.dpsHeaders.getHeaders().get(CloudTaskHeaders.APPENGINE_TASK_QUEUE_NAME);
- return queueId.endsWith(Constants.INDEXER_QUEUE_IDENTIFIER);
- }
-
- public String checkOrGetAuthorizationHeader() {
- if (properties.getDeploymentEnvironment() == DeploymentEnvironment.LOCAL) {
- String authHeader = this.dpsHeaders.getAuthorization();
- if (Strings.isNullOrEmpty(authHeader)) {
- throw new AppException(HttpStatus.SC_UNAUTHORIZED, "Invalid authorization header", "Authorization token cannot be empty");
- }
- String user = this.dpsHeaders.getUserEmail();
- if (Strings.isNullOrEmpty(user)) {
- throw new AppException(HttpStatus.SC_UNAUTHORIZED, "Invalid user header", "User header cannot be empty");
- }
- return authHeader;
- } else {
- return "Bearer " + this.serviceAccountJwtClient.getIdToken(tenantInfo.getName());
- }
- }
-}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java
deleted file mode 100644
index 86ea46e4c96cd76609581cfeab61e930cc5332b6..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.util;
-
-import com.auth0.jwt.JWT;
-import com.google.api.client.json.JsonFactory;
-import com.google.api.client.json.jackson2.JacksonFactory;
-import com.google.api.services.iam.v1.Iam;
-import com.google.cloud.iam.credentials.v1.GenerateIdTokenResponse;
-import com.google.cloud.iam.credentials.v1.IamCredentialsClient;
-import com.google.cloud.iam.credentials.v1.ServiceAccountName;
-import java.util.Collections;
-import org.apache.http.HttpStatus;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.search.IdToken;
-import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
-import org.opengroup.osdu.core.common.provider.interfaces.IJwtCache;
-import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
-import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient;
-import org.opengroup.osdu.core.gcp.multitenancy.credentials.IamCredentialsProvider;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-import org.springframework.web.context.annotation.RequestScope;
-
-@Component
-@RequestScope
-public class ServiceAccountJwtGcpClientImpl implements IServiceAccountJwtClient {
-
- private static final String SERVICE_ACCOUNT_NAME_FORMAT = "projects/%s/serviceAccounts/%s";
- private final IamCredentialsProvider iamCredentialsProvider = new IamCredentialsProvider();
-
- private final IndexerConfigurationProperties indexerConfigurationProperties;
- private final ITenantFactory tenantInfoServiceProvider;
- private final IJwtCache cacheService;
- private final JaxRsDpsLog log;
- private final DpsHeaders dpsHeaders;
-
- private Iam iam;
-
- @Autowired
- public ServiceAccountJwtGcpClientImpl(
- IndexerConfigurationProperties indexerConfigurationProperties,
- ITenantFactory tenantInfoServiceProvider,
- IJwtCache cacheService, JaxRsDpsLog log, DpsHeaders dpsHeaders) {
- this.indexerConfigurationProperties = indexerConfigurationProperties;
- this.tenantInfoServiceProvider = tenantInfoServiceProvider;
- this.cacheService = cacheService;
- this.log = log;
- this.dpsHeaders = dpsHeaders;
- }
-
- public String getIdToken(String tenantName) {
- this.log.info("Tenant name received for auth token is: " + tenantName);
- TenantInfo tenant = this.tenantInfoServiceProvider.getTenantInfo(tenantName);
- if (tenant == null) {
- this.log.error("Invalid tenant name receiving from pubsub");
- throw new AppException(HttpStatus.SC_BAD_REQUEST, "Invalid tenant Name",
- "Invalid tenant Name from pubsub");
- }
- try {
-
- IdToken cachedToken = (IdToken) this.cacheService.get(tenant.getServiceAccount());
- // Add the user to DpsHeaders directly
- this.dpsHeaders.put(DpsHeaders.USER_EMAIL, tenant.getServiceAccount());
-
- if (!IdToken.refreshToken(cachedToken)) {
- return cachedToken.getTokenValue();
- }
-
- try (IamCredentialsClient iamCredentialsClient = iamCredentialsProvider
- .getIamCredentialsClient()) {
- ServiceAccountName serviceAccountName = ServiceAccountName
- .parse(String.format(SERVICE_ACCOUNT_NAME_FORMAT, tenant.getServiceAccount()));
- GenerateIdTokenResponse idTokenResponse = iamCredentialsClient
- .generateIdToken(serviceAccountName, Collections
- .emptyList(), indexerConfigurationProperties.getGoogleAudiences(), true);
- String token = idTokenResponse.getToken();
- IdToken idToken = IdToken.builder().tokenValue(token)
- .expirationTimeMillis(JWT.decode(token).getExpiresAt().getTime()).build();
- this.cacheService.put(tenant.getServiceAccount(), idToken);
- return token;
- }
- } catch (AppException e) {
- throw e;
- } catch (Exception e) {
- throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Persistence error",
- "Error generating token", e);
- }
- }
-}
diff --git a/provider/indexer-reference/src/main/resources/application-dev.properties b/provider/indexer-reference/src/main/resources/application-dev.properties
deleted file mode 100644
index ea75c38d4866a1a690158a7e7efbd184c68ad58c..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/main/resources/application-dev.properties
+++ /dev/null
@@ -1,44 +0,0 @@
-#
-# Copyright 2020 Google LLC
-# Copyright 2020 EPAM Systems, Inc
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-GOOGLE_CLOUD_PROJECT=nice-etching-277309
-
-mongo-db-url=localhost:27017
-mongo-db-user=
-mongo-db-password=
-
-indexer-host=os-indexer-dot-nice-etching-277309.uc.r.appspot.com
-storage-hostname=os-storage-dot-nice-etching-277309.uc.r.appspot.com
-
-storage-schema-host=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/schemas
-storage-query-record-host=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/query/records
-storage-query-record-for-conversion-host=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/query/records:batch
-storage-records-batch-size=20
-
-indexer-queue-host=https://os-indexer-queue-dot-nice-etching-277309.uc.r.appspot.com/_dps/task-handlers/enqueue
-
-authorize-api=https://entitlements-dot-nice-etching-277309.uc.r.appspot.com/entitlements/v1
-legaltag-api=https://os-legal-dot-nice-etching-277309.uc.r.appspot.com/api/legal/v1
-crs-api=example.com
-
-## use below values for gcp: opendes
-redis-group-host=127.0.0.1
-redis-search-host=127.0.0.1
-
-google-audiences=689762842995-pv217jo3k8j803kk6gqf52qb5amos3a9.apps.googleusercontent.com
-
-mb-rabbitmq-uri=amqp://guest:guest@127.0.0.1:5672
diff --git a/provider/indexer-reference/src/main/resources/application.properties b/provider/indexer-reference/src/main/resources/application.properties
deleted file mode 100644
index 4e704e197e435d41c794d996275a1de20f89efcc..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/main/resources/application.properties
+++ /dev/null
@@ -1,73 +0,0 @@
-#
-# Copyright 2021 Google LLC
-# Copyright 2021 EPAM Systems, Inc
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-LOG_PREFIX=indexer
-
-server.servlet.contextPath=/api/indexer/v2
-logging.level.org.springframework.web=DEBUG
-server.port=8080
-JAVA_OPTS=-Xms3072m -Xmx3072m
-JAVA_GC_OPTS=-XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45
-
-environment=CLOUD
-
-#default cache settings
-schema-cache-expiration=60
-index-cache-expiration=60
-elastic-cache-expiration=1440
-cursor-cache-expiration=60
-# kinds cache expiration 2*24*60
-kinds-cache-expiration=2880
-# attributes cache expiration 2*24*60
-attributes-cache-expiration=2880
-
-kinds-redis-database=1
-cron-index-cleanup-threshold-days=3
-cron-empty-index-cleanup-threshold-days=7
-
-gae-service=indexer
-key-ring=csqp
-kms-key=searchservice
-
-elastic-settings-id=indexer-service
-
-mongo-db-url=localhost:27017
-mongo-db-user=
-mongo-db-password=
-
-indexer-host=os-indexer-dot-nice-etching-277309.uc.r.appspot.com
-storage-hostname=os-storage-dot-nice-etching-277309.uc.r.appspot.com
-
-storage-schema-host=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/schemas
-storage-query-record-host=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/query/records
-storage-query-record-for-conversion-host=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/query/records:batch
-storage-records-batch-size=20
-
-indexer-queue-host=https://os-indexer-queue-dot-nice-etching-277309.uc.r.appspot.com/_dps/task-handlers/enqueue
-
-authorize-api=https://entitlements-dot-nice-etching-277309.uc.r.appspot.com/entitlements/v1
-legaltag-api=https://os-legal-dot-nice-etching-277309.uc.r.appspot.com/api/legal/v1
-crs-api=example.com
-
-## use below values for gcp: opendes
-redis-group-host=127.0.0.1
-redis-search-host=127.0.0.1
-redis-search-port=6379
-
-google-audiences=689762842995-pv217jo3k8j803kk6gqf52qb5amos3a9.apps.googleusercontent.com
-
-mb-rabbitmq-uri=amqp://guest:guest@127.0.0.1:5672
\ No newline at end of file
diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java
deleted file mode 100644
index bfc4f3a3330570ea73c59d2721954dd75aded7a5..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.middleware;
-
-import java.io.IOException;
-import java.util.Collections;
-import javax.servlet.FilterChain;
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.mockito.Mockito;
-import org.mockito.junit.MockitoJUnitRunner;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-
-@RunWith(MockitoJUnitRunner.class)
-public class IndexFilterTest {
-
- @InjectMocks
- private IndexFilter indexFilter;
-
- @Mock
- private DpsHeaders dpsHeaders;
-
- @Test
- public void shouldSetCorrectResponseHeaders() throws IOException, ServletException {
- HttpServletRequest httpServletRequest = Mockito.mock(HttpServletRequest.class);
- HttpServletResponse httpServletResponse = Mockito.mock(HttpServletResponse.class);
- FilterChain filterChain = Mockito.mock(FilterChain.class);
- Mockito.when(httpServletRequest.getRequestURI()).thenReturn("https://test.com");
- Mockito.when(httpServletRequest.getMethod()).thenReturn("POST");
- Mockito.when(dpsHeaders.getCorrelationId()).thenReturn("correlation-id-value");
-
- indexFilter.doFilter(httpServletRequest, httpServletResponse, filterChain);
-
- Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Origin", Collections.singletonList("*").toString());
- Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Headers", Collections.singletonList("origin, content-type, accept, authorization, data-partition-id, correlation-id, appkey").toString());
- Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Methods", Collections.singletonList("GET, POST, PUT, DELETE, OPTIONS, HEAD, PATCH").toString());
- Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Credentials", Collections.singletonList("true").toString());
- Mockito.verify(httpServletResponse).addHeader("X-Frame-Options", Collections.singletonList("DENY").toString());
- Mockito.verify(httpServletResponse).addHeader("X-XSS-Protection", Collections.singletonList("1; mode=block").toString());
- Mockito.verify(httpServletResponse).addHeader("X-Content-Type-Options", Collections.singletonList("nosniff").toString());
- Mockito.verify(httpServletResponse).addHeader("Cache-Control", Collections.singletonList("no-cache, no-store, must-revalidate").toString());
- Mockito.verify(httpServletResponse).addHeader("Content-Security-Policy", Collections.singletonList("default-src 'self'").toString());
- Mockito.verify(httpServletResponse).addHeader("Strict-Transport-Security", Collections.singletonList("max-age=31536000; includeSubDomains").toString());
- Mockito.verify(httpServletResponse).addHeader("Expires", Collections.singletonList("0").toString());
- Mockito.verify(httpServletResponse).addHeader("correlation-id", "correlation-id-value");
- Mockito.verify(filterChain).doFilter(httpServletRequest, httpServletResponse);
- }
-}
diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/CronServiceImplTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/CronServiceImplTest.java
deleted file mode 100644
index cadcb0bf01f4e8516756d2cc2e070b71071178f8..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/CronServiceImplTest.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.service;
-
-import static org.mockito.Mockito.any;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-import com.google.common.collect.Lists;
-import java.io.IOException;
-import java.time.Instant;
-import java.time.temporal.ChronoUnit;
-import org.elasticsearch.client.RestHighLevelClient;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.search.IndexInfo;
-import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.opengroup.osdu.indexer.util.ElasticClientHandler;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.springframework.test.context.junit4.SpringRunner;
-
-
-@RunWith(SpringRunner.class)
-@PrepareForTest({RestHighLevelClient.class})
-public class CronServiceImplTest {
-
- @Mock
- private RestHighLevelClient restHighLevelClient;
- @Mock
- private IndicesService indicesService;
- @Mock
- private ElasticClientHandler elasticClientHandler;
- @Mock
- private IndexerConfigurationProperties configurationProperties;
- @Mock
- private IRequestInfo requestInfo;
- @Mock
- private JaxRsDpsLog log;
- @InjectMocks
- private CronServiceImpl sut;
-
- @InjectMocks
- private DpsHeaders dpsHeaders;
-
- @Before
- public void setup() {
-
- when(this.requestInfo.getHeaders()).thenReturn(dpsHeaders);
-
- when(configurationProperties.getCronIndexCleanupThresholdDays()).thenReturn(3);
- when(configurationProperties.getCronEmptyIndexCleanupThresholdDays()).thenReturn(3);
- }
-
- @Test
- public void run_cleanup_when_cron_job_runs_with_correct_pattern() throws Exception {
- final String indexPattern = "tenant1-index-*";
-
- IndexInfo info = IndexInfo.builder().name("tenant1-index-1.0.0").documentCount("10")
- .creationDate(Long.toString(Instant.now().minus(4, ChronoUnit.DAYS).toEpochMilli()))
- .build();
-
- when(this.requestInfo.getPartitionId()).thenReturn("tenant1");
- when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient);
- when(this.indicesService.getIndexInfo(this.restHighLevelClient, indexPattern))
- .thenReturn(Lists.newArrayList(info));
-
- this.sut.cleanupIndices(indexPattern);
-
- verify(this.indicesService, times(1)).deleteIndex(restHighLevelClient, "tenant1-index-1.0.0");
- verify(this.indicesService, times(1)).getIndexInfo(restHighLevelClient, indexPattern);
- }
-
- @Test(expected = IOException.class)
- public void run_cleanup_when_cron_job_runs_with_wrong_pattern() throws Exception {
- IOException exception = new IOException("blah");
- when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient);
- when(this.indicesService.getIndexInfo(this.restHighLevelClient, "tenant1-test-*"))
- .thenThrow(exception);
-
- this.sut.cleanupIndices("tenant1-test-*");
-
- verify(this.indicesService, times(0)).deleteIndex(any(), any());
- }
-
- @Test
- public void run_cleanup_when_backend_does_not_have_empty_stale_indices() throws Exception {
- IndexInfo info = IndexInfo.builder().name("tenant1-index-1.0.0").documentCount("10")
- .creationDate(Long.toString(Instant.now().minus(8, ChronoUnit.DAYS).toEpochMilli()))
- .build();
-
- when(this.requestInfo.getPartitionId()).thenReturn("tenant1");
- when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient);
- when(this.indicesService.getIndexInfo(this.restHighLevelClient, null))
- .thenReturn(Lists.newArrayList(info));
-
- this.sut.cleanupEmptyStaleIndices();
-
- verify(this.indicesService, times(0)).deleteIndex(restHighLevelClient, null);
- verify(this.indicesService, times(1)).getIndexInfo(restHighLevelClient, null);
- }
-
- @Test
- public void run_cleanup_when_backend_have_empty_stale_indices() throws Exception {
- IndexInfo info = IndexInfo.builder().name("tenant1-index-1.0.0").documentCount("0")
- .creationDate(Long.toString(Instant.now().minus(8, ChronoUnit.DAYS).toEpochMilli()))
- .build();
-
- when(this.requestInfo.getPartitionId()).thenReturn("tenant1");
- when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient);
- when(this.indicesService.getIndexInfo(this.restHighLevelClient, null))
- .thenReturn(Lists.newArrayList(info));
-
- this.sut.cleanupEmptyStaleIndices();
-
- verify(this.indicesService, times(1)).deleteIndex(restHighLevelClient, "tenant1-index-1.0.0");
- verify(this.indicesService, times(1)).getIndexInfo(restHighLevelClient, null);
- }
-
- @Test(expected = IOException.class)
- public void run_cleanup_when_backend_throws_exception() throws Exception {
- IOException exception = new IOException("blah");
- when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient);
- when(this.indicesService.getIndexInfo(this.restHighLevelClient, null)).thenThrow(exception);
-
- this.sut.cleanupEmptyStaleIndices();
-
- verify(this.indicesService, times(0)).deleteIndex(any(), any());
- }
-}
\ No newline at end of file
diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceTest.java
deleted file mode 100644
index b09fbc40ddb38ffd327be67375a4b7718d687722..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceTest.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.service;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.mockito.Mockito.when;
-
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.search.ClusterSettings;
-import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
-import org.opengroup.osdu.core.common.multitenancy.ITenantInfoService;
-import org.opengroup.osdu.core.common.provider.interfaces.IElasticCredentialsCache;
-import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.springframework.test.context.junit4.SpringRunner;
-
-@RunWith(SpringRunner.class)
-public class ElasticSettingServiceTest {
-
- @Mock
- private ITenantInfoService tenantInfoService;
- @Mock
- private IElasticRepository elasticRepository;
- @Mock
- private IElasticCredentialsCache elasticCredentialCache;
- @Mock
- private IndexerConfigurationProperties configurationProperties;
- @Mock
- private TenantInfo tenantInfo;
- @InjectMocks
- private ElasticSettingServiceImpl sut;
- @Mock
- private ClusterSettings clusterSettings;
- @Mock
- private DpsHeaders headersInfo;
-
- @Mock
- private JaxRsDpsLog log;
-
-
- public String GAE_SERVICE = "indexer";
-
- private final String host = "db5c51c1.us-central1.gcp.cloud.es.io";
- private final int port = 9243;
- private final String credentials = "name:password";
-
- String cacheKey = "";
-
-
- @Before
- public void setup() {
- when(tenantInfo.getName()).thenReturn("tenant1");
- when(this.headersInfo.getPartitionId()).thenReturn("tenant1");
- when(this.tenantInfoService.getTenantInfo()).thenReturn(tenantInfo);
- when(configurationProperties.getGaeService()).thenReturn("indexer");
- clusterSettings = ClusterSettings.builder().host(host).port(port)
- .userNameAndPassword(credentials).build();
- cacheKey = String.format("%s-%s", GAE_SERVICE, tenantInfo.getName());
- }
-
- @Test
- public void should_getValid_clusterSettings_fromCache() {
-
- when(this.elasticCredentialCache.get(cacheKey)).thenReturn(clusterSettings);
-
- ClusterSettings response = this.sut.getElasticClusterInformation();
- assertNotNull(response);
- assertEquals(response.getHost(), host);
- assertEquals(response.getPort(), port);
- assertEquals(response.getUserNameAndPassword(), credentials);
- }
-
- @Test
- public void should_getValid_clusterSettings_fromCosmosDB() {
-
- when(this.elasticCredentialCache.get(cacheKey)).thenReturn(clusterSettings);
-
- when(this.elasticRepository.getElasticClusterSettings(tenantInfo)).thenReturn(clusterSettings);
-
- ClusterSettings response = this.sut.getElasticClusterInformation();
- assertNotNull(response);
- assertEquals(response.getHost(), host);
- assertEquals(response.getPort(), port);
- assertEquals(response.getUserNameAndPassword(), credentials);
- }
-
- @Test(expected = AppException.class)
- public void should_throwAppException_when_tenantClusterInfo_not_found() throws AppException {
-
- when(this.elasticRepository.getElasticClusterSettings(tenantInfo)).thenReturn(null);
-
- this.sut.getElasticClusterInformation();
-
- }
-}
diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImplTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImplTest.java
deleted file mode 100644
index db855559e8c02acc780da73f5436a4081b8e03b2..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImplTest.java
+++ /dev/null
@@ -1,202 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.service;
-
-import static org.junit.Assert.fail;
-import static org.mockito.Mockito.mock;
-import static org.powermock.api.mockito.PowerMockito.when;
-
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
-import java.io.IOException;
-import java.lang.reflect.Type;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.UUID;
-import org.apache.http.HttpEntity;
-import org.apache.http.util.EntityUtils;
-import org.elasticsearch.client.Request;
-import org.elasticsearch.client.Response;
-import org.elasticsearch.client.RestClient;
-import org.elasticsearch.client.RestHighLevelClient;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.ArgumentMatchers;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.indexer.IElasticSettingService;
-import org.opengroup.osdu.core.common.model.search.ClusterSettings;
-import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
-import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
-import org.opengroup.osdu.indexer.logging.AuditLogger;
-import org.opengroup.osdu.indexer.util.ElasticClientHandler;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.springframework.test.context.junit4.SpringRunner;
-
-@RunWith(SpringRunner.class)
-@PrepareForTest({RestHighLevelClient.class, Response.class, RestClient.class, HttpEntity.class,
- EntityUtils.class})
-public class IndexCopyServiceImplTest {
-
- private final String correlationId = UUID.randomUUID().toString();
-
- @Mock
- private HttpEntity httpEntity;
- @Mock
- private HttpEntity httpEntityRequest;
- @Mock
- private IRequestInfo requestInfo;
- @Mock
- private DpsHeaders headersInfo;
- @Mock
- private RestClient restClient;
- @Mock
- private RestHighLevelClient restHighLevelClient;
- @Mock
- private IndicesService indicesService;
- @Mock
- private IMappingService mappingService;
- @Mock
- private ElasticClientHandler elasticClientHandler;
- @Mock
- private ElasticIndexNameResolver elasticIndexNameResolver;
- @Mock
- private Response response;
- @Mock
- private IElasticSettingService elasticSettingService;
- @Mock
- private AuditLogger auditLogger;
- @Mock
- private Map<String, String> httpHeaders;
- @InjectMocks
- private IndexCopyServiceImpl sut;
-
- private ClusterSettings commonCluster;
-
- private Map<String, Object> correctMap;
-
- @Before
- public void setup() {
-
- commonCluster = ClusterSettings.builder().host("commonhost").port(8080)
- .userNameAndPassword("username:pwd").build();
-
- httpHeaders = new HashMap<>();
- httpHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
- httpHeaders.put(DpsHeaders.CORRELATION_ID, correlationId);
- when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(httpHeaders);
- when(response.getEntity()).thenReturn(httpEntity);
-
- Type mapType = new TypeToken<Map<String, Object>>() {
- }.getType();
- String afterFormat = "{\"properties\":{\"id\":{\"type\":\"keyword\"}}}";
- correctMap = new Gson().fromJson(afterFormat, mapType);
-
- restHighLevelClient = mock(RestHighLevelClient.class);
-
- }
-
- @Test(expected = IOException.class)
- public void should_throwIOException_when_indexMappingNotFound() throws Exception {
- IOException exception = new IOException(
- "Fail to get mapping for the given index from common cluster.");
-
- when(this.mappingService.getIndexMapping(ArgumentMatchers.any(), ArgumentMatchers.any()))
- .thenThrow(exception);
-
- this.sut.copyIndex("common:metadata:entity:1.0.0");
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void should_throwIllegalArgExceptionCopyIndexRequest_copyIndexTest() {
- try {
- this.sut.copyIndex(null);
- } catch (IOException e) {
- fail("Should not throw IOException but illegalArgumentException.");
- }
- }
-
- @Test
- public void should_returnIndexMapping_getIndexMappingFromCommonClustertest() {
- String mappingJson = "{\"common-metadata-entity-1.0.0\":{\"mappings\":{\"entity\":{\"properties\":{\"id\":{\"type\":\"keyword\"}}}}}}";
- when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
- try {
- when(this.mappingService.getIndexMapping(ArgumentMatchers.any(), ArgumentMatchers.any()))
- .thenReturn(mappingJson);
- Map<String, Object> resultMap = this.sut.getIndexMappingsFromCommonCluster("test", "test");
- Assert.assertEquals(resultMap, correctMap);
- } catch (Exception ignored) {
- }
- }
-
- @Test
- public void should_returnClusterInfo_getCommonClusterInformationtest() {
- try {
- String[] correctCommonCluster = {"https://commonhost:8080", "username", "pwd"};
-
- when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
-
- when(elasticSettingService.getElasticClusterInformation()).thenReturn(commonCluster);
-
- String[] resultCommonCluster = this.sut.getCommonClusterInformation();
- Assert.assertEquals(correctCommonCluster[0], resultCommonCluster[0]);
- Assert.assertEquals(correctCommonCluster[1], resultCommonCluster[1]);
- Assert.assertEquals(correctCommonCluster[2], resultCommonCluster[2]);
- } catch (IOException ignored) {
- fail("Should not throw this exception " + ignored.getMessage());
- }
- }
-
- @Test(expected = AppException.class)
- public void should_throwException_failToCreateIndexInTenantCluster_createIndexInTenantClustertest() {
- try {
- when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
- when(indicesService
- .createIndex(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(),
- ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(false);
- this.sut.createIndexInTenantCluster("test", "test", "test", correctMap);
- } catch (IOException ignored) {
- fail("Should not throw this exception " + ignored.getMessage());
- }
- }
-
- @Ignore
- public void should_returnTaskIdResponse_reindexRequestSucceed_reindexInTenantClustertest() {
- //TODO: fix the null Response from restHighLevelClient.getLowLevelClient().performRequest().
- try {
- String[] correctCommonCluster = {"https://commonhost:8080", "username", "pwd"};
- Request request = new Request("POST", "/_reindex?wait_for_completion=false");
- request.setEntity(httpEntityRequest);
- when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
- when(indicesService
- .createIndex(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(),
- ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(false);
- when(restHighLevelClient.getLowLevelClient()).thenReturn(restClient);
- when(restClient.performRequest(request)).thenReturn(response);
- when(response.getEntity()).thenReturn(httpEntity);
- Assert.assertEquals(httpEntity,
- this.sut.reindexInTenantCluster("test", "test", correctCommonCluster));
- } catch (IOException ignored) {
- }
- }
-}
diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java
deleted file mode 100644
index bf7fa01a337c13971b47efc97c5b20f96788cb9d..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.service;
-
-import static org.junit.Assert.fail;
-import static org.mockito.Matchers.any;
-import static org.mockito.MockitoAnnotations.initMocks;
-import static org.powermock.api.mockito.PowerMockito.mockStatic;
-import static org.powermock.api.mockito.PowerMockito.when;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
-import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
-import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
-import org.powermock.modules.junit4.PowerMockRunner;
-import org.powermock.modules.junit4.PowerMockRunnerDelegate;
-import org.springframework.test.context.junit4.SpringRunner;
-
-@RunWith(PowerMockRunner.class)
-@PowerMockRunnerDelegate(SpringRunner.class)
-public class ReindexServiceTest {
-
- private final String cursor = "100";
-
- private final String correlationId = UUID.randomUUID().toString();
-
- @Mock
- private IndexerConfigurationProperties configurationProperties;
- @Mock
- private StorageService storageService;
- @Mock
- private IRequestInfo requestInfo;
- @Mock
- private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
- @Mock
- private JaxRsDpsLog log;
- @InjectMocks
- private ReindexServiceImpl sut;
-
- private RecordReindexRequest recordReindexRequest;
- private RecordQueryResponse recordQueryResponse;
-
- private Map<String, String> httpHeaders;
-
- @Before
- public void setup() {
- initMocks(this);
-
- mockStatic(UUID.class);
-
- recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0")
- .cursor(cursor).build();
- recordQueryResponse = new RecordQueryResponse();
-
- httpHeaders = new HashMap<>();
- httpHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
- httpHeaders.put(DpsHeaders.CORRELATION_ID, correlationId);
- DpsHeaders standardHeaders = DpsHeaders.createFromMap(httpHeaders);
- when(requestInfo.getHeaders()).thenReturn(standardHeaders);
- when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(httpHeaders);
- when(requestInfo.getHeadersWithDwdAuthZ()).thenReturn(standardHeaders);
- }
-
- @Test
- public void should_returnNull_givenNullResponseResult_reIndexRecordsTest() {
- try {
- recordQueryResponse.setResults(null);
- when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
-
- String response = sut.reindexRecords(recordReindexRequest, false);
-
- Assert.assertNull(response);
- } catch (Exception e) {
- fail("Should not throw this exception" + e.getMessage());
- }
- }
-
- @Test
- public void should_returnNull_givenEmptyResponseResult_reIndexRecordsTest() {
- try {
- recordQueryResponse.setResults(new ArrayList<>());
- when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
-
- String response = sut.reindexRecords(recordReindexRequest, false);
-
- Assert.assertNull(response);
- } catch (Exception e) {
- fail("Should not throw this exception" + e.getMessage());
- }
- }
-
- @Ignore
- @Test
- public void should_returnRecordQueryRequestPayload_givenValidResponseResult_reIndexRecordsTest() {
- try {
- recordQueryResponse.setCursor(cursor);
- List<String> results = new ArrayList<>();
- results.add("test1");
- recordQueryResponse.setResults(results);
-
- when(configurationProperties.getStorageRecordsBatchSize()).thenReturn(1);
-
- when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
-
- String taskQueuePayload = sut.reindexRecords(recordReindexRequest, false);
-
- Assert.assertEquals("{\"kind\":\"tenant:test:test:1.0.0\",\"cursor\":\"100\"}",
- taskQueuePayload);
- } catch (Exception e) {
- fail("Should not throw exception" + e.getMessage());
- }
- }
-
- @Test
- public void should_returnRecordChangedMessage_givenValidResponseResult_reIndexRecordsTest() {
- try {
- List<String> results = new ArrayList<>();
- results.add("test1");
- recordQueryResponse.setResults(results);
- when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
-
- String taskQueuePayload = sut.reindexRecords(recordReindexRequest, false);
-
- Assert.assertEquals(String.format(
- "{\"data\":\"[{\\\"id\\\":\\\"test1\\\",\\\"kind\\\":\\\"tenant:test:test:1.0.0\\\",\\\"op\\\":\\\"create\\\"}]\",\"attributes\":{\"correlation-id\":\"%s\"}}",
- correlationId), taskQueuePayload);
- } catch (Exception e) {
- fail("Should not throw exception" + e.getMessage());
- }
- }
-}
diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceTest.java
deleted file mode 100644
index 47bfb735e84e23c3681d8b003d8e79a11c4c2455..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceTest.java
+++ /dev/null
@@ -1,276 +0,0 @@
-/*
- * Copyright 2021 Google LLC
- * Copyright 2021 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.service;
-
-import static java.util.Collections.singletonList;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.fail;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.powermock.api.mockito.PowerMockito.when;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
-import java.lang.reflect.Type;
-import java.net.URISyntaxException;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.ArgumentMatchers;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.mockito.Mockito;
-import org.mockito.Spy;
-import org.opengroup.osdu.core.common.http.IUrlFetchService;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.http.HttpResponse;
-import org.opengroup.osdu.core.common.model.indexer.IndexingStatus;
-import org.opengroup.osdu.core.common.model.indexer.JobStatus;
-import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
-import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
-import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
-import org.opengroup.osdu.core.common.model.indexer.Records;
-import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.springframework.http.HttpStatus;
-import org.springframework.test.context.junit4.SpringRunner;
-
-@RunWith(SpringRunner.class)
-public class StorageServiceTest {
-
- @Mock
- private IUrlFetchService urlFetchService;
- @Mock
- private JobStatus jobStatus;
- @Mock
- private JaxRsDpsLog log;
- @Mock
- private IRequestInfo requestInfo;
- @Mock
- private IndexerConfigurationProperties configurationProperties;
- @Spy
- private ObjectMapper objectMapper = new ObjectMapper();
- @InjectMocks
- private StorageServiceImpl sut;
-
- private List<String> ids;
- private static final String RECORD_ID1 = "tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465";
- private static final String RECORDS_ID2 = "tenant1:doc:15e790a69beb4d789b1f979e2af2e813";
-
- @Before
- public void setup() {
-
- String recordChangedMessages =
- "[{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"purge\"},"
- +
- "{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"create\"}]";
-
- when(this.requestInfo.getHeadersMap()).thenReturn(new HashMap<>());
- when(this.requestInfo.getHeaders()).thenReturn(new DpsHeaders());
-
- Type listType = new TypeToken<List<RecordInfo>>() {
- }.getType();
-
- List<RecordInfo> msgs = (new Gson()).fromJson(recordChangedMessages, listType);
- jobStatus.initialize(msgs);
- ids = Arrays.asList(RECORD_ID1, RECORDS_ID2);
-
- when(configurationProperties.getStorageRecordsBatchSize()).thenReturn(20);
- }
-
- @Test
- public void should_return404_givenNullData_getValidStorageRecordsTest()
- throws URISyntaxException {
-
- HttpResponse httpResponse = mock(HttpResponse.class);
- Mockito.when(httpResponse.getBody()).thenReturn(null);
-
- when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
-
- should_return404_getValidStorageRecordsTest();
- }
-
- @Test
- public void should_return404_givenEmptyData_getValidStorageRecordsTest()
- throws URISyntaxException {
-
- String emptyDataFromStorage = "{\"records\":[],\"notFound\":[]}";
-
- HttpResponse httpResponse = mock(HttpResponse.class);
- Mockito.when(httpResponse.getBody()).thenReturn(emptyDataFromStorage);
-
- when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
-
- should_return404_getValidStorageRecordsTest();
- }
-
- @Test
- public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsTest()
- throws URISyntaxException {
-
- String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"], \"conversionStatuses\": []}";
-
- HttpResponse httpResponse = mock(HttpResponse.class);
- Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
-
- when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
- Records storageRecords = this.sut.getStorageRecords(ids);
-
- assertEquals(1, storageRecords.getRecords().size());
- }
-
- @Test
- public void should_logMissingRecord_given_storageMissedRecords() throws URISyntaxException {
-
- String validDataFromStorage = "{\"records\":[{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[]}";
-
- HttpResponse httpResponse = mock(HttpResponse.class);
- Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
-
- when(this.urlFetchService.sendRequest(any())).thenReturn(httpResponse);
- Records storageRecords = this.sut.getStorageRecords(ids);
-
- assertEquals(1, storageRecords.getRecords().size());
- verify(this.jobStatus).addOrUpdateRecordStatus(singletonList(RECORDS_ID2), IndexingStatus.FAIL,
- HttpStatus.NOT_FOUND.value(),
- "Partial response received from Storage service - missing records",
- "Partial response received from Storage service: tenant1:doc:15e790a69beb4d789b1f979e2af2e813");
- }
-
- @Test
- public void should_returnValidJobStatus_givenFailedUnitsConversion_processRecordChangedMessageTest()
- throws URISyntaxException {
- String validDataFromStorage = "{\"records\":[{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[],\"conversionStatuses\":[{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\",\"status\":\"ERROR\",\"errors\":[\"crs conversion failed\"]}]}";
-
- HttpResponse httpResponse = mock(HttpResponse.class);
- Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
-
- when(this.urlFetchService.sendRequest(any())).thenReturn(httpResponse);
- Records storageRecords = this.sut.getStorageRecords(singletonList(RECORDS_ID2));
-
- assertEquals(1, storageRecords.getRecords().size());
- verify(this.jobStatus)
- .addOrUpdateRecordStatus(RECORDS_ID2, IndexingStatus.WARN, HttpStatus.BAD_REQUEST.value(),
- "crs conversion failed", String
- .format("record-id: %s | %s", "tenant1:doc:15e790a69beb4d789b1f979e2af2e813",
- "crs conversion failed"));
- }
-
- @Test
- public void should_returnValidResponse_givenValidRecordQueryRequest_getRecordListByKind()
- throws Exception {
-
- RecordReindexRequest recordReindexRequest = RecordReindexRequest.builder()
- .kind("tenant:test:test:1.0.0").cursor("100").build();
-
- HttpResponse httpResponse = new HttpResponse();
- httpResponse.setBody(new Gson().toJson(recordReindexRequest, RecordReindexRequest.class));
-
- when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
-
- RecordQueryResponse recordQueryResponse = this.sut.getRecordsByKind(recordReindexRequest);
-
- assertEquals("100", recordQueryResponse.getCursor());
- assertNull(recordQueryResponse.getResults());
- }
-
- @Test
- public void should_returnValidResponse_givenValidKind_getSchemaByKind() throws Exception {
-
- String validSchemaFromStorage = "{" +
- " \"kind\": \"tenant:test:test:1.0.0\"," +
- " \"schema\": [" +
- " {" +
- " \"path\": \"msg\"," +
- " \"kind\": \"string\"" +
- " }," +
- " {" +
- " \"path\": \"references.entity\"," +
- " \"kind\": \"string\"" +
- " }" +
- " ]," +
- " \"ext\": null" +
- "}";
- String kind = "tenant:test:test:1.0.0";
-
- HttpResponse httpResponse = new HttpResponse();
- httpResponse.setResponseCode(HttpStatus.OK.value());
- httpResponse.setBody(validSchemaFromStorage);
-
- when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
-
- String recordSchemaResponse = this.sut.getStorageSchema(kind);
-
- assertNotNull(recordSchemaResponse);
- }
-
- @Test
- public void should_returnNullResponse_givenAbsentKind_getSchemaByKind() throws Exception {
-
- String kind = "tenant:test:test:1.0.0";
-
- HttpResponse httpResponse = new HttpResponse();
- httpResponse.setResponseCode(HttpStatus.NOT_FOUND.value());
-
- when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
-
- String recordSchemaResponse = this.sut.getStorageSchema(kind);
-
- assertNull(recordSchemaResponse);
- }
-
- @Test
- public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsWithInvalidConversionTest()
- throws URISyntaxException {
-
- String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"],\"conversionStatuses\": [{\"id\":\"testid\",\"status\":\"ERROR\",\"errors\":[\"conversion error occurred\"] } ]}";
-
- HttpResponse httpResponse = mock(HttpResponse.class);
- Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
-
- when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
- Records storageRecords = this.sut.getStorageRecords(ids);
-
- assertEquals(1, storageRecords.getRecords().size());
-
- assertEquals(1, storageRecords.getConversionStatuses().get(0).getErrors().size());
-
- assertEquals("conversion error occurred",
- storageRecords.getConversionStatuses().get(0).getErrors().get(0));
- }
-
- private void should_return404_getValidStorageRecordsTest() {
- try {
- this.sut.getStorageRecords(ids);
- fail("Should throw exception");
- } catch (AppException e) {
- assertEquals(HttpStatus.NOT_FOUND.value(), e.getError().getCode());
- } catch (Exception e) {
- fail("Should not throw this exception" + e.getMessage());
- }
- }
-}
diff --git a/provider/indexer-reference/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/provider/indexer-reference/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker
deleted file mode 100644
index ca6ee9cea8ec189a088d50559325d4e84ff8ad09..0000000000000000000000000000000000000000
--- a/provider/indexer-reference/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker
+++ /dev/null
@@ -1 +0,0 @@
-mock-maker-inline
\ No newline at end of file
diff --git a/testing/indexer-test-azure/pom.xml b/testing/indexer-test-azure/pom.xml
index 3b87e83e6dd4b382149c774bfe70281872c847e8..efa8c10bb5cb69659d4489153256c857a9025530 100644
--- a/testing/indexer-test-azure/pom.xml
+++ b/testing/indexer-test-azure/pom.xml
@@ -37,6 +37,7 @@
<junit.jupiter.version>5.6.0</junit.jupiter.version>
<elasticsearch.version>7.8.1</elasticsearch.version>
<os-core-common.version>0.14.0-rc8</os-core-common.version>
+ <jackson.version>2.13.2</jackson.version>
</properties>
<dependencies>
@@ -206,12 +207,12 @@
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
- <version>2.12.2</version>
+ <version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
- <version>2.12.2</version>
+ <version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>org.mockito</groupId>