diff --git a/pom.xml b/pom.xml
index 8d6c8317da1c8ef7ddad77b2d371afa6cff7774d..eef38efd9dc660066d903e17802171631c6f969e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -136,6 +136,7 @@
<module>provider/indexer-azure</module>
<module>provider/indexer-gcp</module>
<module>provider/indexer-ibm</module>
+ <module>provider/indexer-reference</module>
</modules>
<repositories>
diff --git a/provider/indexer-reference/Dockerfile b/provider/indexer-reference/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..1e6901f365e2e9c7049c5019b4037ade931b65f0
--- /dev/null
+++ b/provider/indexer-reference/Dockerfile
@@ -0,0 +1,5 @@
+FROM openjdk:8-slim
+WORKDIR /app
+COPY target/indexer-reference-0.0.1-SNAPSHOT-spring-boot.jar indexer-reference.jar
+# Run the web service on container startup.
+CMD java -Djava.security.egd=file:/dev/./urandom -Dserver.port=8080 -jar /app/indexer-reference.jar
diff --git a/provider/indexer-reference/docker-compose.yml b/provider/indexer-reference/docker-compose.yml
new file mode 100644
index 0000000000000000000000000000000000000000..a6838447de241e658325e275434293b2d4e2be09
--- /dev/null
+++ b/provider/indexer-reference/docker-compose.yml
@@ -0,0 +1,12 @@
+version: "3"
+services:
+ os-storage-app:
+ build:
+ args:
+ JAR_FILE: target/indexer-reference-0.0.1-SNAPSHOT-spring-boot.jar
+ context: ""
+ dockerfile: ../Dockerfile
+ image: us.gcr.io/osdu-anthos-02/os-indexer/anthos-indexer-reference
+ ports:
+ - "8080:8080"
+
diff --git a/provider/indexer-reference/kubernetes/deployments/deployment-os-indexer-service.yml b/provider/indexer-reference/kubernetes/deployments/deployment-os-indexer-service.yml
new file mode 100644
index 0000000000000000000000000000000000000000..0157d9875389754f1c640f0d186e02b2ea0020d8
--- /dev/null
+++ b/provider/indexer-reference/kubernetes/deployments/deployment-os-indexer-service.yml
@@ -0,0 +1,179 @@
+apiVersion: v1
+data:
+ AUTHORIZE_API: ${AUTHORIZE_API}
+ MONGO_DB_URL: ${MONGO_DB_URL}
+ MONGO_DB_USER: ${MONGO_DB_USER}
+ MONGO_DB_NAME: ${MONGO_DB_NAME}
+ REGION: ${REGION}
+ LEGALTAG_API: ${LEGALTAG_API}
+ INDEXER_HOST: ${INDEXER_HOST}
+ GOOGLE_CLOUD_PROJECT: ${GOOGLE_CLOUD_PROJECT}
+ STORAGE_HOSTNAME: ${STORAGE_HOSTNAME}
+ STORAGE_SCHEMA_HOST: ${STORAGE_SCHEMA_HOST}
+ STORAGE_QUERY_RECORD_HOST: ${STORAGE_QUERY_RECORD_HOST}
+ STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST: ${STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST}
+ INDEXER_QUEUE_HOST: ${INDEXER_QUEUE_HOST}
+ CRS_API: ${CRS_API}
+ REDIS_GROUP_HOST: ${REDIS_GROUP_HOST}
+ REDIS_SEARCH_HOST: ${REDIS_SEARCH_HOST}
+ GOOGLE_AUDIENCES: ${GOOGLE_AUDIENCES}
+ KEY_RING: ${KEY_RING}
+ KMS_KEY: ${KMS_KEY}
+kind: ConfigMap
+metadata:
+ labels:
+ app: indexer-reference
+ name: indexer-config
+ namespace: default
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ generateName: indexer-reference-anthos
+ labels:
+ app: indexer-reference
+ name: indexer-reference
+ namespace: default
+spec:
+ selector:
+ matchLabels:
+ app: indexer-reference
+ replicas: 1
+ template:
+ metadata:
+ labels:
+ app: indexer-reference
+ spec:
+ containers:
+ - env:
+ - name: AUTHORIZE_API
+ valueFrom:
+ configMapKeyRef:
+ key: AUTHORIZE_API
+ name: indexer-config
+ - name: MONGO_DB_URL
+ valueFrom:
+ configMapKeyRef:
+ key: MONGO_DB_URL
+ name: indexer-config
+ - name: MONGO_DB_USER
+ valueFrom:
+ configMapKeyRef:
+ key: MONGO_DB_USER
+ name: indexer-config
+ - name: MONGO_DB_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: indexer-secret
+ key: mongo.db.password
+ - name: MONGO_DB_NAME
+ valueFrom:
+ configMapKeyRef:
+ key: MONGO_DB_NAME
+ name: indexer-config
+ - name: MB_RABBITMQ_URI
+ valueFrom:
+ secretKeyRef:
+ name: indexer-secret
+ key: mb.rabbitmq.uri
+ - name: REGION
+ valueFrom:
+ configMapKeyRef:
+ key: REGION
+ name: indexer-config
+ - name: LEGALTAG_API
+ valueFrom:
+ configMapKeyRef:
+ key: LEGALTAG_API
+ name: indexer-config
+ - name: INDEXER_HOST
+ valueFrom:
+ configMapKeyRef:
+ key: INDEXER_HOST
+ name: indexer-config
+ - name: GOOGLE_CLOUD_PROJECT
+ valueFrom:
+ configMapKeyRef:
+ key: GOOGLE_CLOUD_PROJECT
+ name: indexer-config
+ - name: STORAGE_HOSTNAME
+ valueFrom:
+ configMapKeyRef:
+ key: STORAGE_HOSTNAME
+ name: indexer-config
+ - name: STORAGE_SCHEMA_HOST
+ valueFrom:
+ configMapKeyRef:
+ key: STORAGE_SCHEMA_HOST
+ name: indexer-config
+ - name: STORAGE_QUERY_RECORD_HOST
+ valueFrom:
+ configMapKeyRef:
+ key: STORAGE_QUERY_RECORD_HOST
+ name: indexer-config
+ - name: STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST
+ valueFrom:
+ configMapKeyRef:
+ key: STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST
+ name: indexer-config
+ - name: INDEXER_QUEUE_HOST
+ valueFrom:
+ configMapKeyRef:
+ key: INDEXER_QUEUE_HOST
+ name: indexer-config
+ - name: CRS_API
+ valueFrom:
+ configMapKeyRef:
+ key: CRS_API
+ name: indexer-config
+ - name: REDIS_GROUP_HOST
+ valueFrom:
+ configMapKeyRef:
+ key: REDIS_GROUP_HOST
+ name: indexer-config
+ - name: REDIS_SEARCH_HOST
+ valueFrom:
+ configMapKeyRef:
+ key: REDIS_SEARCH_HOST
+ name: indexer-config
+ - name: GOOGLE_AUDIENCES
+ valueFrom:
+ configMapKeyRef:
+ key: GOOGLE_AUDIENCES
+ name: indexer-config
+ - name: KEY_RING
+ valueFrom:
+ configMapKeyRef:
+ key: KEY_RING
+ name: indexer-config
+ - name: KMS_KEY
+ valueFrom:
+ configMapKeyRef:
+ key: KMS_KEY
+ name: indexer-config
+ image: us.gcr.io/osdu-anthos-02/os-indexer/anthos-indexer-reference:9966597-dirty
+ name: indexer-reference
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: indexer-reference
+ namespace: default
+spec:
+ ports:
+ - protocol: TCP
+ port: 80
+ targetPort: 8080
+ selector:
+ app: indexer-reference
+ type: LoadBalancer
+---
+apiVersion: v1
+data:
+ mongo.db.password: ${mongo.db.password}
+ mb.rabbitmq.uri: ${mb.rabbitmq.uri}
+kind: Secret
+metadata:
+ name: indexer-secret
+ namespace: default
+type: Opaque
\ No newline at end of file
diff --git a/provider/indexer-reference/pom.xml b/provider/indexer-reference/pom.xml
new file mode 100644
index 0000000000000000000000000000000000000000..99d01082c723b55da8f969a693d301daf9378570
--- /dev/null
+++ b/provider/indexer-reference/pom.xml
@@ -0,0 +1,197 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Copyright 2020 Google LLC
+ ~ Copyright 2020 EPAM Systems, Inc
+ ~
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ https://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.opengroup.osdu.indexer</groupId>
+ <artifactId>indexer-service</artifactId>
+ <version>1.0.4-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>indexer-reference</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+ <name>indexer-reference</name>
+ <description>Indexer Service GCP Anthos</description>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.opengroup.osdu.indexer</groupId>
+ <artifactId>indexer-core</artifactId>
+ <version>1.0.6-SNAPSHOT</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.opengroup.osdu</groupId>
+ <artifactId>core-lib-gcp</artifactId>
+ <version>0.1.17</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.opengroup.osdu</groupId>
+ <artifactId>os-core-common</artifactId>
+ <version>0.0.18</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-data-mongodb</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.cloud</groupId>
+ <artifactId>google-cloud-logging</artifactId>
+ <version>1.72.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.apis</groupId>
+ <artifactId>google-api-services-storage</artifactId>
+ <version>v1-rev150-1.25.0</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava-jdk5</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.google.apis</groupId>
+ <artifactId>google-api-services-cloudkms</artifactId>
+ <version>v1-rev81-1.25.0</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-amqp</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch</artifactId>
+ <version>6.6.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-client</artifactId>
+ <version>6.6.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-high-level-client</artifactId>
+ <version>6.6.2</version>
+ </dependency>
+
+ <!-- Test Dependencies -->
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- Test Dependencies -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.12</version>
+ <scope>test</scope>
+ </dependency>
+ <!-- https://mvnrepository.com/artifact/org.powermock/powermock-api-mockito2 -->
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-mockito2</artifactId>
+ <version>2.0.2</version>
+ <scope>test</scope>
+ </dependency>
+ <!-- https://mvnrepository.com/artifact/org.powermock/powermock-module-junit4 -->
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <version>2.0.2</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <version>2.26.0</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>cobertura-maven-plugin</artifactId>
+ <version>2.7</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.auth0</groupId>
+ <artifactId>java-jwt</artifactId>
+ <version>3.8.1</version>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/com.github.stefanbirkner/system-rules -->
+ <dependency>
+ <groupId>com.github.stefanbirkner</groupId>
+ <artifactId>system-rules</artifactId>
+ <version>1.2.0</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.api.grpc</groupId>
+ <artifactId>proto-google-iam-v1</artifactId>
+ <version>0.12.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-actuator-autoconfigure</artifactId>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>repackage</goal>
+ </goals>
+ <configuration>
+ <classifier>spring-boot</classifier>
+ <mainClass>
+ org.opengroup.osdu.indexer.IndexerAnthosApplication
+ </mainClass>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+
+</project>
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/IndexerAnthosApplication.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/IndexerAnthosApplication.java
new file mode 100644
index 0000000000000000000000000000000000000000..d2c3e68f06d2da43a8988aa470b58d2002f36888
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/IndexerAnthosApplication.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer;
+
+import org.opengroup.osdu.core.gcp.multitenancy.TenantFactory;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration;
+import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.FilterType;
+
+@SpringBootApplication(exclude = {MongoAutoConfiguration.class, SecurityAutoConfiguration.class,
+ ManagementWebSecurityAutoConfiguration.class})
+@Configuration
+@ComponentScan(value = {"org.opengroup.osdu"}, excludeFilters = {
+ @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = TenantFactory.class)})
+public class IndexerAnthosApplication {
+
+ public static void main(String[] args) {
+ SpringApplication.run(IndexerAnthosApplication.class, args);
+ }
+
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/ServletInitializer.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/ServletInitializer.java
new file mode 100644
index 0000000000000000000000000000000000000000..ec6996fe06ba02a1675486c9c22d341389736d32
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/ServletInitializer.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer;
+
+import org.springframework.boot.builder.SpringApplicationBuilder;
+import org.springframework.boot.web.servlet.support.SpringBootServletInitializer;
+
+public class ServletInitializer extends SpringBootServletInitializer {
+
+ @Override
+ protected SpringApplicationBuilder configure(SpringApplicationBuilder application) {
+ return application.sources(IndexerAnthosApplication.class);
+ }
+}
+
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/AttributesCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/AttributesCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..b60b4d14405fbadaaa0e25977b81e08fb66fe905
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/AttributesCache.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import java.util.Set;
+import org.opengroup.osdu.core.common.cache.RedisCache;
+import org.opengroup.osdu.core.common.provider.interfaces.IAttributesCache;
+import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class AttributesCache implements IAttributesCache<String, Set>, AutoCloseable {
+
+ private RedisCache<String, Set> cache;
+
+ @Autowired
+ public AttributesCache(IndexerConfigurationProperties indexerConfigurationProperties) {
+ cache = new RedisCache(indexerConfigurationProperties.getRedisSearchHost(),
+ Integer.parseInt(indexerConfigurationProperties.getRedisSearchPort()),
+ indexerConfigurationProperties.getIndexCacheExpiration() * 60,
+ String.class,
+ Boolean.class);
+ }
+
+ @Override
+ public void put(String key, Set value) {
+ this.cache.put(key, value);
+ }
+
+ @Override
+ public Set get(String key) {
+ return this.cache.get(key);
+ }
+
+ @Override
+ public void delete(String key) {
+ this.cache.delete(key);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+
+ @Override
+ public void close() {
+ this.cache.close();
+ }
+}
\ No newline at end of file
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..6e832b44671a8ac95b65556076bf15122c6990a3
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import com.google.auth.oauth2.AccessToken;
+import org.opengroup.osdu.core.common.cache.RedisCache;
+import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class DatastoreCredentialCache extends RedisCache<String, AccessToken> {
+
+ @Autowired
+ public DatastoreCredentialCache(IndexerConfigurationProperties indexerConfigurationProperties) {
+ super(indexerConfigurationProperties.getRedisSearchHost(),
+ Integer.parseInt(indexerConfigurationProperties.getRedisSearchPort()),
+ 58 * 60,
+ String.class,
+ AccessToken.class);
+ }
+}
\ No newline at end of file
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..24804bc9000a8f10298b5e30a033ed99886c90de
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.RedisCache;
+import org.opengroup.osdu.core.common.model.search.ClusterSettings;
+import org.opengroup.osdu.core.common.provider.interfaces.IElasticCredentialsCache;
+import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class ElasticCredentialsCache implements IElasticCredentialsCache<String, ClusterSettings>,
+ AutoCloseable {
+
+ private RedisCache<String, ClusterSettings> cache;
+
+ @Autowired
+ public ElasticCredentialsCache(IndexerConfigurationProperties indexerConfigurationProperties) {
+ cache = new RedisCache<>(indexerConfigurationProperties.getRedisSearchHost(),
+ Integer.parseInt(indexerConfigurationProperties.getRedisSearchPort()),
+ indexerConfigurationProperties.getIndexCacheExpiration() * 60,
+ String.class,
+ ClusterSettings.class);
+ }
+
+ @Override
+ public void close() throws Exception {
+ this.cache.close();
+ }
+
+ @Override
+ public void put(String s, ClusterSettings o) {
+ this.cache.put(s, o);
+ }
+
+ @Override
+ public ClusterSettings get(String s) {
+ return this.cache.get(s);
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(s);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/IndexCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/IndexCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..de447717623d63ded8d1d1c46efc42817d618183
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/IndexCache.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.RedisCache;
+import org.opengroup.osdu.core.common.provider.interfaces.IIndexCache;
+import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class IndexCache implements IIndexCache<String, Boolean>, AutoCloseable {
+
+ private RedisCache<String, Boolean> cache;
+
+ @Autowired
+ public IndexCache(IndexerConfigurationProperties indexerConfigurationProperties) {
+ cache = new RedisCache<>(indexerConfigurationProperties.getRedisSearchHost(),
+ Integer.parseInt(indexerConfigurationProperties.getRedisSearchPort()),
+ indexerConfigurationProperties.getIndexCacheExpiration() * 60,
+ String.class,
+ Boolean.class);
+ }
+
+ @Override
+ public void close() throws Exception {
+ this.cache.close();
+ }
+
+ @Override
+ public void put(String s, Boolean o) {
+ this.cache.put(s, o);
+ }
+
+ @Override
+ public Boolean get(String s) {
+ return this.cache.get(s);
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(s);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/JwtCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/JwtCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..9a2a02d753dfbbe506a10bab3ecb2d6083876bef
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/JwtCache.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.RedisCache;
+import org.opengroup.osdu.core.common.model.search.IdToken;
+import org.opengroup.osdu.core.common.provider.interfaces.IJwtCache;
+import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+@Component
+public class JwtCache implements IJwtCache<String, IdToken>, AutoCloseable {
+
+ RedisCache<String, IdToken> cache;
+
+ // google service account id_token can be requested only for 1 hr
+ private final static int EXPIRED_AFTER = 59;
+
+ public JwtCache(IndexerConfigurationProperties indexerConfigurationProperties) {
+ cache = new RedisCache<>(indexerConfigurationProperties.getRedisSearchHost(),
+ Integer.parseInt(indexerConfigurationProperties.getRedisSearchPort()),
+ EXPIRED_AFTER * 60,
+ String.class,
+ IdToken.class);
+ }
+
+ @Override
+ public void close() throws Exception {
+ this.cache.close();
+ }
+
+ @Override
+ public void put(String s, IdToken o) {
+ this.cache.put(s, o);
+ }
+
+ @Override
+ public IdToken get(String s) {
+ return this.cache.get(s);
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(s);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/KindsCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/KindsCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..afc2a8c39299158c3a50e1e9f5d1f5b67e5a6095
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/KindsCache.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import java.util.Set;
+import org.opengroup.osdu.core.common.cache.RedisCache;
+import org.opengroup.osdu.core.common.provider.interfaces.IKindsCache;
+import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class KindsCache implements IKindsCache<String, Set>, AutoCloseable {
+
+ private RedisCache<String, Set> cache;
+
+ @Autowired
+ public KindsCache(IndexerConfigurationProperties indexerConfigurationProperties) {
+ cache = new RedisCache<>(indexerConfigurationProperties.getRedisSearchHost(),
+ Integer.parseInt(indexerConfigurationProperties.getRedisSearchPort()),
+ indexerConfigurationProperties.getKindsCacheExpiration() * 60,
+ indexerConfigurationProperties.getKindsRedisDatabase(),
+ String.class,
+ Set.class);
+ }
+
+ @Override
+ public void close() throws Exception {
+ this.cache.close();
+ }
+
+ @Override
+ public void put(String s, Set o) {
+ this.cache.put(s, o);
+ }
+
+ @Override
+ public Set get(String s) {
+ return this.cache.get(s);
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(s);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/SchemaCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/SchemaCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..3d84e22959501decedf8dd08564bc0869bee8d50
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/SchemaCache.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.RedisCache;
+import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class SchemaCache implements ISchemaCache<String, String>, AutoCloseable {
+
+ private RedisCache<String, String> cache;
+
+ @Autowired
+ public SchemaCache(IndexerConfigurationProperties indexerConfigurationProperties) {
+ cache = new RedisCache<>(indexerConfigurationProperties.getRedisSearchHost(),
+ Integer.parseInt(indexerConfigurationProperties.getRedisSearchPort()),
+ indexerConfigurationProperties.getSchemaCacheExpiration() * 60,
+ String.class,
+ String.class);
+ }
+
+ @Override
+ public void close() throws Exception {
+ this.cache.close();
+ }
+
+ @Override
+ public void put(String s, String o) {
+ this.cache.put(s, o);
+ }
+
+ @Override
+ public String get(String s) {
+ return this.cache.get(s);
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(s);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/EntitlementsConfigProperties.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/EntitlementsConfigProperties.java
new file mode 100644
index 0000000000000000000000000000000000000000..1ba4425a4f2f09ff361c00cbcf4ec554811daf34
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/EntitlementsConfigProperties.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.config;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@ConfigurationProperties
+@Getter
+@Setter
+public class EntitlementsConfigProperties {
+
+ private String AuthorizeApi;
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/MongoDBConfigProperties.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/MongoDBConfigProperties.java
new file mode 100644
index 0000000000000000000000000000000000000000..080f76dfc25465b8539e8c0789e81941bd026e07
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/MongoDBConfigProperties.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.config;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@ConfigurationProperties
+@Getter
+@Setter
+public class MongoDBConfigProperties {
+
+ private String mongoDbUrl;
+ private String mongoDbUser;
+ private String mongoDbPassword;
+ private String mongoDbName;
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/RabbitMqConfigProperties.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/RabbitMqConfigProperties.java
new file mode 100644
index 0000000000000000000000000000000000000000..55efbf1a75763e976d71a840335e60b14c8a5ad4
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/config/RabbitMqConfigProperties.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.config;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@ConfigurationProperties
+@Getter
+@Setter
+public class RabbitMqConfigProperties {
+
+ private String mbRabbitMqUri;
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/EntitlementsClientFactory.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/EntitlementsClientFactory.java
new file mode 100644
index 0000000000000000000000000000000000000000..45131176cd1ff35eb2dc411b0073b245c0d3b8e8
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/EntitlementsClientFactory.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.di;
+
+import org.opengroup.osdu.core.common.entitlements.EntitlementsAPIConfig;
+import org.opengroup.osdu.core.common.entitlements.EntitlementsFactory;
+import org.opengroup.osdu.core.common.entitlements.IEntitlementsFactory;
+import org.opengroup.osdu.indexer.config.EntitlementsConfigProperties;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.config.AbstractFactoryBean;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+@Component
+@RequestScope
+@Lazy
+public class EntitlementsClientFactory extends AbstractFactoryBean<IEntitlementsFactory> {
+
+ private EntitlementsConfigProperties entitlementsConfigProperties;
+
+ @Override
+ protected IEntitlementsFactory createInstance() throws Exception {
+ return new EntitlementsFactory(EntitlementsAPIConfig
+ .builder()
+ .rootUrl(entitlementsConfigProperties.getAuthorizeApi())
+ .build());
+ }
+
+ @Override
+ public Class<?> getObjectType() {
+ return IEntitlementsFactory.class;
+ }
+
+ @Autowired
+ public void setEntitlementsConfigProperties(
+ EntitlementsConfigProperties entitlementsConfigProperties) {
+ this.entitlementsConfigProperties = entitlementsConfigProperties;
+ }
+}
\ No newline at end of file
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/RabbitMQFactoryImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/RabbitMQFactoryImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..3103acc8e88eca93f51397fc921499de1fc11c35
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/RabbitMQFactoryImpl.java
@@ -0,0 +1,87 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.di;
+
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+import java.util.concurrent.TimeoutException;
+import javax.annotation.PostConstruct;
+import org.opengroup.osdu.indexer.config.RabbitMqConfigProperties;
+import org.opengroup.osdu.indexer.messagebus.IMessageFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class RabbitMQFactoryImpl implements IMessageFactory {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RabbitMQFactoryImpl.class);
+ private final RabbitMqConfigProperties rabbitMqConfigProperties;
+
+ private Channel channel;
+
+ @Autowired
+ public RabbitMQFactoryImpl(RabbitMqConfigProperties rabbitMqConfigProperties) {
+ this.rabbitMqConfigProperties = rabbitMqConfigProperties;
+ }
+
+ @PostConstruct
+ private void init() {
+ ConnectionFactory factory = new ConnectionFactory();
+ try {
+ String uri = rabbitMqConfigProperties.getMbRabbitMqUri();
+ LOG.debug(String.format("RabbitMQ Channel = %s", uri));
+ factory.setUri(uri);
+ factory.setAutomaticRecoveryEnabled(true);
+ Connection conn = factory.newConnection();
+ this.channel = conn.createChannel();
+ LOG.debug("RabbitMQ Channel was created.");
+ for (String queue : Arrays.asList(DEFAULT_QUEUE_NAME, INDEXER_QUEUE_NAME, LEGAL_QUEUE_NAME)) {
+ channel.queueDeclare(queue, true, false, false, null);
+ LOG.debug(String.format("Queue [%s] was declared.", queue));
+ }
+ } catch (KeyManagementException | NoSuchAlgorithmException | URISyntaxException | IOException | TimeoutException e) {
+ LOG.error(e.getMessage(), e);
+ }
+
+ }
+
+ @Override
+ public void sendMessage(String msg) {
+ this.sendMessage("records", msg);
+ }
+
+ @Override
+ public void sendMessage(String queueName, String msg) {
+ String queueNameWithPrefix = queueName;
+ try {
+ channel.basicPublish("", queueNameWithPrefix, null, msg.getBytes());
+ LOG.info(String.format("[x] Sent '%s' to queue [%s]", msg, queueNameWithPrefix));
+ } catch (IOException e) {
+ LOG.error(String.format("Unable to publish message to [%s]", queueNameWithPrefix));
+ LOG.error(e.getMessage(), e);
+ }
+ }
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..b60b6e590e5ec671110e74fa49949391c219ce23
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryImpl.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.di;
+
+
+import com.google.gson.Gson;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import org.bson.Document;
+import org.opengroup.osdu.core.common.cache.ICache;
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
+import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
+import org.opengroup.osdu.indexer.persistence.MongoDdmsClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Primary;
+import org.springframework.stereotype.Component;
+
+@Primary
+@Component
+public class TenantFactoryImpl implements ITenantFactory {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TenantFactoryImpl.class);
+
+ public static final String MAIN_DATABASE = "main";
+ public static final String TENANT_INFO = "tenantinfo";
+
+
+ private final MongoDdmsClient mongoClient;
+
+ @Autowired
+ public TenantFactoryImpl(MongoDdmsClient mongoClient) {
+ this.mongoClient = mongoClient;
+ }
+
+ private Map<String, TenantInfo> tenants;
+
+ public boolean exists(String tenantName) {
+ if (this.tenants == null) {
+ initTenants();
+ }
+ return this.tenants.containsKey(tenantName);
+ }
+
+ public TenantInfo getTenantInfo(String tenantName) {
+ if (this.tenants == null) {
+ initTenants();
+ }
+ return this.tenants.get(tenantName);
+ }
+
+ public Collection<TenantInfo> listTenantInfo() {
+ if (this.tenants == null) {
+ initTenants();
+ }
+ return this.tenants.values();
+ }
+
+ public <V> ICache<String, V> createCache(String tenantName, String host, int port,
+ int expireTimeSeconds, Class<V> classOfV) {
+ return null;
+ }
+
+ public void flushCache() {
+ }
+
+ private void initTenants() {
+ this.tenants = new HashMap<>();
+ MongoCollection<Document> mongoCollection = mongoClient
+ .getMongoCollection(MAIN_DATABASE, TENANT_INFO);
+ FindIterable<Document> results = mongoCollection.find();
+ if (Objects.isNull(results) && Objects.isNull(results.first())) {
+ LOG.error(String.format("Collection \'%s\' is empty.", results));
+ }
+ for (Document document : results) {
+ TenantInfo tenantInfo = new Gson().fromJson(document.toJson(), TenantInfo.class);
+ this.tenants.put(tenantInfo.getName(), tenantInfo);
+ }
+ }
+
+}
+
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/messagebus/IMessageFactory.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/messagebus/IMessageFactory.java
new file mode 100644
index 0000000000000000000000000000000000000000..21316cc68c9d5fa64615a8616f3f6a0e533f8ed4
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/messagebus/IMessageFactory.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.messagebus;
+
+public interface IMessageFactory {
+ String DEFAULT_QUEUE_NAME = "records";
+ String LEGAL_QUEUE_NAME = "legal";
+ String INDEXER_QUEUE_NAME = "indexer";
+
+ void sendMessage(String msg);
+
+ void sendMessage(String queueName, String msg);
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/middleware/IndexFilter.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/middleware/IndexFilter.java
new file mode 100644
index 0000000000000000000000000000000000000000..12cffcf01340bf7c7ce763e2ba7a19202ed1a8e6
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/middleware/IndexFilter.java
@@ -0,0 +1,127 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.middleware;
+
+import com.google.common.base.Strings;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import lombok.extern.java.Log;
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.core.common.http.ResponseHeaders;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.search.DeploymentEnvironment;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpMethod;
+import org.springframework.stereotype.Component;
+
+@Log
+@Component
+public class IndexFilter implements Filter {
+
+
+ private final IndexerConfigurationProperties indexerConfigurationProperties;
+ private final DpsHeaders dpsHeaders;
+ private final IRequestInfo requestInfo;
+
+ private FilterConfig filterConfig;
+
+ private static final String PATH_SWAGGER = "/swagger.json";
+ private static final String PATH_TASK_HANDLERS = "task-handlers";
+ private static final String PATH_CRON_HANDLERS = "cron-handlers";
+
+ @Autowired
+ public IndexFilter(IndexerConfigurationProperties indexerConfigurationProperties,
+ DpsHeaders dpsHeaders, IRequestInfo requestInfo) {
+ this.indexerConfigurationProperties = indexerConfigurationProperties;
+ this.dpsHeaders = dpsHeaders;
+ this.requestInfo = requestInfo;
+ }
+
+ @Override
+ public void init(FilterConfig filterConfig) throws ServletException {
+ }
+
+ @Override
+ public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse,
+ FilterChain filterChain)
+ throws IOException, ServletException {
+ HttpServletRequest httpRequest = (HttpServletRequest) servletRequest;
+ String uri = httpRequest.getRequestURI().toLowerCase();
+
+ if (httpRequest.getMethod().equalsIgnoreCase(HttpMethod.POST.name()) && uri
+ .contains(PATH_TASK_HANDLERS)) {
+ if (DeploymentEnvironment.valueOf(indexerConfigurationProperties.getEnvironment())
+ != DeploymentEnvironment.LOCAL) {
+ checkWorkerApiAccess(requestInfo);
+ }
+ }
+
+ if (httpRequest.getMethod().equalsIgnoreCase(HttpMethod.GET.name()) && uri
+ .contains(PATH_CRON_HANDLERS)) {
+ checkWorkerApiAccess(requestInfo);
+ }
+
+ filterChain.doFilter(servletRequest, servletResponse);
+
+ HttpServletResponse httpResponse = (HttpServletResponse) servletResponse;
+ Map<String, List<Object>> standardHeaders = ResponseHeaders.STANDARD_RESPONSE_HEADERS;
+ for (Map.Entry<String, List<Object>> header : standardHeaders.entrySet()) {
+ httpResponse.addHeader(header.getKey(), header.getValue().toString());
+ }
+ if (httpResponse.getHeader(DpsHeaders.CORRELATION_ID) == null) {
+ httpResponse.addHeader(DpsHeaders.CORRELATION_ID, dpsHeaders.getCorrelationId());
+ }
+ }
+
+ @Override
+ public void destroy() {
+ }
+
+ private void checkWorkerApiAccess(IRequestInfo requestInfo) {
+ }
+
+ private List<String> validateAccountId(DpsHeaders requestHeaders) {
+ String accountHeader = requestHeaders.getPartitionIdWithFallbackToAccountId();
+ String debuggingInfo = String.format("%s:%s", DpsHeaders.DATA_PARTITION_ID, accountHeader);
+
+ if (Strings.isNullOrEmpty(accountHeader)) {
+ throw new AppException(HttpStatus.SC_BAD_REQUEST, "Bad request",
+ "invalid or empty data partition", debuggingInfo);
+ }
+
+ List<String> dataPartitions = Arrays.asList(accountHeader.trim().split("\\s*,\\s*"));
+ if (dataPartitions.isEmpty() || dataPartitions.size() > 1) {
+ throw new AppException(HttpStatus.SC_BAD_REQUEST, "Bad request",
+ "invalid or empty data partition", debuggingInfo);
+ }
+ return dataPartitions;
+ }
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreCredential.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreCredential.java
new file mode 100644
index 0000000000000000000000000000000000000000..9b7f8b1f5edd751f35dcd850ec57073c594f838f
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreCredential.java
@@ -0,0 +1,123 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.persistence;
+
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.iam.v1.Iam;
+import com.google.api.services.iam.v1.Iam.Projects.ServiceAccounts.SignJwt;
+import com.google.api.services.iam.v1.IamScopes;
+import com.google.api.services.iam.v1.model.SignJwtRequest;
+import com.google.api.services.iam.v1.model.SignJwtResponse;
+import com.google.auth.oauth2.AccessToken;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.gson.JsonObject;
+import java.util.Collections;
+import java.util.Date;
+import org.apache.commons.lang3.time.DateUtils;
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
+import org.opengroup.osdu.core.common.util.Crc32c;
+import org.opengroup.osdu.indexer.cache.DatastoreCredentialCache;
+
+public class DatastoreCredential extends GoogleCredentials {
+
+ private static final long serialVersionUID = 8344377091688956815L;
+ private static final JsonFactory JSON_FACTORY = new JacksonFactory();
+ private Iam iam;
+
+ private final TenantInfo tenant;
+ private final DatastoreCredentialCache cache;
+
+ protected DatastoreCredential(TenantInfo tenant, DatastoreCredentialCache cache) {
+ this.tenant = tenant;
+ this.cache = cache;
+ }
+
+ @Override
+ public AccessToken refreshAccessToken() {
+
+ String cacheKey = this.getCacheKey();
+
+ AccessToken accessToken = this.cache.get(cacheKey);
+
+ if (accessToken != null) {
+ return accessToken;
+ }
+
+ try {
+ SignJwtRequest signJwtRequest = new SignJwtRequest();
+ signJwtRequest.setPayload(this.getPayload());
+
+ String serviceAccountName = String
+ .format("projects/-/serviceAccounts/%s", this.tenant.getServiceAccount());
+
+ SignJwt signJwt = this.getIam().projects().serviceAccounts()
+ .signJwt(serviceAccountName, signJwtRequest);
+
+ SignJwtResponse signJwtResponse = signJwt.execute();
+ String signedJwt = signJwtResponse.getSignedJwt();
+
+ accessToken = new AccessToken(signedJwt, DateUtils.addSeconds(new Date(), 3600));
+
+ this.cache.put(cacheKey, accessToken);
+
+ return accessToken;
+ } catch (Exception e) {
+ throw new RuntimeException("Error creating datastore credential", e);
+ }
+ }
+
+ private String getPayload() {
+ JsonObject payload = new JsonObject();
+ payload.addProperty("iss", this.tenant.getServiceAccount());
+ payload.addProperty("sub", this.tenant.getServiceAccount());
+ payload.addProperty("aud", "https://datastore.googleapis.com/google.datastore.v1.Datastore");
+ payload.addProperty("iat", System.currentTimeMillis() / 1000);
+
+ return payload.toString();
+ }
+
+ protected void setIam(Iam iam) {
+ this.iam = iam;
+ }
+
+ private Iam getIam() throws Exception {
+ if (this.iam == null) {
+
+ GoogleCredential credential = GoogleCredential.getApplicationDefault();
+ if (credential.createScopedRequired()) {
+ credential = credential.createScoped(Collections.singletonList(IamScopes.CLOUD_PLATFORM));
+ }
+
+ Iam.Builder builder = new Iam.Builder(GoogleNetHttpTransport.newTrustedTransport(),
+ JSON_FACTORY,
+ credential).setApplicationName("Search Service");
+
+ this.iam = builder.build();
+ }
+ return this.iam;
+ }
+
+
+ private String getCacheKey() {
+ return Crc32c
+ .hashToBase64EncodedString(String.format("datastoreCredential:%s", this.tenant.getName()));
+ }
+}
\ No newline at end of file
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java
new file mode 100644
index 0000000000000000000000000000000000000000..144edee0eb4c1bd99054bd1a1859588adb9be3ae
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java
@@ -0,0 +1,75 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.persistence;
+
+import com.google.api.gax.retrying.RetrySettings;
+import com.google.cloud.TransportOptions;
+import com.google.cloud.datastore.Datastore;
+import com.google.cloud.datastore.DatastoreOptions;
+import com.google.cloud.http.HttpTransportOptions;
+import java.util.HashMap;
+import java.util.Map;
+import javax.inject.Inject;
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
+import org.opengroup.osdu.indexer.cache.DatastoreCredentialCache;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+import org.threeten.bp.Duration;
+
+@Component
+public class DatastoreFactory {
+
+ @Inject
+ private DatastoreCredentialCache cache;
+
+ @Autowired
+ public DatastoreFactory(DatastoreCredentialCache cache) {
+ this.cache = cache;
+ }
+
+ private static Map<String, Datastore> DATASTORE_CLIENTS = new HashMap<>();
+
+ private static final RetrySettings RETRY_SETTINGS = RetrySettings.newBuilder()
+ .setMaxAttempts(6)
+ .setInitialRetryDelay(Duration.ofSeconds(10))
+ .setMaxRetryDelay(Duration.ofSeconds(32))
+ .setRetryDelayMultiplier(2.0)
+ .setTotalTimeout(Duration.ofSeconds(50))
+ .setInitialRpcTimeout(Duration.ofSeconds(50))
+ .setRpcTimeoutMultiplier(1.0)
+ .setMaxRpcTimeout(Duration.ofSeconds(50))
+ .build();
+
+ private static final TransportOptions TRANSPORT_OPTIONS = HttpTransportOptions.newBuilder()
+ .setReadTimeout(30000)
+ .build();
+
+ public Datastore getDatastoreInstance(TenantInfo tenantInfo) {
+ if (DATASTORE_CLIENTS.get(tenantInfo.getName()) == null) {
+ Datastore googleDatastore = DatastoreOptions.newBuilder()
+ .setCredentials(new DatastoreCredential(tenantInfo, this.cache))
+ .setRetrySettings(RETRY_SETTINGS)
+ .setTransportOptions(TRANSPORT_OPTIONS)
+ .setNamespace(tenantInfo.getName())
+ .setProjectId(tenantInfo.getProjectId())
+ .build().getService();
+ DATASTORE_CLIENTS.put(tenantInfo.getName(), googleDatastore);
+ }
+ return DATASTORE_CLIENTS.get(tenantInfo.getName());
+ }
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryMongoDB.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryMongoDB.java
new file mode 100644
index 0000000000000000000000000000000000000000..1c4d4fba54520a6fe22fc1d32c129dda80309432
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryMongoDB.java
@@ -0,0 +1,87 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.persistence;
+
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import java.util.Objects;
+import org.apache.http.HttpStatus;
+import org.bson.Document;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.search.ClusterSettings;
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
+import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class ElasticRepositoryMongoDB implements IElasticRepository {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ElasticRepositoryMongoDB.class);
+
+ private static final String SCHEMA_DATABASE = "local";
+ private static final String SEARCH_SETTINGS = "SearchSettings";
+ private static final String HOST = "host";
+ private static final String PORT = "port";
+ private static final String XPACK_RESTCLIENT_CONFIGURATION = "configuration";
+
+ private final MongoDdmsClient mongoClient;
+
+ @Autowired
+ public ElasticRepositoryMongoDB(MongoDdmsClient mongoClient) {
+ this.mongoClient = mongoClient;
+ }
+
+ @Override
+ public ClusterSettings getElasticClusterSettings(TenantInfo tenantInfo) {
+ MongoCollection<Document> mongoCollection = mongoClient
+ .getMongoCollection(SCHEMA_DATABASE, SEARCH_SETTINGS);
+
+ FindIterable<Document> results = mongoCollection.find();
+
+ if (Objects.isNull(results) && Objects.isNull(results.first())) {
+ LOG.error(String.format("Collection \'%s\' is empty.", results));
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Cluster setting fetch error",
+ "An error has occurred fetching cluster settings from the database.");
+ }
+
+ Document document = results.first();
+
+ String encryptedConfiguration = document.get(XPACK_RESTCLIENT_CONFIGURATION).toString();
+ String encryptedHost = document.get(HOST).toString();
+ String encryptedPort = document.get(PORT).toString();
+
+ try {
+
+ String host = encryptedHost;//this.kmsClient.decryptString(encryptedHost);
+ String portString = encryptedPort;//this.kmsClient.decryptString(encryptedPort);
+ String usernameAndPassword = encryptedConfiguration;//this.kmsClient.decryptString(encryptedConfiguration);
+
+ int port = Integer.parseInt(portString);
+ ClusterSettings clusterSettings = new ClusterSettings(host, port, usernameAndPassword);
+ clusterSettings.setHttps(false);
+ return clusterSettings;
+
+ } catch (Exception e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Cluster setting fetch error",
+ "An error has occurred fetching cluster settings from the database.", e);
+ }
+ }
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/MongoDdmsClient.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/MongoDdmsClient.java
new file mode 100644
index 0000000000000000000000000000000000000000..6cb02a3a7fbfb36d6a2ada3f3b3906a1116aaaa4
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/MongoDdmsClient.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.persistence;
+
+import com.mongodb.client.MongoCollection;
+import org.bson.Document;
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
+import org.opengroup.osdu.indexer.util.MongoClientHandler;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class MongoDdmsClient {
+
+ private final MongoClientHandler mongoClientHandler;
+
+ @Autowired
+ public MongoDdmsClient(MongoClientHandler mongoClientHandler) {
+ this.mongoClientHandler = mongoClientHandler;
+ }
+
+ public MongoCollection<Document> getMongoCollection(String dbName, String collectionName) {
+ return mongoClientHandler.getMongoClient().getDatabase(dbName)
+ .getCollection(collectionName);
+ }
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..a9a82d8cffb4ea3b86d56d66e210f00575f0a034
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java
@@ -0,0 +1,109 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.publish;
+
+
+import com.google.common.reflect.TypeToken;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonElement;
+import java.lang.reflect.Type;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.elasticsearch.common.Strings;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.indexer.JobStatus;
+import org.opengroup.osdu.core.common.model.indexer.RecordStatus;
+import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
+import org.opengroup.osdu.indexer.messagebus.IMessageFactory;
+import org.opengroup.osdu.indexer.provider.interfaces.IPublisher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+@Component
+@RequestScope
+public class PublisherImpl implements IPublisher {
+
+ private static final Logger LOG = LoggerFactory.getLogger(PublisherImpl.class);
+ private final IMessageFactory mq;
+
+ @Autowired
+ public PublisherImpl(IMessageFactory mq) {
+ this.mq = mq;
+ }
+
+ @Override
+ public void publishStatusChangedTagsToTopic(DpsHeaders headers, JobStatus indexerBatchStatus) {
+
+ String tenant = headers.getPartitionId();
+ if (Strings.isNullOrEmpty(tenant)) {
+ tenant = headers.getAccountId();
+ }
+
+ Map<String, String> message = new HashMap<>();
+ message.put(tenant, headers.getPartitionIdWithFallbackToAccountId());
+ headers.addCorrelationIdIfMissing();
+ message.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
+
+ RecordChangedMessages recordChangedMessages = getRecordChangedMessage(headers,
+ indexerBatchStatus);
+ message.put("data", recordChangedMessages.toString());
+
+ try {
+ LOG.info("Indexer publishes message " + headers.getCorrelationId());
+ mq.sendMessage(IMessageFactory.INDEXER_QUEUE_NAME, message.toString());
+ } catch (Exception e) {
+ LOG.error(e.getMessage(), e);
+ }
+ }
+
+ private RecordChangedMessages getRecordChangedMessage(DpsHeaders headers,
+ JobStatus indexerBatchStatus) {
+
+ Gson gson = new GsonBuilder().create();
+ Map<String, String> attributesMap = new HashMap<>();
+ Type listType = new TypeToken<List<RecordStatus>>() {
+ }.getType();
+
+ JsonElement statusChangedTagsJson = gson
+ .toJsonTree(indexerBatchStatus.getStatusesList(), listType);
+ String statusChangedTagsData = (statusChangedTagsJson.toString());
+
+ String tenant = headers.getPartitionId();
+ // This code it to provide backward compatibility to slb-account-id
+ if (!Strings.isNullOrEmpty(tenant)) {
+ attributesMap
+ .put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
+ } else {
+ attributesMap.put(DpsHeaders.ACCOUNT_ID, headers.getPartitionIdWithFallbackToAccountId());
+ }
+ headers.addCorrelationIdIfMissing();
+ attributesMap.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
+
+ RecordChangedMessages recordChangedMessages = new RecordChangedMessages();
+ // statusChangedTagsData is not ByteString but String
+ recordChangedMessages.setData(statusChangedTagsData);
+ recordChangedMessages.setAttributes(attributesMap);
+
+ return recordChangedMessages;
+ }
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java
new file mode 100644
index 0000000000000000000000000000000000000000..0db2f9e557efcc69fbd0419df57fc3a821e024de
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.security;
+
+import org.springframework.context.annotation.Configuration;
+import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity;
+import org.springframework.security.config.annotation.web.builders.HttpSecurity;
+import org.springframework.security.config.annotation.web.builders.WebSecurity;
+import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
+import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter;
+
+@Configuration
+@EnableWebSecurity
+@EnableGlobalMethodSecurity(prePostEnabled = true)
+public class GSuiteSecurityConfig extends WebSecurityConfigurerAdapter {
+
+ @Override
+ protected void configure(HttpSecurity http) throws Exception {
+ http.httpBasic().disable()
+ .csrf().disable();
+ }
+
+ @Override
+ public void configure(WebSecurity web) throws Exception {
+ web.ignoring().antMatchers("/api-docs")
+ .antMatchers("/swagger");
+ }
+
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..558e30bcc6d7c6653f51f1d023a56b6f015a5c18
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
@@ -0,0 +1,614 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.service;
+
+import com.google.gson.Gson;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+import javax.inject.Inject;
+import org.apache.http.HttpStatus;
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.action.bulk.BulkItemResponse;
+import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.rest.RestStatus;
+import org.opengroup.osdu.core.common.Constants;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.entitlements.Acl;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.http.RequestStatus;
+import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
+import org.opengroup.osdu.core.common.model.indexer.IndexingStatus;
+import org.opengroup.osdu.core.common.model.indexer.JobStatus;
+import org.opengroup.osdu.core.common.model.indexer.OperationType;
+import org.opengroup.osdu.core.common.model.indexer.RecordIndexerPayload;
+import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
+import org.opengroup.osdu.core.common.model.indexer.RecordStatus;
+import org.opengroup.osdu.core.common.model.indexer.Records;
+import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
+import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
+import org.opengroup.osdu.core.common.search.IndicesService;
+import org.opengroup.osdu.indexer.logging.AuditLogger;
+import org.opengroup.osdu.indexer.provider.interfaces.IPublisher;
+import org.opengroup.osdu.indexer.util.ElasticClientHandler;
+import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
+import org.springframework.context.annotation.Primary;
+import org.springframework.stereotype.Service;
+
+@Service
+@Primary
+public class IndexerServiceImpl implements IndexerService {
+
+ private static final TimeValue BULK_REQUEST_TIMEOUT = TimeValue.timeValueMinutes(1);
+
+ private static final List<RestStatus> RETRY_ELASTIC_EXCEPTION = new ArrayList<>(Arrays
+ .asList(RestStatus.TOO_MANY_REQUESTS, RestStatus.BAD_GATEWAY,
+ RestStatus.SERVICE_UNAVAILABLE));
+
+ private final Gson gson = new Gson();
+
+ @Inject
+ private JaxRsDpsLog jaxRsDpsLog;
+ @Inject
+ private AuditLogger auditLogger;
+ @Inject
+ private StorageService storageService;
+ @Inject
+ private IndexSchemaService schemaService;
+ @Inject
+ private IndicesService indicesService;
+ @Inject
+ private IndexerMappingService mappingService;
+ @Inject
+ private IPublisher progressPublisher;
+ @Inject
+ private ElasticClientHandler elasticClientHandler;
+ @Inject
+ private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
+ @Inject
+ private ElasticIndexNameResolver elasticIndexNameResolver;
+ @Inject
+ private StorageIndexerPayloadMapper storageIndexerPayloadMapper;
+ @Inject
+ private IRequestInfo requestInfo;
+ @Inject
+ private JobStatus jobStatus;
+
+ private DpsHeaders headers;
+
+ @Override
+ public JobStatus processRecordChangedMessages(RecordChangedMessages message,
+ List<RecordInfo> recordInfos) throws Exception {
+
+ // this should not happen
+ if (recordInfos.size() == 0) {
+ return null;
+ }
+
+ String errorMessage = "";
+ List<String> retryRecordIds = new LinkedList<>();
+
+ // get auth header with service account Authorization
+ this.headers = this.requestInfo.getHeaders();
+ String authorization = message.getAttributes().get("authorization");
+ headers.put("authorization", authorization);
+ // initialize status for all messages.
+ this.jobStatus.initialize(recordInfos);
+
+ try {
+ // get upsert records
+ Map<String, Map<String, OperationType>> upsertRecordMap = RecordInfo
+ .getUpsertRecordIds(recordInfos);
+ if (upsertRecordMap != null && !upsertRecordMap.isEmpty()) {
+ List<String> upsertFailureRecordIds = processUpsertRecords(upsertRecordMap);
+ retryRecordIds.addAll(upsertFailureRecordIds);
+ }
+
+ // get delete records
+ Map<String, List<String>> deleteRecordMap = RecordInfo.getDeleteRecordIds(recordInfos);
+ if (deleteRecordMap != null && !deleteRecordMap.isEmpty()) {
+ List<String> deleteFailureRecordIds = processDeleteRecords(deleteRecordMap);
+ retryRecordIds.addAll(deleteFailureRecordIds);
+ }
+
+ // process schema change messages
+ Map<String, OperationType> schemaMsgs = RecordInfo.getSchemaMsgs(recordInfos);
+ if (schemaMsgs != null && !schemaMsgs.isEmpty()) {
+ this.schemaService.processSchemaMessages(schemaMsgs);
+ }
+
+ // process failed records
+ if (retryRecordIds.size() > 0) {
+ retryAndEnqueueFailedRecords(recordInfos, retryRecordIds, message);
+ }
+ } catch (IOException e) {
+ errorMessage = e.getMessage();
+ throw new AppException(HttpStatus.SC_GATEWAY_TIMEOUT, "Internal communication failure",
+ errorMessage, e);
+ } catch (AppException e) {
+ errorMessage = e.getMessage();
+ throw e;
+ } catch (Exception e) {
+ errorMessage = "error indexing records";
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Unknown error",
+ "An unknown error has occurred.", e);
+ } finally {
+ this.jobStatus.finalizeRecordStatus(errorMessage);
+ this.updateAuditLog();
+ this.progressPublisher.publishStatusChangedTagsToTopic(this.headers, this.jobStatus);
+ }
+
+ return jobStatus;
+ }
+
+ @Override
+ public void processSchemaMessages(List<RecordInfo> recordInfos) throws IOException {
+ Map<String, OperationType> schemaMsgs = RecordInfo.getSchemaMsgs(recordInfos);
+ if (schemaMsgs != null && !schemaMsgs.isEmpty()) {
+ try (RestHighLevelClient restClient = elasticClientHandler.createRestClient()) {
+ schemaMsgs.entrySet().forEach(msg -> {
+ try {
+ processSchemaEvents(restClient, msg);
+ } catch (IOException | ElasticsearchStatusException e) {
+ throw new AppException(
+ org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR.value(),
+ "unable to process schema delete", e.getMessage());
+ }
+ });
+ }
+ }
+ }
+
+ private void processSchemaEvents(RestHighLevelClient restClient,
+ Map.Entry<String, OperationType> msg) throws IOException, ElasticsearchStatusException {
+ String kind = msg.getKey();
+ String index = elasticIndexNameResolver.getIndexNameFromKind(kind);
+
+ boolean indexExist = indicesService.isIndexExist(restClient, index);
+ if (indexExist && msg.getValue() == OperationType.purge_schema) {
+ indicesService.deleteIndex(restClient, index);
+ }
+ }
+
+ private List<String> processUpsertRecords(Map<String, Map<String, OperationType>> upsertRecordMap)
+ throws Exception {
+ // get schema for kind
+ Map<String, IndexSchema> schemas = this.getSchema(upsertRecordMap);
+
+ if (schemas.isEmpty()) {
+ return new LinkedList<>();
+ }
+
+ // get recordIds with valid upsert index-status
+ List<String> recordIds = this.jobStatus.getIdsByValidUpsertIndexingStatus();
+
+ if (recordIds.isEmpty()) {
+ return new LinkedList<>();
+ }
+
+ // get records via storage api
+ Records storageRecords = this.storageService.getStorageRecords(recordIds);
+ List<String> failedOrRetryRecordIds = new LinkedList<>(storageRecords.getMissingRetryRecords());
+
+ // map storage records to indexer payload
+ RecordIndexerPayload recordIndexerPayload = this
+ .getIndexerPayload(upsertRecordMap, schemas, storageRecords);
+
+ jaxRsDpsLog.info(String.format(
+ "records change messages received : %s | valid storage bulk records: %s | valid index payload: %s",
+ recordIds.size(), storageRecords.getRecords().size(),
+ recordIndexerPayload.getRecords().size()));
+
+ // index records
+ failedOrRetryRecordIds.addAll(processElasticMappingAndUpsertRecords(recordIndexerPayload));
+
+ return failedOrRetryRecordIds;
+ }
+
+ private Map<String, IndexSchema> getSchema(
+ Map<String, Map<String, OperationType>> upsertRecordMap) {
+
+ Map<String, IndexSchema> schemas = new HashMap<>();
+
+ try {
+ for (Map.Entry<String, Map<String, OperationType>> entry : upsertRecordMap.entrySet()) {
+
+ String kind = entry.getKey();
+ IndexSchema schemaObj = this.schemaService.getIndexerInputSchema(kind, false);
+ if (schemaObj.isDataSchemaMissing()) {
+ this.jobStatus.addOrUpdateRecordStatus(entry.getValue().keySet(), IndexingStatus.WARN,
+ HttpStatus.SC_NOT_FOUND, "schema not found",
+ String.format("schema not found | kind: %s", kind));
+ }
+
+ schemas.put(kind, schemaObj);
+ }
+ } catch (AppException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Get schema error",
+ "An error has occurred while getting schema", e);
+ }
+
+ return schemas;
+ }
+
+ private RecordIndexerPayload getIndexerPayload(
+ Map<String, Map<String, OperationType>> upsertRecordMap,
+ Map<String, IndexSchema> kindSchemaMap, Records records) {
+ List<Records.Entity> storageValidRecords = records.getRecords();
+ List<RecordIndexerPayload.Record> indexerPayload = new ArrayList<>();
+ List<IndexSchema> schemas = new ArrayList<>();
+
+ for (Records.Entity storageRecord : storageValidRecords) {
+
+ Map<String, OperationType> idOperationMap = upsertRecordMap.get(storageRecord.getKind());
+
+ // skip if storage returned record with same id but different kind
+ if (idOperationMap == null) {
+ String message = String.format(
+ "storage service returned incorrect record | requested kind: %s | received kind: %s",
+ this.jobStatus.getRecordKindById(storageRecord.getId()), storageRecord.getKind());
+ this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.SKIP,
+ RequestStatus.STORAGE_CONFLICT, message,
+ String.format("%s | record-id: %s", message, storageRecord.getId()));
+ continue;
+ }
+
+ IndexSchema schema = kindSchemaMap.get(storageRecord.getKind());
+ schemas.add(schema);
+
+ // skip indexing of records if data block is empty
+ RecordIndexerPayload.Record document = prepareIndexerPayload(schema, storageRecord,
+ idOperationMap);
+ if (document != null) {
+ indexerPayload.add(document);
+ }
+ }
+
+ // this should only happen if storage service returned WRONG records with kind for all the records in the messages
+ if (indexerPayload.isEmpty()) {
+ throw new AppException(RequestStatus.STORAGE_CONFLICT, "Indexer error",
+ "upsert record failed, storage service returned incorrect records");
+ }
+
+ return RecordIndexerPayload.builder().records(indexerPayload).schemas(schemas).build();
+ }
+
+ private RecordIndexerPayload.Record prepareIndexerPayload(IndexSchema schemaObj,
+ Records.Entity storageRecord, Map<String, OperationType> idToOperationMap) {
+
+ RecordIndexerPayload.Record document = null;
+
+ try {
+ Map<String, Object> storageRecordData = storageRecord.getData();
+ document = new RecordIndexerPayload.Record();
+ if (storageRecordData == null || storageRecordData.isEmpty()) {
+ String message = "empty or null data block found in the storage record";
+ this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.WARN,
+ HttpStatus.SC_NOT_FOUND, message,
+ String.format("record-id: %s | %s", storageRecord.getId(), message));
+ } else if (schemaObj.isDataSchemaMissing()) {
+ document.setSchemaMissing(true);
+ } else {
+ Map<String, Object> dataMap = this.storageIndexerPayloadMapper
+ .mapDataPayload(schemaObj, storageRecordData, storageRecord.getId());
+ if (dataMap.isEmpty()) {
+ document.setMappingMismatch(true);
+ String message = String.format(
+ "complete schema mismatch: none of the data attribute can be mapped | data: %s",
+ storageRecordData);
+ this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.WARN,
+ HttpStatus.SC_NOT_FOUND, message,
+ String.format("record-id: %s | %s", storageRecord.getId(), message));
+ }
+ document.setData(dataMap);
+ }
+ } catch (AppException e) {
+ this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL,
+ HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getMessage());
+ jaxRsDpsLog
+ .warning(String.format("record-id: %s | %s", storageRecord.getId(), e.getMessage()), e);
+ } catch (Exception e) {
+ this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL,
+ HttpStatus.SC_INTERNAL_SERVER_ERROR,
+ String.format("error parsing records against schema, error-message: %s", e.getMessage()));
+ jaxRsDpsLog.error(String
+ .format("record-id: %s | error parsing records against schema, error-message: %s",
+ storageRecord.getId(), e.getMessage()), e);
+ }
+
+ try {
+ // index individual parts of kind
+ String[] kindParts = storageRecord.getKind().split(":");
+
+ document.setKind(storageRecord.getKind());
+ document.setNamespace(kindParts[0] + ":" + kindParts[1]);
+ document.setType(kindParts[2]);
+ document.setId(storageRecord.getId());
+ document.setVersion(storageRecord.getVersion());
+ document.setAcl(storageRecord.getAcl());
+ document.setLegal(storageRecord.getLegal());
+ RecordStatus recordStatus = this.jobStatus.getJobStatusByRecordId(storageRecord.getId());
+ if (recordStatus.getIndexProgress().getStatusCode() == 0) {
+ recordStatus.getIndexProgress().setStatusCode(HttpStatus.SC_OK);
+ }
+ document.setIndexProgress(recordStatus.getIndexProgress());
+ if (storageRecord.getAncestry() != null) {
+ document.setAncestry(storageRecord.getAncestry());
+ }
+ document.setOperationType(idToOperationMap.get(storageRecord.getId()));
+ } catch (Exception e) {
+ this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL,
+ HttpStatus.SC_INTERNAL_SERVER_ERROR,
+ String.format("error parsing meta data, error-message: %s", e.getMessage()));
+ jaxRsDpsLog.error(String.format("record-id: %s | error parsing meta data, error-message: %s",
+ storageRecord.getId(), e.getMessage()), e);
+ }
+ return document;
+ }
+
+ private List<String> processElasticMappingAndUpsertRecords(
+ RecordIndexerPayload recordIndexerPayload) throws Exception {
+
+ try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
+ List<IndexSchema> schemas = recordIndexerPayload.getSchemas();
+ if (schemas == null || schemas.isEmpty()) {
+ return new LinkedList<>();
+ }
+
+ // process the schema
+ this.cacheOrCreateElasticMapping(schemas, restClient);
+
+ // process the records
+ return this.upsertRecords(recordIndexerPayload.getRecords(), restClient);
+ }
+ }
+
+ private void cacheOrCreateElasticMapping(List<IndexSchema> schemas,
+ RestHighLevelClient restClient) throws Exception {
+
+ for (IndexSchema schema : schemas) {
+ String index = this.elasticIndexNameResolver.getIndexNameFromKind(schema.getKind());
+
+ // check if index exist
+ if (this.indicesService.isIndexExist(restClient, index)) {
+ continue;
+ }
+
+ // create index
+ Map<String, Object> mapping = this.mappingService.getIndexMappingFromRecordSchema(schema);
+ if (!this.indicesService.createIndex(restClient, index, null, schema.getType(), mapping)) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error",
+ "Error creating index.",
+ String.format("Failed to get confirmation from elastic server for index: %s", index));
+ }
+ }
+ }
+
+ private List<String> upsertRecords(List<RecordIndexerPayload.Record> records,
+ RestHighLevelClient restClient) throws AppException {
+ if (records == null || records.isEmpty()) {
+ return new LinkedList<>();
+ }
+
+ BulkRequest bulkRequest = new BulkRequest();
+ bulkRequest.timeout(BULK_REQUEST_TIMEOUT);
+
+ for (RecordIndexerPayload.Record record : records) {
+ if ((record.getData() == null || record.getData().isEmpty()) && !record
+ .skippedDataIndexing()) {
+ // it will come here when schema is missing
+ // TODO: rollback once we know what is causing the problem
+ jaxRsDpsLog.warning(String.format("data not found for record: %s", record));
+ }
+
+ OperationType operation = record.getOperationType();
+ Map<String, Object> sourceMap = getSourceMap(record);
+ String index = this.elasticIndexNameResolver.getIndexNameFromKind(record.getKind());
+
+ if (operation == OperationType.create) {
+ IndexRequest indexRequest = new IndexRequest(index, record.getType(), record.getId())
+ .source(this.gson.toJson(sourceMap), XContentType.JSON);
+ bulkRequest.add(indexRequest);
+ } else if (operation == OperationType.update) {
+ UpdateRequest updateRequest = new UpdateRequest(index, record.getType(), record.getId())
+ .upsert(this.gson.toJson(sourceMap), XContentType.JSON);
+ bulkRequest.add(updateRequest);
+ }
+ }
+
+ return processBulkRequest(restClient, bulkRequest);
+ }
+
+ private List<String> processDeleteRecords(Map<String, List<String>> deleteRecordMap)
+ throws Exception {
+ BulkRequest bulkRequest = new BulkRequest();
+ bulkRequest.timeout(BULK_REQUEST_TIMEOUT);
+
+ for (Map.Entry<String, List<String>> record : deleteRecordMap.entrySet()) {
+
+ String[] kindParts = record.getKey().split(":");
+ String type = kindParts[2];
+
+ String index = this.elasticIndexNameResolver.getIndexNameFromKind(record.getKey());
+
+ for (String id : record.getValue()) {
+ DeleteRequest deleteRequest = new DeleteRequest(index, type, id);
+ bulkRequest.add(deleteRequest);
+ }
+ }
+
+ try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
+ return processBulkRequest(restClient, bulkRequest);
+ }
+ }
+
+ private List<String> processBulkRequest(RestHighLevelClient restClient, BulkRequest bulkRequest)
+ throws AppException {
+
+ List<String> failureRecordIds = new LinkedList<>();
+ if (bulkRequest.numberOfActions() == 0) {
+ return failureRecordIds;
+ }
+
+ try {
+ BulkResponse bulkResponse = restClient.bulk(bulkRequest, RequestOptions.DEFAULT);
+
+ // log failed bulk requests
+ ArrayList<String> bulkFailures = new ArrayList<>();
+ int succeededResponses = 0;
+ int failedResponses = 0;
+
+ for (BulkItemResponse bulkItemResponse : bulkResponse.getItems()) {
+ if (bulkItemResponse.isFailed()) {
+ BulkItemResponse.Failure failure = bulkItemResponse.getFailure();
+ bulkFailures.add(String.format("elasticsearch bulk service status: %s id: %s message: %s",
+ failure.getStatus(), failure.getId(), failure.getMessage()));
+ this.jobStatus.addOrUpdateRecordStatus(bulkItemResponse.getId(), IndexingStatus.FAIL,
+ failure.getStatus().getStatus(), bulkItemResponse.getFailureMessage());
+ if (RETRY_ELASTIC_EXCEPTION.contains(bulkItemResponse.status())) {
+ failureRecordIds.add(bulkItemResponse.getId());
+ }
+ failedResponses++;
+ } else {
+ succeededResponses++;
+ this.jobStatus.addOrUpdateRecordStatus(bulkItemResponse.getId(), IndexingStatus.SUCCESS,
+ HttpStatus.SC_OK, "Indexed Successfully");
+ }
+ }
+ if (!bulkFailures.isEmpty()) {
+ this.jaxRsDpsLog.warning(bulkFailures);
+ }
+
+ jaxRsDpsLog.info(String
+ .format("records in elasticsearch service bulk request: %s | successful: %s | failed: %s",
+ bulkRequest.numberOfActions(), succeededResponses, failedResponses));
+ } catch (IOException e) {
+ // throw explicit 504 for IOException
+ throw new AppException(HttpStatus.SC_GATEWAY_TIMEOUT, "Elastic error",
+ "Request cannot be completed in specified time.", e);
+ } catch (ElasticsearchStatusException e) {
+ throw new AppException(e.status().getStatus(), "Elastic error", e.getMessage(), e);
+ } catch (Exception e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error",
+ "Error indexing records.", e);
+ }
+ return failureRecordIds;
+ }
+
+ private Map<String, Object> getSourceMap(RecordIndexerPayload.Record record) {
+
+ Map<String, Object> indexerPayload = new HashMap<>();
+
+ // get the key and get the corresponding object from the individualRecord object
+ if (record.getData() != null) {
+ Map<String, Object> data = new HashMap<>();
+ for (Map.Entry<String, Object> entry : record.getData().entrySet()) {
+ data.put(entry.getKey(), entry.getValue());
+ }
+ indexerPayload.put(Constants.DATA, data);
+ }
+
+ indexerPayload.put(RecordMetaAttribute.ID.getValue(), record.getId());
+ indexerPayload.put(RecordMetaAttribute.KIND.getValue(), record.getKind());
+ indexerPayload.put(RecordMetaAttribute.NAMESPACE.getValue(), record.getNamespace());
+ indexerPayload.put(RecordMetaAttribute.TYPE.getValue(), record.getType());
+ indexerPayload.put(RecordMetaAttribute.VERSION.getValue(), record.getVersion());
+ indexerPayload.put(RecordMetaAttribute.ACL.getValue(), record.getAcl());
+ indexerPayload.put(RecordMetaAttribute.X_ACL.getValue(), Acl.flattenAcl(record.getAcl()));
+ indexerPayload.put(RecordMetaAttribute.LEGAL.getValue(), record.getLegal());
+ indexerPayload.put(RecordMetaAttribute.INDEX_STATUS.getValue(), record.getIndexProgress());
+ if (record.getAncestry() != null) {
+ indexerPayload.put(RecordMetaAttribute.ANCESTRY.getValue(), record.getAncestry());
+ }
+ return indexerPayload;
+ }
+
+ private void retryAndEnqueueFailedRecords(List<RecordInfo> recordInfos,
+ List<String> failuresRecordIds, RecordChangedMessages message) throws IOException {
+
+ jaxRsDpsLog.info(String.format(
+ "queuing bulk failed records back to task-queue for retry | count: %s | records: %s",
+ failuresRecordIds.size(), failuresRecordIds));
+ List<RecordInfo> retryRecordInfos = new LinkedList<>();
+ for (String recordId : failuresRecordIds) {
+ for (RecordInfo origMessage : recordInfos) {
+ if (origMessage.getId().equalsIgnoreCase(recordId)) {
+ retryRecordInfos.add(origMessage);
+ }
+ }
+ }
+
+ RecordChangedMessages newMessage = RecordChangedMessages.builder()
+ .messageId(message.getMessageId())
+ .publishTime(message.getPublishTime())
+ .data(this.gson.toJson(retryRecordInfos))
+ .attributes(message.getAttributes()).build();
+
+ String payLoad = this.gson.toJson(newMessage);
+ this.indexerQueueTaskBuilder.createWorkerTask(payLoad, this.headers);
+ }
+
+ private void updateAuditLog() {
+ logAuditEvents(OperationType.create, this.auditLogger::indexCreateRecordSuccess,
+ this.auditLogger::indexCreateRecordFail);
+ logAuditEvents(OperationType.update, this.auditLogger::indexUpdateRecordSuccess,
+ this.auditLogger::indexUpdateRecordFail);
+ logAuditEvents(OperationType.purge, this.auditLogger::indexPurgeRecordSuccess,
+ this.auditLogger::indexPurgeRecordFail);
+ logAuditEvents(OperationType.delete, this.auditLogger::indexDeleteRecordSuccess,
+ this.auditLogger::indexDeleteRecordFail);
+ }
+
+ private void logAuditEvents(OperationType operationType, Consumer<List<String>> successEvent,
+ Consumer<List<String>> failedEvent) {
+ List<RecordStatus> succeededRecords = this.jobStatus
+ .getRecordStatuses(IndexingStatus.SUCCESS, operationType);
+ if (!succeededRecords.isEmpty()) {
+ successEvent.accept(succeededRecords.stream().map(RecordStatus::succeededAuditLogMessage)
+ .collect(Collectors.toList()));
+ }
+ List<RecordStatus> skippedRecords = this.jobStatus
+ .getRecordStatuses(IndexingStatus.SKIP, operationType);
+ List<RecordStatus> failedRecords = this.jobStatus
+ .getRecordStatuses(IndexingStatus.FAIL, operationType);
+ failedRecords.addAll(skippedRecords);
+ if (!failedRecords.isEmpty()) {
+ failedEvent.accept(failedRecords.stream().map(RecordStatus::failedAuditLogMessage)
+ .collect(Collectors.toList()));
+ }
+ }
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/DpsHeaderFactoryGcp.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/DpsHeaderFactoryGcp.java
new file mode 100644
index 0000000000000000000000000000000000000000..86fb43449ea75340340b0ee43de9e85304c79697
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/DpsHeaderFactoryGcp.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.util;
+
+import com.google.common.base.Strings;
+import java.util.Collections;
+import java.util.Map;
+import java.util.stream.Collectors;
+import javax.inject.Inject;
+import javax.servlet.http.HttpServletRequest;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.gcp.model.AppEngineHeaders;
+import org.opengroup.osdu.core.gcp.util.TraceIdExtractor;
+import org.springframework.context.annotation.Primary;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+@Component
+@RequestScope
+@Primary
+public class DpsHeaderFactoryGcp extends DpsHeaders {
+
+ @Inject
+ public DpsHeaderFactoryGcp(HttpServletRequest request) {
+
+ Map<String, String> headers = Collections
+ .list(request.getHeaderNames())
+ .stream()
+ .collect(Collectors.toMap(h -> h, request::getHeader));
+
+ String traceContext = headers.get(AppEngineHeaders.CLOUD_TRACE_CONTEXT);
+
+ if (!Strings.isNullOrEmpty(traceContext)) {
+ headers.put(AppEngineHeaders.TRACE_ID, TraceIdExtractor.getTraceId(traceContext));
+ }
+
+ this.addFromMap(headers);
+
+ // Add Correlation ID if missing
+ this.addCorrelationIdIfMissing();
+ }
+}
\ No newline at end of file
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/MongoClientHandler.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/MongoClientHandler.java
new file mode 100644
index 0000000000000000000000000000000000000000..8e53b593ec171e2962ca3b286d328e89a1a7716c
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/MongoClientHandler.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.util;
+
+import com.mongodb.ConnectionString;
+import com.mongodb.MongoClientSettings;
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoClients;
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.indexer.config.MongoDBConfigProperties;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class MongoClientHandler {
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ org.opengroup.osdu.indexer.util.MongoClientHandler.class);
+ private static final String MONGO_PREFIX = "mongodb://";
+ private static final String MONGO_OPTIONS = "retryWrites=true&w=majority&maxIdleTimeMS=10000";
+
+ private com.mongodb.client.MongoClient mongoClient = null;
+ private MongoDBConfigProperties mongoDBConfigProperties;
+
+ private MongoClient getOrInitMongoClient() throws RuntimeException {
+ if (mongoClient != null) {
+ return mongoClient;
+ }
+
+ final String connectionString = String.format("%s%s:%s@%s/?%s",
+ MONGO_PREFIX,
+ mongoDBConfigProperties.getMongoDbUser(),
+ mongoDBConfigProperties.getMongoDbPassword(),
+ mongoDBConfigProperties.getMongoDbUrl(),
+ MONGO_OPTIONS);
+ ConnectionString connString = new ConnectionString(connectionString);
+ MongoClientSettings settings = MongoClientSettings.builder()
+ .applyConnectionString(connString)
+ .retryWrites(true)
+ .build();
+ try {
+ mongoClient = MongoClients.create(settings);
+ } catch (Exception ex) {
+ LOG.error("Error connecting MongoDB", ex);
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Error connecting MongoDB",
+ ex.getMessage(), ex);
+ }
+ return mongoClient;
+ }
+
+ public MongoClient getMongoClient() {
+ if (mongoClient == null) {
+ getOrInitMongoClient();
+ }
+ return mongoClient;
+ }
+
+ @Autowired
+ public void setMongoDBConfigProperties(MongoDBConfigProperties mongoDBConfigProperties) {
+ this.mongoDBConfigProperties = mongoDBConfigProperties;
+ }
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..1e3c3f244cb37470a91c6a72185a2007c2578698
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java
@@ -0,0 +1,124 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.util;
+
+import static org.opengroup.osdu.core.common.model.http.DpsHeaders.AUTHORIZATION;
+
+import com.google.common.base.Strings;
+import java.util.Map;
+import lombok.extern.java.Log;
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.core.common.Constants;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.search.DeploymentEnvironment;
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient;
+import org.opengroup.osdu.core.gcp.model.AppEngineHeaders;
+import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+@Log
+@Component
+@RequestScope
+public class RequestInfoImpl implements IRequestInfo {
+
+ private final IndexerConfigurationProperties indexerConfigurationProperties;
+ private final TenantInfo tenantInfo;
+ private final IServiceAccountJwtClient serviceAccountJwtClient;
+ private final DpsHeaders dpsHeaders;
+
+ @Autowired
+ public RequestInfoImpl(IndexerConfigurationProperties indexerConfigurationProperties,
+ TenantInfo tenantInfo, IServiceAccountJwtClient serviceAccountJwtClient,
+ DpsHeaders dpsHeaders) {
+ this.indexerConfigurationProperties = indexerConfigurationProperties;
+ this.tenantInfo = tenantInfo;
+ this.serviceAccountJwtClient = serviceAccountJwtClient;
+ this.dpsHeaders = dpsHeaders;
+ }
+
+ private static final String expectedCronHeaderValue = "true";
+
+ @Override
+ public DpsHeaders getHeaders() {
+
+ return this.dpsHeaders;
+ }
+
+ @Override
+ public String getPartitionId() {
+ return this.dpsHeaders.getPartitionId();
+ }
+
+ @Override
+ public Map<String, String> getHeadersMap() {
+ return this.dpsHeaders.getHeaders();
+ }
+
+ @Override
+ public Map<String, String> getHeadersMapWithDwdAuthZ() {
+ return getHeadersWithDwdAuthZ().getHeaders();
+ }
+
+ @Override
+ public DpsHeaders getHeadersWithDwdAuthZ() {
+ // Update DpsHeaders so that service account creds are passed down
+ this.dpsHeaders.put(AUTHORIZATION, this.checkOrGetAuthorizationHeader());
+ return this.dpsHeaders;
+ }
+
+ @Override
+ public boolean isCronRequest() {
+ String appEngineCronHeader = this.dpsHeaders.getHeaders()
+ .getOrDefault(AppEngineHeaders.CRON_SERVICE, null);
+ return expectedCronHeaderValue.equalsIgnoreCase(appEngineCronHeader);
+ }
+
+ @Override
+ public boolean isTaskQueueRequest() {
+ if (!this.dpsHeaders.getHeaders().containsKey(AppEngineHeaders.TASK_QUEUE_NAME)) {
+ return false;
+ }
+
+ String queueId = this.dpsHeaders.getHeaders().get(AppEngineHeaders.TASK_QUEUE_NAME);
+ return queueId.endsWith(Constants.INDEXER_QUEUE_IDENTIFIER);
+ }
+
+ public String checkOrGetAuthorizationHeader() {
+ if (DeploymentEnvironment.valueOf(indexerConfigurationProperties.getEnvironment())
+ == DeploymentEnvironment.LOCAL) {
+ String authHeader = this.dpsHeaders.getAuthorization();
+ if (Strings.isNullOrEmpty(authHeader)) {
+ throw new AppException(HttpStatus.SC_UNAUTHORIZED, "Invalid authorization header",
+ "Authorization token cannot be empty");
+ }
+ String user = this.dpsHeaders.getUserEmail();
+ if (Strings.isNullOrEmpty(user)) {
+ throw new AppException(HttpStatus.SC_UNAUTHORIZED, "Invalid user header",
+ "User header cannot be empty");
+ }
+ return authHeader;
+ } else {
+ return "Bearer " + this.serviceAccountJwtClient.getIdToken(tenantInfo.getName());
+ }
+ }
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..98605a1509c597b4b5f7332560374ff072248d44
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java
@@ -0,0 +1,202 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.util;
+
+import com.auth0.jwt.JWT;
+import com.auth0.jwt.exceptions.JWTDecodeException;
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.iam.v1.Iam;
+import com.google.api.services.iam.v1.IamScopes;
+import com.google.api.services.iam.v1.model.SignJwtRequest;
+import com.google.api.services.iam.v1.model.SignJwtResponse;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.http.HttpHeaders;
+import org.apache.http.HttpStatus;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.ContentType;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.message.BasicNameValuePair;
+import org.apache.http.util.EntityUtils;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.search.IdToken;
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
+import org.opengroup.osdu.core.common.provider.interfaces.IJwtCache;
+import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
+import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient;
+import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+@Component
+@RequestScope
+public class ServiceAccountJwtGcpClientImpl implements IServiceAccountJwtClient {
+
+ private static final String JWT_AUDIENCE = "https://www.googleapis.com/oauth2/v4/token";
+ private static final String SERVICE_ACCOUNT_NAME_FORMAT = "projects/%s/serviceAccounts/%s";
+
+ private final JsonFactory JSON_FACTORY = new JacksonFactory();
+ private final IndexerConfigurationProperties indexerConfigurationProperties;
+ private final ITenantFactory tenantInfoServiceProvider;
+ private final IJwtCache cacheService;
+ private final JaxRsDpsLog log;
+ private final DpsHeaders dpsHeaders;
+
+ private Iam iam;
+
+ @Autowired
+ public ServiceAccountJwtGcpClientImpl(
+ IndexerConfigurationProperties indexerConfigurationProperties,
+ ITenantFactory tenantInfoServiceProvider,
+ IJwtCache cacheService, JaxRsDpsLog log, DpsHeaders dpsHeaders) {
+ this.indexerConfigurationProperties = indexerConfigurationProperties;
+ this.tenantInfoServiceProvider = tenantInfoServiceProvider;
+ this.cacheService = cacheService;
+ this.log = log;
+ this.dpsHeaders = dpsHeaders;
+ }
+
+ public String getIdToken(String tenantName) {
+ this.log.info("Tenant name received for auth token is: " + tenantName);
+ TenantInfo tenant = this.tenantInfoServiceProvider.getTenantInfo(tenantName);
+ if (tenant == null) {
+ this.log.error("Invalid tenant name receiving from pubsub");
+ throw new AppException(HttpStatus.SC_BAD_REQUEST, "Invalid tenant Name",
+ "Invalid tenant Name from pubsub");
+ }
+ try {
+
+ IdToken cachedToken = (IdToken) this.cacheService.get(tenant.getServiceAccount());
+ // Add the user to DpsHeaders directly
+ this.dpsHeaders.put(DpsHeaders.USER_EMAIL, tenant.getServiceAccount());
+
+ if (!IdToken.refreshToken(cachedToken)) {
+ return cachedToken.getTokenValue();
+ }
+
+ // Getting signed JWT
+ Map<String, Object> signJwtPayload = this.getJWTCreationPayload(tenant);
+
+ SignJwtRequest signJwtRequest = new SignJwtRequest();
+ signJwtRequest.setPayload(JSON_FACTORY.toString(signJwtPayload));
+
+ String serviceAccountName = String
+ .format(SERVICE_ACCOUNT_NAME_FORMAT, tenant.getProjectId(), tenant.getServiceAccount());
+
+ Iam.Projects.ServiceAccounts.SignJwt signJwt = this.getIam().projects().serviceAccounts()
+ .signJwt(serviceAccountName, signJwtRequest);
+ SignJwtResponse signJwtResponse = signJwt.execute();
+ String signedJwt = signJwtResponse.getSignedJwt();
+
+ // Getting id token
+ List<NameValuePair> postParameters = new ArrayList<>();
+ postParameters
+ .add(new BasicNameValuePair("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer"));
+ postParameters.add(new BasicNameValuePair("assertion", signedJwt));
+
+ HttpPost post = new HttpPost(JWT_AUDIENCE);
+ post.setHeader(HttpHeaders.CONTENT_TYPE,
+ ContentType.APPLICATION_FORM_URLENCODED.getMimeType());
+ post.setEntity(new UrlEncodedFormEntity(postParameters, "UTF-8"));
+
+ try (CloseableHttpClient httpclient = HttpClients.createDefault();
+ CloseableHttpResponse httpResponse = httpclient.execute(post)) {
+ JsonObject jsonContent = new JsonParser()
+ .parse(EntityUtils.toString(httpResponse.getEntity()))
+ .getAsJsonObject();
+
+ if (!jsonContent.has("id_token")) {
+ log.error(String.format("Google IAM response: %s", jsonContent.toString()));
+ throw new AppException(HttpStatus.SC_FORBIDDEN, "Access denied",
+ "The user is not authorized to perform this action");
+ }
+
+ String token = jsonContent.get("id_token").getAsString();
+ IdToken idToken = IdToken.builder().tokenValue(token)
+ .expirationTimeMillis(JWT.decode(token).getExpiresAt().getTime()).build();
+
+ this.cacheService.put(tenant.getServiceAccount(), idToken);
+
+ return token;
+ }
+ } catch (JWTDecodeException e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Persistence error",
+ "Invalid token, error decoding", e);
+ } catch (AppException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Persistence error",
+ "Error generating token",
+ e);
+ }
+ }
+
+ public Iam getIam() throws Exception {
+
+ if (this.iam == null) {
+ HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport();
+
+ // Authenticate using Google Application Default Credentials.
+ GoogleCredential credential = GoogleCredential.getApplicationDefault();
+ if (credential.createScopedRequired()) {
+ List<String> scopes = new ArrayList<>();
+ // Enable full Cloud Platform scope.
+ scopes.add(IamScopes.CLOUD_PLATFORM);
+ credential = credential.createScoped(scopes);
+ }
+
+ // Create IAM API object associated with the authenticated transport.
+ this.iam = new Iam.Builder(httpTransport, JSON_FACTORY, credential)
+ .setApplicationName(indexerConfigurationProperties.getIndexerHost())
+ .build();
+ }
+
+ return this.iam;
+ }
+
+ private Map<String, Object> getJWTCreationPayload(TenantInfo tenantInfo) {
+
+ Map<String, Object> payload = new HashMap<>();
+ String googleAudience = indexerConfigurationProperties.getGoogleAudiences();
+ if (googleAudience.contains(",")) {
+ googleAudience = googleAudience.split(",")[0];
+ }
+ payload.put("target_audience", googleAudience);
+ payload.put("exp", System.currentTimeMillis() / 1000 + 3600);
+ payload.put("iat", System.currentTimeMillis() / 1000);
+ payload.put("iss", tenantInfo.getServiceAccount());
+ payload.put("aud", JWT_AUDIENCE);
+
+ return payload;
+ }
+}
diff --git a/provider/indexer-reference/src/main/resources/application-dev.properties b/provider/indexer-reference/src/main/resources/application-dev.properties
new file mode 100644
index 0000000000000000000000000000000000000000..ea75c38d4866a1a690158a7e7efbd184c68ad58c
--- /dev/null
+++ b/provider/indexer-reference/src/main/resources/application-dev.properties
@@ -0,0 +1,44 @@
+#
+# Copyright 2020 Google LLC
+# Copyright 2020 EPAM Systems, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+GOOGLE_CLOUD_PROJECT=nice-etching-277309
+
+mongo-db-url=localhost:27017
+mongo-db-user=
+mongo-db-password=
+
+indexer-host=os-indexer-dot-nice-etching-277309.uc.r.appspot.com
+storage-hostname=os-storage-dot-nice-etching-277309.uc.r.appspot.com
+
+storage-schema-host=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/schemas
+storage-query-record-host=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/query/records
+storage-query-record-for-conversion-host=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/query/records:batch
+storage-records-batch-size=20
+
+indexer-queue-host=https://os-indexer-queue-dot-nice-etching-277309.uc.r.appspot.com/_dps/task-handlers/enqueue
+
+authorize-api=https://entitlements-dot-nice-etching-277309.uc.r.appspot.com/entitlements/v1
+legaltag-api=https://os-legal-dot-nice-etching-277309.uc.r.appspot.com/api/legal/v1
+crs-api=example.com
+
+## use below values for gcp: opendes
+redis-group-host=127.0.0.1
+redis-search-host=127.0.0.1
+
+google-audiences=689762842995-pv217jo3k8j803kk6gqf52qb5amos3a9.apps.googleusercontent.com
+
+mb-rabbitmq-uri=amqp://guest:guest@127.0.0.1:5672
diff --git a/provider/indexer-reference/src/main/resources/application.properties b/provider/indexer-reference/src/main/resources/application.properties
new file mode 100644
index 0000000000000000000000000000000000000000..8690c57a906c0aae8cf6ecc5b0c282be392bbafb
--- /dev/null
+++ b/provider/indexer-reference/src/main/resources/application.properties
@@ -0,0 +1,76 @@
+#
+# Copyright 2021 Google LLC
+# Copyright 2021 EPAM Systems, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+LOG_PREFIX=indexer
+
+server.servlet.contextPath=/api/indexer/v2
+logging.level.org.springframework.web=DEBUG
+server.port=8080
+JAVA_OPTS=-Xms3072m -Xmx3072m
+JAVA_GC_OPTS=-XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45
+
+environment=CLOUD
+
+#default cache settings
+schema-cache-expiration=60
+index-cache-expiration=60
+elastic-cache-expiration=1440
+cursor-cache-expiration=60
+# kinds cache expiration 2*24*60
+kinds-cache-expiration=2880
+# attributes cache expiration 2*24*60
+attributes-cache-expiration=2880
+
+kinds-redis-database=1
+cron-index-cleanup-threshold-days=3
+cron-empty-index-cleanup-threshold-days=7
+
+gae-service=indexer
+key-ring=csqp
+kms-key=searchservice
+
+elastic-datastore-kind=searchsettings
+elastic-datastore-id=indexer-service
+
+mongo-db-url=localhost:27017
+mongo-db-user=
+mongo-db-password=
+
+indexer-host=os-indexer-dot-nice-etching-277309.uc.r.appspot.com
+storage-hostname=os-storage-dot-nice-etching-277309.uc.r.appspot.com
+
+storage-schema-host=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/schemas
+storage-query-record-host=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/query/records
+storage-query-record-for-conversion-host=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/query/records:batch
+storage-records-batch-size=20
+
+indexer-queue-host=https://os-indexer-queue-dot-nice-etching-277309.uc.r.appspot.com/_dps/task-handlers/enqueue
+
+authorize-api=https://entitlements-dot-nice-etching-277309.uc.r.appspot.com/entitlements/v1
+legaltag-api=https://os-legal-dot-nice-etching-277309.uc.r.appspot.com/api/legal/v1
+crs-api=example.com
+
+## use below values for gcp: opendes
+redis-group-host=127.0.0.1
+redis-search-host=127.0.0.1
+
+google-audiences=689762842995-pv217jo3k8j803kk6gqf52qb5amos3a9.apps.googleusercontent.com
+
+mb-rabbitmq-uri=amqp://guest:guest@127.0.0.1:5672
+
+
+
diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..9ec5fb7a30044f7f105c9a97229cb37322f218b9
--- /dev/null
+++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java
@@ -0,0 +1,81 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.middleware;
+
+import java.io.IOException;
+import java.util.Collections;
+import javax.servlet.FilterChain;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.MockitoJUnitRunner;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+
+@RunWith(MockitoJUnitRunner.class)
+public class IndexFilterTest {
+
+ @InjectMocks
+ private IndexFilter indexFilter;
+
+ @Mock
+ private DpsHeaders dpsHeaders;
+
+ @Test
+ public void shouldSetCorrectResponseHeaders() throws IOException, ServletException {
+ HttpServletRequest httpServletRequest = Mockito.mock(HttpServletRequest.class);
+ HttpServletResponse httpServletResponse = Mockito.mock(HttpServletResponse.class);
+ FilterChain filterChain = Mockito.mock(FilterChain.class);
+ Mockito.when(httpServletRequest.getRequestURI()).thenReturn("https://test.com");
+ Mockito.when(httpServletRequest.getMethod()).thenReturn("POST");
+ Mockito.when(dpsHeaders.getCorrelationId()).thenReturn("correlation-id-value");
+
+ indexFilter.doFilter(httpServletRequest, httpServletResponse, filterChain);
+
+ Mockito.verify(httpServletResponse)
+ .addHeader("Access-Control-Allow-Origin", Collections.singletonList("*").toString());
+ Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Headers", Collections
+ .singletonList(
+ "origin, content-type, accept, authorization, data-partition-id, correlation-id, appkey")
+ .toString());
+ Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Methods",
+ Collections.singletonList("GET, POST, PUT, DELETE, OPTIONS, HEAD").toString());
+ Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Credentials",
+ Collections.singletonList("true").toString());
+ Mockito.verify(httpServletResponse)
+ .addHeader("X-Frame-Options", Collections.singletonList("DENY").toString());
+ Mockito.verify(httpServletResponse)
+ .addHeader("X-XSS-Protection", Collections.singletonList("1; mode=block").toString());
+ Mockito.verify(httpServletResponse)
+ .addHeader("X-Content-Type-Options", Collections.singletonList("nosniff").toString());
+ Mockito.verify(httpServletResponse).addHeader("Cache-Control",
+ Collections.singletonList("no-cache, no-store, must-revalidate").toString());
+ Mockito.verify(httpServletResponse).addHeader("Content-Security-Policy",
+ Collections.singletonList("default-src 'self'").toString());
+ Mockito.verify(httpServletResponse).addHeader("Strict-Transport-Security",
+ Collections.singletonList("max-age=31536000; includeSubDomains").toString());
+ Mockito.verify(httpServletResponse)
+ .addHeader("Expires", Collections.singletonList("0").toString());
+ Mockito.verify(httpServletResponse).addHeader("correlation-id", "correlation-id-value");
+ Mockito.verify(filterChain).doFilter(httpServletRequest, httpServletResponse);
+ }
+}
diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/CronServiceImplTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/CronServiceImplTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..e1c53edb8ab313727f54a905a3494a74cb3f6961
--- /dev/null
+++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/CronServiceImplTest.java
@@ -0,0 +1,152 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.service;
+
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.google.common.collect.Lists;
+import java.io.IOException;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.search.IndexInfo;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.opengroup.osdu.core.common.search.IndicesService;
+import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.opengroup.osdu.indexer.util.ElasticClientHandler;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.springframework.test.context.junit4.SpringRunner;
+
+
+@RunWith(SpringRunner.class)
+@PrepareForTest({RestHighLevelClient.class})
+public class CronServiceImplTest {
+
+ @Mock
+ private RestHighLevelClient restHighLevelClient;
+ @Mock
+ private IndicesService indicesService;
+ @Mock
+ private ElasticClientHandler elasticClientHandler;
+ @Mock
+ private IndexerConfigurationProperties configurationProperties;
+ @Mock
+ private IRequestInfo requestInfo;
+ @Mock
+ private JaxRsDpsLog log;
+ @InjectMocks
+ private CronServiceImpl sut;
+
+ @InjectMocks
+ private DpsHeaders dpsHeaders;
+
+ @Before
+ public void setup() {
+
+ when(this.requestInfo.getHeaders()).thenReturn(dpsHeaders);
+
+ when(configurationProperties.getCronIndexCleanupThresholdDays()).thenReturn(3);
+ when(configurationProperties.getCronEmptyIndexCleanupThresholdDays()).thenReturn(3);
+ }
+
+ @Test
+ public void run_cleanup_when_cron_job_runs_with_correct_pattern() throws Exception {
+ final String indexPattern = "tenant1-index-*";
+
+ IndexInfo info = IndexInfo.builder().name("tenant1-index-1.0.0").documentCount("10")
+ .creationDate(Long.toString(Instant.now().minus(4, ChronoUnit.DAYS).toEpochMilli()))
+ .build();
+
+ when(this.requestInfo.getPartitionId()).thenReturn("tenant1");
+ when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient);
+ when(this.indicesService.getIndexInfo(this.restHighLevelClient, indexPattern))
+ .thenReturn(Lists.newArrayList(info));
+
+ this.sut.cleanupIndices(indexPattern);
+
+ verify(this.indicesService, times(1)).deleteIndex(restHighLevelClient, "tenant1-index-1.0.0");
+ verify(this.indicesService, times(1)).getIndexInfo(restHighLevelClient, indexPattern);
+ }
+
+ @Test(expected = IOException.class)
+ public void run_cleanup_when_cron_job_runs_with_wrong_pattern() throws Exception {
+ IOException exception = new IOException("blah");
+ when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient);
+ when(this.indicesService.getIndexInfo(this.restHighLevelClient, "tenant1-test-*"))
+ .thenThrow(exception);
+
+ this.sut.cleanupIndices("tenant1-test-*");
+
+ verify(this.indicesService, times(0)).deleteIndex(any(), any());
+ }
+
+ @Test
+ public void run_cleanup_when_backend_does_not_have_empty_stale_indices() throws Exception {
+ IndexInfo info = IndexInfo.builder().name("tenant1-index-1.0.0").documentCount("10")
+ .creationDate(Long.toString(Instant.now().minus(8, ChronoUnit.DAYS).toEpochMilli()))
+ .build();
+
+ when(this.requestInfo.getPartitionId()).thenReturn("tenant1");
+ when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient);
+ when(this.indicesService.getIndexInfo(this.restHighLevelClient, null))
+ .thenReturn(Lists.newArrayList(info));
+
+ this.sut.cleanupEmptyStaleIndices();
+
+ verify(this.indicesService, times(0)).deleteIndex(restHighLevelClient, null);
+ verify(this.indicesService, times(1)).getIndexInfo(restHighLevelClient, null);
+ }
+
+ @Test
+ public void run_cleanup_when_backend_have_empty_stale_indices() throws Exception {
+ IndexInfo info = IndexInfo.builder().name("tenant1-index-1.0.0").documentCount("0")
+ .creationDate(Long.toString(Instant.now().minus(8, ChronoUnit.DAYS).toEpochMilli()))
+ .build();
+
+ when(this.requestInfo.getPartitionId()).thenReturn("tenant1");
+ when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient);
+ when(this.indicesService.getIndexInfo(this.restHighLevelClient, null))
+ .thenReturn(Lists.newArrayList(info));
+
+ this.sut.cleanupEmptyStaleIndices();
+
+ verify(this.indicesService, times(1)).deleteIndex(restHighLevelClient, "tenant1-index-1.0.0");
+ verify(this.indicesService, times(1)).getIndexInfo(restHighLevelClient, null);
+ }
+
+ @Test(expected = IOException.class)
+ public void run_cleanup_when_backend_throws_exception() throws Exception {
+ IOException exception = new IOException("blah");
+ when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient);
+ when(this.indicesService.getIndexInfo(this.restHighLevelClient, null)).thenThrow(exception);
+
+ this.sut.cleanupEmptyStaleIndices();
+
+ verify(this.indicesService, times(0)).deleteIndex(any(), any());
+ }
+}
\ No newline at end of file
diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..b09fbc40ddb38ffd327be67375a4b7718d687722
--- /dev/null
+++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceTest.java
@@ -0,0 +1,118 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.service;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.Mockito.when;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.search.ClusterSettings;
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
+import org.opengroup.osdu.core.common.multitenancy.ITenantInfoService;
+import org.opengroup.osdu.core.common.provider.interfaces.IElasticCredentialsCache;
+import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository;
+import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.springframework.test.context.junit4.SpringRunner;
+
+@RunWith(SpringRunner.class)
+public class ElasticSettingServiceTest {
+
+ @Mock
+ private ITenantInfoService tenantInfoService;
+ @Mock
+ private IElasticRepository elasticRepository;
+ @Mock
+ private IElasticCredentialsCache elasticCredentialCache;
+ @Mock
+ private IndexerConfigurationProperties configurationProperties;
+ @Mock
+ private TenantInfo tenantInfo;
+ @InjectMocks
+ private ElasticSettingServiceImpl sut;
+ @Mock
+ private ClusterSettings clusterSettings;
+ @Mock
+ private DpsHeaders headersInfo;
+
+ @Mock
+ private JaxRsDpsLog log;
+
+
+ public String GAE_SERVICE = "indexer";
+
+ private final String host = "db5c51c1.us-central1.gcp.cloud.es.io";
+ private final int port = 9243;
+ private final String credentials = "name:password";
+
+ String cacheKey = "";
+
+
+ @Before
+ public void setup() {
+ when(tenantInfo.getName()).thenReturn("tenant1");
+ when(this.headersInfo.getPartitionId()).thenReturn("tenant1");
+ when(this.tenantInfoService.getTenantInfo()).thenReturn(tenantInfo);
+ when(configurationProperties.getGaeService()).thenReturn("indexer");
+ clusterSettings = ClusterSettings.builder().host(host).port(port)
+ .userNameAndPassword(credentials).build();
+ cacheKey = String.format("%s-%s", GAE_SERVICE, tenantInfo.getName());
+ }
+
+ @Test
+ public void should_getValid_clusterSettings_fromCache() {
+
+ when(this.elasticCredentialCache.get(cacheKey)).thenReturn(clusterSettings);
+
+ ClusterSettings response = this.sut.getElasticClusterInformation();
+ assertNotNull(response);
+ assertEquals(response.getHost(), host);
+ assertEquals(response.getPort(), port);
+ assertEquals(response.getUserNameAndPassword(), credentials);
+ }
+
+ @Test
+ public void should_getValid_clusterSettings_fromCosmosDB() {
+
+ when(this.elasticCredentialCache.get(cacheKey)).thenReturn(clusterSettings);
+
+ when(this.elasticRepository.getElasticClusterSettings(tenantInfo)).thenReturn(clusterSettings);
+
+ ClusterSettings response = this.sut.getElasticClusterInformation();
+ assertNotNull(response);
+ assertEquals(response.getHost(), host);
+ assertEquals(response.getPort(), port);
+ assertEquals(response.getUserNameAndPassword(), credentials);
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throwAppException_when_tenantClusterInfo_not_found() throws AppException {
+
+ when(this.elasticRepository.getElasticClusterSettings(tenantInfo)).thenReturn(null);
+
+ this.sut.getElasticClusterInformation();
+
+ }
+}
diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImplTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImplTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..6a438da28b492752abe329f329c0cbd85ec09977
--- /dev/null
+++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImplTest.java
@@ -0,0 +1,203 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.service;
+
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import java.io.IOException;
+import java.lang.reflect.Type;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.http.HttpEntity;
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentMatchers;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.indexer.IElasticSettingService;
+import org.opengroup.osdu.core.common.model.search.ClusterSettings;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
+import org.opengroup.osdu.core.common.search.IndicesService;
+import org.opengroup.osdu.indexer.logging.AuditLogger;
+import org.opengroup.osdu.indexer.util.ElasticClientHandler;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.springframework.test.context.junit4.SpringRunner;
+
+@RunWith(SpringRunner.class)
+@PrepareForTest({RestHighLevelClient.class, Response.class, RestClient.class, HttpEntity.class,
+ EntityUtils.class})
+public class IndexCopyServiceImplTest {
+
+ private final String correlationId = UUID.randomUUID().toString();
+
+ @Mock
+ private HttpEntity httpEntity;
+ @Mock
+ private HttpEntity httpEntityRequest;
+ @Mock
+ private IRequestInfo requestInfo;
+ @Mock
+ private DpsHeaders headersInfo;
+ @Mock
+ private RestClient restClient;
+ @Mock
+ private RestHighLevelClient restHighLevelClient;
+ @Mock
+ private IndicesService indicesService;
+ @Mock
+ private IndexerMappingService mappingService;
+ @Mock
+ private ElasticClientHandler elasticClientHandler;
+ @Mock
+ private ElasticIndexNameResolver elasticIndexNameResolver;
+ @Mock
+ private Response response;
+ @Mock
+ private IElasticSettingService elasticSettingService;
+ @Mock
+ private AuditLogger auditLogger;
+ @Mock
+ private Map<String, String> httpHeaders;
+ @InjectMocks
+ private IndexCopyServiceImpl sut;
+
+ private ClusterSettings commonCluster;
+
+ private Map<String, Object> correctMap;
+
+ @Before
+ public void setup() {
+
+ commonCluster = ClusterSettings.builder().host("commonhost").port(8080)
+ .userNameAndPassword("username:pwd").build();
+
+ httpHeaders = new HashMap<>();
+ httpHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
+ httpHeaders.put(DpsHeaders.CORRELATION_ID, correlationId);
+ when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(httpHeaders);
+ when(response.getEntity()).thenReturn(httpEntity);
+
+ Type mapType = new TypeToken<Map<String, Object>>() {
+ }.getType();
+ String afterFormat = "{\"properties\":{\"id\":{\"type\":\"keyword\"}}}";
+ correctMap = new Gson().fromJson(afterFormat, mapType);
+
+ restHighLevelClient = mock(RestHighLevelClient.class);
+
+ }
+
+ @Test(expected = IOException.class)
+ public void should_throwIOException_when_indexMappingNotFound() throws Exception {
+ IOException exception = new IOException(
+ "Fail to get mapping for the given index from common cluster.");
+
+ when(this.mappingService.getIndexMapping(ArgumentMatchers.any(), ArgumentMatchers.any()))
+ .thenThrow(exception);
+
+ this.sut.copyIndex("common:metadata:entity:1.0.0");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void should_throwIllegalArgExceptionCopyIndexRequest_copyIndexTest() {
+ try {
+ this.sut.copyIndex(null);
+ } catch (IOException e) {
+ fail("Should not throw IOException but illegalArgumentException.");
+ }
+ }
+
+ @Test
+ public void should_returnIndexMapping_getIndexMappingFromCommonClustertest() {
+ String mappingJson = "{\"common-metadata-entity-1.0.0\":{\"mappings\":{\"entity\":{\"properties\":{\"id\":{\"type\":\"keyword\"}}}}}}";
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ try {
+ when(this.mappingService.getIndexMapping(ArgumentMatchers.any(), ArgumentMatchers.any()))
+ .thenReturn(mappingJson);
+ Map<String, Object> resultMap = this.sut.getIndexMappingsFromCommonCluster("test", "test");
+ Assert.assertEquals(resultMap, correctMap);
+ } catch (Exception ignored) {
+ }
+ }
+
+ @Test
+ public void should_returnClusterInfo_getCommonClusterInformationtest() {
+ try {
+ String[] correctCommonCluster = {"https://commonhost:8080", "username", "pwd"};
+
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+
+ when(elasticSettingService.getElasticClusterInformation()).thenReturn(commonCluster);
+
+ String[] resultCommonCluster = this.sut.getCommonClusterInformation();
+ Assert.assertEquals(correctCommonCluster[0], resultCommonCluster[0]);
+ Assert.assertEquals(correctCommonCluster[1], resultCommonCluster[1]);
+ Assert.assertEquals(correctCommonCluster[2], resultCommonCluster[2]);
+ } catch (IOException ignored) {
+ fail("Should not throw this exception " + ignored.getMessage());
+ }
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throwException_failToCreateIndexInTenantCluster_createIndexInTenantClustertest() {
+ try {
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ when(indicesService
+ .createIndex(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(),
+ ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(false);
+ this.sut.createIndexInTenantCluster("test", "test", "test", correctMap);
+ } catch (IOException ignored) {
+ fail("Should not throw this exception " + ignored.getMessage());
+ }
+ }
+
+ @Ignore
+ public void should_returnTaskIdResponse_reindexRequestSucceed_reindexInTenantClustertest() {
+ //TODO: fix the null Response from restHighLevelClient.getLowLevelClient().performRequest().
+ try {
+ String[] correctCommonCluster = {"https://commonhost:8080", "username", "pwd"};
+ Request request = new Request("POST", "/_reindex?wait_for_completion=false");
+ request.setEntity(httpEntityRequest);
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ when(indicesService
+ .createIndex(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(),
+ ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(false);
+ when(restHighLevelClient.getLowLevelClient()).thenReturn(restClient);
+ when(restClient.performRequest(request)).thenReturn(response);
+ when(response.getEntity()).thenReturn(httpEntity);
+ Assert.assertEquals(httpEntity,
+ this.sut.reindexInTenantCluster("test", "test", correctCommonCluster));
+ } catch (IOException ignored) {
+ }
+ }
+}
diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..4ce6eb4e8af4a45eddbc163baad9bb9cd6c2ef12
--- /dev/null
+++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java
@@ -0,0 +1,445 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.service;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.anyBoolean;
+import static org.mockito.Mockito.anyString;
+import static org.mockito.Mockito.eq;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.MockitoAnnotations.initMocks;
+import static org.powermock.api.mockito.PowerMockito.mock;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.http.HttpStatus;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.http.RequestStatus;
+import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
+import org.opengroup.osdu.core.common.model.indexer.OperationType;
+import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
+import org.opengroup.osdu.core.common.search.IndicesService;
+import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache;
+import org.opengroup.osdu.indexer.service.impl.SchemaProviderImpl;
+import org.opengroup.osdu.indexer.util.ElasticClientHandler;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.springframework.test.context.junit4.SpringRunner;
+
+@RunWith(SpringRunner.class)
+@PrepareForTest({RestHighLevelClient.class})
+public class IndexerSchemaServiceTest {
+
+ private final String kind = "tenant:test:test:1.0.0";
+ private final String emptySchema = null;
+ private final String someSchema = "{\"kind\":\"tenant:test:test:1.0.0\", \"schema\":[{\"path\":\"test-path\", \"kind\":\"tenant:test:test:1.0.0\"}]}";
+
+ @Mock
+ private JaxRsDpsLog log;
+ @Mock
+ private SchemaProviderImpl schemaProvider;
+ @Mock
+ private ElasticClientHandler elasticClientHandler;
+ @Mock
+ private ElasticIndexNameResolver elasticIndexNameResolver;
+ @Mock
+ private IndexerMappingService mappingService;
+ @Mock
+ private IndicesService indicesService;
+ @Mock
+ private ISchemaCache schemaCache;
+ @InjectMocks
+ private IndexSchemaServiceImpl sut;
+
+ @Before
+ public void setup() {
+ initMocks(this);
+ RestHighLevelClient restHighLevelClient = mock(RestHighLevelClient.class);
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ }
+
+ @Test
+ public void should_returnNull_givenEmptySchema_getIndexerInputSchemaSchemaTest()
+ throws Exception {
+ when(schemaProvider.getSchema(any())).thenReturn(emptySchema);
+
+ IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false);
+
+ Assert.assertNotNull(indexSchema);
+ }
+
+ @Test
+ public void should_returnValidResponse_givenValidSchema_getIndexerInputSchemaTest()
+ throws Exception {
+ when(schemaProvider.getSchema(any())).thenReturn(someSchema);
+
+ IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false);
+
+ Assert.assertEquals(kind, indexSchema.getKind());
+ }
+
+ @Test
+ public void should_returnValidResponse_givenValidSchemaWithCacheHit_getIndexerInputSchemaTest()
+ throws Exception {
+ when(schemaProvider.getSchema(any())).thenReturn(someSchema);
+ when(this.schemaCache.get(kind + "_flattened")).thenReturn(someSchema);
+
+ IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false);
+
+ Assert.assertEquals(kind, indexSchema.getKind());
+ }
+
+ @Test
+ public void should_throw500_givenInvalidSchemaCacheHit_getIndexerInputSchemaTest() {
+ try {
+ String invalidSchema = "{}}";
+ when(schemaProvider.getSchema(any())).thenReturn(invalidSchema);
+
+ this.sut.getIndexerInputSchema(kind, false);
+ fail("Should throw exception");
+ } catch (AppException e) {
+ Assert.assertEquals(HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getError().getCode());
+ Assert.assertEquals("An error has occurred while normalizing the schema.",
+ e.getError().getMessage());
+ } catch (Exception e) {
+ fail("Should not throw exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_return_basic_schema_when_storage_returns_no_schema() {
+ IndexSchema returnedSchema = this.sut.getIndexerInputSchema(kind, false);
+
+ assertNotNull(returnedSchema.getDataSchema());
+ assertNotNull(returnedSchema);
+ assertEquals(kind, returnedSchema.getKind());
+ }
+
+ @Test
+ public void should_create_schema_when_storage_returns_valid_schema()
+ throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"startDate\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"endDate\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"type \"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"itemguid\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.create_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind))
+ .thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(false);
+ when(this.schemaProvider.getSchema(kind)).thenReturn(storageSchema);
+
+ this.sut.processSchemaMessages(schemaMessages);
+
+ verify(this.mappingService, times(1)).getIndexMappingFromRecordSchema(any());
+ verify(this.indicesService, times(1)).createIndex(any(), any(), any(), any(), any());
+ verifyNoMoreInteractions(this.mappingService);
+ }
+
+ @Test
+ public void should_merge_mapping_when_storage_returns_valid_schema()
+ throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"startDate\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.create_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind))
+ .thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.schemaProvider.getSchema(kind)).thenReturn(storageSchema);
+
+ this.sut.processSchemaMessages(schemaMessages);
+
+ verify(this.indicesService, times(0)).createIndex(any(), any(), any(), any(), any());
+ verify(this.mappingService, times(1)).createMapping(any(), any(), any(), anyBoolean());
+ verifyNoMoreInteractions(this.mappingService);
+ }
+
+ @Test
+ public void should_throw_mapping_conflict_when_elastic_backend_cannot_process_schema_changes()
+ throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String reason = String
+ .format("Could not create type mapping %s/completion.", kind.replace(":", "-"));
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.create_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind))
+ .thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.schemaProvider.getSchema(kind)).thenReturn(storageSchema);
+ when(this.mappingService.createMapping(any(), any(), any(), anyBoolean()))
+ .thenThrow(new AppException(HttpStatus.SC_BAD_REQUEST, reason, ""));
+
+ try {
+ this.sut.processSchemaMessages(schemaMessages);
+ } catch (AppException e) {
+ assertEquals(RequestStatus.SCHEMA_CONFLICT, e.getError().getCode());
+ assertEquals("error creating or merging index mapping", e.getError().getMessage());
+ assertEquals(reason, e.getError().getReason());
+ } catch (Exception e) {
+ fail("Should not throw this exception " + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_throw_genericAppException_when_elastic_backend_cannot_process_schema_changes()
+ throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String reason = String
+ .format("Could not create type mapping %s/completion.", kind.replace(":", "-"));
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.create_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind))
+ .thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.schemaProvider.getSchema(kind)).thenReturn(storageSchema);
+ when(this.mappingService.createMapping(any(), any(), any(), anyBoolean()))
+ .thenThrow(new AppException(HttpStatus.SC_FORBIDDEN, reason, "blah"));
+
+ try {
+ this.sut.processSchemaMessages(schemaMessages);
+ } catch (AppException e) {
+ assertEquals(HttpStatus.SC_FORBIDDEN, e.getError().getCode());
+ assertEquals("blah", e.getError().getMessage());
+ assertEquals(reason, e.getError().getReason());
+ } catch (Exception e) {
+ fail("Should not throw this exception " + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_log_and_do_nothing_when_storage_returns_invalid_schema()
+ throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"" +
+ "}";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.create_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind))
+ .thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.schemaProvider.getSchema(kind)).thenReturn(storageSchema);
+
+ this.sut.processSchemaMessages(schemaMessages);
+
+ verify(this.log).warning(eq("schema not found for kind: tenant1:avocet:completion:1.0.0"));
+ }
+
+ @Test
+ public void should_invalidateCache_when_purge_schema_and_schema_found_in_cache()
+ throws IOException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.purge_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind))
+ .thenReturn(kind.replace(":", "-"));
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.schemaCache.get(kind)).thenReturn("schema");
+ when(this.schemaCache.get(kind + "_flattened")).thenReturn("flattened schema");
+
+ this.sut.processSchemaMessages(schemaMessages);
+
+ verify(this.schemaCache, times(2)).get(anyString());
+ verify(this.schemaCache, times(2)).delete(anyString());
+ }
+
+ @Test
+ public void should_log_warning_when_purge_schema_and_schema_not_found_in_cache()
+ throws IOException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.purge_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind))
+ .thenReturn(kind.replace(":", "-"));
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(false);
+
+ this.sut.processSchemaMessages(schemaMessages);
+
+ verify(this.log).warning(eq(String.format("Kind: %s not found", kind)));
+ }
+
+ @Test
+ public void should_sync_schema_with_storage() throws Exception {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind))
+ .thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.indicesService.deleteIndex(any(), any())).thenReturn(true);
+ when(this.schemaProvider.getSchema(kind)).thenReturn(storageSchema);
+
+ this.sut.syncIndexMappingWithStorageSchema(kind);
+
+ verify(this.mappingService, times(1)).getIndexMappingFromRecordSchema(any());
+ verify(this.indicesService, times(1)).isIndexExist(any(), any());
+ verify(this.indicesService, times(1)).deleteIndex(any(), any());
+ verify(this.indicesService, times(1)).createIndex(any(), any(), any(), any(), any());
+ verifyNoMoreInteractions(this.mappingService);
+ }
+
+ @Test
+ public void should_throw_exception_while_snapshot_running_sync_schema_with_storage()
+ throws Exception {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind))
+ .thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.indicesService.deleteIndex(any(), any()))
+ .thenThrow(new AppException(HttpStatus.SC_CONFLICT, "Index deletion error", "blah"));
+
+ try {
+ this.sut.syncIndexMappingWithStorageSchema(kind);
+ } catch (AppException e) {
+ assertEquals(HttpStatus.SC_CONFLICT, e.getError().getCode());
+ assertEquals("blah", e.getError().getMessage());
+ assertEquals("Index deletion error", e.getError().getReason());
+ } catch (Exception e) {
+ fail("Should not throw this exception " + e.getMessage());
+ }
+
+ verify(this.indicesService, times(1)).isIndexExist(any(), any());
+ verify(this.indicesService, times(1)).deleteIndex(any(), any());
+ verify(this.mappingService, never()).getIndexMappingFromRecordSchema(any());
+ verify(this.indicesService, never()).createIndex(any(), any(), any(), any(), any());
+ }
+
+ @Test
+ public void should_return_true_while_if_forceClean_requested() throws IOException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind))
+ .thenReturn(kind.replace(":", "-"));
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+
+ assertTrue(this.sut.isStorageSchemaSyncRequired(kind, true));
+ }
+
+ @Test
+ public void should_return_true_while_if_forceClean_notRequested_and_indexNotFound()
+ throws IOException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind))
+ .thenReturn(kind.replace(":", "-"));
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(false);
+
+ assertTrue(this.sut.isStorageSchemaSyncRequired(kind, false));
+ }
+
+ @Test
+ public void should_return_false_while_if_forceClean_notRequested_and_indexExist()
+ throws IOException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind))
+ .thenReturn(kind.replace(":", "-"));
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+
+ assertFalse(this.sut.isStorageSchemaSyncRequired(kind, false));
+ }
+}
diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..bf7fa01a337c13971b47efc97c5b20f96788cb9d
--- /dev/null
+++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java
@@ -0,0 +1,161 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.service;
+
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.MockitoAnnotations.initMocks;
+import static org.powermock.api.mockito.PowerMockito.mockStatic;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
+import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
+import org.powermock.modules.junit4.PowerMockRunner;
+import org.powermock.modules.junit4.PowerMockRunnerDelegate;
+import org.springframework.test.context.junit4.SpringRunner;
+
+@RunWith(PowerMockRunner.class)
+@PowerMockRunnerDelegate(SpringRunner.class)
+public class ReindexServiceTest {
+
+ private final String cursor = "100";
+
+ private final String correlationId = UUID.randomUUID().toString();
+
+ @Mock
+ private IndexerConfigurationProperties configurationProperties;
+ @Mock
+ private StorageService storageService;
+ @Mock
+ private IRequestInfo requestInfo;
+ @Mock
+ private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
+ @Mock
+ private JaxRsDpsLog log;
+ @InjectMocks
+ private ReindexServiceImpl sut;
+
+ private RecordReindexRequest recordReindexRequest;
+ private RecordQueryResponse recordQueryResponse;
+
+ private Map<String, String> httpHeaders;
+
+ @Before
+ public void setup() {
+ initMocks(this);
+
+ mockStatic(UUID.class);
+
+ recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0")
+ .cursor(cursor).build();
+ recordQueryResponse = new RecordQueryResponse();
+
+ httpHeaders = new HashMap<>();
+ httpHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
+ httpHeaders.put(DpsHeaders.CORRELATION_ID, correlationId);
+ DpsHeaders standardHeaders = DpsHeaders.createFromMap(httpHeaders);
+ when(requestInfo.getHeaders()).thenReturn(standardHeaders);
+ when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(httpHeaders);
+ when(requestInfo.getHeadersWithDwdAuthZ()).thenReturn(standardHeaders);
+ }
+
+ @Test
+ public void should_returnNull_givenNullResponseResult_reIndexRecordsTest() {
+ try {
+ recordQueryResponse.setResults(null);
+ when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
+
+ String response = sut.reindexRecords(recordReindexRequest, false);
+
+ Assert.assertNull(response);
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_returnNull_givenEmptyResponseResult_reIndexRecordsTest() {
+ try {
+ recordQueryResponse.setResults(new ArrayList<>());
+ when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
+
+ String response = sut.reindexRecords(recordReindexRequest, false);
+
+ Assert.assertNull(response);
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Ignore
+ @Test
+ public void should_returnRecordQueryRequestPayload_givenValidResponseResult_reIndexRecordsTest() {
+ try {
+ recordQueryResponse.setCursor(cursor);
+ List<String> results = new ArrayList<>();
+ results.add("test1");
+ recordQueryResponse.setResults(results);
+
+ when(configurationProperties.getStorageRecordsBatchSize()).thenReturn(1);
+
+ when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
+
+ String taskQueuePayload = sut.reindexRecords(recordReindexRequest, false);
+
+ Assert.assertEquals("{\"kind\":\"tenant:test:test:1.0.0\",\"cursor\":\"100\"}",
+ taskQueuePayload);
+ } catch (Exception e) {
+ fail("Should not throw exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_returnRecordChangedMessage_givenValidResponseResult_reIndexRecordsTest() {
+ try {
+ List<String> results = new ArrayList<>();
+ results.add("test1");
+ recordQueryResponse.setResults(results);
+ when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
+
+ String taskQueuePayload = sut.reindexRecords(recordReindexRequest, false);
+
+ Assert.assertEquals(String.format(
+ "{\"data\":\"[{\\\"id\\\":\\\"test1\\\",\\\"kind\\\":\\\"tenant:test:test:1.0.0\\\",\\\"op\\\":\\\"create\\\"}]\",\"attributes\":{\"correlation-id\":\"%s\"}}",
+ correlationId), taskQueuePayload);
+ } catch (Exception e) {
+ fail("Should not throw exception" + e.getMessage());
+ }
+ }
+}
diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..4b46abf2e3380742203290ead746c39dff3a4049
--- /dev/null
+++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceTest.java
@@ -0,0 +1,272 @@
+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.service;
+
+import static java.util.Collections.singletonList;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import java.lang.reflect.Type;
+import java.net.URISyntaxException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentMatchers;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.opengroup.osdu.core.common.http.IUrlFetchService;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.http.HttpResponse;
+import org.opengroup.osdu.core.common.model.indexer.IndexingStatus;
+import org.opengroup.osdu.core.common.model.indexer.JobStatus;
+import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
+import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
+import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
+import org.opengroup.osdu.core.common.model.indexer.Records;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.springframework.http.HttpStatus;
+import org.springframework.test.context.junit4.SpringRunner;
+
+@RunWith(SpringRunner.class)
+public class StorageServiceTest {
+
+ @Mock
+ private IUrlFetchService urlFetchService;
+ @Mock
+ private JobStatus jobStatus;
+ @Mock
+ private JaxRsDpsLog log;
+ @Mock
+ private IRequestInfo requestInfo;
+ @Mock
+ private IndexerConfigurationProperties configurationProperties;
+ @InjectMocks
+ private StorageServiceImpl sut;
+
+ private List<String> ids;
+ private static final String RECORD_ID1 = "tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465";
+ private static final String RECORDS_ID2 = "tenant1:doc:15e790a69beb4d789b1f979e2af2e813";
+
+ @Before
+ public void setup() {
+
+ String recordChangedMessages =
+ "[{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"purge\"},"
+ +
+ "{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"create\"}]";
+
+ when(this.requestInfo.getHeadersMap()).thenReturn(new HashMap<>());
+ when(this.requestInfo.getHeaders()).thenReturn(new DpsHeaders());
+
+ Type listType = new TypeToken<List<RecordInfo>>() {
+ }.getType();
+
+ List<RecordInfo> msgs = (new Gson()).fromJson(recordChangedMessages, listType);
+ jobStatus.initialize(msgs);
+ ids = Arrays.asList(RECORD_ID1, RECORDS_ID2);
+
+ when(configurationProperties.getStorageRecordsBatchSize()).thenReturn(20);
+ }
+
+ @Test
+ public void should_return404_givenNullData_getValidStorageRecordsTest()
+ throws URISyntaxException {
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(null);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ should_return404_getValidStorageRecordsTest();
+ }
+
+ @Test
+ public void should_return404_givenEmptyData_getValidStorageRecordsTest()
+ throws URISyntaxException {
+
+ String emptyDataFromStorage = "{\"records\":[],\"notFound\":[]}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(emptyDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ should_return404_getValidStorageRecordsTest();
+ }
+
+ @Test
+ public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsTest()
+ throws URISyntaxException {
+
+ String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"], \"conversionStatuses\": []}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+ Records storageRecords = this.sut.getStorageRecords(ids);
+
+ assertEquals(1, storageRecords.getRecords().size());
+ }
+
+ @Test
+ public void should_logMissingRecord_given_storageMissedRecords() throws URISyntaxException {
+
+ String validDataFromStorage = "{\"records\":[{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[]}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(any())).thenReturn(httpResponse);
+ Records storageRecords = this.sut.getStorageRecords(ids);
+
+ assertEquals(1, storageRecords.getRecords().size());
+ verify(this.jobStatus).addOrUpdateRecordStatus(singletonList(RECORDS_ID2), IndexingStatus.FAIL,
+ HttpStatus.NOT_FOUND.value(),
+ "Partial response received from Storage service - missing records",
+ "Partial response received from Storage service: tenant1:doc:15e790a69beb4d789b1f979e2af2e813");
+ }
+
+ @Test
+ public void should_returnValidJobStatus_givenFailedUnitsConversion_processRecordChangedMessageTest()
+ throws URISyntaxException {
+ String validDataFromStorage = "{\"records\":[{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[],\"conversionStatuses\":[{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\",\"status\":\"ERROR\",\"errors\":[\"crs conversion failed\"]}]}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(any())).thenReturn(httpResponse);
+ Records storageRecords = this.sut.getStorageRecords(singletonList(RECORDS_ID2));
+
+ assertEquals(1, storageRecords.getRecords().size());
+ verify(this.jobStatus)
+ .addOrUpdateRecordStatus(RECORDS_ID2, IndexingStatus.WARN, HttpStatus.BAD_REQUEST.value(),
+ "crs conversion failed", String
+ .format("record-id: %s | %s", "tenant1:doc:15e790a69beb4d789b1f979e2af2e813",
+ "crs conversion failed"));
+ }
+
+ @Test
+ public void should_returnValidResponse_givenValidRecordQueryRequest_getRecordListByKind()
+ throws Exception {
+
+ RecordReindexRequest recordReindexRequest = RecordReindexRequest.builder()
+ .kind("tenant:test:test:1.0.0").cursor("100").build();
+
+ HttpResponse httpResponse = new HttpResponse();
+ httpResponse.setBody(new Gson().toJson(recordReindexRequest, RecordReindexRequest.class));
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ RecordQueryResponse recordQueryResponse = this.sut.getRecordsByKind(recordReindexRequest);
+
+ assertEquals("100", recordQueryResponse.getCursor());
+ assertNull(recordQueryResponse.getResults());
+ }
+
+ @Test
+ public void should_returnValidResponse_givenValidKind_getSchemaByKind() throws Exception {
+
+ String validSchemaFromStorage = "{" +
+ " \"kind\": \"tenant:test:test:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"msg\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"references.entity\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]," +
+ " \"ext\": null" +
+ "}";
+ String kind = "tenant:test:test:1.0.0";
+
+ HttpResponse httpResponse = new HttpResponse();
+ httpResponse.setResponseCode(HttpStatus.OK.value());
+ httpResponse.setBody(validSchemaFromStorage);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ String recordSchemaResponse = this.sut.getStorageSchema(kind);
+
+ assertNotNull(recordSchemaResponse);
+ }
+
+ @Test
+ public void should_returnNullResponse_givenAbsentKind_getSchemaByKind() throws Exception {
+
+ String kind = "tenant:test:test:1.0.0";
+
+ HttpResponse httpResponse = new HttpResponse();
+ httpResponse.setResponseCode(HttpStatus.NOT_FOUND.value());
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ String recordSchemaResponse = this.sut.getStorageSchema(kind);
+
+ assertNull(recordSchemaResponse);
+ }
+
+ @Test
+ public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsWithInvalidConversionTest()
+ throws URISyntaxException {
+
+ String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"],\"conversionStatuses\": [{\"id\":\"testid\",\"status\":\"ERROR\",\"errors\":[\"conversion error occurred\"] } ]}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+ Records storageRecords = this.sut.getStorageRecords(ids);
+
+ assertEquals(1, storageRecords.getRecords().size());
+
+ assertEquals(1, storageRecords.getConversionStatuses().get(0).getErrors().size());
+
+ assertEquals("conversion error occurred",
+ storageRecords.getConversionStatuses().get(0).getErrors().get(0));
+ }
+
+ private void should_return404_getValidStorageRecordsTest() {
+ try {
+ this.sut.getStorageRecords(ids);
+ fail("Should throw exception");
+ } catch (AppException e) {
+ assertEquals(HttpStatus.NOT_FOUND.value(), e.getError().getCode());
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+}
diff --git a/provider/indexer-reference/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/provider/indexer-reference/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker
new file mode 100644
index 0000000000000000000000000000000000000000..ca6ee9cea8ec189a088d50559325d4e84ff8ad09
--- /dev/null
+++ b/provider/indexer-reference/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker
@@ -0,0 +1 @@
+mock-maker-inline
\ No newline at end of file