diff --git a/pom.xml b/pom.xml
index 0bf363b5630407611fdd47070d23aa7fcde57d83..463794ff03568206e6348128bf108e965315e728 100644
--- a/pom.xml
+++ b/pom.xml
@@ -130,6 +130,7 @@
<module>provider/indexer-azure</module>
<module>provider/indexer-gcp</module>
<module>provider/indexer-ibm</module>
+ <module>provider/indexer-reference</module>
</modules>
<repositories>
diff --git a/provider/indexer-reference/Dockerfile b/provider/indexer-reference/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..1e6901f365e2e9c7049c5019b4037ade931b65f0
--- /dev/null
+++ b/provider/indexer-reference/Dockerfile
@@ -0,0 +1,5 @@
+FROM openjdk:8-slim
+WORKDIR /app
+COPY target/indexer-reference-0.0.1-SNAPSHOT-spring-boot.jar indexer-reference.jar
+# Run the web service on container startup.
+CMD java -Djava.security.egd=file:/dev/./urandom -Dserver.port=8080 -jar /app/indexer-reference.jar
diff --git a/provider/indexer-reference/docker-compose.yml b/provider/indexer-reference/docker-compose.yml
new file mode 100644
index 0000000000000000000000000000000000000000..a6838447de241e658325e275434293b2d4e2be09
--- /dev/null
+++ b/provider/indexer-reference/docker-compose.yml
@@ -0,0 +1,12 @@
+version: "3"
+services:
+ os-storage-app:
+ build:
+ args:
+ JAR_FILE: target/indexer-reference-0.0.1-SNAPSHOT-spring-boot.jar
+ context: ""
+ dockerfile: ../Dockerfile
+ image: us.gcr.io/osdu-anthos-02/os-indexer/anthos-indexer-reference
+ ports:
+ - "8080:8080"
+
diff --git a/provider/indexer-reference/kubernetes/deployments/deployment-os-indexer-service.yml b/provider/indexer-reference/kubernetes/deployments/deployment-os-indexer-service.yml
new file mode 100644
index 0000000000000000000000000000000000000000..0157d9875389754f1c640f0d186e02b2ea0020d8
--- /dev/null
+++ b/provider/indexer-reference/kubernetes/deployments/deployment-os-indexer-service.yml
@@ -0,0 +1,179 @@
+apiVersion: v1
+data:
+ AUTHORIZE_API: ${AUTHORIZE_API}
+ MONGO_DB_URL: ${MONGO_DB_URL}
+ MONGO_DB_USER: ${MONGO_DB_USER}
+ MONGO_DB_NAME: ${MONGO_DB_NAME}
+ REGION: ${REGION}
+ LEGALTAG_API: ${LEGALTAG_API}
+ INDEXER_HOST: ${INDEXER_HOST}
+ GOOGLE_CLOUD_PROJECT: ${GOOGLE_CLOUD_PROJECT}
+ STORAGE_HOSTNAME: ${STORAGE_HOSTNAME}
+ STORAGE_SCHEMA_HOST: ${STORAGE_SCHEMA_HOST}
+ STORAGE_QUERY_RECORD_HOST: ${STORAGE_QUERY_RECORD_HOST}
+ STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST: ${STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST}
+ INDEXER_QUEUE_HOST: ${INDEXER_QUEUE_HOST}
+ CRS_API: ${CRS_API}
+ REDIS_GROUP_HOST: ${REDIS_GROUP_HOST}
+ REDIS_SEARCH_HOST: ${REDIS_SEARCH_HOST}
+ GOOGLE_AUDIENCES: ${GOOGLE_AUDIENCES}
+ KEY_RING: ${KEY_RING}
+ KMS_KEY: ${KMS_KEY}
+kind: ConfigMap
+metadata:
+ labels:
+ app: indexer-reference
+ name: indexer-config
+ namespace: default
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ generateName: indexer-reference-anthos
+ labels:
+ app: indexer-reference
+ name: indexer-reference
+ namespace: default
+spec:
+ selector:
+ matchLabels:
+ app: indexer-reference
+ replicas: 1
+ template:
+ metadata:
+ labels:
+ app: indexer-reference
+ spec:
+ containers:
+ - env:
+ - name: AUTHORIZE_API
+ valueFrom:
+ configMapKeyRef:
+ key: AUTHORIZE_API
+ name: indexer-config
+ - name: MONGO_DB_URL
+ valueFrom:
+ configMapKeyRef:
+ key: MONGO_DB_URL
+ name: indexer-config
+ - name: MONGO_DB_USER
+ valueFrom:
+ configMapKeyRef:
+ key: MONGO_DB_USER
+ name: indexer-config
+ - name: MONGO_DB_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: indexer-secret
+ key: mongo.db.password
+ - name: MONGO_DB_NAME
+ valueFrom:
+ configMapKeyRef:
+ key: MONGO_DB_NAME
+ name: indexer-config
+ - name: MB_RABBITMQ_URI
+ valueFrom:
+ secretKeyRef:
+ name: indexer-secret
+ key: mb.rabbitmq.uri
+ - name: REGION
+ valueFrom:
+ configMapKeyRef:
+ key: REGION
+ name: indexer-config
+ - name: LEGALTAG_API
+ valueFrom:
+ configMapKeyRef:
+ key: LEGALTAG_API
+ name: indexer-config
+ - name: INDEXER_HOST
+ valueFrom:
+ configMapKeyRef:
+ key: INDEXER_HOST
+ name: indexer-config
+ - name: GOOGLE_CLOUD_PROJECT
+ valueFrom:
+ configMapKeyRef:
+ key: GOOGLE_CLOUD_PROJECT
+ name: indexer-config
+ - name: STORAGE_HOSTNAME
+ valueFrom:
+ configMapKeyRef:
+ key: STORAGE_HOSTNAME
+ name: indexer-config
+ - name: STORAGE_SCHEMA_HOST
+ valueFrom:
+ configMapKeyRef:
+ key: STORAGE_SCHEMA_HOST
+ name: indexer-config
+ - name: STORAGE_QUERY_RECORD_HOST
+ valueFrom:
+ configMapKeyRef:
+ key: STORAGE_QUERY_RECORD_HOST
+ name: indexer-config
+ - name: STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST
+ valueFrom:
+ configMapKeyRef:
+ key: STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST
+ name: indexer-config
+ - name: INDEXER_QUEUE_HOST
+ valueFrom:
+ configMapKeyRef:
+ key: INDEXER_QUEUE_HOST
+ name: indexer-config
+ - name: CRS_API
+ valueFrom:
+ configMapKeyRef:
+ key: CRS_API
+ name: indexer-config
+ - name: REDIS_GROUP_HOST
+ valueFrom:
+ configMapKeyRef:
+ key: REDIS_GROUP_HOST
+ name: indexer-config
+ - name: REDIS_SEARCH_HOST
+ valueFrom:
+ configMapKeyRef:
+ key: REDIS_SEARCH_HOST
+ name: indexer-config
+ - name: GOOGLE_AUDIENCES
+ valueFrom:
+ configMapKeyRef:
+ key: GOOGLE_AUDIENCES
+ name: indexer-config
+ - name: KEY_RING
+ valueFrom:
+ configMapKeyRef:
+ key: KEY_RING
+ name: indexer-config
+ - name: KMS_KEY
+ valueFrom:
+ configMapKeyRef:
+ key: KMS_KEY
+ name: indexer-config
+ image: us.gcr.io/osdu-anthos-02/os-indexer/anthos-indexer-reference:9966597-dirty
+ name: indexer-reference
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: indexer-reference
+ namespace: default
+spec:
+ ports:
+ - protocol: TCP
+ port: 80
+ targetPort: 8080
+ selector:
+ app: indexer-reference
+ type: LoadBalancer
+---
+apiVersion: v1
+data:
+ mongo.db.password: ${mongo.db.password}
+ mb.rabbitmq.uri: ${mb.rabbitmq.uri}
+kind: Secret
+metadata:
+ name: indexer-secret
+ namespace: default
+type: Opaque
\ No newline at end of file
diff --git a/provider/indexer-reference/pom.xml b/provider/indexer-reference/pom.xml
new file mode 100644
index 0000000000000000000000000000000000000000..99d01082c723b55da8f969a693d301daf9378570
--- /dev/null
+++ b/provider/indexer-reference/pom.xml
@@ -0,0 +1,197 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Copyright 2020 Google LLC
+ ~ Copyright 2020 EPAM Systems, Inc
+ ~
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ https://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.opengroup.osdu.indexer</groupId>
+ <artifactId>indexer-service</artifactId>
+ <version>1.0.4-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>indexer-reference</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+ <name>indexer-reference</name>
+ <description>Indexer Service GCP Anthos</description>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.opengroup.osdu.indexer</groupId>
+ <artifactId>indexer-core</artifactId>
+ <version>1.0.6-SNAPSHOT</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.opengroup.osdu</groupId>
+ <artifactId>core-lib-gcp</artifactId>
+ <version>0.1.17</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.opengroup.osdu</groupId>
+ <artifactId>os-core-common</artifactId>
+ <version>0.0.18</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-data-mongodb</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.cloud</groupId>
+ <artifactId>google-cloud-logging</artifactId>
+ <version>1.72.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.apis</groupId>
+ <artifactId>google-api-services-storage</artifactId>
+ <version>v1-rev150-1.25.0</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava-jdk5</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.google.apis</groupId>
+ <artifactId>google-api-services-cloudkms</artifactId>
+ <version>v1-rev81-1.25.0</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-amqp</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch</artifactId>
+ <version>6.6.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-client</artifactId>
+ <version>6.6.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-high-level-client</artifactId>
+ <version>6.6.2</version>
+ </dependency>
+
+ <!-- Test Dependencies -->
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- Test Dependencies -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.12</version>
+ <scope>test</scope>
+ </dependency>
+ <!-- https://mvnrepository.com/artifact/org.powermock/powermock-api-mockito2 -->
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-mockito2</artifactId>
+ <version>2.0.2</version>
+ <scope>test</scope>
+ </dependency>
+ <!-- https://mvnrepository.com/artifact/org.powermock/powermock-module-junit4 -->
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <version>2.0.2</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <version>2.26.0</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>cobertura-maven-plugin</artifactId>
+ <version>2.7</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.auth0</groupId>
+ <artifactId>java-jwt</artifactId>
+ <version>3.8.1</version>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/com.github.stefanbirkner/system-rules -->
+ <dependency>
+ <groupId>com.github.stefanbirkner</groupId>
+ <artifactId>system-rules</artifactId>
+ <version>1.2.0</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.api.grpc</groupId>
+ <artifactId>proto-google-iam-v1</artifactId>
+ <version>0.12.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-actuator-autoconfigure</artifactId>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>repackage</goal>
+ </goals>
+ <configuration>
+ <classifier>spring-boot</classifier>
+ <mainClass>
+ org.opengroup.osdu.indexer.IndexerAnthosApplication
+ </mainClass>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+
+</project>
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/IndexerAnthosApplication.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/IndexerAnthosApplication.java
new file mode 100644
index 0000000000000000000000000000000000000000..a87c9d4577a573359f5d34682be962078b11bc67
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/IndexerAnthosApplication.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer;
+
+import org.opengroup.osdu.core.gcp.multitenancy.TenantFactory;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration;
+import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.FilterType;
+
+@SpringBootApplication(exclude = {MongoAutoConfiguration.class, SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
+@Configuration
+@ComponentScan(value = {"org.opengroup.osdu"}, excludeFilters = {
+ @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = TenantFactory.class)})
+public class IndexerAnthosApplication {
+
+ public static void main(String[] args) {
+ SpringApplication.run(IndexerAnthosApplication.class, args);
+ }
+
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/ServletInitializer.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/ServletInitializer.java
new file mode 100644
index 0000000000000000000000000000000000000000..ccc931148ed7a9826fb8cc7854c85be5f898d796
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/ServletInitializer.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer;
+
+import org.springframework.boot.builder.SpringApplicationBuilder;
+import org.springframework.boot.web.servlet.support.SpringBootServletInitializer;
+
+public class ServletInitializer extends SpringBootServletInitializer {
+ @Override
+ protected SpringApplicationBuilder configure(SpringApplicationBuilder application) {
+ return application.sources(IndexerAnthosApplication.class);
+ }
+}
+
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/AttributesCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/AttributesCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..bdfa51cd624d4afedfa08547fdda56a06c1f7051
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/AttributesCache.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.RedisCache;
+import org.opengroup.osdu.core.common.provider.interfaces.IAttributesCache;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+import java.util.Set;
+
+@Component
+public class AttributesCache implements IAttributesCache<String,Set>, AutoCloseable {
+
+ private RedisCache<String, Set> cache;
+
+ public AttributesCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST,
+ @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT,
+ @Value("${INDEX_CACHE_EXPIRATION}") final String INDEX_CACHE_EXPIRATION) {
+
+ cache = new RedisCache(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT),
+ Integer.parseInt(INDEX_CACHE_EXPIRATION) * 60, String.class, Boolean.class);
+ }
+
+ @Override
+ public void put(String key, Set value) {
+ this.cache.put(key, value);
+ }
+
+ @Override
+ public Set get(String key) {
+ return this.cache.get(key);
+ }
+
+ @Override
+ public void delete(String key) {
+ this.cache.delete(key);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+
+ @Override
+ public void close() {
+ this.cache.close();
+ }
+}
\ No newline at end of file
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..d17e26f7b832658226f9cd32b11426b4e54fe1fc
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/DatastoreCredentialCache.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import com.google.auth.oauth2.AccessToken;
+import org.opengroup.osdu.core.common.cache.RedisCache;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+@Component
+public class DatastoreCredentialCache extends RedisCache<String, AccessToken> {
+
+ // Datastore credentials are only valid for 1hr, release the key 2 minutes before the expiration
+ public DatastoreCredentialCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST, @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT) {
+ super(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), 58 * 60, String.class, AccessToken.class);
+ }
+}
\ No newline at end of file
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..7a5d5f6f32d8a76eb6172b30047d893a718277d4
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/ElasticCredentialsCache.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.RedisCache;
+import org.opengroup.osdu.core.common.model.search.ClusterSettings;
+import org.opengroup.osdu.core.common.provider.interfaces.IElasticCredentialsCache;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+@Component
+public class ElasticCredentialsCache implements IElasticCredentialsCache<String, ClusterSettings>, AutoCloseable {
+
+ private RedisCache<String, ClusterSettings> cache;
+
+ public ElasticCredentialsCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST,
+ @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT,
+ @Value("${ELASTIC_CACHE_EXPIRATION}") final String ELASTIC_CACHE_EXPIRATION) {
+ cache = new RedisCache<>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT),
+ Integer.parseInt(ELASTIC_CACHE_EXPIRATION) * 60, String.class, ClusterSettings.class);
+ }
+
+ @Override
+ public void close() throws Exception {
+ this.cache.close();
+ }
+
+ @Override
+ public void put(String s, ClusterSettings o) {
+ this.cache.put(s,o);
+ }
+
+ @Override
+ public ClusterSettings get(String s) {
+ return this.cache.get(s);
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(s);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/IndexCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/IndexCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..d9b7197fb040234eac6627e926e6e2b88a69fcc3
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/IndexCache.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.RedisCache;
+import org.opengroup.osdu.core.common.provider.interfaces.IIndexCache;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+@Component
+public class IndexCache implements IIndexCache<String, Boolean>, AutoCloseable {
+ private RedisCache<String, Boolean> cache;
+
+ public IndexCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST,
+ @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT,
+ @Value("${INDEX_CACHE_EXPIRATION}") final String INDEX_CACHE_EXPIRATION) {
+ cache = new RedisCache<>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT),
+ Integer.parseInt(INDEX_CACHE_EXPIRATION) * 60, String.class, Boolean.class);
+ }
+
+ @Override
+ public void close() throws Exception {
+ this.cache.close();
+ }
+
+ @Override
+ public void put(String s, Boolean o) {
+ this.cache.put(s, o);
+ }
+
+ @Override
+ public Boolean get(String s) {
+ return this.cache.get(s);
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(s);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/JwtCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/JwtCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..2b442b622e4e3dca4266d93dbe5ea7dea8def433
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/JwtCache.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.RedisCache;
+import org.opengroup.osdu.core.common.model.search.IdToken;
+import org.opengroup.osdu.core.common.provider.interfaces.IJwtCache;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+@Component
+public class JwtCache implements IJwtCache<String, IdToken>, AutoCloseable {
+ RedisCache<String, IdToken> cache;
+
+ // google service account id_token can be requested only for 1 hr
+ private final static int EXPIRED_AFTER = 59;
+
+ public JwtCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST, @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT) {
+ cache = new RedisCache<>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT),
+ EXPIRED_AFTER * 60, String.class, IdToken.class);
+ }
+
+ @Override
+ public void close() throws Exception {
+ this.cache.close();
+ }
+
+ @Override
+ public void put(String s, IdToken o) {
+ this.cache.put(s, o);
+ }
+
+ @Override
+ public IdToken get(String s) {
+ return this.cache.get(s);
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(s);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/KindsCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/KindsCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..d70210a0c570e66dfa97c0e5ac3629f02c8c488c
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/KindsCache.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.RedisCache;
+import org.opengroup.osdu.core.common.provider.interfaces.IKindsCache;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import java.util.Set;
+
+@Component
+public class KindsCache implements IKindsCache<String, Set>, AutoCloseable {
+ private RedisCache<String, Set> cache;
+
+ public KindsCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST,
+ @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT,
+ @Value("${KINDS_CACHE_EXPIRATION}") final String KINDS_CACHE_EXPIRATION,
+ @Value("${KINDS_REDIS_DATABASE}") final String KINDS_REDIS_DATABASE) {
+ cache = new RedisCache<>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT),
+ Integer.parseInt(KINDS_CACHE_EXPIRATION) * 60,
+ Integer.parseInt(KINDS_REDIS_DATABASE), String.class, Set.class);
+ }
+
+ @Override
+ public void close() throws Exception {
+ this.cache.close();
+ }
+
+ @Override
+ public void put(String s, Set o) {
+ this.cache.put(s, o);
+ }
+
+ @Override
+ public Set get(String s) {
+ return this.cache.get(s);
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(s);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/SchemaCache.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/SchemaCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..1fc8b411471a71cd670e9f22d3981c2855a85dcc
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/cache/SchemaCache.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.RedisCache;
+import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+@Component
+public class SchemaCache implements ISchemaCache<String, String>, AutoCloseable {
+ private RedisCache<String, String> cache;
+
+ public SchemaCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST,
+ @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT,
+ @Value("${SCHEMA_CACHE_EXPIRATION}") final String SCHEMA_CACHE_EXPIRATION) {
+ cache = new RedisCache<>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT),
+ Integer.parseInt(SCHEMA_CACHE_EXPIRATION) * 60, String.class, String.class);
+ }
+
+ @Override
+ public void close() throws Exception {
+ this.cache.close();
+ }
+
+ @Override
+ public void put(String s, String o) {
+ this.cache.put(s, o);
+ }
+
+ @Override
+ public String get(String s) {
+ return this.cache.get(s);
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(s);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/EntitlementsClientFactory.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/EntitlementsClientFactory.java
new file mode 100644
index 0000000000000000000000000000000000000000..8460aa5f0da3f175cffb3d54e1b0bdf9cdc2b33d
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/EntitlementsClientFactory.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.di;
+
+import org.opengroup.osdu.core.common.entitlements.EntitlementsAPIConfig;
+import org.opengroup.osdu.core.common.entitlements.EntitlementsFactory;
+import org.opengroup.osdu.core.common.entitlements.IEntitlementsFactory;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.beans.factory.config.AbstractFactoryBean;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+@Component
+@RequestScope
+@Lazy
+public class EntitlementsClientFactory extends AbstractFactoryBean<IEntitlementsFactory> {
+
+ @Value("${AUTHORIZE_API}")
+ private String AUTHORIZE_API;
+
+ @Value("${AUTHORIZE_API_KEY:}")
+ private String AUTHORIZE_API_KEY;
+
+ @Override
+ protected IEntitlementsFactory createInstance() throws Exception {
+
+ return new EntitlementsFactory(EntitlementsAPIConfig
+ .builder()
+ .rootUrl(AUTHORIZE_API)
+ .apiKey(AUTHORIZE_API_KEY)
+ .build());
+ }
+
+ @Override
+ public Class<?> getObjectType() {
+ return IEntitlementsFactory.class;
+ }
+}
\ No newline at end of file
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/RabbitMQFactoryImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/RabbitMQFactoryImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..20236aaf68467a6441e23ba129331eb3f8907550
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/RabbitMQFactoryImpl.java
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.di;
+
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+import java.util.concurrent.TimeoutException;
+import javax.annotation.PostConstruct;
+import org.opengroup.osdu.indexer.messagebus.IMessageFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+@Component
+public class RabbitMQFactoryImpl implements IMessageFactory {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RabbitMQFactoryImpl.class);
+
+ @Value("${mb.rabbitmq.uri}")
+ private String uri;
+
+ private Channel channel;
+
+ @PostConstruct
+ private void init() {
+ ConnectionFactory factory = new ConnectionFactory();
+ try {
+ LOG.debug("RabbitMQ Channel " + uri);
+ factory.setUri(uri);
+ factory.setAutomaticRecoveryEnabled(true);
+ Connection conn = factory.newConnection();
+ this.channel = conn.createChannel();
+ LOG.debug("RabbitMQ Channel was created.");
+ for (String queue : Arrays.asList(DEFAULT_QUEUE_NAME, INDEXER_QUEUE_NAME, LEGAL_QUEUE_NAME)) {
+ channel.queueDeclare(queue, true, false, false, null);
+ LOG.debug("Queue [" + queue + "] was declared.");
+ }
+ } catch (KeyManagementException | NoSuchAlgorithmException | URISyntaxException | IOException | TimeoutException e) {
+ LOG.error(e.getMessage(), e);
+ }
+
+ }
+
+ @Override
+ public void sendMessage(String msg) {
+ this.sendMessage("records", msg);
+ }
+
+ @Override
+ public void sendMessage(String queueName, String msg) {
+ String queueNameWithPrefix = queueName;
+ try {
+ channel.basicPublish("", queueNameWithPrefix, null, msg.getBytes());
+ LOG.info(" [x] Sent '" + msg + "' to queue [" + queueNameWithPrefix + "]");
+ } catch (IOException e) {
+ LOG.error("Unable to publish message to [" + queueNameWithPrefix + "]");
+ LOG.error(e.getMessage(), e);
+ }
+ }
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..8aad1a1a1edadc6cb355859a4e21aaea0520748e
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/di/TenantFactoryImpl.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.di;
+
+
+import com.google.gson.Gson;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import org.bson.Document;
+import org.opengroup.osdu.core.common.cache.ICache;
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
+import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
+import org.opengroup.osdu.indexer.persistence.MongoDdmsClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Primary;
+import org.springframework.stereotype.Component;
+
+@Primary
+@Component
+public class TenantFactoryImpl implements ITenantFactory {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TenantFactoryImpl.class);
+
+ public static final String MAIN_DATABASE = "main";
+ public static final String TENANT_INFO = "tenantinfo";
+
+ @Autowired
+ private MongoDdmsClient mongoClient;
+
+ private Map<String, TenantInfo> tenants;
+
+ public boolean exists(String tenantName) {
+ if (this.tenants == null) {
+ initTenants();
+ }
+ return this.tenants.containsKey(tenantName);
+ }
+
+ public TenantInfo getTenantInfo(String tenantName) {
+ if (this.tenants == null) {
+ initTenants();
+ }
+ return this.tenants.get(tenantName);
+ }
+
+ public Collection<TenantInfo> listTenantInfo() {
+ if (this.tenants == null) {
+ initTenants();
+ }
+ return this.tenants.values();
+ }
+
+ public <V> ICache<String, V> createCache(String tenantName, String host, int port,
+ int expireTimeSeconds, Class<V> classOfV) {
+ return null;
+ }
+
+ public void flushCache() {
+ }
+
+ private void initTenants() {
+ this.tenants = new HashMap<>();
+ MongoCollection<Document> mongoCollection = mongoClient
+ .getMongoCollection(MAIN_DATABASE, TENANT_INFO);
+ FindIterable<Document> results = mongoCollection.find();
+ if (Objects.isNull(results) && Objects.isNull(results.first())) {
+ LOG.error(String.format("Collection \'%s\' is empty.", results));
+ }
+ for (Document document : results) {
+ TenantInfo tenantInfo = new Gson().fromJson(document.toJson(),TenantInfo.class);
+ this.tenants.put(tenantInfo.getName(), tenantInfo);
+ }
+ }
+
+}
+
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/kms/KmsClient.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/kms/KmsClient.java
new file mode 100644
index 0000000000000000000000000000000000000000..2adc6336b182750c215b84de565f37e7ebb8da74
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/kms/KmsClient.java
@@ -0,0 +1,105 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.kms;
+
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.http.javanet.NetHttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.cloudkms.v1.CloudKMS;
+import com.google.api.services.cloudkms.v1.CloudKMSScopes;
+import com.google.api.services.cloudkms.v1.model.DecryptRequest;
+import com.google.api.services.cloudkms.v1.model.DecryptResponse;
+import com.google.api.services.cloudkms.v1.model.EncryptRequest;
+import com.google.api.services.cloudkms.v1.model.EncryptResponse;
+import org.opengroup.osdu.core.common.provider.interfaces.IKmsClient;
+import org.opengroup.osdu.core.common.search.Preconditions;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+@Component
+@RequestScope
+public class KmsClient implements IKmsClient {
+
+ @Value("${GOOGLE_CLOUD_PROJECT}")
+ private String GOOGLE_CLOUD_PROJECT;
+
+ @Value("${KMS_KEY}")
+ private String KMS_KEY;
+
+ @Value("${KEY_RING}")
+ private String KEY_RING;
+
+ private static final String KEY_NAME = "projects/%s/locations/global/keyRings/%s/cryptoKeys/%s";
+
+ /**
+ * Encrypts the given plaintext using the specified crypto key.
+ * Google KMS automatically uses the new primary key version to encrypt data, so this could be directly used for key rotation
+ */
+ public String encryptString(String textToBeEncrypted) throws IOException {
+ Preconditions.checkNotNullOrEmpty(textToBeEncrypted, "textToBeEncrypted cannot be null");
+
+ byte[] plaintext = textToBeEncrypted.getBytes(StandardCharsets.UTF_8);
+ String resourceName = String.format(KEY_NAME, GOOGLE_CLOUD_PROJECT, KEY_RING, KMS_KEY);
+ CloudKMS kms = createAuthorizedClient();
+ EncryptRequest request = new EncryptRequest().encodePlaintext(plaintext);
+ EncryptResponse response = kms.projects().locations().keyRings().cryptoKeys()
+ .encrypt(resourceName, request)
+ .execute();
+ return response.getCiphertext();
+ }
+
+ /**
+ * Decrypts the provided ciphertext with the specified crypto key.
+ * Google KMS automatically uses the correct key version to decrypt data, as long as the key version is not disabled
+ */
+ public String decryptString(String textToBeDecrypted) throws IOException {
+ Preconditions.checkNotNullOrEmpty(textToBeDecrypted, "textToBeDecrypted cannot be null");
+
+ CloudKMS kms = createAuthorizedClient();
+ String cryptoKeyName = String.format(KEY_NAME, GOOGLE_CLOUD_PROJECT, KEY_RING, KMS_KEY);
+ DecryptRequest request = new DecryptRequest().setCiphertext(textToBeDecrypted);
+ DecryptResponse response = kms.projects().locations().keyRings().cryptoKeys()
+ .decrypt(cryptoKeyName, request)
+ .execute();
+ return new String(response.decodePlaintext(), StandardCharsets.UTF_8).trim();
+ }
+
+ /**
+ * Creates an authorized CloudKMS client service using Application Default Credentials.
+ *
+ * @return an authorized CloudKMS client
+ * @throws IOException if there's an error getting the default credentials.
+ */
+ private CloudKMS createAuthorizedClient() throws IOException {
+ HttpTransport transport = new NetHttpTransport();
+ JsonFactory jsonFactory = new JacksonFactory();
+ GoogleCredential credential = GoogleCredential.getApplicationDefault();
+ if (credential.createScopedRequired()) {
+ credential = credential.createScoped(CloudKMSScopes.all());
+ }
+ return new CloudKMS.Builder(transport, jsonFactory, credential)
+ .setApplicationName("CloudKMS snippets")
+ .build();
+ }
+}
\ No newline at end of file
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/messagebus/IMessageFactory.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/messagebus/IMessageFactory.java
new file mode 100644
index 0000000000000000000000000000000000000000..1679a6d63d3b005eb5a807f881299aa11cd377c3
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/messagebus/IMessageFactory.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.messagebus;
+
+public interface IMessageFactory {
+ String DEFAULT_QUEUE_NAME = "records";
+ String LEGAL_QUEUE_NAME = "legal";
+ String INDEXER_QUEUE_NAME = "indexer";
+
+ void sendMessage(String msg);
+
+ void sendMessage(String queueName, String msg);
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/middleware/IndexFilter.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/middleware/IndexFilter.java
new file mode 100644
index 0000000000000000000000000000000000000000..95bbd9be85eb877f5b5025ca2056c80eb14111da
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/middleware/IndexFilter.java
@@ -0,0 +1,121 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.middleware;
+
+import com.google.common.base.Strings;
+import lombok.extern.java.Log;
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.http.ResponseHeaders;
+import org.opengroup.osdu.core.common.model.search.DeploymentEnvironment;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.http.HttpMethod;
+import org.springframework.stereotype.Component;
+
+import javax.inject.Inject;
+import javax.servlet.*;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+@Log
+@Component
+public class IndexFilter implements Filter {
+
+ @Inject
+ private DpsHeaders dpsHeaders;
+
+ @Inject
+ private IRequestInfo requestInfo;
+
+ @Value("${DEPLOYMENT_ENVIRONMENT}")
+ private String DEPLOYMENT_ENVIRONMENT;
+
+ private FilterConfig filterConfig;
+
+ private static final String PATH_SWAGGER = "/swagger.json";
+ private static final String PATH_TASK_HANDLERS = "task-handlers";
+ private static final String PATH_CRON_HANDLERS = "cron-handlers";
+
+ @Override
+ public void init(FilterConfig filterConfig) throws ServletException {
+ }
+
+ @Override
+ public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain)
+ throws IOException, ServletException {
+ HttpServletRequest httpRequest = (HttpServletRequest) servletRequest;
+ String uri = httpRequest.getRequestURI().toLowerCase();
+
+ if (httpRequest.getMethod().equalsIgnoreCase(HttpMethod.POST.name()) && uri.contains(PATH_TASK_HANDLERS)) {
+ if (DeploymentEnvironment.valueOf(DEPLOYMENT_ENVIRONMENT) != DeploymentEnvironment.LOCAL) {
+ checkWorkerApiAccess(requestInfo);
+ }
+ }
+
+ if (httpRequest.getMethod().equalsIgnoreCase(HttpMethod.GET.name()) && uri.contains(PATH_CRON_HANDLERS)) {
+ checkWorkerApiAccess(requestInfo);
+ }
+
+// if (!httpRequest.isSecure()) {
+// throw new AppException(302, "Redirect", "HTTP is not supported. Use HTTPS.");
+// }
+
+ filterChain.doFilter(servletRequest, servletResponse);
+
+ HttpServletResponse httpResponse = (HttpServletResponse) servletResponse;
+ Map<String, List<Object>> standardHeaders = ResponseHeaders.STANDARD_RESPONSE_HEADERS;
+ for (Map.Entry<String, List<Object>> header : standardHeaders.entrySet()) {
+ httpResponse.addHeader(header.getKey(), header.getValue().toString());
+ }
+ if (httpResponse.getHeader(DpsHeaders.CORRELATION_ID) == null) {
+ httpResponse.addHeader(DpsHeaders.CORRELATION_ID, dpsHeaders.getCorrelationId());
+ }
+
+ }
+
+ @Override
+ public void destroy() {
+ }
+
+ private void checkWorkerApiAccess(IRequestInfo requestInfo) {
+// if (requestInfo.isTaskQueueRequest()) return;
+// throw AppException.createForbidden("invalid user agent, AppEngine Task Queue only");
+ }
+
+ private List<String> validateAccountId(DpsHeaders requestHeaders) {
+ String accountHeader = requestHeaders.getPartitionIdWithFallbackToAccountId();
+ String debuggingInfo = String.format("%s:%s", DpsHeaders.DATA_PARTITION_ID, accountHeader);
+
+ if (Strings.isNullOrEmpty(accountHeader)) {
+ throw new AppException(HttpStatus.SC_BAD_REQUEST, "Bad request", "invalid or empty data partition", debuggingInfo);
+ }
+
+ List<String> dataPartitions = Arrays.asList(accountHeader.trim().split("\\s*,\\s*"));
+ if (dataPartitions.isEmpty() || dataPartitions.size() > 1) {
+ throw new AppException(HttpStatus.SC_BAD_REQUEST, "Bad request", "invalid or empty data partition", debuggingInfo);
+ }
+ return dataPartitions;
+ }
+
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreCredential.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreCredential.java
new file mode 100644
index 0000000000000000000000000000000000000000..c3285bd78e05d172f3923a667fa5593be86fdb5c
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreCredential.java
@@ -0,0 +1,119 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.persistence;
+
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.iam.v1.Iam;
+import com.google.api.services.iam.v1.Iam.Projects.ServiceAccounts.SignJwt;
+import com.google.api.services.iam.v1.IamScopes;
+import com.google.api.services.iam.v1.model.SignJwtRequest;
+import com.google.api.services.iam.v1.model.SignJwtResponse;
+import com.google.auth.oauth2.AccessToken;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.gson.JsonObject;
+import java.util.Collections;
+import java.util.Date;
+import org.apache.commons.lang3.time.DateUtils;
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
+import org.opengroup.osdu.core.common.util.Crc32c;
+import org.opengroup.osdu.indexer.cache.DatastoreCredentialCache;
+
+public class DatastoreCredential extends GoogleCredentials {
+
+ private static final long serialVersionUID = 8344377091688956815L;
+ private static final JsonFactory JSON_FACTORY = new JacksonFactory();
+ private Iam iam;
+
+ private final TenantInfo tenant;
+ private final DatastoreCredentialCache cache;
+
+ protected DatastoreCredential(TenantInfo tenant, DatastoreCredentialCache cache) {
+ this.tenant = tenant;
+ this.cache = cache;
+ }
+
+ @Override
+ public AccessToken refreshAccessToken() {
+
+ String cacheKey = this.getCacheKey();
+
+ AccessToken accessToken = this.cache.get(cacheKey);
+
+ if (accessToken != null) {
+ return accessToken;
+ }
+
+ try {
+ SignJwtRequest signJwtRequest = new SignJwtRequest();
+ signJwtRequest.setPayload(this.getPayload());
+
+ String serviceAccountName = String.format("projects/-/serviceAccounts/%s", this.tenant.getServiceAccount());
+
+ SignJwt signJwt = this.getIam().projects().serviceAccounts().signJwt(serviceAccountName, signJwtRequest);
+
+ SignJwtResponse signJwtResponse = signJwt.execute();
+ String signedJwt = signJwtResponse.getSignedJwt();
+
+ accessToken = new AccessToken(signedJwt, DateUtils.addSeconds(new Date(), 3600));
+
+ this.cache.put(cacheKey, accessToken);
+
+ return accessToken;
+ } catch (Exception e) {
+ throw new RuntimeException("Error creating datastore credential", e);
+ }
+ }
+
+ private String getPayload() {
+ JsonObject payload = new JsonObject();
+ payload.addProperty("iss", this.tenant.getServiceAccount());
+ payload.addProperty("sub", this.tenant.getServiceAccount());
+ payload.addProperty("aud", "https://datastore.googleapis.com/google.datastore.v1.Datastore");
+ payload.addProperty("iat", System.currentTimeMillis() / 1000);
+
+ return payload.toString();
+ }
+
+ protected void setIam(Iam iam) {
+ this.iam = iam;
+ }
+
+ private Iam getIam() throws Exception {
+ if (this.iam == null) {
+
+ GoogleCredential credential = GoogleCredential.getApplicationDefault();
+ if (credential.createScopedRequired()) {
+ credential = credential.createScoped(Collections.singletonList(IamScopes.CLOUD_PLATFORM));
+ }
+
+ Iam.Builder builder = new Iam.Builder(GoogleNetHttpTransport.newTrustedTransport(), JSON_FACTORY,
+ credential).setApplicationName("Search Service");
+
+ this.iam = builder.build();
+ }
+ return this.iam;
+ }
+
+
+ private String getCacheKey() {
+ return Crc32c.hashToBase64EncodedString(String.format("datastoreCredential:%s", this.tenant.getName()));
+ }
+}
\ No newline at end of file
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java
new file mode 100644
index 0000000000000000000000000000000000000000..db90ee7efb2dfee6fb3995d09b0fe158a03ad9e1
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/DatastoreFactory.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.persistence;
+
+import com.google.api.gax.retrying.RetrySettings;
+import com.google.cloud.TransportOptions;
+import com.google.cloud.datastore.Datastore;
+import com.google.cloud.datastore.DatastoreOptions;
+import com.google.cloud.http.HttpTransportOptions;
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
+import org.opengroup.osdu.indexer.cache.DatastoreCredentialCache;
+import org.springframework.stereotype.Component;
+import org.threeten.bp.Duration;
+
+import javax.inject.Inject;
+import java.util.HashMap;
+import java.util.Map;
+
+@Component
+public class DatastoreFactory {
+
+ @Inject
+ private DatastoreCredentialCache cache;
+
+ private static Map<String, Datastore> DATASTORE_CLIENTS = new HashMap<>();
+
+ private static final RetrySettings RETRY_SETTINGS = RetrySettings.newBuilder()
+ .setMaxAttempts(6)
+ .setInitialRetryDelay(Duration.ofSeconds(10))
+ .setMaxRetryDelay(Duration.ofSeconds(32))
+ .setRetryDelayMultiplier(2.0)
+ .setTotalTimeout(Duration.ofSeconds(50))
+ .setInitialRpcTimeout(Duration.ofSeconds(50))
+ .setRpcTimeoutMultiplier(1.0)
+ .setMaxRpcTimeout(Duration.ofSeconds(50))
+ .build();
+
+ private static final TransportOptions TRANSPORT_OPTIONS = HttpTransportOptions.newBuilder()
+ .setReadTimeout(30000)
+ .build();
+
+ public Datastore getDatastoreInstance(TenantInfo tenantInfo) {
+ if (DATASTORE_CLIENTS.get(tenantInfo.getName()) == null) {
+ Datastore googleDatastore = DatastoreOptions.newBuilder()
+ .setCredentials(new DatastoreCredential(tenantInfo, this.cache))
+ .setRetrySettings(RETRY_SETTINGS)
+ .setTransportOptions(TRANSPORT_OPTIONS)
+ .setNamespace(tenantInfo.getName())
+ .setProjectId(tenantInfo.getProjectId())
+ .build().getService();
+ DATASTORE_CLIENTS.put(tenantInfo.getName(), googleDatastore);
+ }
+ return DATASTORE_CLIENTS.get(tenantInfo.getName());
+ }
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryMongoDB.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryMongoDB.java
new file mode 100644
index 0000000000000000000000000000000000000000..a3ab89c5d9dc1824af87ad823e547befbc6c4fa8
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/ElasticRepositoryMongoDB.java
@@ -0,0 +1,88 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.persistence;
+
+import com.google.api.client.googleapis.json.GoogleJsonResponseException;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import java.util.Objects;
+import org.apache.http.HttpStatus;
+import org.bson.Document;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.search.ClusterSettings;
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
+import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository;
+import org.opengroup.osdu.core.common.provider.interfaces.IKmsClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class ElasticRepositoryMongoDB implements IElasticRepository {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ElasticRepositoryMongoDB.class);
+
+ private static final String SCHEMA_DATABASE = "local";
+ private static final String SEARCH_SETTINGS = "SearchSettings";
+ private static final String HOST = "host";
+ private static final String PORT = "port";
+ private static final String XPACK_RESTCLIENT_CONFIGURATION = "configuration";
+
+ @Autowired
+ private MongoDdmsClient mongoClient;
+
+ @Autowired
+ private IKmsClient kmsClient;
+
+ @Override
+ public ClusterSettings getElasticClusterSettings(TenantInfo tenantInfo) {
+ MongoCollection<Document> mongoCollection = mongoClient
+ .getMongoCollection(SCHEMA_DATABASE, SEARCH_SETTINGS);
+
+ FindIterable<Document> results = mongoCollection.find();
+
+ if (Objects.isNull(results) && Objects.isNull(results.first())) {
+ LOG.error(String.format("Collection \'%s\' is empty.", results));
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Cluster setting fetch error",
+ "An error has occurred fetching cluster settings from the database.");
+ }
+
+ Document document = results.first();
+
+ String encryptedConfiguration = document.get(XPACK_RESTCLIENT_CONFIGURATION).toString();
+ String encryptedHost = document.get(HOST).toString();
+ String encryptedPort = document.get(PORT).toString();
+
+ try {
+
+ String host = encryptedHost;//this.kmsClient.decryptString(encryptedHost);
+ String portString = encryptedPort;//this.kmsClient.decryptString(encryptedPort);
+ String usernameAndPassword = encryptedConfiguration;//this.kmsClient.decryptString(encryptedConfiguration);
+
+ int port = Integer.parseInt(portString);
+ ClusterSettings clusterSettings = new ClusterSettings(host, port, usernameAndPassword);
+ clusterSettings.setHttps(false);
+ return clusterSettings;
+
+ } catch (Exception e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Cluster setting fetch error",
+ "An error has occurred fetching cluster settings from the database.", e);
+ }
+ }
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/MongoDdmsClient.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/MongoDdmsClient.java
new file mode 100644
index 0000000000000000000000000000000000000000..43b3fd5b1b519006aa781af227f594c81f2d459a
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/persistence/MongoDdmsClient.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.persistence;
+
+import com.mongodb.client.MongoCollection;
+import org.bson.Document;
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
+import org.opengroup.osdu.indexer.util.MongoClientHandler;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class MongoDdmsClient {
+
+ @Autowired
+ private MongoClientHandler mongoClientHandler;
+
+ @Autowired
+ private TenantInfo tenantInfo;
+
+ public MongoCollection<Document> getMongoCollection(String dbName, String collectionName) {
+ return mongoClientHandler.getMongoClient().getDatabase(dbName)
+ .getCollection(collectionName);
+ }
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..d2a4d74cd3ee5969714eb84317bcef27813bcb75
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/publish/PublisherImpl.java
@@ -0,0 +1,103 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.publish;
+
+
+import com.google.common.reflect.TypeToken;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonElement;
+import java.lang.reflect.Type;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.elasticsearch.common.Strings;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.indexer.JobStatus;
+import org.opengroup.osdu.core.common.model.indexer.RecordStatus;
+import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
+import org.opengroup.osdu.indexer.messagebus.IMessageFactory;
+import org.opengroup.osdu.indexer.provider.interfaces.IPublisher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+@Component
+@RequestScope
+public class PublisherImpl implements IPublisher {
+
+ private static final Logger LOG = LoggerFactory.getLogger(PublisherImpl.class);
+
+ @Autowired
+ IMessageFactory mq;
+
+
+ @Override
+ public void publishStatusChangedTagsToTopic(DpsHeaders headers, JobStatus indexerBatchStatus) throws Exception {
+
+ String tenant = headers.getPartitionId();
+ if (Strings.isNullOrEmpty(tenant)) {
+ tenant = headers.getAccountId();
+ }
+
+ Map<String, String> message = new HashMap<>();
+ message.put(tenant, headers.getPartitionIdWithFallbackToAccountId());
+ headers.addCorrelationIdIfMissing();
+ message.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
+
+ RecordChangedMessages recordChangedMessages = getRecordChangedMessage(headers, indexerBatchStatus);
+ message.put("data", recordChangedMessages.toString());
+
+ try {
+ LOG.info("Indexer publishes message " + headers.getCorrelationId());
+ mq.sendMessage(IMessageFactory.INDEXER_QUEUE_NAME, message.toString());
+ } catch (Exception e) {
+ LOG.error(e.getMessage(), e);
+ }
+ }
+
+ private RecordChangedMessages getRecordChangedMessage(DpsHeaders headers, JobStatus indexerBatchStatus) {
+
+ Gson gson = new GsonBuilder().create();
+ Map<String, String> attributesMap = new HashMap<>();
+ Type listType = new TypeToken<List<RecordStatus>>() {
+ }.getType();
+
+ JsonElement statusChangedTagsJson = gson.toJsonTree(indexerBatchStatus.getStatusesList(), listType);
+ String statusChangedTagsData = (statusChangedTagsJson.toString());
+
+ String tenant = headers.getPartitionId();
+ // This code it to provide backward compatibility to slb-account-id
+ if (!Strings.isNullOrEmpty(tenant)) {
+ attributesMap.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
+ } else {
+ attributesMap.put(DpsHeaders.ACCOUNT_ID, headers.getPartitionIdWithFallbackToAccountId());
+ }
+ headers.addCorrelationIdIfMissing();
+ attributesMap.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
+
+ RecordChangedMessages recordChangedMessages = new RecordChangedMessages();
+ // statusChangedTagsData is not ByteString but String
+ recordChangedMessages.setData(statusChangedTagsData);
+ recordChangedMessages.setAttributes(attributesMap);
+
+ return recordChangedMessages;
+ }
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java
new file mode 100644
index 0000000000000000000000000000000000000000..7927d6f3b55656168683b497d67597d446f0246b
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/security/GSuiteSecurityConfig.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.security;
+
+import org.springframework.context.annotation.Configuration;
+import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity;
+import org.springframework.security.config.annotation.web.builders.HttpSecurity;
+import org.springframework.security.config.annotation.web.builders.WebSecurity;
+import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
+import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter;
+
+@Configuration
+@EnableWebSecurity
+@EnableGlobalMethodSecurity(prePostEnabled = true)
+public class GSuiteSecurityConfig extends WebSecurityConfigurerAdapter {
+
+ @Override
+ protected void configure(HttpSecurity http) throws Exception {
+
+ http.httpBasic().disable()
+ .csrf().disable(); //disable default authN. AuthN handled by endpoints proxy
+
+ }
+
+ @Override
+ public void configure(WebSecurity web) throws Exception {
+ web.ignoring().antMatchers("/api-docs")
+ .antMatchers("/swagger");
+ }
+
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..0b558b08ccb00f8666d4e98ecf4961bef37d69c9
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
@@ -0,0 +1,525 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.service;
+
+import com.google.gson.Gson;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+import javax.inject.Inject;
+import org.apache.http.HttpStatus;
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.action.bulk.BulkItemResponse;
+import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.rest.RestStatus;
+import org.opengroup.osdu.core.common.Constants;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.entitlements.Acl;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.http.RequestStatus;
+import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
+import org.opengroup.osdu.core.common.model.indexer.IndexingStatus;
+import org.opengroup.osdu.core.common.model.indexer.JobStatus;
+import org.opengroup.osdu.core.common.model.indexer.OperationType;
+import org.opengroup.osdu.core.common.model.indexer.RecordIndexerPayload;
+import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
+import org.opengroup.osdu.core.common.model.indexer.RecordStatus;
+import org.opengroup.osdu.core.common.model.indexer.Records;
+import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
+import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
+import org.opengroup.osdu.core.common.search.IndicesService;
+import org.opengroup.osdu.indexer.logging.AuditLogger;
+import org.opengroup.osdu.indexer.provider.interfaces.IPublisher;
+import org.opengroup.osdu.indexer.util.ElasticClientHandler;
+import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
+import org.springframework.context.annotation.Primary;
+import org.springframework.stereotype.Service;
+
+@Service
+@Primary
+public class IndexerServiceImpl implements IndexerService {
+
+ private static final TimeValue BULK_REQUEST_TIMEOUT = TimeValue.timeValueMinutes(1);
+
+ private static final List<RestStatus> RETRY_ELASTIC_EXCEPTION = new ArrayList<>(Arrays.asList(RestStatus.TOO_MANY_REQUESTS, RestStatus.BAD_GATEWAY, RestStatus.SERVICE_UNAVAILABLE));
+
+ private final Gson gson = new Gson();
+
+ @Inject
+ private JaxRsDpsLog jaxRsDpsLog;
+ @Inject
+ private AuditLogger auditLogger;
+ @Inject
+ private StorageService storageService;
+ @Inject
+ private IndexSchemaService schemaService;
+ @Inject
+ private IndicesService indicesService;
+ @Inject
+ private IndexerMappingService mappingService;
+ @Inject
+ private IPublisher progressPublisher;
+ @Inject
+ private ElasticClientHandler elasticClientHandler;
+ @Inject
+ private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
+ @Inject
+ private ElasticIndexNameResolver elasticIndexNameResolver;
+ @Inject
+ private StorageIndexerPayloadMapper storageIndexerPayloadMapper;
+ @Inject
+ private IRequestInfo requestInfo;
+ @Inject
+ private JobStatus jobStatus;
+
+ private DpsHeaders headers;
+
+ @Override
+ public JobStatus processRecordChangedMessages(RecordChangedMessages message, List<RecordInfo> recordInfos) throws Exception {
+
+ // this should not happen
+ if (recordInfos.size() == 0) return null;
+
+ String errorMessage = "";
+ List<String> retryRecordIds = new LinkedList<>();
+
+ // get auth header with service account Authorization
+ this.headers = this.requestInfo.getHeaders();
+ String authorization = message.getAttributes().get("authorization");
+ headers.put("authorization", authorization);
+ // initialize status for all messages.
+ this.jobStatus.initialize(recordInfos);
+
+ try {
+ // get upsert records
+ Map<String, Map<String, OperationType>> upsertRecordMap = RecordInfo.getUpsertRecordIds(recordInfos);
+ if (upsertRecordMap != null && !upsertRecordMap.isEmpty()) {
+ List<String> upsertFailureRecordIds = processUpsertRecords(upsertRecordMap);
+ retryRecordIds.addAll(upsertFailureRecordIds);
+ }
+
+ // get delete records
+ Map<String, List<String>> deleteRecordMap = RecordInfo.getDeleteRecordIds(recordInfos);
+ if (deleteRecordMap != null && !deleteRecordMap.isEmpty()) {
+ List<String> deleteFailureRecordIds = processDeleteRecords(deleteRecordMap);
+ retryRecordIds.addAll(deleteFailureRecordIds);
+ }
+
+ // process schema change messages
+ Map<String, OperationType> schemaMsgs = RecordInfo.getSchemaMsgs(recordInfos);
+ if (schemaMsgs != null && !schemaMsgs.isEmpty()) {
+ this.schemaService.processSchemaMessages(schemaMsgs);
+ }
+
+ // process failed records
+ if (retryRecordIds.size() > 0) {
+ retryAndEnqueueFailedRecords(recordInfos, retryRecordIds, message);
+ }
+ } catch (IOException e) {
+ errorMessage = e.getMessage();
+ throw new AppException(HttpStatus.SC_GATEWAY_TIMEOUT, "Internal communication failure", errorMessage, e);
+ } catch (AppException e) {
+ errorMessage = e.getMessage();
+ throw e;
+ } catch (Exception e) {
+ errorMessage = "error indexing records";
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Unknown error", "An unknown error has occurred.", e);
+ } finally {
+ this.jobStatus.finalizeRecordStatus(errorMessage);
+ this.updateAuditLog();
+ this.progressPublisher.publishStatusChangedTagsToTopic(this.headers, this.jobStatus);
+ }
+
+ return jobStatus;
+ }
+
+ @Override
+ public void processSchemaMessages(List<RecordInfo> recordInfos) throws IOException {
+ Map<String, OperationType> schemaMsgs = RecordInfo.getSchemaMsgs(recordInfos);
+ if (schemaMsgs != null && !schemaMsgs.isEmpty()) {
+ try (RestHighLevelClient restClient = elasticClientHandler.createRestClient()) {
+ schemaMsgs.entrySet().forEach(msg -> {
+ try {
+ processSchemaEvents(restClient, msg);
+ } catch (IOException | ElasticsearchStatusException e) {
+ throw new AppException(org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR.value(), "unable to process schema delete", e.getMessage());
+ }
+ });
+ }
+ }
+ }
+
+ private void processSchemaEvents(RestHighLevelClient restClient,
+ Map.Entry<String, OperationType> msg) throws IOException, ElasticsearchStatusException {
+ String kind = msg.getKey();
+ String index = elasticIndexNameResolver.getIndexNameFromKind(kind);
+
+ boolean indexExist = indicesService.isIndexExist(restClient, index);
+ if (indexExist && msg.getValue() == OperationType.purge_schema) {
+ indicesService.deleteIndex(restClient, index);
+ }
+ }
+
+ private List<String> processUpsertRecords(Map<String, Map<String, OperationType>> upsertRecordMap) throws Exception {
+ // get schema for kind
+ Map<String, IndexSchema> schemas = this.getSchema(upsertRecordMap);
+
+ if (schemas.isEmpty()) return new LinkedList<>();
+
+ // get recordIds with valid upsert index-status
+ List<String> recordIds = this.jobStatus.getIdsByValidUpsertIndexingStatus();
+
+ if (recordIds.isEmpty()) return new LinkedList<>();
+
+ // get records via storage api
+ Records storageRecords = this.storageService.getStorageRecords(recordIds);
+ List<String> failedOrRetryRecordIds = new LinkedList<>(storageRecords.getMissingRetryRecords());
+
+ // map storage records to indexer payload
+ RecordIndexerPayload recordIndexerPayload = this.getIndexerPayload(upsertRecordMap, schemas, storageRecords);
+
+ jaxRsDpsLog.info(String.format("records change messages received : %s | valid storage bulk records: %s | valid index payload: %s", recordIds.size(), storageRecords.getRecords().size(), recordIndexerPayload.getRecords().size()));
+
+ // index records
+ failedOrRetryRecordIds.addAll(processElasticMappingAndUpsertRecords(recordIndexerPayload));
+
+ return failedOrRetryRecordIds;
+ }
+
+ private Map<String, IndexSchema> getSchema(Map<String, Map<String, OperationType>> upsertRecordMap) {
+
+ Map<String, IndexSchema> schemas = new HashMap<>();
+
+ try {
+ for (Map.Entry<String, Map<String, OperationType>> entry : upsertRecordMap.entrySet()) {
+
+ String kind = entry.getKey();
+ IndexSchema schemaObj = this.schemaService.getIndexerInputSchema(kind, false);
+ if (schemaObj.isDataSchemaMissing()) {
+ this.jobStatus.addOrUpdateRecordStatus(entry.getValue().keySet(), IndexingStatus.WARN, HttpStatus.SC_NOT_FOUND, "schema not found", String.format("schema not found | kind: %s", kind));
+ }
+
+ schemas.put(kind, schemaObj);
+ }
+ } catch (AppException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Get schema error", "An error has occurred while getting schema", e);
+ }
+
+ return schemas;
+ }
+
+ private RecordIndexerPayload getIndexerPayload(Map<String, Map<String, OperationType>> upsertRecordMap, Map<String, IndexSchema> kindSchemaMap, Records records) {
+ List<Records.Entity> storageValidRecords = records.getRecords();
+ List<RecordIndexerPayload.Record> indexerPayload = new ArrayList<>();
+ List<IndexSchema> schemas = new ArrayList<>();
+
+ for (Records.Entity storageRecord : storageValidRecords) {
+
+ Map<String, OperationType> idOperationMap = upsertRecordMap.get(storageRecord.getKind());
+
+ // skip if storage returned record with same id but different kind
+ if (idOperationMap == null) {
+ String message = String.format("storage service returned incorrect record | requested kind: %s | received kind: %s", this.jobStatus.getRecordKindById(storageRecord.getId()), storageRecord.getKind());
+ this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.SKIP, RequestStatus.STORAGE_CONFLICT, message, String.format("%s | record-id: %s", message, storageRecord.getId()));
+ continue;
+ }
+
+ IndexSchema schema = kindSchemaMap.get(storageRecord.getKind());
+ schemas.add(schema);
+
+ // skip indexing of records if data block is empty
+ RecordIndexerPayload.Record document = prepareIndexerPayload(schema, storageRecord, idOperationMap);
+ if (document != null) {
+ indexerPayload.add(document);
+ }
+ }
+
+ // this should only happen if storage service returned WRONG records with kind for all the records in the messages
+ if (indexerPayload.isEmpty()) {
+ throw new AppException(RequestStatus.STORAGE_CONFLICT, "Indexer error", "upsert record failed, storage service returned incorrect records");
+ }
+
+ return RecordIndexerPayload.builder().records(indexerPayload).schemas(schemas).build();
+ }
+
+ private RecordIndexerPayload.Record prepareIndexerPayload(IndexSchema schemaObj, Records.Entity storageRecord, Map<String, OperationType> idToOperationMap) {
+
+ RecordIndexerPayload.Record document = null;
+
+ try {
+ Map<String, Object> storageRecordData = storageRecord.getData();
+ document = new RecordIndexerPayload.Record();
+ if (storageRecordData == null || storageRecordData.isEmpty()) {
+ String message = "empty or null data block found in the storage record";
+ this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.WARN, HttpStatus.SC_NOT_FOUND, message, String.format("record-id: %s | %s", storageRecord.getId(), message));
+ } else if (schemaObj.isDataSchemaMissing()) {
+ document.setSchemaMissing(true);
+ } else {
+ Map<String, Object> dataMap = this.storageIndexerPayloadMapper.mapDataPayload(schemaObj, storageRecordData, storageRecord.getId());
+ if (dataMap.isEmpty()) {
+ document.setMappingMismatch(true);
+ String message = String.format("complete schema mismatch: none of the data attribute can be mapped | data: %s", storageRecordData);
+ this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.WARN, HttpStatus.SC_NOT_FOUND, message, String.format("record-id: %s | %s", storageRecord.getId(), message));
+ }
+ document.setData(dataMap);
+ }
+ } catch (AppException e) {
+ this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL, HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getMessage());
+ jaxRsDpsLog.warning(String.format("record-id: %s | %s", storageRecord.getId(), e.getMessage()), e);
+ } catch (Exception e) {
+ this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL, HttpStatus.SC_INTERNAL_SERVER_ERROR, String.format("error parsing records against schema, error-message: %s", e.getMessage()));
+ jaxRsDpsLog.error(String.format("record-id: %s | error parsing records against schema, error-message: %s", storageRecord.getId(), e.getMessage()), e);
+ }
+
+ try {
+ // index individual parts of kind
+ String[] kindParts = storageRecord.getKind().split(":");
+
+ document.setKind(storageRecord.getKind());
+ document.setNamespace(kindParts[0] + ":" + kindParts[1]);
+ document.setType(kindParts[2]);
+ document.setId(storageRecord.getId());
+ document.setVersion(storageRecord.getVersion());
+ document.setAcl(storageRecord.getAcl());
+ document.setLegal(storageRecord.getLegal());
+ RecordStatus recordStatus = this.jobStatus.getJobStatusByRecordId(storageRecord.getId());
+ if (recordStatus.getIndexProgress().getStatusCode() == 0) {
+ recordStatus.getIndexProgress().setStatusCode(HttpStatus.SC_OK);
+ }
+ document.setIndexProgress(recordStatus.getIndexProgress());
+ if (storageRecord.getAncestry() != null) document.setAncestry(storageRecord.getAncestry());
+ document.setOperationType(idToOperationMap.get(storageRecord.getId()));
+ } catch (Exception e) {
+ this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.FAIL, HttpStatus.SC_INTERNAL_SERVER_ERROR, String.format("error parsing meta data, error-message: %s", e.getMessage()));
+ jaxRsDpsLog.error(String.format("record-id: %s | error parsing meta data, error-message: %s", storageRecord.getId(), e.getMessage()), e);
+ }
+ return document;
+ }
+
+ private List<String> processElasticMappingAndUpsertRecords(RecordIndexerPayload recordIndexerPayload) throws Exception {
+
+ try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
+ List<IndexSchema> schemas = recordIndexerPayload.getSchemas();
+ if (schemas == null || schemas.isEmpty()) {
+ return new LinkedList<>();
+ }
+
+ // process the schema
+ this.cacheOrCreateElasticMapping(schemas, restClient);
+
+ // process the records
+ return this.upsertRecords(recordIndexerPayload.getRecords(), restClient);
+ }
+ }
+
+ private void cacheOrCreateElasticMapping(List<IndexSchema> schemas, RestHighLevelClient restClient) throws Exception {
+
+ for (IndexSchema schema : schemas) {
+ String index = this.elasticIndexNameResolver.getIndexNameFromKind(schema.getKind());
+
+ // check if index exist
+ if (this.indicesService.isIndexExist(restClient, index)) continue;
+
+ // create index
+ Map<String, Object> mapping = this.mappingService.getIndexMappingFromRecordSchema(schema);
+ if (!this.indicesService.createIndex(restClient, index, null, schema.getType(), mapping)) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error", "Error creating index.", String.format("Failed to get confirmation from elastic server for index: %s", index));
+ }
+ }
+ }
+
+ private List<String> upsertRecords(List<RecordIndexerPayload.Record> records, RestHighLevelClient restClient) throws AppException {
+ if (records == null || records.isEmpty()) return new LinkedList<>();
+
+ BulkRequest bulkRequest = new BulkRequest();
+ bulkRequest.timeout(BULK_REQUEST_TIMEOUT);
+
+ for (RecordIndexerPayload.Record record : records) {
+ if ((record.getData() == null || record.getData().isEmpty()) && !record.skippedDataIndexing()) {
+ // it will come here when schema is missing
+ // TODO: rollback once we know what is causing the problem
+ jaxRsDpsLog.warning(String.format("data not found for record: %s", record));
+ }
+
+ OperationType operation = record.getOperationType();
+ Map<String, Object> sourceMap = getSourceMap(record);
+ String index = this.elasticIndexNameResolver.getIndexNameFromKind(record.getKind());
+
+ if (operation == OperationType.create) {
+ IndexRequest indexRequest = new IndexRequest(index, record.getType(), record.getId()).source(this.gson.toJson(sourceMap), XContentType.JSON);
+ bulkRequest.add(indexRequest);
+ } else if (operation == OperationType.update) {
+ UpdateRequest updateRequest = new UpdateRequest(index, record.getType(), record.getId()).upsert(this.gson.toJson(sourceMap), XContentType.JSON);
+ bulkRequest.add(updateRequest);
+ }
+ }
+
+ return processBulkRequest(restClient, bulkRequest);
+ }
+
+ private List<String> processDeleteRecords(Map<String, List<String>> deleteRecordMap) throws Exception {
+ BulkRequest bulkRequest = new BulkRequest();
+ bulkRequest.timeout(BULK_REQUEST_TIMEOUT);
+
+ for (Map.Entry<String, List<String>> record : deleteRecordMap.entrySet()) {
+
+ String[] kindParts = record.getKey().split(":");
+ String type = kindParts[2];
+
+ String index = this.elasticIndexNameResolver.getIndexNameFromKind(record.getKey());
+
+ for (String id : record.getValue()) {
+ DeleteRequest deleteRequest = new DeleteRequest(index, type, id);
+ bulkRequest.add(deleteRequest);
+ }
+ }
+
+ try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
+ return processBulkRequest(restClient, bulkRequest);
+ }
+ }
+
+ private List<String> processBulkRequest(RestHighLevelClient restClient, BulkRequest bulkRequest) throws AppException {
+
+ List<String> failureRecordIds = new LinkedList<>();
+ if (bulkRequest.numberOfActions() == 0) return failureRecordIds;
+
+
+
+ try {
+ BulkResponse bulkResponse = restClient.bulk(bulkRequest, RequestOptions.DEFAULT);
+
+ // log failed bulk requests
+ ArrayList<String> bulkFailures = new ArrayList<>();
+ int succeededResponses = 0;
+ int failedResponses = 0;
+
+ for (BulkItemResponse bulkItemResponse : bulkResponse.getItems()) {
+ if (bulkItemResponse.isFailed()) {
+ BulkItemResponse.Failure failure = bulkItemResponse.getFailure();
+ bulkFailures.add(String.format("elasticsearch bulk service status: %s id: %s message: %s", failure.getStatus(), failure.getId(), failure.getMessage()));
+ this.jobStatus.addOrUpdateRecordStatus(bulkItemResponse.getId(), IndexingStatus.FAIL, failure.getStatus().getStatus(), bulkItemResponse.getFailureMessage());
+ if (RETRY_ELASTIC_EXCEPTION.contains(bulkItemResponse.status())) {
+ failureRecordIds.add(bulkItemResponse.getId());
+ }
+ failedResponses++;
+ } else {
+ succeededResponses++;
+ this.jobStatus.addOrUpdateRecordStatus(bulkItemResponse.getId(), IndexingStatus.SUCCESS, HttpStatus.SC_OK, "Indexed Successfully");
+ }
+ }
+ if (!bulkFailures.isEmpty()) this.jaxRsDpsLog.warning(bulkFailures);
+
+ jaxRsDpsLog.info(String.format("records in elasticsearch service bulk request: %s | successful: %s | failed: %s", bulkRequest.numberOfActions(), succeededResponses, failedResponses));
+ } catch (IOException e) {
+ // throw explicit 504 for IOException
+ throw new AppException(HttpStatus.SC_GATEWAY_TIMEOUT, "Elastic error", "Request cannot be completed in specified time.", e);
+ } catch (ElasticsearchStatusException e) {
+ throw new AppException(e.status().getStatus(), "Elastic error", e.getMessage(), e);
+ } catch (Exception e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error", "Error indexing records.", e);
+ }
+ return failureRecordIds;
+ }
+
+ private Map<String, Object> getSourceMap(RecordIndexerPayload.Record record) {
+
+ Map<String, Object> indexerPayload = new HashMap<>();
+
+ // get the key and get the corresponding object from the individualRecord object
+ if (record.getData() != null) {
+ Map<String, Object> data = new HashMap<>();
+ for (Map.Entry<String, Object> entry : record.getData().entrySet()) {
+ data.put(entry.getKey(), entry.getValue());
+ }
+ indexerPayload.put(Constants.DATA, data);
+ }
+
+ indexerPayload.put(RecordMetaAttribute.ID.getValue(), record.getId());
+ indexerPayload.put(RecordMetaAttribute.KIND.getValue(), record.getKind());
+ indexerPayload.put(RecordMetaAttribute.NAMESPACE.getValue(), record.getNamespace());
+ indexerPayload.put(RecordMetaAttribute.TYPE.getValue(), record.getType());
+ indexerPayload.put(RecordMetaAttribute.VERSION.getValue(), record.getVersion());
+ indexerPayload.put(RecordMetaAttribute.ACL.getValue(), record.getAcl());
+ indexerPayload.put(RecordMetaAttribute.X_ACL.getValue(), Acl.flattenAcl(record.getAcl()));
+ indexerPayload.put(RecordMetaAttribute.LEGAL.getValue(), record.getLegal());
+ indexerPayload.put(RecordMetaAttribute.INDEX_STATUS.getValue(), record.getIndexProgress());
+ if (record.getAncestry() != null) {
+ indexerPayload.put(RecordMetaAttribute.ANCESTRY.getValue(), record.getAncestry());
+ }
+ return indexerPayload;
+ }
+
+ private void retryAndEnqueueFailedRecords(List<RecordInfo> recordInfos, List<String> failuresRecordIds, RecordChangedMessages message) throws IOException {
+
+ jaxRsDpsLog.info(String.format("queuing bulk failed records back to task-queue for retry | count: %s | records: %s", failuresRecordIds.size(), failuresRecordIds));
+ List<RecordInfo> retryRecordInfos = new LinkedList<>();
+ for (String recordId : failuresRecordIds) {
+ for (RecordInfo origMessage : recordInfos) {
+ if (origMessage.getId().equalsIgnoreCase(recordId)) {
+ retryRecordInfos.add(origMessage);
+ }
+ }
+ }
+
+ RecordChangedMessages newMessage = RecordChangedMessages.builder()
+ .messageId(message.getMessageId())
+ .publishTime(message.getPublishTime())
+ .data(this.gson.toJson(retryRecordInfos))
+ .attributes(message.getAttributes()).build();
+
+ String payLoad = this.gson.toJson(newMessage);
+ this.indexerQueueTaskBuilder.createWorkerTask(payLoad, this.headers);
+ }
+
+ private void updateAuditLog() {
+ logAuditEvents(OperationType.create, this.auditLogger::indexCreateRecordSuccess, this.auditLogger::indexCreateRecordFail);
+ logAuditEvents(OperationType.update, this.auditLogger::indexUpdateRecordSuccess, this.auditLogger::indexUpdateRecordFail);
+ logAuditEvents(OperationType.purge, this.auditLogger::indexPurgeRecordSuccess, this.auditLogger::indexPurgeRecordFail);
+ logAuditEvents(OperationType.delete, this.auditLogger::indexDeleteRecordSuccess, this.auditLogger::indexDeleteRecordFail);
+ }
+
+ private void logAuditEvents(OperationType operationType, Consumer<List<String>> successEvent, Consumer<List<String>> failedEvent) {
+ List<RecordStatus> succeededRecords = this.jobStatus.getRecordStatuses(IndexingStatus.SUCCESS, operationType);
+ if(!succeededRecords.isEmpty()) {
+ successEvent.accept(succeededRecords.stream().map(RecordStatus::succeededAuditLogMessage).collect(Collectors.toList()));
+ }
+ List<RecordStatus> skippedRecords = this.jobStatus.getRecordStatuses(IndexingStatus.SKIP, operationType);
+ List<RecordStatus> failedRecords = this.jobStatus.getRecordStatuses(IndexingStatus.FAIL, operationType);
+ failedRecords.addAll(skippedRecords);
+ if(!failedRecords.isEmpty()) {
+ failedEvent.accept(failedRecords.stream().map(RecordStatus::failedAuditLogMessage).collect(Collectors.toList()));
+ }
+ }
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/DpsHeaderFactoryGcp.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/DpsHeaderFactoryGcp.java
new file mode 100644
index 0000000000000000000000000000000000000000..0e3c201656b9c61bdcb537c552d16fd50bef9188
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/DpsHeaderFactoryGcp.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.util;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.inject.Inject;
+
+import com.google.common.base.Strings;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+
+import org.opengroup.osdu.core.gcp.model.AppEngineHeaders;
+import org.opengroup.osdu.core.gcp.util.TraceIdExtractor;
+import org.springframework.context.annotation.Primary;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+@Component
+@RequestScope
+@Primary
+public class DpsHeaderFactoryGcp extends DpsHeaders {
+
+ @Inject
+ public DpsHeaderFactoryGcp(HttpServletRequest request) {
+
+ Map<String, String> headers = Collections
+ .list(request.getHeaderNames())
+ .stream()
+ .collect(Collectors.toMap(h -> h, request::getHeader));
+
+ String traceContext = headers.get(AppEngineHeaders.CLOUD_TRACE_CONTEXT);
+
+ if(!Strings.isNullOrEmpty(traceContext)){
+ headers.put(AppEngineHeaders.TRACE_ID, TraceIdExtractor.getTraceId(traceContext));
+ }
+
+ this.addFromMap(headers);
+
+ // Add Correlation ID if missing
+ this.addCorrelationIdIfMissing();
+ }
+}
\ No newline at end of file
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/MongoClientHandler.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/MongoClientHandler.java
new file mode 100644
index 0000000000000000000000000000000000000000..b60ef1da6a11f4f21c4206448743ecb05e81c59e
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/MongoClientHandler.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.util;
+
+import com.mongodb.ConnectionString;
+import com.mongodb.MongoClientSettings;
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoClients;
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+@Component
+public class MongoClientHandler {
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ org.opengroup.osdu.indexer.util.MongoClientHandler.class);
+ private static final String MONGO_PREFIX = "mongodb://";
+ private static final String MONGO_OPTIONS = "retryWrites=true&w=majority&maxIdleTimeMS=10000";
+
+ private MongoClient mongoClient = null;
+
+ @Value("${mongo.db.url:#{null}}")
+ private String dbUrl;
+
+ @Value("${mongo.db.user:#{null}}")
+ private String dbUser;
+
+ @Value("${mongo.db.password:#{null}}")
+ private String dbPassword;
+
+ private MongoClient getOrInitMongoClient() throws RuntimeException {
+ if (mongoClient != null) {
+ return mongoClient;
+ }
+
+ final String connectionString = String.format("%s%s:%s@%s/?%s",
+ MONGO_PREFIX,
+ dbUser,
+ dbPassword,
+ dbUrl,
+ MONGO_OPTIONS);
+ ConnectionString connString = new ConnectionString(connectionString);
+ MongoClientSettings settings = MongoClientSettings.builder()
+ .applyConnectionString(connString)
+ .retryWrites(true)
+ .build();
+ try {
+ mongoClient = MongoClients.create(settings);
+ } catch (Exception ex) {
+ LOG.error("Error connecting MongoDB", ex);
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Error connecting MongoDB",
+ ex.getMessage(), ex);
+ }
+ return mongoClient;
+ }
+
+ public MongoClient getMongoClient() {
+ if (mongoClient == null) {
+ getOrInitMongoClient();
+ }
+ return mongoClient;
+ }
+
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..2396dd1e55b99ee7853d4e9b94e752be5a559261
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/RequestInfoImpl.java
@@ -0,0 +1,117 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.util;
+
+import com.google.common.base.Strings;
+import lombok.extern.java.Log;
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.core.common.Constants;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.search.DeploymentEnvironment;
+import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient;
+import org.opengroup.osdu.core.gcp.model.AppEngineHeaders;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+import javax.inject.Inject;
+import java.util.Map;
+
+import static org.opengroup.osdu.core.common.model.http.DpsHeaders.AUTHORIZATION;
+
+
+@Log
+@Component
+@RequestScope
+public class RequestInfoImpl implements IRequestInfo {
+
+ @Inject
+ private DpsHeaders dpsHeaders;
+
+ @Inject
+ private IServiceAccountJwtClient serviceAccountJwtClient;
+
+ @Inject
+ private TenantInfo tenantInfo;
+
+ @Value("${DEPLOYMENT_ENVIRONMENT}")
+ private String DEPLOYMENT_ENVIRONMENT;
+
+ private static final String expectedCronHeaderValue = "true";
+
+ @Override
+ public DpsHeaders getHeaders() {
+
+ return this.dpsHeaders;
+ }
+
+ @Override
+ public String getPartitionId() {
+ return this.dpsHeaders.getPartitionId();
+ }
+
+ @Override
+ public Map<String, String> getHeadersMap() {
+ return this.dpsHeaders.getHeaders();
+ }
+
+ @Override
+ public Map<String, String> getHeadersMapWithDwdAuthZ() {
+ return getHeadersWithDwdAuthZ().getHeaders();
+ }
+
+ @Override
+ public DpsHeaders getHeadersWithDwdAuthZ() {
+ // Update DpsHeaders so that service account creds are passed down
+ this.dpsHeaders.put(AUTHORIZATION, this.checkOrGetAuthorizationHeader());
+ return this.dpsHeaders;
+ }
+
+ @Override
+ public boolean isCronRequest() {
+ String appEngineCronHeader = this.dpsHeaders.getHeaders().getOrDefault(AppEngineHeaders.CRON_SERVICE, null);
+ return expectedCronHeaderValue.equalsIgnoreCase(appEngineCronHeader);
+ }
+
+ @Override
+ public boolean isTaskQueueRequest() {
+ if (!this.dpsHeaders.getHeaders().containsKey(AppEngineHeaders.TASK_QUEUE_NAME)) return false;
+
+ String queueId = this.dpsHeaders.getHeaders().get(AppEngineHeaders.TASK_QUEUE_NAME);
+ return queueId.endsWith(Constants.INDEXER_QUEUE_IDENTIFIER);
+ }
+
+ public String checkOrGetAuthorizationHeader() {
+ if (DeploymentEnvironment.valueOf(DEPLOYMENT_ENVIRONMENT) == DeploymentEnvironment.LOCAL) {
+ String authHeader = this.dpsHeaders.getAuthorization();
+ if (Strings.isNullOrEmpty(authHeader)) {
+ throw new AppException(HttpStatus.SC_UNAUTHORIZED, "Invalid authorization header", "Authorization token cannot be empty");
+ }
+ String user = this.dpsHeaders.getUserEmail();
+ if (Strings.isNullOrEmpty(user)) {
+ throw new AppException(HttpStatus.SC_UNAUTHORIZED, "Invalid user header", "User header cannot be empty");
+ }
+ return authHeader;
+ } else {
+ return "Bearer " + this.serviceAccountJwtClient.getIdToken(tenantInfo.getName());
+ }
+ }
+}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..6da39c9ed5a45a7f792936e2c7b88bdb8ee8e236
--- /dev/null
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImpl.java
@@ -0,0 +1,196 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.util;
+
+import com.auth0.jwt.JWT;
+import com.auth0.jwt.exceptions.JWTDecodeException;
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.iam.v1.Iam;
+import com.google.api.services.iam.v1.IamScopes;
+import com.google.api.services.iam.v1.model.SignJwtRequest;
+import com.google.api.services.iam.v1.model.SignJwtResponse;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import java.io.FileInputStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import javax.inject.Inject;
+import org.apache.http.HttpHeaders;
+import org.apache.http.HttpStatus;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.ContentType;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.message.BasicNameValuePair;
+import org.apache.http.util.EntityUtils;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.search.IdToken;
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
+import org.opengroup.osdu.core.common.provider.interfaces.IJwtCache;
+import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
+import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+@Component
+@RequestScope
+public class ServiceAccountJwtGcpClientImpl implements IServiceAccountJwtClient {
+
+ private static final String JWT_AUDIENCE = "https://www.googleapis.com/oauth2/v4/token";
+ private static final String SERVICE_ACCOUNT_NAME_FORMAT = "projects/%s/serviceAccounts/%s";
+
+ private final JsonFactory JSON_FACTORY = new JacksonFactory();
+
+ private Iam iam;
+
+ @Inject
+ private ITenantFactory tenantInfoServiceProvider;
+ @Inject
+ private IJwtCache cacheService;
+ @Inject
+ private JaxRsDpsLog log;
+ @Inject
+ private DpsHeaders dpsHeaders;
+
+ @Value("${GOOGLE_AUDIENCES}")
+ public String GOOGLE_AUDIENCES;
+
+ @Value("${INDEXER_HOST}")
+ public String INDEXER_HOST;
+
+ public String getIdToken(String tenantName) {
+ this.log.info("Tenant name received for auth token is: " + tenantName);
+ TenantInfo tenant = this.tenantInfoServiceProvider.getTenantInfo(tenantName);
+ if (tenant == null) {
+ this.log.error("Invalid tenant name receiving from pubsub");
+ throw new AppException(HttpStatus.SC_BAD_REQUEST, "Invalid tenant Name", "Invalid tenant Name from pubsub");
+ }
+ try {
+
+ IdToken cachedToken = (IdToken) this.cacheService.get(tenant.getServiceAccount());
+ // Add the user to DpsHeaders directly
+ this.dpsHeaders.put(DpsHeaders.USER_EMAIL, tenant.getServiceAccount());
+
+ if (!IdToken.refreshToken(cachedToken)) {
+ return cachedToken.getTokenValue();
+ }
+
+ // Getting signed JWT
+ Map<String, Object> signJwtPayload = this.getJWTCreationPayload(tenant);
+
+ SignJwtRequest signJwtRequest = new SignJwtRequest();
+ signJwtRequest.setPayload(JSON_FACTORY.toString(signJwtPayload));
+
+ String serviceAccountName = String
+ .format(SERVICE_ACCOUNT_NAME_FORMAT, tenant.getProjectId(), tenant.getServiceAccount());
+
+ Iam.Projects.ServiceAccounts.SignJwt signJwt = this.getIam().projects().serviceAccounts()
+ .signJwt(serviceAccountName, signJwtRequest);
+ SignJwtResponse signJwtResponse = signJwt.execute();
+ String signedJwt = signJwtResponse.getSignedJwt();
+
+ // Getting id token
+ List<NameValuePair> postParameters = new ArrayList<>();
+ postParameters.add(new BasicNameValuePair("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer"));
+ postParameters.add(new BasicNameValuePair("assertion", signedJwt));
+
+ HttpPost post = new HttpPost(JWT_AUDIENCE);
+ post.setHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_FORM_URLENCODED.getMimeType());
+ post.setEntity(new UrlEncodedFormEntity(postParameters, "UTF-8"));
+
+ try (CloseableHttpClient httpclient = HttpClients.createDefault();
+ CloseableHttpResponse httpResponse = httpclient.execute(post)) {
+ JsonObject jsonContent = new JsonParser().parse(EntityUtils.toString(httpResponse.getEntity()))
+ .getAsJsonObject();
+
+ if (!jsonContent.has("id_token")) {
+ log.error(String.format("Google IAM response: %s", jsonContent.toString()));
+ throw new AppException(HttpStatus.SC_FORBIDDEN, "Access denied",
+ "The user is not authorized to perform this action");
+ }
+
+ String token = jsonContent.get("id_token").getAsString();
+ IdToken idToken = IdToken.builder().tokenValue(token)
+ .expirationTimeMillis(JWT.decode(token).getExpiresAt().getTime()).build();
+
+ this.cacheService.put(tenant.getServiceAccount(), idToken);
+
+ return token;
+ }
+ } catch (JWTDecodeException e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Persistence error",
+ "Invalid token, error decoding", e);
+ } catch (AppException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Persistence error", "Error generating token",
+ e);
+ }
+ }
+
+ public Iam getIam() throws Exception {
+
+ if (this.iam == null) {
+ HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport();
+
+ // Authenticate using Google Application Default Credentials.
+ GoogleCredential credential = GoogleCredential.getApplicationDefault();
+ if (credential.createScopedRequired()) {
+ List<String> scopes = new ArrayList<>();
+ // Enable full Cloud Platform scope.
+ scopes.add(IamScopes.CLOUD_PLATFORM);
+ credential = credential.createScoped(scopes);
+ }
+
+ // Create IAM API object associated with the authenticated transport.
+ this.iam = new Iam.Builder(httpTransport, JSON_FACTORY, credential)
+ .setApplicationName(INDEXER_HOST)
+ .build();
+ }
+
+ return this.iam;
+ }
+
+ private Map<String, Object> getJWTCreationPayload(TenantInfo tenantInfo) {
+
+ Map<String, Object> payload = new HashMap<>();
+ String googleAudience = GOOGLE_AUDIENCES;
+ if (googleAudience.contains(",")) {
+ googleAudience = googleAudience.split(",")[0];
+ }
+ payload.put("target_audience", googleAudience);
+ payload.put("exp", System.currentTimeMillis() / 1000 + 3600);
+ payload.put("iat", System.currentTimeMillis() / 1000);
+ payload.put("iss", tenantInfo.getServiceAccount());
+ payload.put("aud", JWT_AUDIENCE);
+
+ return payload;
+ }
+}
diff --git a/provider/indexer-reference/src/main/resources/application-dev.properties b/provider/indexer-reference/src/main/resources/application-dev.properties
new file mode 100644
index 0000000000000000000000000000000000000000..09ea204df6cd8cb54bceb57f3b3b3e2d44613b34
--- /dev/null
+++ b/provider/indexer-reference/src/main/resources/application-dev.properties
@@ -0,0 +1,40 @@
+#
+# Copyright 2020 Google LLC
+# Copyright 2020 EPAM Systems, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+GOOGLE_CLOUD_PROJECT=nice-etching-277309
+
+INDEXER_HOST=os-indexer-dot-nice-etching-277309.uc.r.appspot.com
+STORAGE_HOSTNAME=os-storage-dot-nice-etching-277309.uc.r.appspot.com
+
+STORAGE_SCHEMA_HOST=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/schemas
+STORAGE_QUERY_RECORD_HOST=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/query/records
+STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/query/records:batch
+STORAGE_RECORDS_BATCH_SIZE=20
+
+INDEXER_QUEUE_HOST=https://os-indexer-queue-dot-nice-etching-277309.uc.r.appspot.com/_dps/task-handlers/enqueue
+
+AUTHORIZE_API=https://entitlements-dot-nice-etching-277309.uc.r.appspot.com/entitlements/v1
+LEGALTAG_API=https://os-legal-dot-nice-etching-277309.uc.r.appspot.com/api/legal/v1
+CRS_API=example.com
+
+## use below values for gcp: opendes
+REDIS_GROUP_HOST=127.0.0.1
+REDIS_SEARCH_HOST=127.0.0.1
+
+GOOGLE_AUDIENCES=689762842995-pv217jo3k8j803kk6gqf52qb5amos3a9.apps.googleusercontent.com
+
+mb.rabbitmq.uri=amqp://guest:guest@127.0.0.1:5672
diff --git a/provider/indexer-reference/src/main/resources/application.properties b/provider/indexer-reference/src/main/resources/application.properties
new file mode 100644
index 0000000000000000000000000000000000000000..4d62d4abed0705ab986b75574fbbb0ec3e22f418
--- /dev/null
+++ b/provider/indexer-reference/src/main/resources/application.properties
@@ -0,0 +1,78 @@
+#
+# Copyright 2020 Google LLC
+# Copyright 2020 EPAM Systems, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+LOG_PREFIX=indexer
+
+server.servlet.contextPath=/api/indexer/v2
+logging.level.org.springframework.web=DEBUG
+server.port=8080
+JAVA_OPTS=-Xms3072m -Xmx3072m
+JAVA_GC_OPTS=-XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45
+
+DEPLOYMENT_ENVIRONMENT=CLOUD
+
+REDIS_GROUP_PORT=6379
+REDIS_SEARCH_PORT=6379
+DEFAULT_DATA_COUNTRY=US
+
+#Default Cache Settings
+SCHEMA_CACHE_EXPIRATION=60
+INDEX_CACHE_EXPIRATION=60
+ELASTIC_CACHE_EXPIRATION=1440
+CURSOR_CACHE_EXPIRATION=60
+# Kinds Cache expiration 2*24*60
+KINDS_CACHE_EXPIRATION=2880
+# Attributes Cache expiration 2*24*60
+ATTRIBUTES_CACHE_EXPIRATION=2880
+
+KINDS_REDIS_DATABASE=1
+CRON_INDEX_CLEANUP_THRESHOLD_DAYS=3
+CRON_EMPTY_INDEX_CLEANUP_THRESHOLD_DAYS=7
+
+GAE_SERVICE=indexer
+KEY_RING=csqp
+KMS_KEY=searchService
+
+ELASTIC_DATASTORE_KIND=SearchSettings
+ELASTIC_DATASTORE_ID=indexer-service
+
+mongo.db.url=localhost:27017
+mongo.db.user=
+mongo.db.password=
+
+INDEXER_HOST=os-indexer-dot-nice-etching-277309.uc.r.appspot.com
+STORAGE_HOSTNAME=os-storage-dot-nice-etching-277309.uc.r.appspot.com
+
+STORAGE_SCHEMA_HOST=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/schemas
+STORAGE_QUERY_RECORD_HOST=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/query/records
+STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/query/records:batch
+STORAGE_RECORDS_BATCH_SIZE=20
+
+INDEXER_QUEUE_HOST=https://os-indexer-queue-dot-nice-etching-277309.uc.r.appspot.com/_dps/task-handlers/enqueue
+
+AUTHORIZE_API=https://entitlements-dot-nice-etching-277309.uc.r.appspot.com/entitlements/v1
+LEGALTAG_API=https://os-legal-dot-nice-etching-277309.uc.r.appspot.com/api/legal/v1
+CRS_API=example.com
+
+## use below values for gcp: opendes
+REDIS_GROUP_HOST=127.0.0.1
+REDIS_SEARCH_HOST=127.0.0.1
+
+GOOGLE_AUDIENCES=689762842995-pv217jo3k8j803kk6gqf52qb5amos3a9.apps.googleusercontent.com
+
+mb.rabbitmq.uri=amqp://guest:guest@127.0.0.1:5672
+
diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..7ebfab8dba438872ec892048d02cea3dd0d9e1a7
--- /dev/null
+++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.middleware;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.MockitoJUnitRunner;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+
+import javax.servlet.FilterChain;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import java.io.IOException;
+import java.util.Collections;
+
+@RunWith(MockitoJUnitRunner.class)
+public class IndexFilterTest {
+
+ @InjectMocks
+ private IndexFilter indexFilter;
+
+ @Mock
+ private DpsHeaders dpsHeaders;
+
+ @Test
+ public void shouldSetCorrectResponseHeaders() throws IOException, ServletException {
+ HttpServletRequest httpServletRequest = Mockito.mock(HttpServletRequest.class);
+ HttpServletResponse httpServletResponse = Mockito.mock(HttpServletResponse.class);
+ FilterChain filterChain = Mockito.mock(FilterChain.class);
+ Mockito.when(httpServletRequest.getRequestURI()).thenReturn("https://test.com");
+ Mockito.when(httpServletRequest.getMethod()).thenReturn("POST");
+ Mockito.when(dpsHeaders.getCorrelationId()).thenReturn("correlation-id-value");
+
+ indexFilter.doFilter(httpServletRequest, httpServletResponse, filterChain);
+
+ Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Origin", Collections.singletonList("*").toString());
+ Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Headers", Collections.singletonList("origin, content-type, accept, authorization, data-partition-id, correlation-id, appkey").toString());
+ Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Methods", Collections.singletonList("GET, POST, PUT, DELETE, OPTIONS, HEAD").toString());
+ Mockito.verify(httpServletResponse).addHeader("Access-Control-Allow-Credentials", Collections.singletonList("true").toString());
+ Mockito.verify(httpServletResponse).addHeader("X-Frame-Options", Collections.singletonList("DENY").toString());
+ Mockito.verify(httpServletResponse).addHeader("X-XSS-Protection", Collections.singletonList("1; mode=block").toString());
+ Mockito.verify(httpServletResponse).addHeader("X-Content-Type-Options", Collections.singletonList("nosniff").toString());
+ Mockito.verify(httpServletResponse).addHeader("Cache-Control", Collections.singletonList("no-cache, no-store, must-revalidate").toString());
+ Mockito.verify(httpServletResponse).addHeader("Content-Security-Policy", Collections.singletonList("default-src 'self'").toString());
+ Mockito.verify(httpServletResponse).addHeader("Strict-Transport-Security", Collections.singletonList("max-age=31536000; includeSubDomains").toString());
+ Mockito.verify(httpServletResponse).addHeader("Expires", Collections.singletonList("0").toString());
+ Mockito.verify(httpServletResponse).addHeader("correlation-id", "correlation-id-value");
+ Mockito.verify(filterChain).doFilter(httpServletRequest, httpServletResponse);
+ }
+}
diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/CronServiceImplTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/CronServiceImplTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..80e25bbb400b88632fec13016fdbfc6bd622b4c4
--- /dev/null
+++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/CronServiceImplTest.java
@@ -0,0 +1,138 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.service;
+
+import com.google.common.collect.Lists;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.search.IndexInfo;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.opengroup.osdu.core.common.search.IndicesService;
+import org.opengroup.osdu.indexer.util.ElasticClientHandler;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+
+import static org.mockito.Mockito.*;
+
+
+@RunWith(SpringRunner.class)
+@PrepareForTest({RestHighLevelClient.class})
+public class CronServiceImplTest {
+
+ @Mock
+ private RestHighLevelClient restHighLevelClient;
+ @Mock
+ private IndicesService indicesService;
+ @Mock
+ private ElasticClientHandler elasticClientHandler;
+ @Mock
+ private IRequestInfo requestInfo;
+ @Mock
+ private JaxRsDpsLog log;
+ @InjectMocks
+ private CronServiceImpl sut;
+
+ @InjectMocks
+ private DpsHeaders dpsHeaders;
+
+ @Before
+ public void setup() {
+
+ when(this.requestInfo.getHeaders()).thenReturn(dpsHeaders);
+
+ ReflectionTestUtils.setField(this.sut, "CRON_INDEX_CLEANUP_THRESHOLD_DAYS", "3");
+ ReflectionTestUtils.setField(this.sut, "CRON_EMPTY_INDEX_CLEANUP_THRESHOLD_DAYS", "7");
+ }
+
+ @Test
+ public void run_cleanup_when_cron_job_runs_with_correct_pattern() throws Exception {
+ final String indexPattern = "tenant1-index-*";
+
+ IndexInfo info = IndexInfo.builder().name("tenant1-index-1.0.0").documentCount("10").creationDate(Long.toString(Instant.now().minus(4, ChronoUnit.DAYS).toEpochMilli())).build();
+
+ when(this.requestInfo.getPartitionId()).thenReturn("tenant1");
+ when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient);
+ when(this.indicesService.getIndexInfo(this.restHighLevelClient, indexPattern)).thenReturn(Lists.newArrayList(info));
+
+ this.sut.cleanupIndices(indexPattern);
+
+ verify(this.indicesService, times(1)).deleteIndex(restHighLevelClient, "tenant1-index-1.0.0");
+ verify(this.indicesService, times(1)).getIndexInfo(restHighLevelClient, indexPattern);
+ }
+
+ @Test(expected = IOException.class)
+ public void run_cleanup_when_cron_job_runs_with_wrong_pattern() throws Exception {
+ IOException exception = new IOException("blah");
+ when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient);
+ when(this.indicesService.getIndexInfo(this.restHighLevelClient, "tenant1-test-*")).thenThrow(exception);
+
+ this.sut.cleanupIndices("tenant1-test-*");
+
+ verify(this.indicesService, times(0)).deleteIndex(any(), any());
+ }
+
+ @Test
+ public void run_cleanup_when_backend_does_not_have_empty_stale_indices() throws Exception {
+ IndexInfo info = IndexInfo.builder().name("tenant1-index-1.0.0").documentCount("10").creationDate(Long.toString(Instant.now().minus(8, ChronoUnit.DAYS).toEpochMilli())).build();
+
+ when(this.requestInfo.getPartitionId()).thenReturn("tenant1");
+ when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient);
+ when(this.indicesService.getIndexInfo(this.restHighLevelClient, null)).thenReturn(Lists.newArrayList(info));
+
+ this.sut.cleanupEmptyStaleIndices();
+
+ verify(this.indicesService, times(0)).deleteIndex(restHighLevelClient, null);
+ verify(this.indicesService, times(1)).getIndexInfo(restHighLevelClient, null);
+ }
+
+ @Test
+ public void run_cleanup_when_backend_have_empty_stale_indices() throws Exception {
+ IndexInfo info = IndexInfo.builder().name("tenant1-index-1.0.0").documentCount("0").creationDate(Long.toString(Instant.now().minus(8, ChronoUnit.DAYS).toEpochMilli())).build();
+
+ when(this.requestInfo.getPartitionId()).thenReturn("tenant1");
+ when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient);
+ when(this.indicesService.getIndexInfo(this.restHighLevelClient, null)).thenReturn(Lists.newArrayList(info));
+
+ this.sut.cleanupEmptyStaleIndices();
+
+ verify(this.indicesService, times(1)).deleteIndex(restHighLevelClient, "tenant1-index-1.0.0");
+ verify(this.indicesService, times(1)).getIndexInfo(restHighLevelClient, null);
+ }
+
+ @Test(expected = IOException.class)
+ public void run_cleanup_when_backend_throws_exception() throws Exception {
+ IOException exception = new IOException("blah");
+ when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient);
+ when(this.indicesService.getIndexInfo(this.restHighLevelClient, null)).thenThrow(exception);
+
+ this.sut.cleanupEmptyStaleIndices();
+
+ verify(this.indicesService, times(0)).deleteIndex(any(), any());
+ }
+}
\ No newline at end of file
diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..e83cb64880b35e56be406b186c81185ddff353cf
--- /dev/null
+++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceTest.java
@@ -0,0 +1,114 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.service;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.model.search.ClusterSettings;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.provider.interfaces.IElasticCredentialsCache;
+import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository;
+import org.opengroup.osdu.core.common.multitenancy.ITenantInfoService;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.Mockito.when;
+
+@RunWith(SpringRunner.class)
+public class ElasticSettingServiceTest {
+
+ @Mock
+ private ITenantInfoService tenantInfoService;
+ @Mock
+ private IElasticRepository elasticRepository;
+ @Mock
+ private IElasticCredentialsCache elasticCredentialCache;
+ @Mock
+ private TenantInfo tenantInfo;
+ @InjectMocks
+ private ElasticSettingServiceImpl sut;
+ @Mock
+ private ClusterSettings clusterSettings;
+ @Mock
+ private DpsHeaders headersInfo;
+
+ @Mock
+ private JaxRsDpsLog log;
+
+
+ public String GAE_SERVICE = "indexer";
+
+ private final String host = "db5c51c1.us-central1.gcp.cloud.es.io";
+ private final int port = 9243;
+ private final String credentials = "name:password";
+
+ String cacheKey = "";
+
+
+ @Before
+ public void setup() {
+ when(tenantInfo.getName()).thenReturn("tenant1");
+ when(this.headersInfo.getPartitionId()).thenReturn("tenant1");
+ when(this.tenantInfoService.getTenantInfo()).thenReturn(tenantInfo);
+ sut.GAE_SERVICE = "indexer";
+ clusterSettings = ClusterSettings.builder().host(host).port(port).userNameAndPassword(credentials).build();
+ cacheKey = String.format("%s-%s", GAE_SERVICE, tenantInfo.getName());
+ }
+
+ @Test
+ public void should_getValid_clusterSettings_fromCache() {
+
+ when(this.elasticCredentialCache.get(cacheKey)).thenReturn(clusterSettings);
+
+ ClusterSettings response = this.sut.getElasticClusterInformation();
+ assertNotNull(response);
+ assertEquals(response.getHost(), host);
+ assertEquals(response.getPort(), port);
+ assertEquals(response.getUserNameAndPassword(), credentials);
+ }
+
+ @Test
+ public void should_getValid_clusterSettings_fromCosmosDB() {
+
+ when(this.elasticCredentialCache.get(cacheKey)).thenReturn(clusterSettings);
+
+ when(this.elasticRepository.getElasticClusterSettings(tenantInfo)).thenReturn(clusterSettings);
+
+ ClusterSettings response = this.sut.getElasticClusterInformation();
+ assertNotNull(response);
+ assertEquals(response.getHost(), host);
+ assertEquals(response.getPort(), port);
+ assertEquals(response.getUserNameAndPassword(), credentials);
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throwAppException_when_tenantClusterInfo_not_found() throws AppException {
+
+ when(this.elasticRepository.getElasticClusterSettings(tenantInfo)).thenReturn(null);
+
+ this.sut.getElasticClusterInformation();
+
+ }
+}
diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImplTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImplTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..9ef4a6adbe21c802885ee6b4caf93840b2728c1e
--- /dev/null
+++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexCopyServiceImplTest.java
@@ -0,0 +1,193 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.service;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentMatchers;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.model.search.ClusterSettings;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.indexer.logging.AuditLogger;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.opengroup.osdu.core.common.model.indexer.IElasticSettingService;
+import org.opengroup.osdu.core.common.search.IndicesService;
+import org.opengroup.osdu.indexer.util.ElasticClientHandler;
+import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.io.IOException;
+import java.lang.reflect.Type;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@RunWith(SpringRunner.class)
+@PrepareForTest({RestHighLevelClient.class, Response.class, RestClient.class, HttpEntity.class, EntityUtils.class})
+public class IndexCopyServiceImplTest {
+ private final String correlationId = UUID.randomUUID().toString();
+
+ @Mock
+ private HttpEntity httpEntity;
+ @Mock
+ private HttpEntity httpEntityRequest;
+ @Mock
+ private IRequestInfo requestInfo;
+ @Mock
+ private DpsHeaders headersInfo;
+ @Mock
+ private RestClient restClient;
+ @Mock
+ private RestHighLevelClient restHighLevelClient;
+ @Mock
+ private IndicesService indicesService;
+ @Mock
+ private IndexerMappingService mappingService;
+ @Mock
+ private ElasticClientHandler elasticClientHandler;
+ @Mock
+ private ElasticIndexNameResolver elasticIndexNameResolver;
+ @Mock
+ private Response response;
+ @Mock
+ private IElasticSettingService elasticSettingService;
+ @Mock
+ private AuditLogger auditLogger;
+ @Mock
+ private Map<String, String> httpHeaders;
+ @InjectMocks
+ private IndexCopyServiceImpl sut;
+
+ private ClusterSettings commonCluster;
+
+ private Map<String, Object> correctMap;
+
+ @Before
+ public void setup() {
+
+ commonCluster = ClusterSettings.builder().host("commonhost").port(8080).userNameAndPassword("username:pwd").build();
+
+ httpHeaders = new HashMap<>();
+ httpHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
+ httpHeaders.put(DpsHeaders.CORRELATION_ID, correlationId);
+ when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(httpHeaders);
+ when(response.getEntity()).thenReturn(httpEntity);
+
+ Type mapType = new TypeToken<Map<String, Object>>() {}.getType();
+ String afterFormat = "{\"properties\":{\"id\":{\"type\":\"keyword\"}}}";
+ correctMap = new Gson().fromJson(afterFormat, mapType);
+
+ restHighLevelClient = mock(RestHighLevelClient.class);
+
+ }
+
+ @Test(expected = IOException.class)
+ public void should_throwIOException_when_indexMappingNotFound() throws Exception {
+ IOException exception = new IOException("Fail to get mapping for the given index from common cluster.");
+
+ when(this.mappingService.getIndexMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenThrow(exception);
+
+ this.sut.copyIndex("common:metadata:entity:1.0.0");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void should_throwIllegalArgExceptionCopyIndexRequest_copyIndexTest() {
+ try {
+ this.sut.copyIndex(null);
+ } catch (IOException e) {
+ fail("Should not throw IOException but illegalArgumentException.");
+ }
+ }
+
+ @Test
+ public void should_returnIndexMapping_getIndexMappingFromCommonClustertest() {
+ String mappingJson = "{\"common-metadata-entity-1.0.0\":{\"mappings\":{\"entity\":{\"properties\":{\"id\":{\"type\":\"keyword\"}}}}}}";
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ try {
+ when(this.mappingService.getIndexMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(mappingJson);
+ Map<String, Object> resultMap = this.sut.getIndexMappingsFromCommonCluster("test", "test");
+ Assert.assertEquals(resultMap, correctMap);
+ } catch (Exception ignored) {
+ }
+ }
+
+ @Test
+ public void should_returnClusterInfo_getCommonClusterInformationtest() {
+ try {
+ String[] correctCommonCluster = {"https://commonhost:8080", "username", "pwd"};
+
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+
+ when(elasticSettingService.getElasticClusterInformation()).thenReturn(commonCluster);
+
+ String[] resultCommonCluster = this.sut.getCommonClusterInformation();
+ Assert.assertEquals(correctCommonCluster[0], resultCommonCluster[0]);
+ Assert.assertEquals(correctCommonCluster[1], resultCommonCluster[1]);
+ Assert.assertEquals(correctCommonCluster[2], resultCommonCluster[2]);
+ } catch (IOException ignored) {
+ fail("Should not throw this exception " + ignored.getMessage());
+ }
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throwException_failToCreateIndexInTenantCluster_createIndexInTenantClustertest() {
+ try {
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ when(indicesService.createIndex(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(false);
+ this.sut.createIndexInTenantCluster("test", "test", "test", correctMap);
+ } catch (IOException ignored) {
+ fail("Should not throw this exception " + ignored.getMessage());
+ }
+ }
+
+ @Ignore
+ public void should_returnTaskIdResponse_reindexRequestSucceed_reindexInTenantClustertest() {
+ //TODO: fix the null Response from restHighLevelClient.getLowLevelClient().performRequest().
+ try {
+ String[] correctCommonCluster = {"https://commonhost:8080", "username", "pwd"};
+ Request request = new Request("POST", "/_reindex?wait_for_completion=false");
+ request.setEntity(httpEntityRequest);
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ when(indicesService.createIndex(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(false);
+ when(restHighLevelClient.getLowLevelClient()).thenReturn(restClient);
+ when(restClient.performRequest(request)).thenReturn(response);
+ when(response.getEntity()).thenReturn(httpEntity);
+ Assert.assertEquals(httpEntity, this.sut.reindexInTenantCluster("test", "test", correctCommonCluster));
+ } catch (IOException ignored) {
+ }
+ }
+}
diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..d091cf83b22ce4fc1b6bd3c3d78042fc88ec85e6
--- /dev/null
+++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java
@@ -0,0 +1,341 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.service;
+
+import com.google.gson.Gson;
+import org.apache.http.StatusLine;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
+import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData;
+import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.*;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.reindex.BulkByScrollResponse;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
+import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
+import org.opengroup.osdu.core.common.search.Config;
+import org.opengroup.osdu.indexer.util.ElasticClientHandler;
+import org.opengroup.osdu.indexer.util.TypeMapper;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.io.IOException;
+import java.util.*;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.MockitoAnnotations.initMocks;
+import static org.powermock.api.mockito.PowerMockito.mockStatic;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@Ignore
+@RunWith(SpringRunner.class)
+@PrepareForTest({ RestHighLevelClient.class, IndicesClient.class, Config.class })
+public class IndexerMappingServiceTest {
+
+ private final String kind = "tenant:test:test:1.0.0";
+ private final String index = "tenant-test-test-1.0.0";
+ private final String type = "test";
+ private final String mappingValid = "{\"dynamic\":false,\"properties\":{\"data\":{\"properties\":{\"Msg\":{\"type\":\"text\",\"analyzer\":\"de_indexer_analyzer\",\"search_analyzer\":\"de_search_analyzer\"},\"Location\":{\"type\":\"geo_point\"}}},\"id\":{\"type\":\"keyword\"},\"acl\":{\"properties\":{\"viewers\":{\"type\":\"keyword\"},\"owners\":{\"type\":\"keyword\"}}}}}";
+
+ @Mock
+ private RestClient restClient;
+ @Mock
+ private Response response;
+ @Mock
+ private StatusLine statusLine;
+
+ @InjectMocks
+ private IndexerMappingServiceImpl sut;
+
+ @Mock
+ private ElasticClientHandler elasticClientHandler;
+
+ @InjectMocks
+ private RestHighLevelClient restHighLevelClient;
+
+ @InjectMocks
+ private IndexSchema indexSchema;
+ @InjectMocks
+ private IndicesClient indicesClient;
+
+ @InjectMocks
+ private AcknowledgedResponse mappingResponse;
+
+ @Before
+ public void setup() throws IOException {
+ initMocks(this);
+ mockStatic(Config.class);
+ when(Config.isPreDemo()).thenReturn(true);
+ Map<String, String> dataMapping = new HashMap<>();
+ dataMapping.put("Location", "geo_point");
+ dataMapping.put("Msg", "text");
+ Map<String, Object> metaMapping = new HashMap<>();
+ metaMapping.put(RecordMetaAttribute.ID.getValue(), "keyword");
+ metaMapping.put(RecordMetaAttribute.ACL.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.ACL));
+ this.indexSchema = IndexSchema.builder().kind(kind).type(type).dataSchema(dataMapping).metaSchema(metaMapping)
+ .build();
+
+ this.indicesClient = PowerMockito.mock(IndicesClient.class);
+ this.restHighLevelClient = PowerMockito.mock(RestHighLevelClient.class);
+
+ when(this.restHighLevelClient.getLowLevelClient()).thenReturn(restClient);
+ when(this.restClient.performRequest(any())).thenReturn(response);
+ when(this.response.getStatusLine()).thenReturn(statusLine);
+ when(this.statusLine.getStatusCode()).thenReturn(200);
+ }
+
+ @Test
+ public void should_returnValidMapping_givenFalseMerge_createMappingTest() {
+ try {
+ String mapping = this.sut.createMapping(restHighLevelClient, indexSchema, index, false);
+ assertEquals(mappingValid, mapping);
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_returnValidMapping_givenTrueMerge_createMappingTest() {
+ try {
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
+
+ String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true);
+ assertEquals(this.mappingValid, mapping);
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_returnValidMapping_givenExistType_createMappingTest() {
+ try {
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
+
+ IndexerMappingServiceImpl indexerMappingServiceLocal = PowerMockito.spy(new IndexerMappingServiceImpl());
+ doReturn(false).when(indexerMappingServiceLocal).isTypeExist(any(), any(), any());
+ String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true);
+ assertEquals(this.mappingValid, mapping);
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_update_indices_field_with_keyword_when_valid_indices() throws Exception {
+ try {
+ Set<String> indices = new HashSet<String>();
+ indices.add("indices 1");
+ GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ when(this.indicesClient.getFieldMapping(any(), any())).thenReturn(getFieldMappingsResponse);
+ XContentBuilder builder = XContentFactory.jsonBuilder();
+ builder.startObject();
+ builder.field("any field", new HashMap());
+ builder.endObject();
+ BytesReference bytesReference = BytesReference.bytes(builder);
+ FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
+ Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ mapBuilder.put("data.any field", mappingMetaData);
+ Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ mappingBuilder.put("any index 1", mapBuilder);
+ mappingBuilder.put("any index 2", mapBuilder);
+ Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ mapping.put("indices 1", mappingBuilder);
+ when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
+ BulkByScrollResponse response = mock(BulkByScrollResponse.class);
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class));
+ when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+
+ this.sut.updateIndexMappingForIndicesOfSameType( indices,"any field");
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throw_exception_if_someIndex_is_invalid_andWeIndexfield_with_keyword() throws Exception {
+ try {
+ Set<String> indices = new HashSet<String>();
+ indices.add("invalid 1");
+ GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ when(this.indicesClient.getFieldMapping(any(), any())).thenReturn(getFieldMappingsResponse);
+ XContentBuilder builder = XContentFactory.jsonBuilder();
+ builder.startObject();
+ builder.field("any field", new HashMap());
+ builder.endObject();
+ BytesReference bytesReference = BytesReference.bytes(builder);
+ FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
+ Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ mapBuilder.put("data.any field", mappingMetaData);
+ Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ mappingBuilder.put("any index 1", mapBuilder);
+ mappingBuilder.put("any index 2", mapBuilder);
+ Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ mapping.put("indices 1", mappingBuilder);
+ when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
+ BulkByScrollResponse response = mock(BulkByScrollResponse.class);
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class));
+ when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+
+ this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
+ } catch (Exception e) {
+ throw e;
+ }
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throw_exception_if_type_of_index_is_invalid_andWeIndexfield_with_keyword() throws Exception {
+ try {
+ Set<String> indices = new HashSet<String>();
+ indices.add("indices 1");
+ GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ when(this.indicesClient.getFieldMapping(any(), any())).thenReturn(getFieldMappingsResponse);
+ XContentBuilder builder = XContentFactory.jsonBuilder();
+ builder.startObject();
+ builder.field("any field", new HashMap());
+ builder.endObject();
+ BytesReference bytesReference = BytesReference.bytes(builder);
+ FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
+ Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ mapBuilder.put("data.any field", mappingMetaData);
+ Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ mappingBuilder.put("any index 1", mapBuilder);
+ mappingBuilder.put("any index 2", mapBuilder);
+ Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ mapping.put("indices 1", mappingBuilder);
+ when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
+ BulkByScrollResponse response = mock(BulkByScrollResponse.class);
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class));
+ when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field invalid");
+ } catch (Exception e) {
+ throw e;
+ }
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throw_exception_if_elastic_search_failedToFetch_andWeIndexfield_with_keyword() throws Exception {
+ try {
+
+ Set<String> indices = new HashSet<String>();
+ indices.add("indices 1");
+ indices.add("indices Invalid");
+ GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ when(this.indicesClient.getFieldMapping(any(), any())).thenThrow(new ElasticsearchException(""));
+ XContentBuilder builder = XContentFactory.jsonBuilder();
+ builder.startObject();
+ builder.field("any field", new HashMap());
+ builder.endObject();
+ BytesReference bytesReference = BytesReference.bytes(builder);
+ FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
+ Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ mapBuilder.put("data.any field", mappingMetaData);
+ Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ mappingBuilder.put("any index 1", mapBuilder);
+ mappingBuilder.put("any index 2", mapBuilder);
+ Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ mapping.put("indices 1", mappingBuilder);
+ when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
+ BulkByScrollResponse response = mock(BulkByScrollResponse.class);
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class));
+ when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
+ } catch (AppException e) {
+ throw e;
+ }
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throw_exception_when_elastic_failedToIndex_indices_field_with_keyword() {
+ try {
+ Set<String> indices = new HashSet<String>();
+ indices.add("indices 1");
+ indices.add("indices Invalid");
+ GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ when(this.indicesClient.getFieldMapping(any(), any())).thenReturn(getFieldMappingsResponse);
+ XContentBuilder builder = XContentFactory.jsonBuilder();
+ builder.startObject();
+ builder.field("any field", new HashMap());
+ builder.endObject();
+ BytesReference bytesReference = BytesReference.bytes(builder);
+ FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
+ Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ mapBuilder.put("data.any field", mappingMetaData);
+ Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ mappingBuilder.put("any index 1", mapBuilder);
+ mappingBuilder.put("any index 2", mapBuilder);
+ Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ mapping.put("indices 1", mappingBuilder);
+ when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(any(), any(RequestOptions.class));
+ BulkByScrollResponse response = mock(BulkByScrollResponse.class);
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(any(), any(RequestOptions.class));
+ when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
+ when(this.indicesClient.putMapping(any(), any(RequestOptions.class))).thenThrow(new ElasticsearchException(""));
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
+ } catch (AppException e) {
+ throw e;
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+
+ @Test
+ public void should_returnDocumentMapping_givenValidIndexSchema() {
+
+ try {
+ Map<String, Object> documentMapping = this.sut.getIndexMappingFromRecordSchema(this.indexSchema);
+ String documentMappingJson = new Gson().toJson(documentMapping);
+ assertEquals(this.mappingValid, documentMappingJson);
+
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+}
diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..e1dd9c641f4c25c50949edec3f6021921040aa2f
--- /dev/null
+++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java
@@ -0,0 +1,403 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.service;
+
+import org.apache.http.HttpStatus;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.indexer.OperationType;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache;
+import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
+import org.opengroup.osdu.core.common.model.http.RequestStatus;
+import org.opengroup.osdu.core.common.search.IndicesService;
+import org.opengroup.osdu.indexer.util.ElasticClientHandler;
+import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.*;
+import static org.mockito.MockitoAnnotations.initMocks;
+import static org.powermock.api.mockito.PowerMockito.mock;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@RunWith(SpringRunner.class)
+@PrepareForTest({RestHighLevelClient.class})
+public class IndexerSchemaServiceTest {
+
+ private final String kind = "tenant:test:test:1.0.0";
+ private final String emptySchema = null;
+ private final String someSchema = "{\"kind\":\"tenant:test:test:1.0.0\", \"schema\":[{\"path\":\"test-path\", \"kind\":\"tenant:test:test:1.0.0\"}]}";
+
+ @Mock
+ private JaxRsDpsLog log;
+ @Mock
+ private StorageService storageService;
+ @Mock
+ private ElasticClientHandler elasticClientHandler;
+ @Mock
+ private ElasticIndexNameResolver elasticIndexNameResolver;
+ @Mock
+ private IndexerMappingService mappingService;
+ @Mock
+ private IndicesService indicesService;
+ @Mock
+ private ISchemaCache schemaCache;
+ @InjectMocks
+ private IndexSchemaServiceImpl sut;
+
+ @Before
+ public void setup() {
+ initMocks(this);
+ RestHighLevelClient restHighLevelClient = mock(RestHighLevelClient.class);
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ }
+
+ @Test
+ public void should_returnNull_givenEmptySchema_getIndexerInputSchemaSchemaTest() throws Exception {
+ when(storageService.getStorageSchema(any())).thenReturn(emptySchema);
+
+ IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false);
+
+ Assert.assertNotNull(indexSchema);
+ }
+
+ @Test
+ public void should_returnValidResponse_givenValidSchema_getIndexerInputSchemaTest() throws Exception {
+ when(storageService.getStorageSchema(any())).thenReturn(someSchema);
+
+ IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false);
+
+ Assert.assertEquals(kind, indexSchema.getKind());
+ }
+
+ @Test
+ public void should_returnValidResponse_givenValidSchemaWithCacheHit_getIndexerInputSchemaTest() throws Exception {
+ when(storageService.getStorageSchema(any())).thenReturn(someSchema);
+ when(this.schemaCache.get(kind + "_flattened")).thenReturn(someSchema);
+
+ IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false);
+
+ Assert.assertEquals(kind, indexSchema.getKind());
+ }
+
+ @Test
+ public void should_throw500_givenInvalidSchemaCacheHit_getIndexerInputSchemaTest() {
+ try {
+ String invalidSchema = "{}}";
+ when(storageService.getStorageSchema(any())).thenReturn(invalidSchema);
+
+ this.sut.getIndexerInputSchema(kind, false);
+ fail("Should throw exception");
+ } catch (AppException e) {
+ Assert.assertEquals(HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getError().getCode());
+ Assert.assertEquals("An error has occurred while normalizing the schema.", e.getError().getMessage());
+ } catch (Exception e) {
+ fail("Should not throw exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_return_basic_schema_when_storage_returns_no_schema() {
+ IndexSchema returnedSchema = this.sut.getIndexerInputSchema(kind, false);
+
+ assertNotNull(returnedSchema.getDataSchema());
+ assertNotNull(returnedSchema);
+ assertEquals(kind, returnedSchema.getKind());
+ }
+
+ @Test
+ public void should_create_schema_when_storage_returns_valid_schema() throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"startDate\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"endDate\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"type \"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"itemguid\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.create_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(false);
+ when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
+
+ this.sut.processSchemaMessages(schemaMessages);
+
+ verify(this.mappingService, times(1)).getIndexMappingFromRecordSchema(any());
+ verify(this.indicesService, times(1)).createIndex(any(), any(), any(), any(), any());
+ verifyNoMoreInteractions(this.mappingService);
+ }
+
+ @Test
+ public void should_merge_mapping_when_storage_returns_valid_schema() throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"startDate\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.create_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
+
+ this.sut.processSchemaMessages(schemaMessages);
+
+ verify(this.indicesService, times(0)).createIndex(any(), any(), any(), any(), any());
+ verify(this.mappingService, times(1)).createMapping(any(), any(), any(), anyBoolean());
+ verifyNoMoreInteractions(this.mappingService);
+ }
+
+ @Test
+ public void should_throw_mapping_conflict_when_elastic_backend_cannot_process_schema_changes() throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String reason = String.format("Could not create type mapping %s/completion.", kind.replace(":", "-"));
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.create_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
+ when(this.mappingService.createMapping(any(), any(), any(), anyBoolean())).thenThrow(new AppException(HttpStatus.SC_BAD_REQUEST, reason, ""));
+
+ try {
+ this.sut.processSchemaMessages(schemaMessages);
+ } catch (AppException e) {
+ assertEquals(e.getError().getCode(), RequestStatus.SCHEMA_CONFLICT);
+ assertEquals(e.getError().getMessage(), "error creating or merging index mapping");
+ assertEquals(e.getError().getReason(), reason);
+ } catch (Exception e) {
+ fail("Should not throw this exception " + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_throw_genericAppException_when_elastic_backend_cannot_process_schema_changes() throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String reason = String.format("Could not create type mapping %s/completion.", kind.replace(":", "-"));
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.create_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
+ when(this.mappingService.createMapping(any(), any(), any(), anyBoolean())).thenThrow(new AppException(HttpStatus.SC_FORBIDDEN, reason, "blah"));
+
+ try {
+ this.sut.processSchemaMessages(schemaMessages);
+ } catch (AppException e) {
+ assertEquals(e.getError().getCode(), HttpStatus.SC_FORBIDDEN);
+ assertEquals(e.getError().getMessage(), "blah");
+ assertEquals(e.getError().getReason(), reason);
+ } catch (Exception e) {
+ fail("Should not throw this exception " + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_log_and_do_nothing_when_storage_returns_invalid_schema() throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"" +
+ "}";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.create_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
+
+ this.sut.processSchemaMessages(schemaMessages);
+
+ verify(this.log).warning(eq("schema not found for kind: tenant1:avocet:completion:1.0.0"));
+ }
+
+ @Test
+ public void should_invalidateCache_when_purge_schema_and_schema_found_in_cache() throws IOException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.purge_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.schemaCache.get(kind)).thenReturn("schema");
+ when(this.schemaCache.get(kind + "_flattened")).thenReturn("flattened schema");
+
+ this.sut.processSchemaMessages(schemaMessages);
+
+ verify(this.schemaCache, times(2)).get(anyString());
+ verify(this.schemaCache, times(2)).delete(anyString());
+ }
+
+ @Test
+ public void should_log_warning_when_purge_schema_and_schema_not_found_in_cache() throws IOException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.purge_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(false);
+
+ this.sut.processSchemaMessages(schemaMessages);
+
+ verify(this.log).warning(eq(String.format("Kind: %s not found", kind)));
+ }
+
+ @Test
+ public void should_sync_schema_with_storage() throws Exception {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.indicesService.deleteIndex(any(), any())).thenReturn(true);
+ when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
+
+ this.sut.syncIndexMappingWithStorageSchema(kind);
+
+ verify(this.mappingService, times(1)).getIndexMappingFromRecordSchema(any());
+ verify(this.indicesService, times(1)).isIndexExist(any(), any());
+ verify(this.indicesService, times(1)).deleteIndex(any(), any());
+ verify(this.indicesService, times(1)).createIndex(any(), any(), any(), any(), any());
+ verifyNoMoreInteractions(this.mappingService);
+ }
+
+ @Test
+ public void should_throw_exception_while_snapshot_running_sync_schema_with_storage() throws Exception {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.indicesService.deleteIndex(any(), any())).thenThrow(new AppException(HttpStatus.SC_CONFLICT, "Index deletion error", "blah"));
+
+ try {
+ this.sut.syncIndexMappingWithStorageSchema(kind);
+ } catch (AppException e) {
+ assertEquals(e.getError().getCode(), HttpStatus.SC_CONFLICT);
+ assertEquals(e.getError().getMessage(), "blah");
+ assertEquals(e.getError().getReason(), "Index deletion error");
+ } catch (Exception e) {
+ fail("Should not throw this exception " + e.getMessage());
+ }
+
+ verify(this.indicesService, times(1)).isIndexExist(any(), any());
+ verify(this.indicesService, times(1)).deleteIndex(any(), any());
+ verify(this.mappingService, never()).getIndexMappingFromRecordSchema(any());
+ verify(this.indicesService, never()).createIndex(any(), any(), any(), any(), any());
+ }
+
+ @Test
+ public void should_return_true_while_if_forceClean_requested() throws IOException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+
+ assertTrue(this.sut.isStorageSchemaSyncRequired(kind, true));
+ }
+
+ @Test
+ public void should_return_true_while_if_forceClean_notRequested_and_indexNotFound() throws IOException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(false);
+
+ assertTrue(this.sut.isStorageSchemaSyncRequired(kind, false));
+ }
+
+ @Test
+ public void should_return_false_while_if_forceClean_notRequested_and_indexExist() throws IOException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+
+ assertFalse(this.sut.isStorageSchemaSyncRequired(kind, false));
+ }
+}
diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..f2a8886f1372faf894698a8272e35ee8c608efda
--- /dev/null
+++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceTest.java
@@ -0,0 +1,354 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.service;//// Copyright 2017-2019, Schlumberger
+////
+//// Licensed under the Apache License, Version 2.0 (the "License");
+//// you may not use this file except in compliance with the License.
+//// You may obtain a copy of the License at
+////
+//// http://www.apache.org/licenses/LICENSE-2.0
+////
+//// Unless required by applicable law or agreed to in writing, software
+//// distributed under the License is distributed on an "AS IS" BASIS,
+//// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//// See the License for the specific language governing permissions and
+//// limitations under the License.
+//
+//package org.opendes.indexer.service;
+//
+//import com.google.gson.Gson;
+//import com.google.gson.reflect.TypeToken;
+//import org.elasticsearch.action.bulk.BulkItemResponse;
+//import org.elasticsearch.action.bulk.BulkResponse;
+//import org.elasticsearch.client.RequestOptions;
+//import org.elasticsearch.client.RestHighLevelClient;
+//import org.junit.Before;
+//import org.junit.Ignore;
+//import org.junit.Test;
+//import org.junit.runner.RunWith;
+//import org.mockito.InjectMocks;
+//import org.mockito.Mock;
+//import org.mockito.Spy;
+//import org.opendes.client.api.DpsHeaders;
+//import org.opendes.core.logging.JaxRsDpsLog;
+//import org.opendes.core.model.DeploymentEnvironment;
+//import org.opendes.core.model.RecordChangedMessages;
+//import org.opendes.core.service.IndicesService;
+//import org.opendes.core.util.Config;
+//import org.opendes.core.util.ElasticClientHandler;
+//import org.opendes.core.util.ElasticIndexNameResolver;
+//import org.opendes.core.util.HeadersUtil;
+//import org.opendes.indexer.logging.AuditLogger;
+//import org.opendes.indexer.model.*;
+//import org.opendes.indexer.publish.IPublisher;
+//import org.opendes.indexer.util.IRequestInfo;
+//import org.opendes.indexer.util.IndexerQueueTaskBuilder;
+//import org.opendes.indexer.util.JobStatus;
+//import org.opendes.indexer.util.RecordInfo;
+//import org.powermock.core.classloader.annotations.PrepareForTest;
+//import javax.inject.Inject;
+//import org.springframework.context.annotation.Lazy;
+//import org.springframework.test.context.junit4.SpringRunner;
+//
+//import java.io.IOException;
+//import java.lang.reflect.Type;
+//import java.util.*;
+//
+//import static java.util.Collections.singletonList;
+//import static org.junit.Assert.*;
+//import static org.mockito.Matchers.any;
+//import static org.mockito.Mockito.verify;
+//import static org.mockito.Mockito.when;
+//import static org.powermock.api.mockito.PowerMockito.mock;
+//import static org.powermock.api.mockito.PowerMockito.mockStatic;
+//
+//@Ignore
+//@RunWith(SpringRunner.class)
+//@PrepareForTest({RestHighLevelClient.class, BulkResponse.class, StorageAcl.class, HeadersUtil.class, Config.class})
+//public class IndexerServiceTest {
+//
+// private final String pubsubMsg = "[{\"id\":\"tenant1:doc:test1\",\"kind\":\"tenant1:testindexer1:well:1.0.0\",\"op\":\"update\"}," +
+// "{\"id\":\"tenant1:doc:test2\",\"kind\":\"tenant1:testindexer2:well:1.0.0\",\"op\":\"create\"}]";
+// private final String kind1 = "tenant1:testindexer1:well:1.0.0";
+// private final String kind2 = "tenant1:testindexer2:well:1.0.0";
+// private final String recordId1 = "tenant1:doc:test1";
+// private final String recordId2 = "tenant1:doc:test2";
+// private final String failureMassage = "test failure";
+//
+// @Mock
+// private IndexSchemaService indexSchemaService;
+// @Mock
+// private IndicesService indicesService;
+// @Mock
+// private IndexerMappingService indexerMappingService;
+// @Mock
+// private StorageService storageService;
+// @Mock
+// private IPublisher publisherImpl;
+// @Mock
+// private RestHighLevelClient restHighLevelClient;
+// @Mock
+// private ElasticClientHandler elasticClientHandler;
+// @Mock
+// private BulkResponse bulkResponse;
+// @Mock
+// private IRequestInfo requestInfo;
+// @Mock
+// private ElasticIndexNameResolver elasticIndexNameResolver;
+// @Mock
+// private AttributeParsingServiceImpl attributeParsingServiceImpl;
+// @Mock
+// private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
+// @Mock
+// private JaxRsDpsLog log;
+// @Mock
+// private AuditLogger auditLogger;
+// @InjectMocks
+// private IndexerServiceImpl sut;
+// @InjectMocks @Spy
+// private JobStatus jobStatus = new JobStatus();
+//
+// @Inject
+// @Lazy
+// private DpsHeaders dpsHeaders;
+// private RecordChangedMessages recordChangedMessages;
+// private List<RecordInfo> recordInfos;
+//
+// @Before
+// public void setup() throws IOException {
+//
+// mockStatic(StorageAcl.class);
+// mockStatic(Config.class);
+//
+// when(Config.getDeploymentEnvironment()).thenReturn(DeploymentEnvironment.LOCAL);
+// when(Config.getElasticClusterName()).thenReturn("CLUSTER");
+// when(Config.getElasticServerAddress()).thenReturn("testsite");
+//
+// dpsHeaders = new DpsHeaders();
+// dpsHeaders.put(AppEngineHeaders.TASK_QUEUE_RETRY_COUNT, "1");
+// dpsHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
+// when(requestInfo.getHeaders()).thenReturn(dpsHeaders);
+// when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(dpsHeaders.getHeaders());
+//
+// Type listType = new TypeToken<List<RecordInfo>>() {}.getType();
+// recordInfos = (new Gson()).fromJson(pubsubMsg, listType);
+//
+// when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+// when(restHighLevelClient.bulk(any(), any(RequestOptions.class))).thenReturn(bulkResponse);
+//
+// BulkItemResponse[] responses = new BulkItemResponse[]{prepareResponseFail(), prepareResponseSuccess()};
+// when(bulkResponse.getItems()).thenReturn(responses);
+// Map<String, String> attr = new HashMap<>();
+// attr.put(DpsHeaders.ACCOUNT_ID, "slb");
+// recordChangedMessages = RecordChangedMessages.builder().attributes(attr).messageId("xxxx").publishTime("2000-01-02T10:10:44+0000").data("{}").build();
+// when(StorageAcl.flattenAcl(any())).thenReturn(null);
+// }
+//
+// @Test
+// public void should_returnNull_givenEmptyJobSubInfo_processRecordChangedMessageTest() throws Exception {
+// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, new ArrayList<>());
+//
+// assertNull(jobStatus);
+// }
+//
+// @Test
+// public void should_returnValidJobStatus_givenNullSchema_processRecordChangedMessageTest() {
+// try {
+// indexSchemaServiceMock(kind1, null);
+// indexSchemaServiceMock(kind2, null);
+// List<ConversionStatus> conversionStatus = new LinkedList<>();
+// List<Records.Entity> validRecords = new ArrayList<>();
+// Map<String, Object> storageData = new HashMap<>();
+// storageData.put("schema1", "test-value");
+// storageData.put("schema2", "test-value");
+// storageData.put("schema3", "test-value");
+// storageData.put("schema4", "test-value");
+// storageData.put("schema5", "test-value");
+// storageData.put("schema6", "test-value");
+// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build());
+// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build();
+//
+// when(storageService.getStorageRecords(any())).thenReturn(storageRecords);
+// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true);
+//
+// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
+//
+// assertEquals(2, jobStatus.getStatusesList().size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).size());
+// } catch (Exception e) {
+// fail("Should not throw this exception" + e.getMessage());
+// }
+// }
+//
+// @Test
+// public void should_returnValidJobStatus_givenFailedUnitsConversion_processRecordChangedMessageTest() {
+// try {
+// indexSchemaServiceMock(kind1, null);
+// indexSchemaServiceMock(kind2, null);
+// List<ConversionStatus> conversionStatuses = new LinkedList<>();
+// List<String> status=new ArrayList<>();
+// status.add("crs bla bla");
+// ConversionStatus conversionStatus=ConversionStatus.builder().status("ERROR").errors(status).id(recordId2).build();
+// conversionStatuses.add(conversionStatus);
+// List<Records.Entity> validRecords = new ArrayList<>();
+// Map<String, Object> storageData = new HashMap<>();
+// storageData.put("schema1", "test-value");
+// storageData.put("schema2", "test-value");
+// storageData.put("schema3", "test-value");
+// storageData.put("schema4", "test-value");
+// storageData.put("schema5", "test-value");
+// storageData.put("schema6", "test-value");
+// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build());
+// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatuses).build();
+//
+// when(storageService.getStorageRecords(any())).thenReturn(storageRecords);
+// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true);
+//
+// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
+//
+// assertEquals(2, jobStatus.getStatusesList().size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).size());
+// assertTrue(jobStatus.getJobStatusByRecordId(jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).get(0)).getIndexProgress().getTrace().contains("crs bla bla"));
+// } catch (Exception e) {
+// fail("Should not throw this exception" + e.getMessage());
+// }
+// }
+//
+// @Test
+// public void should_returnValidJobStatus_givenNullSchemaForARecord_processRecordChangedMessageTest() {
+// try {
+// List<Records.Entity> validRecords = new ArrayList<>();
+// List<ConversionStatus> conversionStatus = new LinkedList<>();
+// Map<String, Object> storageData = new HashMap<>();
+// storageData.put("schema1", "test-value");
+// storageData.put("schema2", "test-value");
+// storageData.put("schema3", "test-value");
+// storageData.put("schema4", "test-value");
+// storageData.put("schema5", "test-value");
+// storageData.put("schema6", "test-value");
+// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build());
+// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build();
+// when(storageService.getStorageRecords(any())).thenReturn(storageRecords);
+//
+// Map<String, String> schema = createSchema();
+// indexSchemaServiceMock(kind1, schema);
+// indexSchemaServiceMock(kind2, null);
+// when(elasticIndexNameResolver.getIndexNameFromKind(kind2)).thenReturn("tenant1-testindexer2-well-1.0.0");
+// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true);
+// JobStatus jobStatus = sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
+//
+// assertEquals(2, jobStatus.getStatusesList().size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).size());
+// assertEquals("Indexed Successfully", jobStatus.getStatusesList().get(1).getIndexProgress().getTrace().pop());
+// assertEquals("schema not found", jobStatus.getStatusesList().get(1).getIndexProgress().getTrace().pop());
+// } catch (Exception e) {
+// fail("Should not throw this exception" + e.getMessage());
+// }
+// }
+//
+// @Test
+// public void should_returnValidJobStatus_givenValidCreateAndUpdateRecords_processRecordChangedMessagesTest() {
+// try {
+// Map<String, Object> storageData = new HashMap<>();
+// storageData.put("schema1", "test-value");
+// List<ConversionStatus> conversionStatus = new LinkedList<>();
+// List<Records.Entity> validRecords = new ArrayList<>();
+// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build());
+// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build();
+//
+// when(storageService.getStorageRecords(any())).thenReturn(storageRecords);
+// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true);
+// Map<String, String> schema = createSchema();
+// indexSchemaServiceMock(kind2, schema);
+// indexSchemaServiceMock(kind1, null);
+// JobStatus jobStatus = sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
+//
+// assertEquals(2, jobStatus.getStatusesList().size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.SUCCESS).size());
+// } catch (Exception e) {
+// fail("Should not throw this exception" + e.getMessage());
+// }
+// }
+//
+// @Test
+// public void should_properlyUpdateAuditLogs_givenValidCreateAndUpdateRecords() {
+// try {
+// Map<String, Object> storageData = new HashMap<>();
+// List<ConversionStatus> conversionStatus = new LinkedList<>();
+//
+// storageData.put("schema1", "test-value");
+// List<Records.Entity> validRecords = new ArrayList<>();
+// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build());
+// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build();
+//
+// when(this.storageService.getStorageRecords(any())).thenReturn(storageRecords);
+// when(this.indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true);
+// Map<String, String> schema = createSchema();
+// indexSchemaServiceMock(kind2, schema);
+// indexSchemaServiceMock(kind1, null);
+// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
+//
+// assertEquals(2, jobStatus.getStatusesList().size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.SUCCESS).size());
+//
+// verify(this.auditLogger).indexCreateRecordSuccess(singletonList("RecordStatus(id=tenant1:doc:test2, kind=tenant1:testindexer2:well:1.0.0, operationType=create, status=SUCCESS)"));
+// verify(this.auditLogger).indexUpdateRecordFail(singletonList("RecordStatus(id=tenant1:doc:test1, kind=tenant1:testindexer1:well:1.0.0, operationType=update, status=FAIL)"));
+// } catch (Exception e) {
+// fail("Should not throw this exception" + e.getMessage());
+// }
+// }
+//
+// private BulkItemResponse prepareResponseFail() {
+// BulkItemResponse responseFail = mock(BulkItemResponse.class);
+// when(responseFail.isFailed()).thenReturn(true);
+// when(responseFail.getFailureMessage()).thenReturn(failureMassage);
+// when(responseFail.getId()).thenReturn(recordId1);
+// when(responseFail.getFailure()).thenReturn(new BulkItemResponse.Failure("failure index", "failure type", "failure id", new Exception("test failure")));
+// return responseFail;
+// }
+//
+// private BulkItemResponse prepareResponseSuccess() {
+// BulkItemResponse responseSuccess = mock(BulkItemResponse.class);
+// when(responseSuccess.getId()).thenReturn(recordId2);
+// return responseSuccess;
+// }
+//
+// private void indexSchemaServiceMock(String kind, Map<String, String> schema) {
+// if (schema == null) {
+// IndexSchema indexSchema = IndexSchema.builder().kind(kind).dataSchema(null).build();
+// when(indexSchemaService.getIndexerInputSchema(kind)).thenReturn(indexSchema);
+// } else {
+// IndexSchema indexSchema = IndexSchema.builder().kind(kind).dataSchema(schema).build();
+// when(indexSchemaService.getIndexerInputSchema(kind)).thenReturn(indexSchema);
+// }
+// }
+//
+// private Map<String, String> createSchema() {
+// Map<String, String> schema = new HashMap<>();
+// schema.put("schema1", "keyword");
+// schema.put("schema2", "boolean");
+// schema.put("schema3", "date");
+// schema.put("schema6", "object");
+// return schema;
+// }
+//}
diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..502699c256cc6b3eb0301e175dacb320ac9cce12
--- /dev/null
+++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java
@@ -0,0 +1,154 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.service;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
+import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.opengroup.osdu.core.common.search.Config;
+import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+import org.powermock.modules.junit4.PowerMockRunnerDelegate;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.util.*;
+
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.MockitoAnnotations.initMocks;
+import static org.powermock.api.mockito.PowerMockito.mockStatic;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@RunWith(PowerMockRunner.class)
+@PowerMockRunnerDelegate(SpringRunner.class)
+@PrepareForTest({Config.class})
+public class ReindexServiceTest {
+
+ private final String cursor = "100";
+
+ private final String correlationId = UUID.randomUUID().toString();
+
+ @Mock
+ private StorageService storageService;
+ @Mock
+ private IRequestInfo requestInfo;
+ @Mock
+ private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
+ @Mock
+ private JaxRsDpsLog log;
+ @InjectMocks
+ private ReindexServiceImpl sut;
+
+ private RecordReindexRequest recordReindexRequest;
+ private RecordQueryResponse recordQueryResponse;
+
+ private Map<String, String> httpHeaders;
+
+ @Before
+ public void setup() {
+ initMocks(this);
+
+ mockStatic(UUID.class);
+
+ recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor(cursor).build();
+ recordQueryResponse = new RecordQueryResponse();
+
+ httpHeaders = new HashMap<>();
+ httpHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
+ httpHeaders.put(DpsHeaders.CORRELATION_ID, correlationId);
+ DpsHeaders standardHeaders = DpsHeaders.createFromMap(httpHeaders);
+ when(requestInfo.getHeaders()).thenReturn(standardHeaders);
+ when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(httpHeaders);
+ when(requestInfo.getHeadersWithDwdAuthZ()).thenReturn(standardHeaders);
+ }
+
+ @Test
+ public void should_returnNull_givenNullResponseResult_reIndexRecordsTest() {
+ try {
+ recordQueryResponse.setResults(null);
+ when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
+
+ String response = sut.reindexRecords(recordReindexRequest, false);
+
+ Assert.assertNull(response);
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_returnNull_givenEmptyResponseResult_reIndexRecordsTest() {
+ try {
+ recordQueryResponse.setResults(new ArrayList<>());
+ when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
+
+ String response = sut.reindexRecords(recordReindexRequest, false);
+
+ Assert.assertNull(response);
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+ @Ignore
+ @Test
+ public void should_returnRecordQueryRequestPayload_givenValidResponseResult_reIndexRecordsTest() {
+ try {
+ recordQueryResponse.setCursor(cursor);
+ List<String> results = new ArrayList<>();
+ results.add("test1");
+ recordQueryResponse.setResults(results);
+
+ mockStatic(Config.class);
+ when(Config.getStorageRecordsBatchSize()).thenReturn(1);
+
+ when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
+
+ String taskQueuePayload = sut.reindexRecords(recordReindexRequest, false);
+
+ Assert.assertEquals("{\"kind\":\"tenant:test:test:1.0.0\",\"cursor\":\"100\"}", taskQueuePayload);
+ } catch (Exception e) {
+ fail("Should not throw exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_returnRecordChangedMessage_givenValidResponseResult_reIndexRecordsTest() {
+ try {
+ List<String> results = new ArrayList<>();
+ results.add("test1");
+ recordQueryResponse.setResults(results);
+ when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
+
+ String taskQueuePayload = sut.reindexRecords(recordReindexRequest, false);
+
+ Assert.assertEquals(String.format("{\"data\":\"[{\\\"id\\\":\\\"test1\\\",\\\"kind\\\":\\\"tenant:test:test:1.0.0\\\",\\\"op\\\":\\\"create\\\"}]\",\"attributes\":{\"correlation-id\":\"%s\"}}", correlationId), taskQueuePayload);
+ } catch (Exception e) {
+ fail("Should not throw exception" + e.getMessage());
+ }
+ }
+}
diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..e8388a141ee52a42bcdd1d0db27025f92ccbc78e
--- /dev/null
+++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceTest.java
@@ -0,0 +1,251 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.service;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentMatchers;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.indexer.IndexingStatus;
+import org.opengroup.osdu.core.common.model.indexer.JobStatus;
+import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
+import org.opengroup.osdu.core.common.model.http.HttpResponse;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.opengroup.osdu.core.common.http.IUrlFetchService;
+import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
+import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
+import org.opengroup.osdu.core.common.model.indexer.Records;
+import org.springframework.http.HttpStatus;
+import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import java.lang.reflect.Type;
+import java.net.URISyntaxException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+
+import static java.util.Collections.singletonList;
+import static org.junit.Assert.*;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@RunWith(SpringRunner.class)
+public class StorageServiceTest {
+
+ @Mock
+ private IUrlFetchService urlFetchService;
+ @Mock
+ private JobStatus jobStatus;
+ @Mock
+ private JaxRsDpsLog log;
+ @Mock
+ private IRequestInfo requestInfo;
+ @InjectMocks
+ private StorageServiceImpl sut;
+
+ private List<String> ids;
+ private static final String RECORD_ID1 = "tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465";
+ private static final String RECORDS_ID2 = "tenant1:doc:15e790a69beb4d789b1f979e2af2e813";
+
+ @Before
+ public void setup() {
+
+ String recordChangedMessages = "[{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"purge\"}," +
+ "{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"create\"}]";
+
+ when(this.requestInfo.getHeadersMap()).thenReturn(new HashMap<>());
+ when(this.requestInfo.getHeaders()).thenReturn(new DpsHeaders());
+
+ Type listType = new TypeToken<List<RecordInfo>>() {}.getType();
+
+ List<RecordInfo> msgs = (new Gson()).fromJson(recordChangedMessages, listType);
+ jobStatus.initialize(msgs);
+ ids = Arrays.asList(RECORD_ID1, RECORDS_ID2);
+
+ ReflectionTestUtils.setField(this.sut, "STORAGE_RECORDS_BATCH_SIZE", "20");
+ }
+
+ @Test
+ public void should_return404_givenNullData_getValidStorageRecordsTest() throws URISyntaxException {
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(null);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ should_return404_getValidStorageRecordsTest();
+ }
+
+ @Test
+ public void should_return404_givenEmptyData_getValidStorageRecordsTest() throws URISyntaxException {
+
+ String emptyDataFromStorage = "{\"records\":[],\"notFound\":[]}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(emptyDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ should_return404_getValidStorageRecordsTest();
+ }
+
+ @Test
+ public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsTest() throws URISyntaxException {
+
+ String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"], \"conversionStatuses\": []}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+ Records storageRecords = this.sut.getStorageRecords(ids);
+
+ assertEquals(1, storageRecords.getRecords().size());
+ }
+
+ @Test
+ public void should_logMissingRecord_given_storageMissedRecords() throws URISyntaxException {
+
+ String validDataFromStorage = "{\"records\":[{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[]}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(any())).thenReturn(httpResponse);
+ Records storageRecords = this.sut.getStorageRecords(ids);
+
+ assertEquals(1, storageRecords.getRecords().size());
+ verify(this.jobStatus).addOrUpdateRecordStatus(singletonList(RECORDS_ID2), IndexingStatus.FAIL, HttpStatus.NOT_FOUND.value(), "Partial response received from Storage service - missing records", "Partial response received from Storage service: tenant1:doc:15e790a69beb4d789b1f979e2af2e813");
+ }
+
+ @Test
+ public void should_returnValidJobStatus_givenFailedUnitsConversion_processRecordChangedMessageTest() throws URISyntaxException {
+ String validDataFromStorage = "{\"records\":[{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[],\"conversionStatuses\":[{\"id\":\"tenant1:doc:15e790a69beb4d789b1f979e2af2e813\",\"status\":\"ERROR\",\"errors\":[\"crs conversion failed\"]}]}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(any())).thenReturn(httpResponse);
+ Records storageRecords = this.sut.getStorageRecords(singletonList(RECORDS_ID2));
+
+ assertEquals(1, storageRecords.getRecords().size());
+ verify(this.jobStatus).addOrUpdateRecordStatus(RECORDS_ID2, IndexingStatus.WARN, HttpStatus.BAD_REQUEST.value(), "crs conversion failed", String.format("record-id: %s | %s", "tenant1:doc:15e790a69beb4d789b1f979e2af2e813", "crs conversion failed"));
+ }
+
+ @Test
+ public void should_returnValidResponse_givenValidRecordQueryRequest_getRecordListByKind() throws Exception {
+
+ RecordReindexRequest recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor("100").build();
+
+ HttpResponse httpResponse = new HttpResponse();
+ httpResponse.setBody(new Gson().toJson(recordReindexRequest, RecordReindexRequest.class));
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ RecordQueryResponse recordQueryResponse = this.sut.getRecordsByKind(recordReindexRequest);
+
+ assertEquals("100", recordQueryResponse.getCursor());
+ assertNull(recordQueryResponse.getResults());
+ }
+
+ @Test
+ public void should_returnValidResponse_givenValidKind_getSchemaByKind() throws Exception {
+
+ String validSchemaFromStorage = "{" +
+ " \"kind\": \"tenant:test:test:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"msg\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"references.entity\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]," +
+ " \"ext\": null" +
+ "}";
+ String kind = "tenant:test:test:1.0.0";
+
+ HttpResponse httpResponse = new HttpResponse();
+ httpResponse.setResponseCode(HttpStatus.OK.value());
+ httpResponse.setBody(validSchemaFromStorage);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ String recordSchemaResponse = this.sut.getStorageSchema(kind);
+
+ assertNotNull(recordSchemaResponse);
+ }
+
+ @Test
+ public void should_returnNullResponse_givenAbsentKind_getSchemaByKind() throws Exception {
+
+ String kind = "tenant:test:test:1.0.0";
+
+ HttpResponse httpResponse = new HttpResponse();
+ httpResponse.setResponseCode(HttpStatus.NOT_FOUND.value());
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ String recordSchemaResponse = this.sut.getStorageSchema(kind);
+
+ assertNull(recordSchemaResponse);
+ }
+
+ @Test
+ public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsWithInvalidConversionTest() throws URISyntaxException {
+
+ String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"],\"conversionStatuses\": [{\"id\":\"testid\",\"status\":\"ERROR\",\"errors\":[\"conversion error occurred\"] } ]}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any())).thenReturn(httpResponse);
+ Records storageRecords = this.sut.getStorageRecords(ids);
+
+ assertEquals(1, storageRecords.getRecords().size());
+
+ assertEquals(1, storageRecords.getConversionStatuses().get(0).getErrors().size());
+
+ assertEquals("conversion error occurred", storageRecords.getConversionStatuses().get(0).getErrors().get(0));
+ }
+
+ private void should_return404_getValidStorageRecordsTest() {
+ try {
+ this.sut.getStorageRecords(ids);
+ fail("Should throw exception");
+ } catch (AppException e) {
+ assertEquals(HttpStatus.NOT_FOUND.value(), e.getError().getCode());
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+}
diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImplTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImplTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..58dbef93eb28c9c3c471bf47193d883964fd831a
--- /dev/null
+++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/util/ServiceAccountJwtGcpClientImplTest.java
@@ -0,0 +1,171 @@
+/*
+ * Copyright 2020 Google LLC
+ * Copyright 2020 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.util;
+
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.javanet.NetHttpTransport;
+import com.google.api.services.iam.v1.Iam;
+import com.google.api.services.iam.v1.model.SignJwtResponse;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Spy;
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.search.DeploymentEnvironment;
+import org.opengroup.osdu.core.common.model.search.IdToken;
+import org.opengroup.osdu.core.common.provider.interfaces.IJwtCache;
+import org.opengroup.osdu.core.common.search.Config;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.MockitoAnnotations.initMocks;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@Ignore
+@RunWith(SpringRunner.class)
+@PrepareForTest({GoogleNetHttpTransport.class, GoogleCredential.class, NetHttpTransport.class, SignJwtResponse.class, Iam.Builder.class, HttpClients.class, EntityUtils.class, Config.class})
+public class ServiceAccountJwtGcpClientImplTest {
+
+ private static final String JWT_TOKEN = "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Ik1UVXlPREE0TXpFd09BPT0ifQ.eyJzdWIiOiJtemh1OUBzbGIuY29tIiwiaXNzIjoic2F1dGgtcHJldmlldy5zbGIuY29tIiwiYXVkIjoidGVzdC1zbGJkZXYtZGV2cG9ydGFsLnNsYmFwcC5jb20iLCJpYXQiOjE1MjgxNDg5MTUsImV4cCI6MTUyODIzNTMxNSwicHJvdmlkZXIiOiJzbGIuY29tIiwiY2xpZW50IjoidGVzdC1zbGJkZXYtZGV2cG9ydGFsLnNsYmFwcC5jb20iLCJ1c2VyaWQiOiJtemh1OUBzbGIuY29tIiwiZW1haWwiOiJtemh1OUBzbGIuY29tIiwiYXV0aHoiOiJ7XCJhY2NvdW50Q291bnRyeVwiOntcImNvZGVcIjpcInVzXCIsXCJpZFwiOjU3MTU5OTkxMDE4MTI3MzYsXCJuYW1lXCI6XCJVbml0ZWQgU3RhdGVzIG9mIEFtZXJpY2FcIn0sXCJhY2NvdW50SWRcIjo1NjkxODc4ODMzOTEzODU2LFwiYWNjb3VudE5hbWVcIjpcIlNJUyBJbnRlcm5hbCBIUVwiLFwiY3JlYXRlZFwiOlwiMjAxOC0wNS0wM1QxNzoyNTo1NS40NDNaXCIsXCJkZXBhcnRtZW50TWFuYWdlclwiOm51bGwsXCJzdWJzY3JpcHRpb25zXCI6W3tcImFjY291bnRJZFwiOjU2OTE4Nzg4MzM5MTM4NTYsXCJjb250cmFjdElkXCI6NTc1MTcwMDIxMjE1NDM2OCxcImNyZWF0ZWRcIjpcIjIwMTgtMDUtMDNUMTc6MzM6MDkuNTczWlwiLFwiY3JtQ29udHJhY3RJZFwiOlwiU0lTLUlOVEVSTkFMLUhRLVFBXCIsXCJjcm1Db250cmFjdEl0ZW1JZFwiOlwiZGV2bGlcIixcImV4cGlyYXRpb25cIjpcIjE5NzAtMDEtMDFUMDA6MDA6MDAuMDAwWlwiLFwiaWRcIjo1MDc5Mjg4NTA0MTIzMzkyLFwicHJvZHVjdFwiOntcImNvZGVcIjpcImRldmVsb3Blci1saWdodFwiLFwiY29tY2F0TmFtZVwiOlwiTm90IGluIENvbUNhdFwiLFwiZmVhdHVyZVNldHNcIjpbe1wiYXBwbGljYXRpb25cIjp7XCJjb2RlXCI6XCJhcGlkZXZlbG9wZXJwb3J0YWxcIixcImlkXCI6NTE2ODkzMDY5NTkzODA0OCxcIm5hbWVcIjpcIkFQSSBEZXZlbG9wZXIgUG9ydGFsXCIsXCJ0eXBlXCI6XCJXZWJBcHBcIn0sXCJjbGFpbXNcIjpudWxsLFwiaWRcIjo1MTkxNTcyMjg3MTI3NTUyLFwibmFtZVwiOlwiRGV2ZWxvcGVyXCIsXCJ0eXBlXCI6XCJCQVNFXCJ9XSxcImlkXCI6NTE1MDczMDE1MTI2NDI1NixcIm5hbWVcIjpcIkRldmVsb3BlciBQb3J0YWxcIixcInBhcnROdW1iZXJcIjpcIlNERUwtUEItU1VCVVwifX1dLFwidXNlckVtYWlsXCI6XCJtemh1OUBzbGIuY29tXCIsXCJ1c2VyTmFtZVwiOlwiTWluZ3lhbmcgWmh1XCJ9XG4iLCJsYXN0bmFtZSI6IlpodSIsImZpcnN0bmFtZSI6Ik1pbmd5YW5nIiwiY291bnRyeSI6IiIsImNvbXBhbnkiOiIiLCJqb2J0aXRsZSI6IiIsInN1YmlkIjoiNDE3YjczMjktYmMwNy00OTFmLWJiYzQtZTQ1YjRhMWFiYjVjLVd3U0c0dyIsImlkcCI6ImNvcnAyIiwiaGQiOiJzbGIuY29tIn0.WQfGr1Xu-6IdaXdoJ9Fwzx8O2el1UkFPWo1vk_ujiAfdOjAR46UG5SrBC7mzC7gYRyK3a4fimBmbv3uRVJjTNXdxXRLZDw0SvXUMIOqjUGLom491ESbrtka_Xz7vGO-tWyDcEQDTfFzQ91LaVN7XdzL18_EDTXZoPhKb-zquyk9WLQxP9Mw-3Yh-UrbvC9nl1-GRn1IVbzp568kqkpOVUFM9alYSGw-oMGDZNt1DIYOJnpGaw2RB5B3AKvNivZH_Xdac7ZTzQbsDOt8B8DL2BphuxcJ9jshCJkM2SHQ15uErv8sfnzMwdF08e_0QcC_30I8eX9l8yOu6TnwwqlXunw";
+
+ @Mock
+ private JaxRsDpsLog log;
+ @Mock
+ private GoogleCredential credential;
+ @Mock
+ private NetHttpTransport httpTransport;
+ @Mock
+ private SignJwtResponse signJwtResponse;
+ @Mock
+ private Iam iam;
+ @Mock
+ private Iam.Projects iamProject;
+ @Mock
+ private Iam.Projects.ServiceAccounts iamProjectServiceAccounts;
+ @Mock
+ private Iam.Projects.ServiceAccounts.SignJwt signJwt;
+ @Mock
+ private CloseableHttpClient httpClient;
+ @Mock
+ private CloseableHttpResponse httpResponse;
+// @InjectMocks
+// private TenantInfoServiceImpl tenantInfoServiceProvider;
+// @Mock
+// private TenantInfoServiceImpl tenantInfoService;
+ @Mock
+ private IJwtCache cacheService;
+ @InjectMocks @Spy
+ private ServiceAccountJwtGcpClientImpl sut;
+ @Before
+ public void setup() throws Exception {
+ initMocks(this);
+
+// mockStatic(GoogleNetHttpTransport.class);
+// mockStatic(GoogleCredential.class);
+// mockStatic(HttpClients.class);
+// mockStatic(EntityUtils.class);
+// mockStatic(Config.class);
+
+ when(GoogleNetHttpTransport.newTrustedTransport()).thenReturn(httpTransport);
+ when(GoogleCredential.getApplicationDefault()).thenReturn(credential);
+ when(credential.createScopedRequired()).thenReturn(true);
+ when(credential.createScoped(any())).thenReturn(credential);
+ when(HttpClients.createDefault()).thenReturn(httpClient);
+ when(httpClient.execute(any())).thenReturn(httpResponse);
+ when(Config.getDeploymentEnvironment()).thenReturn(DeploymentEnvironment.LOCAL);
+ when(Config.getGoogleAudiences()).thenReturn("aud");
+
+// when(this.tenantInfoServiceProvider).thenReturn(this.tenantInfoService);
+
+ TenantInfo tenantInfo = new TenantInfo();
+ tenantInfo.setServiceAccount("tenant");
+// when(this.tenantInfoService.getTenantInfo()).thenReturn(tenantInfo);
+
+ when(this.sut.getIam()).thenReturn(iam);
+ when(this.iam.projects()).thenReturn(iamProject);
+ when(this.iamProject.serviceAccounts()).thenReturn(iamProjectServiceAccounts);
+ when(this.iamProjectServiceAccounts.signJwt(any(), any())).thenReturn(signJwt);
+ when(this.signJwt.execute()).thenReturn(signJwtResponse);
+ when(this.signJwtResponse.getSignedJwt()).thenReturn("testJwt");
+
+ }
+
+ @Test
+ public void should_returnCachedToken_givenCachedToken_getIdTokenTest() {
+ String tokenValue = "tokenValue";
+ IdToken idToken = IdToken.builder().tokenValue(tokenValue).expirationTimeMillis(System.currentTimeMillis() + 10000000L).build();
+ when(this.cacheService.get(any())).thenReturn(idToken);
+
+ String returnedIdToken = this.sut.getIdToken(tokenValue);
+
+ Assert.assertEquals(tokenValue, returnedIdToken);
+ }
+
+ @Test
+ public void should_returnValidToken_getIdTokenTest() throws Exception {
+ when(EntityUtils.toString(any())).thenReturn(String.format("{\"id_token\":\"%s\"}", JWT_TOKEN));
+
+ String returnedToken = this.sut.getIdToken("tenant");
+
+ Assert.assertEquals(JWT_TOKEN, returnedToken);
+ }
+
+ @Test
+ public void should_return500_given_invalidJWTResponse_getIdTokenException() {
+ try {
+ when(EntityUtils.toString(any())).thenReturn(String.format("{\"id_token\":\"%s\"}", "invalid jwt"));
+
+ this.sut.getIdToken("tenant");
+ fail("Should throw exception");
+ } catch (AppException e) {
+ Assert.assertEquals(HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getError().getCode());
+ Assert.assertEquals("Invalid token, error decoding", e.getError().getMessage());
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_return403_given_missingIdTokenResponse_getIdTokenException() {
+ try {
+ when(EntityUtils.toString(any())).thenReturn("{}");
+
+ this.sut.getIdToken("tenant");
+ fail("Should throw exception");
+ } catch (AppException e) {
+ Assert.assertEquals(HttpStatus.SC_FORBIDDEN, e.getError().getCode());
+ Assert.assertEquals("The user is not authorized to perform this action", e.getError().getMessage());
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+}
diff --git a/provider/indexer-reference/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/provider/indexer-reference/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker
new file mode 100644
index 0000000000000000000000000000000000000000..ca6ee9cea8ec189a088d50559325d4e84ff8ad09
--- /dev/null
+++ b/provider/indexer-reference/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker
@@ -0,0 +1 @@
+mock-maker-inline
\ No newline at end of file