diff --git a/provider/indexer-ibm/azure-build.yml b/provider/indexer-ibm/azure-build.yml
new file mode 100644
index 0000000000000000000000000000000000000000..705d5f2168a7f1c6a077d21d9c69066ed673f303
--- /dev/null
+++ b/provider/indexer-ibm/azure-build.yml
@@ -0,0 +1,55 @@
+# Maven
+# Build your Java project and run tests with Apache Maven.
+# Add steps that analyze code, save build artifacts, deploy, and more:
+# https://docs.microsoft.com/azure/devops/pipelines/languages/java
+
+trigger:
+ branches:
+ include:
+ - master
+ paths:
+ exclude:
+ - README.md
+ - .gitignore
+
+pool:
+ name: Hosted Ubuntu 1604
+ demands: maven
+
+steps:
+- task: Maven@3
+ displayName: 'build, test, code coverage'
+ inputs:
+ mavenPomFile: 'pom.xml'
+ options: '--settings ./indexer-core/maven/settings.xml -DVSTS_FEED_TOKEN=$(VSTS_FEED_TOKEN)'
+ testResultsFiles: '**/*/TEST-*.xml'
+ codeCoverageToolOption: JaCoCo
+ goals: 'install'
+
+- task: Maven@3
+ displayName: 'build, test, code coverage'
+ inputs:
+ mavenPomFile: 'pom.xml'
+ options: '--settings ./provider/indexer-azure/maven/settings.xml -DVSTS_FEED_TOKEN=$(VSTS_FEED_TOKEN) -P indexer-azure'
+ testResultsFiles: '**/*/TEST-*.xml'
+ codeCoverageToolOption: JaCoCo
+ goals: 'package'
+
+- task: CopyFiles@2
+ displayName: 'Copy Azure artifacts for maven deploy to: $(build.artifactstagingdirectory)'
+ inputs:
+ SourceFolder:
+ Contents: |
+ pom.xml
+ provider/indexer-azure/maven/settings.xml
+ provider/indexer-azure/pom.xml
+ provider/indexer-azure/target/*-spring-boot.jar
+ TargetFolder: '$(build.artifactstagingdirectory)'
+
+- task: PublishBuildArtifacts@1
+ displayName: 'Publish Artifact: drop'
+ inputs:
+ PathtoPublish: '$(build.artifactstagingdirectory)'
+ ArtifactName: 'drop'
+ publishLocation: 'Container'
+ condition: succeededOrFailed()
diff --git a/provider/indexer-ibm/maven/settings.xml b/provider/indexer-ibm/maven/settings.xml
new file mode 100644
index 0000000000000000000000000000000000000000..79c58d344efccb52d3f026e28ce38a368da983aa
--- /dev/null
+++ b/provider/indexer-ibm/maven/settings.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Copyright 2019 IBM Corp. All Rights Reserved.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
+ <servers>
+ <server>
+ <id>azure-auth</id>
+ <configuration>
+ <tenant>${AZURE_DEPLOY_TENANT}</tenant>
+ <client>${AZURE_DEPLOY_CLIENT_ID}</client>
+ <key>${AZURE_DEPLOY_CLIENT_SECRET}</key>
+ <environment>AZURE</environment>
+ </configuration>
+ </server>
+ </servers>
+</settings>
\ No newline at end of file
diff --git a/provider/indexer-ibm/pom.xml b/provider/indexer-ibm/pom.xml
new file mode 100644
index 0000000000000000000000000000000000000000..fa907326f3cd04c65cb8921f0b442527b949a6e6
--- /dev/null
+++ b/provider/indexer-ibm/pom.xml
@@ -0,0 +1,221 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Copyright 2019 IBM Corp. All Rights Reserved.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.opengroup.osdu.indexer</groupId>
+ <artifactId>indexer-service</artifactId>
+ <version>1.0.2</version>
+ <relativePath>../..</relativePath>
+ </parent>
+
+ <artifactId>indexer-ibm</artifactId>
+ <version>1.0.0</version>
+ <name>indexer-ibm</name>
+ <description>Indexer Service IBM</description>
+ <packaging>jar</packaging>
+
+ <properties>
+ <azure.version>2.1.7</azure.version>
+ <azure.appservice.resourcegroup></azure.appservice.resourcegroup>
+ <azure.appservice.plan></azure.appservice.plan>
+ <azure.appservice.appname></azure.appservice.appname>
+ <azure.appservice.subscription></azure.appservice.subscription>
+ </properties>
+
+ <dependencies>
+
+ <!-- OSDU core service dependencies -->
+ <dependency>
+ <groupId>org.opengroup.osdu</groupId>
+ <artifactId>os-core-common</artifactId>
+ <version>0.0.8</version>
+ </dependency>
+ <dependency>
+ <groupId>org.opengroup.osdu.indexer</groupId>
+ <artifactId>indexer-core</artifactId>
+ <version>1.0.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.opengroup.osdu</groupId>
+ <artifactId>os-core-lib-ibm</artifactId>
+ <version>0.1.1</version>
+ </dependency>
+
+
+
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-security</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.security.oauth</groupId>
+ <artifactId>spring-security-oauth2</artifactId>
+ <version>2.3.6.RELEASE</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-jwt</artifactId>
+ <version>1.0.10.RELEASE</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-oauth2-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-oauth2-jose</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch</artifactId>
+ <version>6.6.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-client</artifactId>
+ <version>6.6.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-high-level-client</artifactId>
+ <version>6.6.2</version>
+ </dependency>
+
+ <!-- Test Dependencies -->
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.auth0</groupId>
+ <artifactId>java-jwt</artifactId>
+ <version>3.8.1</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.12</version>
+ <scope>test</scope>
+ </dependency>
+ <!-- https://mvnrepository.com/artifact/org.powermock/powermock-api-mockito2 -->
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-mockito2</artifactId>
+ <version>2.0.2</version>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/org.powermock/powermock-module-junit4 -->
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <version>2.0.2</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <version>3.0.0</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>cobertura-maven-plugin</artifactId>
+ <version>2.7</version>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>repackage</goal>
+ </goals>
+ <configuration>
+ <classifier>spring-boot</classifier>
+ <mainClass>
+ org.opengroup.osdu.indexer.ibm.IndexerAzureApplication
+ </mainClass>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>com.microsoft.azure</groupId>
+ <artifactId>azure-webapp-maven-plugin</artifactId>
+ <version>1.7.0</version>
+ <configuration>
+ <schemaVersion>V2</schemaVersion>
+ <authentication>
+ <serverId>azure-auth</serverId>
+ </authentication>
+ <subscriptionId>${azure.appservice.subscription}</subscriptionId>
+ <resourceGroup>${azure.appservice.resourcegroup}</resourceGroup>
+ <appServicePlanName>${azure.appservice.plan}</appServicePlanName>
+ <appName>${azure.appservice.appname}</appName>
+ <appSettings>
+ <property>
+ <name>JAVA_OPTS</name>
+ <value>-Dserver.port=80</value>
+ </property>
+ </appSettings>
+ <deployment>
+ <resources>
+ <resource>
+ <directory>${project.basedir}/target</directory>
+ <includes>
+ <include>*spring-boot.jar</include>
+ </includes>
+ </resource>
+ </resources>
+ </deployment>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.4.2</version>
+ <configuration>
+ <useSystemClassLoader>false</useSystemClassLoader>
+ <threadCount>1</threadCount>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+
+</project>
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/IndexerIBMApplication.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/IndexerIBMApplication.java
new file mode 100644
index 0000000000000000000000000000000000000000..9d00f3c4fe8c822a9f8c13b5ae38271cad36d9e4
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/IndexerIBMApplication.java
@@ -0,0 +1,30 @@
+// Copyright 2019 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.Configuration;
+
+@SpringBootApplication
+@ComponentScan({"org.opengroup.osdu.core.common", "org.opengroup.osdu.indexer"})
+public class IndexerIBMApplication {
+
+ public static void main(String[] args) {
+ SpringApplication.run(IndexerIBMApplication.class, args);
+ }
+
+}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/ServletInitializer.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/ServletInitializer.java
new file mode 100644
index 0000000000000000000000000000000000000000..434f5e70980b6d220956f3de3b0e01c4a4ad8091
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/ServletInitializer.java
@@ -0,0 +1,26 @@
+// Copyright 2019 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm;
+
+import org.springframework.boot.builder.SpringApplicationBuilder;
+import org.springframework.boot.web.servlet.support.SpringBootServletInitializer;
+
+public class ServletInitializer extends SpringBootServletInitializer {
+ @Override
+ protected SpringApplicationBuilder configure(SpringApplicationBuilder application) {
+ return application.sources(IndexerIBMApplication.class);
+ }
+}
+
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/api/AADController.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/api/AADController.java
new file mode 100644
index 0000000000000000000000000000000000000000..9b4ed05b81c7b4a9c4f1e13d586c485f892745c8
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/api/AADController.java
@@ -0,0 +1,42 @@
+// Copyright 2019 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.api;
+
+import org.opengroup.osdu.indexer.ibm.util.ServiceAccountJwtClientImpl;
+
+import org.springframework.http.*;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+
+import javax.inject.Inject;
+
+@Controller
+public class AADController {
+
+ @Inject
+ ServiceAccountJwtClientImpl service;
+
+
+ @RequestMapping("/obo_api")
+ @PostMapping
+ public ResponseEntity<String> callOboApi() throws Throwable {
+
+ String token = service.getIdToken("common");
+
+ return new ResponseEntity<String>(token, HttpStatus.OK);
+ }
+}
+
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/AttributesCache.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/AttributesCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..b62709872251c5c34c2df8ac8f5986489d568b3b
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/AttributesCache.java
@@ -0,0 +1,55 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.cache;
+
+import org.opengroup.osdu.core.common.model.core.cache.VmCache;
+import org.opengroup.osdu.core.common.spi.coreis.IAttributesCache;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import java.util.Set;
+
+@Component
+public class AttributesCache implements IAttributesCache<String,Set> {
+
+ private VmCache<String, Set> cache;
+
+ public AttributesCache(@Value("${INDEX_CACHE_EXPIRATION}") final String INDEX_CACHE_EXPIRATION,
+ @Value("${MAX_CACHE_VALUE_SIZE}") final String MAX_CACHE_VALUE_SIZE) {
+ cache = new VmCache<>(Integer.parseInt(INDEX_CACHE_EXPIRATION) * 60,
+ Integer.parseInt(MAX_CACHE_VALUE_SIZE));
+ }
+
+ @Override
+ public void put(String key, Set value) {
+ this.cache.put(key, value);
+ }
+
+ @Override
+ public Set get(String key) {
+ return this.cache.get(key);
+ }
+
+ @Override
+ public void delete(String key) {
+ this.cache.delete(key);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+
+}
\ No newline at end of file
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/CosmosDBSchemaCache.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/CosmosDBSchemaCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..2ed935d6094604433dbf5064b08c2b95e12816c3
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/CosmosDBSchemaCache.java
@@ -0,0 +1,27 @@
+//// Copyright 2019 IBM Corp. All Rights Reserved.
+////
+//// Licensed under the Apache License, Version 2.0 (the "License");
+//// you may not use this file except in compliance with the License.
+//// You may obtain a copy of the License at
+////
+//// http://www.apache.org/licenses/LICENSE-2.0
+////
+//// Unless required by applicable law or agreed to in writing, software
+//// distributed under the License is distributed on an "AS IS" BASIS,
+//// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//// See the License for the specific language governing permissions and
+//// limitations under the License.
+//
+//package org.opengroup.osdu.indexer.ibm.cache;
+//
+//
+//import org.opengroup.osdu.core.common.model.core.cache.VmCache;
+//import org.opengroup.osdu.core.common.model.storage.Schema;
+//import org.springframework.stereotype.Component;
+//
+//@Component
+//public class CosmosDBSchemaCache extends VmCache<String, Schema> {
+// public CosmosDBSchemaCache() {
+// super(5 * 60, 1000);
+// }
+//}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/ElasticCredentialsCache.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/ElasticCredentialsCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..709095c67415a6f250b8ab08ffa2f5dc2790ceb2
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/ElasticCredentialsCache.java
@@ -0,0 +1,39 @@
+package org.opengroup.osdu.indexer.ibm.cache;
+
+import org.opengroup.osdu.core.common.model.core.cache.VmCache;
+import org.opengroup.osdu.core.common.model.core.ClusterSettings;
+import org.opengroup.osdu.core.common.spi.coreis.IElasticCredentialsCache;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+@Component
+public class ElasticCredentialsCache implements IElasticCredentialsCache<String, ClusterSettings> {
+
+ private VmCache<String, ClusterSettings> cache;
+
+ public ElasticCredentialsCache(@Value("${ELASTIC_CACHE_EXPIRATION}") final String ELASTIC_CACHE_EXPIRATION,
+ @Value("${MAX_CACHE_VALUE_SIZE}") final String MAX_CACHE_VALUE_SIZE) {
+ cache = new VmCache<>(Integer.parseInt(ELASTIC_CACHE_EXPIRATION) * 60,
+ Integer.parseInt(MAX_CACHE_VALUE_SIZE));
+ }
+
+ @Override
+ public void put(String s, ClusterSettings o) {
+ this.cache.put(s,o);
+ }
+
+ @Override
+ public ClusterSettings get(String s) {
+ return this.cache.get(s);
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(s);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/IndexCache.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/IndexCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..df452efe3f75ceec4289fb43661f48b8903bdc72
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/IndexCache.java
@@ -0,0 +1,37 @@
+package org.opengroup.osdu.indexer.ibm.cache;
+
+import org.opengroup.osdu.core.common.model.core.cache.VmCache;
+import org.opengroup.osdu.core.common.spi.coreis.IIndexCache;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+@Component
+public class IndexCache implements IIndexCache<String, Boolean> {
+ private VmCache<String, Boolean> cache;
+
+ public IndexCache(@Value("${INDEX_CACHE_EXPIRATION}") final String INDEX_CACHE_EXPIRATION,
+ @Value("${MAX_CACHE_VALUE_SIZE}") final String MAX_CACHE_VALUE_SIZE) {
+ cache = new VmCache<>(Integer.parseInt(INDEX_CACHE_EXPIRATION) * 60,
+ Integer.parseInt(MAX_CACHE_VALUE_SIZE));
+ }
+
+ @Override
+ public void put(String s, Boolean o) {
+ this.cache.put(s, o);
+ }
+
+ @Override
+ public Boolean get(String s) {
+ return this.cache.get(s);
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(s);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/JwtCache.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/JwtCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..170be5bcc6cea742867c5060557db1f7fff9912c
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/JwtCache.java
@@ -0,0 +1,39 @@
+package org.opengroup.osdu.indexer.ibm.cache;
+
+import org.opengroup.osdu.core.common.model.core.cache.VmCache;
+import org.opengroup.osdu.core.common.model.coreis.IdToken;
+import org.opengroup.osdu.core.common.spi.coreis.IJwtCache;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+@Component
+public class JwtCache implements IJwtCache<String, IdToken> {
+ private VmCache<String, IdToken> cache;
+
+ // Azure service account id_token can be requested only for 1 hr
+ private final static int EXPIRED_AFTER = 59;
+
+ public JwtCache(@Value("${MAX_CACHE_VALUE_SIZE}") final String MAX_CACHE_VALUE_SIZE) {
+ cache = new VmCache<>(EXPIRED_AFTER * 60, Integer.parseInt(MAX_CACHE_VALUE_SIZE));
+ }
+
+ @Override
+ public void put(String s, IdToken o) {
+ this.cache.put(s, o);
+ }
+
+ @Override
+ public IdToken get(String s) {
+ return this.cache.get(s);
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(s);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/KindsCache.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/KindsCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..7effffb32b9a665cd3d451e9a1dea12b17a3a1a6
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/KindsCache.java
@@ -0,0 +1,39 @@
+package org.opengroup.osdu.indexer.ibm.cache;
+
+import org.opengroup.osdu.core.common.model.core.cache.VmCache;
+import org.opengroup.osdu.core.common.spi.coreis.IKindsCache;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import java.util.Set;
+
+@Component
+public class KindsCache implements IKindsCache<String, Set> {
+ private VmCache<String, Set> cache;
+
+ public KindsCache(@Value("${KINDS_CACHE_EXPIRATION}") final String KINDS_CACHE_EXPIRATION,
+ @Value("${MAX_CACHE_VALUE_SIZE}") final String MAX_CACHE_VALUE_SIZE) {
+ cache = new VmCache<>(Integer.parseInt(KINDS_CACHE_EXPIRATION) * 60,
+ Integer.parseInt(MAX_CACHE_VALUE_SIZE));
+ }
+
+ @Override
+ public void put(String s, Set o) {
+ this.cache.put(s, o);
+ }
+
+ @Override
+ public Set get(String s) {
+ return this.cache.get(s);
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(s);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/SchemaCache.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/SchemaCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..7610b1d51c8ef4df20cf42cdc2f8d5935720e025
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/SchemaCache.java
@@ -0,0 +1,37 @@
+package org.opengroup.osdu.indexer.ibm.cache;
+
+import org.opengroup.osdu.core.common.model.core.cache.VmCache;
+import org.opengroup.osdu.core.common.spi.indexer.ISchemaCache;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+@Component
+public class SchemaCache implements ISchemaCache<String, String> {
+ private VmCache<String, String> cache;
+
+ public SchemaCache(@Value("${SCHEMA_CACHE_EXPIRATION}") final String SCHEMA_CACHE_EXPIRATION,
+ @Value("${MAX_CACHE_VALUE_SIZE}") final String MAX_CACHE_VALUE_SIZE) {
+ cache = new VmCache<>(Integer.parseInt(SCHEMA_CACHE_EXPIRATION) * 60,
+ Integer.parseInt(MAX_CACHE_VALUE_SIZE));
+ }
+
+ @Override
+ public void put(String s, String o) {
+ this.cache.put(s, o);
+ }
+
+ @Override
+ public String get(String s) {
+ return this.cache.get(s);
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(s);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/di/DpsLogFactory.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/di/DpsLogFactory.java
new file mode 100644
index 0000000000000000000000000000000000000000..53594660ffd1fabcc9848154703ab7b77711b067
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/di/DpsLogFactory.java
@@ -0,0 +1,35 @@
+// Copyright 2019 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.di;
+
+import org.opengroup.osdu.core.common.model.core.DpsLog;
+import org.opengroup.osdu.core.common.service.core.Log;
+import org.springframework.beans.factory.FactoryBean;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+@Component
+@RequestScope
+public class DpsLogFactory implements FactoryBean<DpsLog> {
+ @Override
+ public DpsLog getObject() throws Exception {
+ return new Log();
+ }
+
+ @Override
+ public Class<?> getObjectType() {
+ return DpsLog.class;
+ }
+}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/di/EntitlementsFactoryIBM.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/di/EntitlementsFactoryIBM.java
new file mode 100644
index 0000000000000000000000000000000000000000..3310e78b20ee7a74d85caf0725e5dd4749b336ce
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/di/EntitlementsFactoryIBM.java
@@ -0,0 +1,30 @@
+//// Copyright 2019 IBM Corp. All Rights Reserved.
+////
+//// Licensed under the Apache License, Version 2.0 (the "License");
+//// you may not use this file except in compliance with the License.
+//// You may obtain a copy of the License at
+////
+//// http://www.apache.org/licenses/LICENSE-2.0
+////
+//// Unless required by applicable law or agreed to in writing, software
+//// distributed under the License is distributed on an "AS IS" BASIS,
+//// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//// See the License for the specific language governing permissions and
+//// limitations under the License.
+//
+//package org.opengroup.osdu.indexer.ibm.di;
+//
+//import org.opengroup.osdu.core.common.model.core.DpsHeaders;
+//import org.opengroup.osdu.core.common.service.core.entitlements.IEntitlementsFactory;
+//import org.opengroup.osdu.core.common.service.core.entitlements.IEntitlementsService;
+//import org.springframework.context.annotation.Primary;
+//import org.springframework.stereotype.Component;
+//
+//@Component
+//@Primary
+//public class EntitlementsFactoryIBM implements IEntitlementsFactory {
+// @Override
+// public IEntitlementsService create(DpsHeaders dpsHeaders) {
+// return new EntitlementsServiceIBM(dpsHeaders);
+// }
+//}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/di/EntitlementsServiceIBM.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/di/EntitlementsServiceIBM.java
new file mode 100644
index 0000000000000000000000000000000000000000..e9fbe77dd0c8675ef4ac451e2ec18ce955084119
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/di/EntitlementsServiceIBM.java
@@ -0,0 +1,115 @@
+// Copyright 2019 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.di;
+
+import org.apache.http.HttpStatus;
+
+import org.opengroup.osdu.core.common.model.core.DpsHeaders;
+import org.opengroup.osdu.core.common.model.core.entitlements.*;
+import org.opengroup.osdu.core.common.service.core.entitlements.IEntitlementsService;
+import org.opengroup.osdu.core.common.service.core.HttpResponse;
+import org.springframework.security.core.Authentication;
+import org.springframework.security.core.GrantedAuthority;
+import org.springframework.security.core.context.SecurityContextHolder;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+public class EntitlementsServiceIBM implements IEntitlementsService {
+ //service principals don't have UPN or email
+ static final String INTEGRATION_TEST_ADMIN_ROLE = "data.test.admin@opendes.ibm.com";
+ public static final String PREFIX = "ROLE_";
+ DpsHeaders headers;
+
+ public EntitlementsServiceIBM(DpsHeaders headers){
+ this.headers = headers;
+ }
+
+ @Override
+ public MemberInfo addMember(GroupEmail groupEmail, MemberInfo memberInfo) throws EntitlementsException {
+ return null;
+ }
+
+ @Override
+ public Members getMembers(GroupEmail groupEmail, GetMembers getMembers) throws EntitlementsException {
+ return null;
+ }
+
+ @Override
+ public Groups getGroups() throws EntitlementsException {
+ //TODO implement
+ return null;
+ }
+
+// @Override
+// public Groups getGroups() throws EntitlementsException {
+// final Authentication auth = SecurityContextHolder.getContext().getAuthentication();
+// final UserPrincipal current = (UserPrincipal) auth.getPrincipal();
+// String email = current.getUpn();
+//
+// List<GroupInfo> giList = new ArrayList();
+// Collection<? extends GrantedAuthority> authorities = auth.getAuthorities();
+// for(GrantedAuthority authority : authorities)
+// {
+// GroupInfo gi = new GroupInfo();
+// String role = authority.getAuthority();
+// if (role.startsWith(PREFIX)){
+// role = role.substring(PREFIX.length());
+// }
+// gi.setName(role);
+// if ((email == null || email.isEmpty()) && role.equalsIgnoreCase(INTEGRATION_TEST_ADMIN_ROLE)) {
+// email = INTEGRATION_TEST_ADMIN_ROLE;
+// gi.setEmail(email);
+// giList.add(0, gi);
+// }
+// else {
+// gi.setEmail(email);
+// giList.add(gi);
+// }
+// }
+// if (giList.size() > 0)
+// {
+// Groups groups = new Groups();
+// groups.setGroups(giList);
+// groups.setDesId(email);
+// return groups;
+// }
+//
+// HttpResponse response = new HttpResponse();
+// response.setResponseCode(HttpStatus.SC_INTERNAL_SERVER_ERROR);
+// throw new EntitlementsException("no authorities found", response);
+// }
+
+ @Override
+ public GroupInfo createGroup(CreateGroup createGroup) throws EntitlementsException {
+ return null;
+ }
+
+ @Override
+ public void deleteMember(String s, String s1) throws EntitlementsException {
+
+ }
+
+ @Override
+ public Groups authorizeAny(String... strings) throws EntitlementsException {
+ return null;
+ }
+
+ @Override
+ public void authenticate() throws EntitlementsException {
+
+ }
+}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/di/TenantFactoryImpl.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/di/TenantFactoryImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..2e828556cb0e4b0c715bd733ebc458530bf65292
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/di/TenantFactoryImpl.java
@@ -0,0 +1,114 @@
+// Copyright 2019 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.di;
+
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.annotation.PostConstruct;
+
+import org.opengroup.osdu.core.common.model.core.ICache;
+import org.opengroup.osdu.core.common.model.core.ITenantFactory;
+import org.opengroup.osdu.core.common.model.core.TenantInfo;
+import org.opengroup.osdu.core.ibm.cloudant.IBMCloudantClientFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+import com.cloudant.client.api.Database;
+
+@Component
+@RequestScope
+public class TenantFactoryImpl implements ITenantFactory {
+
+ private static final Logger logger = LoggerFactory.getLogger(TenantFactoryImpl.class);
+
+ @Autowired
+ private IBMCloudantClientFactory cloudantFactory;
+ private Database db;
+
+ @PostConstruct
+ public void init(){
+ try {
+ db = cloudantFactory.getClient().database(TenantInfoDoc.DB_NAME, true);
+ } catch (MalformedURLException e) {
+ e.printStackTrace();
+ }
+ }
+
+
+ private Map<String, TenantInfo> tenants;
+
+ public boolean exists(String tenantName)
+ {
+ if (this.tenants == null)
+ initTenants();
+ return this.tenants.containsKey(tenantName);
+ }
+
+ public TenantInfo getTenantInfo(String tenantName) {
+ if (this.tenants == null)
+ initTenants();
+ return this.tenants.get(tenantName);
+ }
+
+ public Collection<TenantInfo> listTenantInfo() {
+ if (this.tenants == null)
+ initTenants();
+ return this.tenants.values();
+ }
+
+ public <V> ICache<String, V> createCache(String tenantName, String host, int port, int expireTimeSeconds, Class<V> classOfV)
+ {
+ return null;
+ }
+
+ public void flushCache() {}
+
+ private void initTenants() {
+ this.tenants = new HashMap<>();
+
+
+ // TODO implement limit, cursor
+ // TODO refactor a more efficient way, with index
+
+
+ List<TenantInfoDoc> allDocs = null;
+ try {
+ allDocs = db.getAllDocsRequestBuilder().build().getResponse().getDocsAs(TenantInfoDoc.class);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ if (allDocs == null) {
+ logger.error("Could not get all documents for " + TenantInfoDoc.class.getCanonicalName() + " in the DB.");
+ return;
+ }
+
+ allDocs.forEach(doc -> {
+ TenantInfo ti = new TenantInfo();
+ String tenantName = doc.getId();
+ ti.setName(tenantName);
+ ti.setServiceAccount(doc.getServiceprincipalAppId());
+ this.tenants.put(tenantName, ti) ;
+ });
+ }
+}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/di/TenantInfoDoc.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/di/TenantInfoDoc.java
new file mode 100644
index 0000000000000000000000000000000000000000..4fe1fa25eac53b1fdb7777fd4b0448dee0f9cbff
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/di/TenantInfoDoc.java
@@ -0,0 +1,33 @@
+// Copyright 2019 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.di;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.springframework.data.annotation.Id;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class TenantInfoDoc {
+
+ public static final String DB_NAME = "TenantInfo"; //collection name
+
+ @Id
+ private String id;
+ private String serviceprincipalAppId;
+}
+
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/di/TenantInfoFactory.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/di/TenantInfoFactory.java
new file mode 100644
index 0000000000000000000000000000000000000000..5c45b59c8c0e5ad2a5b7af579bb8be6d7425ef2c
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/di/TenantInfoFactory.java
@@ -0,0 +1,44 @@
+// Copyright 2019 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.di;
+
+import javax.inject.Inject;
+
+import org.opengroup.osdu.core.common.model.core.DpsHeaders;
+import org.opengroup.osdu.core.common.model.core.ITenantFactory;
+import org.opengroup.osdu.core.common.model.core.TenantInfo;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.config.AbstractFactoryBean;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+@Component
+@RequestScope
+public class TenantInfoFactory extends AbstractFactoryBean<TenantInfo> {
+ @Inject
+ private ITenantFactory tenantFactory;
+ @Inject
+ @Qualifier("dpsHeaderFactorySearch")
+ private DpsHeaders headers;
+ @Override
+ protected TenantInfo createInstance() throws Exception {
+ String id = this.headers.getPartitionIdWithFallbackToAccountId();
+ return this.tenantFactory.getTenantInfo(id);
+ }
+ @Override
+ public Class<?> getObjectType() {
+ return TenantInfo.class;
+ }
+}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/model/AADConfiguration.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/model/AADConfiguration.java
new file mode 100644
index 0000000000000000000000000000000000000000..de8a58f27216827338d69abc8390dce86fde40b3
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/model/AADConfiguration.java
@@ -0,0 +1,38 @@
+// Copyright 2019 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.model;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+@Getter
+@Setter
+@Component
+@ConfigurationProperties("aad")
+public class AADConfiguration {
+ String clientId;
+ String authority;
+ String secretKey;
+ String oboApi;
+
+ public String getAuthority(){
+ if (!authority.endsWith("/")) {
+ authority += "/";
+ }
+ return authority;
+ }
+}
\ No newline at end of file
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/model/ElasticSettingSchema.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/model/ElasticSettingSchema.java
new file mode 100644
index 0000000000000000000000000000000000000000..a43caf08d835f9691b9e20981d191b7bea30df20
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/model/ElasticSettingSchema.java
@@ -0,0 +1,40 @@
+// Copyright 2019 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.model;
+
+import javax.validation.constraints.NotEmpty;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class ElasticSettingSchema {
+
+ @NotEmpty
+ private String host;
+
+ @NotEmpty
+ private String port;
+
+ @NotEmpty
+ private String usernameAndPassword;
+
+ @NotEmpty
+ private boolean isHttps;
+
+}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/persistence/ElasticRepositoryBYOC.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/persistence/ElasticRepositoryBYOC.java
new file mode 100644
index 0000000000000000000000000000000000000000..0cc67b97bf949b625a8a65c278695ccb79bfe5af
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/persistence/ElasticRepositoryBYOC.java
@@ -0,0 +1,67 @@
+// Copyright 2019 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.persistence;
+
+import org.apache.http.HttpStatus;
+
+import org.opengroup.osdu.core.common.model.core.TenantInfo;
+import org.opengroup.osdu.indexer.ibm.model.ElasticSettingSchema;
+import org.opengroup.osdu.core.common.model.core.ClusterSettings;
+import org.opengroup.osdu.core.common.spi.coreis.ElasticRepository;
+import org.opengroup.osdu.core.common.model.coreis.AppException;
+import org.opengroup.osdu.core.common.service.coreis.Preconditions;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+import javax.inject.Inject;
+
+@Component
+public class ElasticRepositoryBYOC implements ElasticRepository {
+
+ @Value("${ELASTIC_DATASTORE_KIND}")
+ private String ELASTIC_DATASTORE_KIND;
+
+ @Value("${ELASTIC_DATASTORE_ID}")
+ private String ELASTIC_DATASTORE_ID;
+
+ @Inject
+ private ISchemaRepository schemaRepository;
+
+
+ @Override
+ public ClusterSettings getElasticClusterSettings(TenantInfo tenantInfo) {
+
+ if(tenantInfo == null)
+ throw new AppException(HttpStatus.SC_NOT_FOUND, "TenantInfo is null", "");
+
+ String settingId = tenantInfo.getName().concat("-").concat(ELASTIC_DATASTORE_ID);
+ ElasticSettingSchema schema = this.schemaRepository.get(settingId);
+
+ if (schema == null) {
+ throw new AppException(HttpStatus.SC_NOT_FOUND, "Elastic setting not found", "The requested cluster setting was not found in CosmosDB.", String.format("Elastic setting with key: '%s' does not exist in CosmostDB.", ELASTIC_DATASTORE_KIND));
+ }
+
+ String host = schema.getHost();
+ String portString = schema.getPort();
+ String usernameAndPassword = schema.getUsernameAndPassword();
+
+ Preconditions.checkNotNullOrEmpty(host, "host cannot be null");
+ Preconditions.checkNotNullOrEmpty(portString, "port cannot be null");
+ Preconditions.checkNotNullOrEmpty(usernameAndPassword, "configuration cannot be null");
+
+ int port = Integer.parseInt(portString);
+
+ return new ClusterSettings(host, port, usernameAndPassword, schema.isHttps(), schema.isHttps());
+ }
+}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/persistence/ElasticSettingSchemaRepositoryImpl.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/persistence/ElasticSettingSchemaRepositoryImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..77e58c696df6f25501596fccea28368555a31a2e
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/persistence/ElasticSettingSchemaRepositoryImpl.java
@@ -0,0 +1,74 @@
+// Copyright 2019 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.persistence;
+
+import java.net.MalformedURLException;
+
+import javax.annotation.PostConstruct;
+
+import org.opengroup.osdu.core.ibm.cloudant.IBMCloudantClientFactory;
+import org.opengroup.osdu.indexer.ibm.model.ElasticSettingSchema;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Repository;
+import org.slf4j.Logger;
+import com.cloudant.client.api.Database;
+
+
+@Repository
+public class ElasticSettingSchemaRepositoryImpl implements ISchemaRepository {
+
+ private static final Logger logger = LoggerFactory.getLogger(ElasticSettingSchemaRepositoryImpl.class);
+
+ @Autowired
+ private IBMCloudantClientFactory cloudantFactory;
+ private Database db;
+
+ @PostConstruct
+ public void init(){
+ try {
+ db = cloudantFactory.getClient().database(ElasticSettingsDoc.DB_NAME, true);
+ } catch (MalformedURLException e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ public void add(ElasticSettingSchema schema, String id) {
+ ElasticSettingsDoc sd = new ElasticSettingsDoc();
+ sd.setId(id);
+ sd.setSettingSchema(schema);
+ db.save(sd);
+ }
+
+ @Override
+ public ElasticSettingSchema get(String id) {
+ if (db.contains(id)) {
+ ElasticSettingsDoc sd = db.find(ElasticSettingsDoc.class, id);
+ ElasticSettingSchema newSchema = new ElasticSettingSchema();
+ newSchema.setPort(sd.getSettingSchema().getPort());
+ newSchema.setHost(sd.getSettingSchema().getHost());
+ newSchema.setUsernameAndPassword(sd.getSettingSchema().getUsernameAndPassword());
+ newSchema.setHttps(sd.getSettingSchema().isHttps());
+ return newSchema;
+ } else {
+ logger.error(ElasticSettingsDoc.class + " with id " + id + " was not found in the database.");
+ return null;
+ }
+
+
+
+ }
+}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/persistence/ElasticSettingsDoc.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/persistence/ElasticSettingsDoc.java
new file mode 100644
index 0000000000000000000000000000000000000000..3c320cf351a4d8bfa68f9a366e0ccd2742e54117
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/persistence/ElasticSettingsDoc.java
@@ -0,0 +1,36 @@
+// Copyright 2019 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.persistence;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.opengroup.osdu.indexer.ibm.model.ElasticSettingSchema;
+import org.springframework.data.annotation.Id;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+
+public class ElasticSettingsDoc {
+
+ public static final String DB_NAME = "SearchSettings"; //collection name
+
+ @Id
+ private String id;
+ private ElasticSettingSchema settingSchema;
+}
+
+//interface CosmosDBElasticSettings extends DocumentDbRepository<ElasticSettingsDoc, String> {}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/persistence/ISchemaRepository.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/persistence/ISchemaRepository.java
new file mode 100644
index 0000000000000000000000000000000000000000..6b5f6157de45164de6dd7d8dfcae83bb7f63b5ec
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/persistence/ISchemaRepository.java
@@ -0,0 +1,30 @@
+
+// Copyright 2019 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.persistence;
+
+import org.opengroup.osdu.indexer.ibm.model.ElasticSettingSchema;
+
+public interface ISchemaRepository {
+ String SCHEMA_KIND = "IndexerSchema";
+
+ String SCHEMA = "schema";
+ String KIND = "KIND";
+
+ void add(ElasticSettingSchema schema, String id);
+
+ ElasticSettingSchema get(String id);
+}
+
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/publish/PublisherImpl.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/publish/PublisherImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..285c4bbdfa434b5835f98d68002ff55ea2478ad7
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/publish/PublisherImpl.java
@@ -0,0 +1,111 @@
+// Copyright 2019 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.publish;
+
+
+import java.lang.reflect.Type;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.inject.Inject;
+
+import org.elasticsearch.common.Strings;
+import org.opengroup.osdu.core.common.model.core.DpsHeaders;
+import org.opengroup.osdu.core.common.model.coreis.RecordChangedMessages;
+import org.opengroup.osdu.core.common.model.indexer.JobStatus;
+import org.opengroup.osdu.core.common.model.indexer.RecordStatus;
+import org.opengroup.osdu.core.common.service.coreis.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.spi.indexer.IPublisher;
+import org.opengroup.osdu.core.ibm.messagebus.IRabbitMQFactory;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+import com.google.common.reflect.TypeToken;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonElement;
+
+@Component
+@RequestScope
+public class PublisherImpl implements IPublisher {
+
+
+ @Inject
+ IRabbitMQFactory rabbitMQ;
+
+ @Inject
+ private JaxRsDpsLog logger;
+
+ @Override
+ public void publishStatusChangedTagsToTopic(DpsHeaders headers, JobStatus indexerBatchStatus) throws Exception {
+
+ String tenant = headers.getPartitionId();
+ if (Strings.isNullOrEmpty(tenant))
+ tenant = headers.getAccountId();
+
+
+ Map<String, String> message = new HashMap<>();
+ message.put(tenant, headers.getPartitionIdWithFallbackToAccountId());
+ headers.addCorrelationIdIfMissing();
+ message.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
+
+
+
+ RecordChangedMessages recordChangedMessages = getRecordChangedMessage(headers, indexerBatchStatus);
+ message.put("data", recordChangedMessages.toString());
+
+ try {
+ logger.info("Indexer publishes message " + headers.getCorrelationId());
+ rabbitMQ.sendMessage(message.toString());
+ }
+ catch (Exception e)
+ {
+ logger.error(e.getMessage(), e);
+ }
+ }
+
+ private RecordChangedMessages getRecordChangedMessage(DpsHeaders headers, JobStatus indexerBatchStatus) {
+
+ Gson gson = new GsonBuilder().create();
+ Map<String, String> attributesMap = new HashMap<>();
+ Type listType = new TypeToken<List<RecordStatus>>() {
+ }.getType();
+
+ JsonElement statusChangedTagsJson = gson.toJsonTree(indexerBatchStatus.getStatusesList(), listType);
+ String statusChangedTagsData = (statusChangedTagsJson.toString());
+
+ String tenant = headers.getPartitionId();
+ // This code it to provide backward compatibility to slb-account-id
+ if (!Strings.isNullOrEmpty(tenant)) {
+ attributesMap.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
+ } else {
+ attributesMap.put(DpsHeaders.ACCOUNT_ID, headers.getPartitionIdWithFallbackToAccountId());
+ }
+ headers.addCorrelationIdIfMissing();
+ attributesMap.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
+
+
+ RecordChangedMessages recordChangedMessages = new RecordChangedMessages();
+ // statusChangedTagsData is not ByteString but String
+ recordChangedMessages.setData(statusChangedTagsData);
+ recordChangedMessages.setAttributes(attributesMap);
+
+
+
+
+ return recordChangedMessages;
+ }
+}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/security/BasicAuthSecurityConfig.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/security/BasicAuthSecurityConfig.java
new file mode 100644
index 0000000000000000000000000000000000000000..60678a6e3e1fdb98f36a4622ba35978a47e88b5b
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/security/BasicAuthSecurityConfig.java
@@ -0,0 +1,50 @@
+// Copyright 2019 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.security;
+
+import com.microsoft.azure.spring.autoconfigure.aad.AADAppRoleStatelessAuthenticationFilter;
+import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity;
+import org.springframework.security.config.annotation.web.builders.HttpSecurity;
+import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
+import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter;
+import org.springframework.security.config.http.SessionCreationPolicy;
+import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter;
+
+import javax.inject.Inject;
+
+@EnableWebSecurity
+@EnableGlobalMethodSecurity(prePostEnabled = true)
+public class BasicAuthSecurityConfig extends WebSecurityConfigurerAdapter {
+ @Inject
+ private AADAppRoleStatelessAuthenticationFilter appRoleAuthFilter;
+
+ @Override
+ protected void configure(HttpSecurity http) throws Exception {
+ http.csrf().disable().sessionManagement().sessionCreationPolicy(SessionCreationPolicy.NEVER)
+ .and()
+ .authorizeRequests()
+ .antMatchers("/", "/index.html","/obo_api",
+ "/index-worker", "/_dps/task-handlers", "/_dps/task-handlers/**",
+ "/reindex",
+ "/v2/api-docs",
+ "/swagger-resources/**",
+ "/configuration/security",
+ "/swagger",
+ "/swagger-ui.html",
+ "/webjars/**").permitAll()
+ .anyRequest().authenticated()
+ .and().addFilterBefore(appRoleAuthFilter, UsernamePasswordAuthenticationFilter.class);
+ }
+}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/security/WhoamiController.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/security/WhoamiController.java
new file mode 100644
index 0000000000000000000000000000000000000000..dc969170721deb8407881f43d8e777c2e122736d
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/security/WhoamiController.java
@@ -0,0 +1,39 @@
+// Copyright 2019 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.security;
+
+import org.springframework.security.core.Authentication;
+import org.springframework.security.core.context.SecurityContextHolder;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.ResponseBody;
+
+@Controller
+public class WhoamiController {
+ @RequestMapping(value = {"/", "/whoami"})
+ @ResponseBody
+ public String whoami() {
+ final Authentication auth = SecurityContextHolder.getContext().getAuthentication();
+
+ String userName = auth.getName();
+ String roles = String.valueOf(auth.getAuthorities());
+ String details = String.valueOf(auth.getPrincipal());
+
+ return "user: " + userName + "<BR>" +
+ "roles: " + roles + "<BR>" +
+ "details: " + details;
+ }
+}
+
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/HeadersInfoIBMImpl.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/HeadersInfoIBMImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..782cfeb304028f1b1811f3356911a754a8af0e24
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/HeadersInfoIBMImpl.java
@@ -0,0 +1,97 @@
+// Copyright 2019 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.util;
+
+import lombok.extern.java.Log;
+import org.opengroup.osdu.core.common.model.core.DpsHeaders;
+import org.opengroup.osdu.core.common.model.coreis.SlbHeaders;
+import org.opengroup.osdu.core.common.spi.coreis.IHeadersInfo;
+import org.opengroup.osdu.core.common.service.coreis.Preconditions;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+import javax.inject.Inject;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+@Log
+@Component
+@RequestScope
+public class HeadersInfoIBMImpl implements IHeadersInfo {
+
+ @Inject
+ @Qualifier("dpsHeaderFactorySearch")
+ private DpsHeaders headersMap;
+
+ @Value("${indexer.queue.key}")
+ private String queueKey;
+
+ //ToDo this should be moved to Azure client-lib
+ public static final String INDEXER_QUEUE_KEY = "x-functions-key";
+
+ private static final HashSet<String> FORBIDDEN_FROM_LOGGING = new HashSet<>();
+ static {
+ FORBIDDEN_FROM_LOGGING.add(DpsHeaders.AUTHORIZATION);
+ FORBIDDEN_FROM_LOGGING.add(DpsHeaders.ON_BEHALF_OF);
+ // FORBIDDEN_FROM_LOGGING.add(INDEXER_QUEUE_KEY);
+ }
+
+ @Override
+ public DpsHeaders getHeaders() {
+ if (headersMap == null) {
+ log.warning("Headers Map DpsHeaders is null");
+ }
+ DpsHeaders headers = this.getCoreServiceHeaders(headersMap.getHeaders());
+ return headers;
+ }
+
+ @Override
+ public String getUser() {
+ return getHeaders().getUserEmail();
+ }
+
+ @Override
+ public String getPartitionId() {
+ return getHeaders().getPartitionIdWithFallbackToAccountId();
+ }
+
+ @Override
+ public String getPrimaryPartitionId() {
+ return getHeadersMap().get(SlbHeaders.PRIMARY_PARTITION_ID);
+ }
+
+ @Override
+ public Map<String, String> getHeadersMap() {
+ return getHeaders().getHeaders();
+ }
+
+ @Override
+ public DpsHeaders getCoreServiceHeaders(Map<String, String> input) {
+ Preconditions.checkNotNull(input, "input headers cannot be null");
+
+ DpsHeaders output = DpsHeaders.createFromMap(input);
+ output.put(INDEXER_QUEUE_KEY,queueKey);
+ return output;
+ }
+
+ @Override
+ public String toString() {
+ return this.getHeadersMap().entrySet().stream().filter(map -> !FORBIDDEN_FROM_LOGGING.contains(map.getKey().toLowerCase())).map(Map.Entry::toString).collect(Collectors.joining(" | "));
+ }
+
+}
\ No newline at end of file
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/RequestInfoImpl.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/RequestInfoImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..01f94956590ac0cc6c664ca336c12463c9bd7243
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/RequestInfoImpl.java
@@ -0,0 +1,110 @@
+// Copyright 2019 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.util;
+
+import com.google.common.base.Strings;
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.core.common.model.core.DpsHeaders;
+import org.opengroup.osdu.core.common.model.core.TenantInfo;
+import org.opengroup.osdu.core.common.model.coreis.AppException;
+import org.opengroup.osdu.core.common.model.coreis.DeploymentEnvironment;
+import org.opengroup.osdu.core.common.spi.coreis.IHeadersInfo;
+import org.opengroup.osdu.core.common.spi.coreis.IRequestInfo;
+import org.opengroup.osdu.core.common.spi.coreis.IServiceAccountJwtClient;
+import org.opengroup.osdu.core.common.model.coreis.AppException;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+import javax.inject.Inject;
+import java.util.Map;
+
+import static org.opengroup.osdu.core.common.model.core.DpsHeaders.AUTHORIZATION;
+import static org.opengroup.osdu.indexer.ibm.util.HeadersInfoIBMImpl.INDEXER_QUEUE_KEY;
+
+
+@Component
+@RequestScope
+public class RequestInfoImpl implements IRequestInfo {
+
+ @Inject
+ @Qualifier("dpsHeaderFactorySearch")
+ private DpsHeaders dpsHeaders;
+
+ @Inject
+ private IHeadersInfo headersInfo;
+ @Inject
+ private IServiceAccountJwtClient serviceAccountJwtClient;
+
+ @Inject
+ @Qualifier("TenantInfoFactorySearch")
+ private TenantInfo tenantInfo;
+
+ @Value("${DEPLOYMENT_ENVIRONMENT}")
+ private String DEPLOYMENT_ENVIRONMENT;
+
+
+ @Override
+ public DpsHeaders getHeaders() {
+
+ return this.headersInfo.getHeaders();
+ }
+
+ @Override
+ public String getPartitionId() {
+ return this.headersInfo.getPartitionId();
+ }
+
+ @Override
+ public Map<String, String> getHeadersMap() {
+ return this.headersInfo.getHeadersMap();
+ }
+
+ @Override
+ public Map<String, String> getHeadersMapWithDwdAuthZ() {
+ return getHeadersWithDwdAuthZ().getHeaders();
+ }
+
+ @Override
+ public DpsHeaders getHeadersWithDwdAuthZ() {
+ this.dpsHeaders.put(AUTHORIZATION, this.checkOrGetAuthorizationHeader());
+ return this.headersInfo.getHeaders();
+ }
+
+ @Override
+ public boolean isCronRequest() { return false;}
+
+ @Override
+ public boolean isTaskQueueRequest() {
+ if (!this.headersInfo.getHeadersMap().containsKey(INDEXER_QUEUE_KEY)) return false;
+
+// String queueId = this.headersInfo.getHeadersMap().get(AppEngineHeaders.TASK_QUEUE_NAME);
+// return queueId.endsWith(Constants.INDEXER_QUEUE_IDENTIFIER);
+ return false;
+ }
+
+ public String checkOrGetAuthorizationHeader() {
+ if (DeploymentEnvironment.valueOf(DEPLOYMENT_ENVIRONMENT) == DeploymentEnvironment.LOCAL) {
+ String authHeader = this.headersInfo.getHeaders().getAuthorization();
+ if (Strings.isNullOrEmpty(authHeader)) {
+ throw new AppException(HttpStatus.SC_UNAUTHORIZED, "Invalid authorization header", "Authorization token cannot be empty");
+ }
+ return authHeader;
+ } else {
+ return "Bearer " + this.serviceAccountJwtClient.getIdToken(tenantInfo.getName());
+ }
+ }
+}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/ServiceAccountJwtClientImpl.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/ServiceAccountJwtClientImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..6ca3101dd4ae52350f465cf0b6dfd1a3e9f8bae2
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/ServiceAccountJwtClientImpl.java
@@ -0,0 +1,119 @@
+// Copyright 2019 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.util;
+
+import com.auth0.jwt.JWT;
+import com.auth0.jwt.exceptions.JWTDecodeException;
+import com.microsoft.aad.adal4j.AuthenticationContext;
+import com.microsoft.aad.adal4j.AuthenticationResult;
+import com.microsoft.aad.adal4j.ClientCredential;
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.core.common.model.core.DpsHeaders;
+import org.opengroup.osdu.core.common.model.core.ITenantFactory;
+import org.opengroup.osdu.core.common.model.core.TenantInfo;
+import org.opengroup.osdu.core.common.model.coreis.AppException;
+import org.opengroup.osdu.core.common.service.coreis.JaxRsDpsLog;
+import org.opengroup.osdu.indexer.ibm.model.AADConfiguration;
+
+import org.opengroup.osdu.core.common.model.coreis.IdToken;
+import org.opengroup.osdu.core.common.spi.coreis.IJwtCache;
+import org.opengroup.osdu.core.common.spi.coreis.IHeadersInfo;
+import org.opengroup.osdu.core.common.spi.coreis.IServiceAccountJwtClient;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+import javax.inject.Inject;
+import java.net.MalformedURLException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+@Component
+@RequestScope
+public class ServiceAccountJwtClientImpl implements IServiceAccountJwtClient {
+
+ @Inject
+ private ITenantFactory tenantInfoServiceProvider;
+ @Inject
+ private IHeadersInfo headersInfoAzure;
+ @Inject
+ @Qualifier("dpsHeaderFactorySearch")
+ private DpsHeaders dpsHeaders;
+ @Inject
+ private IJwtCache cacheService;
+ @Inject
+ private JaxRsDpsLog log;
+
+ @Inject
+ private AADConfiguration configuration;
+
+ public String getIdToken(String tenantName) {
+ this.log.info("Tenant name received for auth token is: " + tenantName);
+ TenantInfo tenant = this.tenantInfoServiceProvider.getTenantInfo(tenantName);
+ if (tenant == null) {
+ this.log.error("Invalid tenant name receiving from azure");
+ throw new AppException(HttpStatus.SC_BAD_REQUEST, "Invalid tenant Name", "Invalid tenant Name from azure");
+ }
+ String ACCESS_TOKEN = "";
+ try {
+
+ IdToken cachedToken = (IdToken) this.cacheService.get(tenant.getServiceAccount());
+ this.headersInfoAzure.getHeaders().put(DpsHeaders.USER_EMAIL, tenant.getServiceAccount());
+ this.dpsHeaders.put(DpsHeaders.USER_EMAIL, tenant.getServiceAccount());
+
+ if (!IdToken.refreshToken(cachedToken)) {
+ return cachedToken.getTokenValue();
+ }
+
+ ExecutorService service = Executors.newFixedThreadPool(1);
+ AuthenticationContext context = null;
+
+ try {
+ context = new AuthenticationContext(configuration.getAuthority(), false, service);
+ ClientCredential credential = new ClientCredential(configuration.getClientId(), configuration.getSecretKey());
+ Future<AuthenticationResult> future = context.acquireToken(configuration.getOboApi(), credential, null);
+
+ ACCESS_TOKEN = future.get().getAccessToken();
+
+ if (future == null) {
+ log.error(String.format("Azure Authentication: %s", future.get().getAccessToken()));
+ throw new AppException(HttpStatus.SC_FORBIDDEN, "Access denied", "The user is not authorized to perform this action");
+ }
+ IdToken idToken = IdToken.builder().tokenValue(ACCESS_TOKEN).expirationTimeMillis(JWT.decode(ACCESS_TOKEN).getExpiresAt().getTime()).build();
+
+ this.cacheService.put(tenant.getServiceAccount(), idToken);
+
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ } catch (ExecutionException e) {
+ e.printStackTrace();
+ } catch (MalformedURLException e) {
+ e.printStackTrace();
+ } finally {
+ service.shutdown();
+ }
+ } catch (JWTDecodeException e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Persistence error", "Invalid token, error decoding", e);
+ } catch (AppException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Persistence error", "Error generating token", e);
+ }
+
+ return ACCESS_TOKEN;
+ }
+}
diff --git a/provider/indexer-ibm/src/main/resources/application.properties b/provider/indexer-ibm/src/main/resources/application.properties
new file mode 100644
index 0000000000000000000000000000000000000000..a79e32ddcde40517593b25bcef9d90b778ce2c65
--- /dev/null
+++ b/provider/indexer-ibm/src/main/resources/application.properties
@@ -0,0 +1,86 @@
+#server.servlet.contextPath=/api/indexer/v2/
+logging.level.org.springframework.web=DEBUG
+server.port=8080
+JAVA_HEAP_OPTS=-Xms4096M -Xmx4096M
+JAVA_GC_OPTS=-XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45
+
+DEFAULT_DATA_COUNTRY=US
+
+AUTHORIZE_API=https://opendesproxy.azurewebsites.net/entitlements/v1
+AUTHORIZE_API_KEY=dummy
+LEGALTAG_API=https://opendesproxy.azurewebsites.net/legal/v1
+LEGALTAG_API_KEY=dummy
+CRS_API=https://opendesproxy.azurewebsites.net/crs/v1
+CRS_API_KEY=dummy
+
+#Default Cache Settings
+SCHEMA_CACHE_EXPIRATION=60
+INDEX_CACHE_EXPIRATION=60
+ELASTIC_CACHE_EXPIRATION=1440
+CURSOR_CACHE_EXPIRATION=60
+# Kinds Cache expiration 2*24*60
+KINDS_CACHE_EXPIRATION=2880
+# Attributes Cache expiration 2*24*60
+ATTRIBUTES_CACHE_EXPIRATION=2880
+# Maximum size of cache value
+MAX_CACHE_VALUE_SIZE=1000
+
+KINDS_REDIS_DATABASE=1
+CRON_INDEX_CLEANUP_THRESHOLD_DAYS=3
+CRON_EMPTY_INDEX_CLEANUP_THRESHOLD_DAYS=7
+
+STORAGE_SCHEMA_HOST=https://opendesmvp.azurewebsites.net/schemas
+STORAGE_QUERY_RECORD_HOST=https://opendesmvp.azurewebsites.net/query/records
+STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=https://opendesmvp.azurewebsites.net/query/records:batch
+STORAGE_RECORDS_BATCH_SIZE=20
+
+INDEXER_QUEUE_HOST=https://requeuefunction-cd1.azurewebsites.net/api/re-enqueue
+
+#MSAL AAD
+aad.authority=https://login.microsoftonline.com/1668106e-2ae0-456a-bb7d-64b52104db99/oauth2/token
+aad.clientId=b70451c6-936f-4430-8680-0985817ac751
+aad.secretKey=b5GZtbS.bB7iSzLTb+Nb-AOb9G@1@LZo
+aad.oboApi=api://ffb32b1a-c6fe-4ed0-819b-6ca483de3640
+
+spring.security.oauth2.client.registration.azure.client-id=b70451c6-936f-4430-8680-0985817ac751
+spring.security.oauth2.client.registration.azure.client-secret=b5GZtbS.bB7iSzLTb+Nb-AOb9G@1@LZo
+spring.security.oauth2.client.registration.azure.client-name=Azure
+
+
+#spring.security.user.name=opendes@byoc.local
+#spring.security.user.password=123
+#spring.security.user.roles=service.indexer.admin
+
+
+# Azure AD configuration
+azure.activedirectory.client-id=ffb32b1a-c6fe-4ed0-819b-6ca483de3640
+azure.activedirectory.tenant-id=1668106e-2ae0-456a-bb7d-64b52104db99
+azure.activedirectory.AppIdUri=api://${azure.activedirectory.client-id}
+azure.activedirectory.session-stateless=true
+
+#azure.activedirectory.active-directory-groups=ADMIN, VIEWER
+
+# Azure CosmosDB configuration
+azure.cosmosdb.uri=https://opendescosmosdb.documents.azure.com:443/
+azure.cosmosdb.key=AQK0VbnMGn76h9Ypw5aoirKMwoAYJBRY2XENn1dAYypoea83weVyjYcGaRDeEX7HqXIVggaDuFAhQ4Y1Y9lKDQ==
+azure.cosmosdb.database=opendesdb
+
+
+# Azure Service Bus configuration
+azure.servicebus.connection-string=Endpoint=sb://pliuopendes.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=km8Nscc0gf299Ck6npmM3D14VU5Tx1lJYRdlHcExIvY=
+azure.servicebus.topic-name=recordstopic
+
+#Indexer-Queue-header
+indexer.queue.key=abcd
+
+REDIS_GROUP_HOST=127.0.0.1
+REDIS_GROUP_PORT=6379
+REDIS_SEARCH_HOST=localhost
+REDIS_SEARCH_PORT=6379
+
+ELASTIC_DATASTORE_KIND=SearchSettings
+ELASTIC_DATASTORE_ID=indexer-service
+
+GAE_SERVICE=indexer
+
+DEPLOYMENT_ENVIRONMENT=CLOUD
\ No newline at end of file
diff --git a/provider/indexer-ibm/src/main/resources/logback-spring.xml b/provider/indexer-ibm/src/main/resources/logback-spring.xml
new file mode 100644
index 0000000000000000000000000000000000000000..e70b3be45b9f89b107f0e0197dede6b2afa5da2a
--- /dev/null
+++ b/provider/indexer-ibm/src/main/resources/logback-spring.xml
@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Copyright 2019 IBM Corp. All Rights Reserved.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<configuration>
+
+ <appender name="Console"
+ class="ch.qos.logback.core.ConsoleAppender">
+ <layout class="ch.qos.logback.classic.PatternLayout">
+ <Pattern>
+ %black(%d{ISO8601}) %highlight(%-5level) [%blue(%t)] %yellow(%C{1.}): %msg%n%throwable
+ </Pattern>
+ </layout>
+ </appender>
+
+ <appender name="aiAppender"
+ class="com.microsoft.applicationinsights.logback.ApplicationInsightsAppender">
+ <instrumentationKey>${appinsights_key}</instrumentationKey>
+ </appender>
+
+ <!-- servicebus throws exceptions when idle -->
+ <logger name="com.microsoft.azure.servicebus.primitives.RequestResponseLink" level="error" />
+
+ <root level="warn">
+ <appender-ref ref="Console" />
+ <appender-ref ref="aiAppender" />
+ </root>
+
+</configuration>
\ No newline at end of file
diff --git a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/CronServiceImplTest.java b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/CronServiceImplTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..58d97805083f4552d49de81cefc3524a90d55265
--- /dev/null
+++ b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/CronServiceImplTest.java
@@ -0,0 +1,141 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.service;
+
+import com.google.common.collect.Lists;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.model.core.DpsHeaders;
+import org.opengroup.osdu.indexer.service.CronServiceImpl;
+import org.opengroup.osdu.core.common.service.coreis.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.coreis.IndexInfo;
+import org.opengroup.osdu.core.common.spi.coreis.IRequestInfo;
+import org.opengroup.osdu.core.common.service.coreis.IndicesService;
+import org.opengroup.osdu.core.common.service.coreis.Config;
+import org.opengroup.osdu.is.core.util.ElasticClientHandler;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.*;
+import static org.powermock.api.mockito.PowerMockito.mockStatic;
+
+
+@Ignore
+@RunWith(SpringRunner.class)
+@PrepareForTest({RestHighLevelClient.class})
+public class CronServiceImplTest {
+
+ @Mock
+ private RestHighLevelClient restHighLevelClient;
+ @Mock
+ private IndicesService indicesService;
+ @Mock
+ private ElasticClientHandler elasticClientHandler;
+ @Mock
+ private IRequestInfo requestInfo;
+ @Mock
+ private JaxRsDpsLog log;
+ @InjectMocks
+ private CronServiceImpl sut;
+
+ @InjectMocks
+ private DpsHeaders dpsHeaders;
+
+ @Before
+ public void setup() {
+ mockStatic(Config.class);
+
+ when(this.requestInfo.getHeaders()).thenReturn(dpsHeaders);
+
+ when(Config.getIndexCleanupThresholdDays()).thenReturn(3);
+ when(Config.getEmptyIndexCleanupThresholdDays()).thenReturn(7);
+ }
+
+ @Test
+ public void run_cleanup_when_cron_job_runs_with_correct_pattern() throws Exception {
+ final String indexPattern = "tenant1-index-*";
+
+ IndexInfo info = IndexInfo.builder().name("tenant1-index-1.0.0").documentCount("10").creationDate(Long.toString(Instant.now().minus(4, ChronoUnit.DAYS).toEpochMilli())).build();
+
+ when(this.requestInfo.getPartitionId()).thenReturn("tenant1");
+ when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient);
+ when(this.indicesService.getIndexInfo(this.restHighLevelClient, indexPattern)).thenReturn(Lists.newArrayList(info));
+
+ this.sut.cleanupIndices(indexPattern);
+
+ verify(this.indicesService, times(1)).deleteIndex(restHighLevelClient, "tenant1-index-1.0.0");
+ verify(this.indicesService, times(1)).getIndexInfo(restHighLevelClient, indexPattern);
+ }
+
+ @Test(expected = IOException.class)
+ public void run_cleanup_when_cron_job_runs_with_wrong_pattern() throws Exception {
+ IOException exception = new IOException("blah");
+ when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient);
+ when(this.indicesService.getIndexInfo(this.restHighLevelClient, "tenant1-test-*")).thenThrow(exception);
+
+ this.sut.cleanupIndices("tenant1-test-*");
+
+ verify(this.indicesService, times(0)).deleteIndex(any(), any());
+ }
+
+ @Test
+ public void run_cleanup_when_backend_does_not_have_empty_stale_indices() throws Exception {
+ IndexInfo info = IndexInfo.builder().name("tenant1-index-1.0.0").documentCount("10").creationDate(Long.toString(Instant.now().minus(8, ChronoUnit.DAYS).toEpochMilli())).build();
+
+ when(this.requestInfo.getPartitionId()).thenReturn("tenant1");
+ when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient);
+ when(this.indicesService.getIndexInfo(this.restHighLevelClient, null)).thenReturn(Lists.newArrayList(info));
+
+ this.sut.cleanupEmptyStaleIndices();
+
+ verify(this.indicesService, times(0)).deleteIndex(restHighLevelClient, null);
+ verify(this.indicesService, times(1)).getIndexInfo(restHighLevelClient, null);
+ }
+
+ @Test
+ public void run_cleanup_when_backend_have_empty_stale_indices() throws Exception {
+ IndexInfo info = IndexInfo.builder().name("tenant1-index-1.0.0").documentCount("0").creationDate(Long.toString(Instant.now().minus(8, ChronoUnit.DAYS).toEpochMilli())).build();
+
+ when(this.requestInfo.getPartitionId()).thenReturn("tenant1");
+ when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient);
+ when(this.indicesService.getIndexInfo(this.restHighLevelClient, null)).thenReturn(Lists.newArrayList(info));
+
+ this.sut.cleanupEmptyStaleIndices();
+
+ verify(this.indicesService, times(1)).deleteIndex(restHighLevelClient, "tenant1-index-1.0.0");
+ verify(this.indicesService, times(1)).getIndexInfo(restHighLevelClient, null);
+ }
+
+ @Test(expected = IOException.class)
+ public void run_cleanup_when_backend_throws_exception() throws Exception {
+ IOException exception = new IOException("blah");
+ when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient);
+ when(this.indicesService.getIndexInfo(this.restHighLevelClient, null)).thenThrow(exception);
+
+ this.sut.cleanupEmptyStaleIndices();
+
+ verify(this.indicesService, times(0)).deleteIndex(any(), any());
+ }
+}
\ No newline at end of file
diff --git a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/ElasticSettingServiceTest.java b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/ElasticSettingServiceTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..67e481d3065b7ba39b0d2cb830abba556b63bbab
--- /dev/null
+++ b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/ElasticSettingServiceTest.java
@@ -0,0 +1,112 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.service;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.model.core.TenantInfo;
+import org.opengroup.osdu.indexer.service.ElasticSettingServiceImpl;
+import org.opengroup.osdu.core.common.service.coreis.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.core.ClusterSettings;
+import org.opengroup.osdu.core.common.spi.coreis.IElasticCredentialsCache;
+import org.opengroup.osdu.core.common.spi.coreis.ElasticRepository;
+import org.opengroup.osdu.core.common.spi.coreis.IHeadersInfo;
+import org.opengroup.osdu.core.common.service.coreis.TenantInfoService;
+import org.opengroup.osdu.core.common.model.coreis.AppException;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.Mockito.when;
+
+@RunWith(SpringRunner.class)
+public class ElasticSettingServiceTest {
+
+ @Mock
+ private TenantInfoService tenantInfoService;
+ @Mock
+ private ElasticRepository elasticRepository;
+ @Mock
+ private IElasticCredentialsCache elasticCredentialCache;
+ @Mock
+ private TenantInfo tenantInfo;
+ @InjectMocks
+ private ElasticSettingServiceImpl sut;
+ @Mock
+ private ClusterSettings clusterSettings;
+ @Mock
+ private IHeadersInfo headersInfo;
+
+ @Mock
+ private JaxRsDpsLog log;
+
+
+ public String GAE_SERVICE = "indexer";
+
+ private final String host = "db5c51c1.us-central1.gcp.cloud.es.io";
+ private final int port = 9243;
+ private final String credentials = "name:password";
+
+ String cacheKey = "";
+
+
+ @Before
+ public void setup() {
+ when(tenantInfo.getName()).thenReturn("tenant1");
+ when(this.headersInfo.getPartitionId()).thenReturn("tenant1");
+ when(this.tenantInfoService.getTenantInfo()).thenReturn(tenantInfo);
+ sut.GAE_SERVICE = "indexer";
+ clusterSettings = ClusterSettings.builder().host(host).port(port).userNameAndPassword(credentials).build();
+ cacheKey = String.format("%s-%s", GAE_SERVICE, tenantInfo.getName());
+ }
+
+ @Test
+ public void should_getValid_clusterSettings_fromCache() {
+
+ when(this.elasticCredentialCache.get(cacheKey)).thenReturn(clusterSettings);
+
+ ClusterSettings response = this.sut.getElasticClusterInformation();
+ assertNotNull(response);
+ assertEquals(response.getHost(), host);
+ assertEquals(response.getPort(), port);
+ assertEquals(response.getUserNameAndPassword(), credentials);
+ }
+
+ @Test
+ public void should_getValid_clusterSettings_fromCosmosDB() {
+
+ when(this.elasticCredentialCache.get(cacheKey)).thenReturn(clusterSettings);
+
+ when(this.elasticRepository.getElasticClusterSettings(tenantInfo)).thenReturn(clusterSettings);
+
+ ClusterSettings response = this.sut.getElasticClusterInformation();
+ assertNotNull(response);
+ assertEquals(response.getHost(), host);
+ assertEquals(response.getPort(), port);
+ assertEquals(response.getUserNameAndPassword(), credentials);
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throwAppException_when_tenantClusterInfo_not_found() throws AppException {
+
+ when(this.elasticRepository.getElasticClusterSettings(tenantInfo)).thenReturn(null);
+
+ this.sut.getElasticClusterInformation();
+
+ }
+}
diff --git a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexCopyServiceImplTest.java b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexCopyServiceImplTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..6ae884ff6d18155d6f69fc4376ec97dad9fe8ec5
--- /dev/null
+++ b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexCopyServiceImplTest.java
@@ -0,0 +1,194 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.service;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import org.apache.http.HttpEntity;
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentMatchers;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.model.core.DpsHeaders;
+import org.opengroup.osdu.core.common.model.indexer.AuditLogger;
+import org.opengroup.osdu.indexer.service.IndexCopyServiceImpl;
+import org.opengroup.osdu.indexer.service.IndexerMappingService;
+import org.opengroup.osdu.core.common.model.core.ClusterSettings;
+import org.opengroup.osdu.core.common.spi.coreis.IHeadersInfo;
+import org.opengroup.osdu.core.common.spi.coreis.IRequestInfo;
+import org.opengroup.osdu.is.core.service.ElasticSettingService;
+import org.opengroup.osdu.core.common.service.coreis.IndicesService;
+import org.opengroup.osdu.core.common.model.coreis.AppException;
+import org.opengroup.osdu.is.core.util.ElasticClientHandler;
+import org.opengroup.osdu.core.common.service.coreis.ElasticIndexNameResolver;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.io.IOException;
+import java.lang.reflect.Type;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@RunWith(SpringRunner.class)
+@PrepareForTest({RestHighLevelClient.class, Response.class, RestClient.class, HttpEntity.class, EntityUtils.class})
+public class IndexCopyServiceImplTest {
+ private final String correlationId = UUID.randomUUID().toString();
+
+ @Mock
+ private HttpEntity httpEntity;
+ @Mock
+ private HttpEntity httpEntityRequest;
+ @Mock
+ private IRequestInfo requestInfo;
+ @Mock
+ private IHeadersInfo headersInfo;
+ @Mock
+ private RestClient restClient;
+ @Mock
+ private RestHighLevelClient restHighLevelClient;
+ @Mock
+ private IndicesService indicesService;
+ @Mock
+ private IndexerMappingService mappingService;
+ @Mock
+ private ElasticClientHandler elasticClientHandler;
+ @Mock
+ private ElasticIndexNameResolver elasticIndexNameResolver;
+ @Mock
+ private Response response;
+ @Mock
+ private ElasticSettingService elasticSettingService;
+ @Mock
+ private AuditLogger auditLogger;
+ @Mock
+ private Map<String, String> httpHeaders;
+ @InjectMocks
+ private IndexCopyServiceImpl sut;
+
+ private ClusterSettings commonCluster;
+
+ private Map<String, Object> correctMap;
+
+ @Before
+ public void setup() {
+
+ commonCluster = ClusterSettings.builder().host("commonhost").port(8080).userNameAndPassword("username:pwd").build();
+
+ httpHeaders = new HashMap<>();
+ httpHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
+ httpHeaders.put(DpsHeaders.CORRELATION_ID, correlationId);
+ DpsHeaders standardHeaders = DpsHeaders.createFromMap(httpHeaders);
+ when(headersInfo.getHeaders()).thenReturn(standardHeaders);
+ when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(httpHeaders);
+ when(response.getEntity()).thenReturn(httpEntity);
+
+ Type mapType = new TypeToken<Map<String, Object>>() {}.getType();
+ String afterFormat = "{\"properties\":{\"id\":{\"type\":\"keyword\"}}}";
+ correctMap = new Gson().fromJson(afterFormat, mapType);
+
+ restHighLevelClient = mock(RestHighLevelClient.class);
+
+ }
+
+ @Test(expected = IOException.class)
+ public void should_throwIOException_when_indexMappingNotFound() throws Exception {
+ IOException exception = new IOException("Fail to get mapping for the given index from common cluster.");
+
+ when(this.mappingService.getIndexMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenThrow(exception);
+
+ this.sut.copyIndex("common:metadata:entity:1.0.0");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void should_throwIllegalArgExceptionCopyIndexRequest_copyIndexTest() {
+ try {
+ this.sut.copyIndex(null);
+ } catch (IOException e) {
+ fail("Should not throw IOException but illegalArgumentException.");
+ }
+ }
+
+ @Test
+ public void should_returnIndexMapping_getIndexMappingFromCommonClustertest() {
+ String mappingJson = "{\"common-metadata-entity-1.0.0\":{\"mappings\":{\"entity\":{\"properties\":{\"id\":{\"type\":\"keyword\"}}}}}}";
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ try {
+ when(this.mappingService.getIndexMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(mappingJson);
+ Map<String, Object> resultMap = this.sut.getIndexMappingsFromCommonCluster("test", "test");
+ Assert.assertEquals(resultMap, correctMap);
+ } catch (Exception ignored) {
+ }
+ }
+
+ @Test
+ public void should_returnClusterInfo_getCommonClusterInformationtest() {
+ try {
+ String[] correctCommonCluster = {"https://commonhost:8080", "username", "pwd"};
+
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+
+ when(elasticSettingService.getElasticClusterInformation()).thenReturn(commonCluster);
+
+ String[] resultCommonCluster = this.sut.getCommonClusterInformation();
+ Assert.assertEquals(correctCommonCluster[0], resultCommonCluster[0]);
+ Assert.assertEquals(correctCommonCluster[1], resultCommonCluster[1]);
+ Assert.assertEquals(correctCommonCluster[2], resultCommonCluster[2]);
+ } catch (IOException ignored) {
+ fail("Should not throw this exception " + ignored.getMessage());
+ }
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throwException_failToCreateIndexInTenantCluster_createIndexInTenantClustertest() {
+ try {
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ when(indicesService.createIndex(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(false);
+ this.sut.createIndexInTenantCluster("test", "test", "test", correctMap);
+ } catch (IOException ignored) {
+ fail("Should not throw this exception " + ignored.getMessage());
+ }
+ }
+
+ @Ignore
+ public void should_returnTaskIdResponse_reindexRequestSucceed_reindexInTenantClustertest() {
+ //TODO: fix the null Response from restHighLevelClient.getLowLevelClient().performRequest().
+ try {
+ String[] correctCommonCluster = {"https://commonhost:8080", "username", "pwd"};
+ Request request = new Request("POST", "/_reindex?wait_for_completion=false");
+ request.setEntity(httpEntityRequest);
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ when(indicesService.createIndex(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(false);
+ when(restHighLevelClient.getLowLevelClient()).thenReturn(restClient);
+ when(restClient.performRequest(request)).thenReturn(response);
+ when(response.getEntity()).thenReturn(httpEntity);
+ Assert.assertEquals(httpEntity, this.sut.reindexInTenantCluster("test", "test", correctCommonCluster));
+ } catch (IOException ignored) {
+ }
+ }
+}
diff --git a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerMappingServiceTest.java b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerMappingServiceTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..754677b3bf3ae463f49891cf1a31763a4f9d4a61
--- /dev/null
+++ b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerMappingServiceTest.java
@@ -0,0 +1,316 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.service;
+
+import org.apache.http.StatusLine;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
+import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData;
+import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.*;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.reindex.BulkByScrollResponse;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentMatchers;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
+import org.opengroup.osdu.indexer.service.IndexerMappingServiceImpl;
+import org.opengroup.osdu.core.common.model.coreis.RecordMetaAttribute;
+import org.opengroup.osdu.core.common.model.coreis.AppException;
+import org.opengroup.osdu.is.core.util.ElasticClientHandler;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.io.IOException;
+import java.util.*;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@Ignore
+@RunWith(SpringRunner.class)
+@PrepareForTest({ RestHighLevelClient.class, IndicesClient.class })
+public class IndexerMappingServiceTest {
+
+ private final String kind = "tenant:test:test:1.0.0";
+ private final String index = "tenant-test-test-1.0.0";
+ private final String type = "test";
+ private final String mappingValid = "{\"dynamic\":false,\"properties\":{\"data\":{\"properties\":{\"Location\":{\"type\":\"geo_point\"}}},\"id\":{\"type\":\"keyword\"}}}";
+
+ @Mock
+ private RestClient restClient;
+ @Mock
+ private Response response;
+ @Mock
+ private StatusLine statusLine;
+
+ @InjectMocks
+ private IndexerMappingServiceImpl sut;
+
+ @Mock
+ private ElasticClientHandler elasticClientHandler;
+
+ @InjectMocks
+ private RestHighLevelClient restHighLevelClient;
+
+ @InjectMocks
+ private IndexSchema indexSchema;
+ @InjectMocks
+ private IndicesClient indicesClient;
+
+ @InjectMocks
+ private AcknowledgedResponse mappingResponse;
+
+ @Before
+ public void setup() throws IOException {
+ Map<String, String> dataMapping = new HashMap<>();
+ dataMapping.put("Location", "geo_point");
+ Map<String, Object> metaMapping = new HashMap<>();
+ metaMapping.put(RecordMetaAttribute.ID.getValue(), "keyword");
+ this.indexSchema = IndexSchema.builder().kind(kind).type(type).dataSchema(dataMapping).metaSchema(metaMapping)
+ .build();
+
+ this.indicesClient = PowerMockito.mock(IndicesClient.class);
+ this.restHighLevelClient = PowerMockito.mock(RestHighLevelClient.class);
+
+ when(this.restHighLevelClient.getLowLevelClient()).thenReturn(restClient);
+ when(this.restClient.performRequest(ArgumentMatchers.any())).thenReturn(response);
+ when(this.response.getStatusLine()).thenReturn(statusLine);
+ when(this.statusLine.getStatusCode()).thenReturn(200);
+ }
+
+ @Test
+ public void should_returnValidMapping_givenFalseMerge_createMappingTest() {
+ try {
+ String mapping = this.sut.createMapping(restHighLevelClient, indexSchema, index, false);
+ assertEquals(mappingValid, mapping);
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_returnValidMapping_givenTrueMerge_createMappingTest() {
+ try {
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+
+ String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true);
+ assertEquals(this.mappingValid, mapping);
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_returnValidMapping_givenExistType_createMappingTest() {
+ try {
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+
+ IndexerMappingServiceImpl indexerMappingServiceLocal = PowerMockito.spy(new IndexerMappingServiceImpl());
+ doReturn(false).when(indexerMappingServiceLocal).isTypeExist(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any());
+ String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true);
+ assertEquals(this.mappingValid, mapping);
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_update_indices_field_with_keyword_when_valid_indices() throws Exception {
+ try {
+ Set<String> indices = new HashSet<String>();
+ indices.add("indices 1");
+ GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
+ XContentBuilder builder = XContentFactory.jsonBuilder();
+ builder.startObject();
+ builder.field("any field", new HashMap());
+ builder.endObject();
+ BytesReference bytesReference = BytesReference.bytes(builder);
+ FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
+ Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ mapBuilder.put("data.any field", mappingMetaData);
+ Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ mappingBuilder.put("any index 1", mapBuilder);
+ mappingBuilder.put("any index 2", mapBuilder);
+ Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ mapping.put("indices 1", mappingBuilder);
+ when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ BulkByScrollResponse response = mock(BulkByScrollResponse.class);
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+
+ this.sut.updateIndexMappingForIndicesOfSameType( indices,"any field");
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throw_exception_if_someIndex_is_invalid_andWeIndexfield_with_keyword() throws Exception {
+ try {
+ Set<String> indices = new HashSet<String>();
+ indices.add("invalid 1");
+ GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
+ XContentBuilder builder = XContentFactory.jsonBuilder();
+ builder.startObject();
+ builder.field("any field", new HashMap());
+ builder.endObject();
+ BytesReference bytesReference = BytesReference.bytes(builder);
+ FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
+ Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ mapBuilder.put("data.any field", mappingMetaData);
+ Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ mappingBuilder.put("any index 1", mapBuilder);
+ mappingBuilder.put("any index 2", mapBuilder);
+ Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ mapping.put("indices 1", mappingBuilder);
+ when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ BulkByScrollResponse response = mock(BulkByScrollResponse.class);
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+
+ this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
+ } catch (Exception e) {
+ throw e;
+ }
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throw_exception_if_type_of_index_is_invalid_andWeIndexfield_with_keyword() throws Exception {
+ try {
+ Set<String> indices = new HashSet<String>();
+ indices.add("indices 1");
+ GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
+ XContentBuilder builder = XContentFactory.jsonBuilder();
+ builder.startObject();
+ builder.field("any field", new HashMap());
+ builder.endObject();
+ BytesReference bytesReference = BytesReference.bytes(builder);
+ FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
+ Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ mapBuilder.put("data.any field", mappingMetaData);
+ Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ mappingBuilder.put("any index 1", mapBuilder);
+ mappingBuilder.put("any index 2", mapBuilder);
+ Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ mapping.put("indices 1", mappingBuilder);
+ when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ BulkByScrollResponse response = mock(BulkByScrollResponse.class);
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field invalid");
+ } catch (Exception e) {
+ throw e;
+ }
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throw_exception_if_elastic_search_failedToFetch_andWeIndexfield_with_keyword() throws Exception {
+ try {
+
+ Set<String> indices = new HashSet<String>();
+ indices.add("indices 1");
+ indices.add("indices Invalid");
+ GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenThrow(new ElasticsearchException(""));
+ XContentBuilder builder = XContentFactory.jsonBuilder();
+ builder.startObject();
+ builder.field("any field", new HashMap());
+ builder.endObject();
+ BytesReference bytesReference = BytesReference.bytes(builder);
+ FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
+ Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ mapBuilder.put("data.any field", mappingMetaData);
+ Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ mappingBuilder.put("any index 1", mapBuilder);
+ mappingBuilder.put("any index 2", mapBuilder);
+ Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ mapping.put("indices 1", mappingBuilder);
+ when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ BulkByScrollResponse response = mock(BulkByScrollResponse.class);
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
+ } catch (AppException e) {
+ throw e;
+ }
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throw_exception_when_elastic_failedToIndex_indices_field_with_keyword() {
+ try {
+ Set<String> indices = new HashSet<String>();
+ indices.add("indices 1");
+ indices.add("indices Invalid");
+ GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
+ XContentBuilder builder = XContentFactory.jsonBuilder();
+ builder.startObject();
+ builder.field("any field", new HashMap());
+ builder.endObject();
+ BytesReference bytesReference = BytesReference.bytes(builder);
+ FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
+ Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ mapBuilder.put("data.any field", mappingMetaData);
+ Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ mappingBuilder.put("any index 1", mapBuilder);
+ mappingBuilder.put("any index 2", mapBuilder);
+ Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ mapping.put("indices 1", mappingBuilder);
+ when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ BulkByScrollResponse response = mock(BulkByScrollResponse.class);
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
+ when(this.indicesClient.putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class))).thenThrow(new ElasticsearchException(""));
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
+ } catch (AppException e) {
+ throw e;
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+}
diff --git a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerSchemaServiceTest.java b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerSchemaServiceTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..8427f8c0b8b6f5811725e58d2b73af0a119b5d6a
--- /dev/null
+++ b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerSchemaServiceTest.java
@@ -0,0 +1,329 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.service;
+
+import org.apache.http.HttpStatus;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
+import org.opengroup.osdu.core.common.model.indexer.OperationType;
+import org.opengroup.osdu.core.common.spi.indexer.ISchemaCache;
+import org.opengroup.osdu.indexer.service.IndexSchemaServiceImpl;
+import org.opengroup.osdu.indexer.service.IndexerMappingService;
+import org.opengroup.osdu.indexer.service.StorageService;
+import org.opengroup.osdu.core.common.model.coreis.RequestStatus;
+import org.opengroup.osdu.core.common.service.coreis.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.service.coreis.IndicesService;
+import org.opengroup.osdu.core.common.model.coreis.AppException;
+import org.opengroup.osdu.is.core.util.ElasticClientHandler;
+import org.opengroup.osdu.core.common.service.coreis.ElasticIndexNameResolver;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.*;
+import static org.mockito.ArgumentMatchers.*;
+import static org.mockito.Mockito.*;
+import static org.mockito.MockitoAnnotations.initMocks;
+import static org.powermock.api.mockito.PowerMockito.mock;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@Ignore
+@RunWith(SpringRunner.class)
+@PrepareForTest({RestHighLevelClient.class})
+public class IndexerSchemaServiceTest {
+
+ private final String kind = "tenant:test:test:1.0.0";
+ private final String emptySchema = null;
+ private final String someSchema = "{\"kind\":\"tenant:test:test:1.0.0\", \"schema\":[{\"path\":\"test-path\", \"kind\":\"tenant:test:test:1.0.0\"}]}";
+
+ @Mock
+ private JaxRsDpsLog log;
+ @Mock
+ private StorageService storageService;
+ @Mock
+ private ElasticClientHandler elasticClientHandler;
+ @Mock
+ private ElasticIndexNameResolver elasticIndexNameResolver;
+ @Mock
+ private IndexerMappingService mappingService;
+ @Mock
+ private IndicesService indicesService;
+ @Mock
+ private ISchemaCache schemaCache;
+ @InjectMocks
+ private IndexSchemaServiceImpl sut;
+
+ @Before
+ public void setup() {
+ initMocks(this);
+ RestHighLevelClient restHighLevelClient = mock(RestHighLevelClient.class);
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ }
+
+ @Test
+ public void should_returnNull_givenEmptySchema_getIndexerInputSchemaSchemaTest() throws Exception {
+ when(storageService.getStorageSchema(any())).thenReturn(emptySchema);
+
+ IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind);
+
+ Assert.assertNotNull(indexSchema);
+ }
+
+ @Test
+ public void should_returnValidResponse_givenValidSchema_getIndexerInputSchemaTest() throws Exception {
+ when(storageService.getStorageSchema(any())).thenReturn(someSchema);
+
+ IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind);
+
+ Assert.assertEquals(kind, indexSchema.getKind());
+ }
+
+ @Test
+ public void should_returnValidResponse_givenValidSchemaWithCacheHit_getIndexerInputSchemaTest() throws Exception {
+ when(storageService.getStorageSchema(any())).thenReturn(someSchema);
+ when(this.schemaCache.get(kind + "_flattened")).thenReturn(someSchema);
+
+ IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind);
+
+ Assert.assertEquals(kind, indexSchema.getKind());
+ }
+
+ @Test
+ public void should_throw500_givenInvalidSchemaCacheHit_getIndexerInputSchemaTest() {
+ try {
+ String invalidSchema = "{}}";
+ when(storageService.getStorageSchema(any())).thenReturn(invalidSchema);
+
+ this.sut.getIndexerInputSchema(kind);
+ fail("Should throw exception");
+ } catch (AppException e) {
+ Assert.assertEquals(HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getError().getCode());
+ Assert.assertEquals("An error has occurred while normalizing the schema.", e.getError().getMessage());
+ } catch (Exception e) {
+ fail("Should not throw exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_return_basic_schema_when_storage_returns_no_schema() {
+ IndexSchema returnedSchema = this.sut.getIndexerInputSchema(kind);
+
+ assertNotNull(returnedSchema.getDataSchema());
+ assertNotNull(returnedSchema);
+ assertEquals(kind, returnedSchema.getKind());
+ }
+
+ @Test
+ public void should_create_schema_when_storage_returns_valid_schema() throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"startDate\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"endDate\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"type \"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"itemguid\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.create_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(false);
+ when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
+
+ this.sut.processSchemaMessages(schemaMessages);
+
+ verify(this.mappingService, times(1)).getIndexMappingFromRecordSchema(any());
+ verify(this.indicesService, times(1)).createIndex(any(), any(), any(), any(), any());
+ verifyNoMoreInteractions(this.mappingService);
+ }
+
+ @Test
+ public void should_merge_mapping_when_storage_returns_valid_schema() throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"startDate\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.create_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
+
+ this.sut.processSchemaMessages(schemaMessages);
+
+ verify(this.indicesService, times(0)).createIndex(any(), any(), any(), any(), any());
+ verify(this.mappingService, times(1)).createMapping(any(), any(), any(), anyBoolean());
+ verifyNoMoreInteractions(this.mappingService);
+ }
+
+ @Test
+ public void should_throw_mapping_conflict_when_elastic_backend_cannot_process_schema_changes() throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String reason = String.format("Could not create type mapping %s/completion.", kind.replace(":", "-"));
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.create_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
+ when(this.mappingService.createMapping(any(), any(), any(), anyBoolean())).thenThrow(new AppException(HttpStatus.SC_BAD_REQUEST, reason, ""));
+
+ try {
+ this.sut.processSchemaMessages(schemaMessages);
+ } catch (AppException e){
+ assertEquals(e.getError().getCode(), RequestStatus.SCHEMA_CONFLICT);
+ assertEquals(e.getError().getMessage(), "error creating or merging index mapping");
+ assertEquals(e.getError().getReason(), reason);
+ } catch (Exception e) {
+ fail("Should not throw this exception " + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_throw_genericAppException_when_elastic_backend_cannot_process_schema_changes() throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String reason = String.format("Could not create type mapping %s/completion.", kind.replace(":", "-"));
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.create_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
+ when(this.mappingService.createMapping(any(), any(), any(), anyBoolean())).thenThrow(new AppException(HttpStatus.SC_FORBIDDEN, reason, "blah"));
+
+ try {
+ this.sut.processSchemaMessages(schemaMessages);
+ } catch (AppException e){
+ assertEquals(e.getError().getCode(), HttpStatus.SC_FORBIDDEN);
+ assertEquals(e.getError().getMessage(), "blah");
+ assertEquals(e.getError().getReason(), reason);
+ } catch (Exception e) {
+ fail("Should not throw this exception " + e.getMessage());
+ }
+ }
+
+
+ @Test
+ public void should_log_and_do_nothing_when_storage_returns_invalid_schema() throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"" +
+ "}";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.create_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
+
+ this.sut.processSchemaMessages(schemaMessages);
+
+ verify(this.log).warning(eq("schema not found for kind: tenant1:avocet:completion:1.0.0"));
+ }
+
+ @Test
+ public void should_invalidateCache_when_purge_schema_and_schema_found_in_cache() throws IOException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.purge_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.schemaCache.get(kind)).thenReturn("schema");
+ when(this.schemaCache.get(kind + "_flattened")).thenReturn("flattened schema");
+
+ this.sut.processSchemaMessages(schemaMessages);
+
+ verify(this.schemaCache, times(2)).get(anyString());
+ verify(this.schemaCache, times(2)).delete(anyString());
+ }
+
+ @Test
+ public void should_log_warning_when_purge_schema_and_schema_not_found_in_cache() throws IOException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.purge_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(false);
+
+ this.sut.processSchemaMessages(schemaMessages);
+
+ verify(this.log).warning(eq(String.format("Kind: %s not found", kind)));
+ }
+}
diff --git a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerServiceTest.java b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerServiceTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..0043a0d898b93ace6e7a49c7f2e67b6f17f41726
--- /dev/null
+++ b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerServiceTest.java
@@ -0,0 +1,337 @@
+package org.opengroup.osdu.indexer.ibm.service;//// Copyright 2017-2019, Schlumberger
+////
+//// Licensed under the Apache License, Version 2.0 (the "License");
+//// you may not use this file except in compliance with the License.
+//// You may obtain a copy of the License at
+////
+//// http://www.apache.org/licenses/LICENSE-2.0
+////
+//// Unless required by applicable law or agreed to in writing, software
+//// distributed under the License is distributed on an "AS IS" BASIS,
+//// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//// See the License for the specific language governing permissions and
+//// limitations under the License.
+//
+//package org.opendes.indexer.service;
+//
+//import com.google.gson.Gson;
+//import com.google.gson.reflect.TypeToken;
+//import org.elasticsearch.action.bulk.BulkItemResponse;
+//import org.elasticsearch.action.bulk.BulkResponse;
+//import org.elasticsearch.client.RequestOptions;
+//import org.elasticsearch.client.RestHighLevelClient;
+//import org.junit.Before;
+//import org.junit.Ignore;
+//import org.junit.Test;
+//import org.junit.runner.RunWith;
+//import org.mockito.InjectMocks;
+//import org.mockito.Mock;
+//import org.mockito.Spy;
+//import org.opendes.client.api.DpsHeaders;
+//import org.opendes.core.logging.JaxRsDpsLog;
+//import org.opendes.core.model.DeploymentEnvironment;
+//import org.opendes.core.model.RecordChangedMessages;
+//import org.opendes.core.service.IndicesService;
+//import org.opendes.core.util.Config;
+//import org.opendes.core.util.ElasticClientHandler;
+//import org.opendes.core.util.ElasticIndexNameResolver;
+//import org.opendes.core.util.HeadersUtil;
+//import org.opendes.indexer.logging.AuditLogger;
+//import org.opendes.indexer.model.*;
+//import org.opendes.indexer.publish.IPublisher;
+//import org.opendes.indexer.util.IRequestInfo;
+//import org.opendes.indexer.util.IndexerQueueTaskBuilder;
+//import org.opendes.indexer.util.JobStatus;
+//import org.opendes.indexer.util.RecordInfo;
+//import org.powermock.core.classloader.annotations.PrepareForTest;
+//import org.springframework.context.annotation.Lazy;
+//import org.springframework.test.context.junit4.SpringRunner;
+//
+//import javax.inject.Inject;
+//import java.io.IOException;
+//import java.lang.reflect.Type;
+//import java.util.*;
+//
+//import static java.util.Collections.singletonList;
+//import static org.junit.Assert.*;
+//import static org.mockito.Matchers.any;
+//import static org.mockito.Mockito.verify;
+//import static org.mockito.Mockito.when;
+//import static org.powermock.api.mockito.PowerMockito.mock;
+//import static org.powermock.api.mockito.PowerMockito.mockStatic;
+//
+//@Ignore
+//@RunWith(SpringRunner.class)
+//@PrepareForTest({RestHighLevelClient.class, BulkResponse.class, StorageAcl.class, HeadersUtil.class, Config.class})
+//public class IndexerServiceTest {
+//
+// private final String pubsubMsg = "[{\"id\":\"tenant1:doc:test1\",\"kind\":\"tenant1:testindexer1:well:1.0.0\",\"op\":\"update\"}," +
+// "{\"id\":\"tenant1:doc:test2\",\"kind\":\"tenant1:testindexer2:well:1.0.0\",\"op\":\"create\"}]";
+// private final String kind1 = "tenant1:testindexer1:well:1.0.0";
+// private final String kind2 = "tenant1:testindexer2:well:1.0.0";
+// private final String recordId1 = "tenant1:doc:test1";
+// private final String recordId2 = "tenant1:doc:test2";
+// private final String failureMassage = "test failure";
+//
+// @Mock
+// private IndexSchemaService indexSchemaService;
+// @Mock
+// private IndicesService indicesService;
+// @Mock
+// private IndexerMappingService indexerMappingService;
+// @Mock
+// private StorageService storageService;
+// @Mock
+// private IPublisher publisherImpl;
+// @Mock
+// private RestHighLevelClient restHighLevelClient;
+// @Mock
+// private ElasticClientHandler elasticClientHandler;
+// @Mock
+// private BulkResponse bulkResponse;
+// @Mock
+// private IRequestInfo requestInfo;
+// @Mock
+// private ElasticIndexNameResolver elasticIndexNameResolver;
+// @Mock
+// private AttributeParsingServiceImpl attributeParsingServiceImpl;
+// @Mock
+// private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
+// @Mock
+// private JaxRsDpsLog log;
+// @Mock
+// private AuditLogger auditLogger;
+// @InjectMocks
+// private IndexerServiceImpl sut;
+// @InjectMocks @Spy
+// private JobStatus jobStatus = new JobStatus();
+//
+// @Inject
+// @Lazy
+// private DpsHeaders dpsHeaders;
+// private RecordChangedMessages recordChangedMessages;
+// private List<RecordInfo> recordInfos;
+//
+// @Before
+// public void setup() throws IOException {
+//
+// mockStatic(StorageAcl.class);
+// mockStatic(Config.class);
+//
+// when(Config.getDeploymentEnvironment()).thenReturn(DeploymentEnvironment.LOCAL);
+// when(Config.getElasticClusterName()).thenReturn("CLUSTER");
+// when(Config.getElasticServerAddress()).thenReturn("testsite");
+//
+// dpsHeaders = new DpsHeaders();
+// dpsHeaders.put(AppEngineHeaders.TASK_QUEUE_RETRY_COUNT, "1");
+// dpsHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
+// when(requestInfo.getHeaders()).thenReturn(dpsHeaders);
+// when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(dpsHeaders.getHeaders());
+//
+// Type listType = new TypeToken<List<RecordInfo>>() {}.getType();
+// recordInfos = (new Gson()).fromJson(pubsubMsg, listType);
+//
+// when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+// when(restHighLevelClient.bulk(any(), any(RequestOptions.class))).thenReturn(bulkResponse);
+//
+// BulkItemResponse[] responses = new BulkItemResponse[]{prepareResponseFail(), prepareResponseSuccess()};
+// when(bulkResponse.getItems()).thenReturn(responses);
+// Map<String, String> attr = new HashMap<>();
+// attr.put(DpsHeaders.ACCOUNT_ID, "slb");
+// recordChangedMessages = RecordChangedMessages.builder().attributes(attr).messageId("xxxx").publishTime("2000-01-02T10:10:44+0000").data("{}").build();
+// when(StorageAcl.flattenAcl(any())).thenReturn(null);
+// }
+//
+// @Test
+// public void should_returnNull_givenEmptyJobSubInfo_processRecordChangedMessageTest() throws Exception {
+// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, new ArrayList<>());
+//
+// assertNull(jobStatus);
+// }
+//
+// @Test
+// public void should_returnValidJobStatus_givenNullSchema_processRecordChangedMessageTest() {
+// try {
+// indexSchemaServiceMock(kind1, null);
+// indexSchemaServiceMock(kind2, null);
+// List<ConversionStatus> conversionStatus = new LinkedList<>();
+// List<Records.Entity> validRecords = new ArrayList<>();
+// Map<String, Object> storageData = new HashMap<>();
+// storageData.put("schema1", "test-value");
+// storageData.put("schema2", "test-value");
+// storageData.put("schema3", "test-value");
+// storageData.put("schema4", "test-value");
+// storageData.put("schema5", "test-value");
+// storageData.put("schema6", "test-value");
+// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build());
+// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build();
+//
+// when(storageService.getStorageRecords(any())).thenReturn(storageRecords);
+// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true);
+//
+// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
+//
+// assertEquals(2, jobStatus.getStatusesList().size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).size());
+// } catch (Exception e) {
+// fail("Should not throw this exception" + e.getMessage());
+// }
+// }
+//
+// @Test
+// public void should_returnValidJobStatus_givenFailedUnitsConversion_processRecordChangedMessageTest() {
+// try {
+// indexSchemaServiceMock(kind1, null);
+// indexSchemaServiceMock(kind2, null);
+// List<ConversionStatus> conversionStatuses = new LinkedList<>();
+// List<String> status=new ArrayList<>();
+// status.add("crs bla bla");
+// ConversionStatus conversionStatus=ConversionStatus.builder().status("ERROR").errors(status).id(recordId2).build();
+// conversionStatuses.add(conversionStatus);
+// List<Records.Entity> validRecords = new ArrayList<>();
+// Map<String, Object> storageData = new HashMap<>();
+// storageData.put("schema1", "test-value");
+// storageData.put("schema2", "test-value");
+// storageData.put("schema3", "test-value");
+// storageData.put("schema4", "test-value");
+// storageData.put("schema5", "test-value");
+// storageData.put("schema6", "test-value");
+// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build());
+// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatuses).build();
+//
+// when(storageService.getStorageRecords(any())).thenReturn(storageRecords);
+// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true);
+//
+// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
+//
+// assertEquals(2, jobStatus.getStatusesList().size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).size());
+// assertTrue(jobStatus.getJobStatusByRecordId(jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).get(0)).getIndexProgress().getTrace().contains("crs bla bla"));
+// } catch (Exception e) {
+// fail("Should not throw this exception" + e.getMessage());
+// }
+// }
+//
+// @Test
+// public void should_returnValidJobStatus_givenNullSchemaForARecord_processRecordChangedMessageTest() {
+// try {
+// List<Records.Entity> validRecords = new ArrayList<>();
+// List<ConversionStatus> conversionStatus = new LinkedList<>();
+// Map<String, Object> storageData = new HashMap<>();
+// storageData.put("schema1", "test-value");
+// storageData.put("schema2", "test-value");
+// storageData.put("schema3", "test-value");
+// storageData.put("schema4", "test-value");
+// storageData.put("schema5", "test-value");
+// storageData.put("schema6", "test-value");
+// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build());
+// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build();
+// when(storageService.getStorageRecords(any())).thenReturn(storageRecords);
+//
+// Map<String, String> schema = createSchema();
+// indexSchemaServiceMock(kind1, schema);
+// indexSchemaServiceMock(kind2, null);
+// when(elasticIndexNameResolver.getIndexNameFromKind(kind2)).thenReturn("tenant1-testindexer2-well-1.0.0");
+// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true);
+// JobStatus jobStatus = sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
+//
+// assertEquals(2, jobStatus.getStatusesList().size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).size());
+// assertEquals("Indexed Successfully", jobStatus.getStatusesList().get(1).getIndexProgress().getTrace().pop());
+// assertEquals("schema not found", jobStatus.getStatusesList().get(1).getIndexProgress().getTrace().pop());
+// } catch (Exception e) {
+// fail("Should not throw this exception" + e.getMessage());
+// }
+// }
+//
+// @Test
+// public void should_returnValidJobStatus_givenValidCreateAndUpdateRecords_processRecordChangedMessagesTest() {
+// try {
+// Map<String, Object> storageData = new HashMap<>();
+// storageData.put("schema1", "test-value");
+// List<ConversionStatus> conversionStatus = new LinkedList<>();
+// List<Records.Entity> validRecords = new ArrayList<>();
+// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build());
+// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build();
+//
+// when(storageService.getStorageRecords(any())).thenReturn(storageRecords);
+// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true);
+// Map<String, String> schema = createSchema();
+// indexSchemaServiceMock(kind2, schema);
+// indexSchemaServiceMock(kind1, null);
+// JobStatus jobStatus = sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
+//
+// assertEquals(2, jobStatus.getStatusesList().size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.SUCCESS).size());
+// } catch (Exception e) {
+// fail("Should not throw this exception" + e.getMessage());
+// }
+// }
+//
+// @Test
+// public void should_properlyUpdateAuditLogs_givenValidCreateAndUpdateRecords() {
+// try {
+// Map<String, Object> storageData = new HashMap<>();
+// List<ConversionStatus> conversionStatus = new LinkedList<>();
+//
+// storageData.put("schema1", "test-value");
+// List<Records.Entity> validRecords = new ArrayList<>();
+// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build());
+// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build();
+//
+// when(this.storageService.getStorageRecords(any())).thenReturn(storageRecords);
+// when(this.indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true);
+// Map<String, String> schema = createSchema();
+// indexSchemaServiceMock(kind2, schema);
+// indexSchemaServiceMock(kind1, null);
+// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
+//
+// assertEquals(2, jobStatus.getStatusesList().size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.SUCCESS).size());
+//
+// verify(this.auditLogger).indexCreateRecordSuccess(singletonList("RecordStatus(id=tenant1:doc:test2, kind=tenant1:testindexer2:well:1.0.0, operationType=create, status=SUCCESS)"));
+// verify(this.auditLogger).indexUpdateRecordFail(singletonList("RecordStatus(id=tenant1:doc:test1, kind=tenant1:testindexer1:well:1.0.0, operationType=update, status=FAIL)"));
+// } catch (Exception e) {
+// fail("Should not throw this exception" + e.getMessage());
+// }
+// }
+//
+// private BulkItemResponse prepareResponseFail() {
+// BulkItemResponse responseFail = mock(BulkItemResponse.class);
+// when(responseFail.isFailed()).thenReturn(true);
+// when(responseFail.getFailureMessage()).thenReturn(failureMassage);
+// when(responseFail.getId()).thenReturn(recordId1);
+// when(responseFail.getFailure()).thenReturn(new BulkItemResponse.Failure("failure index", "failure type", "failure id", new Exception("test failure")));
+// return responseFail;
+// }
+//
+// private BulkItemResponse prepareResponseSuccess() {
+// BulkItemResponse responseSuccess = mock(BulkItemResponse.class);
+// when(responseSuccess.getId()).thenReturn(recordId2);
+// return responseSuccess;
+// }
+//
+// private void indexSchemaServiceMock(String kind, Map<String, String> schema) {
+// if (schema == null) {
+// IndexSchema indexSchema = IndexSchema.builder().kind(kind).dataSchema(null).build();
+// when(indexSchemaService.getIndexerInputSchema(kind)).thenReturn(indexSchema);
+// } else {
+// IndexSchema indexSchema = IndexSchema.builder().kind(kind).dataSchema(schema).build();
+// when(indexSchemaService.getIndexerInputSchema(kind)).thenReturn(indexSchema);
+// }
+// }
+//
+// private Map<String, String> createSchema() {
+// Map<String, String> schema = new HashMap<>();
+// schema.put("schema1", "keyword");
+// schema.put("schema2", "boolean");
+// schema.put("schema3", "date");
+// schema.put("schema6", "object");
+// return schema;
+// }
+//}
diff --git a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/ReindexServiceTest.java b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/ReindexServiceTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..56134e570be0cbfedd61fcb8c229505cc826374a
--- /dev/null
+++ b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/ReindexServiceTest.java
@@ -0,0 +1,145 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.service;
+
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentMatchers;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.model.core.DpsHeaders;
+import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
+import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
+import org.opengroup.osdu.indexer.service.ReindexServiceImpl;
+import org.opengroup.osdu.indexer.service.StorageService;
+import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
+import org.opengroup.osdu.core.common.service.coreis.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.spi.coreis.IRequestInfo;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.util.*;
+
+import static org.junit.Assert.fail;
+import static org.mockito.MockitoAnnotations.initMocks;
+import static org.powermock.api.mockito.PowerMockito.mockStatic;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@Ignore
+@RunWith(SpringRunner.class)
+public class ReindexServiceTest {
+
+ private final String cursor = "100";
+
+ private final String correlationId = UUID.randomUUID().toString();
+
+ @Mock
+ private StorageService storageService;
+
+ @Mock
+ private Map<String, String> httpHeaders;
+ @Mock
+ private IRequestInfo requestInfo;
+ @Mock
+ private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
+ @Mock
+ private JaxRsDpsLog log;
+ @InjectMocks
+ private ReindexServiceImpl sut;
+
+ private RecordReindexRequest recordReindexRequest;
+ private RecordQueryResponse recordQueryResponse;
+
+ @Before
+ public void setup() {
+ initMocks(this);
+
+ mockStatic(UUID.class);
+
+ recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor(cursor).build();
+ recordQueryResponse = new RecordQueryResponse();
+
+ httpHeaders = new HashMap<>();
+ httpHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
+ httpHeaders.put(DpsHeaders.CORRELATION_ID, correlationId);
+ DpsHeaders standardHeaders = DpsHeaders.createFromMap(httpHeaders);
+ when(requestInfo.getHeaders()).thenReturn(standardHeaders);
+ when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(httpHeaders);
+ }
+
+ @Test
+ public void should_returnNull_givenNullResponseResult_reIndexRecordsTest() {
+ try {
+ recordQueryResponse.setResults(null);
+ when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
+
+ String response = sut.reindexRecords(recordReindexRequest);
+
+ Assert.assertNull(response);
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_returnNull_givenEmptyResponseResult_reIndexRecordsTest() {
+ try {
+ recordQueryResponse.setResults(new ArrayList<>());
+ when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
+
+ String response = sut.reindexRecords(recordReindexRequest);
+
+ Assert.assertNull(response);
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_returnRecordQueryRequestPayload_givenValidResponseResult_reIndexRecordsTest() {
+ try {
+ recordQueryResponse.setCursor(cursor);
+ List<String> results = new ArrayList<>();
+ results.add("test1");
+ recordQueryResponse.setResults(results);
+ when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
+
+ String taskQueuePayload = sut.reindexRecords(recordReindexRequest);
+
+ Assert.assertEquals("{\"kind\":\"tenant:test:test:1.0.0\",\"cursor\":\"100\"}", taskQueuePayload);
+ } catch (Exception e) {
+ fail("Should not throw exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_returnRecordChangedMessage_givenValidResponseResult_reIndexRecordsTest() {
+ try {
+ List<String> results = new ArrayList<>();
+ results.add("test1");
+ recordQueryResponse.setResults(results);
+ when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
+
+ String taskQueuePayload = sut.reindexRecords(recordReindexRequest);
+
+ Assert.assertEquals(String.format("{\"data\":\"[{\\\"id\\\":\\\"test1\\\",\\\"kind\\\":\\\"tenant:test:test:1.0.0\\\",\\\"op\\\":\\\"create\\\"}]\",\"attributes\":{\"slb-correlation-id\":\"%s\"}}", correlationId), taskQueuePayload);
+ } catch (Exception e) {
+ fail("Should not throw exception" + e.getMessage());
+ }
+ }
+}
diff --git a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/StorageServiceTest.java b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/StorageServiceTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..68f50a26dd70274ac79305478a134d14003fd651
--- /dev/null
+++ b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/StorageServiceTest.java
@@ -0,0 +1,211 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.service;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentMatchers;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
+import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
+import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
+import org.opengroup.osdu.core.common.model.indexer.Records;
+import org.opengroup.osdu.indexer.service.StorageServiceImpl;
+import org.opengroup.osdu.core.common.model.indexer.JobStatus;
+import org.opengroup.osdu.core.common.model.coreis.HttpResponse;
+import org.opengroup.osdu.core.common.service.coreis.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.spi.coreis.IRequestInfo;
+import org.opengroup.osdu.core.common.service.coreis.UrlFetchService;
+import org.opengroup.osdu.core.common.model.coreis.AppException;
+import org.springframework.http.HttpStatus;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.lang.reflect.Type;
+import java.net.URISyntaxException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+
+import static org.junit.Assert.*;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@Ignore
+@RunWith(SpringRunner.class)
+public class StorageServiceTest {
+
+ @Mock
+ private UrlFetchService urlFetchService;
+ @Mock
+ private JobStatus jobStatus;
+ @Mock
+ private JaxRsDpsLog log;
+ @Mock
+ private IRequestInfo requestInfo;
+ @InjectMocks
+ private StorageServiceImpl sut;
+
+ private List<String> ids;
+
+ @Before
+ public void setup() {
+
+ String recordChangedMessages = "[{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"purge\"}," +
+ "{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"create\"}]";
+
+ when(this.requestInfo.getHeadersMap()).thenReturn(new HashMap<>());
+
+ Type listType = new TypeToken<List<RecordInfo>>() {}.getType();
+
+ List<RecordInfo> msgs = (new Gson()).fromJson(recordChangedMessages, listType);
+ jobStatus.initialize(msgs);
+ ids = Arrays.asList("tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465", "tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465");
+ }
+
+ @Test
+ public void should_return404_givenNullData_getValidStorageRecordsTest() throws URISyntaxException {
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(null);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ should_return404_getValidStorageRecordsTest();
+ }
+
+ @Test
+ public void should_return404_givenEmptyData_getValidStorageRecordsTest() throws URISyntaxException {
+
+ String emptyDataFromStorage = "{\"records\":[],\"notFound\":[]}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(emptyDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ should_return404_getValidStorageRecordsTest();
+ }
+
+ @Test
+ public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsTest() throws URISyntaxException {
+
+ String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"]}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(httpResponse);
+ Records storageRecords = this.sut.getStorageRecords(ids);
+
+ assertEquals(1, storageRecords.getRecords().size());
+ }
+
+ @Test
+ public void should_returnValidResponse_givenValidRecordQueryRequest_getRecordListByKind() throws Exception {
+
+ RecordReindexRequest recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor("100").build();
+
+ HttpResponse httpResponse = new HttpResponse();
+ httpResponse.setBody(new Gson().toJson(recordReindexRequest, RecordReindexRequest.class));
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ RecordQueryResponse recordQueryResponse = this.sut.getRecordsByKind(recordReindexRequest);
+
+ assertEquals("100", recordQueryResponse.getCursor());
+ assertNull(recordQueryResponse.getResults());
+ }
+
+ @Test
+ public void should_returnValidResponse_givenValidKind_getSchemaByKind() throws Exception {
+
+ String validSchemaFromStorage = "{" +
+ " \"kind\": \"tenant:test:test:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"msg\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"references.entity\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]," +
+ " \"ext\": null" +
+ "}";
+ String kind = "tenant:test:test:1.0.0";
+
+ HttpResponse httpResponse = new HttpResponse();
+ httpResponse.setResponseCode(HttpStatus.OK.value());
+ httpResponse.setBody(validSchemaFromStorage);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ String recordSchemaResponse = this.sut.getStorageSchema(kind);
+
+ assertNotNull(recordSchemaResponse);
+ }
+
+ @Test
+ public void should_returnNullResponse_givenAbsentKind_getSchemaByKind() throws Exception {
+
+ String kind = "tenant:test:test:1.0.0";
+
+ HttpResponse httpResponse = new HttpResponse();
+ httpResponse.setResponseCode(HttpStatus.NOT_FOUND.value());
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ String recordSchemaResponse = this.sut.getStorageSchema(kind);
+
+ assertNull(recordSchemaResponse);
+ }
+
+ @Test
+ public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsWithInvalidConversionTest() throws URISyntaxException {
+
+ String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"],\"conversionStatuses\": [{\"id\":\"testid\",\"status\":\"ERROR\",\"errors\":[\"conversion error occured\"] } ]}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(httpResponse);
+ Records storageRecords = this.sut.getStorageRecords(ids);
+
+ assertEquals(1, storageRecords.getRecords().size());
+
+ assertEquals(1, storageRecords.getConversionStatuses().get(0).getErrors().size());
+
+ assertEquals("conversion error occured", storageRecords.getConversionStatuses().get(0).getErrors().get(0));
+ }
+
+ private void should_return404_getValidStorageRecordsTest() {
+ try {
+ this.sut.getStorageRecords(ids);
+ fail("Should throw exception");
+ } catch (AppException e) {
+ assertEquals(HttpStatus.NOT_FOUND, e.getError().getCode());
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+}
diff --git a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/TenantInfoServiceTest.java b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/TenantInfoServiceTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..ba94fb4a96805a474e7c9bc82dcfa4ac13d225b4
--- /dev/null
+++ b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/TenantInfoServiceTest.java
@@ -0,0 +1,90 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.service;
+
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.model.core.DpsHeaders;
+import org.opengroup.osdu.core.common.model.core.ITenantFactory;
+import org.opengroup.osdu.core.common.model.core.TenantInfo;
+import org.opengroup.osdu.indexer.service.TenantInfoServiceImpl;
+import org.opengroup.osdu.core.common.spi.coreis.IHeadersInfo;
+import org.opengroup.osdu.core.common.model.coreis.AppException;
+import org.springframework.http.HttpHeaders;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.Mockito.when;
+
+@RunWith(SpringRunner.class)
+public class TenantInfoServiceTest {
+
+ private static final String HEADER_NAME = "ANY_HEADER";
+ private static final String HEADER_VALUE = "ANY_VALUE";
+
+ @Mock
+ private ITenantFactory tenantFactory;
+ @Mock
+ private IHeadersInfo headersInfo;
+ @InjectMocks
+ private TenantInfoServiceImpl sut;
+
+ @Mock
+ private TenantInfo info;
+
+ @Mock
+ private HttpHeaders httpHeaders;
+
+ @InjectMocks
+ private DpsHeaders HEADERS;
+
+ @Before
+ public void setup() {
+ HEADERS.put(HEADER_NAME, HEADER_VALUE);
+ }
+
+ @Ignore
+ @Test
+ public void should_return_validTenant_given_validAccountId() {
+
+ when(this.info.getName()).thenReturn("tenant1");
+ when(tenantFactory.getTenantInfo("tenant1")).thenReturn(info);
+
+ when(this.headersInfo.getHeaders()).thenReturn(HEADERS);
+
+ when(this.headersInfo.getPartitionId()).thenReturn("tenant1");
+
+ when(this.sut.getTenantInfo()).thenReturn(info);
+
+ assertNotNull(this.sut.getTenantInfo());
+ assertEquals("tenant1", this.sut.getTenantInfo().getName());
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throwException_given_invalidAccountId() {
+
+ when(this.info.getName()).thenReturn("tenant2");
+ when(tenantFactory.getTenantInfo("tenant1")).thenReturn(null);
+
+ when(this.sut.getTenantInfo()).thenReturn(info);
+
+ assertNotNull(this.sut.getTenantInfo());
+ }
+}
\ No newline at end of file
diff --git a/testing/indexer-test-ibm/pom.xml b/testing/indexer-test-ibm/pom.xml
new file mode 100644
index 0000000000000000000000000000000000000000..ab4667f601481135b68eeeb2c1db0256429b84be
--- /dev/null
+++ b/testing/indexer-test-ibm/pom.xml
@@ -0,0 +1,130 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>org.opengroup.osdu.indexer</groupId>
+ <artifactId>indexer-test-ibm</artifactId>
+ <version>0.0.1</version>
+ <packaging>jar</packaging>
+
+ <properties>
+ <maven.compiler.target>1.8</maven.compiler.target>
+ <maven.compiler.source>1.8</maven.compiler.source>
+ <cucumber.version>1.2.5</cucumber.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.google.api-client</groupId>
+ <artifactId>google-api-client</artifactId>
+ <version>1.28.0</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.opengroup.osdu.indexer</groupId>
+ <artifactId>indexer-test-core</artifactId>
+ <version>0.0.1</version>
+ </dependency>
+
+ <!-- Cucumber -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.12</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>info.cukes</groupId>
+ <artifactId>cucumber-java</artifactId>
+ <version>${cucumber.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>info.cukes</groupId>
+ <artifactId>cucumber-junit</artifactId>
+ <version>${cucumber.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- Gson: Java to Json conversion -->
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>2.8.5</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.jaxrs</groupId>
+ <artifactId>jackson-jaxrs-json-provider</artifactId>
+ <version>2.9.9</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.glassfish</groupId>
+ <artifactId>javax.json</artifactId>
+ <version>1.1.4</version>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-client</artifactId>
+ <version>1.19.4</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ <version>1.18.2</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>2.6</version>
+ </dependency>
+
+ <!--Elasticsearch-->
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch</artifactId>
+ <version>6.6.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-client</artifactId>
+ <version>6.6.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-high-level-client</artifactId>
+ <version>6.6.2</version>
+ </dependency>
+
+ <!--Logging-->
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-to-slf4j</artifactId>
+ <version>2.11.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-jdk14</artifactId>
+ <version>1.8.0-beta4</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>27.1-jre</version>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..4978ddfccad197628432d4ccb8b5985b4af5017e
--- /dev/null
+++ b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java
@@ -0,0 +1,13 @@
+package org.opengroup.osdu.step_definitions.index.record;
+
+import cucumber.api.CucumberOptions;
+import cucumber.api.junit.Cucumber;
+import org.junit.runner.RunWith;
+
+@RunWith(Cucumber.class)
+@CucumberOptions(
+ features = "classpath:features/indexrecord/IndexRecord.feature",
+ glue = {"classpath:org.opengroup.osdu.step_definitions/index/record"},
+ plugin = {"pretty", "junit:target/cucumber-reports/TEST-indexrecord.xml"})
+public class RunTest {
+}
\ No newline at end of file
diff --git a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
new file mode 100644
index 0000000000000000000000000000000000000000..2d402c72f1fa06162e6f37deeae286498f3d30d1
--- /dev/null
+++ b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -0,0 +1,52 @@
+package org.opengroup.osdu.step_definitions.index.record;
+
+import lombok.extern.java.Log;
+import org.opengroup.osdu.common.RecordSteps;
+import org.opengroup.osdu.util.IBMHTTPClient;
+
+import cucumber.api.Scenario;
+import cucumber.api.java.Before;
+import cucumber.api.DataTable;
+import cucumber.api.java.en.Given;
+import cucumber.api.java.en.Then;
+import cucumber.api.java.en.When;
+
+@Log
+public class Steps extends RecordSteps {
+
+ public Steps() {
+ super(new IBMHTTPClient());
+ }
+
+ @Before
+ public void before(Scenario scenario) {
+ this.scenario = scenario;
+ this.httpClient = new IBMHTTPClient();
+ }
+
+ @Given("^the schema is created with the following kind$")
+ public void the_schema_is_created_with_the_following_kind(DataTable dataTable) {
+ super.the_schema_is_created_with_the_following_kind(dataTable);
+ }
+
+ @When("^I ingest records with the \"(.*?)\" with \"(.*?)\" for a given \"(.*?)\"$")
+ public void i_ingest_records_with_the_for_a_given(String record, String dataGroup, String kind) {
+ super.i_ingest_records_with_the_for_a_given(record, dataGroup, kind);
+ }
+
+ @Then("^I should get the (\\d+) documents for the \"([^\"]*)\" in the Elastic Search$")
+ public void i_should_get_the_documents_for_the_in_the_Elastic_Search(int expectedCount, String index) throws Throwable {
+ super.i_should_get_the_documents_for_the_in_the_Elastic_Search(expectedCount, index);
+ }
+
+ @Then("^I should get the elastic \"(.*?)\" for the \"([^\"]*)\" and \"([^\"]*)\" in the Elastic Search$")
+ public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String type, String index) throws Throwable {
+ super.i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(expectedMapping, type, index);
+ }
+
+ @Then("^I should get the (\\d+) documents for the \"([^\"]*)\" in the Elastic Search with out \"(.*?)\"$")
+ public void iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(int expectedCount, String index, String skippedAttributes) throws Throwable {
+ super.iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(expectedCount, index, skippedAttributes);
+ }
+
+}
\ No newline at end of file
diff --git a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/util/IBMHTTPClient.java b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/util/IBMHTTPClient.java
new file mode 100644
index 0000000000000000000000000000000000000000..6affdc4049b0d8884d3c7bb395978c2e8adfc41a
--- /dev/null
+++ b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/util/IBMHTTPClient.java
@@ -0,0 +1,23 @@
+package org.opengroup.osdu.util;
+
+import lombok.ToString;
+import lombok.extern.java.Log;
+
+import java.io.IOException;
+
+@Log
+@ToString
+public class IBMHTTPClient extends HTTPClient {
+
+ private static String token = null;
+
+ @Override
+ public synchronized String getAccessToken() {
+ if(token == null) {
+ //FIXME need to get this from somewhere
+ token = "Basic " + Config.getStorageBasicAuthToken();
+ return token;
+ }
+ return token;
+ }
+}
\ No newline at end of file
diff --git a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/util/JwtTokenUtil.java b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/util/JwtTokenUtil.java
new file mode 100644
index 0000000000000000000000000000000000000000..7c13aaae6539701a93daf5793be42e12a2e9da3b
--- /dev/null
+++ b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/util/JwtTokenUtil.java
@@ -0,0 +1,103 @@
+package org.opengroup.osdu.util;
+
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.client.json.webtoken.JsonWebSignature;
+import com.google.api.client.json.webtoken.JsonWebToken;
+import com.google.api.client.util.Clock;
+import com.google.common.base.Strings;
+import com.google.gson.Gson;
+import lombok.Data;
+import org.apache.commons.io.Charsets;
+import org.apache.commons.io.IOUtils;
+import org.apache.http.HttpResponse;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.message.BasicNameValuePair;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.GeneralSecurityException;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.List;
+
+class JwtTokenUtil {
+
+ private static String accessToken;
+
+ static String getAccessToken() throws IOException {
+
+ if (Strings.isNullOrEmpty(accessToken)) {
+ accessToken = getServiceAccountAccessToken(getJwtForIntegrationTesterAccount());
+ }
+ return accessToken;
+ }
+
+ private static String getServiceAccountAccessToken(String key) throws IOException {
+
+ try (CloseableHttpClient httpClient = HttpClientBuilder.create().build()) {
+
+ List<NameValuePair> parameters = new ArrayList<>();
+ parameters.add(new BasicNameValuePair("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer"));
+ parameters.add(new BasicNameValuePair("assertion", key));
+
+ HttpPost postRequest = new HttpPost("https://www.googleapis.com/oauth2/v4/token");
+ postRequest.addHeader("Content-Type", "application/x-www-form-urlencoded");
+ postRequest.setEntity(new UrlEncodedFormEntity(parameters));
+
+ HttpResponse response = httpClient.execute(postRequest);
+ String responseContent = IOUtils.toString(response.getEntity().getContent(), Charsets.toCharset("UTF-8"));
+
+ JwtTokenUtil.ResponseToken responseToken = new Gson().fromJson(responseContent, JwtTokenUtil.ResponseToken.class);
+
+ return responseToken.getId_token();
+ }
+ }
+
+ private static String getJwtForIntegrationTesterAccount() throws IOException {
+ String serviceAccountFile = Config.getKeyValue();
+ return getJwt(serviceAccountFile);
+ }
+
+ private static String getJwt(String serviceAccountFile) throws IOException {
+
+ String targetAudience = Config.getTargetAudience();
+ long currentTime = Clock.SYSTEM.currentTimeMillis();
+
+ InputStream stream = new ByteArrayInputStream(Base64.getDecoder().decode(serviceAccountFile));
+ GoogleCredential credential = GoogleCredential.fromStream(stream);
+
+ JsonWebSignature.Header header = new JsonWebSignature.Header();
+ header.setAlgorithm("RS256");
+ header.setType("JWT");
+ header.setKeyId(credential.getServiceAccountPrivateKeyId());
+
+ JsonWebSignature.Payload payload = new JsonWebToken.Payload();
+ payload.setIssuedAtTimeSeconds(currentTime / 1000);
+ payload.setExpirationTimeSeconds(currentTime / 1000 + 3600);
+ payload.setAudience("https://www.googleapis.com/oauth2/v4/token");
+ payload.setIssuer(credential.getServiceAccountId());
+ payload.set("target_audience", targetAudience);
+
+ JsonFactory jsonFactory = JacksonFactory.getDefaultInstance();
+ String signedJwt = null;
+ try {
+ signedJwt = JsonWebSignature.signUsingRsaSha256(credential.getServiceAccountPrivateKey(), jsonFactory, header, payload);
+ } catch (GeneralSecurityException e) {
+ e.printStackTrace();
+ }
+
+ return signedJwt;
+ }
+
+ @Data
+ class ResponseToken {
+ public String id_token;
+ }
+}