merged from master

indexer-service-azure/pom.xml

@@ -16,24 +16,38 @@
     <description>Indexer Service Azure</description>
     <packaging>jar</packaging>
 
-    <dependencies>
-        <dependency>
-            <groupId>org.opendes.core</groupId>
-            <artifactId>indexer-search-core-azure</artifactId>
-            <version>0.0.1-SNAPSHOT</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>org.opendes.core</groupId>
-                    <artifactId>indexer-search-core-root</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
+    <properties>
+        <azure.version>2.1.7</azure.version>
+    </properties>
 
+    <dependencies>
         <dependency>
             <groupId>org.opendes.indexer</groupId>
             <artifactId>indexer-service-root</artifactId>
             <version>1.0-SNAPSHOT</version>
         </dependency>
+
+        <dependency>
+            <groupId>com.microsoft.azure</groupId>
+            <artifactId>azure-active-directory-spring-boot-starter</artifactId>
+            <version>${azure.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.microsoft.azure</groupId>
+            <artifactId>azure-cosmosdb-spring-boot-starter</artifactId>
+            <version>${azure.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.microsoft.azure</groupId >
+            <artifactId>azure-storage-spring-boot-starter</artifactId>
+            <version>${azure.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.microsoft.azure</groupId>
+            <artifactId>azure-servicebus-spring-boot-starter</artifactId>
+            <version>${azure.version}</version>
+        </dependency>
+
     </dependencies>
 
     <build>

indexer-service-azure/src/main/java/org/opendes/indexer/azure/di/EntitlementsClientFactory.java

+package org.opendes.indexer.azure.di;
+
+import org.opendes.client.api.entitlements.IEntitlementsFactory;
+import org.springframework.beans.factory.config.AbstractFactoryBean;
+import org.springframework.context.annotation.Primary;
+import org.springframework.stereotype.Component;
+
+@Component("AzureEntitlementsClientFactory")
+@Primary
+public class EntitlementsClientFactory extends AbstractFactoryBean<IEntitlementsFactory> {
+
+    @Override
+    protected IEntitlementsFactory createInstance() throws Exception {
+        return new EntitlementsFactoryAzure();
+    }
+
+    @Override
+    public Class<?> getObjectType() {
+        return IEntitlementsFactory.class;
+    }
+}

indexer-service-azure/src/main/java/org/opendes/indexer/azure/di/EntitlementsFactoryAzure.java

+package org.opendes.indexer.azure.di;
+
+import org.opendes.client.api.DpsHeaders;
+import org.opendes.client.api.entitlements.IEntitlementsFactory;
+import org.opendes.client.api.entitlements.IEntitlementsService;
+
+public class EntitlementsFactoryAzure implements IEntitlementsFactory {
+    @Override
+    public IEntitlementsService create(DpsHeaders headers) {
+        return new EntitlementsServiceAzure(headers);
+    }
+}

indexer-service-azure/src/main/java/org/opendes/indexer/azure/di/EntitlementsServiceAzure.java

+package org.opendes.indexer.azure.di;
+
+import com.microsoft.azure.spring.autoconfigure.aad.UserPrincipal;
+import org.apache.http.HttpStatus;
+import org.opendes.client.api.DpsHeaders;
+import org.opendes.client.api.entitlements.EntitlementsException;
+import org.opendes.client.api.entitlements.IEntitlementsService;
+import org.opendes.client.api.entitlements.models.*;
+import org.opendes.client.httpclient.HttpResponse;
+import org.opendes.core.model.SearchServiceRole;
+import org.springframework.security.core.Authentication;
+import org.springframework.security.core.GrantedAuthority;
+import org.springframework.security.core.context.SecurityContextHolder;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+public class EntitlementsServiceAzure implements IEntitlementsService {
+    DpsHeaders headers;
+
+    public EntitlementsServiceAzure(DpsHeaders headers){
+        this.headers = headers;
+    }
+
+    @Override
+    public MemberInfo addMember(GroupEmail groupEmail, MemberInfo memberInfo) throws EntitlementsException {
+        return null;
+    }
+
+    @Override
+    public Members getMembers(GroupEmail groupEmail, GetMembers getMembers) throws EntitlementsException {
+        return null;
+    }
+
+    @Override
+    public Groups getGroups() throws EntitlementsException {
+        final Authentication auth = SecurityContextHolder.getContext().getAuthentication();
+        final UserPrincipal current = (UserPrincipal) auth.getPrincipal();
+        String email = String.valueOf(current.getUpn());
+
+        List<GroupInfo> giList = new ArrayList();
+        Collection<? extends GrantedAuthority> authorities = auth.getAuthorities();
+        for(GrantedAuthority authority : authorities)
+        {
+            GroupInfo gi = new GroupInfo();
+            String role = authority.getAuthority();
+            if (role.startsWith(SearchServiceRole.PREFIX)){
+                role = role.substring(SearchServiceRole.PREFIX.length());
+            }
+            gi.setName(role);
+            gi.setEmail(email);
+            giList.add(gi);
+        }
+        if (giList.size() > 0)
+        {
+            Groups groups = new Groups();
+            groups.setGroups(giList);
+            groups.setDesId(email);
+            return groups;
+        }
+
+        HttpResponse response = new HttpResponse();
+        response.setResponseCode(HttpStatus.SC_INTERNAL_SERVER_ERROR);
+        throw new EntitlementsException("no authorities found", response);
+    }
+
+    @Override
+    public GroupInfo createGroup(CreateGroup createGroup) throws EntitlementsException {
+        return null;
+    }
+
+    @Override
+    public void deleteMember(String s, String s1) throws EntitlementsException {
+
+    }
+
+    @Override
+    public Groups authorizeAny(String... strings) throws EntitlementsException {
+        return null;
+    }
+
+    @Override
+    public void authenticate() throws EntitlementsException {
+
+    }
+}

indexer-service-azure/src/main/java/org/opendes/indexer/azure/di/TenantFactoryImpl.java

+package org.opendes.indexer.azure.di;
+
+import org.opendes.client.cache.ICache;
+import org.opendes.client.multitenancy.ITenantFactory;
+import org.opendes.client.multitenancy.TenantInfo;
+import org.springframework.stereotype.Component;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+@Component
+public class TenantFactoryImpl implements ITenantFactory {
+    public static final String DefaultTenantName = "common";
+    private List<TenantInfo> tenants;
+
+    public TenantFactoryImpl()
+    {
+        TenantInfo ti = new TenantInfo();
+        ti.setName(DefaultTenantName);
+        this.tenants = new ArrayList<>();
+        this.tenants.add(ti);
+    }
+
+    public boolean exists(String tenantName)
+    {
+        return true;
+    }
+
+    public TenantInfo getTenantInfo(String tenantName) {
+        // we are not checking tenantName yet, we have only 1 tenant
+        return this.tenants.get(0);
+    }
+
+    public Collection<TenantInfo> listTenantInfo() {
+        return this.tenants;
+    }
+
+    public <V> ICache<String, V> createCache(String tenantName, String host, int port, int expireTimeSeconds, Class<V> classOfV)
+    {
+        return null;
+    }
+
+    public void flushCache() {}
+}

indexer-service-azure/src/main/java/org/opendes/indexer/azure/kms/KmsClientImpl.java

+package org.opendes.indexer.azure.kms;
+
+import org.opendes.core.kms.IKmsClient;
+
+import java.io.IOException;
+
+public class KmsClientImpl implements IKmsClient {
+    @Override
+    public String encryptString(String textToBeEncrypted) throws IOException {
+        return textToBeEncrypted;
+    }
+
+    @Override
+    public String decryptString(String textToBeDecrypted) throws IOException {
+        return textToBeDecrypted;
+    }
+}

indexer-service-azure/src/main/java/org/opendes/indexer/azure/model/ISchemaRepository.java

+
+
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.azure.model;
+
+public interface ISchemaRepository {
+    String SCHEMA_KIND = "IndexerSchema";
+
+    String SCHEMA = "schema";
+    String USER = "user";
+    String EXTENSION = "extension";
+
+    void add(Schema schema, String user);
+
+    Schema get(String kind);
+}
+

indexer-service-azure/src/main/java/org/opendes/indexer/azure/model/Schema.java

+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.azure.model;
+
+        import java.util.Map;
+
+        import javax.validation.Valid;
+        import javax.validation.constraints.NotNull;
+
+        import lombok.AllArgsConstructor;
+        import lombok.Data;
+        import lombok.NoArgsConstructor;
+
+        import io.swagger.annotations.ApiModelProperty;
+        import org.opendes.indexer.SwaggerDoc;
+        import org.opendes.core.validation.ValidKind;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class Schema {
+
+    @ValidKind
+    @NotNull
+    @ApiModelProperty(value = SwaggerDoc.SCHEMA_REQUEST_KIND,
+            required = true,
+            example = SwaggerDoc.RECORD_KIND_EXAMPLE)
+    private String kind;
+
+    @Valid
+    private SchemaItem[] schema;
+
+    private Map<String, Object> ext;
+}
\ No newline at end of file

indexer-service-azure/src/main/java/org/opendes/indexer/azure/model/SchemaItem.java

+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.azure.model;
+
+import java.util.Map;
+
+import javax.validation.constraints.NotEmpty;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonInclude.Include;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class SchemaItem {
+
+    @NotEmpty
+    private String path;
+
+    @NotEmpty
+    private String kind;
+
+    @JsonInclude(value = Include.NON_NULL)
+    private Map<String, Object> ext;
+}

indexer-service-azure/src/main/java/org/opendes/indexer/azure/persistence/ElasticRepositoryCosmosDB.java

+package org.opendes.indexer.azure.persistence;
+
+import org.opendes.client.multitenancy.TenantInfo;
+import org.opendes.core.model.ClusterSettings;
+import org.opendes.core.persistence.ElasticRepository;
+
+public class ElasticRepositoryCosmosDB implements ElasticRepository {
+    @Override
+    public ClusterSettings getElasticClusterSettings(TenantInfo tenantInfo) {
+        return null;
+    }
+}

indexer-service-azure/src/main/java/org/opendes/indexer/azure/persistence/SchemaRepositoryImpl.java

+package org.opendes.indexer.azure.persistence;
+
+import com.microsoft.azure.spring.data.cosmosdb.core.mapping.Document;
+import com.microsoft.azure.spring.data.cosmosdb.core.mapping.PartitionKey;
+import com.microsoft.azure.spring.data.cosmosdb.repository.DocumentDbRepository;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.opendes.indexer.azure.model.Schema;
+import org.opendes.indexer.azure.model.SchemaItem;
+import org.opendes.indexer.azure.model.ISchemaRepository;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.data.annotation.Id;
+import org.springframework.stereotype.Repository;
+
+import java.util.Map;
+import java.util.Optional;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Document(collection = "StorageSchema") //collection name
+class SchemaDoc {
+
+    @PartitionKey
+    @Id
+    private String kind;
+    private Map<String,Object> extension;
+    private String user;
+    private SchemaItem[] schemaItems;
+}
+
+interface CosmosDB extends DocumentDbRepository<SchemaDoc, String> {}
+
+@Repository
+public class SchemaRepositoryImpl implements ISchemaRepository {
+
+    @Autowired
+    private CosmosDB db;
+
+    @Override
+    public void add(Schema schema, String user) {
+        SchemaDoc sd = new SchemaDoc();
+        sd.setKind(schema.getKind());
+        sd.setExtension(schema.getExt());
+        sd.setUser(user);
+        sd.setSchemaItems(schema.getSchema());
+        db.save(sd);
+    }
+
+    @Override
+    public Schema get(String kind) {
+        Optional<SchemaDoc> sd = db.findById(kind);
+        if (!sd.isPresent())
+            return null;
+
+        Schema newSchema = new Schema();
+        newSchema.setKind(kind);
+        newSchema.setSchema(sd.get().getSchemaItems());
+        newSchema.setExt(sd.get().getExtension());
+        return newSchema;
+    }
+}

indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/RequestInfoImpl.java

 package org.opendes.indexer.azure.util;
 
 import com.google.common.base.Strings;
-import com.slb.core.model.AppEngineHeaders;
-import com.slb.core.model.DeploymentEnvironment;
-import com.slb.core.util.HeadersInfo;
 import org.apache.http.HttpStatus;
 import org.opendes.client.api.DpsHeaders;
+import org.opendes.client.multitenancy.TenantInfo;
+import org.opendes.core.model.DeploymentEnvironment;
+import org.opendes.core.util.AppException;
+import org.opendes.core.util.Config;
+import org.opendes.core.util.HeadersInfo;
 import org.opendes.indexer.util.IRequestInfo;
 import org.springframework.beans.factory.annotation.Autowired;
 
@@ -22,9 +24,11 @@ public class RequestInfoImpl implements IRequestInfo {
 
     private static final String expectedCronHeaderValue = "true";
 
+    @Autowired
+    private TenantInfo tenantInfo;
+
     @Override
     public DpsHeaders getHeaders() {
-
         return this.headersInfo.getHeaders();
     }
 
@@ -51,18 +55,10 @@ public class RequestInfoImpl implements IRequestInfo {
     }
 
     @Override
-    public boolean isCronRequest() {
-        String appEngineCronHeader = this.headersInfo.getHeadersMap().getOrDefault(AppEngineHeaders.CRON_SERVICE, null);
-        return expectedCronHeaderValue.equalsIgnoreCase(appEngineCronHeader);
-    }
+    public boolean isCronRequest() { return false; }
 
     @Override
-    public boolean isTaskQueueRequest() {
-        if (!this.headersInfo.getHeadersMap().containsKey(AppEngineHeaders.TASK_QUEUE_NAME)) return false;
-
-        String queueId = this.headersInfo.getHeadersMap().get(AppEngineHeaders.TASK_QUEUE_NAME);
-        return queueId.endsWith(Constants.INDEXER_QUEUE_IDENTIFIER);
-    }
+    public boolean isTaskQueueRequest() { return false; }
 
     private String checkOrGetAuthorizationHeader() {
         if (Config.getDeploymentEnvironment() == DeploymentEnvironment.LOCAL) {
@@ -72,7 +68,8 @@ public class RequestInfoImpl implements IRequestInfo {
             }
             return authHeader;
         } else {
-            return "Bearer " + this.serviceAccountJwtClient.getIdToken();
+            return "Bearer " + this.serviceAccountJwtClient.getIdToken(tenantInfo.getName());
         }
     }
+
 }

indexer-service-azure/src/main/java/org/opendes/indexer/azure/util/ServiceAccountJwtClientImpl.java

+package org.opendes.indexer.azure.util;
+
+import org.opendes.core.util.IServiceAccountJwtClient;
+import org.springframework.stereotype.Component;
+
+@Component
+public class ServiceAccountJwtClientImpl implements IServiceAccountJwtClient {
+    @Override
+    public String getIdToken(String tenantName){
+        return "common";
+    }
+}

indexer-service-gcp/pom.xml

@@ -18,49 +18,93 @@
 
     <dependencies>
         <dependency>
-            <groupId>org.opendes.core</groupId>
-            <artifactId>indexer-search-core-gcp</artifactId>
-            <version>1.1-SNAPSHOT</version>
+            <groupId>org.opendes.indexer</groupId>
+            <artifactId>indexer-service-root</artifactId>
+            <version>1.0-SNAPSHOT</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.os</groupId>
+            <artifactId>client-lib-gcp</artifactId>
+            <version>0.0.8</version>
             <exclusions>
                 <exclusion>
-                    <groupId>org.opendes.core</groupId>
-                    <artifactId>indexer-search-core-root</artifactId>
+                    <groupId>org.os</groupId>
+                    <artifactId>client-lib</artifactId>
                 </exclusion>
             </exclusions>
         </dependency>
 
         <dependency>
-            <groupId>org.opendes.indexer</groupId>
-            <artifactId>indexer-service-root</artifactId>
-            <version>1.0-SNAPSHOT</version>
+            <groupId>com.google.cloud</groupId>
+            <artifactId>google-cloud-datastore</artifactId>
+            <version>1.72.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.cloud</groupId>
+            <artifactId>google-cloud-logging</artifactId>
+            <version>1.72.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.apis</groupId>
+            <artifactId>google-api-services-storage</artifactId>
+            <version>v1-rev150-1.25.0</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.google.guava</groupId>
+                    <artifactId>guava-jdk5</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>com.google.apis</groupId>
+            <artifactId>google-api-services-cloudkms</artifactId>
+            <version>v1-rev81-1.25.0</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/com.google.cloud/google-cloud-pubsub -->
+        <dependency>
+            <groupId>com.google.cloud</groupId>
+            <artifactId>google-cloud-pubsub</artifactId>
+            <version>1.60.0</version>
         </dependency>
-
-<!--        <dependency>-->
-<!--            <groupId>com.google.cloud</groupId>-->
-<!--            <artifactId>google-cloud-datastore</artifactId>-->
-<!--            <version>1.72.0</version>-->
-<!--        </dependency>-->
-<!--        <dependency>-->
-<!--            <groupId>com.google.cloud</groupId>-->
-<!--            <artifactId>google-cloud-logging</artifactId>-->
-<!--            <version>1.72.0</version>-->
-<!--        </dependency>-->
         <dependency>
             <groupId>com.google.appengine.tools</groupId>
             <artifactId>appengine-gcs-client</artifactId>
             <version>0.8</version>
         </dependency>
-
-        <!--        <dependency>-->
-<!--            <groupId>com.google.apis</groupId>-->
-<!--            <artifactId>google-api-services-storage</artifactId>-->
-<!--            <version>v1-rev150-1.25.0</version>-->
-<!--            <exclusions>-->
-<!--                <exclusion>-->
-<!--                    <groupId>com.google.guava</groupId>-->
-<!--                    <artifactId>guava-jdk5</artifactId>-->
-<!--                </exclusion>-->
-<!--            </exclusions>-->
+<!--        <dependency>-->
+<!--            <groupId>org.mockito</groupId>-->
+<!--            <artifactId>mockito-core</artifactId>-->
+<!--            <scope>test</scope>-->
+<!--        </dependency>-->
+<!--        <dependency>-->
+<!--            <groupId>junit</groupId>-->
+<!--            <artifactId>junit</artifactId>-->
+<!--            <scope>test</scope>-->
+<!--        </dependency>-->
+<!--        <dependency>-->
+<!--            <groupId>org.powermock</groupId>-->
+<!--            <artifactId>powermock-core</artifactId>-->
+<!--            <version>2.0.2</version>-->
+<!--            <scope>test</scope>-->
+<!--        </dependency>-->
+<!--        <dependency>-->
+<!--            <groupId>org.powermock</groupId>-->
+<!--            <artifactId>powermock-api-mockito2</artifactId>-->
+<!--            <version>2.0.2</version>-->
+<!--            <scope>test</scope>-->
+<!--        </dependency>-->
+<!--        <dependency>-->
+<!--            <groupId>org.springframework</groupId>-->
+<!--            <artifactId>spring-test</artifactId>-->
+<!--            <version>5.1.9.RELEASE</version>-->
+<!--            <scope>test</scope>-->
+<!--        </dependency>-->
+<!--        <dependency>-->
+<!--            <groupId>org.springframework</groupId>-->
+<!--            <artifactId>spring-test</artifactId>-->
+<!--            <version>5.1.9.RELEASE</version>-->
+<!--            <scope>test</scope>-->
 <!--        </dependency>-->
     </dependencies>
 

indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/IndexerGcpApplication.java

@@ -10,4 +10,4 @@ public class IndexerGcpApplication {
         SpringApplication.run(IndexerGcpApplication.class, args);
     }
 
-}
\ No newline at end of file
+}

indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/cache/DatastoreCredentialCache.java

+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.gcp.cache;
+
+import com.google.auth.oauth2.AccessToken;
+import org.opendes.client.cache.RedisCache;
+import org.springframework.beans.factory.annotation.Value;
+
+public class DatastoreCredentialCache extends RedisCache<String, AccessToken> {
+
+	// Datastore credentials are only valid for 1hr, release the key 2 minutes before the expiration
+	public DatastoreCredentialCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST, @Value("${REDIS_SEARCH_PORT}") final String REDIS_SEARCH_PORT) {
+		super(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), 58 * 60, String.class, AccessToken.class);
+	}
+}
\ No newline at end of file

indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/kms/KmsClient.java

+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.gcp.kms;
+
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.http.javanet.NetHttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.cloudkms.v1.CloudKMS;
+import com.google.api.services.cloudkms.v1.CloudKMSScopes;
+import com.google.api.services.cloudkms.v1.model.DecryptRequest;
+import com.google.api.services.cloudkms.v1.model.DecryptResponse;
+import com.google.api.services.cloudkms.v1.model.EncryptRequest;
+import com.google.api.services.cloudkms.v1.model.EncryptResponse;
+import org.opendes.core.kms.IKmsClient;
+import org.opendes.core.util.Config;
+import org.opendes.core.util.Preconditions;
+import org.springframework.beans.factory.annotation.Value;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+public class KmsClient implements IKmsClient {
+
+    @Value("${KMS_KEY}")
+    private String KMS_KEY;
+
+    @Value("${KEY_RING}")
+    private String KEY_RING;
+
+    private static final String KEY_NAME = "projects/%s/locations/global/keyRings/%s/cryptoKeys/%s";
+
+    /**
+     * Encrypts the given plaintext using the specified crypto key.
+     * Google KMS automatically uses the new primary key version to encrypt data, so this could be directly used for key rotation
+     */
+    public String encryptString(String textToBeEncrypted) throws IOException {
+        Preconditions.checkNotNullOrEmpty(textToBeEncrypted, "textToBeEncrypted cannot be null");
+
+        byte[] plaintext = textToBeEncrypted.getBytes(StandardCharsets.UTF_8);
+        String resourceName = String.format(KEY_NAME, Config.getGoogleCloudProjectId(), KEY_RING, KMS_KEY);
+        CloudKMS kms = createAuthorizedClient();
+        EncryptRequest request = new EncryptRequest().encodePlaintext(plaintext);
+        EncryptResponse response = kms.projects().locations().keyRings().cryptoKeys()
+                .encrypt(resourceName, request)
+                .execute();
+        return response.getCiphertext();
+    }
+
+    /**
+     * Decrypts the provided ciphertext with the specified crypto key.
+     * Google KMS automatically uses the correct key version to decrypt data, as long as the key version is not disabled
+     */
+    public String decryptString(String textToBeDecrypted) throws IOException {
+        Preconditions.checkNotNullOrEmpty(textToBeDecrypted, "textToBeDecrypted cannot be null");
+
+        CloudKMS kms = createAuthorizedClient();
+        String cryptoKeyName = String.format(KEY_NAME, Config.getGoogleCloudProjectId(), KEY_RING, KMS_KEY);
+        DecryptRequest request = new DecryptRequest().setCiphertext(textToBeDecrypted);
+        DecryptResponse response = kms.projects().locations().keyRings().cryptoKeys()
+                .decrypt(cryptoKeyName, request)
+                .execute();
+        return new String(response.decodePlaintext(), StandardCharsets.UTF_8).trim();
+    }
+
+    /**
+     * Creates an authorized CloudKMS client service using Application Default Credentials.
+     *
+     * @return an authorized CloudKMS client
+     * @throws IOException if there's an error getting the default credentials.
+     */
+    private CloudKMS createAuthorizedClient() throws IOException {
+        HttpTransport transport = new NetHttpTransport();
+        JsonFactory jsonFactory = new JacksonFactory();
+        GoogleCredential credential = GoogleCredential.getApplicationDefault();
+        if (credential.createScopedRequired()) {
+            credential = credential.createScoped(CloudKMSScopes.all());
+        }
+        return new CloudKMS.Builder(transport, jsonFactory, credential)
+                .setApplicationName("CloudKMS snippets")
+                .build();
+    }
+}
\ No newline at end of file

indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/model/AppEngineHeaders.java

+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.gcp.model;
+
+public class AppEngineHeaders {
+    public static final String DATA_GROUPS = "X-Data-Groups";
+    public static final String TASK_QUEUE_RETRY_COUNT = "X-AppEngine-TaskExecutionCount";
+    public static final String TASK_QUEUE_NAME = "X-AppEngine-QueueName";
+    public static final String CITY_LAT_LONG = "X-AppEngine-CityLatLong";
+    public static final String COUNTRY = "X-AppEngine-Country";
+    public static final String REGION = "X-AppEngine-Region";
+    public static final String CITY = "X-AppEngine-City";
+    public static final String CLOUD_TRACE_CONTEXT = "X-Cloud-Trace-Context";
+    public static final String TRACE_ID = "X-Trace-Id";
+    public static final String CRON_SERVICE = "X-AppEngine-Cron";
+}
\ No newline at end of file

indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/persistence/DatastoreCredential.java

+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.gcp.persistence;
+
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.iam.v1.Iam;
+import com.google.api.services.iam.v1.Iam.Projects.ServiceAccounts.SignJwt;
+import com.google.api.services.iam.v1.model.SignJwtRequest;
+import com.google.api.services.iam.v1.model.SignJwtResponse;
+import com.google.auth.oauth2.AccessToken;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.gson.JsonObject;
+import org.apache.commons.lang3.time.DateUtils;
+import org.opendes.client.cryptographic.Crc32c;
+import org.opendes.client.multitenancy.TenantInfo;
+import org.opendes.indexer.gcp.cache.DatastoreCredentialCache;
+
+import java.util.Date;
+
+public class DatastoreCredential extends GoogleCredentials {
+
+	private static final long serialVersionUID = 8344377091688956815L;
+	private static final JsonFactory JSON_FACTORY = new JacksonFactory();
+	private Iam iam;
+
+	private final TenantInfo tenant;
+	private final DatastoreCredentialCache cache;
+
+	protected DatastoreCredential(TenantInfo tenant, DatastoreCredentialCache cache) {
+		this.tenant = tenant;
+		this.cache = cache;
+	}
+
+	@Override
+	public AccessToken refreshAccessToken() {
+
+		String cacheKey = this.getCacheKey();
+
+		AccessToken accessToken = this.cache.get(cacheKey);
+
+		if (accessToken != null) {
+			return accessToken;
+		}
+
+		try {
+			SignJwtRequest signJwtRequest = new SignJwtRequest();
+			signJwtRequest.setPayload(this.getPayload());
+
+			String serviceAccountName = String.format("projects/-/serviceAccounts/%s", this.tenant.getServiceAccount());
+
+			SignJwt signJwt = this.getIam().projects().serviceAccounts().signJwt(serviceAccountName, signJwtRequest);
+
+			SignJwtResponse signJwtResponse = signJwt.execute();
+			String signedJwt = signJwtResponse.getSignedJwt();
+
+			accessToken = new AccessToken(signedJwt, DateUtils.addSeconds(new Date(), 3600));
+
+			this.cache.put(cacheKey, accessToken);
+
+			return accessToken;
+		} catch (Exception e) {
+			throw new RuntimeException("Error creating datastore credential", e);
+		}
+	}
+
+	private String getPayload() {
+		JsonObject payload = new JsonObject();
+		payload.addProperty("iss", this.tenant.getServiceAccount());
+		payload.addProperty("sub", this.tenant.getServiceAccount());
+		payload.addProperty("aud", "https://datastore.googleapis.com/google.datastore.v1.Datastore");
+		payload.addProperty("iat", System.currentTimeMillis() / 1000);
+
+		return payload.toString();
+	}
+
+	protected void setIam(Iam iam) {
+		this.iam = iam;
+	}
+
+	private Iam getIam() throws Exception {
+		if (this.iam == null) {
+
+			Iam.Builder builder = new Iam.Builder(GoogleNetHttpTransport.newTrustedTransport(), JSON_FACTORY,
+					GoogleCredential.getApplicationDefault()).setApplicationName("Search Service");
+
+			this.iam = builder.build();
+		}
+		return this.iam;
+	}
+
+	private String getCacheKey() {
+		return Crc32c.hashToBase64EncodedString(String.format("datastoreCredential:%s", this.tenant.getName()));
+	}
+}
\ No newline at end of file

indexer-service-gcp/src/main/java/org/opendes/indexer/gcp/persistence/DatastoreFactory.java

+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opendes.indexer.gcp.persistence;
+
+import com.google.api.gax.retrying.RetrySettings;
+import com.google.cloud.TransportOptions;
+import com.google.cloud.datastore.Datastore;
+import com.google.cloud.datastore.DatastoreOptions;
+import com.google.cloud.http.HttpTransportOptions;
+import org.opendes.client.multitenancy.TenantInfo;
+import org.opendes.indexer.gcp.cache.DatastoreCredentialCache;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.threeten.bp.Duration;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class DatastoreFactory {
+
+    @Autowired
+    private DatastoreCredentialCache cache;
+
+    private static Map<String, Datastore> DATASTORE_CLIENTS = new HashMap<>();
+
+    private static final RetrySettings RETRY_SETTINGS = RetrySettings.newBuilder()
+            .setMaxAttempts(6)
+            .setInitialRetryDelay(Duration.ofSeconds(10))
+            .setMaxRetryDelay(Duration.ofSeconds(32))
+            .setRetryDelayMultiplier(2.0)
+            .setTotalTimeout(Duration.ofSeconds(50))
+            .setInitialRpcTimeout(Duration.ofSeconds(50))
+            .setRpcTimeoutMultiplier(1.0)
+            .setMaxRpcTimeout(Duration.ofSeconds(50))
+            .build();
+
+    private static final TransportOptions TRANSPORT_OPTIONS = HttpTransportOptions.newBuilder()
+            .setReadTimeout(30000)
+            .build();
+
+    public Datastore getDatastoreInstance(TenantInfo tenantInfo) {
+        if (DATASTORE_CLIENTS.get(tenantInfo.getName()) == null) {
+            Datastore googleDatastore = DatastoreOptions.newBuilder()
+                    .setCredentials(new DatastoreCredential(tenantInfo, this.cache))
+                    .setRetrySettings(RETRY_SETTINGS)
+                    .setTransportOptions(TRANSPORT_OPTIONS)
+                    .setNamespace(tenantInfo.getName())
+                    .setProjectId(tenantInfo.getProjectId())
+                    .build().getService();
+            DATASTORE_CLIENTS.put(tenantInfo.getName(), googleDatastore);
+        }
+        return DATASTORE_CLIENTS.get(tenantInfo.getName());
+    }
+}