Local development of Search Service for Anthos PoC

pom.xml

@@ -50,6 +50,7 @@
         <module>provider/search-aws</module>
         <module>provider/search-azure</module>
         <module>provider/search-ibm</module>
+        <module>provider/search-reference</module>
         <!--<module>testing/integration-tests</module>-->
     </modules>
 

provider/search-reference/pom.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.opengroup.osdu</groupId>
+        <artifactId>os-search</artifactId>
+        <version>0.0.5-SNAPSHOT</version>
+        <relativePath>../../pom.xml</relativePath>
+    </parent>
+
+    <groupId>org.opengroup.osdu</groupId>
+    <artifactId>search-reference</artifactId>
+    <version>0.0.5-SNAPSHOT</version>
+    <description>MongoDB implementation of Search service APIs</description>
+    <packaging>jar</packaging>
+
+    <properties>
+        <maven.compiler.target>1.8</maven.compiler.target>
+        <maven.compiler.source>1.8</maven.compiler.source>
+        <maven.war.plugin>2.6</maven.war.plugin>
+        <appengine.maven.plugin>1.0.0</appengine.maven.plugin>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <failOnMissingWebXml>false</failOnMissingWebXml>
+        <project.main.basedir>${project.parent.basedir}</project.main.basedir>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.opengroup.osdu</groupId>
+            <artifactId>search-core</artifactId>
+            <version>0.0.5-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>org.opengroup.osdu</groupId>
+            <artifactId>os-core-common</artifactId>
+            <version>0.0.18</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-data-mongodb</artifactId>
+            <version>2.1.7.RELEASE</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-test</artifactId>
+            <version>2.1.7.RELEASE</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.powermock</groupId>
+            <artifactId>powermock-api-mockito2</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.powermock</groupId>
+            <artifactId>powermock-module-junit4</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-all</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <!--Elasticsearch-->
+        <dependency>
+            <groupId>org.elasticsearch</groupId>
+            <artifactId>elasticsearch</artifactId>
+            <version>6.6.2</version>
+        </dependency>
+        <dependency>
+            <groupId>org.elasticsearch.client</groupId>
+            <artifactId>elasticsearch-rest-client</artifactId>
+            <version>6.6.2</version>
+        </dependency>
+        <dependency>
+            <groupId>org.elasticsearch.client</groupId>
+            <artifactId>elasticsearch-rest-high-level-client</artifactId>
+            <version>6.6.2</version>
+        </dependency>
+        <dependency>
+            <groupId>org.locationtech.jts.io</groupId>
+            <artifactId>jts-io-common</artifactId>
+            <version>1.15.0</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+                <configuration>
+                    <classifier>spring-boot</classifier>
+                    <mainClass>
+                        org.opengroup.osdu.search.SearchApplication
+                    </mainClass>
+                    <profiles>
+                        <profile>
+                            <id>local</id>
+                            <activation>
+                                <activeByDefault>true</activeByDefault>
+                            </activation>
+                            <properties>
+                                <spring.profiles.active>local</spring.profiles.active>
+                            </properties>
+                        </profile>
+                        <profile>
+                            <id>dev</id>
+                            <properties>
+                                <spring.profiles.active>dev</spring.profiles.active>
+                            </properties>
+                        </profile>
+                    </profiles>
+                </configuration>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>repackage</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>
\ No newline at end of file

provider/search-reference/src/main/java/org/opengroup/osdu/search/provider/reference/SearchReferenceApplication.java

+package org.opengroup.osdu.search.provider.reference;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.context.annotation.ComponentScan;
+
+@ComponentScan({"org.opengroup.osdu"})
+@SpringBootApplication(
+    exclude = {
+        org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration.class,
+        org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration.class})
+public class SearchReferenceApplication {
+  public static void main(String[] args) {
+    SpringApplication.run(SearchReferenceApplication.class, args);
+  }
+}

provider/search-reference/src/main/java/org/opengroup/osdu/search/provider/reference/cache/CursorCacheImpl.java

+package org.opengroup.osdu.search.provider.reference.cache;
+
+import org.opengroup.osdu.core.common.cache.RedisCache;
+import org.opengroup.osdu.core.common.model.search.CursorSettings;
+import org.opengroup.osdu.search.cache.CursorCache;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+@Component
+public class CursorCacheImpl extends RedisCache<String, CursorSettings> implements CursorCache {
+    
+    public CursorCacheImpl(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST, @Value("${REDIS_SEARCH_PORT}") final int REDIS_SEARCH_PORT, @Value("${CURSOR_CACHE_EXPIRATION}") final int CURSOR_CACHE_EXPIRATION) {
+        super(REDIS_SEARCH_HOST, REDIS_SEARCH_PORT, CURSOR_CACHE_EXPIRATION, String.class, CursorSettings.class);
+    }
+}

provider/search-reference/src/main/java/org/opengroup/osdu/search/provider/reference/cache/ElasticCredentialsCache.java

+package org.opengroup.osdu.search.provider.reference.cache;
+
+import org.opengroup.osdu.core.common.cache.RedisCache;
+import org.opengroup.osdu.core.common.model.search.ClusterSettings;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+@Component
+public class ElasticCredentialsCache extends RedisCache<String, ClusterSettings> {
+
+  public ElasticCredentialsCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST,
+                                 @Value("${REDIS_SEARCH_PORT}") final int REDIS_SEARCH_PORT,
+                                 @Value("${ELASTIC_CACHE_EXPIRATION}") final int ELASTIC_CACHE_EXPIRATION) {
+    super(REDIS_SEARCH_HOST, REDIS_SEARCH_PORT, ELASTIC_CACHE_EXPIRATION * 60, String.class, ClusterSettings.class);
+  }
+}

provider/search-reference/src/main/java/org/opengroup/osdu/search/provider/reference/cache/FieldTypeMappingCache.java

+package org.opengroup.osdu.search.provider.reference.cache;
+
+import java.util.HashSet;
+import org.opengroup.osdu.core.common.cache.RedisCache;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+@Component
+public class FieldTypeMappingCache extends RedisCache<String, HashSet> {
+
+  public FieldTypeMappingCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST,
+                               @Value("${REDIS_SEARCH_PORT}") final int REDIS_SEARCH_PORT) {
+    super(REDIS_SEARCH_HOST, REDIS_SEARCH_PORT, 1440 * 60, String.class, HashSet.class);
+  }
+}

provider/search-reference/src/main/java/org/opengroup/osdu/search/provider/reference/di/EntitlementsClientFactory.java

+package org.opengroup.osdu.search.provider.reference.di;
+
+import org.opengroup.osdu.core.common.entitlements.EntitlementsAPIConfig;
+import org.opengroup.osdu.core.common.entitlements.EntitlementsFactory;
+import org.opengroup.osdu.core.common.entitlements.IEntitlementsFactory;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.beans.factory.config.AbstractFactoryBean;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+@Component
+@RequestScope
+public class EntitlementsClientFactory extends AbstractFactoryBean<IEntitlementsFactory> {
+
+	@Value("${AUTHORIZE_API}")
+	private String AUTHORIZE_API;
+
+	@Value("${AUTHORIZE_API_KEY:}")
+	private String AUTHORIZE_API_KEY;
+
+	@Override
+	protected IEntitlementsFactory createInstance() throws Exception {
+
+		return new EntitlementsFactory(EntitlementsAPIConfig
+				.builder()
+				.rootUrl(AUTHORIZE_API)
+				.apiKey(AUTHORIZE_API_KEY)
+				.build());
+	}
+
+	@Override
+	public Class<?> getObjectType() {
+		return IEntitlementsFactory.class;
+	}
+}
\ No newline at end of file

provider/search-reference/src/main/java/org/opengroup/osdu/search/provider/reference/di/TenantFactoryImpl.java

+package org.opengroup.osdu.search.provider.reference.di;
+
+import static org.opengroup.osdu.search.provider.reference.provider.persistence.ElasticRepositoryReference.SEARCH_DATABASE;
+import com.google.gson.Gson;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import org.bson.Document;
+import org.opengroup.osdu.core.common.cache.ICache;
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
+import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
+import org.opengroup.osdu.search.provider.reference.model.TenantInfoDocument;
+import org.opengroup.osdu.search.provider.reference.provider.persistence.MongoDdmsClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class TenantFactoryImpl implements ITenantFactory {
+
+  private static final Logger LOG = LoggerFactory.getLogger(TenantFactoryImpl.class);
+  public static final String TENANT_INFO = "TenantInfo";
+
+  @Autowired
+  private MongoDdmsClient mongoClient;
+
+  private Map<String, TenantInfo> tenants;
+
+  public boolean exists(String tenantName) {
+    if (this.tenants == null) {
+      initTenants();
+    }
+    return this.tenants.containsKey(tenantName);
+  }
+
+  public TenantInfo getTenantInfo(String tenantName) {
+    if (this.tenants == null) {
+      initTenants();
+    }
+    return this.tenants.get(tenantName);
+  }
+
+  public Collection<TenantInfo> listTenantInfo() {
+    if (this.tenants == null) {
+      initTenants();
+    }
+    return this.tenants.values();
+  }
+
+  public <V> ICache<String, V> createCache(String tenantName, String host, int port,
+      int expireTimeSeconds, Class<V> classOfV) {
+    return null;
+  }
+
+  public void flushCache() {
+  }
+
+  private void initTenants() {
+    this.tenants = new HashMap<>();
+    MongoCollection<Document> mongoCollection = mongoClient
+        .getMongoCollection(SEARCH_DATABASE, TENANT_INFO);
+    FindIterable<Document> results = mongoCollection.find();
+    if (Objects.isNull(results) && Objects.isNull(results.first())) {
+      LOG.error(String.format("Collection \'%s\' is empty.", results));
+    }
+    for (Document document : results) {
+      TenantInfoDocument tenantInfoDocument = new Gson()
+          .fromJson(document.toJson(), TenantInfoDocument.class);
+      TenantInfo tenantInfo = convertToTenantInfo(tenantInfoDocument);
+      this.tenants.put(tenantInfo.getName(), tenantInfo);
+    }
+  }
+
+  private TenantInfo convertToTenantInfo(TenantInfoDocument tenantInfoDocument) {
+    TenantInfo tenantInfo = new TenantInfo();
+    tenantInfo.setName(tenantInfoDocument.getId());
+    return tenantInfo;
+  }
+}
+

provider/search-reference/src/main/java/org/opengroup/osdu/search/provider/reference/model/ElasticSettingSchema.java

+package org.opengroup.osdu.search.provider.reference.model;
+
+import javax.validation.constraints.NotEmpty;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class ElasticSettingSchema {
+
+    @NotEmpty
+    private String host;
+
+    @NotEmpty
+    private String port;
+
+    @NotEmpty
+    private String usernameAndPassword;
+
+    @NotEmpty
+    private boolean isHttps;  
+
+}

provider/search-reference/src/main/java/org/opengroup/osdu/search/provider/reference/model/TenantInfoDocument.java

+package org.opengroup.osdu.search.provider.reference.model;
+
+import java.util.List;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.springframework.data.mongodb.core.mapping.Document;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Document(collection = "TenantInfo")
+public class TenantInfoDocument {
+
+  private String id;
+  private List<String> groups;
+}

provider/search-reference/src/main/java/org/opengroup/osdu/search/provider/reference/provider/impl/CcsQueryServiceImpl.java

+package org.opengroup.osdu.search.provider.reference.provider.impl;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import javax.inject.Inject;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.search.CcsQueryRequest;
+import org.opengroup.osdu.core.common.model.search.CcsQueryResponse;
+import org.opengroup.osdu.core.common.model.search.QueryRequest;
+import org.opengroup.osdu.core.common.model.search.QueryResponse;
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
+import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository;
+import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
+import org.opengroup.osdu.core.common.search.Config;
+import org.opengroup.osdu.search.provider.interfaces.ICcsQueryService;
+import org.opengroup.osdu.search.provider.interfaces.IQueryService;
+import org.springframework.stereotype.Service;
+
+// TODO: Remove this temporary implementation when ECE CCS is utilized
+@Service
+public class CcsQueryServiceImpl implements ICcsQueryService {
+
+    @Inject
+    private DpsHeaders dpsHeaders;
+
+    @Inject
+    private ITenantFactory tenantStorageFactory;
+
+    @Inject
+    private IElasticRepository elasticRepository;
+
+    @Inject
+    private IQueryService queryService;
+
+    @Override
+    public CcsQueryResponse makeRequest(final CcsQueryRequest ccsQueryRequest) throws Exception {
+        List<String> accounts = Arrays.asList(dpsHeaders.getPartitionIdWithFallbackToAccountId().trim().split("\\s*,\\s*"));
+        List<QueryResponse> tenantResponses = getTenantResponses(accounts, ccsQueryRequest);
+        return convertQueryResponseToCcsQueryResponse(getCompoundResponse(tenantResponses));
+    }
+
+    private List<QueryResponse> getTenantResponses(final List<String> accounts, final CcsQueryRequest ccsQueryRequest) throws Exception {
+        List<QueryResponse> tenantResponses = new ArrayList<>();
+        if (Config.isSmartSearchCcsDisabled() || accounts.size() == 1) {
+            TenantInfo tenant = tenantStorageFactory.getTenantInfo(this.dpsHeaders.getPartitionIdWithFallbackToAccountId());
+            tenantResponses.add(queryService.queryIndex(convertCcsQueryRequestToQueryRequest(ccsQueryRequest),
+                    elasticRepository.getElasticClusterSettings(tenant)));
+        } else {
+            ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
+            List<Future<QueryResponse>> futureResponses = new ArrayList<>();
+            for (String account : accounts) {
+                futureResponses.add(executorService.submit(() -> {
+                    TenantInfo tenant = tenantStorageFactory.getTenantInfo(account);
+                    return queryService.queryIndex(convertCcsQueryRequestToQueryRequest(ccsQueryRequest),
+                            elasticRepository.getElasticClusterSettings(tenant));
+                }));
+            }
+            for (Future<QueryResponse> futureResponse : futureResponses) {
+                // TODO: Check why it can be null, this doesn't look right and leads to redundant null checks in other services
+                QueryResponse tenantResponse = futureResponse.get();
+                if (tenantResponse != null) {
+                    tenantResponses.add(tenantResponse);
+                }
+            }
+            executorService.shutdown();
+        }
+        return tenantResponses;
+    }
+
+    private QueryResponse getCompoundResponse(final List<QueryResponse> tenantResponses) {
+        QueryResponse response = new QueryResponse();
+        if (tenantResponses.isEmpty()) {
+            return response;
+        } else {
+            for (QueryResponse tenantResponse : tenantResponses) {
+                response.setTotalCount(response.getTotalCount() + tenantResponse.getTotalCount());
+            }
+            tenantResponses.sort(Comparator.comparingLong(QueryResponse::getTotalCount).reversed());
+            QueryResponse largestResponse = tenantResponses.remove(0);
+            List<Map<String, Object>> results = new LinkedList<>();
+            for (Map<String, Object> result : largestResponse.getResults()) {
+                results.add(result);
+                int index = largestResponse.getResults().indexOf(result);
+                for (QueryResponse tenantResponse : tenantResponses) {
+                    if (index < tenantResponse.getResults().size()) {
+                        results.add(tenantResponse.getResults().get(index));
+                    }
+                }
+            }
+            response.setResults(results);
+            return response;
+        }
+    }
+
+    private QueryRequest convertCcsQueryRequestToQueryRequest(final CcsQueryRequest ccsQueryRequest) {
+        QueryRequest queryRequest = new QueryRequest();
+        queryRequest.setFrom(ccsQueryRequest.getFrom());
+        queryRequest.setKind(ccsQueryRequest.getKind());
+        queryRequest.setLimit(ccsQueryRequest.getLimit());
+        queryRequest.setQuery(ccsQueryRequest.getQuery());
+        queryRequest.setQueryAsOwner(ccsQueryRequest.isQueryAsOwner());
+        return queryRequest;
+    }
+
+    private CcsQueryResponse convertQueryResponseToCcsQueryResponse(final QueryResponse queryResponse) {
+        CcsQueryResponse ccsQueryResponse = new CcsQueryResponse();
+        ccsQueryResponse.setResults(queryResponse.getResults());
+        ccsQueryResponse.setTotalCount(queryResponse.getTotalCount());
+        return ccsQueryResponse;
+    }
+}

provider/search-reference/src/main/java/org/opengroup/osdu/search/provider/reference/provider/impl/CrossTenantInfoServiceImpl.java

+package org.opengroup.osdu.search.provider.reference.provider.impl;
+
+import java.util.LinkedList;
+import java.util.List;
+import javax.inject.Inject;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
+import org.opengroup.osdu.core.common.multitenancy.ITenantInfoService;
+import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
+import org.opengroup.osdu.search.provider.interfaces.ICrossTenantInfoService;
+import org.springframework.stereotype.Service;
+
+@Service
+public class CrossTenantInfoServiceImpl implements ITenantInfoService, ICrossTenantInfoService {
+
+    @Inject
+    private ITenantFactory tenantFactory;
+
+    @Inject
+    private DpsHeaders headers;
+
+    @Override
+    public TenantInfo getTenantInfo() {
+        String primaryAccountId = this.headers.getPartitionIdWithFallbackToAccountId();
+        TenantInfo tenantInfo = this.tenantFactory.getTenantInfo(primaryAccountId);
+        if (tenantInfo == null) {
+            throw AppException.createUnauthorized(String.format("could not retrieve tenant info for data partition id: %s", primaryAccountId));
+        }
+        return tenantInfo;
+    }
+
+    @Override
+    public List<TenantInfo> getAllTenantsFromPartitionId() {
+        List<TenantInfo> tenantInfos = new LinkedList<>();
+
+        String[] accountIdList = headers.getPartitionIdWithFallbackToAccountId().split(",");
+        //Get all tenant values requested by user
+        for (String accountId : accountIdList) {
+            TenantInfo tenantInfo = tenantFactory.getTenantInfo(accountId);
+            tenantInfos.add(tenantInfo);
+        }
+        return tenantInfos;
+    }
+
+}

provider/search-reference/src/main/java/org/opengroup/osdu/search/provider/reference/provider/impl/ProviderHeaderServiceImpl.java

+package org.opengroup.osdu.search.provider.reference.provider.impl;
+
+import org.opengroup.osdu.search.provider.interfaces.IProviderHeaderService;
+import org.springframework.stereotype.Service;
+
+//TODO: Move to search-gcp once available
+@Service
+public class ProviderHeaderServiceImpl implements IProviderHeaderService {
+    private static final String DATA_GROUPS = "X-Data-Groups";
+    private static final String CRON_SERVICE = "X-AppEngine-Cron";
+