Commit b9b19153 authored by Igor Filippov (EPAM)'s avatar Igor Filippov (EPAM)
Browse files

Local development of Search Service for Anthos PoC

parent e798bfea
......@@ -50,6 +50,7 @@
<module>provider/search-aws</module>
<module>provider/search-azure</module>
<module>provider/search-ibm</module>
<module>provider/search-reference</module>
<!--<module>testing/integration-tests</module>-->
</modules>
......
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-search</artifactId>
<version>0.0.5-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>search-reference</artifactId>
<version>0.0.5-SNAPSHOT</version>
<description>MongoDB implementation of Search service APIs</description>
<packaging>jar</packaging>
<properties>
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.war.plugin>2.6</maven.war.plugin>
<appengine.maven.plugin>1.0.0</appengine.maven.plugin>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<failOnMissingWebXml>false</failOnMissingWebXml>
<project.main.basedir>${project.parent.basedir}</project.main.basedir>
</properties>
<dependencies>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>search-core</artifactId>
<version>0.0.5-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
<version>0.0.18</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-mongodb</artifactId>
<version>2.1.7.RELEASE</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<version>2.1.7.RELEASE</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-api-mockito2</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-module-junit4</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<scope>test</scope>
</dependency>
<!--Elasticsearch-->
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>6.6.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
<version>6.6.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>6.6.2</version>
</dependency>
<dependency>
<groupId>org.locationtech.jts.io</groupId>
<artifactId>jts-io-common</artifactId>
<version>1.15.0</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<classifier>spring-boot</classifier>
<mainClass>
org.opengroup.osdu.search.SearchApplication
</mainClass>
<profiles>
<profile>
<id>local</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<spring.profiles.active>local</spring.profiles.active>
</properties>
</profile>
<profile>
<id>dev</id>
<properties>
<spring.profiles.active>dev</spring.profiles.active>
</properties>
</profile>
</profiles>
</configuration>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
\ No newline at end of file
package org.opengroup.osdu.search.provider.reference;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.ComponentScan;
@ComponentScan({"org.opengroup.osdu"})
@SpringBootApplication(
exclude = {
org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration.class,
org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration.class})
public class SearchReferenceApplication {
public static void main(String[] args) {
SpringApplication.run(SearchReferenceApplication.class, args);
}
}
package org.opengroup.osdu.search.provider.reference.cache;
import org.opengroup.osdu.core.common.cache.RedisCache;
import org.opengroup.osdu.core.common.model.search.CursorSettings;
import org.opengroup.osdu.search.cache.CursorCache;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@Component
public class CursorCacheImpl extends RedisCache<String, CursorSettings> implements CursorCache {
public CursorCacheImpl(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST, @Value("${REDIS_SEARCH_PORT}") final int REDIS_SEARCH_PORT, @Value("${CURSOR_CACHE_EXPIRATION}") final int CURSOR_CACHE_EXPIRATION) {
super(REDIS_SEARCH_HOST, REDIS_SEARCH_PORT, CURSOR_CACHE_EXPIRATION, String.class, CursorSettings.class);
}
}
package org.opengroup.osdu.search.provider.reference.cache;
import org.opengroup.osdu.core.common.cache.RedisCache;
import org.opengroup.osdu.core.common.model.search.ClusterSettings;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@Component
public class ElasticCredentialsCache extends RedisCache<String, ClusterSettings> {
public ElasticCredentialsCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST,
@Value("${REDIS_SEARCH_PORT}") final int REDIS_SEARCH_PORT,
@Value("${ELASTIC_CACHE_EXPIRATION}") final int ELASTIC_CACHE_EXPIRATION) {
super(REDIS_SEARCH_HOST, REDIS_SEARCH_PORT, ELASTIC_CACHE_EXPIRATION * 60, String.class, ClusterSettings.class);
}
}
package org.opengroup.osdu.search.provider.reference.cache;
import java.util.HashSet;
import org.opengroup.osdu.core.common.cache.RedisCache;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@Component
public class FieldTypeMappingCache extends RedisCache<String, HashSet> {
public FieldTypeMappingCache(@Value("${REDIS_SEARCH_HOST}") final String REDIS_SEARCH_HOST,
@Value("${REDIS_SEARCH_PORT}") final int REDIS_SEARCH_PORT) {
super(REDIS_SEARCH_HOST, REDIS_SEARCH_PORT, 1440 * 60, String.class, HashSet.class);
}
}
package org.opengroup.osdu.search.provider.reference.di;
import org.opengroup.osdu.core.common.entitlements.EntitlementsAPIConfig;
import org.opengroup.osdu.core.common.entitlements.EntitlementsFactory;
import org.opengroup.osdu.core.common.entitlements.IEntitlementsFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.beans.factory.config.AbstractFactoryBean;
import org.springframework.stereotype.Component;
import org.springframework.web.context.annotation.RequestScope;
@Component
@RequestScope
public class EntitlementsClientFactory extends AbstractFactoryBean<IEntitlementsFactory> {
@Value("${AUTHORIZE_API}")
private String AUTHORIZE_API;
@Value("${AUTHORIZE_API_KEY:}")
private String AUTHORIZE_API_KEY;
@Override
protected IEntitlementsFactory createInstance() throws Exception {
return new EntitlementsFactory(EntitlementsAPIConfig
.builder()
.rootUrl(AUTHORIZE_API)
.apiKey(AUTHORIZE_API_KEY)
.build());
}
@Override
public Class<?> getObjectType() {
return IEntitlementsFactory.class;
}
}
\ No newline at end of file
package org.opengroup.osdu.search.provider.reference.di;
import static org.opengroup.osdu.search.provider.reference.provider.persistence.ElasticRepositoryReference.SEARCH_DATABASE;
import com.google.gson.Gson;
import com.mongodb.client.FindIterable;
import com.mongodb.client.MongoCollection;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import org.bson.Document;
import org.opengroup.osdu.core.common.cache.ICache;
import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
import org.opengroup.osdu.search.provider.reference.model.TenantInfoDocument;
import org.opengroup.osdu.search.provider.reference.provider.persistence.MongoDdmsClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@Component
public class TenantFactoryImpl implements ITenantFactory {
private static final Logger LOG = LoggerFactory.getLogger(TenantFactoryImpl.class);
public static final String TENANT_INFO = "TenantInfo";
@Autowired
private MongoDdmsClient mongoClient;
private Map<String, TenantInfo> tenants;
public boolean exists(String tenantName) {
if (this.tenants == null) {
initTenants();
}
return this.tenants.containsKey(tenantName);
}
public TenantInfo getTenantInfo(String tenantName) {
if (this.tenants == null) {
initTenants();
}
return this.tenants.get(tenantName);
}
public Collection<TenantInfo> listTenantInfo() {
if (this.tenants == null) {
initTenants();
}
return this.tenants.values();
}
public <V> ICache<String, V> createCache(String tenantName, String host, int port,
int expireTimeSeconds, Class<V> classOfV) {
return null;
}
public void flushCache() {
}
private void initTenants() {
this.tenants = new HashMap<>();
MongoCollection<Document> mongoCollection = mongoClient
.getMongoCollection(SEARCH_DATABASE, TENANT_INFO);
FindIterable<Document> results = mongoCollection.find();
if (Objects.isNull(results) && Objects.isNull(results.first())) {
LOG.error(String.format("Collection \'%s\' is empty.", results));
}
for (Document document : results) {
TenantInfoDocument tenantInfoDocument = new Gson()
.fromJson(document.toJson(), TenantInfoDocument.class);
TenantInfo tenantInfo = convertToTenantInfo(tenantInfoDocument);
this.tenants.put(tenantInfo.getName(), tenantInfo);
}
}
private TenantInfo convertToTenantInfo(TenantInfoDocument tenantInfoDocument) {
TenantInfo tenantInfo = new TenantInfo();
tenantInfo.setName(tenantInfoDocument.getId());
return tenantInfo;
}
}
package org.opengroup.osdu.search.provider.reference.model;
import javax.validation.constraints.NotEmpty;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@AllArgsConstructor
@NoArgsConstructor
public class ElasticSettingSchema {
@NotEmpty
private String host;
@NotEmpty
private String port;
@NotEmpty
private String usernameAndPassword;
@NotEmpty
private boolean isHttps;
}
package org.opengroup.osdu.search.provider.reference.model;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.springframework.data.mongodb.core.mapping.Document;
@Data
@AllArgsConstructor
@NoArgsConstructor
@Document(collection = "TenantInfo")
public class TenantInfoDocument {
private String id;
private List<String> groups;
}
package org.opengroup.osdu.search.provider.reference.provider.impl;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import javax.inject.Inject;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.search.CcsQueryRequest;
import org.opengroup.osdu.core.common.model.search.CcsQueryResponse;
import org.opengroup.osdu.core.common.model.search.QueryRequest;
import org.opengroup.osdu.core.common.model.search.QueryResponse;
import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository;
import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
import org.opengroup.osdu.core.common.search.Config;
import org.opengroup.osdu.search.provider.interfaces.ICcsQueryService;
import org.opengroup.osdu.search.provider.interfaces.IQueryService;
import org.springframework.stereotype.Service;
// TODO: Remove this temporary implementation when ECE CCS is utilized
@Service
public class CcsQueryServiceImpl implements ICcsQueryService {
@Inject
private DpsHeaders dpsHeaders;
@Inject
private ITenantFactory tenantStorageFactory;
@Inject
private IElasticRepository elasticRepository;
@Inject
private IQueryService queryService;
@Override
public CcsQueryResponse makeRequest(final CcsQueryRequest ccsQueryRequest) throws Exception {
List<String> accounts = Arrays.asList(dpsHeaders.getPartitionIdWithFallbackToAccountId().trim().split("\\s*,\\s*"));
List<QueryResponse> tenantResponses = getTenantResponses(accounts, ccsQueryRequest);
return convertQueryResponseToCcsQueryResponse(getCompoundResponse(tenantResponses));
}
private List<QueryResponse> getTenantResponses(final List<String> accounts, final CcsQueryRequest ccsQueryRequest) throws Exception {
List<QueryResponse> tenantResponses = new ArrayList<>();
if (Config.isSmartSearchCcsDisabled() || accounts.size() == 1) {
TenantInfo tenant = tenantStorageFactory.getTenantInfo(this.dpsHeaders.getPartitionIdWithFallbackToAccountId());
tenantResponses.add(queryService.queryIndex(convertCcsQueryRequestToQueryRequest(ccsQueryRequest),
elasticRepository.getElasticClusterSettings(tenant)));
} else {
ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
List<Future<QueryResponse>> futureResponses = new ArrayList<>();
for (String account : accounts) {
futureResponses.add(executorService.submit(() -> {
TenantInfo tenant = tenantStorageFactory.getTenantInfo(account);
return queryService.queryIndex(convertCcsQueryRequestToQueryRequest(ccsQueryRequest),
elasticRepository.getElasticClusterSettings(tenant));
}));
}
for (Future<QueryResponse> futureResponse : futureResponses) {
// TODO: Check why it can be null, this doesn't look right and leads to redundant null checks in other services
QueryResponse tenantResponse = futureResponse.get();
if (tenantResponse != null) {
tenantResponses.add(tenantResponse);
}
}
executorService.shutdown();
}
return tenantResponses;
}
private QueryResponse getCompoundResponse(final List<QueryResponse> tenantResponses) {
QueryResponse response = new QueryResponse();
if (tenantResponses.isEmpty()) {
return response;
} else {
for (QueryResponse tenantResponse : tenantResponses) {
response.setTotalCount(response.getTotalCount() + tenantResponse.getTotalCount());
}
tenantResponses.sort(Comparator.comparingLong(QueryResponse::getTotalCount).reversed());
QueryResponse largestResponse = tenantResponses.remove(0);
List<Map<String, Object>> results = new LinkedList<>();
for (Map<String, Object> result : largestResponse.getResults()) {
results.add(result);
int index = largestResponse.getResults().indexOf(result);
for (QueryResponse tenantResponse : tenantResponses) {
if (index < tenantResponse.getResults().size()) {
results.add(tenantResponse.getResults().get(index));
}
}
}
response.setResults(results);
return response;
}
}
private QueryRequest convertCcsQueryRequestToQueryRequest(final CcsQueryRequest ccsQueryRequest) {
QueryRequest queryRequest = new QueryRequest();
queryRequest.setFrom(ccsQueryRequest.getFrom());
queryRequest.setKind(ccsQueryRequest.getKind());
queryRequest.setLimit(ccsQueryRequest.getLimit());
queryRequest.setQuery(ccsQueryRequest.getQuery());
queryRequest.setQueryAsOwner(ccsQueryRequest.isQueryAsOwner());
return queryRequest;
}
private CcsQueryResponse convertQueryResponseToCcsQueryResponse(final QueryResponse queryResponse) {
CcsQueryResponse ccsQueryResponse = new CcsQueryResponse();
ccsQueryResponse.setResults(queryResponse.getResults());
ccsQueryResponse.setTotalCount(queryResponse.getTotalCount());
return ccsQueryResponse;
}
}
package org.opengroup.osdu.search.provider.reference.provider.impl;
import java.util.LinkedList;
import java.util.List;
import javax.inject.Inject;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
import org.opengroup.osdu.core.common.multitenancy.ITenantInfoService;
import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
import org.opengroup.osdu.search.provider.interfaces.ICrossTenantInfoService;
import org.springframework.stereotype.Service;
@Service
public class CrossTenantInfoServiceImpl implements ITenantInfoService, ICrossTenantInfoService {
@Inject
private ITenantFactory tenantFactory;
@Inject
private DpsHeaders headers;
@Override
public TenantInfo getTenantInfo() {
String primaryAccountId = this.headers.getPartitionIdWithFallbackToAccountId();
TenantInfo tenantInfo = this.tenantFactory.getTenantInfo(primaryAccountId);
if (tenantInfo == null) {
throw AppException.createUnauthorized(String.format("could not retrieve tenant info for data partition id: %s", primaryAccountId));
}
return tenantInfo;
}
@Override
public List<TenantInfo> getAllTenantsFromPartitionId() {
List<TenantInfo> tenantInfos = new LinkedList<>();
String[] accountIdList = headers.getPartitionIdWithFallbackToAccountId().split(",");
//Get all tenant values requested by user
for (String accountId : accountIdList) {
TenantInfo tenantInfo = tenantFactory.getTenantInfo(accountId);
tenantInfos.add(tenantInfo);
}
return tenantInfos;
}
}
package org.opengroup.osdu.search.provider.reference.provider.impl;
import org.opengroup.osdu.search.provider.interfaces.IProviderHeaderService;
import org.springframework.stereotype.Service;
//TODO: Move to search-gcp once available
@Service
public class ProviderHeaderServiceImpl implements IProviderHeaderService {
private static final String DATA_GROUPS = "X-Data-Groups";
private static final String CRON_SERVICE = "X-AppEngine-Cron";