Commit aaab65a2 authored by Sanjeev-SLB's avatar Sanjeev-SLB
Browse files

Merge branch 'master' of...

Merge branch 'master' of https://community.opengroup.org/osdu/platform/system/storage into virtual_service
parents 5f37e17d 6145aa71
Pipeline #67612 failed with stages
in 32 minutes and 12 seconds
This diff is collapsed.
......@@ -23,7 +23,7 @@ import org.springframework.stereotype.Component;
import javax.inject.Inject;
@Component
@Component("LegalTagCache")
public class LegalTagCache implements ICache<String, String> {
@Inject
......
......@@ -58,7 +58,7 @@
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
<artifactId>os-core-lib-aws</artifactId>
<version>0.11.0-SNAPSHOT</version>
<version>0.11.0</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
......
......@@ -30,7 +30,7 @@ import org.springframework.stereotype.Component;
import javax.inject.Inject;
import java.util.Map;
@Component
@Component("LegalTagCache")
public class LegalTagCache implements ICache<String, String> {
@Value("${aws.elasticache.cluster.endpoint:null}")
String REDIS_SEARCH_HOST;
......
// Copyright © Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.storage.provider.azure.cache;
import org.opengroup.osdu.core.common.cache.RedisCache;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
import javax.inject.Named;
@Component("CursorCache")
@ConditionalOnProperty(value = "runtime.env.local", havingValue = "false", matchIfMissing = true)
public class CursorRedisCache extends RedisCache<String, String> {
public CursorRedisCache(
final @Named("REDIS_HOST") String host,
final @Named("REDIS_PORT") int port,
final @Named("REDIS_PASSWORD") String password,
final @Named("CURSOR_REDIS_TTL") int timeout,
@Value("${redis.database}") final int database) {
super(host, port, password, timeout, database, String.class, String.class);
}
}
// Copyright © Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.storage.provider.azure.cache;
import org.opengroup.osdu.core.common.cache.VmCache;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
@Component("CursorCache")
@ConditionalOnProperty(value = "runtime.env.local", havingValue = "true")
public class CursorVmCache extends VmCache<String, String> {
public CursorVmCache() {
super(600, 1000);
}
}
\ No newline at end of file
......@@ -20,6 +20,9 @@ public class RedisConfig {
@Value("${redis.group.ttl:30}")
public int groupRedisTtl;
@Value("${redis.cursor.ttl:90}")
public int cursorRedisTtl;
@Bean
@Named("REDIS_PORT")
public int getRedisPort() {
......@@ -36,6 +39,10 @@ public class RedisConfig {
@Named("GROUP_REDIS_TTL")
public int getGroupRedisTtl() { return groupRedisTtl; }
@Bean
@Named("CURSOR_REDIS_TTL")
public int getCursorRedisTtl() { return cursorRedisTtl; }
@Bean
@Named("REDIS_HOST")
public String redisHost(SecretClient kv) {
......
......@@ -15,16 +15,21 @@
package org.opengroup.osdu.storage.provider.azure.repository;
import com.azure.cosmos.CosmosException;
import com.google.common.base.Strings;
import com.lambdaworks.redis.RedisException;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.azure.query.CosmosStorePageRequest;
import org.opengroup.osdu.core.common.cache.ICache;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.storage.DatastoreQueryResult;
import org.opengroup.osdu.core.common.model.storage.RecordState;
import org.opengroup.osdu.core.common.util.Crc32c;
import org.opengroup.osdu.storage.provider.azure.RecordMetadataDoc;
import org.opengroup.osdu.storage.provider.azure.SchemaDoc;
import org.opengroup.osdu.storage.provider.interfaces.IQueryRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
......@@ -46,6 +51,10 @@ public class QueryRepository implements IQueryRepository {
@Autowired
private JaxRsDpsLog logger;
@Autowired
@Qualifier("CursorCache")
private ICache<String, String> cursorCache;
@Override
public DatastoreQueryResult getAllKinds(Integer limit, String cursor) {
......@@ -96,18 +105,26 @@ public class QueryRepository implements IQueryRepository {
}
@Override
public DatastoreQueryResult getAllRecordIdsFromKind(String kind, Integer limit, String cursor) {
public DatastoreQueryResult getAllRecordIdsFromKind(String kind, Integer limit, String hashedCursorKey) {
Assert.notNull(kind, "kind must not be null");
boolean paginated = false;
boolean paginated = false;
int numRecords = PAGE_SIZE;
if (limit != null) {
numRecords = limit > 0 ? limit : PAGE_SIZE;
paginated = true;
}
if (cursor != null && !cursor.isEmpty()) {
String cursor = null;
if (hashedCursorKey != null && !hashedCursorKey.isEmpty()) {
paginated = true;
try {
cursor = this.cursorCache.get(hashedCursorKey);
} catch (RedisException ex) {
this.logger.error(String.format("Error getting key %s from redis: %s", hashedCursorKey, ex.getMessage()), ex);
}
if (Strings.isNullOrEmpty(cursor)) throw this.getInvalidCursorException();
}
String status = RecordState.active.toString();
Sort sort = Sort.by(Sort.Direction.ASC, "id");
......@@ -124,7 +141,12 @@ public class QueryRepository implements IQueryRepository {
if (pageable instanceof CosmosStorePageRequest) {
continuation = ((CosmosStorePageRequest) pageable).getRequestContinuation();
}
dqr.setCursor(continuation);
if (!Strings.isNullOrEmpty(continuation)) {
String hashedCursor = Crc32c.hashToBase64EncodedString(continuation);
this.cursorCache.put(hashedCursor, continuation);
dqr.setCursor(hashedCursor);
}
docs = docPage.getContent();
} else {
docs = record.findByMetadata_kindAndMetadata_status(kind, status);
......
......@@ -41,7 +41,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-gcp</artifactId>
<version>0.11.0-SNAPSHOT</version>
<version>0.11.0</version>
</dependency>
<dependency>
......
......@@ -22,7 +22,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@Component
@Component("LegalTagCache")
public class LegalTagCache implements ICache<String, String> {
@Autowired
......
......@@ -90,7 +90,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-gcp</artifactId>
<version>0.11.0-SNAPSHOT</version>
<version>0.11.0</version>
</dependency>
</dependencies>
......
......@@ -31,6 +31,7 @@ import org.opengroup.osdu.core.common.model.storage.Record;
import org.opengroup.osdu.core.common.model.storage.RecordIdWithVersion;
import org.opengroup.osdu.core.common.model.storage.RecordMetadata;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Service;
import java.util.List;
......@@ -46,6 +47,7 @@ public class LegalServiceImpl implements ILegalService {
@Autowired
private DpsHeaders headers;
@Autowired
@Qualifier("LegalTagCache")
private ICache<String, String> cache;
@Autowired
private ILegalFactory factory;
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment