Commit d05029b6 authored by neelesh thakur's avatar neelesh thakur
Browse files

rebase

parents 34bbf55b 9f99ca92
Pipeline #69389 passed with stages
in 36 minutes and 36 seconds
This diff is collapsed.
......@@ -19,4 +19,6 @@ policy:
enabled: #{POLICY_ENABLED}#
schemaEndpoints:
disabled: #{SCHEMA_ENDPOINTS_DISABLED}#
\ No newline at end of file
disabled: #{SCHEMA_ENDPOINTS_DISABLED}#
istioDnsHost: #{ISTIO_DNS_HOST}#
\ No newline at end of file
---
# Source: /devops/azure/chart/templates/virtual-service.yaml
apiVersion: networking.istio.io/v1alpha3
kind: VirtualService
metadata:
name: {{ .Chart.Name }}
namespace: osdu
spec:
hosts:
- "{{ .Values.istioDnsHost }}"
gateways:
- istio-gateway
http:
- match:
- uri:
prefix: "/api/{{ .Chart.Name }}/v2"
route:
- destination:
host: {{ .Chart.Name }}
port:
number: 80
corsPolicy:
maxAge: "60m"
allowCredentials: true
allowHeaders:
- Authorization
- Data-Partition-Id
- Correlation-Id
- Content-Type
allowMethods:
- POST
- GET
- PUT
- PATCH
- DELETE
allowOrigins:
- prefix: "*"
\ No newline at end of file
......@@ -26,4 +26,6 @@ policy:
enabled: true
schemaEndpoints:
disabled: false
\ No newline at end of file
disabled: false
istioDnsHost: ""
\ No newline at end of file
......@@ -9,7 +9,7 @@ spec:
selector:
matchLabels:
app: "{{ .Values.conf.app_name }}"
replicas: 1
replicas: {{ .Values.conf.replicas }}
template:
metadata:
labels:
......@@ -36,4 +36,3 @@ spec:
cpu: "{{ .Values.data.limits_cpu }}"
memory: "{{ .Values.data.limits_memory }}"
serviceAccountName: "{{ .Values.data.serviceAccountName }}"
......@@ -13,3 +13,4 @@ data:
conf:
configmap: "storage-config"
app_name: "storage"
replicas: 3
......@@ -23,7 +23,7 @@ import org.springframework.stereotype.Component;
import javax.inject.Inject;
@Component
@Component("LegalTagCache")
public class LegalTagCache implements ICache<String, String> {
@Inject
......
......@@ -30,7 +30,7 @@ import org.springframework.stereotype.Component;
import javax.inject.Inject;
import java.util.Map;
@Component
@Component("LegalTagCache")
public class LegalTagCache implements ICache<String, String> {
@Value("${aws.elasticache.cluster.endpoint:null}")
String REDIS_SEARCH_HOST;
......
......@@ -33,7 +33,7 @@
<azure.appservice.plan />
<azure.appservice.appname />
<azure.appservice.subscription />
<osdu.corelibazure.version>0.11.0-rc2</osdu.corelibazure.version>
<osdu.corelibazure.version>0.12.0-rc6</osdu.corelibazure.version>
<osdu.storage-core.version>0.12.0-SNAPSHOT</osdu.storage-core.version>
<junit.version>4.12</junit.version>
<mockito.version>1.10.19</mockito.version>
......
......@@ -12,32 +12,25 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.storage.provider.azure.util;
import com.azure.cosmos.implementation.RequestRateTooLargeException;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.storage.util.GlobalExceptionMapper;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.ControllerAdvice;
import org.springframework.web.bind.annotation.ExceptionHandler;
@ControllerAdvice
public class GlobalAzureExceptionMapper {
private GlobalExceptionMapper mapper;
public GlobalAzureExceptionMapper(GlobalExceptionMapper mapper) {
this.mapper = mapper;
package org.opengroup.osdu.storage.provider.azure.cache;
import org.opengroup.osdu.core.common.cache.RedisCache;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
import javax.inject.Named;
@Component("CursorCache")
@ConditionalOnProperty(value = "runtime.env.local", havingValue = "false", matchIfMissing = true)
public class CursorRedisCache extends RedisCache<String, String> {
public CursorRedisCache(
final @Named("REDIS_HOST") String host,
final @Named("REDIS_PORT") int port,
final @Named("REDIS_PASSWORD") String password,
final @Named("CURSOR_REDIS_TTL") int timeout,
@Value("${redis.database}") final int database) {
super(host, port, password, timeout, database, String.class, String.class);
}
@ExceptionHandler(RequestRateTooLargeException.class)
protected ResponseEntity<Object> handleCosmosdbException(Exception e) {
return mapper.getErrorResponse(
new AppException(HttpStatus.SERVICE_UNAVAILABLE.value(), "Too many requests on cosmosdb.",
"Request rate is large. Please retry this request later", e));
}
}
// Copyright © Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.storage.provider.azure.cache;
import org.opengroup.osdu.core.common.cache.VmCache;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
@Component("CursorCache")
@ConditionalOnProperty(value = "runtime.env.local", havingValue = "true")
public class CursorVmCache extends VmCache<String, String> {
public CursorVmCache() {
super(600, 1000);
}
}
\ No newline at end of file
......@@ -20,6 +20,9 @@ public class RedisConfig {
@Value("${redis.group.ttl:30}")
public int groupRedisTtl;
@Value("${redis.cursor.ttl:90}")
public int cursorRedisTtl;
@Bean
@Named("REDIS_PORT")
public int getRedisPort() {
......@@ -36,6 +39,10 @@ public class RedisConfig {
@Named("GROUP_REDIS_TTL")
public int getGroupRedisTtl() { return groupRedisTtl; }
@Bean
@Named("CURSOR_REDIS_TTL")
public int getCursorRedisTtl() { return cursorRedisTtl; }
@Bean
@Named("REDIS_HOST")
public String redisHost(SecretClient kv) {
......
......@@ -15,16 +15,21 @@
package org.opengroup.osdu.storage.provider.azure.repository;
import com.azure.cosmos.CosmosException;
import com.google.common.base.Strings;
import com.lambdaworks.redis.RedisException;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.azure.query.CosmosStorePageRequest;
import org.opengroup.osdu.core.common.cache.ICache;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.storage.DatastoreQueryResult;
import org.opengroup.osdu.core.common.model.storage.RecordState;
import org.opengroup.osdu.core.common.util.Crc32c;
import org.opengroup.osdu.storage.provider.azure.RecordMetadataDoc;
import org.opengroup.osdu.storage.provider.azure.SchemaDoc;
import org.opengroup.osdu.storage.provider.interfaces.IQueryRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
......@@ -46,6 +51,10 @@ public class QueryRepository implements IQueryRepository {
@Autowired
private JaxRsDpsLog logger;
@Autowired
@Qualifier("CursorCache")
private ICache<String, String> cursorCache;
@Override
public DatastoreQueryResult getAllKinds(Integer limit, String cursor) {
......@@ -96,18 +105,26 @@ public class QueryRepository implements IQueryRepository {
}
@Override
public DatastoreQueryResult getAllRecordIdsFromKind(String kind, Integer limit, String cursor) {
public DatastoreQueryResult getAllRecordIdsFromKind(String kind, Integer limit, String hashedCursorKey) {
Assert.notNull(kind, "kind must not be null");
boolean paginated = false;
boolean paginated = false;
int numRecords = PAGE_SIZE;
if (limit != null) {
numRecords = limit > 0 ? limit : PAGE_SIZE;
paginated = true;
}
if (cursor != null && !cursor.isEmpty()) {
String cursor = null;
if (hashedCursorKey != null && !hashedCursorKey.isEmpty()) {
paginated = true;
try {
cursor = this.cursorCache.get(hashedCursorKey);
} catch (RedisException ex) {
this.logger.error(String.format("Error getting key %s from redis: %s", hashedCursorKey, ex.getMessage()), ex);
}
if (Strings.isNullOrEmpty(cursor)) throw this.getInvalidCursorException();
}
String status = RecordState.active.toString();
Sort sort = Sort.by(Sort.Direction.ASC, "id");
......@@ -124,7 +141,12 @@ public class QueryRepository implements IQueryRepository {
if (pageable instanceof CosmosStorePageRequest) {
continuation = ((CosmosStorePageRequest) pageable).getRequestContinuation();
}
dqr.setCursor(continuation);
if (!Strings.isNullOrEmpty(continuation)) {
String hashedCursor = Crc32c.hashToBase64EncodedString(continuation);
this.cursorCache.put(hashedCursor, continuation);
dqr.setCursor(hashedCursor);
}
docs = docPage.getContent();
} else {
docs = record.findByMetadata_kindAndMetadata_status(kind, status);
......
// Copyright © Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.storage.provider.azure.util;
import com.azure.cosmos.implementation.RequestRateTooLargeException;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.runners.MockitoJUnitRunner;
import org.opengroup.osdu.core.common.model.http.AppError;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.storage.util.GlobalExceptionMapper;
import org.springframework.http.ResponseEntity;
import static org.junit.Assert.assertEquals;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.when;
import static org.springframework.http.HttpStatus.SERVICE_UNAVAILABLE;
@RunWith(MockitoJUnitRunner.class)
public class GlobalAzureExceptionMapperTest {
@InjectMocks
private GlobalAzureExceptionMapper sut;
@Mock
private GlobalExceptionMapper mapper;
@Test
public void should_returnServiceUnavailable_with_correct_reason_when_RequestRateTooLargeException_Is_Captured() {
RequestRateTooLargeException exception = Mockito.mock(RequestRateTooLargeException.class);
AppError expectedBody = new AppError(SERVICE_UNAVAILABLE.value(), "Too many requests on cosmosdb.", "Cosmosdb error.");
when(mapper.getErrorResponse(any(AppException.class))).thenReturn(new ResponseEntity<>(expectedBody, SERVICE_UNAVAILABLE));
ResponseEntity response = this.sut.handleCosmosdbException(exception);
assertEquals(503, response.getStatusCodeValue());
assertEquals("Too many requests on cosmosdb.", ((AppError)response.getBody()).getReason());
}
}
......@@ -208,8 +208,15 @@ public class GoogleCloudStorage implements ICloudStorage {
// inconsistency then cleanup and check the access again
// This makes all the APIs robust to inconsistent data, but will add some
// latency
if (!this.hasAccessRobustToDataCorruption(bucket, record, this.storageFactory.getStorage(this.headers.getUserEmail(), tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName(), properties.isEnableImpersonalization()))) {
return false;
try {
if (!this.hasAccessRobustToDataCorruption(bucket, record,
this.storageFactory.getStorage(this.headers.getUserEmail(),
tenant.getServiceAccount(), tenant.getProjectId(), tenant.getName(),
properties.isEnableImpersonalization()))) {
return false;
}
} catch (StorageException exception) {
throw new AppException(HttpStatus.SC_FORBIDDEN, ACCESS_DENIED_ERROR_REASON, ACCESS_DENIED_ERROR_MSG, e);
}
}
}
......
......@@ -22,7 +22,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@Component
@Component("LegalTagCache")
public class LegalTagCache implements ICache<String, String> {
@Autowired
......
......@@ -31,6 +31,7 @@ import org.opengroup.osdu.core.common.model.storage.Record;
import org.opengroup.osdu.core.common.model.storage.RecordIdWithVersion;
import org.opengroup.osdu.core.common.model.storage.RecordMetadata;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Service;
import java.util.List;
......@@ -46,6 +47,7 @@ public class LegalServiceImpl implements ILegalService {
@Autowired
private DpsHeaders headers;
@Autowired
@Qualifier("LegalTagCache")
private ICache<String, String> cache;
@Autowired
private ILegalFactory factory;
......
......@@ -19,6 +19,8 @@ import javax.validation.ValidationException;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException;
import javassist.NotFoundException;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.Ordered;
......@@ -35,6 +37,8 @@ import org.springframework.web.context.request.WebRequest;
import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler;
import org.opengroup.osdu.core.common.model.http.AppException;
import java.io.IOException;
@Order(Ordered.HIGHEST_PRECEDENCE)
@ControllerAdvice
public class GlobalExceptionMapper extends ResponseEntityExceptionHandler {
......@@ -77,6 +81,17 @@ public class GlobalExceptionMapper extends ResponseEntityExceptionHandler {
new AppException(HttpStatus.FORBIDDEN.value(), "Access denied", e.getMessage(), e));
}
@ExceptionHandler(IOException.class)
public ResponseEntity<Object> handleIOException(IOException e) {
if (StringUtils.containsIgnoreCase(ExceptionUtils.getRootCauseMessage(e), "Broken pipe")) {
this.logger.warning("Client closed the connection while request still being processed");
return null;
} else {
return this.getErrorResponse(
new AppException(HttpStatus.SERVICE_UNAVAILABLE.value(), "Unknown error", e.getMessage(), e));
}
}
@Override
@NonNull
protected ResponseEntity<Object> handleHttpRequestMethodNotSupported(@NonNull HttpRequestMethodNotSupportedException e,
......
......@@ -16,6 +16,7 @@ package org.opengroup.osdu.storage.util;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.mockito.Mockito.verify;
import javax.validation.ValidationException;
......@@ -36,6 +37,8 @@ import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.springframework.http.ResponseEntity;
import org.mockito.runners.MockitoJUnitRunner;
import java.io.IOException;
@RunWith(MockitoJUnitRunner.class)
public class GlobalExceptionMapperTest {
......@@ -109,4 +112,22 @@ public class GlobalExceptionMapperTest {
assertEquals(AppError.class, response.getBody().getClass());
assertEquals("Unrecognized property.", ((AppError)response.getBody()).getReason());
}
@Test
public void should_returnNullResponse_when_BrokenPipeIOExceptionIsCaptured() {
IOException ioException = new IOException("Broken pipe");
ResponseEntity response = this.sut.handleIOException(ioException);
assertNull(response);
}
@Test
public void should_returnServiceUnavailable_when_IOExceptionIsCaptured() {
IOException ioException = new IOException("Not broken yet");
ResponseEntity response = this.sut.handleIOException(ioException);
assertEquals(HttpStatus.SC_SERVICE_UNAVAILABLE, response.getStatusCodeValue());
}
}
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment