Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
Menu
Open sidebar
Open Subsurface Data Universe Software
Platform
System
Storage
Commits
d05029b6
Commit
d05029b6
authored
Oct 01, 2021
by
neelesh thakur
Browse files
rebase
parents
34bbf55b
9f99ca92
Pipeline
#69389
passed with stages
in 36 minutes and 36 seconds
Changes
19
Pipelines
3
Expand all
Hide whitespace changes
Inline
Side-by-side
NOTICE
View file @
d05029b6
This diff is collapsed.
Click to expand it.
devops/azure/chart/helm-config.yaml
View file @
d05029b6
...
...
@@ -19,4 +19,6 @@ policy:
enabled
:
#{POLICY_ENABLED}#
schemaEndpoints
:
disabled
:
#{SCHEMA_ENDPOINTS_DISABLED}#
\ No newline at end of file
disabled
:
#{SCHEMA_ENDPOINTS_DISABLED}#
istioDnsHost
:
#{ISTIO_DNS_HOST}#
\ No newline at end of file
devops/azure/chart/templates/virtual-service.yaml
0 → 100644
View file @
d05029b6
---
# Source: /devops/azure/chart/templates/virtual-service.yaml
apiVersion
:
networking.istio.io/v1alpha3
kind
:
VirtualService
metadata
:
name
:
{{
.Chart.Name
}}
namespace
:
osdu
spec
:
hosts
:
-
"
{{
.Values.istioDnsHost
}}"
gateways
:
-
istio-gateway
http
:
-
match
:
-
uri
:
prefix
:
"
/api/{{
.Chart.Name
}}/v2"
route
:
-
destination
:
host
:
{{
.Chart.Name
}}
port
:
number
:
80
corsPolicy
:
maxAge
:
"
60m"
allowCredentials
:
true
allowHeaders
:
-
Authorization
-
Data-Partition-Id
-
Correlation-Id
-
Content-Type
allowMethods
:
-
POST
-
GET
-
PUT
-
PATCH
-
DELETE
allowOrigins
:
-
prefix
:
"
*"
\ No newline at end of file
devops/azure/chart/values.yaml
View file @
d05029b6
...
...
@@ -26,4 +26,6 @@ policy:
enabled
:
true
schemaEndpoints
:
disabled
:
false
\ No newline at end of file
disabled
:
false
istioDnsHost
:
"
"
\ No newline at end of file
devops/gcp/deploy/templates/storage-deploy.yml
View file @
d05029b6
...
...
@@ -9,7 +9,7 @@ spec:
selector
:
matchLabels
:
app
:
"
{{
.Values.conf.app_name
}}"
replicas
:
1
replicas
:
{{
.Values.conf.replicas
}}
template
:
metadata
:
labels
:
...
...
@@ -36,4 +36,3 @@ spec:
cpu
:
"
{{
.Values.data.limits_cpu
}}"
memory
:
"
{{
.Values.data.limits_memory
}}"
serviceAccountName
:
"
{{
.Values.data.serviceAccountName
}}"
devops/gcp/deploy/values.yaml
View file @
d05029b6
...
...
@@ -13,3 +13,4 @@ data:
conf
:
configmap
:
"
storage-config"
app_name
:
"
storage"
replicas
:
3
provider/storage-aws-mongodb/src/main/java/org/opengroup/osdu/storage/provider/mongodb/cache/LegalTagCache.java
View file @
d05029b6
...
...
@@ -23,7 +23,7 @@ import org.springframework.stereotype.Component;
import
javax.inject.Inject
;
@Component
@Component
(
"LegalTagCache"
)
public
class
LegalTagCache
implements
ICache
<
String
,
String
>
{
@Inject
...
...
provider/storage-aws/src/main/java/org/opengroup/osdu/storage/provider/aws/cache/LegalTagCache.java
View file @
d05029b6
...
...
@@ -30,7 +30,7 @@ import org.springframework.stereotype.Component;
import
javax.inject.Inject
;
import
java.util.Map
;
@Component
@Component
(
"LegalTagCache"
)
public
class
LegalTagCache
implements
ICache
<
String
,
String
>
{
@Value
(
"${aws.elasticache.cluster.endpoint:null}"
)
String
REDIS_SEARCH_HOST
;
...
...
provider/storage-azure/pom.xml
View file @
d05029b6
...
...
@@ -33,7 +33,7 @@
<azure.appservice.plan
/>
<azure.appservice.appname
/>
<azure.appservice.subscription
/>
<osdu.corelibazure.version>
0.1
1
.0-rc
2
</osdu.corelibazure.version>
<osdu.corelibazure.version>
0.1
2
.0-rc
6
</osdu.corelibazure.version>
<osdu.storage-core.version>
0.12.0-SNAPSHOT
</osdu.storage-core.version>
<junit.version>
4.12
</junit.version>
<mockito.version>
1.10.19
</mockito.version>
...
...
provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/
util/GlobalAzureExceptionMapper
.java
→
provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/
cache/CursorRedisCache
.java
View file @
d05029b6
...
...
@@ -12,32 +12,25 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package
org.opengroup.osdu.storage.provider.azure.util
;
import
com.azure.cosmos.implementation.RequestRateTooLargeException
;
import
org.opengroup.osdu.core.common.model.http.AppException
;
import
org.opengroup.osdu.storage.util.GlobalExceptionMapper
;
import
org.springframework.http.HttpStatus
;
import
org.springframework.http.ResponseEntity
;
import
org.springframework.web.bind.annotation.ControllerAdvice
;
import
org.springframework.web.bind.annotation.ExceptionHandler
;
@ControllerAdvice
public
class
GlobalAzureExceptionMapper
{
private
GlobalExceptionMapper
mapper
;
public
GlobalAzureExceptionMapper
(
GlobalExceptionMapper
mapper
)
{
this
.
mapper
=
mapper
;
package
org.opengroup.osdu.storage.provider.azure.cache
;
import
org.opengroup.osdu.core.common.cache.RedisCache
;
import
org.springframework.beans.factory.annotation.Value
;
import
org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
;
import
org.springframework.stereotype.Component
;
import
javax.inject.Named
;
@Component
(
"CursorCache"
)
@ConditionalOnProperty
(
value
=
"runtime.env.local"
,
havingValue
=
"false"
,
matchIfMissing
=
true
)
public
class
CursorRedisCache
extends
RedisCache
<
String
,
String
>
{
public
CursorRedisCache
(
final
@Named
(
"REDIS_HOST"
)
String
host
,
final
@Named
(
"REDIS_PORT"
)
int
port
,
final
@Named
(
"REDIS_PASSWORD"
)
String
password
,
final
@Named
(
"CURSOR_REDIS_TTL"
)
int
timeout
,
@Value
(
"${redis.database}"
)
final
int
database
)
{
super
(
host
,
port
,
password
,
timeout
,
database
,
String
.
class
,
String
.
class
);
}
@ExceptionHandler
(
RequestRateTooLargeException
.
class
)
protected
ResponseEntity
<
Object
>
handleCosmosdbException
(
Exception
e
)
{
return
mapper
.
getErrorResponse
(
new
AppException
(
HttpStatus
.
SERVICE_UNAVAILABLE
.
value
(),
"Too many requests on cosmosdb."
,
"Request rate is large. Please retry this request later"
,
e
));
}
}
provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/cache/CursorVmCache.java
0 → 100644
View file @
d05029b6
// Copyright © Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package
org.opengroup.osdu.storage.provider.azure.cache
;
import
org.opengroup.osdu.core.common.cache.VmCache
;
import
org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
;
import
org.springframework.stereotype.Component
;
@Component
(
"CursorCache"
)
@ConditionalOnProperty
(
value
=
"runtime.env.local"
,
havingValue
=
"true"
)
public
class
CursorVmCache
extends
VmCache
<
String
,
String
>
{
public
CursorVmCache
()
{
super
(
600
,
1000
);
}
}
\ No newline at end of file
provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/di/RedisConfig.java
View file @
d05029b6
...
...
@@ -20,6 +20,9 @@ public class RedisConfig {
@Value
(
"${redis.group.ttl:30}"
)
public
int
groupRedisTtl
;
@Value
(
"${redis.cursor.ttl:90}"
)
public
int
cursorRedisTtl
;
@Bean
@Named
(
"REDIS_PORT"
)
public
int
getRedisPort
()
{
...
...
@@ -36,6 +39,10 @@ public class RedisConfig {
@Named
(
"GROUP_REDIS_TTL"
)
public
int
getGroupRedisTtl
()
{
return
groupRedisTtl
;
}
@Bean
@Named
(
"CURSOR_REDIS_TTL"
)
public
int
getCursorRedisTtl
()
{
return
cursorRedisTtl
;
}
@Bean
@Named
(
"REDIS_HOST"
)
public
String
redisHost
(
SecretClient
kv
)
{
...
...
provider/storage-azure/src/main/java/org/opengroup/osdu/storage/provider/azure/repository/QueryRepository.java
View file @
d05029b6
...
...
@@ -15,16 +15,21 @@
package
org.opengroup.osdu.storage.provider.azure.repository
;
import
com.azure.cosmos.CosmosException
;
import
com.google.common.base.Strings
;
import
com.lambdaworks.redis.RedisException
;
import
org.apache.http.HttpStatus
;
import
org.opengroup.osdu.azure.query.CosmosStorePageRequest
;
import
org.opengroup.osdu.core.common.cache.ICache
;
import
org.opengroup.osdu.core.common.logging.JaxRsDpsLog
;
import
org.opengroup.osdu.core.common.model.http.AppException
;
import
org.opengroup.osdu.core.common.model.storage.DatastoreQueryResult
;
import
org.opengroup.osdu.core.common.model.storage.RecordState
;
import
org.opengroup.osdu.core.common.util.Crc32c
;
import
org.opengroup.osdu.storage.provider.azure.RecordMetadataDoc
;
import
org.opengroup.osdu.storage.provider.azure.SchemaDoc
;
import
org.opengroup.osdu.storage.provider.interfaces.IQueryRepository
;
import
org.springframework.beans.factory.annotation.Autowired
;
import
org.springframework.beans.factory.annotation.Qualifier
;
import
org.springframework.data.domain.Page
;
import
org.springframework.data.domain.Pageable
;
import
org.springframework.data.domain.Sort
;
...
...
@@ -46,6 +51,10 @@ public class QueryRepository implements IQueryRepository {
@Autowired
private
JaxRsDpsLog
logger
;
@Autowired
@Qualifier
(
"CursorCache"
)
private
ICache
<
String
,
String
>
cursorCache
;
@Override
public
DatastoreQueryResult
getAllKinds
(
Integer
limit
,
String
cursor
)
{
...
...
@@ -96,18 +105,26 @@ public class QueryRepository implements IQueryRepository {
}
@Override
public
DatastoreQueryResult
getAllRecordIdsFromKind
(
String
kind
,
Integer
limit
,
String
c
ursor
)
{
public
DatastoreQueryResult
getAllRecordIdsFromKind
(
String
kind
,
Integer
limit
,
String
hashedC
ursor
Key
)
{
Assert
.
notNull
(
kind
,
"kind must not be null"
);
boolean
paginated
=
false
;
boolean
paginated
=
false
;
int
numRecords
=
PAGE_SIZE
;
if
(
limit
!=
null
)
{
numRecords
=
limit
>
0
?
limit
:
PAGE_SIZE
;
paginated
=
true
;
}
if
(
cursor
!=
null
&&
!
cursor
.
isEmpty
())
{
String
cursor
=
null
;
if
(
hashedCursorKey
!=
null
&&
!
hashedCursorKey
.
isEmpty
())
{
paginated
=
true
;
try
{
cursor
=
this
.
cursorCache
.
get
(
hashedCursorKey
);
}
catch
(
RedisException
ex
)
{
this
.
logger
.
error
(
String
.
format
(
"Error getting key %s from redis: %s"
,
hashedCursorKey
,
ex
.
getMessage
()),
ex
);
}
if
(
Strings
.
isNullOrEmpty
(
cursor
))
throw
this
.
getInvalidCursorException
();
}
String
status
=
RecordState
.
active
.
toString
();
Sort
sort
=
Sort
.
by
(
Sort
.
Direction
.
ASC
,
"id"
);
...
...
@@ -124,7 +141,12 @@ public class QueryRepository implements IQueryRepository {
if
(
pageable
instanceof
CosmosStorePageRequest
)
{
continuation
=
((
CosmosStorePageRequest
)
pageable
).
getRequestContinuation
();
}
dqr
.
setCursor
(
continuation
);
if
(!
Strings
.
isNullOrEmpty
(
continuation
))
{
String
hashedCursor
=
Crc32c
.
hashToBase64EncodedString
(
continuation
);
this
.
cursorCache
.
put
(
hashedCursor
,
continuation
);
dqr
.
setCursor
(
hashedCursor
);
}
docs
=
docPage
.
getContent
();
}
else
{
docs
=
record
.
findByMetadata_kindAndMetadata_status
(
kind
,
status
);
...
...
provider/storage-azure/src/test/java/org/opengroup/osdu/storage/provider/azure/util/GlobalAzureExceptionMapperTest.java
deleted
100644 → 0
View file @
34bbf55b
// Copyright © Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package
org.opengroup.osdu.storage.provider.azure.util
;
import
com.azure.cosmos.implementation.RequestRateTooLargeException
;
import
org.junit.Test
;
import
org.junit.runner.RunWith
;
import
org.mockito.InjectMocks
;
import
org.mockito.Mock
;
import
org.mockito.Mockito
;
import
org.mockito.runners.MockitoJUnitRunner
;
import
org.opengroup.osdu.core.common.model.http.AppError
;
import
org.opengroup.osdu.core.common.model.http.AppException
;
import
org.opengroup.osdu.storage.util.GlobalExceptionMapper
;
import
org.springframework.http.ResponseEntity
;
import
static
org
.
junit
.
Assert
.
assertEquals
;
import
static
org
.
mockito
.
Matchers
.
any
;
import
static
org
.
mockito
.
Mockito
.
when
;
import
static
org
.
springframework
.
http
.
HttpStatus
.
SERVICE_UNAVAILABLE
;
@RunWith
(
MockitoJUnitRunner
.
class
)
public
class
GlobalAzureExceptionMapperTest
{
@InjectMocks
private
GlobalAzureExceptionMapper
sut
;
@Mock
private
GlobalExceptionMapper
mapper
;
@Test
public
void
should_returnServiceUnavailable_with_correct_reason_when_RequestRateTooLargeException_Is_Captured
()
{
RequestRateTooLargeException
exception
=
Mockito
.
mock
(
RequestRateTooLargeException
.
class
);
AppError
expectedBody
=
new
AppError
(
SERVICE_UNAVAILABLE
.
value
(),
"Too many requests on cosmosdb."
,
"Cosmosdb error."
);
when
(
mapper
.
getErrorResponse
(
any
(
AppException
.
class
))).
thenReturn
(
new
ResponseEntity
<>(
expectedBody
,
SERVICE_UNAVAILABLE
));
ResponseEntity
response
=
this
.
sut
.
handleCosmosdbException
(
exception
);
assertEquals
(
503
,
response
.
getStatusCodeValue
());
assertEquals
(
"Too many requests on cosmosdb."
,
((
AppError
)
response
.
getBody
()).
getReason
());
}
}
provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/GoogleCloudStorage.java
View file @
d05029b6
...
...
@@ -208,8 +208,15 @@ public class GoogleCloudStorage implements ICloudStorage {
// inconsistency then cleanup and check the access again
// This makes all the APIs robust to inconsistent data, but will add some
// latency
if
(!
this
.
hasAccessRobustToDataCorruption
(
bucket
,
record
,
this
.
storageFactory
.
getStorage
(
this
.
headers
.
getUserEmail
(),
tenant
.
getServiceAccount
(),
tenant
.
getProjectId
(),
tenant
.
getName
(),
properties
.
isEnableImpersonalization
())))
{
return
false
;
try
{
if
(!
this
.
hasAccessRobustToDataCorruption
(
bucket
,
record
,
this
.
storageFactory
.
getStorage
(
this
.
headers
.
getUserEmail
(),
tenant
.
getServiceAccount
(),
tenant
.
getProjectId
(),
tenant
.
getName
(),
properties
.
isEnableImpersonalization
())))
{
return
false
;
}
}
catch
(
StorageException
exception
)
{
throw
new
AppException
(
HttpStatus
.
SC_FORBIDDEN
,
ACCESS_DENIED_ERROR_REASON
,
ACCESS_DENIED_ERROR_MSG
,
e
);
}
}
}
...
...
provider/storage-gcp/src/main/java/org/opengroup/osdu/storage/provider/gcp/cache/LegalTagCache.java
View file @
d05029b6
...
...
@@ -22,7 +22,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import
org.springframework.beans.factory.annotation.Value
;
import
org.springframework.stereotype.Component
;
@Component
@Component
(
"LegalTagCache"
)
public
class
LegalTagCache
implements
ICache
<
String
,
String
>
{
@Autowired
...
...
storage-core/src/main/java/org/opengroup/osdu/storage/service/LegalServiceImpl.java
View file @
d05029b6
...
...
@@ -31,6 +31,7 @@ import org.opengroup.osdu.core.common.model.storage.Record;
import
org.opengroup.osdu.core.common.model.storage.RecordIdWithVersion
;
import
org.opengroup.osdu.core.common.model.storage.RecordMetadata
;
import
org.springframework.beans.factory.annotation.Autowired
;
import
org.springframework.beans.factory.annotation.Qualifier
;
import
org.springframework.stereotype.Service
;
import
java.util.List
;
...
...
@@ -46,6 +47,7 @@ public class LegalServiceImpl implements ILegalService {
@Autowired
private
DpsHeaders
headers
;
@Autowired
@Qualifier
(
"LegalTagCache"
)
private
ICache
<
String
,
String
>
cache
;
@Autowired
private
ILegalFactory
factory
;
...
...
storage-core/src/main/java/org/opengroup/osdu/storage/util/GlobalExceptionMapper.java
View file @
d05029b6
...
...
@@ -19,6 +19,8 @@ import javax.validation.ValidationException;
import
com.fasterxml.jackson.core.JsonProcessingException
;
import
com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException
;
import
javassist.NotFoundException
;
import
org.apache.commons.lang3.StringUtils
;
import
org.apache.commons.lang3.exception.ExceptionUtils
;
import
org.opengroup.osdu.core.common.logging.JaxRsDpsLog
;
import
org.springframework.beans.factory.annotation.Autowired
;
import
org.springframework.core.Ordered
;
...
...
@@ -35,6 +37,8 @@ import org.springframework.web.context.request.WebRequest;
import
org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler
;
import
org.opengroup.osdu.core.common.model.http.AppException
;
import
java.io.IOException
;
@Order
(
Ordered
.
HIGHEST_PRECEDENCE
)
@ControllerAdvice
public
class
GlobalExceptionMapper
extends
ResponseEntityExceptionHandler
{
...
...
@@ -77,6 +81,17 @@ public class GlobalExceptionMapper extends ResponseEntityExceptionHandler {
new
AppException
(
HttpStatus
.
FORBIDDEN
.
value
(),
"Access denied"
,
e
.
getMessage
(),
e
));
}
@ExceptionHandler
(
IOException
.
class
)
public
ResponseEntity
<
Object
>
handleIOException
(
IOException
e
)
{
if
(
StringUtils
.
containsIgnoreCase
(
ExceptionUtils
.
getRootCauseMessage
(
e
),
"Broken pipe"
))
{
this
.
logger
.
warning
(
"Client closed the connection while request still being processed"
);
return
null
;
}
else
{
return
this
.
getErrorResponse
(
new
AppException
(
HttpStatus
.
SERVICE_UNAVAILABLE
.
value
(),
"Unknown error"
,
e
.
getMessage
(),
e
));
}
}
@Override
@NonNull
protected
ResponseEntity
<
Object
>
handleHttpRequestMethodNotSupported
(
@NonNull
HttpRequestMethodNotSupportedException
e
,
...
...
storage-core/src/test/java/org/opengroup/osdu/storage/util/GlobalExceptionMapperTest.java
View file @
d05029b6
...
...
@@ -16,6 +16,7 @@ package org.opengroup.osdu.storage.util;
import
static
org
.
junit
.
Assert
.
assertEquals
;
import
static
org
.
junit
.
Assert
.
assertNotNull
;
import
static
org
.
junit
.
Assert
.
assertNull
;
import
static
org
.
mockito
.
Mockito
.
verify
;
import
javax.validation.ValidationException
;
...
...
@@ -36,6 +37,8 @@ import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import
org.springframework.http.ResponseEntity
;
import
org.mockito.runners.MockitoJUnitRunner
;
import
java.io.IOException
;
@RunWith
(
MockitoJUnitRunner
.
class
)
public
class
GlobalExceptionMapperTest
{
...
...
@@ -109,4 +112,22 @@ public class GlobalExceptionMapperTest {
assertEquals
(
AppError
.
class
,
response
.
getBody
().
getClass
());
assertEquals
(
"Unrecognized property."
,
((
AppError
)
response
.
getBody
()).
getReason
());
}
@Test
public
void
should_returnNullResponse_when_BrokenPipeIOExceptionIsCaptured
()
{
IOException
ioException
=
new
IOException
(
"Broken pipe"
);
ResponseEntity
response
=
this
.
sut
.
handleIOException
(
ioException
);
assertNull
(
response
);
}
@Test
public
void
should_returnServiceUnavailable_when_IOExceptionIsCaptured
()
{
IOException
ioException
=
new
IOException
(
"Not broken yet"
);
ResponseEntity
response
=
this
.
sut
.
handleIOException
(
ioException
);
assertEquals
(
HttpStatus
.
SC_SERVICE_UNAVAILABLE
,
response
.
getStatusCodeValue
());
}
}
\ No newline at end of file
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment