Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
Menu
Open sidebar
Open Subsurface Data Universe Software
Platform
System
Schema
Commits
9e1788d1
Commit
9e1788d1
authored
May 17, 2021
by
Matt Wise
Browse files
Merge branch 'aws-multitenant-v2' into 'master'
Multitenant/Data Split See merge request
!107
parents
4de76a9a
8630e50c
Pipeline
#40370
canceled with stages
in 1 minute and 58 seconds
Changes
14
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
provider/schema-aws/pom.xml
View file @
9e1788d1
...
...
@@ -27,7 +27,7 @@
<packaging>
jar
</packaging>
<properties>
<aws.version>
1.11.
637
</aws.version>
<aws.version>
1.11.
1018
</aws.version>
</properties>
<dependencies>
...
...
@@ -67,7 +67,7 @@
<dependency>
<groupId>
org.opengroup.osdu.core.aws
</groupId>
<artifactId>
os-core-lib-aws
</artifactId>
<version>
0.
3
.1
7
</version>
<version>
0.
9
.1
-SNAPSHOT
</version>
</dependency>
<dependency>
...
...
provider/schema-aws/src/main/java/org/opengroup/osdu/schema/provider/aws/config/AwsServiceConfig.java
View file @
9e1788d1
...
...
@@ -14,17 +14,12 @@
package
org.opengroup.osdu.schema.provider.aws.config
;
import
lombok.AccessLevel
;
import
lombok.Getter
;
import
lombok.Setter
;
import
org.opengroup.osdu.core.aws.ssm.ParameterStorePropertySource
;
import
org.opengroup.osdu.core.aws.ssm.SSMConfig
;
import
org.opengroup.osdu.core.common.logging.JaxRsDpsLog
;
import
org.springframework.beans.factory.annotation.Value
;
import
org.springframework.stereotype.Component
;
import
javax.annotation.PostConstruct
;
import
javax.inject.Inject
;
import
lombok.AccessLevel
;
import
lombok.Getter
;
import
lombok.Setter
;
@Component
public
class
AwsServiceConfig
{
...
...
@@ -39,49 +34,10 @@ public class AwsServiceConfig {
@Setter
(
AccessLevel
.
PROTECTED
)
public
String
environment
;
@Value
(
"${aws.dynamodb.table.prefix}"
)
@Getter
()
@Setter
(
AccessLevel
.
PROTECTED
)
public
String
dynamoDbTablePrefix
;
@Value
(
"${aws.dynamodb.endpoint}"
)
@Getter
()
@Setter
(
AccessLevel
.
PROTECTED
)
public
String
dynamoDbEndpoint
;
@Value
(
"${aws.s3.endpoint}"
)
@Getter
()
@Setter
(
AccessLevel
.
PROTECTED
)
public
String
s3Endpoint
;
@Value
(
"${aws.s3.bucket}"
)
@Getter
()
@Setter
(
AccessLevel
.
PROTECTED
)
public
String
s3DataBucket
;
@Value
(
"${aws.ssm}"
)
@Getter
()
@Setter
(
AccessLevel
.
PROTECTED
)
public
Boolean
ssmEnabled
;
/*@Inject
protected JaxRsDpsLog logger;*/
@PostConstruct
public
void
init
()
{
if
(
ssmEnabled
)
{
SSMConfig
ssmConfig
=
new
SSMConfig
();
ParameterStorePropertySource
ssm
=
ssmConfig
.
amazonSSM
();
// String s3DataBucketParameter = "/osdu/" + environment + "/global-s3-bucket";
String
s3DataBucketParameter
=
"/osdu/"
+
environment
+
"/schema/schema-s3-bucket-name"
;
try
{
s3DataBucket
=
ssm
.
getProperty
(
s3DataBucketParameter
).
toString
();
}
catch
(
Exception
e
)
{
//logger.error(String.format("SSM property %s not found", s3DataBucketParameter));
System
.
out
.
println
(
String
.
format
(
"SSM property %s not found"
,
s3DataBucketParameter
));
}
}
}
}
provider/schema-aws/src/main/java/org/opengroup/osdu/schema/provider/aws/impl/schemainfostore/AwsAuthorityStore.java
View file @
9e1788d1
...
...
@@ -13,7 +13,8 @@
// limitations under the License.
package
org.opengroup.osdu.schema.provider.aws.impl.schemainfostore
;
import
org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelper
;
import
org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelperFactory
;
import
org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelperV2
;
import
org.opengroup.osdu.core.common.logging.JaxRsDpsLog
;
import
org.opengroup.osdu.core.common.model.http.DpsHeaders
;
import
org.opengroup.osdu.schema.constants.SchemaConstants
;
...
...
@@ -24,9 +25,9 @@ import org.opengroup.osdu.schema.model.Authority;
import
org.opengroup.osdu.schema.provider.aws.config.AwsServiceConfig
;
import
org.opengroup.osdu.schema.provider.aws.models.AuthorityDoc
;
import
org.opengroup.osdu.schema.provider.interfaces.schemainfostore.IAuthorityStore
;
import
org.springframework.beans.factory.annotation.Value
;
import
org.springframework.stereotype.Repository
;
import
javax.annotation.PostConstruct
;
import
javax.inject.Inject
;
import
java.text.MessageFormat
;
...
...
@@ -40,30 +41,34 @@ public class AwsAuthorityStore implements IAuthorityStore {
private
JaxRsDpsLog
logger
;
@Inject
private
AwsServiceConfig
serviceConfig
;
private
DynamoDBQueryHelperFactory
dynamoDBQueryHelperFactory
;
private
DynamoDBQueryHelper
queryHelper
;
@Value
(
"${aws.dynamodb.authorityTable.ssm.relativePath}"
)
String
authorityTableParameterRelativePath
;
@PostConstruct
public
void
init
()
{
// TODO: serviceConfig.environment isn't correct and needs to be table prefix. Maybe the "-" will fix it
queryHelper
=
new
DynamoDBQueryHelper
(
serviceConfig
.
getDynamoDbEndpoint
(),
serviceConfig
.
getAmazonRegion
(),
serviceConfig
.
getDynamoDbTablePrefix
());
private
DynamoDBQueryHelperV2
getAuthorityTableQueryHelper
()
{
return
dynamoDBQueryHelperFactory
.
getQueryHelperForPartition
(
headers
,
authorityTableParameterRelativePath
);
}
@Override
public
Authority
get
(
String
authorityId
)
throws
NotFoundException
,
ApplicationException
{
String
id
=
headers
.
getPartitionId
()
+
":"
+
authorityId
;
AuthorityDoc
result
=
queryHelper
.
loadByPrimaryKey
(
AuthorityDoc
.
class
,
id
);
if
(
result
==
null
)
{
throw
new
NotFoundException
(
SchemaConstants
.
INVALID_INPUT
);
}
return
result
.
getAuthority
();
DynamoDBQueryHelperV2
queryHelper
=
getAuthorityTableQueryHelper
();
String
id
=
headers
.
getPartitionId
()
+
":"
+
authorityId
;
AuthorityDoc
result
=
queryHelper
.
loadByPrimaryKey
(
AuthorityDoc
.
class
,
id
);
if
(
result
==
null
)
{
throw
new
NotFoundException
(
SchemaConstants
.
INVALID_INPUT
);
}
return
result
.
getAuthority
();
}
@Override
public
Authority
create
(
Authority
authority
)
throws
ApplicationException
,
BadRequestException
{
public
Authority
create
(
Authority
authority
)
throws
ApplicationException
,
BadRequestException
{
DynamoDBQueryHelperV2
queryHelper
=
getAuthorityTableQueryHelper
();
String
id
=
headers
.
getPartitionId
()
+
":"
+
authority
.
getAuthorityId
();
AuthorityDoc
doc
=
new
AuthorityDoc
();
...
...
provider/schema-aws/src/main/java/org/opengroup/osdu/schema/provider/aws/impl/schemainfostore/AwsEntityTypeStore.java
View file @
9e1788d1
...
...
@@ -13,7 +13,8 @@
// limitations under the License.
package
org.opengroup.osdu.schema.provider.aws.impl.schemainfostore
;
import
org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelper
;
import
org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelperFactory
;
import
org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelperV2
;
import
org.opengroup.osdu.core.common.logging.JaxRsDpsLog
;
import
org.opengroup.osdu.core.common.model.http.DpsHeaders
;
import
org.opengroup.osdu.schema.constants.SchemaConstants
;
...
...
@@ -21,12 +22,11 @@ import org.opengroup.osdu.schema.exceptions.ApplicationException;
import
org.opengroup.osdu.schema.exceptions.BadRequestException
;
import
org.opengroup.osdu.schema.exceptions.NotFoundException
;
import
org.opengroup.osdu.schema.model.EntityType
;
import
org.opengroup.osdu.schema.provider.aws.config.AwsServiceConfig
;
import
org.opengroup.osdu.schema.provider.aws.models.EntityTypeDoc
;
import
org.opengroup.osdu.schema.provider.interfaces.schemainfostore.IEntityTypeStore
;
import
org.springframework.beans.factory.annotation.Value
;
import
org.springframework.stereotype.Repository
;
import
javax.annotation.PostConstruct
;
import
javax.inject.Inject
;
import
java.text.MessageFormat
;
...
...
@@ -40,19 +40,20 @@ public class AwsEntityTypeStore implements IEntityTypeStore {
private
JaxRsDpsLog
log
;
@Inject
private
AwsServiceConfig
serviceConfig
;
private
DynamoDBQueryHelperFactory
dynamoDBQueryHelperFactory
;
private
DynamoDBQueryHelper
queryHelper
;
@Value
(
"${aws.dynamodb.entityTypeTable.ssm.relativePath}"
)
String
entityTypeTableParameterRelativePath
;
@PostConstruct
public
void
init
()
{
queryHelper
=
new
DynamoDBQueryHelper
(
serviceConfig
.
getDynamoDbEndpoint
(),
serviceConfig
.
getAmazonRegion
(),
serviceConfig
.
getDynamoDbTablePrefix
());
private
DynamoDBQueryHelperV2
getEntityTypeTableQueryHelper
()
{
return
dynamoDBQueryHelperFactory
.
getQueryHelperForPartition
(
headers
,
entityTypeTableParameterRelativePath
);
}
@Override
public
EntityType
get
(
String
entityTypeId
)
throws
NotFoundException
,
ApplicationException
{
DynamoDBQueryHelperV2
queryHelper
=
getEntityTypeTableQueryHelper
();
String
id
=
headers
.
getPartitionId
()
+
":"
+
entityTypeId
;
EntityTypeDoc
result
=
queryHelper
.
loadByPrimaryKey
(
EntityTypeDoc
.
class
,
id
);
if
(
result
==
null
)
{
...
...
@@ -63,6 +64,9 @@ public class AwsEntityTypeStore implements IEntityTypeStore {
@Override
public
EntityType
create
(
EntityType
entityType
)
throws
BadRequestException
,
ApplicationException
{
DynamoDBQueryHelperV2
queryHelper
=
getEntityTypeTableQueryHelper
();
String
id
=
headers
.
getPartitionId
()
+
":"
+
entityType
.
getEntityTypeId
();
EntityTypeDoc
doc
=
new
EntityTypeDoc
();
...
...
provider/schema-aws/src/main/java/org/opengroup/osdu/schema/provider/aws/impl/schemainfostore/AwsSchemaInfoStore.java
View file @
9e1788d1
...
...
@@ -25,11 +25,11 @@ import java.util.Set;
import
java.util.TreeMap
;
import
java.util.stream.Collectors
;
import
javax.annotation.PostConstruct
;
import
javax.inject.Inject
;
import
org.joda.time.DateTime
;
import
org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelper
;
import
org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelperFactory
;
import
org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelperV2
;
import
org.opengroup.osdu.core.common.logging.JaxRsDpsLog
;
import
org.opengroup.osdu.core.common.model.http.DpsHeaders
;
import
org.opengroup.osdu.core.common.model.tenant.TenantInfo
;
...
...
@@ -42,7 +42,6 @@ import org.opengroup.osdu.schema.model.QueryParams;
import
org.opengroup.osdu.schema.model.SchemaIdentity
;
import
org.opengroup.osdu.schema.model.SchemaInfo
;
import
org.opengroup.osdu.schema.model.SchemaRequest
;
import
org.opengroup.osdu.schema.provider.aws.config.AwsServiceConfig
;
import
org.opengroup.osdu.schema.provider.aws.models.SchemaInfoDoc
;
import
org.opengroup.osdu.schema.provider.interfaces.schemainfostore.ISchemaInfoStore
;
import
org.opengroup.osdu.schema.provider.interfaces.schemastore.ISchemaStore
;
...
...
@@ -64,28 +63,34 @@ public class AwsSchemaInfoStore implements ISchemaInfoStore {
private
ITenantFactory
tenantFactory
;
@Inject
private
JaxRsDpsLog
log
;
private
JaxRsDpsLog
log
;
@Inject
private
AwsServiceConfig
serviceConfig
;
private
ISchemaStore
schemaStore
;
@Inject
private
ISchemaStore
schemaS
tor
e
;
private
DynamoDBQueryHelperFactory
dynamoDBQueryHelperFac
tor
y
;
private
DynamoDBQueryHelper
queryHelper
;
@Value
(
"${aws.dynamodb.schemaInfoTable.ssm.relativePath}"
)
String
schemaInfoTableParameterRelativePath
;
@Value
(
"${shared.tenant.name:common}"
)
private
String
sharedTenant
;
@PostConstruct
public
void
init
()
{
queryHelper
=
new
DynamoDBQueryHelper
(
serviceConfig
.
getDynamoDbEndpoint
(),
serviceConfig
.
getAmazonRegion
(),
serviceConfig
.
getDynamoDbTablePrefix
());
private
DynamoDBQueryHelperV2
getSchemaInfoTableQueryHelper
()
{
return
dynamoDBQueryHelperFactory
.
getQueryHelperForPartition
(
headers
,
schemaInfoTableParameterRelativePath
);
}
private
DynamoDBQueryHelperV2
getSchemaInfoTableQueryHelper
(
String
dataPartitionId
)
{
return
dynamoDBQueryHelperFactory
.
getQueryHelperForPartition
(
dataPartitionId
,
schemaInfoTableParameterRelativePath
);
}
@Override
public
SchemaInfo
getSchemaInfo
(
String
schemaId
)
throws
ApplicationException
,
NotFoundException
{
DynamoDBQueryHelperV2
queryHelper
=
getSchemaInfoTableQueryHelper
();
String
id
=
headers
.
getPartitionId
()
+
":"
+
schemaId
;
SchemaInfoDoc
result
=
queryHelper
.
loadByPrimaryKey
(
SchemaInfoDoc
.
class
,
id
);
if
(
result
==
null
)
{
...
...
@@ -101,6 +106,8 @@ public class AwsSchemaInfoStore implements ISchemaInfoStore {
// createdBy and createdOn. This causes the need to query the entity twice which is inefficient.
// This should be fixed
DynamoDBQueryHelperV2
queryHelper
=
getSchemaInfoTableQueryHelper
();
String
partitionId
=
headers
.
getPartitionId
();
String
userEmail
=
headers
.
getUserEmail
();
String
id
=
partitionId
+
":"
+
schema
.
getSchemaInfo
().
getSchemaIdentity
().
getId
();
...
...
@@ -128,6 +135,9 @@ public class AwsSchemaInfoStore implements ISchemaInfoStore {
@Override
public
SchemaInfo
createSchemaInfo
(
SchemaRequest
schema
)
throws
ApplicationException
,
BadRequestException
{
DynamoDBQueryHelperV2
queryHelper
=
getSchemaInfoTableQueryHelper
();
String
partitionId
=
headers
.
getPartitionId
();
String
userEmail
=
headers
.
getUserEmail
();
String
id
=
partitionId
+
":"
+
schema
.
getSchemaInfo
().
getSchemaIdentity
().
getId
();
...
...
@@ -158,6 +168,9 @@ public class AwsSchemaInfoStore implements ISchemaInfoStore {
@Override
public
String
getLatestMinorVerSchema
(
SchemaInfo
schemaInfo
)
throws
ApplicationException
{
DynamoDBQueryHelperV2
queryHelper
=
getSchemaInfoTableQueryHelper
();
String
dataPartitionId
=
headers
.
getPartitionId
();
SchemaInfoDoc
fullSchemaInfoDoc
=
SchemaInfoDoc
.
mapFrom
(
schemaInfo
,
headers
.
getPartitionId
());
...
...
@@ -193,6 +206,9 @@ public class AwsSchemaInfoStore implements ISchemaInfoStore {
// TODO: how should the system handle empty query params i.e. &scope=
// is it equal to empty string or should the qualifier be removed?
DynamoDBQueryHelperV2
queryHelper
=
getSchemaInfoTableQueryHelper
(
tenantId
);
List
<
String
>
filters
=
new
ArrayList
<>();
Map
<
String
,
AttributeValue
>
valueMap
=
new
HashMap
<>();
...
...
@@ -254,7 +270,8 @@ public class AwsSchemaInfoStore implements ISchemaInfoStore {
@Override
public
boolean
isUnique
(
String
schemaId
,
String
tenantId
)
throws
ApplicationException
{
public
boolean
isUnique
(
String
schemaId
,
String
tenantId
)
throws
ApplicationException
{
Set
<
String
>
tenantList
=
new
HashSet
<>();
tenantList
.
add
(
sharedTenant
);
tenantList
.
add
(
tenantId
);
...
...
@@ -267,6 +284,8 @@ public class AwsSchemaInfoStore implements ISchemaInfoStore {
}
for
(
String
tenant
:
tenantList
)
{
DynamoDBQueryHelperV2
queryHelper
=
getSchemaInfoTableQueryHelper
(
tenant
);
String
id
=
tenant
+
":"
+
schemaId
;
SchemaInfoDoc
schemaInfoDoc
=
new
SchemaInfoDoc
();
schemaInfoDoc
.
setId
(
id
);
...
...
@@ -280,6 +299,8 @@ public class AwsSchemaInfoStore implements ISchemaInfoStore {
@Override
public
boolean
cleanSchema
(
String
schemaId
)
throws
ApplicationException
{
DynamoDBQueryHelperV2
queryHelper
=
getSchemaInfoTableQueryHelper
();
String
id
=
headers
.
getPartitionId
()
+
":"
+
schemaId
;
SchemaInfoDoc
schemaInfoDoc
=
new
SchemaInfoDoc
();
schemaInfoDoc
.
setId
(
id
);
...
...
@@ -294,6 +315,9 @@ public class AwsSchemaInfoStore implements ISchemaInfoStore {
private
void
validateSupersededById
(
SchemaIdentity
superseding_schema
,
String
tenantId
)
throws
ApplicationException
,
BadRequestException
{
DynamoDBQueryHelperV2
queryHelper
=
getSchemaInfoTableQueryHelper
(
tenantId
);
if
(
superseding_schema
!=
null
)
{
String
id
=
tenantId
+
":"
+
superseding_schema
.
getId
();
SchemaInfoDoc
schemaInfoDoc
=
new
SchemaInfoDoc
();
...
...
provider/schema-aws/src/main/java/org/opengroup/osdu/schema/provider/aws/impl/schemainfostore/AwsSourceStore.java
View file @
9e1788d1
...
...
@@ -13,7 +13,8 @@
// limitations under the License.
package
org.opengroup.osdu.schema.provider.aws.impl.schemainfostore
;
import
org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelper
;
import
org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelperFactory
;
import
org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelperV2
;
import
org.opengroup.osdu.core.common.logging.JaxRsDpsLog
;
import
org.opengroup.osdu.core.common.model.http.DpsHeaders
;
import
org.opengroup.osdu.schema.constants.SchemaConstants
;
...
...
@@ -21,12 +22,11 @@ import org.opengroup.osdu.schema.exceptions.ApplicationException;
import
org.opengroup.osdu.schema.exceptions.BadRequestException
;
import
org.opengroup.osdu.schema.exceptions.NotFoundException
;
import
org.opengroup.osdu.schema.model.Source
;
import
org.opengroup.osdu.schema.provider.aws.config.AwsServiceConfig
;
import
org.opengroup.osdu.schema.provider.aws.models.SourceDoc
;
import
org.opengroup.osdu.schema.provider.interfaces.schemainfostore.ISourceStore
;
import
org.springframework.beans.factory.annotation.Value
;
import
org.springframework.stereotype.Repository
;
import
javax.annotation.PostConstruct
;
import
javax.inject.Inject
;
import
java.text.MessageFormat
;
...
...
@@ -42,19 +42,20 @@ public class AwsSourceStore implements ISourceStore {
private
JaxRsDpsLog
log
;
@Inject
private
AwsServiceConfig
serviceConfig
;
private
DynamoDBQueryHelperFactory
dynamoDBQueryHelperFactory
;
private
DynamoDBQueryHelper
queryHelper
;
@Value
(
"${aws.dynamodb.sourceTable.ssm.relativePath}"
)
String
sourceTableParameterRelativePath
;
@PostConstruct
public
void
init
()
{
queryHelper
=
new
DynamoDBQueryHelper
(
serviceConfig
.
getDynamoDbEndpoint
(),
serviceConfig
.
getAmazonRegion
(),
serviceConfig
.
getDynamoDbTablePrefix
());
private
DynamoDBQueryHelperV2
getSourceTableQueryHelper
()
{
return
dynamoDBQueryHelperFactory
.
getQueryHelperForPartition
(
headers
,
sourceTableParameterRelativePath
);
}
@Override
public
Source
get
(
String
sourceId
)
throws
NotFoundException
,
ApplicationException
{
DynamoDBQueryHelperV2
queryHelper
=
getSourceTableQueryHelper
();
String
id
=
headers
.
getPartitionId
()
+
":"
+
sourceId
;
SourceDoc
result
=
queryHelper
.
loadByPrimaryKey
(
SourceDoc
.
class
,
id
);
if
(
result
==
null
)
{
...
...
@@ -76,6 +77,9 @@ public class AwsSourceStore implements ISourceStore {
}
catch
(
NotFoundException
e
)
{
}
try
{
DynamoDBQueryHelperV2
queryHelper
=
getSourceTableQueryHelper
();
String
id
=
headers
.
getPartitionId
()
+
":"
+
source
.
getSourceId
();
SourceDoc
sourceDoc
=
new
SourceDoc
(
id
,
headers
.
getPartitionId
(),
source
);
queryHelper
.
save
(
sourceDoc
);
...
...
provider/schema-aws/src/main/java/org/opengroup/osdu/schema/provider/aws/impl/schemastore/AwsSchemaStore.java
View file @
9e1788d1
...
...
@@ -17,14 +17,17 @@ import com.amazonaws.services.s3.AmazonS3;
import
com.amazonaws.services.s3.model.AmazonS3Exception
;
import
com.amazonaws.services.s3.model.ObjectMetadata
;
import
com.amazonaws.services.s3.model.PutObjectRequest
;
import
org.opengroup.osdu.core.aws.s3.IS3ClientFactory
;
import
org.opengroup.osdu.core.aws.s3.S3ClientWithBucket
;
import
org.opengroup.osdu.core.aws.s3.S3Config
;
import
org.opengroup.osdu.core.common.logging.JaxRsDpsLog
;
import
org.opengroup.osdu.core.common.model.http.DpsHeaders
;
import
org.opengroup.osdu.schema.constants.SchemaConstants
;
import
org.opengroup.osdu.schema.exceptions.ApplicationException
;
import
org.opengroup.osdu.schema.exceptions.NotFoundException
;
import
org.opengroup.osdu.schema.provider.aws.config.AwsServiceConfig
;
import
org.opengroup.osdu.schema.provider.interfaces.schemastore.ISchemaStore
;
import
org.springframework.beans.factory.annotation.Value
;
import
org.springframework.stereotype.Repository
;
import
javax.annotation.PostConstruct
;
...
...
@@ -40,23 +43,32 @@ public class AwsSchemaStore implements ISchemaStore {
private
DpsHeaders
headers
;
@Inject
private
AwsServiceConfig
serviceConfig
;
private
JaxRsDpsLog
logger
;
@Inject
private
JaxRsDpsLog
logger
;
private
IS3ClientFactory
s3ClientFactory
;
private
AmazonS3
s3
;
@Value
(
"${aws.s3.schemaBucket.ssm.relativePath}"
)
private
String
s3SchemaBucketParameterRelativePath
;
private
S3ClientWithBucket
getS3ClientWithBucket
()
{
String
dataPartitionId
=
headers
.
getPartitionIdWithFallbackToAccountId
();
return
getS3ClientWithBucket
(
dataPartitionId
);
}
private
S3ClientWithBucket
getS3ClientWithBucket
(
String
dataPartitionId
)
{
return
s3ClientFactory
.
getS3ClientForPartition
(
dataPartitionId
,
s3SchemaBucketParameterRelativePath
);
}
@PostConstruct
public
void
init
()
{
s3
=
new
S3Config
(
serviceConfig
.
getS3Endpoint
(),
serviceConfig
.
getAmazonRegion
()).
amazonS3
();
}
@Override
public
String
createSchema
(
String
filePath
,
String
content
)
throws
ApplicationException
{
// should this be headers.getPartitionIdWithFallbackToAccountId ??
String
path
=
resolvePath
(
headers
.
getPartitionId
(),
filePath
);
S3ClientWithBucket
s3ClientWithBucket
=
getS3ClientWithBucket
();
AmazonS3
s3
=
s3ClientWithBucket
.
getS3Client
();
String
path
=
resolvePath
(
headers
.
getPartitionIdWithFallbackToAccountId
(),
filePath
);
byte
[]
bytes
=
content
.
getBytes
(
StandardCharsets
.
UTF_8
);
int
bytesSize
=
bytes
.
length
;
...
...
@@ -67,7 +79,7 @@ public class AwsSchemaStore implements ISchemaStore {
String
bucket
;
try
{
bucket
=
s
erviceConfig
.
s3DataBucket
;
bucket
=
s
3ClientWithBucket
.
getBucketName
()
;
PutObjectRequest
req
=
new
PutObjectRequest
(
bucket
,
path
,
newStream
,
metadata
);
s3
.
putObject
(
req
);
...
...
@@ -83,11 +95,14 @@ public class AwsSchemaStore implements ISchemaStore {
// first this method is called with the callers partitionid, then if not found, its called with the
// common project id which is "common". Not sure why this isn't passed into the createSchema call.
S3ClientWithBucket
s3ClientWithBucket
=
getS3ClientWithBucket
(
dataPartitionId
);
AmazonS3
s3
=
s3ClientWithBucket
.
getS3Client
();
String
content
;
String
path
=
resolvePath
(
dataPartitionId
,
filePath
);
try
{
content
=
s3
.
getObjectAsString
(
s
erviceConfig
.
s3DataBucket
,
path
);
content
=
s3
.
getObjectAsString
(
s
3ClientWithBucket
.
getBucketName
()
,
path
);
}
catch
(
AmazonS3Exception
ex
)
{
if
(
ex
.
getErrorCode
().
equals
(
"NoSuchKey"
))
{
// or could be ex.getStatusCode == 404 (depends)
logger
.
error
(
"AwsSchemaStore"
,
String
.
format
(
SchemaConstants
.
SCHEMA_NOT_PRESENT
,
ex
.
getErrorMessage
()));
...
...
@@ -112,8 +127,12 @@ public class AwsSchemaStore implements ISchemaStore {
@Override
public
boolean
cleanSchemaProject
(
String
schemaId
)
throws
ApplicationException
{
logger
.
info
(
"Delete schema: "
+
schemaId
);
S3ClientWithBucket
s3ClientWithBucket
=
getS3ClientWithBucket
();
AmazonS3
s3
=
s3ClientWithBucket
.
getS3Client
();
try
{
s3
.
deleteObject
(
s
erviceConfig
.
s3DataBucket
,
resolvePath
(
headers
.
getPartitionId
(),
schemaId
));
s3
.
deleteObject
(
s
3ClientWithBucket
.
getBucketName
()
,
resolvePath
(
headers
.
getPartitionId
WithFallbackToAccountId
(),
schemaId
));
logger
.
info
(
"Schema deleted: "
+
schemaId
);
return
true
;
}
catch
(
Exception
e
)
{
...
...
provider/schema-aws/src/main/resources/application-unittest.properties
View file @
9e1788d1
...
...
@@ -17,7 +17,7 @@ logging.level.org.springframework.web=${LOG_LEVEL:INFO}
server.port
=
${APPLICATION_PORT:8080}
aws.resource.prefix
=
${ENVIRONMENT}
aws.ssm
=
${SSM_ENABLED:True}
AUTHORIZE_API
=
${ENTITLEMENTS_BASE_URL}/api/entitlements/v
1
AUTHORIZE_API
=
${ENTITLEMENTS_BASE_URL}/api/entitlements/v
2
AUTHORIZE_API_KEY
=
LOG_PREFIX
=
schema
...
...
provider/schema-aws/src/main/resources/application.properties
View file @
9e1788d1
...
...
@@ -16,15 +16,21 @@ server.servlet.context-path=/api/schema-service/v1
logging.level.org.springframework.web
=
${LOG_LEVEL:INFO}
server.port
=
${APPLICATION_PORT:8080}
aws.resource.prefix
=
${ENVIRONMENT}
aws.ssm
=
${SSM_ENABLED:True}
AUTHORIZE_API
=
${ENTITLEMENTS_BASE_URL}/api/entitlements/v
1
AUTHORIZE_API
=
${ENTITLEMENTS_BASE_URL}/api/entitlements/v
2
AUTHORIZE_API_KEY
=
PARTITION_API
=
${ENTITLEMENTS_BASE_URL}/api/partition/v1
LOG_PREFIX
=
schema
# s3 data bucket
aws.s3.endpoint
=
s3.${AWS_REGION}.amazonaws.com
aws.s3.bucket
=
${S3_BUCKET:""}
aws.ssm
=
${SSM_ENABLED:True}
aws.parameter.prefix
=
/osdu/${ENVIRONMENT}
aws.environment
=
${ENVIRONMENT}
## AWS s3 Configuraiton
#Tenant Specific S3 Bucket Configuration
aws.s3.schemaBucket.ssm.relativePath
=
${SCHEMA_BUCKET_SSM_RELATIVE_PATH:schema/s3DataBucket}