Commit a2ba56fd authored by Rucha Deshpande's avatar Rucha Deshpande
Browse files

Merge branch 'aws-multitenant-v2' into 'master'

multitenant-v2

See merge request !84
parents bb86f110 a4d01a7d
Pipeline #42479 failed with stages
in 39 minutes and 44 seconds
......@@ -45,7 +45,6 @@ The following software have components provided under the terms of this license:
- AWS Java SDK for Amazon SNS (from https://aws.amazon.com/sdkforjava)
- AWS Java SDK for Amazon SQS (from https://aws.amazon.com/sdkforjava)
- AWS Java SDK for the AWS Simple Systems Management (SSM) Service (from https://aws.amazon.com/sdkforjava)
- AWS SDK for Java - BOM (from https://aws.amazon.com/sdkforjava)
- AWS SDK for Java - Core (from https://aws.amazon.com/sdkforjava)
- Adapter: RxJava (from )
- Apache Commons BeanUtils (from http://commons.apache.org/proper/commons-beanutils/)
......
......@@ -30,7 +30,7 @@
<relativePath>../../pom.xml</relativePath>
</parent>
<properties>
<aws.version>1.11.637</aws.version>
<aws.version>1.11.1018</aws.version>
</properties>
<dependencies>
......@@ -64,7 +64,7 @@
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
<artifactId>os-core-lib-aws</artifactId>
<version>0.3.17</version>
<version>0.9.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
......@@ -120,12 +120,7 @@
<version>2.0.2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-sns</artifactId>
<version>1.11.651</version>
<scope>compile</scope>
</dependency>
</dependencies>
......
......@@ -17,18 +17,18 @@ package org.opengroup.osdu.register.provider.aws.action;
import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBDeleteExpression;
import com.amazonaws.services.dynamodbv2.model.AttributeValue;
import com.amazonaws.services.dynamodbv2.model.ConditionalCheckFailedException;
import org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelper;
import org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelperFactory;
import org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelperV2;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.register.action.model.Action;
import org.opengroup.osdu.register.provider.aws.config.AwsServiceConfig;
import org.opengroup.osdu.register.provider.aws.util.DocUtil;
import org.opengroup.osdu.register.provider.interfaces.action.IActionRepo;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Repository;
import javax.annotation.PostConstruct;
import javax.inject.Inject;
import java.util.ArrayList;
import java.util.HashMap;
......@@ -41,71 +41,54 @@ public class AwsActionRepo implements IActionRepo {
@Autowired
private DpsHeaders dpsHeaders;
private DpsHeaders headers;
@Autowired
private JaxRsDpsLog logger;
@Inject
private AwsServiceConfig serviceConfig;
private DynamoDBQueryHelperFactory dynamoDBQueryHelperFactory;
private DynamoDBQueryHelper queryHelper;
@Value("${aws.dynamodb.actionTable.ssm.relativePath}")
String actionTableParameterRelativePath;
@Autowired
DocUtil docUtil;
@PostConstruct
public void init() {
queryHelper = new DynamoDBQueryHelper(serviceConfig.getDynamoDbEndpoint(),
serviceConfig.getAmazonRegion(),
serviceConfig.getDynamoDbTablePrefix());
private DynamoDBQueryHelperV2 getActionQueryHelper() {
return dynamoDBQueryHelperFactory.getQueryHelperForPartition(headers, actionTableParameterRelativePath);
}
@Override
public List<Action> getAllActions() {
DynamoDBQueryHelperV2 queryHelper = getActionQueryHelper();
List<String> filters = new ArrayList<>();
Map<String, AttributeValue> valueMap = new HashMap<>();
filters.add("dataPartitionId = :dataPartitionId");
valueMap.put(":dataPartitionId", new AttributeValue().withS(dpsHeaders.getPartitionId()));
valueMap.put(":dataPartitionId", new AttributeValue().withS(headers.getPartitionId()));
String filterExpression = String.join(" and ", filters);
logger.info(String.format("Action query filter expression: %s", filterExpression));
List<ActionDoc> results = queryHelper.scanTable(ActionDoc.class, filterExpression, valueMap);
List<Action> actionsList = results.stream().map(ActionDoc::mapto).collect(Collectors.toList());
//Alternative implementation
/* List<Action> actionsList = new ArrayList<Action>();
for (ActionDoc actionDoc : results){
actionsList.add(ActionDoc.mapto(actionDoc));
}*/
return actionsList;
return results.stream().map(ActionDoc::mapto).collect(Collectors.toList());
}
@Override
public Action createAction(Action action) {
if(action.getId() == null){
DynamoDBQueryHelperV2 queryHelper = getActionQueryHelper();
if (action.getId() == null) {
logger.error("Action id cannot be null");
throw new AppException(400, "Bad Request", "Action id cannot be null");
}
ActionDoc doc = ActionDoc.mapFrom(action, dpsHeaders.getPartitionId());
ActionDoc doc = ActionDoc.mapFrom(action, headers.getPartitionId());
try {
queryHelper.saveWithHashCondition(doc,docUtil.getHashKey());
}
catch(ConditionalCheckFailedException e){
queryHelper.saveWithHashCondition(doc, DocUtil.getHashKey());
} catch (ConditionalCheckFailedException e) {
logger.error(String.format("An action already exists with the id: %s", action.getId()));
throw new AppException(409, "Conflict", String.format("An action already exists with the id: %s", action.getId()));
}
catch(AppException e)
{
} catch (AppException e) {
throw new AppException(e.getError().getCode(), e.getError().getReason(), e.getMessage());
}
......@@ -115,27 +98,27 @@ public class AwsActionRepo implements IActionRepo {
@Override
public boolean delete(String id) {
try{
ActionDoc objectTodelete = new ActionDoc();
objectTodelete.setId(id);
DynamoDBDeleteExpression deleteExpression = new DynamoDBDeleteExpression()
.withConditionExpression("attribute_exists(id)");
queryHelper.deleteByObjectWithCondition(objectTodelete,deleteExpression);
DynamoDBQueryHelperV2 queryHelper = getActionQueryHelper();
try {
ActionDoc objectToDelete = new ActionDoc();
objectToDelete.setId(id);
DynamoDBDeleteExpression deleteExpression = new DynamoDBDeleteExpression()
.withConditionExpression("attribute_exists(id)");
queryHelper.deleteByObjectWithCondition(objectToDelete, deleteExpression);
} catch (ConditionalCheckFailedException e) {
logger.error(String.format("Failed to delete: %s Object does not exist", id));
throw new AppException(404, "ObjectNotFound", String.format("The Action you are trying to delete: %s does not exist", id));
} catch (AppException e) {
throw new AppException(e.getError().getCode(), e.getError().getReason(), e.getMessage());
}
return true;
}
catch(ConditionalCheckFailedException e){
logger.error(String.format("Failed to delete: %s Object does not exist",id));
throw new AppException(404, "ObjectNotFound", String.format("The Action you are trying to delete: %s does not exist", id));
}
catch(AppException e)
{
throw new AppException(e.getError().getCode(), e.getError().getReason(), e.getMessage());
}
return true;
}
@Override
public Action get(String id){
public Action get(String id) {
DynamoDBQueryHelperV2 queryHelper = getActionQueryHelper();
ActionDoc doc = queryHelper.loadByPrimaryKey(ActionDoc.class, id);
if (doc == null) {
logger.error(String.format("Action with id %s does not exist.", id));
......@@ -145,6 +128,4 @@ public class AwsActionRepo implements IActionRepo {
}
}
}
......@@ -46,18 +46,6 @@ public class AwsServiceConfig {
@Setter(AccessLevel.PROTECTED)
public String amazonRegion;
@Value("${aws.dynamodb.table.prefix}")
@Getter()
@Setter(AccessLevel.PROTECTED)
public String dynamoDbTablePrefix;
@Value("${aws.dynamodb.endpoint}")
@Getter()
@Setter(AccessLevel.PROTECTED)
public String dynamoDbEndpoint;
@Value("${aws.ssm}")
@Getter()
@Setter(AccessLevel.PROTECTED)
......
......@@ -15,23 +15,20 @@ package org.opengroup.osdu.register.provider.aws.ddms;
import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBDeleteExpression;
import com.amazonaws.services.dynamodbv2.datamodeling.PaginatedQueryList;
import com.amazonaws.services.dynamodbv2.model.ConditionalCheckFailedException;
import org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelper;
import org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelperFactory;
import org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelperV2;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.register.ddms.model.Ddms;
import org.opengroup.osdu.register.provider.aws.action.ActionDoc;
import org.opengroup.osdu.register.provider.aws.config.AwsServiceConfig;
import org.opengroup.osdu.register.provider.aws.util.DocUtil;
import org.opengroup.osdu.register.provider.interfaces.ddms.IDdmsRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Repository;
import javax.annotation.PostConstruct;
import javax.inject.Inject;
import java.util.ArrayList;
import java.util.List;
@Repository
......@@ -39,35 +36,31 @@ public class AwsDdmsRepo implements IDdmsRepository {
@Autowired
private DpsHeaders dpsHeaders;
private DpsHeaders headers;
@Autowired
private JaxRsDpsLog logger;
@Inject
private AwsServiceConfig serviceConfig;
private DynamoDBQueryHelperFactory dynamoDBQueryHelperFactory;
private DynamoDBQueryHelper queryHelper;
@Value("${aws.dynamodb.ddmsTable.ssm.relativePath}")
String ddmsTableParameterRelativePath;
@Autowired
DocUtil docUtil;
@PostConstruct
public void init() {
queryHelper = new DynamoDBQueryHelper(serviceConfig.getDynamoDbEndpoint(),
serviceConfig.getAmazonRegion(),
serviceConfig.getDynamoDbTablePrefix());
private DynamoDBQueryHelperV2 getDDMSQueryHelper() {
return dynamoDBQueryHelperFactory.getQueryHelperForPartition(headers, ddmsTableParameterRelativePath);
}
@Override
public Ddms create(Ddms ddms) {
DdmsDoc doc = DdmsDoc.mapFrom(ddms, dpsHeaders.getPartitionId());
DdmsDoc doc = DdmsDoc.mapFrom(ddms, headers.getPartitionId());
DynamoDBQueryHelperV2 queryHelper = getDDMSQueryHelper();
try {
queryHelper.saveWithHashCondition(doc,docUtil.getHashKey());
queryHelper.saveWithHashCondition(doc,DocUtil.getHashKey());
}
catch(ConditionalCheckFailedException e){
logger.error(String.format("An DDMS already exists with the id: %s", ddms.getId()));
......@@ -77,13 +70,12 @@ public class AwsDdmsRepo implements IDdmsRepository {
{
throw new AppException(e.getError().getCode(), e.getError().getReason(), e.getMessage());
}
return ddms;
}
@Override
public Ddms get(String id) {
DynamoDBQueryHelperV2 queryHelper = getDDMSQueryHelper();
DdmsDoc doc = queryHelper.loadByPrimaryKey(DdmsDoc.class, id);
if (doc == null) {
logger.error(String.format("Ddms with id %s does not exist.", id));
......@@ -91,32 +83,27 @@ public class AwsDdmsRepo implements IDdmsRepository {
} else {
return DdmsDoc.mapto(doc);
}
}
@Override
public List<Ddms> query(String entityType) {
String dataPartitionId = dpsHeaders.getPartitionId();
DynamoDBQueryHelperV2 queryHelper = getDDMSQueryHelper();
String dataPartitionId = headers.getPartitionId();
DdmsDoc gsiQuery = new DdmsDoc();
String key = String.format("%s:%s",dataPartitionId,entityType);
gsiQuery.setPartitionIdEntityType(key);
List<Ddms> ddmsList = docUtil.getDdmsList(queryHelper,gsiQuery);
return ddmsList;
return docUtil.getDdmsList(queryHelper,gsiQuery);
}
@Override
public boolean delete(String id) {
DynamoDBQueryHelperV2 queryHelper = getDDMSQueryHelper();
try{
DdmsDoc objectTodelete = new DdmsDoc();
objectTodelete.setId(id);
DdmsDoc objectToDelete = new DdmsDoc();
objectToDelete.setId(id);
DynamoDBDeleteExpression deleteExpression = new DynamoDBDeleteExpression()
.withConditionExpression("attribute_exists(id)");
queryHelper.deleteByObjectWithCondition(objectTodelete,deleteExpression);
queryHelper.deleteByObjectWithCondition(objectToDelete,deleteExpression);
}
catch(ConditionalCheckFailedException e){
logger.error(String.format("Failed to delete: %s Object does not exist",id));
......
......@@ -13,18 +13,11 @@
// limitations under the License.
package org.opengroup.osdu.register.provider.aws.subscriber;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.PostConstruct;
import javax.inject.Inject;
import org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelper;
import org.opengroup.osdu.core.aws.ssm.ParameterStorePropertySource;
import org.opengroup.osdu.core.aws.ssm.SSMConfig;
import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBDeleteExpression;
import com.amazonaws.services.dynamodbv2.model.AttributeValue;
import com.amazonaws.services.dynamodbv2.model.ConditionalCheckFailedException;
import org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelperFactory;
import org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelperV2;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
......@@ -34,70 +27,68 @@ import org.opengroup.osdu.register.provider.interfaces.subscriber.ISubscriptionR
import org.opengroup.osdu.register.subscriber.model.Secret;
import org.opengroup.osdu.register.subscriber.model.Subscription;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Repository;
import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBDeleteExpression;
import com.amazonaws.services.dynamodbv2.model.AttributeValue;
import com.amazonaws.services.dynamodbv2.model.ConditionalCheckFailedException;
import javax.annotation.PostConstruct;
import javax.inject.Inject;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Repository
public class AwsSubscriptionRepo implements ISubscriptionRepository {
@Autowired
private DpsHeaders dpsHeaders;
private DpsHeaders headers;
@Autowired
private JaxRsDpsLog logger;
private DynamoDBQueryHelper queryHelper;
@Inject
private AwsServiceConfig serviceConfig;
private DynamoDBQueryHelperFactory dynamoDBQueryHelperFactory;
@Value("${aws.dynamodb.subscriptionTable.ssm.relativePath}")
String subscriptionTableParameterRelativePath;
private String amazonSNSTopicArn_records_changed;
private SSMConfig ssmConfig;
private ParameterStorePropertySource ssm;
@Autowired
private SubscriptionHelper snsHelper;
@Autowired
private KmsHelper kmsHelper;
@Inject
private AwsServiceConfig serviceConfig;
@PostConstruct
public void init() {
amazonSNSTopicArn_records_changed = serviceConfig.getSnsTopicArn();
}
@Autowired
DocUtil docUtil;
@PostConstruct
public void init() {
// TODO: serviceConfig.environment isn't correct and needs to be table prefix. Maybe the "-" will fix it
queryHelper = new DynamoDBQueryHelper(serviceConfig.getDynamoDbEndpoint(),
serviceConfig.getAmazonRegion(),
serviceConfig.getDynamoDbTablePrefix());
amazonSNSTopicArn_records_changed=serviceConfig.getSnsTopicArn();
private DynamoDBQueryHelperV2 getSubscriptionQueryHelper() {
return dynamoDBQueryHelperFactory.getQueryHelperForPartition(headers, subscriptionTableParameterRelativePath);
}
@Override
public Subscription create(Subscription s) throws Exception {
public Subscription create(Subscription s) throws AppException {
DynamoDBQueryHelperV2 queryHelper = getSubscriptionQueryHelper();
String pushEndPoint = s.getPushEndpoint();
String subscriptionArn = snsHelper.createPushSubscription(amazonSNSTopicArn_records_changed,pushEndPoint);
String subscriptionArn = snsHelper.createPushSubscription(amazonSNSTopicArn_records_changed, pushEndPoint);
ByteBuffer encryptedSecretValue = kmsHelper.encrypt(s.getSecret().toString());
SubscriptionDoc doc = SubscriptionDoc.mapFrom(s, dpsHeaders.getPartitionId(),subscriptionArn,encryptedSecretValue);
SubscriptionDoc doc = SubscriptionDoc.mapFrom(s, headers.getPartitionId(), subscriptionArn, encryptedSecretValue);
try {
queryHelper.saveWithHashCondition(doc,docUtil.getHashKey());
}
catch(ConditionalCheckFailedException e){
queryHelper.saveWithHashCondition(doc, DocUtil.getHashKey());
} catch (ConditionalCheckFailedException e) {
logger.error(String.format("A subscription already exists with the id: %s", s.getId()));
throw new AppException(409, "Conflict", String.format("A subscription already exists with the id: %s", s.getId()));
}
catch(AppException e)
{
} catch (AppException e) {
throw new AppException(e.getError().getCode(), e.getError().getReason(), e.getMessage());
}
return s;
......@@ -106,66 +97,57 @@ public class AwsSubscriptionRepo implements ISubscriptionRepository {
@Override
public Subscription get(String id) {
DynamoDBQueryHelperV2 queryHelper = getSubscriptionQueryHelper();
SubscriptionDoc doc = queryHelper.loadByPrimaryKey(SubscriptionDoc.class, id);
if (doc == null) {
logger.error(String.format("Subscription with id %s does not exist.", id));
throw new AppException(404, "Not found", String.format("Subscription with id %s does not exist.", id));
} else {
String secretValue = kmsHelper.decrypt(doc.getSecretValue());
Secret secret = docUtil.createSecret(doc.getSecretType(),secretValue);
Subscription s = SubscriptionDoc.mapTo(doc,secret);
return s;
Secret secret = docUtil.createSecret(doc.getSecretType(), secretValue);
return SubscriptionDoc.mapTo(doc, secret);
}
}
@Override
public List<Subscription> query(String notificationId) {
String key=String.format("%s:%s",dpsHeaders.getPartitionId(),notificationId);
DynamoDBQueryHelperV2 queryHelper = getSubscriptionQueryHelper();
String key = String.format("%s:%s", headers.getPartitionId(), notificationId);
SubscriptionDoc gsiQuery = new SubscriptionDoc();
gsiQuery.setPartitionIdNotificationId(key);
List<Subscription> subsList = docUtil.getSubscriptionList(queryHelper,gsiQuery);
return subsList;
return docUtil.getSubscriptionList(queryHelper, gsiQuery);
}
@Override
public boolean delete(String id) {
SubscriptionDoc doc=null;
String snsSubscriptionArn="";
SubscriptionDoc doc;
String snsSubscriptionArn;
try {
//Need this to retrieve the subscription Arn
doc = queryHelper.loadByPrimaryKey(SubscriptionDoc.class, id);
DynamoDBQueryHelperV2 queryHelper = getSubscriptionQueryHelper();
doc = queryHelper.loadByPrimaryKey(SubscriptionDoc.class, id);
if (doc == null) {
logger.error(String.format("Subscription with id %s does not exist.", id));
throw new AppException(404, "Not found", String.format("Subscription with id %s does not exist.", id));
}
else {
} else {
snsSubscriptionArn = doc.getSnssubscriptionArn();
// queryHelper.deleteByPrimaryKey(SubscriptionDoc.class, id);
SubscriptionDoc objectTodelete = new SubscriptionDoc();
objectTodelete.setId(id);
// queryHelper.deleteByPrimaryKey(SubscriptionDoc.class, id);
SubscriptionDoc objectToDelete = new SubscriptionDoc();
objectToDelete.setId(id);
DynamoDBDeleteExpression deleteExpression = new DynamoDBDeleteExpression()
.withConditionExpression("attribute_exists(id)");
queryHelper.deleteByObjectWithCondition(objectTodelete,deleteExpression);
queryHelper.deleteByObjectWithCondition(objectToDelete, deleteExpression);
}
}
catch(ConditionalCheckFailedException e){
logger.error(String.format("Failed to delete: %s Object does not exist",id));
} catch (ConditionalCheckFailedException e) {
logger.error(String.format("Failed to delete: %s Object does not exist", id));
throw new AppException(404, "ObjectNotFound", String.format("The Subscription you are trying to delete: %s does not exist", id));
}
catch(AppException e)
{
} catch (AppException e) {
throw new AppException(e.getError().getCode(), e.getError().getReason(), e.getMessage());
}
//delete the SNS subscription
//this will throw 404 if SNS Subscription not found
snsHelper.deletePushSubscription(snsSubscriptionArn);
......@@ -175,17 +157,16 @@ public class AwsSubscriptionRepo implements ISubscriptionRepository {
@Override
public boolean patch(Subscription subscription, Secret secret) {