Commit d0f0b320 authored by Bill Wang's avatar Bill Wang
Browse files

Squashed commit of the following

commit e459ce00 
Author: zhijie wang <wanzhiji@amazon.com> 
Date: Tue May 04 2021 16:26:08 GMT-0500 (Central Daylight Time) 

    Merge remote-tracking branch 'origin' into multi-tenant


commit 04db74fc 
Author: zhijie wang <wanzhiji@amazon.com> 
Date: Mon May 03 2021 11:28:05 GMT-0500 (Central Daylight Time) 

    revert minor changes


commit ebdcbd40 
Author: zhijie wang <wanzhiji@amazon.com> 
Date: Mon May 03 2021 11:27:13 GMT-0500 (Central Daylight Time) 

    add back sns


commit 60bbc092 
Author: David Diederich <d.diederich@opengroup.org> 
Date: Fri Apr 16 2021 14:46:57 GMT-0500 (Central Daylight Time) 

    Update version of default branch to 0.9.0-SNAPSHOT


commit f44e8c92 
Author: zhijie wang <wanzhiji@amazon.com> 
Date: Sat May 01 2021 14:08:19 GMT-0500 (Central Daylight Time) 

    fix typo


commit bb22bab2 
Author: zhijie wang <wanzhiji@amazon.com> 
Date: Sat May 01 2021 13:01:08 GMT-0500 (Central Daylight Time) 

    add partition api reference


commit 49f7a97c 
Author: zhijie wang <wanzhiji@amazon.com> 
Date: Sat May 01 2021 11:43:14 GMT-0500 (Central Daylight Time) 

    Add back GoogleServiceAccount


commit 816b83de 
Author: zhijie wang <wanzhiji@amazon.com> 
Date: Fri Apr 30 2021 11:43:06 GMT-0500 (Central Daylight Time) 

    remove dynmo info in service config


commit 591d9b36 
Author: zhijie wang <wanzhiji@amazon.com> 
Date: Tue Apr 27 2021 17:05:43 GMT-0500 (Central Daylight Time) 

    update unit test


commit e0f5c6e0 
Author: zhijie wang <wanzhiji@amazon.com> 
Date: Tue Apr 27 2021 16:42:26 GMT-0500 (Central Daylight Time) 

    initial commit
parent bb86f110
......@@ -64,7 +64,7 @@
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
<artifactId>os-core-lib-aws</artifactId>
<version>0.3.17</version>
<version>0.9.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
......
......@@ -17,18 +17,18 @@ package org.opengroup.osdu.register.provider.aws.action;
import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBDeleteExpression;
import com.amazonaws.services.dynamodbv2.model.AttributeValue;
import com.amazonaws.services.dynamodbv2.model.ConditionalCheckFailedException;
import org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelper;
import org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelperFactory;
import org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelperV2;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.register.action.model.Action;
import org.opengroup.osdu.register.provider.aws.config.AwsServiceConfig;
import org.opengroup.osdu.register.provider.aws.util.DocUtil;
import org.opengroup.osdu.register.provider.interfaces.action.IActionRepo;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Repository;
import javax.annotation.PostConstruct;
import javax.inject.Inject;
import java.util.ArrayList;
import java.util.HashMap;
......@@ -41,71 +41,54 @@ public class AwsActionRepo implements IActionRepo {
@Autowired
private DpsHeaders dpsHeaders;
private DpsHeaders headers;
@Autowired
private JaxRsDpsLog logger;
@Inject
private AwsServiceConfig serviceConfig;
private DynamoDBQueryHelperFactory dynamoDBQueryHelperFactory;
private DynamoDBQueryHelper queryHelper;
@Value("${aws.dynamodb.actionTable.ssm.relativePath}")
String actionTableParameterRelativePath;
@Autowired
DocUtil docUtil;
@PostConstruct
public void init() {
queryHelper = new DynamoDBQueryHelper(serviceConfig.getDynamoDbEndpoint(),
serviceConfig.getAmazonRegion(),
serviceConfig.getDynamoDbTablePrefix());
private DynamoDBQueryHelperV2 getActionQueryHelper() {
return dynamoDBQueryHelperFactory.getQueryHelperForPartition(headers, actionTableParameterRelativePath);
}
@Override
public List<Action> getAllActions() {
DynamoDBQueryHelperV2 queryHelper = getActionQueryHelper();
List<String> filters = new ArrayList<>();
Map<String, AttributeValue> valueMap = new HashMap<>();
filters.add("dataPartitionId = :dataPartitionId");
valueMap.put(":dataPartitionId", new AttributeValue().withS(dpsHeaders.getPartitionId()));
valueMap.put(":dataPartitionId", new AttributeValue().withS(headers.getPartitionId()));
String filterExpression = String.join(" and ", filters);
logger.info(String.format("Action query filter expression: %s", filterExpression));
List<ActionDoc> results = queryHelper.scanTable(ActionDoc.class, filterExpression, valueMap);
List<Action> actionsList = results.stream().map(ActionDoc::mapto).collect(Collectors.toList());
//Alternative implementation
/* List<Action> actionsList = new ArrayList<Action>();
for (ActionDoc actionDoc : results){
actionsList.add(ActionDoc.mapto(actionDoc));
}*/
return actionsList;
return results.stream().map(ActionDoc::mapto).collect(Collectors.toList());
}
@Override
public Action createAction(Action action) {
if(action.getId() == null){
DynamoDBQueryHelperV2 queryHelper = getActionQueryHelper();
if (action.getId() == null) {
logger.error("Action id cannot be null");
throw new AppException(400, "Bad Request", "Action id cannot be null");
}
ActionDoc doc = ActionDoc.mapFrom(action, dpsHeaders.getPartitionId());
ActionDoc doc = ActionDoc.mapFrom(action, headers.getPartitionId());
try {
queryHelper.saveWithHashCondition(doc,docUtil.getHashKey());
}
catch(ConditionalCheckFailedException e){
queryHelper.saveWithHashCondition(doc, DocUtil.getHashKey());
} catch (ConditionalCheckFailedException e) {
logger.error(String.format("An action already exists with the id: %s", action.getId()));
throw new AppException(409, "Conflict", String.format("An action already exists with the id: %s", action.getId()));
}
catch(AppException e)
{
} catch (AppException e) {
throw new AppException(e.getError().getCode(), e.getError().getReason(), e.getMessage());
}
......@@ -115,27 +98,27 @@ public class AwsActionRepo implements IActionRepo {
@Override
public boolean delete(String id) {
try{
ActionDoc objectTodelete = new ActionDoc();
objectTodelete.setId(id);
DynamoDBDeleteExpression deleteExpression = new DynamoDBDeleteExpression()
.withConditionExpression("attribute_exists(id)");
queryHelper.deleteByObjectWithCondition(objectTodelete,deleteExpression);
DynamoDBQueryHelperV2 queryHelper = getActionQueryHelper();
try {
ActionDoc objectToDelete = new ActionDoc();
objectToDelete.setId(id);
DynamoDBDeleteExpression deleteExpression = new DynamoDBDeleteExpression()
.withConditionExpression("attribute_exists(id)");
queryHelper.deleteByObjectWithCondition(objectToDelete, deleteExpression);
} catch (ConditionalCheckFailedException e) {
logger.error(String.format("Failed to delete: %s Object does not exist", id));
throw new AppException(404, "ObjectNotFound", String.format("The Action you are trying to delete: %s does not exist", id));
} catch (AppException e) {
throw new AppException(e.getError().getCode(), e.getError().getReason(), e.getMessage());
}
return true;
}
catch(ConditionalCheckFailedException e){
logger.error(String.format("Failed to delete: %s Object does not exist",id));
throw new AppException(404, "ObjectNotFound", String.format("The Action you are trying to delete: %s does not exist", id));
}
catch(AppException e)
{
throw new AppException(e.getError().getCode(), e.getError().getReason(), e.getMessage());
}
return true;
}
@Override
public Action get(String id){
public Action get(String id) {
DynamoDBQueryHelperV2 queryHelper = getActionQueryHelper();
ActionDoc doc = queryHelper.loadByPrimaryKey(ActionDoc.class, id);
if (doc == null) {
logger.error(String.format("Action with id %s does not exist.", id));
......@@ -145,6 +128,4 @@ public class AwsActionRepo implements IActionRepo {
}
}
}
......@@ -46,18 +46,6 @@ public class AwsServiceConfig {
@Setter(AccessLevel.PROTECTED)
public String amazonRegion;
@Value("${aws.dynamodb.table.prefix}")
@Getter()
@Setter(AccessLevel.PROTECTED)
public String dynamoDbTablePrefix;
@Value("${aws.dynamodb.endpoint}")
@Getter()
@Setter(AccessLevel.PROTECTED)
public String dynamoDbEndpoint;
@Value("${aws.ssm}")
@Getter()
@Setter(AccessLevel.PROTECTED)
......
......@@ -15,23 +15,20 @@ package org.opengroup.osdu.register.provider.aws.ddms;
import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBDeleteExpression;
import com.amazonaws.services.dynamodbv2.datamodeling.PaginatedQueryList;
import com.amazonaws.services.dynamodbv2.model.ConditionalCheckFailedException;
import org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelper;
import org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelperFactory;
import org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelperV2;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.register.ddms.model.Ddms;
import org.opengroup.osdu.register.provider.aws.action.ActionDoc;
import org.opengroup.osdu.register.provider.aws.config.AwsServiceConfig;
import org.opengroup.osdu.register.provider.aws.util.DocUtil;
import org.opengroup.osdu.register.provider.interfaces.ddms.IDdmsRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Repository;
import javax.annotation.PostConstruct;
import javax.inject.Inject;
import java.util.ArrayList;
import java.util.List;
@Repository
......@@ -39,35 +36,31 @@ public class AwsDdmsRepo implements IDdmsRepository {
@Autowired
private DpsHeaders dpsHeaders;
private DpsHeaders headers;
@Autowired
private JaxRsDpsLog logger;
@Inject
private AwsServiceConfig serviceConfig;
private DynamoDBQueryHelperFactory dynamoDBQueryHelperFactory;
private DynamoDBQueryHelper queryHelper;
@Value("${aws.dynamodb.ddmsTable.ssm.relativePath}")
String ddmsTableParameterRelativePath;
@Autowired
DocUtil docUtil;
@PostConstruct
public void init() {
queryHelper = new DynamoDBQueryHelper(serviceConfig.getDynamoDbEndpoint(),
serviceConfig.getAmazonRegion(),
serviceConfig.getDynamoDbTablePrefix());
private DynamoDBQueryHelperV2 getDDMSQueryHelper() {
return dynamoDBQueryHelperFactory.getQueryHelperForPartition(headers, ddmsTableParameterRelativePath);
}
@Override
public Ddms create(Ddms ddms) {
DdmsDoc doc = DdmsDoc.mapFrom(ddms, dpsHeaders.getPartitionId());
DdmsDoc doc = DdmsDoc.mapFrom(ddms, headers.getPartitionId());
DynamoDBQueryHelperV2 queryHelper = getDDMSQueryHelper();
try {
queryHelper.saveWithHashCondition(doc,docUtil.getHashKey());
queryHelper.saveWithHashCondition(doc,DocUtil.getHashKey());
}
catch(ConditionalCheckFailedException e){
logger.error(String.format("An DDMS already exists with the id: %s", ddms.getId()));
......@@ -77,13 +70,12 @@ public class AwsDdmsRepo implements IDdmsRepository {
{
throw new AppException(e.getError().getCode(), e.getError().getReason(), e.getMessage());
}
return ddms;
}
@Override
public Ddms get(String id) {
DynamoDBQueryHelperV2 queryHelper = getDDMSQueryHelper();
DdmsDoc doc = queryHelper.loadByPrimaryKey(DdmsDoc.class, id);
if (doc == null) {
logger.error(String.format("Ddms with id %s does not exist.", id));
......@@ -91,32 +83,27 @@ public class AwsDdmsRepo implements IDdmsRepository {
} else {
return DdmsDoc.mapto(doc);
}
}
@Override
public List<Ddms> query(String entityType) {
String dataPartitionId = dpsHeaders.getPartitionId();
DynamoDBQueryHelperV2 queryHelper = getDDMSQueryHelper();
String dataPartitionId = headers.getPartitionId();
DdmsDoc gsiQuery = new DdmsDoc();
String key = String.format("%s:%s",dataPartitionId,entityType);
gsiQuery.setPartitionIdEntityType(key);
List<Ddms> ddmsList = docUtil.getDdmsList(queryHelper,gsiQuery);
return ddmsList;
return docUtil.getDdmsList(queryHelper,gsiQuery);
}
@Override
public boolean delete(String id) {
DynamoDBQueryHelperV2 queryHelper = getDDMSQueryHelper();
try{
DdmsDoc objectTodelete = new DdmsDoc();
objectTodelete.setId(id);
DdmsDoc objectToDelete = new DdmsDoc();
objectToDelete.setId(id);
DynamoDBDeleteExpression deleteExpression = new DynamoDBDeleteExpression()
.withConditionExpression("attribute_exists(id)");
queryHelper.deleteByObjectWithCondition(objectTodelete,deleteExpression);
queryHelper.deleteByObjectWithCondition(objectToDelete,deleteExpression);
}
catch(ConditionalCheckFailedException e){
logger.error(String.format("Failed to delete: %s Object does not exist",id));
......
......@@ -13,18 +13,11 @@
// limitations under the License.
package org.opengroup.osdu.register.provider.aws.subscriber;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.PostConstruct;
import javax.inject.Inject;
import org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelper;
import org.opengroup.osdu.core.aws.ssm.ParameterStorePropertySource;
import org.opengroup.osdu.core.aws.ssm.SSMConfig;
import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBDeleteExpression;
import com.amazonaws.services.dynamodbv2.model.AttributeValue;
import com.amazonaws.services.dynamodbv2.model.ConditionalCheckFailedException;
import org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelperFactory;
import org.opengroup.osdu.core.aws.dynamodb.DynamoDBQueryHelperV2;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
......@@ -34,70 +27,68 @@ import org.opengroup.osdu.register.provider.interfaces.subscriber.ISubscriptionR
import org.opengroup.osdu.register.subscriber.model.Secret;
import org.opengroup.osdu.register.subscriber.model.Subscription;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Repository;
import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBDeleteExpression;
import com.amazonaws.services.dynamodbv2.model.AttributeValue;
import com.amazonaws.services.dynamodbv2.model.ConditionalCheckFailedException;
import javax.annotation.PostConstruct;
import javax.inject.Inject;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Repository
public class AwsSubscriptionRepo implements ISubscriptionRepository {
@Autowired
private DpsHeaders dpsHeaders;
private DpsHeaders headers;
@Autowired
private JaxRsDpsLog logger;
private DynamoDBQueryHelper queryHelper;
@Inject
private AwsServiceConfig serviceConfig;
private DynamoDBQueryHelperFactory dynamoDBQueryHelperFactory;
@Value("${aws.dynamodb.subscriptionTable.ssm.relativePath}")
String subscriptionTableParameterRelativePath;
private String amazonSNSTopicArn_records_changed;
private SSMConfig ssmConfig;
private ParameterStorePropertySource ssm;
@Autowired
private SubscriptionHelper snsHelper;
@Autowired
private KmsHelper kmsHelper;
@Inject
private AwsServiceConfig serviceConfig;
@PostConstruct
public void init() {
amazonSNSTopicArn_records_changed = serviceConfig.getSnsTopicArn();
}
@Autowired
DocUtil docUtil;
@PostConstruct
public void init() {
// TODO: serviceConfig.environment isn't correct and needs to be table prefix. Maybe the "-" will fix it
queryHelper = new DynamoDBQueryHelper(serviceConfig.getDynamoDbEndpoint(),
serviceConfig.getAmazonRegion(),
serviceConfig.getDynamoDbTablePrefix());
amazonSNSTopicArn_records_changed=serviceConfig.getSnsTopicArn();
private DynamoDBQueryHelperV2 getSubscriptionQueryHelper() {
return dynamoDBQueryHelperFactory.getQueryHelperForPartition(headers, subscriptionTableParameterRelativePath);
}
@Override
public Subscription create(Subscription s) throws Exception {
public Subscription create(Subscription s) throws AppException {
DynamoDBQueryHelperV2 queryHelper = getSubscriptionQueryHelper();
String pushEndPoint = s.getPushEndpoint();
String subscriptionArn = snsHelper.createPushSubscription(amazonSNSTopicArn_records_changed,pushEndPoint);
String subscriptionArn = snsHelper.createPushSubscription(amazonSNSTopicArn_records_changed, pushEndPoint);
ByteBuffer encryptedSecretValue = kmsHelper.encrypt(s.getSecret().toString());
SubscriptionDoc doc = SubscriptionDoc.mapFrom(s, dpsHeaders.getPartitionId(),subscriptionArn,encryptedSecretValue);
SubscriptionDoc doc = SubscriptionDoc.mapFrom(s, headers.getPartitionId(), subscriptionArn, encryptedSecretValue);
try {
queryHelper.saveWithHashCondition(doc,docUtil.getHashKey());
}
catch(ConditionalCheckFailedException e){
queryHelper.saveWithHashCondition(doc, DocUtil.getHashKey());
} catch (ConditionalCheckFailedException e) {
logger.error(String.format("A subscription already exists with the id: %s", s.getId()));
throw new AppException(409, "Conflict", String.format("A subscription already exists with the id: %s", s.getId()));
}
catch(AppException e)
{
} catch (AppException e) {
throw new AppException(e.getError().getCode(), e.getError().getReason(), e.getMessage());
}
return s;
......@@ -106,66 +97,57 @@ public class AwsSubscriptionRepo implements ISubscriptionRepository {
@Override
public Subscription get(String id) {
DynamoDBQueryHelperV2 queryHelper = getSubscriptionQueryHelper();
SubscriptionDoc doc = queryHelper.loadByPrimaryKey(SubscriptionDoc.class, id);
if (doc == null) {
logger.error(String.format("Subscription with id %s does not exist.", id));
throw new AppException(404, "Not found", String.format("Subscription with id %s does not exist.", id));
} else {
String secretValue = kmsHelper.decrypt(doc.getSecretValue());
Secret secret = docUtil.createSecret(doc.getSecretType(),secretValue);
Subscription s = SubscriptionDoc.mapTo(doc,secret);
return s;
Secret secret = docUtil.createSecret(doc.getSecretType(), secretValue);
return SubscriptionDoc.mapTo(doc, secret);
}
}
@Override
public List<Subscription> query(String notificationId) {
String key=String.format("%s:%s",dpsHeaders.getPartitionId(),notificationId);
DynamoDBQueryHelperV2 queryHelper = getSubscriptionQueryHelper();
String key = String.format("%s:%s", headers.getPartitionId(), notificationId);
SubscriptionDoc gsiQuery = new SubscriptionDoc();
gsiQuery.setPartitionIdNotificationId(key);
List<Subscription> subsList = docUtil.getSubscriptionList(queryHelper,gsiQuery);
return subsList;
return docUtil.getSubscriptionList(queryHelper, gsiQuery);
}
@Override
public boolean delete(String id) {
SubscriptionDoc doc=null;
String snsSubscriptionArn="";
SubscriptionDoc doc;
String snsSubscriptionArn;
try {
//Need this to retrieve the subscription Arn
doc = queryHelper.loadByPrimaryKey(SubscriptionDoc.class, id);
DynamoDBQueryHelperV2 queryHelper = getSubscriptionQueryHelper();
doc = queryHelper.loadByPrimaryKey(SubscriptionDoc.class, id);
if (doc == null) {
logger.error(String.format("Subscription with id %s does not exist.", id));
throw new AppException(404, "Not found", String.format("Subscription with id %s does not exist.", id));
}
else {
} else {
snsSubscriptionArn = doc.getSnssubscriptionArn();
// queryHelper.deleteByPrimaryKey(SubscriptionDoc.class, id);
SubscriptionDoc objectTodelete = new SubscriptionDoc();
objectTodelete.setId(id);
// queryHelper.deleteByPrimaryKey(SubscriptionDoc.class, id);
SubscriptionDoc objectToDelete = new SubscriptionDoc();
objectToDelete.setId(id);
DynamoDBDeleteExpression deleteExpression = new DynamoDBDeleteExpression()
.withConditionExpression("attribute_exists(id)");
queryHelper.deleteByObjectWithCondition(objectTodelete,deleteExpression);
queryHelper.deleteByObjectWithCondition(objectToDelete, deleteExpression);
}
}
catch(ConditionalCheckFailedException e){
logger.error(String.format("Failed to delete: %s Object does not exist",id));
} catch (ConditionalCheckFailedException e) {
logger.error(String.format("Failed to delete: %s Object does not exist", id));
throw new AppException(404, "ObjectNotFound", String.format("The Subscription you are trying to delete: %s does not exist", id));
}
catch(AppException e)
{
} catch (AppException e) {
throw new AppException(e.getError().getCode(), e.getError().getReason(), e.getMessage());
}
//delete the SNS subscription
//this will throw 404 if SNS Subscription not found
snsHelper.deletePushSubscription(snsSubscriptionArn);
......@@ -175,17 +157,16 @@ public class AwsSubscriptionRepo implements ISubscriptionRepository {
@Override
public boolean patch(Subscription subscription, Secret secret) {
SubscriptionDoc doc=null;
DynamoDBQueryHelperV2 queryHelper = getSubscriptionQueryHelper();
SubscriptionDoc doc;
String id = subscription.getId();
try {
doc = queryHelper.loadByPrimaryKey(SubscriptionDoc.class, id);
}catch(AppException e)
{
if(e.getError().getCode() == 404) {
} catch (AppException e) {
if (e.getError().getCode() == 404) {
logger.error("Could not find subscription with Id %s ", id);
throw e;
}
else {
} else {
logger.error(e.getMessage());
throw new AppException(e.getError().getCode(), e.getError().getReason(), e.getMessage());
}
......@@ -193,44 +174,41 @@ public class AwsSubscriptionRepo implements ISubscriptionRepository {
//update the secret
doc.setSecretType(secret.getSecretType());
doc.setSecretValue(kmsHelper.encrypt(secret.toString()));
try{
try {
queryHelper.save(doc);
} catch (AppException e) {
String msg="Error while saving the updated secret";
throw new AppException(e.getError().getCode(), e.getError