Commit 141702ab authored by Rucha Deshpande's avatar Rucha Deshpande
Browse files

AWS Notification int. tests

commit 0b9beed3 
Author: Rucha Deshpande <deshruch@amazon.com> 
Date: Fri Oct 23 2020 09:10:09 GMT-0500 (Central Daylight Time) 

    Update fossa.yml for aws


commit 3871c9fe 
Author: Rucha Deshpande <deshruch@amazon.com> 
Date: Thu Oct 22 2020 16:35:39 GMT-0500 (Central Daylight Time) 

    update unit test


commit cf057e5a 
Author: Rucha Deshpande <deshruch@amazon.com> 
Date: Thu Oct 22 2020 14:53:11 GMT-0500 (Central Daylight Time) 

    Update chmod permissions


commit e91dcd6a 
Author: Rucha Deshpande <deshruch@amazon.com> 
Date: Thu Oct 22 2020 14:43:24 GMT-0500 (Central Daylight Time) 

    Update to AWS int tests


commit 7f4cb8e1 
Author: Rucha Deshpande <deshruch@amazon.com> 
Date: Thu Oct 22 2020 14:12:13 GMT-0500 (Central Daylight Time) 

    Update for AWS int tests
parent c3b1bb1a
Pipeline #12908 failed with stages
in 2 minutes and 33 seconds
......@@ -23,4 +23,8 @@ analyze:
- name: notification-azure
type: mvn
target: provider/notification-azure/pom.xml
path: .
- name: notification-aws
type: mvn
target: provider/notification-aws/pom.xml
path: .
\ No newline at end of file
......@@ -4,6 +4,11 @@ variables:
OSDU_GCP_SERVICE: notification
OSDU_GCP_ENV_VARS: APP_PROJECT=${OSDU_GCP_PROJECT},APP_ENTITLEMENTS=${OSDU_GCP_ENTITLEMENTS_URL},APP_REGISTER=${OSDU_GCP_REGISTER_URL},APP_GOOGLEAUDIENCE=${GOOGLE_AUDIENCE}
AWS_BUILD_SUBDIR: provider/notification-aws/build-aws
AWS_TEST_SUBDIR: testing/notification-test-aws
AWS_SERVICE: notification
AWS_ENVIRONMENT: dev
include:
- project: 'osdu/platform/ci-cd-pipelines'
ref: 'master'
......@@ -25,5 +30,8 @@ include:
ref: 'master'
file: 'cloud-providers/osdu-gcp-cloudrun.yml'
- project: 'osdu/platform/ci-cd-pipelines'
file: 'cloud-providers/aws.yml'
osdu-gcp-test:
allow_failure: true
......@@ -53,8 +53,8 @@ phases:
- echo "Building primary service assemblies..."
- mvn -B test install -pl notification-core,provider/notification-aws -Ddeployment.environment=prod
#- echo "Building integration testing assemblies and gathering artifacts..."
#- ./testing/notification-test-aws/build-aws/prepare-dist.sh
- echo "Building integration testing assemblies and gathering artifacts..."
- ./testing/notification-test-aws/build-aws/prepare-dist.sh
- echo "Building docker image..."
- docker build -f provider/notification-aws/build-aws/Dockerfile -t ${ECR_IMAGE} .
......
......@@ -37,6 +37,7 @@
<java.version>8</java.version>
<maven.compiler.target>${java.version}</maven.compiler.target>
<maven.compiler.source>${java.version}</maven.compiler.source>
<aws.version>1.11.637</aws.version>
</properties>
<repositories>
......@@ -70,6 +71,13 @@
<version>0.3.11-SNAPSHOT</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk-secretsmanager -->
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-secretsmanager</artifactId>
<version>${aws.version}</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>notification-core</artifactId>
......
......@@ -48,6 +48,13 @@ public class AwsPubsubRequestBodyExtractor implements IPubsubRequestBodyExtracto
@Autowired
private JaxRsDpsLog log;
@Autowired
public AwsPubsubRequestBodyExtractor(HttpServletRequest httpServletRequest) {
this.request = httpServletRequest;
}
public Map<String, String> extractAttributesFromRequestBody() {
if (this.messageContent == null) {
this.messageContent = this.extractPubsubMessageFromRequestBody();
......
// Copyright © 2020 Amazon Web Services
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.notification.provider.aws.impl;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.services.secretsmanager.AWSSecretsManager;
import com.amazonaws.services.secretsmanager.AWSSecretsManagerClientBuilder;
import com.amazonaws.services.secretsmanager.model.*;
import com.amazonaws.services.simplesystemsmanagement.AWSSimpleSystemsManagement;
import com.amazonaws.services.simplesystemsmanagement.AWSSimpleSystemsManagementClientBuilder;
import com.amazonaws.services.simplesystemsmanagement.model.*;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.Setter;
import org.opengroup.osdu.core.aws.iam.IAMConfig;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient;
import org.opengroup.osdu.notification.provider.aws.utils.AwsCognitoClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@Component
public class ServiceAccountJwtAwsClientImpl implements IServiceAccountJwtClient {
@Value("${aws.region}")
@Getter()
@Setter(AccessLevel.PROTECTED)
public String amazonRegion;
@Value("${aws.ssm}")
@Getter()
@Setter(AccessLevel.PROTECTED)
public Boolean ssmEnabled;
@Value("${aws.environment}")
@Getter()
@Setter(AccessLevel.PROTECTED)
public String environment;
@Autowired
private JaxRsDpsLog log;
String password;
String clientid;
String userpoolid;
String serviceprincipaluser;
AwsCognitoClient cognitoClient;
private AWSCredentialsProvider amazonAWSCredentials;
private AWSSimpleSystemsManagement ssmManager;
@PostConstruct
public void init() {
if (ssmEnabled) {
String secretKey = "service_principal_password";
String secretName = "/osdu/" + environment + "/service_principal_password";
String cognito_user_pool_id = "/osdu/" + environment + "/cognito-user-pool-id";
String cognito_client_id = "/osdu/" + environment + "/cognito-client-id";
String service_principal = "/osdu/" + environment + "/service-principal-user";
amazonAWSCredentials = IAMConfig.amazonAWSCredentials();
ssmManager = AWSSimpleSystemsManagementClientBuilder.standard()
.withCredentials(amazonAWSCredentials)
.withRegion(amazonRegion)
.build();
GetParametersRequest paramRequest = new GetParametersRequest()
.withNames(cognito_user_pool_id,cognito_client_id,service_principal)
.withWithDecryption(true);
GetParametersResult paramResult = new GetParametersResult();
paramResult = ssmManager.getParameters(paramRequest);
List<Parameter> paramsResultList = new ArrayList<>();
List<String> paramsResultListInvalid = new ArrayList<>();
paramsResultList = paramResult.getParameters();
paramsResultListInvalid = paramResult.getInvalidParameters();
if(paramsResultListInvalid.size() >0)
{
log.error("SSM did not retrieve all parameters");
}
for (Parameter s : paramsResultList) {
if (s.getName().equalsIgnoreCase(cognito_user_pool_id)) {
userpoolid = s.getValue();
}
if (s.getName().equalsIgnoreCase(cognito_client_id)) {
clientid = s.getValue();
}
if (s.getName().equalsIgnoreCase(service_principal)) {
serviceprincipaluser = s.getValue();
}
}
password = getSecret(secretName,amazonRegion,secretKey);
cognitoClient = new AwsCognitoClient(amazonRegion,clientid,"USER_PASSWORD_AUTH", serviceprincipaluser,password);
cognitoClient.setPassword(serviceprincipaluser,password,userpoolid);
}
}
@Override
public String getIdToken(String s) {
String token ="not used";
String token= getServicePrincipalCredentials();
return token;
}
public String getServicePrincipalCredentials()
{
String token = cognitoClient.getToken(serviceprincipaluser,password,"bearer");
return token;
}
public String getSecret(String secretName, String region,String secretKey) {
String secretVaue="";
// Create a Secrets Manager client
AWSSecretsManager client = AWSSecretsManagerClientBuilder.standard()
.withRegion(region)
.build();
String secret="", decodedBinarySecret="";
GetSecretValueRequest getSecretValueRequest = new GetSecretValueRequest()
.withSecretId(secretName);
GetSecretValueResult getSecretValueResult = null;
try {
getSecretValueResult = client.getSecretValue(getSecretValueRequest);
} catch (DecryptionFailureException e) {
// Secrets Manager can't decrypt the protected secret text using the provided KMS key.
// Deal with the exception here, and/or rethrow at your discretion.
log.error("Error while setting up ServicePrincipalAccount"+e.getMessage());
throw e;
} catch (InternalServiceErrorException e) {
// An error occurred on the server side.
// Deal with the exception here, and/or rethrow at your discretion.
log.error("Error while setting up ServicePrincipalAccount"+e.getMessage());
throw e;
} catch (InvalidParameterException e) {
// You provided an invalid value for a parameter.
// Deal with the exception here, and/or rethrow at your discretion.
log.error("Error while setting up ServicePrincipalAccount"+e.getMessage());
throw e;
} catch (InvalidRequestException e) {
// You provided a parameter value that is not valid for the current state of the resource.
// Deal with the exception here, and/or rethrow at your discretion.
log.error("Error while setting up ServicePrincipalAccount"+e.getMessage());
throw e;
} catch (ResourceNotFoundException e) {
// We can't find the resource that you asked for.
// Deal with the exception here, and/or rethrow at your discretion.
log.error("Error while setting up ServicePrincipalAccount"+e.getMessage());
throw e;
}
// Decrypts secret using the associated KMS CMK.
// Depending on whether the secret is a string or binary, one of these fields will be populated.
if (getSecretValueResult.getSecretString() != null) {
secret = getSecretValueResult.getSecretString();
Map<String, String> secretMap=null;
try
{
secretMap = new ObjectMapper().readValue(secret.getBytes(), Map.class);
} catch (JsonParseException e) {
log.error(e.getMessage());
} catch (JsonMappingException e) {
log.error(e.getMessage());
} catch (IOException e) {
log.error(e.getMessage());
}
secretVaue = secretMap.get(secretKey);
}
return secretVaue;
}
}
// Copyright © 2020 Amazon Web Services
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.notification.provider.aws.utils;
import com.amazonaws.services.cognitoidp.AWSCognitoIdentityProvider;
import com.amazonaws.services.cognitoidp.AWSCognitoIdentityProviderClientBuilder;
import com.amazonaws.services.cognitoidp.model.AdminSetUserPasswordRequest;
import com.amazonaws.services.cognitoidp.model.AdminSetUserPasswordResult;
import com.amazonaws.services.cognitoidp.model.InitiateAuthRequest;
import com.amazonaws.services.cognitoidp.model.InitiateAuthResult;
import org.opengroup.osdu.core.aws.iam.IAMConfig;
import java.util.HashMap;
import java.util.Map;
public class AwsCognitoClient {
// Parameter value locations
private final static String USERNAME_PARAM = "USERNAME";
private final static String PASSWORD_PARAM = "PASSWORD";
private final static String COGNITO_CLIENT_ID_PROPERTY = "AWS_COGNITO_CLIENT_ID";
private final static String COGNITO_AUTH_FLOW_PROPERTY = "AWS_COGNITO_AUTH_FLOW";
private final static String COGNITO_AUTH_PARAMS_USER_PROPERTY = "AWS_COGNITO_AUTH_PARAMS_USER";
private final static String COGNITO_AUTH_PARAMS_PASSWORD_PROPERTY = "AWS_COGNITO_AUTH_PARAMS_PASSWORD";
String awsCognitoClientId;
String awsCognitoAuthFlow;
String awsCognitoAuthParamsUser;
String awsCognitoAuthParamsPassword;
AWSCognitoIdentityProvider provider;
public AwsCognitoClient(String region,String awsCognitoClientId, String awsCognitoAuthFlow, String awsCognitoAuthParamsUser
, String awsCognitoAuthParamsPassword) {
this.awsCognitoClientId = awsCognitoClientId;
this.awsCognitoAuthFlow = awsCognitoAuthFlow;
this.awsCognitoAuthParamsUser = awsCognitoAuthParamsUser;
this.awsCognitoAuthParamsPassword = awsCognitoAuthParamsPassword;
this.provider = generateCognitoClient(region);
}
public String getToken(String username, String password,String tokenType){
Map<String, String> authParameters = new HashMap<>();
authParameters.put(USERNAME_PARAM, username);
authParameters.put(PASSWORD_PARAM, password);
InitiateAuthRequest request = new InitiateAuthRequest();
request.setClientId(awsCognitoClientId);
request.setAuthFlow(awsCognitoAuthFlow);
request.setAuthParameters(authParameters);
String token="";
InitiateAuthResult result = this.provider.initiateAuth(request);
if(tokenType.equals("session"))
token= result.getSession();
else if(tokenType.equals("bearer"))
token= "Bearer "+ result.getAuthenticationResult().getAccessToken();
return token;
}
public AWSCognitoIdentityProvider getProvider() {
return provider;
}
public static AWSCognitoIdentityProvider generateCognitoClient(String region)
{
return AWSCognitoIdentityProviderClientBuilder.standard()
.withCredentials(IAMConfig.amazonAWSCredentials())
.withRegion(region)
.build();
}
public void setPassword(String username, String password,String userPoolId){
AdminSetUserPasswordRequest request = new AdminSetUserPasswordRequest()
.withUsername(username)
.withPassword(password)
.withPermanent(true)
.withUserPoolId(userPoolId);
AdminSetUserPasswordResult result = this.provider.adminSetUserPassword(request);
}
}
......@@ -21,6 +21,8 @@ AUTHORIZE_API=${ENTITLEMENTS_BASE_URL}/api/entitlements/v1
REGISTER_SERVICE_URL=${REGISTER_BASE_URL}/api/register/v1
aws.ssm=${SSM_ENABLED:True}
aws.environment=${RESOURCE_PREFIX}
## AWS DynamoDB configuration
aws.region=${AWS_REGION}
aws.dynamodb.table.prefix=${RESOURCE_PREFIX}-
......
// Copyright © 2020 Amazon Web Services
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.notification.provider.aws;
import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBDeleteExpression;
import com.amazonaws.services.dynamodbv2.model.ConditionalCheckFailedException;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.runners.MockitoJUnitRunner;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.notification.provider.aws.impl.AwsPubsubRequestBodyExtractor;
import javax.servlet.http.HttpServletRequest;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.StringReader;
import java.util.Map;
@RunWith(MockitoJUnitRunner.class)
public class AwsPubsubRequestBodyExtractorTest {
@Mock
private DpsHeaders dpsHeaders;
@Mock
private JaxRsDpsLog logger;
@Mock
private HttpServletRequest request;
@Mock
private AwsPubsubRequestBodyExtractor service;
@Before
public void init() {
}
@Test
public void should_returnValidData_extractDataFromRequestBody() throws IOException {
String stringRequest = "{\n" +
"\t\"Type\": \"Notification\",\n" +
"\t\"message\": {\n" +
"\t\"attributes\": {\n" +
"\t\"correlation-id\": \"39137f49-67d6-4001-a6aa-15521ef4f49e\",\n" +
"\t\"data-partition-id\": \"" + TestUtils.getDataPartitionId() + "\"\n" +
"\t},\n" +
"\t\"data\": \"dGVzdERhdGE=\",\n" +
"\t\"messageId\": \"136969346945\"\n" +
"\t},\n" +
"\t\"subscription\": \""+ "de12345" +"\"\n" +
"}\n";
BufferedReader reader = new BufferedReader(new StringReader(stringRequest));
Mockito.when(request.getReader()).thenReturn(reader);
String expectedData = "testData";
service = new AwsPubsubRequestBodyExtractor(request);
String receivedData = service.extractDataFromRequestBody();
Assert.assertEquals(expectedData,receivedData);
}
@Test
public void should_returnValidNotificationId_FromRequestBody() throws IOException {
String stringRequest = "{\n" +
"\t\"Type\": \"Notification\",\n" +
"\t\"message\": {\n" +
"\t\"attributes\": {\n" +
"\t\"correlation-id\": \"39137f49-67d6-4001-a6aa-15521ef4f49e\",\n" +
"\t\"data-partition-id\": \"" + TestUtils.getDataPartitionId() + "\"\n" +
"\t},\n" +
"\t\"data\": \"dGVzdERhdGE=\",\n" +
"\t\"messageId\": \"136969346945\"\n" +
"\t},\n" +
"\t\"subscription\": \""+ "de12345" +"\"\n" +
"}\n";
BufferedReader reader = new BufferedReader(new StringReader(stringRequest));
Mockito.when(request.getReader()).thenReturn(reader);
String expectedData = "de12345";
service = new AwsPubsubRequestBodyExtractor(request);
String receivedData = service.extractNotificationIdFromRequestBody();
Assert.assertEquals(expectedData,receivedData);
}
@Test
public void should_returnValidAttributes_FromRequestBody() throws IOException {
String stringRequest = "{\n" +
"\t\"Type\": \"Notification\",\n" +
"\t\"message\": {\n" +
"\t\"attributes\": {\n" +
"\t\"correlation-id\": \"39137f49-123-456\",\n" +
"\t\"data-partition-id\": \"" + TestUtils.getDataPartitionId() + "\"\n" +
"\t},\n" +
"\t\"data\": \"dGVzdERhdGE=\",\n" +
"\t\"messageId\": \"136969346945\"\n" +
"\t},\n" +
"\t\"subscription\": \""+ "de12345" +"\"\n" +
"}\n";
BufferedReader reader = new BufferedReader(new StringReader(stringRequest));
Mockito.when(request.getReader()).thenReturn(reader);
String expectedData = "de12345";
service = new AwsPubsubRequestBodyExtractor(request);
// Act
Map<String, String> receivedAttributes = service.extractAttributesFromRequestBody();
// Asset