Commit bd6014d4 authored by Matt Wise's avatar Matt Wise
Browse files

Integration test support for Dataset

commit 79e543c0 
Author: Matt Wise <wsmatth@amazon.com> 
Date: Tue Jan 19 2021 10:42:25 GMT-0600 (Central Standard Time) 

    enable int test preparation in buildspec


commit 4d3e6f02 
Author: Matt Wise <wsmatth@amazon.com> 
Date: Tue Jan 19 2021 10:38:00 GMT-0600 (Central Standard Time) 

    refactor int test location. fully implement


commit b565ca26 
Author: Matt Wise <wsmatth@amazon.com> 
Date: Tue Jan 19 2021 10:23:42 GMT-0600 (Central Standard Time) 

    work on int tests
parent 6776501e
......@@ -65,7 +65,7 @@ phases:
- echo "Skipping integration test build for now..."
# - echo "Building integration testing assemblies and gathering artifacts..."
# - ./testing/dataset-test-aws/build-aws/prepare-dist.sh
- ./testing/dataset-test-aws/build-aws/prepare-dist.sh
- echo "Logging into Docker Hub..."
- docker login -u ${DOCKER_USERNAME} -p ${DOCKER_PASSWORD}
......
# Copyright © 2021 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This script prepares the dist directory for the integration tests.
# Must be run from the root of the repostiory
# This script prepares the dist directory for the integration tests.
# Must be run from the root of the repostiory
set -e
OUTPUT_DIR="${OUTPUT_DIR:-dist}"
INTEGRATION_TEST_OUTPUT_DIR=${INTEGRATION_TEST_OUTPUT_DIR:-$OUTPUT_DIR}/testing/integration
INTEGRATION_TEST_OUTPUT_BIN_DIR=${INTEGRATION_TEST_OUTPUT_DIR:-$INTEGRATION_TEST_OUTPUT_DIR}/bin
INTEGRATION_TEST_SOURCE_DIR=testing
INTEGRATION_TEST_SOURCE_DIR_AWS="$INTEGRATION_TEST_SOURCE_DIR"/dataset-test-aws
INTEGRATION_TEST_SOURCE_DIR_CORE="$INTEGRATION_TEST_SOURCE_DIR"/dataset-test-core
echo "--Source directories variables--"
echo $INTEGRATION_TEST_SOURCE_DIR_AWS
echo $INTEGRATION_TEST_SOURCE_DIR_CORE
echo "--Output directories variables--"
echo $OUTPUT_DIR
echo $INTEGRATION_TEST_OUTPUT_DIR
echo $INTEGRATION_TEST_OUTPUT_BIN_DIR
rm -rf "$INTEGRATION_TEST_OUTPUT_DIR"
mkdir -p "$INTEGRATION_TEST_OUTPUT_DIR" && mkdir -p "$INTEGRATION_TEST_OUTPUT_BIN_DIR"
echo "Building integration testing assemblies and gathering artifacts..."
mvn install -f "$INTEGRATION_TEST_SOURCE_DIR_CORE"/pom.xml
mvn install dependency:copy-dependencies -DskipTests -f "$INTEGRATION_TEST_SOURCE_DIR_AWS"/pom.xml -DincludeGroupIds=org.opengroup.osdu -Dmdep.copyPom
cp "$INTEGRATION_TEST_SOURCE_DIR_AWS"/target/dependency/* "${INTEGRATION_TEST_OUTPUT_BIN_DIR}"
(cd "${INTEGRATION_TEST_OUTPUT_BIN_DIR}" && ls *.jar | sed -e 's/\.jar$//' | xargs -I {} echo mvn install:install-file -Dfile={}.jar -DpomFile={}.pom >> install-deps.sh)
chmod +x "${INTEGRATION_TEST_OUTPUT_BIN_DIR}"/install-deps.sh
mvn clean -f "$INTEGRATION_TEST_SOURCE_DIR_AWS"/pom.xml
cp -R "$INTEGRATION_TEST_SOURCE_DIR_AWS"/* "${INTEGRATION_TEST_OUTPUT_DIR}"/
# Copyright © 2021 Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This script prepares the dist directory for the integration tests.
# Must be run from the root of the repostiory
# This script executes the test and copies reports to the provided output directory
# To call this script from the service working directory
# ./dist/testing/integration/build-aws/run-tests.sh "./reports/"
SCRIPT_SOURCE_DIR=$(dirname "$0")
echo "Script source location"
echo "$SCRIPT_SOURCE_DIR"
(cd "$SCRIPT_SOURCE_DIR"/../bin && ./install-deps.sh)
#### ADD REQUIRED ENVIRONMENT VARIABLES HERE ###############################################
# The following variables are automatically populated from the environment during integration testing
# see os-deploy-aws/build-aws/integration-test-env-variables.py for an updated list
# AWS_COGNITO_CLIENT_ID
# ELASTIC_HOST
# ELASTIC_PORT
# FILE_URL
# LEGAL_URL
# SEARCH_URL
# STORAGE_URL
export LEGAL_BASE_URL=$LEGAL_URL
export DATASET_BASE_URL=$DATASET_BASE_URL
export ENTITLEMENTS_BASE_URL=$ENTITLEMENTS_URL
export STORAGE_BASE_URL=$STORAGE_URL
export TENANT_NAME=opendes
export AWS_COGNITO_AUTH_FLOW=USER_PASSWORD_AUTH
export AWS_COGNITO_AUTH_PARAMS_PASSWORD=$ADMIN_PASSWORD
export AWS_COGNITO_AUTH_PARAMS_USER=$ADMIN_USER
export PROVIDER_KEY=AWS_S3
export AWS_S3_REGION=$AWS_REGION
#### RUN INTEGRATION TEST #########################################################################
mvn test -ntp -f "$SCRIPT_SOURCE_DIR"/../pom.xml
#mvn -Dmaven.surefire.debug test -f "$SCRIPT_SOURCE_DIR"/../pom.xml
TEST_EXIT_CODE=$?
#### COPY TEST REPORTS #########################################################################
if [ -n "$1" ]
then
mkdir -p "$1"
cp -R "$SCRIPT_SOURCE_DIR"/../target/surefire-reports "$1"
fi
exit $TEST_EXIT_CODE
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright © 2021 Amazon Web Services
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.opengroup.osdu.storage</groupId>
<artifactId>dataset-test-aws</artifactId>
<version>0.0.1-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.main.basedir>${project.basedir}</project.main.basedir>
</properties>
<dependencies>
<!-- Internal packages -->
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
<artifactId>os-core-lib-aws</artifactId>
<version>0.3.7</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu.storage</groupId>
<artifactId>dataset-test-core</artifactId>
<version>0.0.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-cognitoidp</artifactId>
<version>1.11.676</version>
</dependency>
<!-- third party Apache 2.0 license packages -->
<dependency>
<groupId>javax.json</groupId>
<artifactId>javax.json-api</artifactId>
<version>1.1.2</version>
</dependency>
<dependency>
<groupId>org.glassfish</groupId>
<artifactId>javax.json</artifactId>
<version>1.1.2</version>
</dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-client</artifactId>
<version>1.19.4</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.8.5</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.9.9.3</version>
</dependency>
<dependency>
<groupId>io.jsonwebtoken</groupId>
<artifactId>jjwt</artifactId>
<version>0.9.1</version>
<exclusions>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.2</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.10</version>
<scope>provided</scope>
</dependency>
<!-- Tests -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
</dependencies>
<repositories>
<repository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</url>
</repository>
</repositories>
<distributionManagement>
<repository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/projects/44/packages/maven</url>
</repository>
<snapshotRepository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/projects/44/packages/maven</url>
</snapshotRepository>
</distributionManagement>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.5</version>
<executions>
<execution>
<phase>integration-test</phase>
<goals>
<goal>test</goal>
</goals>
<configuration>
<excludes>
<exclude>none</exclude>
</excludes>
<includes>
<include>**/storage</include>
</includes>
</configuration>
</execution>
</executions>
<configuration>
<trimStackTrace>false</trimStackTrace>
<systemPropertyVariables>
<buildDirectory>${project.build.directory}</buildDirectory>
</systemPropertyVariables>
</configuration>
</plugin>
</plugins>
</build>
</project>
\ No newline at end of file
// Copyright © 2021 Amazon Web Services
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.dataset;
import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
import com.amazonaws.services.cognitoidp.AWSCognitoIdentityProvider;
import com.amazonaws.services.cognitoidp.AWSCognitoIdentityProviderClientBuilder;
public class AWSCognitoBuilder {
public static AWSCognitoIdentityProvider generateCognitoClient(){
return AWSCognitoIdentityProviderClientBuilder.standard()
.withCredentials(new EnvironmentVariableCredentialsProvider())
.build();
}
}
// Copyright © 2021 Amazon Web Services
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.dataset;
import com.amazonaws.services.cognitoidp.AWSCognitoIdentityProvider;
import com.amazonaws.services.cognitoidp.model.InitiateAuthRequest;
import com.amazonaws.services.cognitoidp.model.InitiateAuthResult;
import java.util.HashMap;
import java.util.Map;
public class AWSCognitoClient {
public static String getTokenForUserWithAccess(){
String clientId = System.getProperty("AWS_COGNITO_CLIENT_ID", System.getenv("AWS_COGNITO_CLIENT_ID"));
String authFlow = System.getProperty("AWS_COGNITO_AUTH_FLOW", System.getenv("AWS_COGNITO_AUTH_FLOW"));
Map<String, String> authParameters = new HashMap<>();
authParameters.put("USERNAME", System.getProperty("AWS_COGNITO_AUTH_PARAMS_USER", System.getenv("AWS_COGNITO_AUTH_PARAMS_USER")));
authParameters.put("PASSWORD", System.getProperty("AWS_COGNITO_AUTH_PARAMS_PASSWORD", System.getenv("AWS_COGNITO_AUTH_PARAMS_PASSWORD")));
AWSCognitoIdentityProvider provider = AWSCognitoBuilder.generateCognitoClient();
InitiateAuthRequest request = new InitiateAuthRequest();
request.setClientId(clientId);
request.setAuthFlow(authFlow);
request.setAuthParameters(authParameters);
InitiateAuthResult result = provider.initiateAuth(request);
return result.getAuthenticationResult().getAccessToken();
}
public static String getTokenForUserWithNoAccess(){
String clientId = System.getProperty("AWS_COGNITO_CLIENT_ID", System.getenv("AWS_COGNITO_CLIENT_ID"));
String authFlow = System.getProperty("AWS_COGNITO_AUTH_FLOW", System.getenv("AWS_COGNITO_AUTH_FLOW"));
Map<String, String> authParameters = new HashMap<>();
authParameters.put("USERNAME", System.getProperty("AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS", System.getenv("AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS")));
authParameters.put("PASSWORD", System.getProperty("AWS_COGNITO_AUTH_PARAMS_PASSWORD", System.getenv("AWS_COGNITO_AUTH_PARAMS_PASSWORD")));
AWSCognitoIdentityProvider provider = AWSCognitoBuilder.generateCognitoClient();
InitiateAuthRequest request = new InitiateAuthRequest();
request.setClientId(clientId);
request.setAuthFlow(authFlow);
request.setAuthParameters(authParameters);
InitiateAuthResult result = provider.initiateAuth(request);
return result.getAuthenticationResult().getAccessToken();
}
}
// Copyright © 2021 Amazon Web Services
// Copyright 2017-2019, Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.dataset;
import com.google.common.base.Strings;
import org.opengroup.osdu.core.aws.cognito.AWSCognitoClient;
public class AWSTestUtils extends TestUtils {
private static String token;
private static String noDataAccesstoken;
private static AWSCognitoClient awsCognitoClient = null;
@Override
public synchronized String getToken() throws Exception {
if (Strings.isNullOrEmpty(token)) {
token = getAwsCognitoClient().getTokenForUserWithAccess();
}
return "Bearer " + token;
}
@Override
public synchronized String getNoDataAccessToken() throws Exception {
if (Strings.isNullOrEmpty(noDataAccesstoken)) {
noDataAccesstoken = getAwsCognitoClient().getTokenForUserWithNoAccess();
}
return "Bearer " + noDataAccesstoken;
}
private AWSCognitoClient getAwsCognitoClient() {
if(awsCognitoClient == null)
awsCognitoClient = new AWSCognitoClient();
return awsCognitoClient;
}
}
// Copyright © 2021 Amazon Web Services
// Copyright 2017-2019, Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.dataset;
import java.util.HashMap;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.opengroup.osdu.dataset.model.IntTestFileDeliveryItemAWSImpl;
import org.opengroup.osdu.dataset.model.IntTestFileUploadLocationAWSImpl;
import org.opengroup.osdu.dataset.model.response.IntTestDatasetRetrievalDeliveryItem;
import org.opengroup.osdu.dataset.util.CloudStorageUtilAws;
public class TestDataset extends Dataset {
private static final AWSTestUtils awsTestUtils = new AWSTestUtils();
public TestDataset() {
cloudStorageUtil = new CloudStorageUtilAws();
}
@BeforeClass
public static void classSetup() throws Exception {
Dataset.classSetup(awsTestUtils.getToken());
}
@AfterClass
public static void classTearDown() throws Exception {
Dataset.classTearDown(awsTestUtils.getToken());
}
@Before
@Override
public void setup() throws Exception {
this.testUtils = new AWSTestUtils();
}
@After
@Override
public void tearDown() throws Exception {
this.testUtils = null;
}
@Override
public void validate_storageLocation(Object storageLocation) {
IntTestFileUploadLocationAWSImpl awsStorageLocation = jsonMapper.convertValue(storageLocation,
IntTestFileUploadLocationAWSImpl.class);
Assert.assertNotNull(awsStorageLocation.getConnectionString());
Assert.assertNotNull(awsStorageLocation.getRegion());
Assert.assertNotNull(awsStorageLocation.getUnsignedUrl());
Assert.assertNotNull(awsStorageLocation.getCreatedAt());
Assert.assertNotNull(awsStorageLocation.getCredentials());
}
@Override
public void validate_dataset_retrieval_delivery_item(IntTestDatasetRetrievalDeliveryItem deliveryItem) {
Assert.assertEquals(deliveryItem.getProviderKey(), TestUtils.providerKey);
IntTestFileDeliveryItemAWSImpl awsDeliveryItem = jsonMapper.convertValue(deliveryItem.getRetrievalProperties(),
IntTestFileDeliveryItemAWSImpl.class);
Assert.assertNotNull(awsDeliveryItem.getConnectionString());
Assert.assertNotNull(awsDeliveryItem.getRegion());
Assert.assertNotNull(awsDeliveryItem.getUnsignedUrl());
Assert.assertNotNull(awsDeliveryItem.getCreatedAt());
Assert.assertNotNull(awsDeliveryItem.getCredentials());
Assert.assertNotNull(awsDeliveryItem.getFileName());
Assert.assertNotNull(awsDeliveryItem.getSignedUrl());
Assert.assertNotNull(awsDeliveryItem.getSignedUrlExpiration());
}
}
\ No newline at end of file
// Copyright © 2021 Amazon Web Services
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.dataset.model;
import com.amazonaws.auth.AWSSessionCredentials;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.Date;
@Data
@NoArgsConstructor
@AllArgsConstructor
@Builder
public class IntTestCredentials implements AWSSessionCredentials {
@JsonProperty("accessKeyId")
private String accessKeyId;
@JsonProperty("secretAccessKey")
private String secretAccessKey;
@JsonProperty("sessionToken")
private String sessionToken;
@JsonProperty("expiration")
private Date expiration;
@Override
@JsonIgnore
public String getAWSAccessKeyId() {
return accessKeyId;
}
@Override
@JsonIgnore
public String getAWSSecretKey() {
return secretAccessKey;
}
@Override
@JsonIgnore
public String getSessionToken() {
return sessionToken;
}
}
\ No newline at end of file
// Copyright © 2021 Amazon Web Services
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//