diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0d301d39a7172eff7cfa3c20a354be0489731534..ab8dfe05c80df9ed3eb66b02d45fa59bb3b2125f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -156,9 +156,6 @@ azure_containerize: azure_test: image: community.opengroup.org:5555/osdu/platform/deployment-and-operations/base-containers-azure/azure-maven17:v0.0.1 -aws-test-java: - image: $CI_REGISTRY/osdu/platform/deployment-and-operations/base-containers-aws/aws-maven/aws-maven:v2.1 - fossa-analyze: image: $CI_REGISTRY/divido/fossa-with-cache:v0.9-jdk17 diff --git a/devops/aws/pipeline/override-stages.yml b/devops/aws/pipeline/override-stages.yml index 3fec4d932234dc0275d52ab8b369e27d2c4b3b41..58c1d60a5f5ea0b135a5f099de53ceb169a8018b 100644 --- a/devops/aws/pipeline/override-stages.yml +++ b/devops/aws/pipeline/override-stages.yml @@ -22,6 +22,42 @@ include: - project: "osdu/platform/ci-cd-pipelines" file: "cloud-providers/aws-maven.yml" +aws-containerize: + extends: + - .aws + - .aws_common_variables + - .aws_variables + stage: containerize + needs: ['compile-and-unit-test'] + script: + - | + if [ -z $AWS_BUILDER_DOCKERFILE_PATH ] && [ -z $AWS_RUNTIME_DOCKERFILE_PATH ]; then + echo Building BUILD_DIR/Dockerfile container + docker build -f $BUILD_DIR/Dockerfile -t $LOCAL_IMAGE_TAG .; + else + echo Building BUILDER->RUNTIME containers; + docker build -t builder:latest --file $AWS_BUILDER_DOCKERFILE_PATH .; + docker build -t $LOCAL_IMAGE_TAG --file $AWS_RUNTIME_DOCKERFILE_PATH --build-arg docker_builder_image=builder .; + fi + + # Push to the local container registry + - export PUSH_ENDPOINT_ECR_TAG=`echo push_endpoint.${CI_COMMIT_SHA} | cut -c 1-120` + - export PUSH_ENDPOINT_IMAGE=${AWS_IMAGE_TAG_BASE}:${PUSH_ENDPOINT_ECR_TAG} + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + - docker push $LOCAL_IMAGE_TAG + # Push to Amazon's container registry + - $(aws ecr get-login --no-include-email --region $AWS_ECR_REGION) + - docker tag $LOCAL_IMAGE_TAG $AWS_IMAGE_TAG_BASE:$CI_COMMIT_SHA + - docker tag $LOCAL_IMAGE_TAG $AWS_IMAGE_TAG_BASE:latest + - docker push $AWS_IMAGE_TAG_BASE:$CI_COMMIT_SHA + - docker push $AWS_IMAGE_TAG_BASE:latest + - export INTEGRATION_TEST_SOURCE_DIR_AWS=$(pwd)/testing/notification-test-aws + - $INTEGRATION_TEST_SOURCE_DIR_AWS/build-aws/build_push_endpoint_image.sh + only: + variables: + - $AWS == '1' + + aws-test-java: image: $CI_REGISTRY/osdu/platform/deployment-and-operations/base-containers-aws/aws-maven/aws-maven:v2.1 extends: @@ -30,7 +66,6 @@ aws-test-java: - .aws_common_variables - .aws_variables stage: integration - allow_failure: true needs: [{ job: 'aws-update-tf', optional: true }, { job: 'aws-update-helm', optional: true }, { job: 'aws-update-eks', optional: true }] retry: 1 before_script: @@ -41,8 +76,8 @@ aws-test-java: - export KUBECONFIG=/tmp/kubeconfig-${RANDOM}.yaml - aws eks update-kubeconfig --region $AWS_REGION --name $EKS_CLUSTER_NAME --role-arn $EKS_CLUSTER_MGMT_ROLE - chmod 600 $KUBECONFIG - - pip3 install boto3 - export PUSH_ENDPOINT_PATH=testing/notification-test-aws/build-aws/push-endpoint + - pip3 install -r $PUSH_ENDPOINT_PATH/requirements.txt - echo "PUSH_ENDPOINT_PATH ${PUSH_ENDPOINT_PATH}" - python3 $PUSH_ENDPOINT_PATH/setup_teardown_endpoint.py --create --environment_file $PUSH_ENDPOINT_PATH/environment_file.sh --kube_file $PUSH_ENDPOINT_PATH/push-endpoint-manifest.yaml - trap "python3 $PUSH_ENDPOINT_PATH/setup_teardown_endpoint.py --delete --environment_file $PUSH_ENDPOINT_PATH/environment_file.sh --kube_file $PUSH_ENDPOINT_PATH/push-endpoint-manifest.yaml" EXIT diff --git a/testing/notification-test-aws/build-aws/build_push_endpoint.sh b/testing/notification-test-aws/build-aws/build_push_endpoint.sh new file mode 100755 index 0000000000000000000000000000000000000000..ffe56380bdf0915103635b415e0be161b93998de --- /dev/null +++ b/testing/notification-test-aws/build-aws/build_push_endpoint.sh @@ -0,0 +1,15 @@ +# Copyright © 2020 Amazon Web Services +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +(cd $INTEGRATION_TEST_SOURCE_DIR_AWS/build-aws/push-endpoint && mvn -B -ntp -s ./maven/settings.xml package) diff --git a/testing/notification-test-aws/build-aws/build_push_endpoint_image.sh b/testing/notification-test-aws/build-aws/build_push_endpoint_image.sh new file mode 100755 index 0000000000000000000000000000000000000000..eca26b546b2c92589e6d45800677e165ab0aa761 --- /dev/null +++ b/testing/notification-test-aws/build-aws/build_push_endpoint_image.sh @@ -0,0 +1,15 @@ +# Copyright © 2020 Amazon Web Services +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +(cd $INTEGRATION_TEST_SOURCE_DIR_AWS/build-aws/push-endpoint && docker build -t $PUSH_ENDPOINT_IMAGE -f Dockerfile --no-cache . && docker push $PUSH_ENDPOINT_IMAGE) diff --git a/testing/notification-test-aws/build-aws/prepare-dist.sh b/testing/notification-test-aws/build-aws/prepare-dist.sh index a82740f3e97968bf7cdfcda58863587ef120288e..73be3acc3c40a4cb465b98edb0595d75e58f3952 100755 --- a/testing/notification-test-aws/build-aws/prepare-dist.sh +++ b/testing/notification-test-aws/build-aws/prepare-dist.sh @@ -36,7 +36,8 @@ echo $INTEGRATION_TEST_OUTPUT_DIR echo $INTEGRATION_TEST_OUTPUT_BIN_DIR # Build the Push Endpoint image -(cd $INTEGRATION_TEST_SOURCE_DIR_AWS/build-aws/push-endpoint && mvn -B -ntp -s ./maven/settings.xml package && docker build -t $PUSH_ENDPOINT_IMAGE -f Dockerfile --no-cache . && docker push $PUSH_ENDPOINT_IMAGE) +$INTEGRATION_TEST_SOURCE_DIR_AWS/build-aws/build_push_endpoint.sh +$INTEGRATION_TEST_SOURCE_DIR_AWS/build-aws/build_push_endpoint_image.sh # Finished building the Push Endpoint image rm -rf "$INTEGRATION_TEST_OUTPUT_DIR" diff --git a/testing/notification-test-aws/build-aws/push-endpoint/requirements.txt b/testing/notification-test-aws/build-aws/push-endpoint/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..457ca83c09870fe2d337d9008cbec08f5a2a1337 --- /dev/null +++ b/testing/notification-test-aws/build-aws/push-endpoint/requirements.txt @@ -0,0 +1,17 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the “Licenseâ€). +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an “AS IS†BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +boto3==1.26.6 +botocore==1.29.6 +colorama==0.4.4 diff --git a/testing/notification-test-aws/build-aws/push-endpoint/setup_teardown_endpoint.py b/testing/notification-test-aws/build-aws/push-endpoint/setup_teardown_endpoint.py index 922e9d2b328c909a2218e973456960b3058f8755..9c1c3bb864b9e367dfd4b8804fd05db053978ca5 100644 --- a/testing/notification-test-aws/build-aws/push-endpoint/setup_teardown_endpoint.py +++ b/testing/notification-test-aws/build-aws/push-endpoint/setup_teardown_endpoint.py @@ -330,9 +330,11 @@ class CommonParameters: PolicyName=self.sa_role_policy_name, PolicyDocument=permissions_policy) role_arn = role["Role"]["Arn"] + print(f"Created role {role_arn}") except self.iam_client.exceptions.EntityAlreadyExistsException: role = self.iam_client.get_role(RoleName=service_role_name) role_arn = role["Role"]["Arn"] + print(f"Using pre-existing role {role_arn}") return role_arn def create_k8s_file(self, queue_urls: typing.List[str], diff --git a/testing/notification-test-aws/build-aws/run-tests.sh b/testing/notification-test-aws/build-aws/run-tests.sh index 5a181f150a5ccac46b6de6286638f4d7ecee6e93..3820f05f7afadcca0708e1426e87c93b40161049 100755 --- a/testing/notification-test-aws/build-aws/run-tests.sh +++ b/testing/notification-test-aws/build-aws/run-tests.sh @@ -31,15 +31,16 @@ echo "$SCRIPT_SOURCE_DIR" # see os-deploy-aws/build-aws/integration-test-env-variables.py for an updated list PUSH_ENDPOINT_PATH=$SCRIPT_SOURCE_DIR/push-endpoint +pip3 install -r $PUSH_ENDPOINT_PATH/requirements.txt python3 $PUSH_ENDPOINT_PATH/setup_teardown_endpoint.py --create --environment_file $PUSH_ENDPOINT_PATH/environment_file.sh --kube_file $PUSH_ENDPOINT_PATH/push-endpoint-manifest.yaml trap "python3 $PUSH_ENDPOINT_PATH/setup_teardown_endpoint.py --delete --environment_file $PUSH_ENDPOINT_PATH/environment_file.sh --kube_file $PUSH_ENDPOINT_PATH/push-endpoint-manifest.yaml" EXIT source $PUSH_ENDPOINT_PATH/environment_file.sh export AWS_COGNITO_AUTH_FLOW=USER_PASSWORD_AUTH -export AWS_COGNITO_AUTH_PARAMS_PASSWORD=$ADMIN_PASSWORD -export AWS_COGNITO_AUTH_PARAMS_USER=$ADMIN_USER -export AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS=$USER_NO_ACCESS +export AWS_COGNITO_AUTH_PARAMS_PASSWORD=${AWS_COGNITO_AUTH_PARAMS_PASSWORD:-ADMIN_PASSWORD} +export AWS_COGNITO_AUTH_PARAMS_USER=${AWS_COGNITO_AUTH_PARAMS_USER:-ADMIN_USER} +export AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS=${AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS:-USER_NO_ACCESS} export ENVIRONMENT=DEV export NOTIFICATION_REGISTER_BASE_URL=$NOTIFICATION_REGISTER_BASE_URL export REGISTER_CUSTOM_PUSH_PATH=/api/push-endpoint/0 diff --git a/testing/notification-test-aws/src/test/java/org/opengroup/osdu/notification/subscriptions/TestNotificationsEndpoint.java b/testing/notification-test-aws/src/test/java/org/opengroup/osdu/notification/subscriptions/TestNotificationsEndpoint.java index 858d7c1c6c3dc2163bcb7b975eb9d34e6e91b2ad..7b850facd7ecefbdca82786923acb3755f8efe34 100644 --- a/testing/notification-test-aws/src/test/java/org/opengroup/osdu/notification/subscriptions/TestNotificationsEndpoint.java +++ b/testing/notification-test-aws/src/test/java/org/opengroup/osdu/notification/subscriptions/TestNotificationsEndpoint.java @@ -85,7 +85,7 @@ public class TestNotificationsEndpoint { private final static ObjectMapper objectMapper = new ObjectMapper(); private static final int MAX_RETRIES = 5; - private static final int MAX_POLL_TIME = 120; + private static final int MAX_POLL_TIME = 240; private static class PushQueue {