Commit 76b3abbe authored by Matt Wise's avatar Matt Wise
Browse files

Squash of EKS and Dynamic Environment Support

add update eks step support

update deployment name

fix syntax

update aws test needs

update ci

update env

fix region

use test aws-maven

use eks deployment name

use v1.2 image of aws-maven

add wait for deployment

work on CI split for ECS/EKS

test_val

implement dynamic variable selection

remove AWS_ECR_REGION, it moved to secrets

add missing before script for aws test

cleanup

leverage ssm/secrets manager for variables

fix syntax

wait up to 5 mins for service deployment

use jq supported build image

fix extra space

fix elastic secret pull

remove whitespace

echo entitlements

syntax

fix base url, fix elastic secret
parent bcf776e0
Pipeline #60781 passed with stages
in 18 seconds
.aws_base_variables:
variables:
# Uncomment these variables and resplace concat-vars below once GL is updated to 13.10+
# AWS_API_GW_DOMAIN: ${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}
# AWS_API_GW_URL: https://${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}
AWS_COGNITO_CLIENT_ID: $AWS_BLUE_COGNITO_CLIENT_ID
AWS_COGNITO_USER_POOL_ID: $AWS_BLUE_COGNITO_USER_POOL_ID
# AWS_RESOURCE_PREFIX: $AWS_BLUE_RESOURCE_PREFIX
# AWS_ELASTIC_HOST: $AWS_SANDBOX_ELASTIC_HOST
# AWS_S3_LEGAL_CONFIG_BUCKET: $AWS_BLUE_S3_LEGAL_CONFIG_BUCKET
AWS_REGION: $AWS_BLUE_REGION
AWS_ECR_REGION: us-east-1
.aws_variables:
.aws_variables:
before_script:
- |
if [ $AWS_DEPLOY_TARGET == 'EKS' ]; then
echo "##### DEPLOYING TO EKS/GREEN ######"
export AWS_INFRA_STACK_NAME="$AWS_GREEN_INFRA_STACK_NAME";
export AWS_REGION="$AWS_GREEN_REGION";
export AWS_DOMAIN="${AWS_GREEN_SUBDOMAIN}.${AWS_BASE_DOMAIN}";
export RESOURCE_PREFIX="$AWS_GREEN_RESOURCE_PREFIX";
export S3_LEGAL_CONFIG_BUCKET="$AWS_GREEN_S3_LEGAL_CONFIG_BUCKET";
export AWS_COGNITO_CLIENT_ID=$(aws ssm get-parameter --name /osdu/${RESOURCE_PREFIX}/cognito-client-id --query Parameter.Value --output text --region $AWS_REGION);
export AWS_COGNITO_USER_POOL_ID=$(aws ssm get-parameter --name /osdu/${RESOURCE_PREFIX}/cognito-user-pool-id --query Parameter.Value --output text --region $AWS_REGION);
export ELASTIC_HOST=$(aws ssm get-parameter --name /osdu/${RESOURCE_PREFIX}/elasticsearch/end-point --query Parameter.Value --output text --region $AWS_REGION);
export ELASTIC_SECRET_STRING=$(aws secretsmanager get-secret-value --secret-id /osdu/${RESOURCE_PREFIX}/elasticsearch/credentials --query SecureString --output json --region $AWS_REGION);
export ELASTIC_USER_NAME=$(echo $ELASTIC_SECRET_STRING | jq -r '.username');
export ELASTIC_PASSWORD=$(echo $ELASTIC_SECRET_STRING | jq -r '.password');
else
echo "##### DEPLOYING TO ECS/BLUE ######"
export AWS_INFRA_STACK_NAME="$AWS_BLUE_INFRA_STACK_NAME";
export AWS_REGION="$AWS_BLUE_REGION";
export AWS_DOMAIN="${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}";
export AWS_COGNITO_CLIENT_ID="$AWS_BLUE_COGNITO_CLIENT_ID";
export AWS_COGNITO_USER_POOL_ID="$AWS_BLUE_COGNITO_USER_POOL_ID";
export RESOURCE_PREFIX="$AWS_BLUE_RESOURCE_PREFIX";
export S3_LEGAL_CONFIG_BUCKET="$AWS_BLUE_S3_LEGAL_CONFIG_BUCKET";
export ELASTIC_HOST="$AWS_BLUE_ELASTIC_HOST";
export ELASTIC_USER_NAME="$AWS_ELASTIC_USERNAME";
export ELASTIC_PASSWORD="$AWS_BLUE_ELASTIC_PASSWORD";
fi
echo "#### USING OSDU BASE URL: $AWS_BASE_URL ####"
export AWS_BASE_URL="https://${AWS_DOMAIN}";
export HOST_URL="${AWS_BASE_URL}/api/legal/v1/"
export LEGALTAG_BASE_URL="${AWS_DOMAIN}"
export STORAGE_URL="${AWS_BASE_URL}/api/storage/v2/"
export LEGAL_URL="${AWS_BASE_URL}/api/legal/v1/"
export SEARCH_HOST="${AWS_BASE_URL}/api/search/v2/"
export STORAGE_HOST="${AWS_BASE_URL}/api/storage/v2/"
export LEGAL_HOST="${AWS_BASE_URL}/api/legal/v1/"
export DELIVERY_HOST="${AWS_BASE_URL}/api/delivery/v2/"
export INDEXER_HOST="${AWS_BASE_URL}/api/indexer/v2/"
export ENTITLEMENTS_URL="${AWS_BASE_URL}/api/entitlements/v2/"
export VIRTUAL_SERVICE_HOST_NAME="${AWS_BASE_URL}"
export HOST="${AWS_BASE_URL}"
# needs trailing slash
export PARTITION_BASE_URL="${AWS_BASE_URL}/"
export WORKFLOW_HOST="${AWS_BASE_URL}/api/workflow/"
export DATA_WORKFLOW_HOST="${AWS_BASE_URL}/api/data-workflow/v1"
export REGISTER_CUSTOM_PUSH_URL="${AWS_BASE_URL}/api/register/v1/awstest/aws/challenge"
export REGISTER_CUSTOM_PUSH_URL1="${AWS_BASE_URL}/api/register/v1/awstest/aws/challenge/1"
# needs trailing slash
export REGISTER_BASE_URL="${AWS_BASE_URL}/"
export REGISTER_CUSTOM_PUSH_URL_HMAC="${AWS_BASE_URL}/api/register/v1/awstest/aws/challenge/1"
export NOTIFICATION_BASE_URL="${AWS_BASE_URL}/api/notification/v1/"
export NOTIFICATION_REGISTER_BASE_URL="${AWS_BASE_URL}"
export FILE_SERVICE_HOST="${AWS_BASE_URL}/api/file/v2"
export STORAGE_BASE_URL="${AWS_BASE_URL}/api/storage/v2/"
export DATASET_BASE_URL="${AWS_BASE_URL}/api/dataset/v1/"
export LEGAL_BASE_URL="${AWS_BASE_URL}/api/legal/v1/"
export ENTITLEMENTS_BASE_URL="${AWS_BASE_URL}/api/entitlements/v2/"
export FILEDMS_BASE_URL="${AWS_BASE_URL}/api/dms/file/v1/"
export SEISMICSTORE_SVC_URL="${AWS_BASE_URL}/api/seismic-store/v3"
export WELLBORE_DDMS_URL="${AWS_BASE_URL}/api/os-wellbore-ddms"
export EDSDMS_BASE_URL="${AWS_BASE_URL}/api/dms/eds/v1/"
export SCHEMA_BASE_URL="${AWS_BASE_URL}/api/schema-service/v1/"
export ENTITLEMENT_V2_URL="${AWS_BASE_URL}/api/entitlements/v2/"
export LEGAL_QUEUE="https://sqs.${AWS_REGION}.amazonaws.com/${AWS_ACCOUNT_ID}/${RESOURCE_PREFIX}-legal-queue"
export AWS_COGNITO_REGION="$AWS_REGION"
export AWS_S3_ENDPOINT="s3.${AWS_REGION}.amazonaws.com"
export AWS_S3_REGION="$AWS_REGION"
export DYNAMO_DB_REGION="$AWS_REGION"
export DYNAMO_DB_ENDPOINT="dynamodb.${AWS_REGION}.amazonaws.com"
export TABLE_PREFIX="$RESOURCE_PREFIX"
export AWS_CLUSTER_NAME="${RESOURCE_PREFIX}-core-cluster"
.aws_common_variables:
variables:
ACCESS_KEY_ID: $AWS_ACCOUNT_ACCESS_KEY_ID
SECRET_ACCESS_KEY: $AWS_ACCOUNT_SECRET_ACCESS_KEY
......@@ -23,8 +98,7 @@
ENVIRONMENT: $AWS_ENVIRONMENT
APPLICATION_NAME: os-$AWS_SERVICE
LOCAL_IMAGE_TAG: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:$CI_COMMIT_SHA
AWS_IMAGE_TAG_BASE: ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_ECR_REGION}.amazonaws.com/os-${AWS_SERVICE}
LEGALTAG_BASE_URL: ${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}
AWS_IMAGE_TAG_BASE: ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_ECR_REGION}.amazonaws.com/os-${AWS_SERVICE}
OTHER_RELEVANT_DATA_COUNTRIES: $AWS_OTHER_RELEVANT_DATA_COUNTRIES
LEGAL_TAG : $AWS_LEGAL_TAG
TENANT_NAME : $AWS_TENANT_NAME
......@@ -32,79 +106,32 @@
PRIVATE_TENANT2: tenant2
SHARED_TENANT: $AWS_DEFAULT_DATA_PARTITION_ID_TENANT2
VENDOR: aws
STORAGE_URL: https://${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/storage/v2/
DOMAIN: example.com
LEGAL_URL: https://${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/legal/v1/
AWS_COGNITO_AUTH_FLOW: $AWS_COGNITO_AUTH_FLOW
AWS_COGNITO_AUTH_PARAMS_PASSWORD: $AWS_COGNITO_AUTH_PARAMS_PASSWORD
AWS_COGNITO_AUTH_PARAMS_USER: $AWS_COGNITO_AUTH_PARAMS_USER
AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS: $AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS
AWS_COGNITO_REGION: $AWS_BLUE_REGION
AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS: $AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS
AWS_ACCOUNT_ID: $AWS_ACCOUNT_ID
DEPLOY_ENV: empty
HOST_URL: https://${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/legal/v1/
DEPLOY_ENV: empty
MY_TENANT: $AWS_TENANT_NAME
ENTITLEMENTS_TEST_TENANT: $AWS_ENTITLEMENTS_TEST_TENANT
AWS_S3_ENDPOINT: s3.${AWS_BLUE_REGION}.amazonaws.com
AWS_S3_REGION: $AWS_BLUE_REGION
LOG_LEVEL: INFO
SKIP_HTTP_TESTS: 'true'
S3_LEGAL_CONFIG_BUCKET: $AWS_BLUE_S3_LEGAL_CONFIG_BUCKET
LEGAL_QUEUE: https://sqs.${AWS_BLUE_REGION}.amazonaws.com/${AWS_ACCOUNT_ID}/${AWS_BLUE_RESOURCE_PREFIX}-legal-queue
TABLE_PREFIX: $AWS_BLUE_RESOURCE_PREFIX
RESOURCE_PREFIX: $AWS_BLUE_RESOURCE_PREFIX
DYNAMO_DB_REGION: $AWS_BLUE_REGION
DYNAMO_DB_ENDPOINT: dynamodb.${AWS_BLUE_REGION}.amazonaws.com
SKIP_HTTP_TESTS: 'true'
DELIVERY_INT_TEST_BUCKET_NAME: $AWS_DELIVERY_INT_TEST_BUCKET_NAME
DEFAULT_DATA_PARTITION_ID_TENANT1: $AWS_DEFAULT_DATA_PARTITION_ID_TENANT1
DEFAULT_DATA_PARTITION_ID_TENANT2: $AWS_DEFAULT_DATA_PARTITION_ID_TENANT2
#used by file->delivery int tests
DATA_PARTITION_ID: int-test-file
SEARCH_HOST: https://${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/search/v2/
STORAGE_HOST: https://${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/storage/v2/
LEGAL_HOST: https://${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/legal/v1/
DELIVERY_HOST: https://${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/delivery/v2/
ENTITLEMENTS_DOMAIN: example.com
ELASTIC_HOST: $AWS_BLUE_ELASTIC_HOST
ELASTIC_PORT: 9200
ELASTIC_USER_NAME: $AWS_ELASTIC_USERNAME
ELASTIC_PASSWORD: $AWS_BLUE_ELASTIC_PASSWORD
INDEXER_HOST: https://${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/indexer/v2/
AWS_CLUSTER_NAME: ${AWS_BLUE_RESOURCE_PREFIX}-core-cluster
ENTITLEMENTS_URL: https://${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/entitlements/v2/
VIRTUAL_SERVICE_HOST_NAME: ${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}
HOST: https://${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}
#needs trailing slash
PARTITION_BASE_URL: https://${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}/
WORKFLOW_HOST: https://${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/workflow/
DATA_WORKFLOW_HOST: https://${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/data-workflow/v1
TEST_DAG_NAME: my_first_dag
REGISTER_CUSTOM_PUSH_URL: https://${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/register/v1/awstest/aws/challenge
REGISTER_CUSTOM_PUSH_URL1: https://${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/register/v1/awstest/aws/challenge/1
#needs trailing slash
REGISTER_BASE_URL: https://${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}/
DATA_PARTITION_ID: int-test-file
ENTITLEMENTS_DOMAIN: example.com
ELASTIC_PORT: 9200
TEST_DAG_NAME: my_first_dag
SUBSCRIBER_SECRET: $AWS_SUBSCRIBER_SECRET
HMAC_SECRET: $AWS_HMAC_SECRET
REGISTER_CUSTOM_PUSH_URL_HMAC: https://${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/register/v1/awstest/aws/challenge/1
NOTIFICATION_BASE_URL: https://${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/notification/v1/
NOTIFICATION_REGISTER_BASE_URL: https://${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}
FILE_SERVICE_HOST: https://${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/file/v2
STORAGE_BASE_URL: https://${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/storage/v2/
DATASET_BASE_URL: https://${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/dataset/v1/
PROVIDER_KEY: AWS_S3
LEGAL_BASE_URL: https://${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/legal/v1/
ENTITLEMENTS_BASE_URL: https://${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/entitlements/v2/
FILEDMS_BASE_URL: https://${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/dms/file/v1/
SEISMICSTORE_SVC_URL: https://${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/seismic-store/v3
WELLBORE_DDMS_URL: https://${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/os-wellbore-ddms
EDSDMS_BASE_URL: https://${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/dms/eds/v1/
AWS_BASE_URL: https://${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}
SCHEMA_BASE_URL: https://${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/schema-service/v1/
HMAC_SECRET: $AWS_HMAC_SECRET
PROVIDER_KEY: AWS_S3
#File Service Variables
TIME_ZONE: UTC
USER_ID: $AWS_COGNITO_AUTH_PARAMS_USER
#Entitlements V2 variables--start--
ENTITLEMENT_V2_URL: https://${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/entitlements/v2/
#Entitlements V2 variables--start--
TENANT: opendes
SERVICE_PRINCIPAL_EMAIL: serviceprincipal@testing.com
#Entitlements V2 variables--end--
......@@ -112,11 +139,11 @@
.aws:
tags: ['osdu-medium']
image: $CI_REGISTRY/divido/aws-maven/aws-maven:v1.1
image: $CI_REGISTRY/divido/aws-maven/aws-maven:v1.3
environment:
name: AWS
extends:
- .aws_base_variables
extends:
- .aws_common_variables
- .aws_variables
before_script:
- mkdir -p ~/.aws
......@@ -129,12 +156,12 @@
aws-containerize:
extends:
- .aws
- .aws_base_variables
- .aws
- .aws_common_variables
- .aws_variables
stage: containerize
needs: ['compile-and-unit-test']
script:
script:
- |
if [ -z $AWS_BUILDER_DOCKERFILE_PATH ] && [ -z $AWS_RUNTIME_DOCKERFILE_PATH ]; then
echo Building BUILD_DIR/Dockerfile container
......@@ -158,19 +185,41 @@ aws-containerize:
variables:
- $AWS == 'true'
aws-update-ecs:
aws-update-eks:
extends:
- .aws
- .aws_base_variables
- .aws_variables
- .aws_common_variables
stage: deploy
needs: ['aws-containerize']
script:
- export KUBECONFIG=/tmp/kubeconfig-${RANDOM}.yaml
- export EKS_CLUSTER_MGMT_ROLE=$(aws cloudformation describe-stacks --region $AWS_REGION --stack-name $AWS_INFRA_STACK_NAME --query "Stacks[0].Outputs[?OutputKey=='MainEKSClusterManagementRole'].OutputValue" --output text)
- export EKS_CLUSTER_NAME=$(aws cloudformation describe-stacks --region $AWS_REGION --stack-name $AWS_INFRA_STACK_NAME --query "Stacks[0].Outputs[?OutputKey=='MainEKSClusterName'].OutputValue" --output text)
- echo Using Role -- $EKS_CLUSTER_MGMT_ROLE
- aws eks update-kubeconfig --region $AWS_REGION --name $EKS_CLUSTER_NAME --role-arn $EKS_CLUSTER_MGMT_ROLE
#Some CLIs require a restrictive KUBECONFIG file
- chmod 644 $KUBECONFIG
- kubectl -n osdu-services rollout restart deployment/${AWS_EKS_DEPLOYMENT_NAME}
- kubectl -n osdu-services rollout status -w deployment/${AWS_EKS_DEPLOYMENT_NAME} --timeout=300s
only:
variables:
- $AWS_SKIP_DEPLOY != 'true' && $AWS == 'true' && $AWS_DEPLOY_TARGET == 'EKS'
aws-update-ecs:
extends:
- .aws
- .aws_common_variables
- .aws_variables
stage: deploy
needs: ['aws-containerize']
script:
- ECS_SERVICE_NAME=$(aws ssm get-parameter --name /osdu/${RESOURCE_PREFIX}/ecs/services/${SERVICE_NAME} --query Parameter.Value --output text --region $AWS_BLUE_REGION)
- ECS_SERVICE_NAME=$(aws ssm get-parameter --name /osdu/${RESOURCE_PREFIX}/ecs/services/${SERVICE_NAME} --query Parameter.Value --output text --region $AWS_REGION)
#limit output to the first 50 lines...the rest is bloat
- export ECS_UPDATE_TMP_FILE=/tmp/ecs-update-log-${RANDOM}.txt
- aws ecs update-service --cluster $AWS_CLUSTER_NAME --service $ECS_SERVICE_NAME --region $AWS_BLUE_REGION --force-new-deployment > $ECS_UPDATE_TMP_FILE
- aws ecs update-service --cluster $AWS_CLUSTER_NAME --service $ECS_SERVICE_NAME --region $AWS_REGION --force-new-deployment > $ECS_UPDATE_TMP_FILE
- cat $ECS_UPDATE_TMP_FILE | head -n 50
only:
variables:
- $AWS_SKIP_DEPLOY != 'true' && $AWS == 'true'
- $AWS_SKIP_DEPLOY != 'true' && $AWS == 'true' && ($AWS_DEPLOY_TARGET == 'ECS' || $AWS_DEPLOY_TARGET == '')
aws-test-java:
extends:
- .maven
- .aws
- .aws_base_variables
- .aws
- .aws_common_variables
- .aws_variables
stage: integration
needs: ['aws-update-ecs']
needs: [{ job: 'aws-update-ecs', optional: true }, { job: 'aws-update-eks', optional: true }]
before_script:
- !reference [.maven, before_script]
- !reference [.aws, before_script]
- !reference [.aws_variables, before_script]
script:
- $MAVEN_BUILD $INTEGRATION_TEST_DIR maven-aws-integration-test-output.txt ${AWS_MAVEN_TEST_COMMAND_OVERRIDE:-test} --update-snapshots
only:
......
aws-test-python:
extends:
- .aws
- .aws_base_variables
- .aws
- .aws_common_variables
- .aws_variables
stage: integration
needs: ['aws-update-ecs']
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment