Commit 783eea11 authored by Rucha Deshpande's avatar Rucha Deshpande
Browse files

add aws seismic cicd

parent 1406d242
Pipeline #31510 passed with stage
in 12 seconds
.aws_base_variables:
variables:
# Uncomment these variables and resplace concat-vars below once GL is updated to 13.10+
# AWS_API_GW_DOMAIN: ${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}
# AWS_API_GW_URL: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}
AWS_COGNITO_CLIENT_ID: $AWS_SANDBOX_COGNITO_CLIENT_ID
AWS_COGNITO_USER_POOL_ID: $AWS_SANDBOX_COGNITO_USER_POOL_ID
# AWS_RESOURCE_PREFIX: $AWS_SANDBOX_RESOURCE_PREFIX
# AWS_ELASTIC_HOST: $AWS_SANDBOX_ELASTIC_HOST
# AWS_S3_LEGAL_CONFIG_BUCKET: $AWS_SANDBOX_S3_LEGAL_CONFIG_BUCKET
AWS_REGION: $AWS_SANDBOX_REGION
AWS_ECR_REGION: us-east-1
.aws_variables:
variables:
ACCESS_KEY_ID: $AWS_ACCOUNT_ACCESS_KEY_ID
SECRET_ACCESS_KEY: $AWS_ACCOUNT_SECRET_ACCESS_KEY
AWS_ACCESS_KEY_ID: $AWS_ACCOUNT_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY: $AWS_ACCOUNT_SECRET_ACCESS_KEY
INTEGRATION_TEST_DIR: $AWS_TEST_SUBDIR
SERVICE_NAME: $AWS_SERVICE
BUILD_DIR: $AWS_BUILD_SUBDIR
ENVIRONMENT: $AWS_ENVIRONMENT
APPLICATION_NAME: os-$AWS_SERVICE
LOCAL_IMAGE_TAG: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:$CI_COMMIT_SHA
AWS_IMAGE_TAG_BASE: ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_ECR_REGION}.amazonaws.com/os-${AWS_SERVICE}
LEGALTAG_BASE_URL: ${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}
OTHER_RELEVANT_DATA_COUNTRIES: $AWS_OTHER_RELEVANT_DATA_COUNTRIES
LEGAL_TAG : $AWS_LEGAL_TAG
TENANT_NAME : $AWS_TENANT_NAME
PRIVATE_TENANT1: $AWS_DEFAULT_DATA_PARTITION_ID_TENANT1
PRIVATE_TENANT2: tenant2
SHARED_TENANT: $AWS_DEFAULT_DATA_PARTITION_ID_TENANT2
VENDOR: aws
STORAGE_URL: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/storage/v2/
DOMAIN: $AWS_TESTING_DOMAIN
LEGAL_URL: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/legal/v1/
AWS_COGNITO_AUTH_FLOW: $AWS_COGNITO_AUTH_FLOW
AWS_COGNITO_AUTH_PARAMS_PASSWORD: $AWS_COGNITO_AUTH_PARAMS_PASSWORD
AWS_COGNITO_AUTH_PARAMS_USER: $AWS_COGNITO_AUTH_PARAMS_USER
AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS: $AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS
AWS_COGNITO_REGION: $AWS_SANDBOX_REGION
AWS_ACCOUNT_ID: $AWS_ACCOUNT_ID
DEPLOY_ENV: empty
HOST_URL: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/legal/v1/
MY_TENANT: $AWS_TENANT_NAME
ENTITLEMENTS_TEST_TENANT: $AWS_ENTITLEMENTS_TEST_TENANT
AWS_S3_ENDPOINT: s3.${AWS_SANDBOX_REGION}.amazonaws.com
AWS_S3_REGION: $AWS_SANDBOX_REGION
LOG_LEVEL: INFO
SKIP_HTTP_TESTS: 'true'
S3_LEGAL_CONFIG_BUCKET: $AWS_SANDBOX_S3_LEGAL_CONFIG_BUCKET
LEGAL_QUEUE: https://sqs.${AWS_SANDBOX_REGION}.amazonaws.com/${AWS_ACCOUNT_ID}/${AWS_SANDBOX_RESOURCE_PREFIX}-legal-queue
TABLE_PREFIX: $AWS_SANDBOX_RESOURCE_PREFIX
RESOURCE_PREFIX: $AWS_SANDBOX_RESOURCE_PREFIX
DYNAMO_DB_REGION: $AWS_SANDBOX_REGION
DYNAMO_DB_ENDPOINT: dynamodb.${AWS_SANDBOX_REGION}.amazonaws.com
DELIVERY_INT_TEST_BUCKET_NAME: $AWS_DELIVERY_INT_TEST_BUCKET_NAME
DEFAULT_DATA_PARTITION_ID_TENANT1: $AWS_DEFAULT_DATA_PARTITION_ID_TENANT1
DEFAULT_DATA_PARTITION_ID_TENANT2: $AWS_DEFAULT_DATA_PARTITION_ID_TENANT2
#used by file->delivery int tests
DATA_PARTITION_ID: int-test-file
SEARCH_HOST: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/search/v2/
STORAGE_HOST: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/storage/v2/
LEGAL_HOST: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/legal/v1/
DELIVERY_HOST: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/delivery/v2/
ENTITLEMENTS_DOMAIN: testing.com
ELASTIC_HOST: $AWS_SANDBOX_ELASTIC_HOST
DEFAULT_ELASTIC_USER_NAME: es
DEFAULT_ELASTIC_PASSWORD: $AWS_SANDBOX_REGION
ELASTIC_PORT: 443
INDEXER_HOST: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/indexer/v2/
AWS_CLUSTER_NAME: ${AWS_SANDBOX_RESOURCE_PREFIX}-core-cluster
ENTITLEMENTS_URL: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/entitlements/v1/
VIRTUAL_SERVICE_HOST_NAME: ${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}
HOST: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}
#needs trailing slash
PARTITION_BASE_URL: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/
WORKFLOW_HOST: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/workflow/
DATA_WORKFLOW_HOST: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/data-workflow/v1
TEST_DAG_NAME: my_first_dag
REGISTER_CUSTOM_PUSH_URL: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/register/v1/awstest/aws/challenge
REGISTER_CUSTOM_PUSH_URL1: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/register/v1/awstest/aws/challenge/1
#needs trailing slash
REGISTER_BASE_URL: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/
SUBSCRIBER_SECRET: $AWS_SUBSCRIBER_SECRET
HMAC_SECRET: $AWS_HMAC_SECRET
REGISTER_CUSTOM_PUSH_URL_HMAC: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/register/v1/awstest/aws/challenge/1
NOTIFICATION_BASE_URL: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/notification/v1/
NOTIFICATION_REGISTER_BASE_URL: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}
FILE_SERVICE_HOST: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/file/v2
STORAGE_BASE_URL: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/storage/v2/
DATASET_BASE_URL: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/dataset/v1/
PROVIDER_KEY: AWS_S3
LEGAL_BASE_URL: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/legal/v1/
ENTITLEMENTS_BASE_URL: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/entitlements/v1/
FILEDMS_BASE_URL: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/dms/file/v1/
SEISMICSTORE_SVC_URL: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/seismic-store/v3
AWS_BASE_URL: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}
.aws:
tags: ['osdu-medium']
image: $CI_REGISTRY/divido/aws-maven/aws-maven:v1.1
environment:
name: AWS
variables:
MAVEN_REPO_PATH: "$CI_PROJECT_DIR/.m2/repository"
MAVEN_CLI_OPTS: "--batch-mode --settings=$CI_PROJECT_DIR/.mvn/community-maven.settings.xml"
extends:
- .aws_base_variables
- .aws_variables
before_script:
- mkdir -p ~/.aws
- |
cat > ~/.aws/credentials <<EOF
[default]
aws_access_key_id = $ACCESS_KEY_ID
aws_secret_access_key = $SECRET_ACCESS_KEY
EOF
aws-update-ecs:
extends:
- .aws
- .aws_base_variables
- .aws_variables
stage: deploy
needs: ['push_runtime_image_aws']
script:
- ECS_SERVICE_NAME=$(aws ssm get-parameter --name /osdu/${RESOURCE_PREFIX}/ecs/services/${SERVICE_NAME} --query Parameter.Value --output text --region $AWS_SANDBOX_REGION)
#limit output to the first 50 lines...the rest is bloat
- aws ecs update-service --cluster $AWS_CLUSTER_NAME --service $ECS_SERVICE_NAME --region $AWS_SANDBOX_REGION --force-new-deployment | head -n 50
only:
variables:
- $AWS_SKIP_DEPLOY != 'true' && $AWS == 'true'
aws-test:
extends:
- .aws
- .aws_base_variables
- .aws_variables
stage: integration
needs: ['aws-update-ecs']
script:
- apt-get update
- apt-get install -y python
- apt-get install -y python-pip
- pip install msal
- svctoken=$(python devops/scripts/aws_jwt_client.py)
- echo $svctoken
- npm install -g newman
- chmod +x ./tests/e2e/run_e2e_tests.sh
- ./tests/e2e/run_e2e_tests.sh --seistore-svc-url=${SEISMICSTORE_SVC_URL} --seistore-svc-api-key="NA" --user-idtoken="$svctoken" --user1-idtoken="$svctoken" --tenant=opendes --subproject=awsdemosubproject --admin-email="${AWS_COGNITO_AUTH_PARAMS_USER}" --datapartition=opendes --legaltag01=opendes-sdmstestlegaltag --legaltag02=opendes-sdmstestlegaltag --subproject-long-name=subprojectlonggggggggggggggggggggggname --VCS-Provider="${ISGITLAB}"
only:
variables:
- $AWS == 'true'
artifacts:
when: always
paths:
- $INTEGRATION_TEST_DIR
expire_in: 14 days
......@@ -55,4 +55,28 @@ push_runtime_image_gcp:
- gcloud builds submit --config provider/$GCP_VENDOR/cloudbuild/cloudbuild.yaml --substitutions=_GCP_SERVICE=$GCP_SERVICE,_APPLICATION_NAME=$GCP_APPLICATION_NAME,_PROVIDER_NAME=$GCP_VENDOR,_SHORT_SHA=$CI_COMMIT_SHORT_SHA,_PORT=$PORT
only:
variables:
- $GCP == 'true'
\ No newline at end of file
- $GCP == 'true'
push_runtime_image_aws:
image: $CI_REGISTRY/divido/aws-maven/aws-maven:v1.1
tags: ["osdu-medium"]
stage: containerize
variables:
LOCAL_IMAGE_TAG: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:$CI_COMMIT_SHA
SERVICE_NAME: $AWS_SERVICE
AWS_IMAGE_TAG_BASE: ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_ECR_REGION}.amazonaws.com/os-${AWS_SERVICE}
script:
- docker build -t builder_sdms:latest --file docker/builder.Dockerfile .
- docker build -t $LOCAL_IMAGE_TAG --file docker/runtime.Dockerfile --build-arg docker_builder_image=builder_sdms .
# Push to the local container registry
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- docker push $LOCAL_IMAGE_TAG
# Push to Amazon's container registry
- $(aws ecr get-login --no-include-email --region $AWS_ECR_REGION)
- docker tag $LOCAL_IMAGE_TAG $AWS_IMAGE_TAG_BASE:$CI_COMMIT_SHA
- docker tag $LOCAL_IMAGE_TAG $AWS_IMAGE_TAG_BASE:latest
- docker push $AWS_IMAGE_TAG_BASE:$CI_COMMIT_SHA
- docker push $AWS_IMAGE_TAG_BASE:latest
only:
variables:
- $AWS == 'true'
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment