aws.yml 6.76 KB
Newer Older
1
2
.aws_variables:
  variables:
Matt Wise's avatar
Matt Wise committed
3
4
5
6
    ACCESS_KEY_ID: $AWS_ACCOUNT_ACCESS_KEY_ID
    SECRET_ACCESS_KEY: $AWS_ACCOUNT_SECRET_ACCESS_KEY
    AWS_ACCESS_KEY_ID: $AWS_ACCOUNT_ACCESS_KEY_ID
    AWS_SECRET_ACCESS_KEY: $AWS_ACCOUNT_SECRET_ACCESS_KEY
7
8
9
10
11
12
13
14
15
16
17
18
19
20
    INTEGRATION_TEST_DIR: $AWS_TEST_SUBDIR
    SERVICE_NAME: $AWS_SERVICE
    BUILD_DIR: $AWS_BUILD_SUBDIR
    ENVIRONMENT: $AWS_ENVIRONMENT
    APPLICATION_NAME: os-$AWS_SERVICE
    LOCAL_IMAGE_TAG: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:$CI_COMMIT_SHA
    AWS_IMAGE_TAG_BASE: $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/os-$AWS_SERVICE
    S3_DATA_BUCKET: $AWS_S3_DATA_BUCKET
    SNS_TOPIC_NAME: $AWS_SNS_TOPIC_NAME
    LEGALTAG_BASE_URL: $AWS_LEGALTAG_BASE_URL
    SNS_TOPIC_NAME: $AWS_SNS_TOPIC_NAME
    OTHER_RELEVANT_DATA_COUNTRIES: $AWS_OTHER_RELEVANT_DATA_COUNTRIES
    LEGAL_TAG : $AWS_LEGAL_TAG
    TENANT_NAME : $AWS_TENANT_NAME
Matt Wise's avatar
Matt Wise committed
21
22
23
24
    PRIVATE_TENANT1: $AWS_DEFAULT_DATA_PARTITION_ID_TENANT1
    PRIVATE_TENANT2: tenant2
    SHARED_TENANT: $AWS_DEFAULT_DATA_PARTITION_ID_TENANT2
    VENDOR: aws
25
26
27
28
    STORAGE_URL: $AWS_STORAGE_URL
    DOMAIN: $AWS_TESTING_DOMAIN
    LEGAL_URL: $AWS_LEGAL_URL
    AWS_COGNITO_CLIENT_ID: $AWS_COGNITO_CLIENT_ID
29
    AWS_COGNITO_USER_POOL_ID: $AWS_COGNITO_USER_POOL_ID
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
    AWS_COGNITO_AUTH_FLOW: $AWS_COGNITO_AUTH_FLOW
    AWS_COGNITO_AUTH_PARAMS_PASSWORD: $AWS_COGNITO_AUTH_PARAMS_PASSWORD
    AWS_COGNITO_AUTH_PARAMS_USER: $AWS_COGNITO_AUTH_PARAMS_USER
    AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS: $AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS
    AWS_ACCOUNT_ID: $AWS_ACCOUNT_ID
    AWS_REGION: $AWS_REGION
    DEPLOY_ENV: $AWS_DEPLOY_ENV
    CACHE_CLUSTER_GROUP_ENDPOINT: $AWS_CACHE_CLUSTER_GROUP_ENDPOINT
    CACHE_CLUSTER_GROUP_PORT: $AWS_CACHE_CLUSTER_GROUP_PORT
    CACHE_CLUSTER_LEGALTAG_ENDPOINT: $AWS_CACHE_CLUSTER_LEGALTAG_ENDPOINT
    CACHE_CLUSTER_LEGALTAG_PORT: $AWS_CACHE_CLUSTER_LEGALTAG_PORT
    CACHE_CLUSTER_SCHEMA_ENDPOINT: $AWS_CACHE_CLUSTER_SCHEMA_ENDPOINT
    CACHE_CLUSTER_SCHEMA_PORT: $AWS_CACHE_CLUSTER_SCHEMA_PORT
    APPLICATION_PORT: $AWS_APPLICATION_PORT
    HOST_URL: $AWS_LEGAL_URL
    MY_TENANT: $AWS_TENANT_NAME
46
    ENTITLEMENTS_TEST_TENANT: $AWS_ENTITLEMENTS_TEST_TENANT
47
48
49
50
51
52
53
    AWS_S3_ENDPOINT: $AWS_S3_ENDPOINT
    AWS_S3_REGION: $AWS_REGION
    LOG_LEVEL: INFO
    SKIP_HTTP_TESTS: $AWS_SKIP_HTTP_TESTS
    S3_LEGAL_CONFIG_BUCKET: $AWS_S3_LEGAL_CONFIG_BUCKET
    LEGAL_QUEUE: $AWS_LEGAL_QUEUE
    TABLE_PREFIX: $AWS_TABLE_PREFIX
54
    RESOURCE_PREFIX: $AWS_TABLE_PREFIX
55
56
57
58
59
60
61
62
63
64
    DYNAMO_DB_REGION: $AWS_DYNAMO_DB_REGION
    DYNAMO_DB_ENDPOINT: $AWS_DYNAMO_DB_ENDPOINT
    DELIVERY_INT_TEST_BUCKET_NAME: $AWS_DELIVERY_INT_TEST_BUCKET_NAME
    DEFAULT_DATA_PARTITION_ID_TENANT1: $AWS_DEFAULT_DATA_PARTITION_ID_TENANT1
    DEFAULT_DATA_PARTITION_ID_TENANT2: $AWS_DEFAULT_DATA_PARTITION_ID_TENANT2
    SEARCH_HOST: $AWS_SEARCH_HOST
    STORAGE_HOST: $AWS_STORAGE_HOST
    LEGAL_HOST: $AWS_LEGAL_HOST
    DELIVERY_HOST: $AWS_DELIVERY_HOST
    ENTITLEMENTS_DOMAIN: $AWS_ENTITLEMENTS_DOMAIN
65
66
67
68
69
    ELASTIC_HOST: $AWS_ELASTIC_HOST
    DEFAULT_ELASTIC_USER_NAME: es
    DEFAULT_ELASTIC_PASSWORD: $AWS_REGION
    ELASTIC_PORT: 443
    INDEXER_HOST: $AWS_INDEXER_HOST
70
71
    AWS_CLUSTER_NAME: $AWS_CLUSTER_NAME
    ENTITLEMENTS_URL: $AWS_ENTITLEMENTS_URL
Matt Wise's avatar
Matt Wise committed
72
73
    VIRTUAL_SERVICE_HOST_NAME: $AWS_API_GATEWAY_HOST
    HOST: $AWS_API_GATEWAY_URL
Matt Wise's avatar
Matt Wise committed
74
    PARTITION_BASE_URL: $AWS_API_GATEWAY_URL/ #needs trailing slash
Matt Wise's avatar
Matt Wise committed
75
    WORKFLOW_HOST: $AWS_WORKFLOW_URL
76
    DATA_WORKFLOW_HOST: $AWS_DATA_WORKFLOW_URL
Matt Wise's avatar
Matt Wise committed
77
    INT_TEST_DAG_NAME: my_first_dag
Matt Wise's avatar
Matt Wise committed
78
79
    REGISTER_CUSTOM_PUSH_URL: ${AWS_REGISTER_BASE_URL}${AWS_REGISTER_CUSTOM_PUSH_PATH}
    REGISTER_CUSTOM_PUSH_URL1: ${AWS_REGISTER_BASE_URL}${AWS_REGISTER_CUSTOM_PUSH_PATH1}
Matt Wise's avatar
Matt Wise committed
80
    REGISTER_BASE_URL: $AWS_API_GATEWAY_URL/ #needs trailing slash
Matt Wise's avatar
Matt Wise committed
81
    SUBSCRIBER_SECRET: $AWS_SUBSCRIBER_SECRET
82
83
84
85
    HMAC_SECRET: $AWS_HMAC_SECRET
    REGISTER_CUSTOM_PUSH_URL_HMAC: $AWS_REGISTER_CUSTOM_PUSH_URL_HMAC
    NOTIFICATION_BASE_URL: $AWS_NOTIFICATION_BASE_URL
    NOTIFICATION_REGISTER_BASE_URL: $AWS_NOTIFICATION_REGISTER_BASE_URL
Matt Wise's avatar
Matt Wise committed
86
    FILE_SERVICE_HOST: $AWS_FILE_SERVICE_HOST
87
88

.aws:
89
  tags: ['osdu-medium']
90
  image: $CI_REGISTRY/divido/aws-maven/aws-maven:v1.1
David Diederich's avatar
David Diederich committed
91
92
  environment:
    name: AWS
93
94
95
96
97
98
  variables:
    MAVEN_REPO_PATH: "$CI_PROJECT_DIR/.m2/repository"
    MAVEN_CLI_OPTS: "--batch-mode --settings=$CI_PROJECT_DIR/.mvn/community-maven.settings.xml"
  cache:
    paths:
      - $MAVEN_REPO_PATH
99
100
  extends:
    - .aws_variables
101
102
103
104
105
  before_script:
    - mkdir -p ~/.aws
    - |
      cat > ~/.aws/credentials <<EOF
      [default]
106
107
      aws_access_key_id = $ACCESS_KEY_ID
      aws_secret_access_key = $SECRET_ACCESS_KEY
108
109
110
      EOF

aws-containerize:
111
112
113
  extends: 
    - .aws
    - .aws_variables
114
  stage: containerize
115
  needs: ['compile-and-unit-test']
116
  script:
117
    - docker build  -f $BUILD_DIR/Dockerfile -t $LOCAL_IMAGE_TAG .
118
119
120
121
122
123
124
125
126
    # Push to the local container registry
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
    - docker push $LOCAL_IMAGE_TAG
    # Push to Amazon's container registry
    - $(aws ecr get-login --no-include-email --region $AWS_REGION)
    - docker tag $LOCAL_IMAGE_TAG $AWS_IMAGE_TAG_BASE:$CI_COMMIT_SHA
    - docker tag $LOCAL_IMAGE_TAG $AWS_IMAGE_TAG_BASE:latest
    - docker push $AWS_IMAGE_TAG_BASE:$CI_COMMIT_SHA
    - docker push $AWS_IMAGE_TAG_BASE:latest
127
128
129
130
  only:
    variables:
      - $AWS == 'true'
  
131
132

aws-update-ecs:
133
134
135
  extends: 
    - .aws
    - .aws_variables
136
  stage: deploy
137
  needs: ['aws-containerize']
138
  script:
139
    - ECS_SERVICE_NAME=$(aws ssm get-parameter --name ecs-$SERVICE_NAME --query Parameter.Value  --output text --region $AWS_REGION)
140
    - aws ecs update-service --cluster $AWS_CLUSTER_NAME --service $ECS_SERVICE_NAME --region $AWS_REGION --force-new-deployment
141
142
  only:
    variables:
143
      - $AWS_SKIP_DEPLOY != 'true' && $AWS == 'true'
144

Matt Wise's avatar
Matt Wise committed
145
aws-test-java:
146
147
148
  extends:
    - .aws
    - .aws_variables
149
150
151
  stage: integration
  needs: ['aws-update-ecs']
  script:
152
153
    - ls -ltr
    - cd $INTEGRATION_TEST_DIR
154
    - mvn $MAVEN_CLI_OPTS -Dmaven.repo.local=$MAVEN_REPO_PATH ${AWS_MAVEN_TEST_COMMAND_OVERRIDE:-test} --update-snapshots -Dorg.slf4j.simpleLogger.defaultLogLevel=info
155
156
  only:
    variables:
157
      - $AWS_SKIP_DEPLOY != 'true' && $AWS_SKIP_TESTS != 'true' && $AWS == 'true' && ($AWS_INT_TEST_TYPE == 'java' || $AWS_INT_TEST_TYPE == null) #Default if not defined
Matt Wise's avatar
Matt Wise committed
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
  artifacts:
    when: on_failure
    paths:
      - $INTEGRATION_TEST_DIR
    expire_in: 1 week

aws-test-python:
  extends:
    - .aws
    - .aws_variables
  stage: integration
  needs: ['aws-update-ecs']
  script:
    - ls -ltr
    - cd $INTEGRATION_TEST_DIR
    - chmod +x ./run-integration-tests.sh
    - ./run-integration-tests.sh
  only:
    variables:
177
      - $AWS_SKIP_DEPLOY != 'true' && $AWS_SKIP_TESTS != 'true' && $AWS == 'true' && $AWS_INT_TEST_TYPE == 'python'
178
179
180
181
182
  artifacts:
    when: on_failure
    paths:
      - $INTEGRATION_TEST_DIR
    expire_in: 1 week