aws.yml 8.62 KB
Newer Older
Matt Wise's avatar
Matt Wise committed
1
.aws_base_variables:
2
  variables:
Matt Wise's avatar
Matt Wise committed
3
    # Uncomment these variables and resplace concat-vars below once GL is updated to 13.10+
Matt Wise's avatar
Matt Wise committed
4
    # AWS_API_GW_DOMAIN: ${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}
Matt Wise's avatar
Matt Wise committed
5
    # AWS_API_GW_URL: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}
Matt Wise's avatar
Matt Wise committed
6
7
    AWS_COGNITO_CLIENT_ID: $AWS_SANDBOX_COGNITO_CLIENT_ID
    AWS_COGNITO_USER_POOL_ID: $AWS_SANDBOX_COGNITO_USER_POOL_ID
Matt Wise's avatar
Matt Wise committed
8
9
10
    # AWS_RESOURCE_PREFIX: $AWS_SANDBOX_RESOURCE_PREFIX
    # AWS_ELASTIC_HOST: $AWS_SANDBOX_ELASTIC_HOST
    # AWS_S3_LEGAL_CONFIG_BUCKET: $AWS_SANDBOX_S3_LEGAL_CONFIG_BUCKET
Matt Wise's avatar
Matt Wise committed
11
    AWS_REGION: $AWS_SANDBOX_REGION
Matt Wise's avatar
Matt Wise committed
12
    AWS_ECR_REGION: us-east-1
Matt Wise's avatar
Matt Wise committed
13

Matt Wise's avatar
Matt Wise committed
14
15
.aws_variables: 
  variables:
Matt Wise's avatar
Matt Wise committed
16
17
18
19
    ACCESS_KEY_ID: $AWS_ACCOUNT_ACCESS_KEY_ID
    SECRET_ACCESS_KEY: $AWS_ACCOUNT_SECRET_ACCESS_KEY
    AWS_ACCESS_KEY_ID: $AWS_ACCOUNT_ACCESS_KEY_ID
    AWS_SECRET_ACCESS_KEY: $AWS_ACCOUNT_SECRET_ACCESS_KEY
20
21
22
23
24
25
    INTEGRATION_TEST_DIR: $AWS_TEST_SUBDIR
    SERVICE_NAME: $AWS_SERVICE
    BUILD_DIR: $AWS_BUILD_SUBDIR
    ENVIRONMENT: $AWS_ENVIRONMENT
    APPLICATION_NAME: os-$AWS_SERVICE
    LOCAL_IMAGE_TAG: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:$CI_COMMIT_SHA
Matt Wise's avatar
Matt Wise committed
26
    AWS_IMAGE_TAG_BASE: ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_ECR_REGION}.amazonaws.com/os-${AWS_SERVICE}
Matt Wise's avatar
Matt Wise committed
27
    LEGALTAG_BASE_URL: ${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}
28
29
30
    OTHER_RELEVANT_DATA_COUNTRIES: $AWS_OTHER_RELEVANT_DATA_COUNTRIES
    LEGAL_TAG : $AWS_LEGAL_TAG
    TENANT_NAME : $AWS_TENANT_NAME
Matt Wise's avatar
Matt Wise committed
31
32
33
34
    PRIVATE_TENANT1: $AWS_DEFAULT_DATA_PARTITION_ID_TENANT1
    PRIVATE_TENANT2: tenant2
    SHARED_TENANT: $AWS_DEFAULT_DATA_PARTITION_ID_TENANT2
    VENDOR: aws
Matt Wise's avatar
Matt Wise committed
35
    STORAGE_URL: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/storage/v2/
36
    DOMAIN: $AWS_TESTING_DOMAIN
Matt Wise's avatar
Matt Wise committed
37
    LEGAL_URL: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/legal/v1/
38
39
40
41
    AWS_COGNITO_AUTH_FLOW: $AWS_COGNITO_AUTH_FLOW
    AWS_COGNITO_AUTH_PARAMS_PASSWORD: $AWS_COGNITO_AUTH_PARAMS_PASSWORD
    AWS_COGNITO_AUTH_PARAMS_USER: $AWS_COGNITO_AUTH_PARAMS_USER
    AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS: $AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS
Matt Wise's avatar
Matt Wise committed
42
    AWS_COGNITO_REGION: $AWS_SANDBOX_REGION
43
    AWS_ACCOUNT_ID: $AWS_ACCOUNT_ID
Matt Wise's avatar
Matt Wise committed
44
    DEPLOY_ENV: empty
Matt Wise's avatar
Matt Wise committed
45
    HOST_URL: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/legal/v1/
46
    MY_TENANT: $AWS_TENANT_NAME
47
    ENTITLEMENTS_TEST_TENANT: $AWS_ENTITLEMENTS_TEST_TENANT
Matt Wise's avatar
Matt Wise committed
48
49
    AWS_S3_ENDPOINT: s3.${AWS_SANDBOX_REGION}.amazonaws.com
    AWS_S3_REGION: $AWS_SANDBOX_REGION
50
    LOG_LEVEL: INFO
Matt Wise's avatar
Matt Wise committed
51
    SKIP_HTTP_TESTS: 'true'
Matt Wise's avatar
Matt Wise committed
52
53
54
55
56
57
    S3_LEGAL_CONFIG_BUCKET: $AWS_SANDBOX_S3_LEGAL_CONFIG_BUCKET
    LEGAL_QUEUE: https://sqs.${AWS_SANDBOX_REGION}.amazonaws.com/${AWS_ACCOUNT_ID}/${AWS_SANDBOX_RESOURCE_PREFIX}-legal-queue
    TABLE_PREFIX: $AWS_SANDBOX_RESOURCE_PREFIX
    RESOURCE_PREFIX: $AWS_SANDBOX_RESOURCE_PREFIX
    DYNAMO_DB_REGION: $AWS_SANDBOX_REGION
    DYNAMO_DB_ENDPOINT: dynamodb.${AWS_SANDBOX_REGION}.amazonaws.com
58
59
60
    DELIVERY_INT_TEST_BUCKET_NAME: $AWS_DELIVERY_INT_TEST_BUCKET_NAME
    DEFAULT_DATA_PARTITION_ID_TENANT1: $AWS_DEFAULT_DATA_PARTITION_ID_TENANT1
    DEFAULT_DATA_PARTITION_ID_TENANT2: $AWS_DEFAULT_DATA_PARTITION_ID_TENANT2
61
62
    #used by file->delivery int tests
    DATA_PARTITION_ID: int-test-file
Matt Wise's avatar
Matt Wise committed
63
64
65
66
    SEARCH_HOST: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/search/v2/
    STORAGE_HOST: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/storage/v2/
    LEGAL_HOST: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/legal/v1/
    DELIVERY_HOST: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/delivery/v2/
Matt Wise's avatar
Matt Wise committed
67
    ENTITLEMENTS_DOMAIN: testing.com
Matt Wise's avatar
Matt Wise committed
68
    ELASTIC_HOST: $AWS_SANDBOX_ELASTIC_HOST
69
    DEFAULT_ELASTIC_USER_NAME: es
Matt Wise's avatar
Matt Wise committed
70
    DEFAULT_ELASTIC_PASSWORD: $AWS_SANDBOX_REGION
71
    ELASTIC_PORT: 443
Matt Wise's avatar
Matt Wise committed
72
73
74
75
    INDEXER_HOST: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/indexer/v2/
    AWS_CLUSTER_NAME: ${AWS_SANDBOX_RESOURCE_PREFIX}-core-cluster
    ENTITLEMENTS_URL: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/entitlements/v1/
    VIRTUAL_SERVICE_HOST_NAME: ${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}
Matt Wise's avatar
Matt Wise committed
76
    HOST: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}
77
78
    #needs trailing slash
    PARTITION_BASE_URL: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/
Matt Wise's avatar
Matt Wise committed
79
80
    WORKFLOW_HOST: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/workflow/v1
    DATA_WORKFLOW_HOST: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/data-workflow/v1
Matt Wise's avatar
Matt Wise committed
81
    INT_TEST_DAG_NAME: my_first_dag
Matt Wise's avatar
Matt Wise committed
82
83
    REGISTER_CUSTOM_PUSH_URL: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/register/v1/awstest/aws/challenge
    REGISTER_CUSTOM_PUSH_URL1: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/register/v1/awstest/aws/challenge/1
84
85
    #needs trailing slash
    REGISTER_BASE_URL: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/
Matt Wise's avatar
Matt Wise committed
86
    SUBSCRIBER_SECRET: $AWS_SUBSCRIBER_SECRET
87
    HMAC_SECRET: $AWS_HMAC_SECRET
Matt Wise's avatar
Matt Wise committed
88
89
90
91
92
93
    REGISTER_CUSTOM_PUSH_URL_HMAC: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/register/v1/awstest/aws/challenge/1
    NOTIFICATION_BASE_URL: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/notification/v1/
    NOTIFICATION_REGISTER_BASE_URL: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}
    FILE_SERVICE_HOST: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/file/v2
    STORAGE_BASE_URL: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/storage/v2/
    DATASET_BASE_URL: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/dataset/v1/
Matt Wise's avatar
Matt Wise committed
94
    PROVIDER_KEY: AWS_S3
Matt Wise's avatar
Matt Wise committed
95
96
97
98
    LEGAL_BASE_URL: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/legal/v1/
    ENTITLEMENTS_BASE_URL: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/entitlements/v1/
    FILEDMS_BASE_URL: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}/api/dms/file/v1/
    AWS_BASE_URL: https://${AWS_SANDBOX_SUBDOMAIN}.${AWS_BASE_DOMAIN}
99
.aws:
100
  tags: ['osdu-medium']
101
  image: $CI_REGISTRY/divido/aws-maven/aws-maven:v1.1
David Diederich's avatar
David Diederich committed
102
103
  environment:
    name: AWS
104
105
106
107
108
109
  variables:
    MAVEN_REPO_PATH: "$CI_PROJECT_DIR/.m2/repository"
    MAVEN_CLI_OPTS: "--batch-mode --settings=$CI_PROJECT_DIR/.mvn/community-maven.settings.xml"
  cache:
    paths:
      - $MAVEN_REPO_PATH
110
  extends:
Matt Wise's avatar
Matt Wise committed
111
    - .aws_base_variables
Matt Wise's avatar
Matt Wise committed
112
    - .aws_variables
113
114
115
116
117
  before_script:
    - mkdir -p ~/.aws
    - |
      cat > ~/.aws/credentials <<EOF
      [default]
118
119
      aws_access_key_id = $ACCESS_KEY_ID
      aws_secret_access_key = $SECRET_ACCESS_KEY
120
121
122
      EOF

aws-containerize:
123
124
  extends: 
    - .aws
Matt Wise's avatar
Matt Wise committed
125
    - .aws_base_variables
126
    - .aws_variables
127
  stage: containerize
128
  needs: ['compile-and-unit-test']
129
  script:
130
    - docker build  -f $BUILD_DIR/Dockerfile -t $LOCAL_IMAGE_TAG .
131
132
133
134
    # Push to the local container registry
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
    - docker push $LOCAL_IMAGE_TAG
    # Push to Amazon's container registry
Matt Wise's avatar
Matt Wise committed
135
    - $(aws ecr get-login --no-include-email --region $AWS_ECR_REGION)
136
137
138
139
    - docker tag $LOCAL_IMAGE_TAG $AWS_IMAGE_TAG_BASE:$CI_COMMIT_SHA
    - docker tag $LOCAL_IMAGE_TAG $AWS_IMAGE_TAG_BASE:latest
    - docker push $AWS_IMAGE_TAG_BASE:$CI_COMMIT_SHA
    - docker push $AWS_IMAGE_TAG_BASE:latest
140
141
142
143
  only:
    variables:
      - $AWS == 'true'
  
144
145

aws-update-ecs:
146
147
  extends: 
    - .aws
Matt Wise's avatar
Matt Wise committed
148
    - .aws_base_variables
149
    - .aws_variables
150
  stage: deploy
151
  needs: ['aws-containerize']
152
  script:
Matt Wise's avatar
Matt Wise committed
153
154
    - ECS_SERVICE_NAME=$(aws ssm get-parameter --name /osdu/${RESOURCE_PREFIX}/ecs/services/${SERVICE_NAME} --query Parameter.Value  --output text --region $AWS_SANDBOX_REGION)
    - aws ecs update-service --cluster $AWS_CLUSTER_NAME --service $ECS_SERVICE_NAME --region $AWS_SANDBOX_REGION --force-new-deployment
155
156
  only:
    variables:
157
      - $AWS_SKIP_DEPLOY != 'true' && $AWS == 'true'
158

Matt Wise's avatar
Matt Wise committed
159
aws-test-java:
160
161
  extends:
    - .aws
Matt Wise's avatar
Matt Wise committed
162
    - .aws_base_variables
163
    - .aws_variables
164
165
166
  stage: integration
  needs: ['aws-update-ecs']
  script:
167
168
    - ls -ltr
    - cd $INTEGRATION_TEST_DIR
169
    - mvn $MAVEN_CLI_OPTS -Dmaven.repo.local=$MAVEN_REPO_PATH ${AWS_MAVEN_TEST_COMMAND_OVERRIDE:-test} --update-snapshots -Dorg.slf4j.simpleLogger.defaultLogLevel=info
170
171
  only:
    variables:
172
173
      #Default if not defined
      - $AWS_SKIP_DEPLOY != 'true' && $AWS_SKIP_TESTS != 'true' && $AWS == 'true' && ($AWS_INT_TEST_TYPE == 'java' || $AWS_INT_TEST_TYPE == null)
Matt Wise's avatar
Matt Wise committed
174
175
176
177
178
179
180
181
182
  artifacts:
    when: on_failure
    paths:
      - $INTEGRATION_TEST_DIR
    expire_in: 1 week

aws-test-python:
  extends:
    - .aws
Matt Wise's avatar
Matt Wise committed
183
    - .aws_base_variables
Matt Wise's avatar
Matt Wise committed
184
185
186
187
188
189
190
191
192
193
    - .aws_variables
  stage: integration
  needs: ['aws-update-ecs']
  script:
    - ls -ltr
    - cd $INTEGRATION_TEST_DIR
    - chmod +x ./run-integration-tests.sh
    - ./run-integration-tests.sh
  only:
    variables:
194
      - $AWS_SKIP_DEPLOY != 'true' && $AWS_SKIP_TESTS != 'true' && $AWS == 'true' && $AWS_INT_TEST_TYPE == 'python'
195
196
197
198
199
  artifacts:
    when: on_failure
    paths:
      - $INTEGRATION_TEST_DIR
    expire_in: 1 week