adding indexer queue.

da320178 · MIchael Nguyen · a596e156 · a596e156 · da320178 · da320178
Commit da320178 authored 5 years ago by MIchael Nguyen
--- a/provider/indexer-aws/CloudFormation/Automated/ecs-indexer-queue.yml
+++ b/provider/indexer-aws/CloudFormation/Automated/ecs-indexer-queue.yml
-# Copyright © Amazon Web Services
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-AWSTemplateFormatVersion: 2010-09-09
-Description: >-
-  CloudFormation template for creating the resources used for the ECS cluster the application will
-  be deployed into. Will create the CodeDeploy application, the ECR repository, and the ECS cluster.
-
-Parameters:
-  Environment:
-    Description: An environment name that will be prefixed to resource names.
-    Type: String
-    AllowedValues:
-      - dev
-      - uat
-      - prod
-    ConstraintDescription: Can only be "dev/uat/prod"
-    Default: dev
-
-  Region:
-    Description: The AWS region to deploy the resources to.
-    Type: String
-    Default: us-east-1
-
-  ApplicationName:
-    Description: >
-      The name of the application, which will be used to generate the ECS cluster name.
-      It will be prefixed with the environment name.
-    Type: String
-    MinLength: '1'
-    MaxLength: '64'
-    AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
-    ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
-    Default: os-indexer
-
-  KeyName:
-    Description: >
-      Name of an existing EC2 KeyPair to enable SSH access to the ECS instances. Note that key pairs cannot
-      be created through CloudFormation, but instead must be uploaded through the AWS Console.
-    Type: AWS::EC2::KeyPair::KeyName
-    Default: ecs_indexer_key
-
-  DesiredCapacity:
-    Description: The default number of instances to launch in the ECS cluster.
-    Type: Number
-    Default: '0'
-
-  MinSize:
-    Description: Maximum number of instances that can be launched in the ECS cluster.
-    Type: Number
-    Default: '0'
-
-  MaxSize:
-    Description: Maximum number of instances that can be launched in the ECS cluster.
-    Type: Number
-    Default: '1'
-
-  InstanceType:
-    Description: EC2 instance type
-    Type: String
-    Default: t3.micro
-    AllowedValues:
-      - m5.large
-      - m5.xlarge
-      - m5.2xlarge
-      - m5.4xlarge
-      - m5.12xlarge
-      - m5.16xlarge
-      - m5.24xlarge
-      - m4.large
-      - m4.xlarge
-      - m4.2xlarge
-      - m4.4xlarge
-      - m4.10xlarge
-      - m4.16xlarge
-      - t3.nano
-      - t3.micro
-      - t3.small
-      - t3.medium
-      - t3.large
-      - t3.xlarge
-      - t3.2xlarge
-      - c5.large
-      - c5.xlarge
-      - c5.2xlarge
-      - c5.4xlarge
-      - c5.12xlarge
-      - c5.16xlarge
-      - c5.24xlarge
-      - r5.large
-      - r5.xlarge
-      - r5.2xlarge
-      - r5.4xlarge
-      - r5.12xlarge
-      - r5.24xlarge
-      - r4.large
-      - r4.xlarge
-      - r4.2xlarge
-      - r4.4xlarge
-      - r4.8xlarge
-      - r4.16xlarge
-      - i3.large
-      - i3.xlarge
-      - i3.2xlarge
-      - i3.4xlarge
-      - i3.10xlarge
-      - i3.16xlarge
-      - x1e.xlarge
-      - x1e.2xlarge
-      - x1e.4xlarge
-      - x1e.8xlarge
-      - x1e.16xlarge
-      - x1e.32xlarge
-    ConstraintDescription: Please choose a valid EC2 instance type for the ECS container instances.
-
-Mappings:
-  # This mapping is for the ECS-optimized edition of the November 13-14, 2019 release of the Amazon Linux 2 AMI
-  # It will need to be periodically updated as new versions are released by Amazon.
-  # The latest ECS-optimized AMI IDs can be found here: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs-optimized_AMI.html
-  # The mapping is used to input the correct AMI ID based on the region the instance is being spun up in.
-  AWSRegionToAMI:
-    us-east-1:
-      AMIID: ami-097e3d1cdb541f43e
-    us-east-2:
-      AMIID: ami-0fbd313043845c4f2
-    us-west-1:
-      AMIID: ami-03d7632ea0ab75eaa
-    us-west-2:
-      AMIID: ami-0fb71e703258ab7eb
-    eu-north-1:
-      AMIID: ami-0f8edbbca6bac13a6
-    eu-west-1:
-      AMIID: ami-0bf45a5f4ab05b949
-    eu-west-2:
-      AMIID: ami-0393b5f363fbd613a
-    eu-west-3:
-      AMIID: ami-03490ca40775a62f0
-    eu-central-1:
-      AMIID: ami-074dc9dd588b6ea52
-    ap-northeast-1:
-      AMIID: ami-0934e28fe3e390537
-    ap-northeast-2:
-      AMIID: ami-0fa5d85859452a178
-    ap-south-1:
-      AMIID: ami-0312d67ff59a3db34
-    ap-southeast-1:
-      AMIID: ami-01f07b3fa86406c96
-    ap-southeast-2:
-      AMIID: ami-07610e278b1ddf331
-    ca-central-1:
-      AMIID: ami-0057d82f917a17334
-    sa-east-1:
-      AMIID: ami-0c947c117562538ee
-
-Resources:
-  CodeDeployApplication:
-    Type: AWS::CodeDeploy::Application
-    Properties:
-      ApplicationName: !Sub ${Environment}-${ApplicationName}-code-deploy
-      ComputePlatform: ECS
-
-  ECRRepository:
-    Type: AWS::ECR::Repository
-    Properties:
-      RepositoryName: !Sub ${Environment}-${ApplicationName}-repository
-      RepositoryPolicyText:
-        Version: "2012-10-17"
-        Statement:
-          - Sid: AllowPushPull
-            Effect: Allow
-            Principal:
-              AWS:
-                - !Sub arn:aws:iam::${AWS::AccountId}:root
-                - Fn::ImportValue:
-                    !Sub "${Environment}-${ApplicationName}-CodeBuildRoleArn"
-                - Fn::ImportValue:
-                    !Sub "${Environment}-${ApplicationName}-CFNRoleArn"
-                - Fn::ImportValue:
-                    !Sub "${Environment}-${ApplicationName}-PipelineRoleArn"
-              Service:
-                - codebuild.amazonaws.com
-            Action:
-              - "ecr:GetDownloadUrlForLayer"
-              - "ecr:BatchGetImage"
-              - "ecr:BatchCheckLayerAvailability"
-              - "ecr:PutImage"
-              - "ecr:InitiateLayerUpload"
-              - "ecr:UploadLayerPart"
-              - "ecr:CompleteLayerUpload"
-
-  CloudWatchLogsGroup:
-    Type: AWS::Logs::LogGroup
-    Properties:
-      LogGroupName: !Join ['-', [ECSLogGroup, !Ref 'ApplicationName']]
-      RetentionInDays: 365
-
-  TaskDefinition:
-    Type: AWS::ECS::TaskDefinition
-    Properties:
-      Family: !Join ['', [!Ref 'AWS::StackName', -, !Ref 'ApplicationName']]
-      ContainerDefinitions:
-        - Name: !Ref 'ApplicationName'
-          Cpu: '10'
-          Essential: 'true'
-          Image: !Sub ${AWS::AccountId}.dkr.ecr.region.amazonaws.com/${Environment}-${ApplicationName}-repository:latest
-          Memory: '300'
-          LogConfiguration:
-            LogDriver: awslogs
-            Options:
-              awslogs-group: !Ref 'CloudWatchLogsGroup'
-              awslogs-region: !Ref 'AWS::Region'
-              awslogs-stream-prefix: !Ref 'ApplicationName'
-          MountPoints:
-            - ContainerPath: /root/.m2
-              SourceVolume: docker-volume
-          PortMappings:
-            - ContainerPort: 80
-      Volumes:
-        - Name: docker-volume
-
-  ECSAutoScalingGroup:
-    Type: AWS::AutoScaling::AutoScalingGroup
-    Properties:
-      VPCZoneIdentifier:
-        - Fn::ImportValue:
-            !Sub "${Environment}-OSDU-PublicSubnet-AZ1"
-        - Fn::ImportValue:
-            !Sub "${Environment}-OSDU-PublicSubnet-AZ2"
-      LaunchConfigurationName: !Ref 'ContainerInstances'
-      MinSize: !Ref 'MinSize'
-      MaxSize: !Ref 'MaxSize'
-      DesiredCapacity: !Ref 'DesiredCapacity'
-    CreationPolicy:
-      ResourceSignal:
-        Timeout: PT15M
-    UpdatePolicy:
-      AutoScalingReplacingUpdate:
-        WillReplace: 'true'
-
-  ContainerInstances:
-    Type: AWS::AutoScaling::LaunchConfiguration
-    Properties:
-      ImageId: !FindInMap [AWSRegionToAMI, !Ref 'AWS::Region', AMIID]
-      SecurityGroups:
-        - Fn::ImportValue:
-            !Sub "${Environment}-${ApplicationName}-EcsSecurityGroupId"
-      InstanceType: !Ref 'InstanceType'
-      IamInstanceProfile: !Ref 'EC2InstanceProfile'
-      KeyName: !Ref 'KeyName'
-      UserData:
-        Fn::Base64: !Sub |
-          #!/bin/bash -xe
-          echo ECS_CLUSTER=${ApplicationECSCluster} >> /etc/ecs/ecs.config
-          yum install -y aws-cfn-bootstrap
-          /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource ECSAutoScalingGroup --region ${AWS::Region}
-
-
-  EC2Role:
-    Type: AWS::IAM::Role
-    Properties:
-      AssumeRolePolicyDocument:
-        Statement:
-          - Effect: Allow
-            Principal:
-              Service: [ec2.amazonaws.com]
-            Action: ['sts:AssumeRole']
-      Path: /
-      Policies:
-        - PolicyName: !Sub ${Environment}-${ApplicationName}-ecs-service
-          PolicyDocument:
-            Statement:
-              - Effect: Allow
-                Action: ['ecs:CreateCluster', 'ecs:DeregisterContainerInstance', 'ecs:DiscoverPollEndpoint',
-                         'ecs:Poll', 'ecs:RegisterContainerInstance', 'ecs:StartTelemetrySession',
-                         'ecs:Submit*', 'logs:CreateLogStream', 'logs:PutLogEvents']
-                Resource: '*'
-
-  AutoscalingRole:
-    Type: AWS::IAM::Role
-    Properties:
-      AssumeRolePolicyDocument:
-        Statement:
-          - Effect: Allow
-            Principal:
-              Service: [application-autoscaling.amazonaws.com]
-            Action: ['sts:AssumeRole']
-      Path: /
-      Policies:
-        - PolicyName: !Sub ${Environment}-${ApplicationName}-service-autoscaling
-          PolicyDocument:
-            Statement:
-              - Effect: Allow
-                Action: ['application-autoscaling:*', 'cloudwatch:DescribeAlarms', 'cloudwatch:PutMetricAlarm',
-                         'ecs:DescribeServices', 'ecs:UpdateService']
-                Resource: '*'
-
-  EC2InstanceProfile:
-    Type: AWS::IAM::InstanceProfile
-    Properties:
-      Path: /
-      Roles: [!Ref 'EC2Role']
-
-  SchedulingRule:
-    Type: AWS::Events::Rule
-    Properties:
-      ScheduleExpression: rate(5 minutes)
-      State: ENABLED
-      Targets:
-        -
-          Arn: !GetAtt ApplicationECSCluster.Arn
-          Id: !Sub ${Environment}-${ApplicationName}-target
-          EcsParemeters:
-            TaskCount: 1
-            TaskDefinitionArn: !Ref 'TaskDefinition'
-
-Outputs:
-  ApplicationECSClusterArn:
-    Description: The ARN of the application's ECS cluster.
-    Value: !GetAtt ApplicationECSCluster.Arn
-    Export:
-      Name: !Sub ${Environment}-${ApplicationName}-EcsClusterArn
-
-  ApplicationECSClusterName:
-    Description: The logical name of the application's ECS cluster.
-    Value: !Ref ApplicationECSCluster
-    Export:
-      Name: !Sub ${Environment}-${ApplicationName}-EcsClusterName
-
-  ECSServiceArn:
-    Description: The ARN of the Indexer Service service in the ECS cluster.
-    Value: !Ref 'Service'
-    Export:
-      Name: !Sub ${Environment}-${ApplicationName}-EcsServiceArn
-
-  ECSALBUrl:
-    Description: The Indexer Service ALB DNS URL.
-    Value: !Join ['', [!GetAtt [ECSALB, DNSName]]]
-    Export:
-      Name: !Sub ${Environment}-${ApplicationName}-EcsAlbUrl
-
-  TaskDefinitionArn:
-    Description: The ARN of the Indexer Service ECS task definition.
-    Value: !Ref 'TaskDefinition'
-    Export:
-      Name: !Sub ${Environment}-${ApplicationName}-EcsTaskDefinitionArn
--- a/provider/indexer-aws/buildspec-post-deploy.yml
+++ b/provider/indexer-aws/buildspec-post-deploy.yml
@@ -69,7 +69,7 @@ phases:
      - mvn clean deploy -f provider/indexer-aws/indexer-queue-aws/pom.xml -Ddeployment.environment=$ENVIRONMENT -Ddeployment.repositorybucket=$S3_MAVEN_REPOSITORY -Daws.accessKeyId=$AWS_ACCESS_KEY_ID_MAVEN -Daws.secretKey=$AWS_SECRET_ACCESS_KEY_MAVEN -Dazure.devops.token=$VSTS_FEED_TOKEN -DaltSnapshotDeploymentRepository=snapshot::default::file:../../local-snapshots-dir -DaltReleaseDeploymentRepository=release::default::file:../../local-release-dir -DaltDeploymentRepository=release::default::file:../../local-release-dir
      - echo os-indexer Docker image build started on `date`...
      - docker build -f provider/indexer-aws/indexer-queue-aws/Dockerfile -t $QUEUE_IMAGE_REPO_NAME:$IMAGE_TAG .
-      - docker tag $IMAGE_REPO_NAME:$IMAGE_TAG $AWS_ACCOUNT_NUMBER.dkr.ecr.$DEPLOYMENT_REGION.amazonaws.com/$QUEUE_IMAGE_REPO_NAME:$IMAGE_TAG
+      - docker tag $QUEUE_IMAGE_REPO_NAME:$IMAGE_TAG $AWS_ACCOUNT_NUMBER.dkr.ecr.$DEPLOYMENT_REGION.amazonaws.com/$QUEUE_IMAGE_REPO_NAME:$IMAGE_TAG
      - echo ...os-indexer Docker image build completed on `date`.
      - echo Pushing the Docker image to ECR...
      - docker push $AWS_ACCOUNT_NUMBER.dkr.ecr.$DEPLOYMENT_REGION.amazonaws.com/$QUEUE_IMAGE_REPO_NAME:$IMAGE_TAG

--- a/provider/indexer-aws/indexer-queue-aws/Dockerfile
+++ b/provider/indexer-aws/indexer-queue-aws/Dockerfile
 FROM amazoncorretto:8

 WORKDIR /
-COPY provider/storage-aws/target/indexer-queue-aws-0.0.1-SNAPSHOT-spring-boot.jar indexer-queue-aws-0.0.1-SNAPSHOT-spring-boot.jar
+COPY provider/indexer-aws/indexer-queue-aws/target/indexer-queue-aws-0.0.1-SNAPSHOT-spring-boot.jar indexer-queue-aws-0.0.1-SNAPSHOT-spring-boot.jar
 EXPOSE 8080

-CMD ["java","-jar", "indexer-aws-0.0.1-SNAPSHOT-spring-boot.jar"]
\ No newline at end of file
+CMD ["java","-jar", "indexer-queue-aws-0.0.1-SNAPSHOT-spring-boot.jar"]
\ No newline at end of file