Skip to content
Snippets Groups Projects
Commit d65d7e4e authored by MIchael Nguyen's avatar MIchael Nguyen
Browse files

removed deployment for indexer queue. not completed.

parent c41456f9
Branches IBM_vul_fix
No related tags found
1 merge request!6Trusted ibm
Hide whitespace changes
Inline Side-by-side
provider/indexer-aws/CloudFormation/Automated/fargate-cluster.yml deleted 100644 → 0
+ 0
− 467
View file @ c41456f9
AWSTemplateFormatVersion: '2010-09-09'
Description: This stack deploys a Fargate cluster that is in a VPC with both
public and private subnets. Containers can be deployed into either
the public subnets or the private subnets, and there are two load
balancers. One is inside the public subnet, which can be used to
send traffic to the containers in the private subnet, and one in
the private subnet, which can be used for private internal traffic
between internal services.
Mappings:
# Hard values for the subnet masks. These masks define
# the range of internal IP addresses that can be assigned.
# The VPC can have all IP's from 10.0.0.0 to 10.0.255.255
# There are four subnets which cover the ranges:
#
# 10.0.0.0 - 10.0.0.255
# 10.0.1.0 - 10.0.1.255
# 10.0.2.0 - 10.0.2.255
# 10.0.3.0 - 10.0.3.255
#
# If you need more IP addresses (perhaps you have so many
# instances that you run out) then you can customize these
# ranges to add more
SubnetConfig:
VPC:
CIDR: '10.0.0.0/16'
PublicOne:
CIDR: '10.0.0.0/24'
PublicTwo:
CIDR: '10.0.1.0/24'
PrivateOne:
CIDR: '10.0.2.0/24'
PrivateTwo:
CIDR: '10.0.3.0/24'
Resources:
# VPC in which containers will be networked.
# It has two public subnets, and two private subnets.
# We distribute the subnets across the first two available subnets
# for the region, for high availability.
VPC:
Type: AWS::EC2::VPC
Properties:
EnableDnsSupport: true
EnableDnsHostnames: true
CidrBlock: !FindInMap ['SubnetConfig', 'VPC', 'CIDR']
# Two public subnets, where containers can have public IP addresses
PublicSubnetOne:
Type: AWS::EC2::Subnet
Properties:
AvailabilityZone:
Fn::Select:
- 0
- Fn::GetAZs: {Ref: 'AWS::Region'}
VpcId: !Ref 'VPC'
CidrBlock: !FindInMap ['SubnetConfig', 'PublicOne', 'CIDR']
MapPublicIpOnLaunch: true
PublicSubnetTwo:
Type: AWS::EC2::Subnet
Properties:
AvailabilityZone:
Fn::Select:
- 1
- Fn::GetAZs: {Ref: 'AWS::Region'}
VpcId: !Ref 'VPC'
CidrBlock: !FindInMap ['SubnetConfig', 'PublicTwo', 'CIDR']
MapPublicIpOnLaunch: true
# Two private subnets where containers will only have private
# IP addresses, and will only be reachable by other members of the
# VPC
PrivateSubnetOne:
Type: AWS::EC2::Subnet
Properties:
AvailabilityZone:
Fn::Select:
- 0
- Fn::GetAZs: {Ref: 'AWS::Region'}
VpcId: !Ref 'VPC'
CidrBlock: !FindInMap ['SubnetConfig', 'PrivateOne', 'CIDR']
PrivateSubnetTwo:
Type: AWS::EC2::Subnet
Properties:
AvailabilityZone:
Fn::Select:
- 1
- Fn::GetAZs: {Ref: 'AWS::Region'}
VpcId: !Ref 'VPC'
CidrBlock: !FindInMap ['SubnetConfig', 'PrivateTwo', 'CIDR']
# Setup networking resources for the public subnets. Containers
# in the public subnets have public IP addresses and the routing table
# sends network traffic via the internet gateway.
InternetGateway:
Type: AWS::EC2::InternetGateway
GatewayAttachement:
Type: AWS::EC2::VPCGatewayAttachment
Properties:
VpcId: !Ref 'VPC'
InternetGatewayId: !Ref 'InternetGateway'
PublicRouteTable:
Type: AWS::EC2::RouteTable
Properties:
VpcId: !Ref 'VPC'
PublicRoute:
Type: AWS::EC2::Route
DependsOn: GatewayAttachement
Properties:
RouteTableId: !Ref 'PublicRouteTable'
DestinationCidrBlock: '0.0.0.0/0'
GatewayId: !Ref 'InternetGateway'
PublicSubnetOneRouteTableAssociation:
Type: AWS::EC2::SubnetRouteTableAssociation
Properties:
SubnetId: !Ref PublicSubnetOne
RouteTableId: !Ref PublicRouteTable
PublicSubnetTwoRouteTableAssociation:
Type: AWS::EC2::SubnetRouteTableAssociation
Properties:
SubnetId: !Ref PublicSubnetTwo
RouteTableId: !Ref PublicRouteTable
# Setup networking resources for the private subnets. Containers
# in these subnets have only private IP addresses, and must use a NAT
# gateway to talk to the internet. We launch two NAT gateways, one for
# each private subnet.
NatGatewayOneAttachment:
Type: AWS::EC2::EIP
DependsOn: GatewayAttachement
Properties:
Domain: vpc
NatGatewayTwoAttachment:
Type: AWS::EC2::EIP
DependsOn: GatewayAttachement
Properties:
Domain: vpc
NatGatewayOne:
Type: AWS::EC2::NatGateway
Properties:
AllocationId: !GetAtt NatGatewayOneAttachment.AllocationId
SubnetId: !Ref PublicSubnetOne
NatGatewayTwo:
Type: AWS::EC2::NatGateway
Properties:
AllocationId: !GetAtt NatGatewayTwoAttachment.AllocationId
SubnetId: !Ref PublicSubnetTwo
PrivateRouteTableOne:
Type: AWS::EC2::RouteTable
Properties:
VpcId: !Ref 'VPC'
PrivateRouteOne:
Type: AWS::EC2::Route
Properties:
RouteTableId: !Ref PrivateRouteTableOne
DestinationCidrBlock: 0.0.0.0/0
NatGatewayId: !Ref NatGatewayOne
PrivateRouteTableOneAssociation:
Type: AWS::EC2::SubnetRouteTableAssociation
Properties:
RouteTableId: !Ref PrivateRouteTableOne
SubnetId: !Ref PrivateSubnetOne
PrivateRouteTableTwo:
Type: AWS::EC2::RouteTable
Properties:
VpcId: !Ref 'VPC'
PrivateRouteTwo:
Type: AWS::EC2::Route
Properties:
RouteTableId: !Ref PrivateRouteTableTwo
DestinationCidrBlock: 0.0.0.0/0
NatGatewayId: !Ref NatGatewayTwo
PrivateRouteTableTwoAssociation:
Type: AWS::EC2::SubnetRouteTableAssociation
Properties:
RouteTableId: !Ref PrivateRouteTableTwo
SubnetId: !Ref PrivateSubnetTwo
# ECS Resources
ECSCluster:
Type: AWS::ECS::Cluster
# A security group for the containers we will run in Fargate.
# Three rules, allowing network traffic from a public facing load
# balancer, a private internal load balancer, and from other members
# of the security group.
#
# Remove any of the following ingress rules that are not needed.
FargateContainerSecurityGroup:
Type: AWS::EC2::SecurityGroup
Properties:
GroupDescription: Access to the Fargate containers
VpcId: !Ref 'VPC'
EcsSecurityGroupIngressFromPublicALB:
Type: AWS::EC2::SecurityGroupIngress
Properties:
Description: Ingress from the public ALB
GroupId: !Ref 'FargateContainerSecurityGroup'
IpProtocol: -1
SourceSecurityGroupId: !Ref 'PublicLoadBalancerSG'
EcsSecurityGroupIngressFromPrivateALB:
Type: AWS::EC2::SecurityGroupIngress
Properties:
Description: Ingress from the private ALB
GroupId: !Ref 'FargateContainerSecurityGroup'
IpProtocol: -1
SourceSecurityGroupId: !Ref 'PrivateLoadBalancerSG'
EcsSecurityGroupIngressFromSelf:
Type: AWS::EC2::SecurityGroupIngress
Properties:
Description: Ingress from other containers in the same security group
GroupId: !Ref 'FargateContainerSecurityGroup'
IpProtocol: -1
SourceSecurityGroupId: !Ref 'FargateContainerSecurityGroup'
# Load balancers for getting traffic to containers.
# This sample template creates two load balancers:
#
# - One public load balancer, hosted in public subnets that is accessible
# to the public, and is intended to route traffic to one or more public
# facing services.
# - One private load balancer, hosted in private subnets, that only
# accepts traffic from other containers in the Fargate cluster, and is
# intended for private services that should not be accessed directly
# by the public.
# A public facing load balancer, this is used for accepting traffic from the public
# internet and directing it to public facing microservices
PublicLoadBalancerSG:
Type: AWS::EC2::SecurityGroup
Properties:
GroupDescription: Access to the public facing load balancer
VpcId: !Ref 'VPC'
SecurityGroupIngress:
# Allow access to ALB from anywhere on the internet
- CidrIp: 0.0.0.0/0
IpProtocol: -1
PublicLoadBalancer:
Type: AWS::ElasticLoadBalancingV2::LoadBalancer
DependsOn: GatewayAttachement
Properties:
Scheme: internet-facing
LoadBalancerAttributes:
- Key: idle_timeout.timeout_seconds
Value: '30'
Subnets:
# The load balancer is placed into the public subnets, so that traffic
# from the internet can reach the load balancer directly via the internet gateway
- !Ref PublicSubnetOne
- !Ref PublicSubnetTwo
SecurityGroups: [!Ref 'PublicLoadBalancerSG']
# A dummy target group is used to setup the ALB to just drop traffic
# initially, before any real service target groups have been added.
DummyTargetGroupPublic:
Type: AWS::ElasticLoadBalancingV2::TargetGroup
Properties:
HealthCheckIntervalSeconds: 6
HealthCheckPath: /
HealthCheckProtocol: HTTP
HealthCheckTimeoutSeconds: 5
HealthyThresholdCount: 2
Name: !Join ['-', [!Ref 'AWS::StackName', 'drop-1']]
Port: 80
Protocol: HTTP
UnhealthyThresholdCount: 2
VpcId: !Ref 'VPC'
PublicLoadBalancerListener:
Type: AWS::ElasticLoadBalancingV2::Listener
DependsOn:
- PublicLoadBalancer
Properties:
DefaultActions:
- TargetGroupArn: !Ref 'DummyTargetGroupPublic'
Type: 'forward'
LoadBalancerArn: !Ref 'PublicLoadBalancer'
Port: 80
Protocol: HTTP
# An internal load balancer, this would be used for a service that is not
# directly accessible to the public, but instead should only receive traffic
# from your other services.
PrivateLoadBalancerSG:
Type: AWS::EC2::SecurityGroup
Properties:
GroupDescription: Access to the internal load balancer
VpcId: !Ref 'VPC'
PrivateLoadBalancerIngressFromECS:
Type: AWS::EC2::SecurityGroupIngress
Properties:
Description: Only accept traffic from a container in the fargate container security group
GroupId: !Ref 'PrivateLoadBalancerSG'
IpProtocol: -1
SourceSecurityGroupId: !Ref 'FargateContainerSecurityGroup'
PrivateLoadBalancer:
Type: AWS::ElasticLoadBalancingV2::LoadBalancer
Properties:
Scheme: internal
LoadBalancerAttributes:
- Key: idle_timeout.timeout_seconds
Value: '30'
Subnets:
# This load balancer is put into the private subnet, so that there is no
# route for the public to even be able to access the private load balancer.
- !Ref PrivateSubnetOne
- !Ref PrivateSubnetTwo
SecurityGroups: [!Ref 'PrivateLoadBalancerSG']
# This dummy target group is used to setup the ALB to just drop traffic
# initially, before any real service target groups have been added.
DummyTargetGroupPrivate:
Type: AWS::ElasticLoadBalancingV2::TargetGroup
Properties:
HealthCheckIntervalSeconds: 6
HealthCheckPath: /
HealthCheckProtocol: HTTP
HealthCheckTimeoutSeconds: 5
HealthyThresholdCount: 2
Name: !Join ['-', [!Ref 'AWS::StackName', 'drop-2']]
Port: 80
Protocol: HTTP
UnhealthyThresholdCount: 2
VpcId: !Ref 'VPC'
PrivateLoadBalancerListener:
Type: AWS::ElasticLoadBalancingV2::Listener
DependsOn:
- PrivateLoadBalancer
Properties:
DefaultActions:
- TargetGroupArn: !Ref 'DummyTargetGroupPrivate'
Type: 'forward'
LoadBalancerArn: !Ref 'PrivateLoadBalancer'
Port: 80
Protocol: HTTP
# This is an IAM role which authorizes ECS to manage resources on your
# account on your behalf, such as updating your load balancer with the
# details of where your containers are, so that traffic can reach your
# containers.
ECSRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Statement:
- Effect: Allow
Principal:
Service: [ecs.amazonaws.com]
Action: ['sts:AssumeRole']
Path: /
Policies:
- PolicyName: ecs-service
PolicyDocument:
Statement:
- Effect: Allow
Action:
# Rules which allow ECS to attach network interfaces to instances
# on your behalf in order for awsvpc networking mode to work right
- 'ec2:AttachNetworkInterface'
- 'ec2:CreateNetworkInterface'
- 'ec2:CreateNetworkInterfacePermission'
- 'ec2:DeleteNetworkInterface'
- 'ec2:DeleteNetworkInterfacePermission'
- 'ec2:Describe*'
- 'ec2:DetachNetworkInterface'
# Rules which allow ECS to update load balancers on your behalf
# with the information sabout how to send traffic to your containers
- 'elasticloadbalancing:DeregisterInstancesFromLoadBalancer'
- 'elasticloadbalancing:DeregisterTargets'
- 'elasticloadbalancing:Describe*'
- 'elasticloadbalancing:RegisterInstancesWithLoadBalancer'
- 'elasticloadbalancing:RegisterTargets'
Resource: '*'
# This is a role which is used by the ECS tasks themselves.
ECSTaskExecutionRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Statement:
- Effect: Allow
Principal:
Service: [ecs-tasks.amazonaws.com]
Action: ['sts:AssumeRole']
Path: /
Policies:
- PolicyName: AmazonECSTaskExecutionRolePolicy
PolicyDocument:
Statement:
- Effect: Allow
Action:
# Allow the ECS Tasks to download images from ECR
- 'ecr:GetAuthorizationToken'
- 'ecr:BatchCheckLayerAvailability'
- 'ecr:GetDownloadUrlForLayer'
- 'ecr:BatchGetImage'
# Allow the ECS tasks to upload logs to CloudWatch
- 'logs:CreateLogStream'
- 'logs:PutLogEvents'
Resource: '*'
# These are the values output by the CloudFormation template. Be careful
# about changing any of them, because of them are exported with specific
# names so that the other task related CF templates can use them.
Outputs:
ClusterName:
Description: The name of the ECS cluster
Value: !Ref 'ECSCluster'
Export:
Name: !Join [ ':', [ !Ref 'AWS::StackName', 'ClusterName' ] ]
InternalUrl:
Description: The url of the internal load balancer
Value: !Join ['', ['http://', !GetAtt 'PrivateLoadBalancer.DNSName']]
Export:
Name: !Join [ ':', [ !Ref 'AWS::StackName', 'InternalUrl' ] ]
ExternalUrl:
Description: The url of the external load balancer
Value: !Join ['', ['http://', !GetAtt 'PublicLoadBalancer.DNSName']]
Export:
Name: !Join [ ':', [ !Ref 'AWS::StackName', 'ExternalUrl' ] ]
ECSRole:
Description: The ARN of the ECS role
Value: !GetAtt 'ECSRole.Arn'
Export:
Name: !Join [ ':', [ !Ref 'AWS::StackName', 'ECSRole' ] ]
ECSTaskExecutionRole:
Description: The ARN of the ECS role
Value: !GetAtt 'ECSTaskExecutionRole.Arn'
Export:
Name: !Join [ ':', [ !Ref 'AWS::StackName', 'ECSTaskExecutionRole' ] ]
PublicListener:
Description: The ARN of the public load balancer's Listener
Value: !Ref PublicLoadBalancerListener
Export:
Name: !Join [ ':', [ !Ref 'AWS::StackName', 'PublicListener' ] ]
PrivateListener:
Description: The ARN of the private load balancer's Listener
Value: !Ref PrivateLoadBalancerListener
Export:
Name: !Join [ ':', [ !Ref 'AWS::StackName', 'PrivateListener' ] ]
VPCId:
Description: The ID of the VPC that this stack is deployed in
Value: !Ref 'VPC'
Export:
Name: !Join [ ':', [ !Ref 'AWS::StackName', 'VPCId' ] ]
PublicSubnetOne:
Description: Public subnet one
Value: !Ref 'PublicSubnetOne'
Export:
Name: !Join [ ':', [ !Ref 'AWS::StackName', 'PublicSubnetOne' ] ]
PublicSubnetTwo:
Description: Public subnet two
Value: !Ref 'PublicSubnetTwo'
Export:
Name: !Join [ ':', [ !Ref 'AWS::StackName', 'PublicSubnetTwo' ] ]
PrivateSubnetOne:
Description: Private subnet one
Value: !Ref 'PrivateSubnetOne'
Export:
Name: !Join [ ':', [ !Ref 'AWS::StackName', 'PrivateSubnetOne' ] ]
PrivateSubnetTwo:
Description: Private subnet two
Value: !Ref 'PrivateSubnetTwo'
Export:
Name: !Join [ ':', [ !Ref 'AWS::StackName', 'PrivateSubnetTwo' ] ]
FargateContainerSecurityGroup:
Description: A security group used to allow Fargate containers to receive traffic
Value: !Ref 'FargateContainerSecurityGroup'
Export:
Name: !Join [ ':', [ !Ref 'AWS::StackName', 'FargateContainerSecurityGroup' ] ]
\ No newline at end of file
provider/indexer-aws/CloudFormation/Automated/fargate-task.yml deleted 100644 → 0
+ 0
− 151
View file @ c41456f9
AWSTemplateFormatVersion: '2010-09-09'
Description: Deploy a service on AWS Fargate, hosted in a private subnet, behind a private load balancer.
Parameters:
StackName:
Type: String
Default: production
Description: The name of the parent Fargate networking stack that you created. Necessary
to locate and reference resources created by that stack.
ServiceName:
Type: String
Default: nginx
Description: A name for the service
ImageUrl:
Type: String
Default: nginx
Description: The url of a docker image that contains the application process that
will handle the traffic for this service
ContainerPort:
Type: Number
Default: 80
Description: What port number the application inside the docker container is binding to
ContainerCpu:
Type: Number
Default: 256
Description: How much CPU to give the container. 1024 is 1 CPU
ContainerMemory:
Type: Number
Default: 512
Description: How much memory in megabytes to give the container
Path:
Type: String
Default: "*"
Description: A path on the public load balancer that this service
should be connected to. Use * to send all load balancer
traffic to this service.
Priority:
Type: Number
Default: 1
Description: The priority for the routing rule added to the load balancer.
This only applies if your have multiple services which have been
assigned to different paths on the load balancer.
DesiredCount:
Type: Number
Default: 2
Description: How many copies of the service task to run
Role:
Type: String
Default: ""
Description: (Optional) An IAM role to give the service's containers if the code within needs to
access other AWS resources like S3 buckets, DynamoDB tables, etc
Conditions:
HasCustomRole: !Not [ !Equals [!Ref 'Role', ''] ]
Resources:
# The task definition. This is a simple metadata description of what
# container to run, and what resource requirements it has.
TaskDefinition:
Type: AWS::ECS::TaskDefinition
Properties:
Family: !Ref 'ServiceName'
Cpu: !Ref 'ContainerCpu'
Memory: !Ref 'ContainerMemory'
NetworkMode: awsvpc
RequiresCompatibilities:
- FARGATE
ExecutionRoleArn:
Fn::ImportValue:
!Join [':', [!Ref 'StackName', 'ECSTaskExecutionRole']]
TaskRoleArn:
Fn::If:
- 'HasCustomRole'
- !Ref 'Role'
- !Ref "AWS::NoValue"
ContainerDefinitions:
- Name: !Ref 'ServiceName'
Cpu: !Ref 'ContainerCpu'
Memory: !Ref 'ContainerMemory'
Image: !Ref 'ImageUrl'
PortMappings:
- ContainerPort: !Ref 'ContainerPort'
# The service. The service is a resource which allows you to run multiple
# copies of a type of task, and gather up their logs and metrics, as well
# as monitor the number of running tasks and replace any that have crashed
Service:
Type: AWS::ECS::Service
DependsOn: LoadBalancerRule
Properties:
ServiceName: !Ref 'ServiceName'
Cluster:
Fn::ImportValue:
!Join [':', [!Ref 'StackName', 'ClusterName']]
LaunchType: FARGATE
DeploymentConfiguration:
MaximumPercent: 200
MinimumHealthyPercent: 75
DesiredCount: !Ref 'DesiredCount'
NetworkConfiguration:
AwsvpcConfiguration:
SecurityGroups:
- Fn::ImportValue:
!Join [':', [!Ref 'StackName', 'FargateContainerSecurityGroup']]
Subnets:
- Fn::ImportValue:
!Join [':', [!Ref 'StackName', 'PrivateSubnetOne']]
- Fn::ImportValue:
!Join [':', [!Ref 'StackName', 'PrivateSubnetTwo']]
TaskDefinition: !Ref 'TaskDefinition'
LoadBalancers:
- ContainerName: !Ref 'ServiceName'
ContainerPort: !Ref 'ContainerPort'
TargetGroupArn: !Ref 'TargetGroup'
# A target group. This is used for keeping track of all the tasks, and
# what IP addresses / port numbers they have. You can query it yourself,
# to use the addresses yourself, but most often this target group is just
# connected to an application load balancer, or network load balancer, so
# it can automatically distribute traffic across all the targets.
TargetGroup:
Type: AWS::ElasticLoadBalancingV2::TargetGroup
Properties:
HealthCheckIntervalSeconds: 6
HealthCheckPath: /
HealthCheckProtocol: HTTP
HealthCheckTimeoutSeconds: 5
HealthyThresholdCount: 2
TargetType: ip
Name: !Ref 'ServiceName'
Port: !Ref 'ContainerPort'
Protocol: HTTP
UnhealthyThresholdCount: 2
VpcId:
Fn::ImportValue:
!Join [':', [!Ref 'StackName', 'VPCId']]
# Create a rule on the load balancer for routing traffic to the target group
LoadBalancerRule:
Type: AWS::ElasticLoadBalancingV2::ListenerRule
Properties:
Actions:
- TargetGroupArn: !Ref 'TargetGroup'
Type: 'forward'
Conditions:
- Field: path-pattern
Values: [!Ref 'Path']
ListenerArn:
Fn::ImportValue:
!Join [':', [!Ref 'StackName', 'PrivateListener']]
Priority: !Ref 'Priority'
\ No newline at end of file
provider/indexer-aws/indexer-queue-aws/Dockerfile deleted 100644 → 0
+ 0
− 6
View file @ c41456f9
FROM amazoncorretto:8
FROM maven:3.5-jdk-8 AS build
COPY src /usr/src/app/src
COPY pom.xml /usr/src/app
RUN mvn -f /usr/src/app/pom.xml clean package
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment