diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 39767ed0de87222e2e1cf9da7617d95a9c417d66..89cdc6c7ab67f2c97f8303912361f7712dab5218 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,6 +1,7 @@
variables:
- AWS_BUILD_SUBDIR: provider/indexer-aws
- AWS_APPLICATION_NAME: os-indexer
+ AWS_BUILD_SUBDIR: provider/indexer-aws/build-aws
+ AWS_TEST_SUBDIR: testing/indexer-test-aws
+ AWS_SERVICE: indexer
AWS_ENVIRONMENT: dev
GCP_BUILD_SUBDIR: provider/indexer-gcp
@@ -20,12 +21,21 @@ variables:
include:
- project: "osdu/platform/ci-cd-pipelines"
file: "standard-setup.yml"
+
- project: "osdu/platform/ci-cd-pipelines"
file: "build/maven.yml"
- - project: "osdu/platform/ci-cd-pipelines"
- ref: "master"
- file: "cloud-providers/azure.yml"
+
- project: "osdu/platform/ci-cd-pipelines"
file: "scanners/fossa.yml"
+
- project: "osdu/platform/ci-cd-pipelines"
file: "scanners/gitlab-ultimate.yml"
+
+ - project: "osdu/platform/ci-cd-pipelines"
+ file: "cloud-providers/aws.yml"
+
+ - project: "osdu/platform/ci-cd-pipelines"
+ file: "publishing/pages.yml"
+
+aws-test:
+ tags: ['aws-internal-test']
diff --git a/provider/indexer-aws/.env.template b/provider/indexer-aws/.env.template
index b55ecdc872088111f1c3bb74f33a75bfb4af3253..c162a98fb2867cc81f200289a3d446c57e918112 100644
--- a/provider/indexer-aws/.env.template
+++ b/provider/indexer-aws/.env.template
@@ -56,6 +56,7 @@ SNS_TOPIC_NAME=
SNS_STORAGE_TOPIC_NAME=
ENVIRONMENT=
AWS_REGION=
+LOG_LEVEL=
##### Integration test-specific - these are only used for integration tests, not the app ###
OTHER_RELEVANT_DATA_COUNTRIES=
diff --git a/provider/indexer-aws/CloudFormation/Automated/cache.yml b/provider/indexer-aws/CloudFormation/Automated/cache.yml
deleted file mode 100644
index 207f9815a887994d91731ced4215c00403587394..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/CloudFormation/Automated/cache.yml
+++ /dev/null
@@ -1,205 +0,0 @@
-# Copyright © Amazon Web Services
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-AWSTemplateFormatVersion: 2010-09-09
-Description: >-
- CloudFormation template for creating the resources used for the tenant info database for OSDU.
- It creates the DynamoDB table and the API Gateway endpoints.
-
-Parameters:
- Environment:
- Description: An environment name that will be prefixed to resource names.
- Type: String
- AllowedValues:
- - dev
- - uat
- - prod
- ConstraintDescription: Can only be "dev/uat/prod"
- Default: dev
-
- Region:
- Description: The AWS region to deploy the resources to.
- Type: String
- Default: us-east-1
-
- ApplicationName:
- Description: >
- The name of the application, which will be used to generate the ECS cluster name.
- It will be prefixed with the environment name.
- Type: String
- MinLength: '1'
- MaxLength: '64'
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: os-indexer
-
- CacheName:
- Description: The name of the cache cluster. Will be prefixed with the environment name.
- Type: String
- MinLength: '1'
- MaxLength: '64'
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: cache
-
- CacheEngine:
- Description: Which caching platform to use. Can be set to 'redis' or 'memcached'.
- Type: String
- AllowedValues:
- - redis
- - memcached
- ConstraintDescription: Can only be "redis" or "memcached"
- Default: redis
-
- NodeInstanceType:
- Description: The instance type for redis cache nodes.
- ConstraintDescription: Must be a valid instance type from the list of allowed values.
- Default: cache.t2.micro
- AllowedValues:
- - cache.m5.large
- - cache.m5.xlarge
- - cache.m5.2xlarge
- - cache.m5.4xlarge
- - cache.m5.12xlarge
- - cache.m5.24xlarge
- - cache.m4.large
- - cache.m4.xlarge
- - cache.m4.2xlarge
- - cache.m4.4xlarge
- - cache.m4.10xlarge
- - cache.t2.micro
- - cache.t2.small
- - cache.t2.medium
- - cache.c1.xlarge
- - cache.r5.large
- - cache.r5.xlarge
- - cache.r5.2xlarge
- - cache.r5.4xlarge
- - cache.r5.12xlarge
- - cache.r5.24xlarge
- - cache.r4.large
- - cache.r4.xlarge
- - cache.r4.2xlarge
- - cache.r4.4xlarge
- - cache.r4.8xlarge
- - cache.r4.16xlarge
- Type: String
-
- NumberOfCacheNodes:
- Description: An integer value specifying the number of node in the redis cache.
- Type: Number
- Default: 1
- MinValue: 1
- MaxValue: 128
-
-Conditions:
- IsSingleNode: !Equals [ !Ref NumberOfCacheNodes, 1 ]
-
- IsClustered: !Not [Condition: IsSingleNode]
-
- IsMemcached: !Equals [ !Ref CacheEngine, memcached ]
-
- IsRedis: !Equals [ !Ref CacheEngine, redis ]
-
-Resources:
- ElastiCacheVpcSecurityGroup:
- Type: AWS::EC2::SecurityGroup
- Properties:
- GroupName: !Sub "${Environment}-${CacheName}-sg"
- GroupDescription: "This is the security group that all of our ElastiCache cluster will be placed into."
- VpcId:
- Fn::ImportValue:
- !Sub "${Environment}-OSDU-VPC"
-
- ElastiCacheVpcSecurityGroupCodeBuildIngress:
- Type: AWS::EC2::SecurityGroupIngress
- Properties:
- GroupId: !Ref ElastiCacheVpcSecurityGroup
- IpProtocol: tcp
- FromPort: "6379"
- ToPort: "6379"
- SourceSecurityGroupId:
- Fn::ImportValue:
- !Sub "${Environment}-OSDU-CodeBuildSecurityGroup"
-
- ElastiCacheVpcSecurityGroupECSIngress:
- Type: AWS::EC2::SecurityGroupIngress
- Properties:
- GroupId: !Ref ElastiCacheVpcSecurityGroup
- IpProtocol: tcp
- FromPort: "6379"
- ToPort: "6379"
- SourceSecurityGroupId:
- Fn::ImportValue:
- !Sub "${Environment}-${ApplicationName}-EcsNetworkSecurityGroupId"
-
- ElastiCacheSubnetGroup:
- Type: 'AWS::ElastiCache::SubnetGroup'
- Properties:
- CacheSubnetGroupName: !Sub ${Environment}-${CacheName}-SubnetGroup
- Description: Redis cache VPC subnet group.
- SubnetIds:
- - Fn::ImportValue:
- !Sub "${Environment}-OSDU-PrivateSubnet-AZ1"
- - Fn::ImportValue:
- !Sub "${Environment}-OSDU-PrivateSubnet-AZ2"
-
- ElastiCacheCluster:
- Type: 'AWS::ElastiCache::CacheCluster'
- DependsOn: ElastiCacheSubnetGroup
- Properties:
- AutoMinorVersionUpgrade: 'false'
- AZMode: single-az # this parameter only affects Memcached clusters
- Engine: !Ref CacheEngine
- CacheNodeType: !Ref NodeInstanceType
- NumCacheNodes: !Ref NumberOfCacheNodes
- ClusterName: !Sub ${Environment}-${CacheName}
- CacheSubnetGroupName: !Ref ElastiCacheSubnetGroup
- VpcSecurityGroupIds:
- - Ref: ElastiCacheVpcSecurityGroup
-
-Outputs:
- # Redis (cluster mode disabled) replication groups don't have this attribute.
- # Therefore, Fn::GetAtt returns a value for this attribute only if the replication
- # group is clustered. Otherwise, Fn::GetAtt fails.
- ElastiCacheConfigurationEndpointUrl:
- Description: The configuration endpoint URL of the cache node.
- Value: !GetAtt ElastiCacheCluster.ConfigurationEndpoint.Address
- Condition: IsClustered
- Export:
- Name: !Sub ${Environment}-${CacheName}-ElastiCacheConfigurationEndpointUrl
-
- # This output is only applicable if the cache engine is set to Memcached
- MemcachedConfigurationEndpointPort:
- Description: The Memcached configuration endpoint port of the cache node.
- Value: !GetAtt ElastiCacheCluster.ConfigurationEndpoint.Port
- Condition: IsMemcached
- Export:
- Name: !Sub ${Environment}-${CacheName}-MemcachedConfigurationEndpointPort
-
- # This output is only applicable if the cache engine is set to Redis
- RedisEndpointAddress:
- Description: The Redis endpoint address of the cache.
- Value: !GetAtt ElastiCacheCluster.RedisEndpoint.Address
- Condition: IsRedis
- Export:
- Name: !Sub ${Environment}-${CacheName}-RedisEndpointAddress
-
- # This output is only applicable if the cache engine is set to Redis
- RedisEndpointPort:
- Description: The Redis endpoint port of the cache.
- Value: !GetAtt ElastiCacheCluster.RedisEndpoint.Port
- Condition: IsRedis
- Export:
- Name: !Sub ${Environment}-${CacheName}-RedisEndpointPort
diff --git a/provider/indexer-aws/CloudFormation/Automated/ecs-cluster.yml b/provider/indexer-aws/CloudFormation/Automated/ecs-cluster.yml
deleted file mode 100644
index 3a357d30c1edaedc60a366f266552ee354850c38..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/CloudFormation/Automated/ecs-cluster.yml
+++ /dev/null
@@ -1,764 +0,0 @@
-# Copyright © Amazon Web Services
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-AWSTemplateFormatVersion: 2010-09-09
-Description: >-
- CloudFormation template for creating the resources used for the ECS cluster the application will
- be deployed into. Will create the CodeDeploy application, the ECR repository, and the ECS cluster.
- This is separated from the rest of the ECS resources in order to avoid a circular dependency.
- Because there can be any number of cache stacks, but only one ECS stack per service, it makes sense to have
- the caches import ECS exports in order to permit access from ECS, rather than the other way around, since
- the number of cache clusters and their names can vary, and would require hardcoding them into the ECS template,
- whereas this way things stay generic and the ECS CloudFormation template does not need to be updated in order to
- add or remove ElastiCache clusters.
-
-Parameters:
- Environment:
- Description: An environment name that will be prefixed to resource names.
- Type: String
- AllowedValues:
- - dev
- - uat
- - prod
- ConstraintDescription: Can only be "dev/uat/prod"
- Default: dev
-
- Region:
- Description: The AWS region to deploy the resources to.
- Type: String
- Default: us-east-1
-
- ApplicationName:
- Description: >
- The name of the application, which will be used to generate the ECS cluster name.
- It will be prefixed with the environment name.
- Type: String
- MinLength: '1'
- MaxLength: '64'
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: os-indexer
-
- KeyName:
- Description: >
- Name of an existing EC2 KeyPair to enable SSH access to the ECS instances. Note that key pairs cannot
- be created through CloudFormation, but instead must be uploaded through the AWS Console.
- Type: AWS::EC2::KeyPair::KeyName
- Default: ecs_indexer_key
-
- DesiredCapacity:
- Description: The default number of instances to launch in the ECS cluster.
- Type: Number
- Default: '1'
-
- MaxSize:
- Description: Maximum number of instances that can be launched in the ECS cluster.
- Type: Number
- Default: '1'
-
- InstanceType:
- Description: EC2 instance type
- Type: String
- Default: t3.large
- AllowedValues:
- - m5.large
- - m5.xlarge
- - m5.2xlarge
- - m5.4xlarge
- - m5.12xlarge
- - m5.16xlarge
- - m5.24xlarge
- - m4.large
- - m4.xlarge
- - m4.2xlarge
- - m4.4xlarge
- - m4.10xlarge
- - m4.16xlarge
- - t3.nano
- - t3.micro
- - t3.small
- - t3.medium
- - t3.large
- - t3.xlarge
- - t3.2xlarge
- - c5.large
- - c5.xlarge
- - c5.2xlarge
- - c5.4xlarge
- - c5.12xlarge
- - c5.16xlarge
- - c5.24xlarge
- - r5.large
- - r5.xlarge
- - r5.2xlarge
- - r5.4xlarge
- - r5.12xlarge
- - r5.24xlarge
- - r4.large
- - r4.xlarge
- - r4.2xlarge
- - r4.4xlarge
- - r4.8xlarge
- - r4.16xlarge
- - i3.large
- - i3.xlarge
- - i3.2xlarge
- - i3.4xlarge
- - i3.10xlarge
- - i3.16xlarge
- - x1e.xlarge
- - x1e.2xlarge
- - x1e.4xlarge
- - x1e.8xlarge
- - x1e.16xlarge
- - x1e.32xlarge
- ConstraintDescription: Please choose a valid EC2 instance type for the ECS container instances.
-
- SchemaCacheName:
- Description: The name of the cache cluster for the schema cache. Will be prefixed with the environment name.
- Type: String
- MinLength: '1'
- MaxLength: '64'
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: indexerSchemaCache
-
- IndexCacheName:
- Description: The name of the cache cluster for the index cache. Will be prefixed with the environment name.
- Type: String
- MinLength: '1'
- MaxLength: '64'
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: indexerIndexCache
-
- ECSPort:
- Description: The port that the ECS Service will listen on.
- Type: Number
- Default: 80
- MinValue: 1
- MaxValue: 65535
-
- SNSTopicName:
- Description: >-
- The name of the Simple Notification Service topic for the OS Indexer Service. Defaults to osdu-indexer-messages.
- Will be prefixed with the environment name.
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: osdu-indexer-messages
- Type: String
- MinLength: '1'
- MaxLength: '64'
-
- ECSCPUAllocation:
- Description: The amount of CPU resources to allocate to each ECS task/container. Scale - 1024 = 1 vCPU core.
- Type: Number
- Default: 1024
- MinValue: 10
- MaxValue: 65535
-
- ECSMemoryAllocation:
- Description: The amount of memory (RAM) to allocate to each ECS task/container. Scale - 1 = 1MB of memory.
- Type: Number
- Default: 2048
- MinValue: 256
- MaxValue: 131072
-
- DomainName:
- Description: >-
- The optional custom DNS name for the ECS service's load balancer. If omitted, the site will only be accessible
- via the ECS service's Application Load Balancer DNS name. This value is used in the creation and signing of
- the service's SSL certificate. Leave blank is not using a custom domain for this deployment.
- Type: String
- Default: ''
-
- HostedZoneName:
- Description: >-
- The name of the hosted zone (ex: for indexer.osdu.slb.com, this would likely be osdu.slb.com).
- Leave blank is not using a custom domain for this deployment.
- Type: String
- Default: ''
-
- ElasticsearchDomainName:
- Description: The name of the Elasticsearch domain. Will be prefixed with the environment name.
- Type: String
- MinLength: '1'
- MaxLength: '64'
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: osdu-indexer
-
-Mappings:
- # This mapping is for the ECS-optimized edition of the November 13-14, 2019 release of the Amazon Linux 2 AMI
- # It will need to be periodically updated as new versions are released by Amazon.
- # The latest ECS-optimized AMI IDs can be found here: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs-optimized_AMI.html
- # The mapping is used to input the correct AMI ID based on the region the instance is being spun up in.
- AWSRegionToAMI:
- us-east-1:
- AMIID: ami-097e3d1cdb541f43e
- us-east-2:
- AMIID: ami-0fbd313043845c4f2
- us-west-1:
- AMIID: ami-03d7632ea0ab75eaa
- us-west-2:
- AMIID: ami-0fb71e703258ab7eb
- eu-north-1:
- AMIID: ami-0f8edbbca6bac13a6
- eu-west-1:
- AMIID: ami-0bf45a5f4ab05b949
- eu-west-2:
- AMIID: ami-0393b5f363fbd613a
- eu-west-3:
- AMIID: ami-03490ca40775a62f0
- eu-central-1:
- AMIID: ami-074dc9dd588b6ea52
- ap-northeast-1:
- AMIID: ami-0934e28fe3e390537
- ap-northeast-2:
- AMIID: ami-0fa5d85859452a178
- ap-south-1:
- AMIID: ami-0312d67ff59a3db34
- ap-southeast-1:
- AMIID: ami-01f07b3fa86406c96
- ap-southeast-2:
- AMIID: ami-07610e278b1ddf331
- ca-central-1:
- AMIID: ami-0057d82f917a17334
- sa-east-1:
- AMIID: ami-0c947c117562538ee
-
-Conditions:
- IncludeCustomDomain: !Not [!Equals [ !Ref DomainName, '' ]]
- IsPortStandardSSL:
- !Or [!Equals [ !Ref ECSPort, '443' ], !Equals [ !Ref ECSPort, '8443' ]]
- IsLoadBalancerHTTPS: !And # HTTPS for ECS requires a custom domain, but CloudFront will still have HTTPS/SSL
- - !Condition IncludeCustomDomain
- - !Condition IsPortStandardSSL
-
-Resources:
- # This sets up a Route 53 record for CloudFront if a custom domain is being used,
- # otherwise a default cloudfront.net value will be used instead
- CloudFrontDNSName:
- Type: AWS::Route53::RecordSetGroup
- Condition: IncludeCustomDomain
- Properties:
- HostedZoneName: !Join ['', [!Ref HostedZoneName, .]] # Route 53 requires a trailing period
- RecordSets:
- - Name: !Ref DomainName
- Type: A
- AliasTarget:
- # This hosted zone ID is for ALL CloudFront distributions, always, and should be hard-coded
- HostedZoneId: Z2FDTNDATAQYW2
- DNSName: !GetAtt ECSCloudFrontDistribution.DomainName
-
- # This sets up a Route 53 record for the ECS ALB origin if a custom domain is being used
- ECSDNSName:
- Type: AWS::Route53::RecordSetGroup
- Condition: IncludeCustomDomain
- Properties:
- HostedZoneName: !Join ['', [!Ref HostedZoneName, .]] # Route 53 requires a trailing period
- RecordSets:
- - Name: !Join ['.', ['origin', !Ref DomainName]] # prefix the ECS origin record with 'origin.'
- Type: A
- AliasTarget:
- HostedZoneId: !GetAtt ECSALB.CanonicalHostedZoneID # this value comes from the ALB attributes
- DNSName: !GetAtt ECSALB.DNSName
- EvaluateTargetHealth: true # Route 53 routes traffic to ECS targets based on their health checks
- DependsOn: ECSALB
-
- CodeDeployApplication:
- Type: AWS::CodeDeploy::Application
- Properties:
- ApplicationName: !Sub ${Environment}-${ApplicationName}-code-deploy
- ComputePlatform: ECS
-
- ECRRepository:
- Type: AWS::ECR::Repository
- Properties:
- RepositoryName: !Sub ${Environment}-${ApplicationName}-repository
- RepositoryPolicyText:
- Version: "2012-10-17"
- Statement:
- - Sid: AllowPushPull
- Effect: Allow
- Principal:
- AWS:
- - !Sub arn:aws:iam::${AWS::AccountId}:root
- - Fn::ImportValue:
- !Sub "${Environment}-CodeBuildRoleArn"
- - Fn::ImportValue:
- !Sub "${Environment}-CFNRoleArn"
- - Fn::ImportValue:
- !Sub "${Environment}-PipelineRoleArn"
- Service:
- - codebuild.amazonaws.com
- Action:
- - "ecr:GetDownloadUrlForLayer"
- - "ecr:BatchGetImage"
- - "ecr:BatchCheckLayerAvailability"
- - "ecr:PutImage"
- - "ecr:InitiateLayerUpload"
- - "ecr:UploadLayerPart"
- - "ecr:CompleteLayerUpload"
-
- ApplicationECSCluster:
- Type: AWS::ECS::Cluster
- Properties:
- ClusterName: !Sub ${Environment}-${ApplicationName}-cluster
- Tags:
- - Key: Environment
- Value: !Ref Environment
-
- CloudWatchLogsGroup:
- Type: AWS::Logs::LogGroup
- Properties:
- LogGroupName: !Join ['-', [ECSLogGroup, !Ref 'ApplicationName']]
- RetentionInDays: 365
-
- TaskDefinition:
- Type: AWS::ECS::TaskDefinition
- Properties:
- Family: !Join ['', [!Ref 'AWS::StackName', -, !Ref 'ApplicationName']]
- ContainerDefinitions:
- - Name: !Ref 'ApplicationName'
- Cpu: !Ref ECSCPUAllocation
- Essential: 'true'
- Image: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${Environment}-${ApplicationName}-repository:latest
- Memory: !Ref ECSMemoryAllocation
- LogConfiguration:
- LogDriver: awslogs
- Options:
- awslogs-group: !Ref 'CloudWatchLogsGroup'
- awslogs-region: !Ref 'AWS::Region'
- awslogs-stream-prefix: !Ref 'ApplicationName'
- MountPoints:
- - ContainerPath: /root/.m2
- SourceVolume: docker-volume
- PortMappings:
- - ContainerPort: !Ref ECSPort
- Environment:
- - Name: AWS_ACCESS_KEY_ID
- Value: '{{resolve:secretsmanager:dev-IndexerServiceIamCredentials:SecretString:access_key}}'
- - Name: AWS_SECRET_KEY
- Value: '{{resolve:secretsmanager:dev-IndexerServiceIamCredentials:SecretString:secret_key}}'
- - Name: ENVIRONMENT
- Value: !Ref Environment
- - Name: VSTS_FEED_USER
- Value: '{{resolve:secretsmanager:dev-VSTSFeedToken:SecretString:vsts_feed_user}}'
- - Name: VSTS_FEED_TOKEN
- Value: '{{resolve:secretsmanager:dev-VSTSFeedToken:SecretString:vsts_feed_token}}'
- - Name: CACHE_CLUSTER_SCHEMA_ENDPOINT
- Value:
- Fn::ImportValue:
- !Sub "${Environment}-${SchemaCacheName}-RedisEndpointAddress"
- - Name: CACHE_CLUSTER_SCHEMA_PORT
- Value:
- Fn::ImportValue:
- !Sub "${Environment}-${SchemaCacheName}-RedisEndpointPort"
- - Name: CACHE_CLUSTER_INDEX_ENDPOINT
- Value:
- Fn::ImportValue:
- !Sub "${Environment}-${IndexCacheName}-RedisEndpointAddress"
- - Name: CACHE_CLUSTER_INDEX_PORT
- Value:
- Fn::ImportValue:
- !Sub "${Environment}-${IndexCacheName}-RedisEndpointPort"
- - Name: APPLICATION_PORT
- Value: !Ref ECSPort
- - Name: AWS_REGION
- Value: !Ref 'AWS::Region'
- - Name: AWS_ACCOUNT_ID
- Value: !Ref 'AWS::AccountId'
- - Name: SNS_TOPIC_NAME
- Value: !Ref SNSTopicName
- - Name: ELASTIC_HOST
- Value:
- Fn::ImportValue:
- !Sub "${Environment}-${ElasticsearchDomainName}-ElasticsearchDomainEndpoint"
- - Name: ELASTIC_PORT
- Value: '443' # the Elasticsearch port is not configurable on AWS, and is always 80 for HTTP and 443 for HTTPS, so there's no value in using a CFN parameter
- - Name: JAVA_HEAP_MEMORY
- Value: !Ref ECSMemoryAllocation
- - Name: STORAGE_HOST
- Value:
- Fn::ImportValue:
- !Sub "${Environment}-os-storage-EcsCloudFrontDomainName"
- - Name: SNS_STORAGE_TOPIC_NAME
- Value:
- Fn::ImportValue:
- !Sub "${Environment}-OSDUStorageSNSTopic"
- Volumes:
- - Name: docker-volume
-
- ECSALB:
- Type: AWS::ElasticLoadBalancingV2::LoadBalancer
- Properties:
- Name: !Sub ECSALB-${ApplicationName}
- Scheme: internet-facing
- LoadBalancerAttributes:
- - Key: idle_timeout.timeout_seconds
- Value: '30'
- Subnets:
- - Fn::ImportValue:
- !Sub "${Environment}-OSDU-PublicSubnet-AZ1"
- - Fn::ImportValue:
- !Sub "${Environment}-OSDU-PublicSubnet-AZ2"
- SecurityGroups:
- - Fn::ImportValue:
- !Sub "${Environment}-${ApplicationName}-EcsNetworkSecurityGroupId"
-
- ALBListener:
- Type: AWS::ElasticLoadBalancingV2::Listener
- DependsOn: ECSServiceRole
- Properties:
- DefaultActions:
- - Type: forward
- TargetGroupArn: !Ref 'ECSTargetGroup'
- LoadBalancerArn: !Ref 'ECSALB'
- Port: !Ref ECSPort
- Protocol: !If [IsLoadBalancerHTTPS, HTTPS, HTTP]
-
- LoadBalancerALBListenerCertificate:
- Type: AWS::ElasticLoadBalancingV2::ListenerCertificate
- Condition: IncludeCustomDomain
- Properties:
- Certificates:
- - Fn::ImportValue:
- !Sub "${Environment}-${ApplicationName}-LoadBalancerSSLCertificateArn"
- ListenerArn: !Ref 'ALBListener'
-
- ECSALBPrimaryListenerRule:
- Type: AWS::ElasticLoadBalancingV2::ListenerRule
- DependsOn: ALBListener
- Properties:
- Actions:
- - Type: forward
- TargetGroupArn: !Ref 'ECSTargetGroup'
- Conditions:
- - Field: path-pattern
- Values: [/]
- ListenerArn: !Ref 'ALBListener'
- Priority: 1
-
- ECSTargetGroup:
- Type: AWS::ElasticLoadBalancingV2::TargetGroup
- DependsOn: ECSALB
- Properties:
- HealthCheckIntervalSeconds: 120
- HealthCheckPath: /api/indexer/v2/liveness_check
- HealthCheckProtocol: !If [IsLoadBalancerHTTPS, HTTPS, HTTP]
- HealthCheckTimeoutSeconds: 5
- HealthyThresholdCount: 2
- Name: !Sub ECSTargetGroup-New-${ApplicationName}
- Port: !Ref ECSPort
- Protocol: !If [IsLoadBalancerHTTPS, HTTPS, HTTP]
- UnhealthyThresholdCount: 2
- VpcId:
- Fn::ImportValue:
- !Sub "${Environment}-OSDU-VPC"
-
- ECSCloudFrontDistribution:
- Type: AWS::CloudFront::Distribution
- DependsOn: ECSALB
- Properties:
- DistributionConfig:
- Comment: 'Cloudfront Distribution pointing ALB Origin'
- Origins:
- - DomainName: !GetAtt 'ECSALB.DNSName'
- Id: !Ref 'ECSALB'
- CustomOriginConfig:
- HTTPPort: !Ref ECSPort # The ports are the same because we'll only ever be accessing the ECS cluster over one protocol, as set in OriginProtocolPolicy below
- HTTPSPort: !Ref ECSPort # The ports are the same because we'll only ever be accessing the ECS cluster over one protocol, as set in OriginProtocolPolicy below
- OriginProtocolPolicy: !If [IsLoadBalancerHTTPS, https-only, http-only] # this only affects the origin, not CloudFront / the user's request
- OriginKeepaliveTimeout: '60'
- OriginReadTimeout: '60'
- OriginSSLProtocols:
- - TLSv1
- - TLSv1.1
- - TLSv1.2
- - SSLv3
- Enabled: true
- HttpVersion: 'http2'
- Aliases:
- - Fn::If:
- - IncludeCustomDomain
- - !Ref DomainName
- - !Ref AWS::NoValue
- DefaultCacheBehavior:
- AllowedMethods:
- - GET
- - HEAD
- - OPTIONS
- - PUT
- - POST
- - PATCH
- - DELETE
- Compress: true
- TargetOriginId: !Ref 'ECSALB'
- DefaultTTL: 5
- MaxTTL: 30
- ForwardedValues:
- QueryString: true
- Cookies:
- Forward: all
- Headers:
- - Authorization
- - Data-Partition-Id
- - Content-Type
- - Kind
- - Limit
- - Cursor
- ViewerProtocolPolicy: redirect-to-https # CloudFront requests will always be HTTPS, regardless of the origin or the request
- ViewerCertificate:
- AcmCertificateArn:
- Fn::If:
- - IncludeCustomDomain
- - Fn::ImportValue:
- !Sub "${Environment}-${ApplicationName}-LoadBalancerSSLCertificateArn"
- - Ref: AWS::NoValue
- CloudFrontDefaultCertificate:
- Fn::If:
- - IncludeCustomDomain
- - Ref: AWS::NoValue
- - true
- SslSupportMethod:
- Fn::If:
- - IncludeCustomDomain
- - sni-only # sni-only is free; 'vip' is the only other option, which allows viewers without Server Name Indication (SNI) support by using dedicated IP addresses, but it costs $600/mo per SSL certificate
- - Ref: AWS::NoValue
- MinimumProtocolVersion:
- Fn::If:
- - IncludeCustomDomain
- - TLSv1
- - Ref: AWS::NoValue # this is not used when using the default CloudFront certificate (which is always TLSv1)
-
- ECSAutoScalingGroup:
- Type: AWS::AutoScaling::AutoScalingGroup
- Properties:
- VPCZoneIdentifier:
- - Fn::ImportValue:
- !Sub "${Environment}-OSDU-PublicSubnet-AZ1"
- - Fn::ImportValue:
- !Sub "${Environment}-OSDU-PublicSubnet-AZ2"
- LaunchConfigurationName: !Ref 'ContainerInstances'
- MinSize: '1'
- MaxSize: !Ref 'MaxSize'
- DesiredCapacity: !Ref 'DesiredCapacity'
- CreationPolicy:
- ResourceSignal:
- Timeout: PT15M
- UpdatePolicy:
- AutoScalingReplacingUpdate:
- WillReplace: 'true'
-
- ContainerInstances:
- Type: AWS::AutoScaling::LaunchConfiguration
- Properties:
- ImageId: !FindInMap [AWSRegionToAMI, !Ref 'AWS::Region', AMIID]
- SecurityGroups:
- - Fn::ImportValue:
- !Sub "${Environment}-${ApplicationName}-EcsNetworkSecurityGroupId"
- InstanceType: !Ref 'InstanceType'
- IamInstanceProfile: !Ref 'EC2InstanceProfile'
- KeyName: !Ref 'KeyName'
- UserData:
- Fn::Base64: !Sub |
- #!/bin/bash -xe
- echo ECS_CLUSTER=${ApplicationECSCluster} >> /etc/ecs/ecs.config
- yum install -y aws-cfn-bootstrap
- /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource ECSAutoScalingGroup --region ${AWS::Region}
-
- Service:
- Type: AWS::ECS::Service
- DependsOn: ALBListener
- Properties:
- Cluster: !Ref 'ApplicationECSCluster'
- DesiredCount: '1'
- LoadBalancers:
- - ContainerName: !Ref 'ApplicationName'
- ContainerPort: !Ref ECSPort
- TargetGroupArn: !Ref 'ECSTargetGroup'
- Role: !Ref 'ECSServiceRole'
- TaskDefinition: !Ref 'TaskDefinition'
-
- ECSServiceRole:
- Type: AWS::IAM::Role
- Properties:
- AssumeRolePolicyDocument:
- Statement:
- - Effect: Allow
- Principal:
- Service: [ecs.amazonaws.com]
- Action: ['sts:AssumeRole']
- Path: /
- Policies:
- - PolicyName: !Sub ${Environment}-${ApplicationName}-ecs-service
- PolicyDocument:
- Statement:
- - Effect: Allow
- Action: ['elasticloadbalancing:DeregisterInstancesFromLoadBalancer', 'elasticloadbalancing:DeregisterTargets',
- 'elasticloadbalancing:Describe*', 'elasticloadbalancing:RegisterInstancesWithLoadBalancer',
- 'elasticloadbalancing:RegisterTargets', 'ec2:Describe*', 'ec2:AuthorizeSecurityGroupIngress']
- Resource: '*'
-
- ServiceScalingTarget:
- Type: AWS::ApplicationAutoScaling::ScalableTarget
- DependsOn: Service
- Properties:
- MaxCapacity: 2
- MinCapacity: 1
- ResourceId: !Join ['', [service/, !Ref 'ApplicationECSCluster', /, !GetAtt [Service, Name]]]
- RoleARN: !GetAtt [AutoscalingRole, Arn]
- ScalableDimension: ecs:service:DesiredCount
- ServiceNamespace: ecs
-
- ServiceScalingPolicy:
- Type: AWS::ApplicationAutoScaling::ScalingPolicy
- Properties:
- PolicyName: !Sub ScalingPolicy-${ApplicationName}
- PolicyType: StepScaling
- ScalingTargetId: !Ref 'ServiceScalingTarget'
- StepScalingPolicyConfiguration:
- AdjustmentType: PercentChangeInCapacity
- Cooldown: 60
- MetricAggregationType: Average
- StepAdjustments:
- - MetricIntervalLowerBound: 0
- ScalingAdjustment: 200
-
- ALB500sAlarmScaleUp:
- Type: AWS::CloudWatch::Alarm
- Properties:
- EvaluationPeriods: '1'
- Statistic: Average
- Threshold: '10'
- AlarmDescription: Alarm triggering ECS to scale up if our ALB generates too many HTTP 500 errors.
- Period: '60'
- AlarmActions: [!Ref 'ServiceScalingPolicy']
- Namespace: AWS/ApplicationELB
- Dimensions:
- - Name: LoadBalancer
- Value: !GetAtt
- - ECSALB
- - LoadBalancerFullName
- ComparisonOperator: GreaterThanThreshold
- MetricName: HTTPCode_ELB_5XX_Count
-
- EC2Role:
- Type: AWS::IAM::Role
- Properties:
- AssumeRolePolicyDocument:
- Statement:
- - Effect: Allow
- Principal:
- Service: [ec2.amazonaws.com]
- Action: ['sts:AssumeRole']
- Path: /
- Policies:
- - PolicyName: !Sub ${Environment}-${ApplicationName}-ecs-service
- PolicyDocument:
- Statement:
- - Effect: Allow
- Action: ['ecs:CreateCluster', 'ecs:DeregisterContainerInstance', 'ecs:DiscoverPollEndpoint',
- 'ecs:Poll', 'ecs:RegisterContainerInstance', 'ecs:StartTelemetrySession',
- 'ecs:Submit*', 'logs:CreateLogStream', 'logs:PutLogEvents', 'ecr:*']
- Resource: '*'
-
- AutoscalingRole:
- Type: AWS::IAM::Role
- Properties:
- AssumeRolePolicyDocument:
- Statement:
- - Effect: Allow
- Principal:
- Service: [application-autoscaling.amazonaws.com]
- Action: ['sts:AssumeRole']
- Path: /
- Policies:
- - PolicyName: !Sub ${Environment}-${ApplicationName}-service-autoscaling
- PolicyDocument:
- Statement:
- - Effect: Allow
- Action: ['application-autoscaling:*', 'cloudwatch:DescribeAlarms', 'cloudwatch:PutMetricAlarm',
- 'ecs:DescribeServices', 'ecs:UpdateService']
- Resource: '*'
-
- EC2InstanceProfile:
- Type: AWS::IAM::InstanceProfile
- Properties:
- Path: /
- Roles: [!Ref 'EC2Role']
-
-Outputs:
- ApplicationECSClusterArn:
- Description: The ARN of the application's ECS cluster.
- Value: !GetAtt ApplicationECSCluster.Arn
- Export:
- Name: !Sub ${Environment}-${ApplicationName}-EcsClusterArn
-
- ApplicationECSClusterName:
- Description: The logical name of the application's ECS cluster.
- Value: !Ref ApplicationECSCluster
- Export:
- Name: !Sub ${Environment}-${ApplicationName}-EcsClusterName
-
- ECSServiceArn:
- Description: The ARN of the Indexer Service service in the ECS cluster.
- Value: !Ref 'Service'
- Export:
- Name: !Sub ${Environment}-${ApplicationName}-EcsServiceArn
-
- ECSServiceName:
- Description: The name of the Legal Service service in the ECS cluster.
- Value: !GetAtt Service.Name
- Export:
- Name: !Sub ${Environment}-${ApplicationName}-EcsServiceName
-
- ECSALBUrl:
- Description: The Indexer Service ALB DNS URL.
- Value: !Join ['', [!GetAtt [ECSALB, DNSName]]]
- Export:
- Name: !Sub ${Environment}-${ApplicationName}-EcsAlbUrl
-
- ECSALBCustomDNSName:
- Description: The custom DNS name of the ECS service's ALB origin.
- Condition: IncludeCustomDomain
- Value: !Join ['.', ['origin', !Ref DomainName]]
- Export:
- Name: !Sub ${Environment}-${ApplicationName}-EcsAlbCustomDnsName
-
- ECSCloudFrontCustomDNSName:
- Description: The custom DNS name of the ECS service's CloudFront Distribution.
- Condition: IncludeCustomDomain
- Value: !Ref DomainName
- Export:
- Name: !Sub ${Environment}-${ApplicationName}-EcsCloudFrontCustomDnsName
-
- ECSCloudFrontDomainName:
- Description: The custom DNS name of the ECS service's CloudFront Distribution.
- Value: !GetAtt ECSCloudFrontDistribution.DomainName
- Export:
- Name: !Sub ${Environment}-${ApplicationName}-EcsCloudFrontDomainName
-
- TaskDefinitionArn:
- Description: The ARN of the Indexer Service ECS task definition.
- Value: !Ref 'TaskDefinition'
- Export:
- Name: !Sub ${Environment}-${ApplicationName}-EcsTaskDefinitionArn
-
- IndexerEC2RoleArn:
- Description: The ARN of the application's EC2 role.
- Value: !GetAtt EC2Role.Arn
- Export:
- Name: !Sub ${Environment}-${ApplicationName}-EC2RoleArn
diff --git a/provider/indexer-aws/CloudFormation/Automated/ecs-network.yml b/provider/indexer-aws/CloudFormation/Automated/ecs-network.yml
deleted file mode 100644
index 0fa408776e7b9a4040531b58afcafc4550e8f239..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/CloudFormation/Automated/ecs-network.yml
+++ /dev/null
@@ -1,150 +0,0 @@
-# Copyright © Amazon Web Services
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-AWSTemplateFormatVersion: 2010-09-09
-Description: >-
- CloudFormation template for creating the network resources used for the ECS cluster the application will
- be deployed into. This is separated from the rest of the ECS resources in order to avoid a circular dependency.
- Because there can be any number of cache stacks, but only one ECS stack per service, it makes sense to have
- the caches import ECS exports in order to permit access from ECS, rather than the other way around, since
- the number of cache clusters and their names can vary, and would require hardcoding them into the ECS template,
- whereas this way things stay generic and the ECS CloudFormation template does not need to be updated in order to
- add or remove ElastiCache clusters.
-
-Parameters:
- Environment:
- Description: An environment name that will be prefixed to resource names.
- Type: String
- AllowedValues:
- - dev
- - uat
- - prod
- ConstraintDescription: Can only be "dev/uat/prod"
- Default: dev
-
- Region:
- Description: The AWS region to deploy the resources to.
- Type: String
- Default: us-east-1
-
- ApplicationName:
- Description: >
- The name of the application, which will be used to generate the ECS cluster name.
- It will be prefixed with the environment name.
- Type: String
- MinLength: '1'
- MaxLength: '64'
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: os-indexer
-
- ECSPort:
- Description: The port that the ECS Service will listen on.
- Type: Number
- Default: 443
- MinValue: 1
- MaxValue: 65535
-
- DomainName:
- Description: >-
- The optional custom DNS name for the service's load balancer. If omitted, the site will only be accessible
- via the ECS service's Application Load Balancer DNS name. This value is used in the creation and signing of
- the service's SSL certificate. Leave blank for none.
- Type: String
- Default: ''
-
- AcmCertificateArn:
- Description: >-
- The Amazon Resource Name (ARN) of an existing AWS Certificate Manager (ACM) certificate.
- If omitted, a new SSL certified will be requested/generated (only if the custom domain name
- parameter is provided, otherwise the ECS service's ALB will not use SSL/HTTPS).
- Type: String
- AllowedPattern: "^(|arn:aws:acm:.*)$"
- Default: ''
-
-Conditions:
- IncludeCustomDomain: !Not [!Equals [ !Ref DomainName, '' ]]
- UseExistingACMSSLCertificate: !And
- - !Not [!Equals [ !Ref AcmCertificateArn, '' ]]
- - !Condition IncludeCustomDomain
- ShouldRequestNewSSLCertificate: !And
- - !Not [!Condition UseExistingACMSSLCertificate]
- - !Condition IncludeCustomDomain
- ShouldExportSSLCertificate: !Or
- - !Condition IncludeCustomDomain
- - !Condition UseExistingACMSSLCertificate
-
-Resources:
- # If an existing SSL certificate is not provided, but a custom domain is, request one
- LoadBalancerSSLCertificate:
- Type: 'AWS::CertificateManager::Certificate'
- Condition: ShouldRequestNewSSLCertificate
- Properties:
- DomainName: !Ref DomainName
- SubjectAlternativeNames:
- - !Join ['.', ['origin', !Ref DomainName]] #
-
- ECSSecurityGroup:
- Type: AWS::EC2::SecurityGroup
- Properties:
- GroupName: !Sub "${Environment}-${ApplicationName}-sg"
- GroupDescription: Indexer Service ECS Security Group
- VpcId:
- Fn::ImportValue:
- !Sub "${Environment}-OSDU-VPC"
-
- # Public access to the specified ECS Listening Port
- ECSSecurityGroupECSListenerInbound:
- Type: AWS::EC2::SecurityGroupIngress
- Properties:
- GroupId: !Ref 'ECSSecurityGroup'
- IpProtocol: tcp
- FromPort: !Ref ECSPort
- ToPort: !Ref ECSPort
- CidrIp: 0.0.0.0/0
-
- # SSH access for instances in our VPC's jump box subnet group
- # TODO: Update when the jump box is created as a part of the Util CFN, for now it is public
- ECSSecurityGroupSSHInbound:
- Type: AWS::EC2::SecurityGroupIngress
- Properties:
- GroupId: !Ref 'ECSSecurityGroup'
- IpProtocol: tcp
- FromPort: '22'
- ToPort: '22'
- CidrIp: 0.0.0.0/0
-
- # Open Application Load Balancer port range to self-access
- ECSSecurityGroupALBports:
- Type: AWS::EC2::SecurityGroupIngress
- Properties:
- GroupId: !Ref 'ECSSecurityGroup'
- IpProtocol: tcp
- FromPort: '31000'
- ToPort: '61000'
- SourceSecurityGroupId: !Ref 'ECSSecurityGroup'
-
-Outputs:
- EcsNetworkSecurityGroupId:
- Description: The ID of the Indexer Service ECS EC2 security group.
- Value: !Ref 'ECSSecurityGroup'
- Export:
- Name: !Sub ${Environment}-${ApplicationName}-EcsNetworkSecurityGroupId
-
- LoadBalancerSSLCertificateArn:
- Condition: ShouldExportSSLCertificate
- Description: The ARN of the SSL certificate to be used for both ECS and CloudFront (includes both DNS names).
- Value: !If [UseExistingACMSSLCertificate, !Ref AcmCertificateArn, !Ref 'LoadBalancerSSLCertificate']
- Export:
- Name: !Sub ${Environment}-${ApplicationName}-LoadBalancerSSLCertificateArn
diff --git a/provider/indexer-aws/CloudFormation/Automated/elasticsearch.yml b/provider/indexer-aws/CloudFormation/Automated/elasticsearch.yml
deleted file mode 100644
index 7a18783ae484ffcb6b248c98e131db4d28ee63a6..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/CloudFormation/Automated/elasticsearch.yml
+++ /dev/null
@@ -1,242 +0,0 @@
-# Copyright © Amazon Web Services
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-AWSTemplateFormatVersion: 2010-09-09
-Description: >-
- CloudFormation template for creating the resources used for the tenant info database for OSDU.
- It creates the DynamoDB table and the API Gateway endpoints.
-
-Parameters:
- Environment:
- Description: An environment name that will be prefixed to resource names.
- Type: String
- AllowedValues:
- - dev
- - uat
- - prod
- ConstraintDescription: Can only be "dev/uat/prod"
- Default: dev
-
- ApplicationName:
- Description: >
- The name of the application, which will be used to generate the ECS cluster name.
- It will be prefixed with the environment name.
- Type: String
- MinLength: '1'
- MaxLength: '64'
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: os-indexer
-
- SearchApplicationName:
- Description: >
- The name of the application, which will be used to generate the ECS cluster name.
- It will be prefixed with the environment name.
- Type: String
- MinLength: '1'
- MaxLength: '64'
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: os-search
-
- Region:
- Description: The AWS region to deploy the resources to.
- Type: String
- Default: us-east-1
-
- ElasticsearchDomainName:
- Description: The name of the Elasticsearch domain. Will be prefixed with the environment name.
- Type: String
- MinLength: '1'
- MaxLength: '64'
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: osdu-indexer
-
- ElasticsearchNodeInstanceType:
- Description: The instance type for the main Elasticsearch nodes.
- ConstraintDescription: Must be a valid instance type from the list of allowed values.
- Default: t2.medium.elasticsearch
- AllowedValues:
- - t2.small.elasticsearch
- - t2.medium.elasticsearch
- - m5.large.elasticsearch
- - m5.xlarge.elasticsearch
- - m5.2xlarge.elasticsearch
- - m5.4xlarge.elasticsearch
- - m5.12xlarge.elasticsearch
- - c5.large.elasticsearch
- - c5.xlarge.elasticsearch
- - c5.2xlarge.elasticsearch
- - c5.4xlarge.elasticsearch
- - c5.9xlarge.elasticsearch
- - c5.18xlarge.elasticsearch
- - r5.large.elasticsearch
- - r5.xlarge.elasticsearch
- - r5.2xlarge.elasticsearch
- - r5.4xlarge.elasticsearch
- - r5.12xlarge.elasticsearch
- - i3.large.elasticsearch
- - i3.xlarge.elasticsearch
- - i3.2xlarge.elasticsearch
- - i3.4xlarge.elasticsearch
- - i3.8xlarge.elasticsearch
- - i3.16xlarge.elasticsearch
- Type: String
-
- DedicatedMasterInstanceType:
- Description: >
- The instance type for the dedicated master nodes. These nodes perform cluster management
- tasks, but doesn't hold data or respond to data upload requests.
- ConstraintDescription: Must be a valid instance type from the list of allowed values.
- Default: t2.medium.elasticsearch
- AllowedValues:
- - t2.small.elasticsearch
- - t2.medium.elasticsearch
- - m5.large.elasticsearch
- - m5.xlarge.elasticsearch
- - m5.2xlarge.elasticsearch
- - m5.4xlarge.elasticsearch
- - m5.12xlarge.elasticsearch
- - c5.large.elasticsearch
- - c5.xlarge.elasticsearch
- - c5.2xlarge.elasticsearch
- - c5.4xlarge.elasticsearch
- - c5.9xlarge.elasticsearch
- - c5.18xlarge.elasticsearch
- - r5.large.elasticsearch
- - r5.xlarge.elasticsearch
- - r5.2xlarge.elasticsearch
- - r5.4xlarge.elasticsearch
- - r5.12xlarge.elasticsearch
- - i3.large.elasticsearch
- - i3.xlarge.elasticsearch
- - i3.2xlarge.elasticsearch
- - i3.4xlarge.elasticsearch
- - i3.8xlarge.elasticsearch
- - i3.16xlarge.elasticsearch
- Type: String
-
- NumberOfElasticsearchNodes:
- Description: An integer value specifying the number of Elasticsearch primary nodes in the cluster.
- Type: Number
- Default: 2
- MinValue: 1
- MaxValue: 40
-
- NumberOfDedicatedMasterNodes:
- Description: An integer value specifying the number of dedicated master nodes.
- Type: Number
- Default: 2
- MinValue: 2
- MaxValue: 5
-
- ZoneAwarenessEnabled:
- Description: >
- When Zone Awareness is enabled, Elasticsearch allocates the nodes and replica
- index shards that belong to a cluster across multiple AZs in the deployment region.
- Type: String
- AllowedValues:
- - true
- - false
- Default: false
-
- ElasticsearchVersion:
- Description: >
- The version of Elasticsearch to deploy on the cluster. Defaults to 6.8. Note
- that an update requires a full replacement of the Elasticsearch cluster.
- Type: String
- AllowedValues:
- - 1.5
- - 2.3
- - 5.1
- - 5.3
- - 5.5
- - 5.6
- - 6.0
- - 6.2
- - 6.3
- - 6.4
- - 6.5
- - 6.6
- - 6.8
- - 6.8
- - 7.1
- Default: 6.8
-
- EBSVolumeSize:
- Description: >
- The size of the EBS volume (per instance; total cluster size = EBS volume size x Instance count)
- Maximum size varies by instance type, from 35GiB for t2 instances, up to 12TiB for r5.12xlarge.
- Type: Number
- Default: 10
- MinValue: 10
- MaxValue: 12000
-
-Resources:
- ElasticsearchDomain:
- Type: AWS::Elasticsearch::Domain
- Properties:
- DomainName: !Sub ${Environment}-${ElasticsearchDomainName}
- ElasticsearchVersion: !Ref ElasticsearchVersion
- ElasticsearchClusterConfig:
- DedicatedMasterEnabled: "true"
- InstanceCount: !Ref NumberOfElasticsearchNodes
- ZoneAwarenessEnabled: !Ref ZoneAwarenessEnabled
- InstanceType: !Ref ElasticsearchNodeInstanceType
- DedicatedMasterType: !Ref DedicatedMasterInstanceType
- DedicatedMasterCount: !Ref NumberOfDedicatedMasterNodes
- EBSOptions:
- EBSEnabled: true
- VolumeSize: !Ref EBSVolumeSize
- VolumeType: "gp2"
- NodeToNodeEncryptionOptions:
- Enabled: false
- SnapshotOptions:
- AutomatedSnapshotStartHour: "0"
- AccessPolicies:
- Version: 2012-10-17
- Statement:
- - Effect: "Allow"
- Principal: "*"
- Resource: '*'
- Action: "*"
- AdvancedOptions:
- rest.action.multi.allow_explicit_index: "true"
- Tags:
- -
- Key: "Environment"
- Value: !Ref Environment
- VPCOptions:
- SubnetIds:
- - Fn::ImportValue:
- !Sub "${Environment}-OSDU-PrivateSubnet-AZ1"
- SecurityGroupIds:
- - Fn::ImportValue:
- !Sub "${Environment}-${ApplicationName}-EcsNetworkSecurityGroupId"
-
-Outputs:
- # Elasticsearch domain ARN
- ElasticsearchDomainArn:
- Description: The ARN of the Elasticsearch domain.
- Value: !GetAtt ElasticsearchDomain.DomainArn
- Export:
- Name: !Sub ${Environment}-${ElasticsearchDomainName}-ElasticsearchDomainArn
-
- # Elasticsearch domain endpoint
- ElasticsearchDomainEndpoint:
- Description: The endpoint URL of the Elasticsearch domain.
- Value: !GetAtt ElasticsearchDomain.DomainEndpoint
- Export:
- Name: !Sub ${Environment}-${ElasticsearchDomainName}-ElasticsearchDomainEndpoint
diff --git a/provider/indexer-aws/CloudFormation/Automated/iam-credentials.yml b/provider/indexer-aws/CloudFormation/Automated/iam-credentials.yml
deleted file mode 100644
index 3d5c9f144d52070bfebb81eacc4f32ba4b193f31..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/CloudFormation/Automated/iam-credentials.yml
+++ /dev/null
@@ -1,114 +0,0 @@
-# Copyright © Amazon Web Services
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-AWSTemplateFormatVersion: 2010-09-09
-Description: >-
- CloudFormation template for creating the resources used for application SDK access for OSDU services.
- It creates the IAM account, access keys, and optional key rotation.
-
-Parameters:
- Environment:
- Description: An environment name that will be prefixed to resource names.
- Type: String
- AllowedValues:
- - dev
- - uat
- - prod
- ConstraintDescription: Can only be "dev/uat/prod"
- Default: dev
-
- Region:
- Description: The AWS region to deploy the resources to.
- Type: String
- Default: us-east-1
-
- IndexerServiceIamUsername:
- Description: The username of the service user for the OS Indexer Service.
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Type: String
- Default: service-user-os-indexer
- MinLength: '1'
- MaxLength: '64'
-
- IndexerServiceIamKeyRotationSerial:
- Description: This integer value can only ever be incremented, and an increase in value results in a rotation of the user's access key.
- Type: Number
- Default: 1
-
-Resources:
- IndexerServiceIamUser:
- Type: AWS::IAM::User
- Properties:
- Policies:
- - PolicyName: !Sub ${Environment}-IndexerServiceUserPolicy
- PolicyDocument:
- Version: '2012-10-17'
- Statement:
- -
- Action:
- - 's3:*'
- - 'sns:*'
- - 'sqs:*'
- - 'dynamodb:*'
- - 'logs:*'
- - 'cloudwatch:*'
- - 'es:*'
- - 'cognito-identity:*'
- - 'cognito-idp:*'
- - 'sts:AssumeRole'
- - "iam:*"
- Effect: Allow
- Resource: '*'
- UserName: !Sub ${Environment}-${IndexerServiceIamUsername}
-
- IndexerServiceIamUserAccessKey:
- Type: AWS::IAM::AccessKey
- DependsOn: IndexerServiceIamUser
- Properties:
- Serial: !Ref IndexerServiceIamKeyRotationSerial # this value can only ever be incremented, and an increase in value results in a rotation of the user's access key
- Status: Active
- UserName: !Sub ${Environment}-${IndexerServiceIamUsername}
-
- IAMCredentialsSecret:
- Type: 'AWS::SecretsManager::Secret'
- Properties:
- Name: !Sub ${Environment}-IndexerServiceIamCredentials
- Description: The IAM service account credentials for the search service.
- SecretString:
- Fn::Sub:
- - '{"access_key":"${AccessKey}","secret_key":"${SecretKey}"}'
- - {AccessKey: !Ref IndexerServiceIamUserAccessKey, SecretKey: !GetAtt IndexerServiceIamUserAccessKey.SecretAccessKey}
- Tags:
- - Key: Environment
- Value: !Ref Environment
-
-Outputs:
- IndexerServiceIamUserAccessKeyId:
- Description: The access key ID for the service user for the Schema Repository.
- Value: !Ref IndexerServiceIamUserAccessKey
- Export:
- Name: !Sub ${Environment}-IndexerServiceIamUserAccessKeyId
-
- IndexerServiceIamUserSecretAccessKey:
- Description: The secret access key for the service user for the Schema Repository.
- Value: !GetAtt IndexerServiceIamUserAccessKey.SecretAccessKey
- Export:
- Name: !Sub ${Environment}-IndexerServiceIamUserSecretAccessKey
-
- IndexerServiceIamUserArn:
- Description: The ARN of the service IAM user account.
- Value: !GetAtt IndexerServiceIamUser.Arn
- Export:
- Name: !Sub ${Environment}-IndexerServiceIamUserArn
diff --git a/provider/indexer-aws/CloudFormation/Automated/sns-topic.yml b/provider/indexer-aws/CloudFormation/Automated/sns-topic.yml
deleted file mode 100644
index f90f9188963232b89978dde65f51597fb79f98e9..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/CloudFormation/Automated/sns-topic.yml
+++ /dev/null
@@ -1,108 +0,0 @@
-# Copyright © Amazon Web Services
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-AWSTemplateFormatVersion: 2010-09-09
-Description: >-
- CloudFormation template for creating the resources used for the sending messages to topic and queues to receive the
- messages for OSDU's indexer service. It creates the SNS Topic and the corresponding SQS Queues with their associated policies.
-
-Parameters:
- Environment:
- Description: an environment name that will be prefixed to resource names.
- Type: String
- AllowedValues:
- - dev
- - uat
- - prod
- ConstraintDescription: Can only be "dev/uat/prod"
- Default: dev
-
- Region:
- Description: The AWS region to deploy the resources to.
- Type: String
- Default: us-east-1
-
- SNSTopicName:
- Description: >-
- The name of the Simple Notification Service topic for the OS Indexer Service. Defaults to osdu-indexer-messages.
- Will be prefixed with the environment name.
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: osdu-indexer-messages
- Type: String
- MinLength: '1'
- MaxLength: '64'
-
- SQSQueueName:
- Description: >-
- The name of the Simple Queue Service queue for the OS Indexer Service. Defaults to osdu-indexer-queue.
- Will be prefixed with the environment name.
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: osdu-indexer-queue
- Type: String
- MinLength: '1'
- MaxLength: '64'
-
-Resources:
- OSDUIndexerSNSTopic:
- Type: 'AWS::SNS::Topic'
- Properties:
- DisplayName: !Sub ${Environment}-${SNSTopicName}
- TopicName: !Sub ${Environment}-${SNSTopicName}
- Subscription:
- - Endpoint:
- Fn::GetAtt:
- - OSDUIndexerSQSQueue
- - Arn
- Protocol: sqs
-
- OSDUIndexerSQSQueue:
- Type: AWS::SQS::Queue
- Properties:
- QueueName: !Sub ${Environment}-${SQSQueueName}
-
- OSDUQueuePolicy:
- Type: AWS::SQS::QueuePolicy
- Properties:
- PolicyDocument:
- Version: "2012-10-17"
- Id: OSDUQueuePolicy
- Statement:
- - Sid: Allow-SendMessage-To-Queues-From-SNS-Topic
- Effect: Allow
- Principal: "*"
- Action:
- - sqs:SendMessage
- - sqs:ReceiveMessage
- Resource: "*"
- Condition:
- ArnEquals:
- aws:SourceArn:
- Ref: OSDUIndexerSNSTopic
- Queues:
- - Ref: OSDUIndexerSQSQueue
-
-Outputs:
- OSDUIndexerSNSTopicTopicName:
- Value: !Sub ${Environment}-${SNSTopicName}
- Description: Topic Name of the Indexer Service Message Bus SNS Topic
- Export:
- Name: !Sub ${Environment}-OSDUIndexerSNSTopic
-
- OSDUIndexerSQSQueueName:
- Value: !Sub ${Environment}-${SQSQueueName}
- Description: Queue Name of Subscribed Indexer Service Message Bus SQS Queue
- Export:
- Name: !Sub ${Environment}-OSDUIndexerSQSQueue
diff --git a/provider/indexer-aws/CloudFormation/JarDeploy/CodePipeline-JarDeploy.yml b/provider/indexer-aws/CloudFormation/JarDeploy/CodePipeline-JarDeploy.yml
deleted file mode 100644
index f75d29cd5944fc25edfcf213bac567194319af4e..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/CloudFormation/JarDeploy/CodePipeline-JarDeploy.yml
+++ /dev/null
@@ -1,249 +0,0 @@
-# Copyright © Amazon Web Services
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-AWSTemplateFormatVersion: 2010-09-09
-
-Description: >
- This CloudFormation script creates the deployment pipeline for OSDU's indexer service. The CodePipeline
- should automatically trigger whenever commits are made on the tracked branch. The start and end
- of the CodePipeline should trigger a SNS alert to keep track of when the deployment has started
- and when it finishes.
-
-Parameters:
- Environment:
- Description: Environment Name. Defaults to 'dev'. Can only be dev/uat/prod.
- Type: String
- AllowedValues:
- - dev
- - uat
- - prod
- Default: dev
-
- DeploymentRegion:
- Description: The AWS region to deploy the application to. The default is us-east-1.
- Type: String
- Default: us-east-1
-
- SNSNotificationEmail:
- Description: The email address to send SNS notifications about the build to.
- Type: String
- Default: barclay.walsh@parivedasolutions.com
-
- CodeCommitRepositoryName:
- Description: The name of the Code Commit Repository that the CodePipeline source is connected to.
- Type: String
- Default: os-indexer
-
- JarServiceBase:
- Description: The name of the service base path for the JAR files (e.g. 'indexer').
- Type: String
- Default: indexer
-
- CodeCommitBranchName:
- Description: The name of the Code Commit branch that the CodePipeline source is connected to.
- Type: String
- Default: dev
-
-Resources:
- ArtifactStoreBucket:
- Type: AWS::S3::Bucket
- DeletionPolicy: Delete
- Properties:
- VersioningConfiguration:
- Status: Enabled
-
- ArtifactStoreBucketPolicy:
- Type: AWS::S3::BucketPolicy
- Properties:
- Bucket: !Ref ArtifactStoreBucket
- PolicyDocument:
- Statement:
- - Action:
- - s3:*
- Effect: Allow
- Resource:
- - !Sub arn:aws:s3:::${ArtifactStoreBucket}
- - !Sub arn:aws:s3:::${ArtifactStoreBucket}/*
- Principal:
- AWS:
- - !Sub arn:aws:iam::${AWS::AccountId}:root
- - !ImportValue
- 'Fn::Sub': '${Environment}-CodeBuildRoleArn'
- - !ImportValue
- 'Fn::Sub': '${Environment}-PipelineRoleArn'
- - !ImportValue
- 'Fn::Sub': '${Environment}-CFNRoleArn'
-
- CachingBucket:
- Type: AWS::S3::Bucket
- DeletionPolicy: Delete
- Properties:
- VersioningConfiguration:
- Status: Enabled
-
- CachingBucketPolicy:
- Type: AWS::S3::BucketPolicy
- Properties:
- Bucket: !Ref CachingBucket
- PolicyDocument:
- Statement:
- - Action:
- - s3:*
- Effect: Allow
- Resource:
- - !Sub arn:aws:s3:::${CachingBucket}
- - !Sub arn:aws:s3:::${CachingBucket}/*
- Principal:
- AWS:
- - !Sub arn:aws:iam::${AWS::AccountId}:root
- - !ImportValue
- 'Fn::Sub': '${Environment}-CodeBuildRoleArn'
- - !ImportValue
- 'Fn::Sub': '${Environment}-PipelineRoleArn'
- - !ImportValue
- 'Fn::Sub': '${Environment}-CFNRoleArn'
-
- SNSCodePipelineDeploymentFailed:
- Type: AWS::SNS::Topic
- Properties:
- Subscription:
- - Endpoint: !Ref SNSNotificationEmail
- Protocol: email
- TopicName: !Sub '${Environment}-OS-Indexer-Deployment-CodePipeline-JarDeploy-Failed'
-
- EventRuleCodePipelineFailed:
- Type: AWS::Events::Rule
- Properties:
- Description: Triggered whenever the CodePipeline deployment stage has failed.
- EventPattern:
- source:
- - "aws.codepipeline"
- detail-type:
- - "CodePipeline Stage Execution State Change"
- detail:
- state:
- - "FAILED"
- pipeline:
- - !Sub '${Environment}-OSDU-OS-Indexer-CodePipeline-JarDeploy'
-
- Name: !Sub ${Environment}-CodePipelineEventRule-${CodeCommitRepositoryName}-JarDeploy
- Targets:
- -
- Arn:
- !Ref SNSCodePipelineDeploymentFailed
- Id: "Deployment-CodePipeline-JarDeploy-Failed"
- InputTransformer:
- InputPathsMap:
- pipeline : "$.detail.pipeline"
- InputTemplate: '"The Pipeline <pipeline> has failed."'
-
- Pipeline:
- Type: AWS::CodePipeline::Pipeline
- Properties:
- ArtifactStore:
- Location: !Ref ArtifactStoreBucket
- Type: S3
- Name: !Sub '${Environment}-OSDU-OS-Indexer-CodePipeline-JarDeploy'
- RoleArn: !ImportValue
- 'Fn::Sub': '${Environment}-PipelineRoleArn'
- Stages:
- - Name: Source
- Actions:
- - Name: Source
- ActionTypeId:
- Category: Source
- Owner: AWS
- Provider: CodeCommit
- Version: '1'
- Configuration:
- BranchName: !Ref CodeCommitBranchName
- RepositoryName: !Ref CodeCommitRepositoryName
- OutputArtifacts:
- - Name: Source
- RunOrder: '1'
-
- - Name: CodeBuild
- Actions:
- - Name: Jar-CodeBuild
- ActionTypeId:
- Category: Build
- Owner: AWS
- Provider: CodeBuild
- Version: '1'
- InputArtifacts:
- - Name: Source
- OutputArtifacts:
- - Name: Jar-CodeBuild
- Configuration:
- ProjectName: !Ref JarCodeBuild
- RunOrder: '2'
-
- JarCodeBuild:
- Type: AWS::CodeBuild::Project
- Properties:
- Name: !Sub ${Environment}-jar-codebuild-${CodeCommitRepositoryName}
- Description: CodeBuild commands which run after the CloudFormation deployment.
- ServiceRole: !ImportValue
- 'Fn::Sub': '${Environment}-CodeBuildRoleArn'
- Artifacts:
- Type: S3
- Location: !Ref ArtifactStoreBucket
- Name: !Sub ${Environment}-jar-codebuild
- Environment:
- Type: LINUX_CONTAINER
- ComputeType: BUILD_GENERAL1_SMALL
- Image: aws/codebuild/standard:2.0
- EnvironmentVariables:
- - Name: ENVIRONMENT
- Type: PLAINTEXT
- Value: !Ref Environment
- - Name: AWS_ACCOUNT_ID
- Type: PLAINTEXT
- Value: !Ref AWS::AccountId
- - Name: AWS_REGION
- Type: PLAINTEXT
- Value: !Ref DeploymentRegion
- - Name: APPLICATION_NAME
- Type: PLAINTEXT
- Value: !Ref CodeCommitRepositoryName
- - Name: JAR_SERVICE_BASE
- Type: PLAINTEXT
- Value: !Ref JarServiceBase
- - Name: M2_REPO_S3_BUCKET
- Type: PLAINTEXT
- Value: !Sub "${Environment}-${AWS::AccountId}-persistent-maven-m2-bucket"
- - Name: JAR_DEPLOY_S3_BUCKET
- Type: PLAINTEXT
- Value: !Sub ${Environment}-${AWS::AccountId}-osdu-jar-deploy
- PrivilegedMode: true
- Source:
- BuildSpec: ./provider/indexer-aws/buildspec-jar-deploy.yml
- Location: !Sub https://git-codecommit.${AWS::Region}.amazonaws.com/v1/repos/${CodeCommitRepositoryName}
- Type: CODECOMMIT
- Cache:
- Type: S3
- Location: !Sub ${CachingBucket}/${Environment}
- TimeoutInMinutes: 15
- VpcConfig:
- SecurityGroupIds:
- - Fn::ImportValue:
- !Sub "${Environment}-OSDU-CodeBuildSecurityGroup"
- Subnets:
- - Fn::ImportValue:
- !Sub "${Environment}-OSDU-PrivateSubnet-AZ1"
- - Fn::ImportValue:
- !Sub "${Environment}-OSDU-PrivateSubnet-AZ2"
- VpcId:
- Fn::ImportValue:
- !Sub "${Environment}-OSDU-VPC"
diff --git a/provider/indexer-aws/CloudFormation/Manual/01-CreateCodePipeline.yml b/provider/indexer-aws/CloudFormation/Manual/01-CreateCodePipeline.yml
deleted file mode 100644
index 0109633f0071acd48c8520bf240dd962068ead11..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/CloudFormation/Manual/01-CreateCodePipeline.yml
+++ /dev/null
@@ -1,358 +0,0 @@
-# Copyright © Amazon Web Services
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-AWSTemplateFormatVersion: 2010-09-09
-Description: >
- This CloudFormation script creates the deployment pipeline for OSDU's indexer
- service. The CodePipeline should automatically trigger whenever commits are
- made on the tracked branch. The start and end of the CodePipeline should
- trigger a SNS alert to keep track of when the deployment has started and when
- it finishes.
-Parameters:
- Environment:
- Description: Environment Name. Defaults to 'dev'. Can only be dev/uat/prod.
- Type: String
- AllowedValues:
- - dev
- - uat
- - prod
- Default: dev
-
- DeploymentRegion:
- Description: The AWS region to deploy the application to. The default is us-east-1.
- Type: String
- Default: us-east-1
-
- SNSNotificationEmail:
- Description: The email address to send SNS notifications about the build to.
- Type: String
- Default: barclay.walsh@parivedasolutions.com
-
- CodeCommitRepositoryName:
- Description: >-
- The name of the Code Commit Repository that the CodePipeline source is
- connected to.
- Type: String
- Default: os-indexer
-
- CodeCommitBranchName:
- Description: >-
- The name of the Code Commit branch that the CodePipeline source is
- connected to.
- Type: String
- Default: dev
-
- MasterStackName:
- Description: The name of the master stack that is being deployed by the CodePipeline.
- Type: String
- Default: os-indexer-master-stack
-
- MasterTemplateName:
- Description: >-
- The name of the master template that is called when creating the master
- stack.
- Type: String
- Default: provider/indexer-aws/CloudFormation/Master/os-indexer-master.yml
-
- SchemaCacheName:
- Description: The name of the cache cluster for the schema cache. Needs to match the value in the environment params JSON. Will be prefixed with the environment name.
- Type: String
- MinLength: '1'
- MaxLength: '64'
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: indexerSchemaCache
-
- IndexCacheName:
- Description: The name of the cache cluster for the index cache. Needs to match the value in the environment params JSON. Will be prefixed with the environment name.
- Type: String
- MinLength: '1'
- MaxLength: '64'
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: indexerIndexCache
-
-Resources:
- ArtifactStoreBucket:
- Type: 'AWS::S3::Bucket'
- DeletionPolicy: Delete
- Properties:
- VersioningConfiguration:
- Status: Enabled
- ArtifactStoreBucketPolicy:
- Type: 'AWS::S3::BucketPolicy'
- Properties:
- Bucket: !Ref ArtifactStoreBucket
- PolicyDocument:
- Statement:
- - Action:
- - 's3:*'
- Effect: Allow
- Resource:
- - !Sub 'arn:aws:s3:::${ArtifactStoreBucket}'
- - !Sub 'arn:aws:s3:::${ArtifactStoreBucket}/*'
- Principal:
- AWS:
- - !Sub 'arn:aws:iam::${AWS::AccountId}:root'
- - !ImportValue
- 'Fn::Sub': '${Environment}-CodeBuildRoleArn'
- - !ImportValue
- 'Fn::Sub': '${Environment}-PipelineRoleArn'
- - !ImportValue
- 'Fn::Sub': '${Environment}-CFNRoleArn'
- CachingBucket:
- Type: AWS::S3::Bucket
- DeletionPolicy: Delete
- Properties:
- VersioningConfiguration:
- Status: Enabled
-
- CachingBucketPolicy:
- Type: AWS::S3::BucketPolicy
- Properties:
- Bucket: !Ref CachingBucket
- PolicyDocument:
- Statement:
- - Action:
- - s3:*
- Effect: Allow
- Resource:
- - !Sub arn:aws:s3:::${CachingBucket}
- - !Sub arn:aws:s3:::${CachingBucket}/*
- Principal:
- AWS:
- - !Sub arn:aws:iam::${AWS::AccountId}:root
- - !ImportValue
- 'Fn::Sub': '${Environment}-CodeBuildRoleArn'
- - !ImportValue
- 'Fn::Sub': '${Environment}-PipelineRoleArn'
- - !ImportValue
- 'Fn::Sub': '${Environment}-CFNRoleArn'
-
- SNSCodePipelineDeploymentFailed:
- Type: 'AWS::SNS::Topic'
- Properties:
- Subscription:
- - Endpoint: !Ref SNSNotificationEmail
- Protocol: email
- TopicName: !Sub '${Environment}-OS-Indexer-Deployment-CodePipeline-Failed'
- EventRuleCodePipelineFailed:
- Type: 'AWS::Events::Rule'
- Properties:
- Description: Triggered whenever the CodePipeline deployment stage has failed.
- EventPattern:
- source:
- - aws.codepipeline
- detail-type:
- - CodePipeline Stage Execution State Change
- detail:
- state:
- - FAILED
- pipeline:
- - !Sub '${Environment}-OSDU-OS-Indexer-CodePipeline'
- Name: !Sub '${Environment}-CodePipelineEventRule-${CodeCommitRepositoryName}'
- Targets:
- - Arn: !Ref SNSCodePipelineDeploymentFailed
- Id: Deployment-CodePipeline-Failed
- InputTransformer:
- InputPathsMap:
- pipeline: $.detail.pipeline
- InputTemplate: '"The Pipeline <pipeline> has failed."'
- Pipeline:
- Type: 'AWS::CodePipeline::Pipeline'
- Properties:
- ArtifactStore:
- Location: !Ref ArtifactStoreBucket
- Type: S3
- Name: !Sub '${Environment}-OSDU-OS-Indexer-CodePipeline'
- RoleArn: !ImportValue
- 'Fn::Sub': '${Environment}-PipelineRoleArn'
- Stages:
- - Name: Source
- Actions:
- - Name: Source
- ActionTypeId:
- Category: Source
- Owner: AWS
- Provider: CodeCommit
- Version: '1'
- Configuration:
- BranchName: !Ref CodeCommitBranchName
- RepositoryName: !Ref CodeCommitRepositoryName
- OutputArtifacts:
- - Name: Source
- RunOrder: '1'
- - Name: Pre-Deployment-CodeBuild
- Actions:
- - Name: Pre-Deployment-CodeBuild
- ActionTypeId:
- Category: Build
- Owner: AWS
- Provider: CodeBuild
- Version: '1'
- InputArtifacts:
- - Name: Source
- OutputArtifacts:
- - Name: Pre-Deployment-CodeBuild
- Configuration:
- ProjectName: !Ref PreDeploymentCodeBuild
- RunOrder: '2'
- - Name: Deployment
- Actions:
- - Name: CloudFormation-Deployment-Master
- ActionTypeId:
- Category: Deploy
- Owner: AWS
- Provider: CloudFormation
- Version: '1'
- InputArtifacts:
- - Name: Source
- Configuration:
- ActionMode: CREATE_UPDATE
- Capabilities: CAPABILITY_NAMED_IAM
- RoleArn: !ImportValue
- 'Fn::Sub': '${Environment}-CFNRoleArn'
- StackName: !Sub '${Environment}-${MasterStackName}'
- TemplatePath: !Sub 'Source::${MasterTemplateName}'
- TemplateConfiguration: !Sub >-
- Source::provider/indexer-aws/CloudFormation/Params/${Environment}.template_configuration.json
- RunOrder: '3'
-
- - Name: Post-Deployment-CodeBuild
- Actions:
- - Name: Post-Deployment-CodeBuild
- ActionTypeId:
- Category: Build
- Owner: AWS
- Provider: CodeBuild
- Version: '1'
- InputArtifacts:
- - Name: Source
- OutputArtifacts:
- - Name: Post-Deployment-CodeBuild
- Configuration:
- ProjectName: !Ref PostDeploymentCodeBuild
- RunOrder: '4'
- PreDeploymentCodeBuild:
- Type: 'AWS::CodeBuild::Project'
- Properties:
- Name: !Sub '${Environment}-pre-deployment-codebuild-${CodeCommitRepositoryName}'
- Description: CodeBuild commands which run prior to the CloudFormation deployment.
- ServiceRole: !ImportValue
- 'Fn::Sub': '${Environment}-CodeBuildRoleArn'
- Artifacts:
- Type: S3
- Location: !Ref ArtifactStoreBucket
- Name: !Sub '${Environment}-pre-deployment-codebuild'
- Environment:
- Type: LINUX_CONTAINER
- ComputeType: BUILD_GENERAL1_SMALL
- Image: aws/codebuild/standard:2.0
- EnvironmentVariables:
- - Name: ENVIRONMENT
- Type: PLAINTEXT
- Value: !Ref Environment
- - Name: AWS_ACCOUNT_ID
- Type: PLAINTEXT
- Value: !Ref 'AWS::AccountId'
- - Name: AWS_REGION
- Type: PLAINTEXT
- Value: !Ref DeploymentRegion
- - Name: CFN_S3_BUCKET
- Value: !ImportValue
- 'Fn::Sub': '${Environment}-S3BucketCloudFormation'
- - Name: APPLICATION_NAME
- Type: PLAINTEXT
- Value: !Ref CodeCommitRepositoryName
- PrivilegedMode: false
- Source:
- BuildSpec: ./provider/indexer-aws/buildspec-pre-deploy.yml
- Location: !Sub >-
- https://git-codecommit.${AWS::Region}.amazonaws.com/v1/repos/${CodeCommitRepositoryName}
- Type: CODECOMMIT
- TimeoutInMinutes: 15
-
- PostDeploymentCodeBuild:
- Type: AWS::CodeBuild::Project
- Properties:
- Name: !Sub ${Environment}-post-deployment-codebuild-${CodeCommitRepositoryName}
- Description: CodeBuild commands which run after the CloudFormation deployment.
- ServiceRole: !ImportValue
- 'Fn::Sub': '${Environment}-CodeBuildRoleArn'
- Artifacts:
- Type: S3
- Location: !Ref ArtifactStoreBucket
- Name: !Sub ${Environment}-post-deployment-codebuild
- Environment:
- Type: LINUX_CONTAINER
- ComputeType: BUILD_GENERAL1_SMALL
- Image: aws/codebuild/standard:2.0
- EnvironmentVariables:
- - Name: ENVIRONMENT
- Type: PLAINTEXT
- Value: !Ref Environment
- - Name: AWS_ACCOUNT_ID
- Type: PLAINTEXT
- Value: !Ref AWS::AccountId
- - Name: AWS_REGION
- Type: PLAINTEXT
- Value: !Ref DeploymentRegion
- - Name: VSTS_FEED_USER
- Type: PLAINTEXT
- Value: '{{resolve:secretsmanager:dev-VSTSFeedToken:SecretString:vsts_feed_user}}'
- - Name: VSTS_FEED_TOKEN
- Type: PLAINTEXT
- Value: '{{resolve:secretsmanager:dev-VSTSFeedToken:SecretString:vsts_feed_token}}'
- - Name: IMAGE_TAG
- Type: PLAINTEXT
- Value: latest
- - Name: IMAGE_REPO_NAME
- Type: PLAINTEXT
- Value: !Sub ${Environment}-${CodeCommitRepositoryName}-repository
- - Name: QUEUE_IMAGE_REPO_NAME
- Type: PLAINTEXT
- Value: !Sub ${Environment}-${CodeCommitRepositoryName}-queue-repository
- - Name: SCHEMA_CACHE_NAME
- Type: PLAINTEXT
- Value: !Ref SchemaCacheName
- - Name: INDEX_CACHE_NAME
- Type: PLAINTEXT
- Value: !Ref IndexCacheName
- - Name: APPLICATION_NAME
- Type: PLAINTEXT
- Value: !Ref CodeCommitRepositoryName
- - Name: M2_REPO_S3_BUCKET
- Type: PLAINTEXT
- Value: !Sub "${Environment}-${AWS::AccountId}-persistent-maven-m2-bucket"
- PrivilegedMode: true
- Source:
- BuildSpec: ./provider/indexer-aws/buildspec-post-deploy.yml
- Location: !Sub https://git-codecommit.${AWS::Region}.amazonaws.com/v1/repos/${CodeCommitRepositoryName}
- Type: CODECOMMIT
- Cache:
- Type: S3
- Location: !Sub ${CachingBucket}/${Environment}
- TimeoutInMinutes: 15
- VpcConfig:
- SecurityGroupIds:
- - Fn::ImportValue:
- !Sub "${Environment}-OSDU-CodeBuildSecurityGroup"
- Subnets:
- - Fn::ImportValue:
- !Sub "${Environment}-OSDU-PrivateSubnet-AZ1"
- - Fn::ImportValue:
- !Sub "${Environment}-OSDU-PrivateSubnet-AZ2"
- VpcId:
- Fn::ImportValue:
- !Sub "${Environment}-OSDU-VPC"
\ No newline at end of file
diff --git a/provider/indexer-aws/CloudFormation/Master/os-indexer-master.yml b/provider/indexer-aws/CloudFormation/Master/os-indexer-master.yml
deleted file mode 100644
index c4a49e0b6ab7580627d96052056f23df5e745680..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/CloudFormation/Master/os-indexer-master.yml
+++ /dev/null
@@ -1,625 +0,0 @@
-# Copyright © Amazon Web Services
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-AWSTemplateFormatVersion: 2010-09-09
-Description: Creates all AWS resources used by OSDU's Indexer Service. Requires having previously setup the CodeCommit repository, as well as the CodePipeline (manual template).
-Parameters:
-
- VersionNumber:
- Description: Version Number for the pom to deploy the jar and Docker Image deployment in the Dockerfile
- Type: String
- Default: '0.0.1'
-
- ServiceName:
- Description: >-
- Service name for jar deployment in the Dockerfile
- Type: String
- Default: 'indexer'
-
- Environment:
- Description: The name of the environment.
- Type: String
- AllowedValues:
- - dev
- - uat
- - prod
- ConstraintDescription: Environment can only be "dev/uat/prod".
- Default: dev
-
- DeploymentRegion:
- Description: The AWS region to deploy the resources to.
- Type: String
- Default: us-east-1
-
- ApplicationName:
- Description: >
- The name of the indexer application, should be equal to the repository name.
- Type: String
- MinLength: '1'
- MaxLength: '64'
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: os-indexer
-
- SearchApplicationName:
- Description: >
- The name of the Search Service application (ex: os-search). Should be the same as the Search Service repo name.
- Type: String
- MinLength: '1'
- MaxLength: '64'
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: os-search
-
- KeyName:
- Description: >
- Name of an existing EC2 KeyPair to enable SSH access to the ECS instances. Note that key pairs cannot
- be created through CloudFormation, but instead must be uploaded through the AWS Console.
- Type: AWS::EC2::KeyPair::KeyName
- Default: ecs_indexer_key
-
- DesiredCapacity:
- Description: The default number of instances to launch in the ECS cluster.
- Type: Number
- Default: '1'
-
- MinSize:
- Description: Maximum number of instances that can be launched in the ECS cluster.
- Type: Number
- Default: '0'
-
- MaxSize:
- Description: Maximum number of instances that can be launched in the ECS cluster.
- Type: Number
- Default: '1'
-
- InstanceType:
- Description: EC2 instance type
- Type: String
- Default: t3.large
- AllowedValues:
- - m5.large
- - m5.xlarge
- - m5.2xlarge
- - m5.4xlarge
- - m5.12xlarge
- - m5.16xlarge
- - m5.24xlarge
- - m4.large
- - m4.xlarge
- - m4.2xlarge
- - m4.4xlarge
- - m4.10xlarge
- - m4.16xlarge
- - t3.nano
- - t3.micro
- - t3.small
- - t3.medium
- - t3.large
- - t3.xlarge
- - t3.2xlarge
- - c5.large
- - c5.xlarge
- - c5.2xlarge
- - c5.4xlarge
- - c5.12xlarge
- - c5.16xlarge
- - c5.24xlarge
- - r5.large
- - r5.xlarge
- - r5.2xlarge
- - r5.4xlarge
- - r5.12xlarge
- - r5.24xlarge
- - r4.large
- - r4.xlarge
- - r4.2xlarge
- - r4.4xlarge
- - r4.8xlarge
- - r4.16xlarge
- - i3.large
- - i3.xlarge
- - i3.2xlarge
- - i3.4xlarge
- - i3.10xlarge
- - i3.16xlarge
- - x1e.xlarge
- - x1e.2xlarge
- - x1e.4xlarge
- - x1e.8xlarge
- - x1e.16xlarge
- - x1e.32xlarge
- ConstraintDescription: Please choose a valid EC2 instance type for the ECS container instances.
-
- IndexerServiceIamUsername:
- Description: The username of the service user for the OS Indexer Service.
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Type: String
- Default: service-user-os-indexer
- MinLength: '1'
- MaxLength: '64'
-
- IndexerServiceIamKeyRotationSerial:
- Description: This integer value can only ever be incremented, and an increase in value results in a rotation of the user's access key.
- Type: Number
- Default: 1
-
- SNSTopicName:
- Description: >-
- The name of the Simple Notification Service topic for the OS Indexer Service. Defaults to osdu-indexer-messages.
- Will be prefixed with the environment name.
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: osdu-indexer-messages
- Type: String
- MinLength: '1'
- MaxLength: '64'
-
- SQSQueueName:
- Description: >-
- The name of the Simple Queue Service queue for the OS Indexer Service. Defaults to osdu-indexer-queue.
- Will be prefixed with the environment name.
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: osdu-indexer-queue
- Type: String
- MinLength: '1'
- MaxLength: '64'
-
- IndexCacheName:
- Description: The name of the cache cluster for the legal tag cache. Will be prefixed with the environment name.
- Type: String
- MinLength: '1'
- MaxLength: '64'
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: indexerIndexCache
-
- IndexCacheEngine:
- Description: Which caching platform to use for the legal tag cache. Can be set to 'redis' or 'memcached'.
- Type: String
- AllowedValues:
- - redis
- - memcached
- ConstraintDescription: Can only be "redis" or "memcached"
- Default: redis
-
- IndexCacheNodeInstanceType:
- Description: The instance type for redis cache nodes for the legal tag cache.
- ConstraintDescription: Must be a valid instance type from the list of allowed values.
- Default: cache.t2.micro
- AllowedValues:
- - cache.m5.large
- - cache.m5.xlarge
- - cache.m5.2xlarge
- - cache.m5.4xlarge
- - cache.m5.12xlarge
- - cache.m5.24xlarge
- - cache.m4.large
- - cache.m4.xlarge
- - cache.m4.2xlarge
- - cache.m4.4xlarge
- - cache.m4.10xlarge
- - cache.t2.micro
- - cache.t2.small
- - cache.t2.medium
- - cache.c1.xlarge
- - cache.r5.large
- - cache.r5.xlarge
- - cache.r5.2xlarge
- - cache.r5.4xlarge
- - cache.r5.12xlarge
- - cache.r5.24xlarge
- - cache.r4.large
- - cache.r4.xlarge
- - cache.r4.2xlarge
- - cache.r4.4xlarge
- - cache.r4.8xlarge
- - cache.r4.16xlarge
- Type: String
-
- IndexCacheNumberOfCacheNodes:
- Description: An integer value specifying the number of node in the redis cache for the legal tag cache.
- Type: Number
- Default: 1
- MinValue: 1
- MaxValue: 128
-
- SchemaCacheName:
- Description: The name of the cache cluster for the schema cache. Will be prefixed with the environment name.
- Type: String
- MinLength: '1'
- MaxLength: '64'
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: indexerSchemaCache
-
- SchemaCacheEngine:
- Description: Which caching platform to use for the schema cache. Can be set to 'redis' or 'memcached'.
- Type: String
- AllowedValues:
- - redis
- - memcached
- ConstraintDescription: Can only be "redis" or "memcached"
- Default: redis
-
- SchemaCacheNodeInstanceType:
- Description: The instance type for redis cache nodes for the schema cache.
- ConstraintDescription: Must be a valid instance type from the list of allowed values.
- Default: cache.t2.micro
- AllowedValues:
- - cache.m5.large
- - cache.m5.xlarge
- - cache.m5.2xlarge
- - cache.m5.4xlarge
- - cache.m5.12xlarge
- - cache.m5.24xlarge
- - cache.m4.large
- - cache.m4.xlarge
- - cache.m4.2xlarge
- - cache.m4.4xlarge
- - cache.m4.10xlarge
- - cache.t2.micro
- - cache.t2.small
- - cache.t2.medium
- - cache.c1.xlarge
- - cache.r5.large
- - cache.r5.xlarge
- - cache.r5.2xlarge
- - cache.r5.4xlarge
- - cache.r5.12xlarge
- - cache.r5.24xlarge
- - cache.r4.large
- - cache.r4.xlarge
- - cache.r4.2xlarge
- - cache.r4.4xlarge
- - cache.r4.8xlarge
- - cache.r4.16xlarge
- Type: String
-
- SchemaCacheNumberOfCacheNodes:
- Description: An integer value specifying the number of node in the redis cache for the schema cache.
- Type: Number
- Default: 1
- MinValue: 1
- MaxValue: 128
-
- ElasticsearchDomainName:
- Description: The name of the Elasticsearch domain. Will be prefixed with the environment name.
- Type: String
- MinLength: '1'
- MaxLength: '64'
- AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
- ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: osdu-indexer
-
- ElasticsearchNodeInstanceType:
- Description: The instance type for the main Elasticsearch nodes.
- ConstraintDescription: Must be a valid instance type from the list of allowed values.
- Default: t2.medium.elasticsearch
- AllowedValues:
- - t2.small.elasticsearch
- - t2.medium.elasticsearch
- - m5.large.elasticsearch
- - m5.xlarge.elasticsearch
- - m5.2xlarge.elasticsearch
- - m5.4xlarge.elasticsearch
- - m5.12xlarge.elasticsearch
- - c5.large.elasticsearch
- - c5.xlarge.elasticsearch
- - c5.2xlarge.elasticsearch
- - c5.4xlarge.elasticsearch
- - c5.9xlarge.elasticsearch
- - c5.18xlarge.elasticsearch
- - r5.large.elasticsearch
- - r5.xlarge.elasticsearch
- - r5.2xlarge.elasticsearch
- - r5.4xlarge.elasticsearch
- - r5.12xlarge.elasticsearch
- - i3.large.elasticsearch
- - i3.xlarge.elasticsearch
- - i3.2xlarge.elasticsearch
- - i3.4xlarge.elasticsearch
- - i3.8xlarge.elasticsearch
- - i3.16xlarge.elasticsearch
- Type: String
-
- DedicatedMasterInstanceType:
- Description: >
- The instance type for the dedicated master nodes. These nodes perform cluster management
- tasks, but doesn't hold data or respond to data upload requests.
- ConstraintDescription: Must be a valid instance type from the list of allowed values.
- Default: t2.medium.elasticsearch
- AllowedValues:
- - t2.small.elasticsearch
- - t2.medium.elasticsearch
- - m5.large.elasticsearch
- - m5.xlarge.elasticsearch
- - m5.2xlarge.elasticsearch
- - m5.4xlarge.elasticsearch
- - m5.12xlarge.elasticsearch
- - c5.large.elasticsearch
- - c5.xlarge.elasticsearch
- - c5.2xlarge.elasticsearch
- - c5.4xlarge.elasticsearch
- - c5.9xlarge.elasticsearch
- - c5.18xlarge.elasticsearch
- - r5.large.elasticsearch
- - r5.xlarge.elasticsearch
- - r5.2xlarge.elasticsearch
- - r5.4xlarge.elasticsearch
- - r5.12xlarge.elasticsearch
- - i3.large.elasticsearch
- - i3.xlarge.elasticsearch
- - i3.2xlarge.elasticsearch
- - i3.4xlarge.elasticsearch
- - i3.8xlarge.elasticsearch
- - i3.16xlarge.elasticsearch
- Type: String
-
- NumberOfElasticsearchNodes:
- Description: An integer value specifying the number of Elasticsearch primary nodes in the cluster.
- Type: Number
- Default: 1
- MinValue: 1
- MaxValue: 40
-
- NumberOfDedicatedMasterNodes:
- Description: An integer value specifying the number of dedicated master nodes.
- Type: Number
- Default: 2
- MinValue: 2
- MaxValue: 5
-
- ZoneAwarenessEnabled:
- Description: >
- When Zone Awareness is enabled, Elasticsearch allocates the nodes and replica
- index shards that belong to a cluster across multiple AZs in the deployment region.
- Type: String
- AllowedValues:
- - true
- - false
- Default: false
-
- ElasticsearchVersion:
- Description: >
- The version of Elasticsearch to deploy on the cluster. Defaults to 6.8. Note
- that an update requires a full replacement of the Elasticsearch cluster.
- Type: String
- AllowedValues:
- - 1.5
- - 2.3
- - 5.1
- - 5.3
- - 5.5
- - 5.6
- - 6.0
- - 6.2
- - 6.3
- - 6.4
- - 6.5
- - 6.6
- - 6.8
- - 6.8
- - 7.1
- Default: 6.8
-
- EBSVolumeSize:
- Description: >
- The size of the EBS volume, in GiB, (per instance; total cluster size =
- EBS volume size x Instance count). Maximum size varies by instance type, from 35GiB
- for t2 instances, up to 12TiB for r5.12xlarge.
- Type: Number
- Default: 10
- MinValue: 10
- MaxValue: 12000
-
- ECSPort:
- Description: The port that the ECS Service will listen on.
- Type: Number
- Default: 80
- MinValue: 1
- MaxValue: 65535
-
- ECSCPUAllocation:
- Description: The amount of CPU resources to allocate to each ECS task/container. Scale - 1024 = 1 vCPU core.
- Type: Number
- Default: 1024
- MinValue: 10
- MaxValue: 65535
-
- ECSMemoryAllocation:
- Description: The amount of memory (RAM) to allocate to each ECS task/container. Scale - 1 = 1MB of memory.
- Type: Number
- Default: 2048
- MinValue: 256
- MaxValue: 131072
-
- DomainName:
- Description: >-
- The optional custom DNS name for the ECS service's load balancer. If omitted, the site will only be accessible
- via the ECS service's Application Load Balancer DNS name. This value is used in the creation and signing of
- the service's SSL certificate. Leave blank is not using a custom domain for this deployment.
- Type: String
- Default: ''
-
- HostedZoneName:
- Description: >-
- The name of the hosted zone (ex: for indexer.osdu.slb.com, this would likely be osdu.slb.com).
- Leave blank is not using a custom domain for this deployment.
- Type: String
- Default: ''
-
- AcmCertificateArn:
- Description: >-
- The Amazon Resource Name (ARN) of an existing AWS Certificate Manager (ACM) certificate.
- If omitted, a new SSL certified will be requested/generated (only if the custom domain name
- parameter is provided, otherwise the ECS service's ALB will not use SSL/HTTPS).
- Type: String
- AllowedPattern: "^(|arn:aws:acm:.*)$"
- Default: ''
-
-Resources:
-
- #### Shared Resources ################################################################
-
- IAMCredentialsStack:
- Type: 'AWS::CloudFormation::Stack'
- Properties:
- TemplateURL: !Sub
- - https://s3.amazonaws.com/${CloudFormationS3Bucket}/${ApplicationName}/Automated/${CFNTemplateFilename}
- - CloudFormationS3Bucket: !ImportValue
- 'Fn::Sub': '${Environment}-S3BucketCloudFormation'
- CFNTemplateFilename: iam-credentials.yml
- Parameters:
- Environment: !Ref Environment
- Region: !Ref DeploymentRegion
- IndexerServiceIamUsername: !Ref IndexerServiceIamUsername
- IndexerServiceIamKeyRotationSerial: !Ref IndexerServiceIamKeyRotationSerial
-
- MessageBusSNSStack:
- Type: 'AWS::CloudFormation::Stack'
- Properties:
- TemplateURL: !Sub
- - https://s3.amazonaws.com/${CloudFormationS3Bucket}/${ApplicationName}/Automated/${CFNTemplateFilename}
- - CloudFormationS3Bucket: !ImportValue
- 'Fn::Sub': '${Environment}-S3BucketCloudFormation'
- CFNTemplateFilename: sns-topic.yml
- Parameters:
- Environment: !Ref Environment
- Region: !Ref DeploymentRegion
- SNSTopicName: !Ref SNSTopicName
- SQSQueueName: !Ref SQSQueueName
-
- #### ECS Resources ###################################################################
-
- ECSNetworkStack:
- Type: 'AWS::CloudFormation::Stack'
- DependsOn: IAMCredentialsStack
- Properties:
- TemplateURL: !Sub
- - https://s3.amazonaws.com/${CloudFormationS3Bucket}/${ApplicationName}/Automated/${CFNTemplateFilename}
- - CloudFormationS3Bucket: !ImportValue
- 'Fn::Sub': '${Environment}-S3BucketCloudFormation'
- CFNTemplateFilename: ecs-network.yml
- Parameters:
- Environment: !Ref Environment
- Region: !Ref DeploymentRegion
- ApplicationName: !Ref ApplicationName
- ECSPort: !Ref ECSPort
- DomainName: !Ref DomainName
- AcmCertificateArn: !Ref AcmCertificateArn
-
- ECSClusterStack:
- Type: 'AWS::CloudFormation::Stack'
- DependsOn: [SchemaCacheStack, IndexCacheStack]
- Properties:
- TemplateURL: !Sub
- - https://s3.amazonaws.com/${CloudFormationS3Bucket}/${ApplicationName}/Automated/${CFNTemplateFilename}
- - CloudFormationS3Bucket: !ImportValue
- 'Fn::Sub': '${Environment}-S3BucketCloudFormation'
- CFNTemplateFilename: ecs-cluster.yml
- Parameters:
- Environment: !Ref Environment
- Region: !Ref DeploymentRegion
- ApplicationName: !Ref ApplicationName
- KeyName: !Ref KeyName
- DesiredCapacity: !Ref DesiredCapacity
- MaxSize: !Ref MaxSize
- InstanceType: !Ref InstanceType
- SchemaCacheName: !Ref SchemaCacheName
- IndexCacheName: !Ref IndexCacheName
- ECSPort: !Ref ECSPort
- SNSTopicName: !Ref SNSTopicName
- ECSMemoryAllocation: !Ref ECSMemoryAllocation
- DomainName: !Ref DomainName
- HostedZoneName: !Ref HostedZoneName
- ElasticsearchDomainName: !Ref ElasticsearchDomainName
-
- #### Caching Resources ###############################################################
-
- IndexCacheStack:
- Type: 'AWS::CloudFormation::Stack'
- DependsOn: ECSNetworkStack
- Properties:
- TemplateURL: !Sub
- - https://s3.amazonaws.com/${CloudFormationS3Bucket}/${ApplicationName}/Automated/${CFNTemplateFilename}
- - CloudFormationS3Bucket: !ImportValue
- 'Fn::Sub': '${Environment}-S3BucketCloudFormation'
- CFNTemplateFilename: cache.yml
- Parameters:
- Environment: !Ref Environment
- Region: !Ref DeploymentRegion
- ApplicationName: !Ref ApplicationName
- CacheName: !Ref IndexCacheName
- CacheEngine: !Ref IndexCacheEngine
- NodeInstanceType: !Ref IndexCacheNodeInstanceType
- NumberOfCacheNodes: !Ref IndexCacheNumberOfCacheNodes
-
- SchemaCacheStack:
- Type: 'AWS::CloudFormation::Stack'
- DependsOn: ECSNetworkStack
- Properties:
- TemplateURL: !Sub
- - https://s3.amazonaws.com/${CloudFormationS3Bucket}/${ApplicationName}/Automated/${CFNTemplateFilename}
- - CloudFormationS3Bucket: !ImportValue
- 'Fn::Sub': '${Environment}-S3BucketCloudFormation'
- CFNTemplateFilename: cache.yml
- Parameters:
- Environment: !Ref Environment
- Region: !Ref DeploymentRegion
- ApplicationName: !Ref ApplicationName
- CacheName: !Ref SchemaCacheName
- CacheEngine: !Ref SchemaCacheEngine
- NodeInstanceType: !Ref SchemaCacheNodeInstanceType
- NumberOfCacheNodes: !Ref SchemaCacheNumberOfCacheNodes
-
- #### Elasticsearch Resources #########################################################
-
- ElasticsearchStack:
- Type: 'AWS::CloudFormation::Stack'
- DependsOn: [IAMCredentialsStack, ECSNetworkStack]
- Properties:
- TemplateURL: !Sub
- - https://s3.amazonaws.com/${CloudFormationS3Bucket}/${ApplicationName}/Automated/${CFNTemplateFilename}
- - CloudFormationS3Bucket: !ImportValue
- 'Fn::Sub': '${Environment}-S3BucketCloudFormation'
- CFNTemplateFilename: elasticsearch.yml
- Parameters:
- Environment: !Ref Environment
- Region: !Ref DeploymentRegion
- ElasticsearchDomainName: !Ref ElasticsearchDomainName
- ElasticsearchNodeInstanceType: !Ref ElasticsearchNodeInstanceType
- DedicatedMasterInstanceType: !Ref DedicatedMasterInstanceType
- NumberOfElasticsearchNodes: !Ref NumberOfElasticsearchNodes
- NumberOfDedicatedMasterNodes: !Ref NumberOfDedicatedMasterNodes
- ZoneAwarenessEnabled: !Ref ZoneAwarenessEnabled
- ElasticsearchVersion: !Ref ElasticsearchVersion
- EBSVolumeSize: !Ref EBSVolumeSize
- ApplicationName: !Ref ApplicationName
- SearchApplicationName: !Ref SearchApplicationName
-
-Outputs:
- JarVersionNumber:
- Description: The service name associated with the JAR package for the Dockerfile.
- Value: !Ref 'VersionNumber'
- Export:
- Name: !Sub ${Environment}-${ApplicationName}-JarVersionNumber
-
- JarServiceName:
- Description: The service name associated with the JAR package for the Dockerfile.
- Value: !Ref 'ServiceName'
- Export:
- Name: !Sub ${Environment}-${ApplicationName}-JarServiceName
diff --git a/provider/indexer-aws/CloudFormation/Params/dev.template_configuration.json b/provider/indexer-aws/CloudFormation/Params/dev.template_configuration.json
deleted file mode 100644
index f33fd69a62180cc5c08e9962a46aae62705cabf5..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/CloudFormation/Params/dev.template_configuration.json
+++ /dev/null
@@ -1,44 +0,0 @@
-{
- "Parameters" : {
- "Environment" : "dev",
- "DeploymentRegion" : "us-east-1",
- "ApplicationName" : "os-indexer",
- "SearchApplicationName" : "os-search",
- "KeyName": "indexer-ecs-keypair",
- "DesiredCapacity": "2",
- "MinSize": "0",
- "MaxSize": "3",
- "InstanceType": "t3.large",
- "IndexerServiceIamUsername": "service-user-os-indexer",
- "IndexerServiceIamKeyRotationSerial": "1",
- "SNSTopicName": "osdu-indexer-messages",
- "SQSQueueName": "osdu-indexer-queue",
- "IndexCacheName": "indexerIndexCache",
- "IndexCacheEngine": "redis",
- "IndexCacheNodeInstanceType": "cache.t2.micro",
- "IndexCacheNumberOfCacheNodes": "1",
- "SchemaCacheName": "indexerSchemaCache",
- "SchemaCacheEngine": "redis",
- "SchemaCacheNodeInstanceType": "cache.t2.micro",
- "SchemaCacheNumberOfCacheNodes": "1",
- "ElasticsearchDomainName": "osdu-indexer",
- "ElasticsearchNodeInstanceType": "t2.medium.elasticsearch",
- "DedicatedMasterInstanceType": "t2.medium.elasticsearch",
- "NumberOfElasticsearchNodes": "4",
- "NumberOfDedicatedMasterNodes": "3",
- "ZoneAwarenessEnabled": "false",
- "ElasticsearchVersion": "6.8",
- "EBSVolumeSize": "10",
- "ECSPort": "443",
- "ECSCPUAllocation": "1024",
- "ECSMemoryAllocation": "3072",
- "DomainName": "",
- "HostedZoneName": "",
- "AcmCertificateArn": "",
- "ServiceName": "indexer",
- "VersionNumber": "1.0.5-SNAPSHOT"
- },
- "Tags" : {
- "Environment" : "dev"
- }
-}
diff --git a/provider/indexer-aws/CloudFormation/Params/prod.template_configuration.json b/provider/indexer-aws/CloudFormation/Params/prod.template_configuration.json
deleted file mode 100644
index cf2353331086afe91f04f7abcaa4c9736a4a5d84..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/CloudFormation/Params/prod.template_configuration.json
+++ /dev/null
@@ -1,44 +0,0 @@
-{
- "Parameters" : {
- "Environment" : "prod",
- "DeploymentRegion" : "us-east-1",
- "ApplicationName" : "os-indexer",
- "SearchApplicationName" : "os-search",
- "KeyName": "indexer-ecs-keypair",
- "DesiredCapacity": "2",
- "MinSize": "0",
- "MaxSize": "3",
- "InstanceType": "t3.large",
- "IndexerServiceIamUsername": "service-user-os-indexer",
- "IndexerServiceIamKeyRotationSerial": "1",
- "SNSTopicName": "osdu-indexer-messages",
- "SQSQueueName": "osdu-indexer-queue",
- "IndexCacheName": "indexerIndexCache",
- "IndexCacheEngine": "redis",
- "IndexCacheNodeInstanceType": "cache.t2.micro",
- "IndexCacheNumberOfCacheNodes": "1",
- "SchemaCacheName": "indexerSchemaCache",
- "SchemaCacheEngine": "redis",
- "SchemaCacheNodeInstanceType": "cache.t2.micro",
- "SchemaCacheNumberOfCacheNodes": "1",
- "ElasticsearchDomainName": "osdu-indexer",
- "ElasticsearchNodeInstanceType": "t2.medium.elasticsearch",
- "DedicatedMasterInstanceType": "t2.medium.elasticsearch",
- "NumberOfElasticsearchNodes": "2",
- "NumberOfDedicatedMasterNodes": "2",
- "ZoneAwarenessEnabled": "false",
- "ElasticsearchVersion": "6.8",
- "EBSVolumeSize": "10",
- "ECSPort": "443",
- "ECSCPUAllocation": "1024",
- "ECSMemoryAllocation": "3072",
- "DomainName": "",
- "HostedZoneName": "",
- "AcmCertificateArn": "",
- "ServiceName": "indexer",
- "VersionNumber": "1.0.5-SNAPSHOT"
- },
- "Tags" : {
- "Environment" : "prod"
- }
-}
diff --git a/provider/indexer-aws/CloudFormation/Params/uat.template_configuration.json b/provider/indexer-aws/CloudFormation/Params/uat.template_configuration.json
deleted file mode 100644
index 10d7331ed0253013feff3eaf96e468aca3b80394..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/CloudFormation/Params/uat.template_configuration.json
+++ /dev/null
@@ -1,44 +0,0 @@
-{
- "Parameters" : {
- "Environment" : "uat",
- "DeploymentRegion" : "us-east-1",
- "ApplicationName" : "os-indexer",
- "SearchApplicationName" : "os-search",
- "KeyName": "indexer-ecs-keypair",
- "DesiredCapacity": "2",
- "MinSize": "0",
- "MaxSize": "3",
- "InstanceType": "t3.large",
- "IndexerServiceIamUsername": "service-user-os-indexer",
- "IndexerServiceIamKeyRotationSerial": "1",
- "SNSTopicName": "osdu-indexer-messages",
- "SQSQueueName": "osdu-indexer-queue",
- "IndexCacheName": "indexerIndexCache",
- "IndexCacheEngine": "redis",
- "IndexCacheNodeInstanceType": "cache.t2.micro",
- "IndexCacheNumberOfCacheNodes": "1",
- "SchemaCacheName": "indexerSchemaCache",
- "SchemaCacheEngine": "redis",
- "SchemaCacheNodeInstanceType": "cache.t2.micro",
- "SchemaCacheNumberOfCacheNodes": "1",
- "ElasticsearchDomainName": "osdu-indexer",
- "ElasticsearchNodeInstanceType": "t2.medium.elasticsearch",
- "DedicatedMasterInstanceType": "t2.medium.elasticsearch",
- "NumberOfElasticsearchNodes": "2",
- "NumberOfDedicatedMasterNodes": "2",
- "ZoneAwarenessEnabled": "false",
- "ElasticsearchVersion": "6.8",
- "EBSVolumeSize": "10",
- "ECSPort": "443",
- "ECSCPUAllocation": "1024",
- "ECSMemoryAllocation": "3072",
- "DomainName": "",
- "HostedZoneName": "",
- "AcmCertificateArn": "",
- "ServiceName": "indexer",
- "VersionNumber": "1.0.5-SNAPSHOT"
- },
- "Tags" : {
- "Environment" : "uat"
- }
-}
diff --git a/provider/indexer-aws/Dockerfile b/provider/indexer-aws/build-aws/Dockerfile
similarity index 74%
rename from provider/indexer-aws/Dockerfile
rename to provider/indexer-aws/build-aws/Dockerfile
index 07b7281a7d160013df35a5a150fdcaa8f5bf2c53..4af0e6ae8f3b38370edf5354972609066316b86b 100644
--- a/provider/indexer-aws/Dockerfile
+++ b/provider/indexer-aws/build-aws/Dockerfile
@@ -12,15 +12,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+# https://docs.spring.io/spring-boot/docs/current/reference/html/deployment.html
FROM amazoncorretto:8
-ARG versionNumber
-ARG service
-ENV serviceName=${service}-aws
-ENV awsJar=${serviceName}-${versionNumber}-spring-boot.jar
-
+ARG JAR_FILE=provider/indexer-aws/target/*spring-boot.jar
WORKDIR /
-COPY provider/${serviceName}/target/${awsJar} ${awsJar}
+COPY ${JAR_FILE} app.jar
EXPOSE 8080
-
-CMD ["sh","-c", " java -jar ${awsJar}"]
+ENTRYPOINT java $JAVA_OPTS -jar /app.jar
\ No newline at end of file
diff --git a/provider/indexer-aws/buildspec-jar-deploy.yml b/provider/indexer-aws/buildspec-jar-deploy.yml
deleted file mode 100644
index 546f75409fd9e37722dd85a7298633e2f3b0932b..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/buildspec-jar-deploy.yml
+++ /dev/null
@@ -1,59 +0,0 @@
-# Copyright © Amazon Web Services
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: 0.2
-
-phases:
- install:
- runtime-versions:
- java: openjdk8
- commands:
- - echo Entered the install phase...
- - apt-get update -y
- - apt-get install -y maven
- - java -version
- - mvn clean # .m2 is not created until the first Maven command
- - cp ./provider/indexer-aws/maven/settings.xml /root/.m2/settings.xml # copy the AWS-specific settings.xml to the CodeBuild instance's .m2 folder
- - cat /root/.m2/settings.xml
- - java -version
- - export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
- - echo $JAVA_HOME
- - mvn -version
- - echo "Look below for M2 bucket name:"
- - echo $M2_REPO_S3_BUCKET
- - aws s3 sync s3://$M2_REPO_S3_BUCKET /root/.m2 # copy previous state of the shared libraries' .m2 folder from S3 to local
- - nohup /usr/local/bin/dockerd --host=unix:///var/run/docker.sock --host=tcp://127.0.0.1:2375 --storage-driver=overlay2& # start the Docker Daemon
- - timeout 15 sh -c "until docker info; do echo .; sleep 1; done" # wait for Docker to be ready before proceeding to the build steps
- build:
- commands:
- - echo os-indexer Java build started on `date`...
- - java -version
- - export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
- - mvn -version
- - echo All environment variables
- - printenv
- - mvn clean test -P indexer-core,indexer-aws
- - echo ...os-indexer Java build completed on `date`.
- - echo os-indexer beginning packaging to jar...
- - mvn clean install -P indexer-core,indexer-aws -Ddeployment.environment=$ENVIRONMENT
- - echo Uploading os-indexer JAR to S3...
- - aws s3 cp provider/$JAR_SERVICE_BASE-aws/target s3://$JAR_DEPLOY_S3_BUCKET/$JAR_SERVICE_BASE-aws --recursive --exclude "*" --include "*.jar" # build and push the JAR(s) to S3
-
-cache:
- paths:
- - '/root/.m2/**/*'
-
-artifacts:
- files:
- - '**/*'
diff --git a/provider/indexer-aws/buildspec-post-deploy.yml b/provider/indexer-aws/buildspec-post-deploy.yml
deleted file mode 100644
index 5a576fa4c687e91a904ee66939f01d99df4b0981..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/buildspec-post-deploy.yml
+++ /dev/null
@@ -1,87 +0,0 @@
-# Copyright © Amazon Web Services
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: 0.2
-
-phases:
- install:
- runtime-versions:
- java: openjdk8
- commands:
- - echo Entered the install phase...
- - apt-get update -y
- - apt-get install -y maven
- - java -version
- - mvn clean # .m2 is not created until the first Maven command
- - cp ./provider/indexer-aws/maven/settings.xml /root/.m2/settings.xml # copy the AWS-specific settings.xml to the CodeBuild instance's .m2 folder
- - cat /root/.m2/settings.xml
- - java -version
- - export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
- - echo $JAVA_HOME
- - mvn -version
- - echo "Look below for M2 bucket name:"
- - echo $M2_REPO_S3_BUCKET
- - aws s3 sync s3://$M2_REPO_S3_BUCKET /root/.m2 # copy previous state of the shared libraries' .m2 folder from S3 to local
- - nohup /usr/local/bin/dockerd --host=unix:///var/run/docker.sock --host=tcp://127.0.0.1:2375 --storage-driver=overlay2& # start the Docker Daemon
- - timeout 15 sh -c "until docker info; do echo .; sleep 1; done" # wait for Docker to be ready before proceeding to the build steps
- pre_build:
- commands:
- - echo Logging in to Amazon ECR...
- - $(aws ecr get-login --no-include-email --region $AWS_REGION)
- - echo $AWS_ACCOUNT_ID
- - REPOSITORY_URI=$AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/$IMAGE_REPO_NAME # build and store the ECR repo URI
- - IMAGE_TAG=build-$(echo $CODEBUILD_BUILD_ID | awk -F":" '{print $2}') # generate a version tag from the commit hash for the Docker image
- - COMMIT_HASH=$(echo $CODEBUILD_RESOLVED_SOURCE_VERSION | cut -c 1-7) # get the commit hash
- build:
- commands:
- - echo Indexer-core Java build started on `date`...
- - echo os-indexer Java build started on `date`...
- - java -version
- - export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
- - mvn -version
- - echo Setting environment variables from CloudFormation Exports... # use the AWS CLI commands to query for the CloudFormation export values created in the previous step and set the required environment variables
- - echo Environment - $ENVIRONMENT
- - echo SchemaCacheName - $SCHEMA_CACHE_NAME
- - echo IndexCacheName - $INDEX_CACHE_NAME
- - echo AWSRegion - $AWS_REGION
- - export VERSIONNUMBER=$(aws cloudformation list-exports --query "Exports[?Name=='$ENVIRONMENT-$APPLICATION_NAME-JarVersionNumber'].[Value]" --output text --region $AWS_REGION)
- - echo VERSIONNUMBER - $VERSIONNUMBER
- - export SERVICE=$(aws cloudformation list-exports --query "Exports[?Name=='$ENVIRONMENT-$APPLICATION_NAME-JarServiceName'].[Value]" --output text --region $AWS_REGION)
- - echo SERVICE - $SERVICE
- - echo ...finished setting environment variables!
- - echo All environment variables
- - printenv
- - mvn clean test -P indexer-core,indexer-aws
- - echo ...os-indexer Java build completed on `date`.
- - echo os-indexer beginning packaging to jar...
- - mvn clean install -P indexer-core,indexer-aws -Ddeployment.environment=$ENVIRONMENT -Dversion.number=$VERSIONNUMBER
- - echo os-indexer Docker image build started on `date`...
- - docker build -f provider/indexer-aws/Dockerfile -t $REPOSITORY_URI:latest --build-arg versionNumber=$VERSIONNUMBER --build-arg service=$SERVICE .
- - docker tag $REPOSITORY_URI:latest $REPOSITORY_URI:$IMAGE_TAG
- - echo ...os-indexer Docker image build completed on `date`.
- - echo Pushing the Docker image to ECR...
- - docker push $REPOSITORY_URI:latest
- - docker push $REPOSITORY_URI:$IMAGE_TAG
- - echo Docker image pushed to ECR successfully!
- - ECS_CLUSTER_NAME=$(aws cloudformation list-exports --query "Exports[?Name=='$ENVIRONMENT-$APPLICATION_NAME-EcsClusterName'].[Value]" --output text --region $AWS_REGION)
- - ECS_SERVICE_NAME=$(aws cloudformation list-exports --query "Exports[?Name=='$ENVIRONMENT-$APPLICATION_NAME-EcsServiceName'].[Value]" --output text --region $AWS_REGION)
- - aws ecs update-service --cluster $ECS_CLUSTER_NAME --service $ECS_SERVICE_NAME --force-new-deployment # force a new deployment with the updated image
-
-cache:
- paths:
- - '/root/.m2/**/*'
-
-artifacts:
- files:
- - '**/*'
diff --git a/provider/indexer-aws/buildspec-pre-deploy.yml b/provider/indexer-aws/buildspec-pre-deploy.yml
deleted file mode 100644
index 14697d9c882a1905a8205f06328611e2d3c478ef..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/buildspec-pre-deploy.yml
+++ /dev/null
@@ -1,61 +0,0 @@
-# Copyright © Amazon Web Services
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: 0.2
-
-phases:
- install:
- runtime-versions:
- java: openjdk8
- docker: 18
- commands:
- - echo Entered the install phase...
- - apt-get update -y
- - apt-get install -y maven
- - java -version
- - echo $JAVA_HOME
- - export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
- - echo $JAVA_HOME
- - mvn clean # .m2 is not created until the first Maven command
- - cp ./indexer-core/maven/settings.xml /root/.m2/settings.xml # replace the default settings.xml with our custom one
- - export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
- - mvn -version
- build:
- commands:
- - echo Starting 'Copying CloudFormation scripts to S3://$CFN_S3_BUCKET/$APPLICATION_NAME'
- - pwd
- - ls
- - aws s3 cp ./provider/indexer-aws/CloudFormation "s3://$CFN_S3_BUCKET/$APPLICATION_NAME" --exclude "*" --include "*.yml" --recursive --debug
- - echo Ending 'Ending CloudFormation scripts to S3://$CFN_S3_BUCKET/$APPLICATION_NAME'
-# - echo os-indexer build started on `date`...
- - pwd
- - ls -R -la
- - java -version
- - echo $JAVA_HOME
- - export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
- - mvn -version
-# - mvn test
- post_build:
- commands:
-# - echo ...os-indexer build completed on `date`
-# - echo os-indexer beginning packaging to jar...
-# - mvn package
-
-cache:
- paths:
- - '/root/.m2/**/*'
-
-artifacts:
- files:
- - '**/*'
diff --git a/provider/indexer-aws/maven/settings.xml b/provider/indexer-aws/maven/settings.xml
index adba9a7602b0887236967efc41b49ccad08a7ca1..0ca9fe8103a2386f2ff0479c788306e1312faa3d 100644
--- a/provider/indexer-aws/maven/settings.xml
+++ b/provider/indexer-aws/maven/settings.xml
@@ -11,6 +11,6 @@
<!-- The generated token was last updated on 01/13/2020 and expires on or before 01/12/2021 -->
<password>${azure.devops.token}</password>
</server>
-
</servers>
+
</settings>
diff --git a/provider/indexer-aws/pom.xml b/provider/indexer-aws/pom.xml
index aa763a675d577afe9e3380606336703d0a09a48c..6130309d7b97ce3d8abd764bd66927219b1e9fe9 100644
--- a/provider/indexer-aws/pom.xml
+++ b/provider/indexer-aws/pom.xml
@@ -50,7 +50,7 @@
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
<artifactId>os-core-lib-aws</artifactId>
- <version>0.0.10</version>
+ <version>0.2.0</version>
</dependency>
<!-- AWS managed packages -->
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/IndexCacheImpl.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/IndexCacheImpl.java
index b8953ba2d6cc3aaee5e99565bbc2417413f5d153..c28acd5c3f9d213c5658737d8d393475c85001a9 100644
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/IndexCacheImpl.java
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/IndexCacheImpl.java
@@ -24,8 +24,8 @@ public class IndexCacheImpl implements IIndexCache<String, Boolean>, AutoCloseab
private RedisCache<String, Boolean> cache;
- public IndexCacheImpl(@Value("${aws.elasticache.cluster.index.endpoint}") final String REDIS_SEARCH_HOST,
- @Value("${aws.elasticache.cluster.index.port}") final String REDIS_SEARCH_PORT,
+ public IndexCacheImpl(@Value("${aws.elasticache.cluster.endpoint}") final String REDIS_SEARCH_HOST,
+ @Value("${aws.elasticache.cluster.port}") final String REDIS_SEARCH_PORT,
@Value("${aws.elasticache.cluster.index.expiration}") final String INDEX_CACHE_EXPIRATION) {
cache = new RedisCache<>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT),
Integer.parseInt(INDEX_CACHE_EXPIRATION) * 60, String.class, Boolean.class);
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/SchemaCacheImpl.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/SchemaCacheImpl.java
index 112b3b2f8b1a591f785edb675b7a84cfe2cf5198..e0a713737ccb356313ff9d5edf43ebeac6fda331 100644
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/SchemaCacheImpl.java
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/SchemaCacheImpl.java
@@ -24,8 +24,8 @@ public class SchemaCacheImpl implements ISchemaCache<String, String>, AutoClosea
private RedisCache<String, String> cache;
- public SchemaCacheImpl(@Value("${aws.elasticache.cluster.schema.endpoint}") final String REDIS_SEARCH_HOST,
- @Value("${aws.elasticache.cluster.schema.port}") final String REDIS_SEARCH_PORT,
+ public SchemaCacheImpl(@Value("${aws.elasticache.cluster.endpoint}") final String REDIS_SEARCH_HOST,
+ @Value("${aws.elasticache.cluster.port}") final String REDIS_SEARCH_PORT,
@Value("${aws.elasticache.cluster.schema.expiration}") final String SCHEMA_CACHE_EXPIRATION) {
cache = new RedisCache<>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT),
Integer.parseInt(SCHEMA_CACHE_EXPIRATION) * 60, String.class, String.class);
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/di/EntitlementsFactoryImpl.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/di/EntitlementsFactoryImpl.java
deleted file mode 100644
index 85517a87d6873f420b325e2201612e17181f3e1e..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/di/EntitlementsFactoryImpl.java
+++ /dev/null
@@ -1,36 +0,0 @@
-// Copyright © Amazon Web Services
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.aws.di;
-
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.entitlements.IEntitlementsFactory;
-import org.opengroup.osdu.core.common.entitlements.IEntitlementsService;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.context.annotation.Primary;
-import org.springframework.stereotype.Component;
-
-@Component
-@Primary
-public class EntitlementsFactoryImpl implements IEntitlementsFactory {
- @Value("${aws.lambda.get-groups-function-name}")
- private String getGroupsFunctionName;
-
- @Override
- public IEntitlementsService create(DpsHeaders headers) {
- EntitlementsServiceImpl service = new EntitlementsServiceImpl(headers);
- service.setEntitlementsServiceHelper(getGroupsFunctionName);
- return service;
- }
-}
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/di/EntitlementsServiceImpl.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/di/EntitlementsServiceImpl.java
deleted file mode 100644
index 3dd657c6699407a4557f4d149fcf2a241d39e762..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/di/EntitlementsServiceImpl.java
+++ /dev/null
@@ -1,138 +0,0 @@
-// Copyright © Amazon Web Services
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.aws.di;
-
-import com.amazonaws.regions.Regions;
-import com.amazonaws.services.lambda.invoke.LambdaFunctionException;
-import com.amazonaws.services.lambda.invoke.LambdaSerializationException;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.opengroup.osdu.core.common.model.entitlements.*;
-import org.opengroup.osdu.core.common.model.entitlements.MemberInfo;
-import org.opengroup.osdu.core.common.model.entitlements.Members;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.entitlements.IEntitlementsService;
-import org.opengroup.osdu.core.aws.entitlements.*;
-import org.opengroup.osdu.core.common.http.HttpResponse;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.springframework.context.annotation.Lazy;
-import org.springframework.http.HttpStatus;
-import sun.reflect.generics.reflectiveObjects.NotImplementedException;
-
-import javax.inject.Inject;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-public class EntitlementsServiceImpl implements IEntitlementsService {
- private DpsHeaders dpsHeaders;
- private EntitlementsServiceHelper entitlementsServiceHelper;
-
- private final static String ACCESS_DENIED = "Access denied";
- private final static String ACCESS_DENIED_MSG = "The user is not authorized to perform this action";
-
- @Inject
- @Lazy
- private JaxRsDpsLog jaxRsDpsLog;
-
- public EntitlementsServiceImpl(DpsHeaders headers){
- this.dpsHeaders = headers;
- }
-
- public void setEntitlementsServiceHelper(String getGroupsFunctionName){
- entitlementsServiceHelper = new EntitlementsServiceHelper(Regions.US_EAST_1, getGroupsFunctionName);
- }
-
- @Override
- public MemberInfo addMember(GroupEmail groupEmail, MemberInfo memberInfo) throws EntitlementsException {
- throw new NotImplementedException();
- }
-
- @Override
- public Members getMembers(GroupEmail groupEmail, GetMembers getMembers) throws EntitlementsException {
- throw new NotImplementedException();
- }
-
- @Override
- public Groups getGroups() throws EntitlementsException {
- Groups groups;
- GroupsRequest request = entitlementsServiceHelper.constructRequest(this.dpsHeaders.getHeaders());
-
- try{
- GroupsResult groupsResult = entitlementsServiceHelper.getGroups(request);
- groups = getGroupsFromResult(groupsResult);
- } catch (JsonProcessingException e) {
- throw new EntitlementsException(e.getMessage(), new HttpResponse());
- } catch (LambdaFunctionException e){
- throw new EntitlementsException(e.getMessage(), new HttpResponse());
- } catch (LambdaSerializationException e){
- throw new EntitlementsException(e.getMessage(), new HttpResponse());
- } catch (IOException e){
- throw new EntitlementsException(e.getMessage(), new HttpResponse());
- }
-
- return groups;
- }
-
- @Override
- public GroupInfo createGroup(CreateGroup createGroup) throws EntitlementsException {
- throw new NotImplementedException();
- }
-
- @Override
- public void deleteMember(String s, String s1) throws EntitlementsException {
- throw new NotImplementedException();
- }
-
- @Override
- public Groups authorizeAny(String... strings) throws EntitlementsException {
- throw new NotImplementedException();
- }
-
- @Override
- public void authenticate() throws EntitlementsException {
- throw new NotImplementedException();
- }
-
- private Groups getGroupsFromResult(GroupsResult result) throws EntitlementsException, IOException {
- ObjectMapper mapper = new ObjectMapper();
- Groups groups = new Groups();
- if(result.statusCode == HttpStatus.OK.value()) {
- TypeReference<List<GroupInfoRaw>> mapType = new TypeReference<List<GroupInfoRaw>>() {};
- List<GroupInfoRaw> groupInfosRaw = mapper.readValue(result.body, mapType);
- List<GroupInfo> groupInfos = new ArrayList<>();
- for(GroupInfoRaw groupInfoRaw : groupInfosRaw){
- GroupInfo groupInfo = new GroupInfo();
- groupInfo.setDescription(groupInfoRaw.groupDescription);
- groupInfo.setEmail(groupInfoRaw.groupEmail);
- groupInfo.setName(groupInfoRaw.groupName);
- groupInfos.add(groupInfo);
- }
- groups.setDesId(result.headers.get(RequestKeys.USER_HEADER_KEY));
- groups.setMemberEmail(result.headers.get(RequestKeys.USER_HEADER_KEY));
- groups.setGroups(groupInfos);
- } else {
- if(result.statusCode == HttpStatus.UNAUTHORIZED.value()){
- throw new AppException(HttpStatus.FORBIDDEN.value(), ACCESS_DENIED, ACCESS_DENIED_MSG);
- } else {
- throw new EntitlementsException(String.format("Getting groups for user returned %s status code",
- result.statusCode), new HttpResponse());
- }
- }
- return groups;
- }
-}
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/persistence/ElasticRepositoryImpl.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/persistence/ElasticRepositoryImpl.java
index 70e64ca21d70497fc21121729350d5b16f2a6a44..b65add33a07eb10ce43a89218344f01ab3dae4b0 100644
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/persistence/ElasticRepositoryImpl.java
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/persistence/ElasticRepositoryImpl.java
@@ -14,17 +14,19 @@
package org.opengroup.osdu.indexer.aws.persistence;
+import org.opengroup.osdu.core.aws.ssm.ParameterStorePropertySource;
+import org.opengroup.osdu.core.aws.ssm.SSMConfig;
import org.opengroup.osdu.core.common.model.search.ClusterSettings;
import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
+import javax.annotation.PostConstruct;
+
@Component
public class ElasticRepositoryImpl implements IElasticRepository {
- // TODO: Will need to be implemented later
-
@Value("${aws.es.host}")
String host;
@@ -33,6 +35,27 @@ public class ElasticRepositoryImpl implements IElasticRepository {
String userNameAndPassword = "testing";
+ @Value("${aws.elasticsearch.port}")
+ String portParameter;
+
+ @Value("${aws.elasticsearch.host}")
+ String hostParameter;
+
+ @Value("${aws.ssm}")
+ String ssmEnabledString;
+
+ private ParameterStorePropertySource ssm;
+
+ @PostConstruct
+ private void postConstruct() {
+ if( Boolean.parseBoolean(ssmEnabledString)) {
+ SSMConfig ssmConfig = new SSMConfig();
+ ssm = ssmConfig.amazonSSM();
+ host = ssm.getProperty(hostParameter).toString();
+ port = Integer.parseInt(ssm.getProperty(portParameter).toString());
+ }
+ }
+
@Override
public ClusterSettings getElasticClusterSettings(TenantInfo tenantInfo) {
return new ClusterSettings(host, port, userNameAndPassword);
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/publish/PublisherImpl.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/publish/PublisherImpl.java
index c2efd5c7e235efec8719edabd21ccf64d7bd7613..d5a5ae5195eb483fa7bc11e7eb06839c293f1367 100644
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/publish/PublisherImpl.java
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/publish/PublisherImpl.java
@@ -17,6 +17,8 @@ package org.opengroup.osdu.indexer.aws.publish;
import com.amazonaws.services.sns.model.MessageAttributeValue;
import com.amazonaws.services.sns.model.PublishRequest;
import com.amazonaws.services.sns.AmazonSNS;
+import org.opengroup.osdu.core.aws.ssm.ParameterStorePropertySource;
+import org.opengroup.osdu.core.aws.ssm.SSMConfig;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.aws.sns.AmazonSNSConfig;
import org.opengroup.osdu.core.aws.sns.PublishRequestBuilder;
@@ -34,16 +36,24 @@ public class PublisherImpl implements IPublisher {
AmazonSNS snsClient;
- @Value("${aws.sns.arn}")
+ private ParameterStorePropertySource ssm;
+
private String amazonSNSTopic;
- @Value("${aws.sns.region}")
+ @Value("${aws.region}")
private String amazonSNSRegion;
+ @Value("${aws.indexer.sns.topic.arn}")
+ private String parameter;
+
+
@Inject
public void init(){
AmazonSNSConfig snsConfig = new AmazonSNSConfig(amazonSNSRegion);
snsClient = snsConfig.AmazonSNS();
+ SSMConfig ssmConfig = new SSMConfig();
+ ssm = ssmConfig.amazonSSM();
+ amazonSNSTopic = ssm.getProperty(parameter).toString();
}
public void publishStatusChangedTagsToTopic(DpsHeaders headers, JobStatus indexerBatchStatus) throws Exception
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java
index bffecf18d7765fe7c52acaddc7dab246c64e13ec..5930c6a3cebbd46d0228b1dbf066162046e7d4fa 100644
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java
@@ -21,6 +21,8 @@ import com.amazonaws.services.sqs.AmazonSQS;
import com.amazonaws.services.sqs.model.SendMessageRequest;
import com.google.gson.Gson;
import org.opengroup.osdu.core.aws.sns.AmazonSNSConfig;
+import org.opengroup.osdu.core.aws.ssm.ParameterStorePropertySource;
+import org.opengroup.osdu.core.aws.ssm.SSMConfig;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.aws.sqs.AmazonSQSConfig;
import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
@@ -39,21 +41,29 @@ public class IndexerQueueTaskBuilderAws extends IndexerQueueTaskBuilder {
private AmazonSNS snsClient;
- @Value("${aws.region}")
- private String region;
+ private ParameterStorePropertySource ssm;
- @Value("${aws.sns.storage.arn}")
private String amazonSNSTopic;
private String retryString = "retry";
private Gson gson;
+ @Value("${aws.region}")
+ private String region;
+
+ @Value("${aws.storage.sns.topic.arn}")
+ String parameter;
+
+
@Inject
public void init() {
AmazonSNSConfig config = new AmazonSNSConfig(region);
snsClient = config.AmazonSNS();
gson =new Gson();
+ SSMConfig ssmConfig = new SSMConfig();
+ ssm = ssmConfig.amazonSSM();
+ amazonSNSTopic = ssm.getProperty(parameter).toString();
}
@Override
diff --git a/provider/indexer-aws/src/main/resources/application.properties b/provider/indexer-aws/src/main/resources/application.properties
index 3ddc087e9e18032a061f0a2d5a8384df645d9a69..78fdb52f7df76dec63d2cdaffa1c5bed067c1d4b 100644
--- a/provider/indexer-aws/src/main/resources/application.properties
+++ b/provider/indexer-aws/src/main/resources/application.properties
@@ -1,38 +1,17 @@
LOG_PREFIX=indexer
server.servlet.contextPath=/api/indexer/v2/
-logging.level.org.springframework.web=DEBUG
+logging.level.org.springframework.web=${LOG_LEVEL}
server.port=${APPLICATION_PORT}
-JAVA_HEAP_OPTS=-Xms${JAVA_HEAP_MEMORY}M -Xmx${JAVA_HEAP_MEMORY}M
-JAVA_GC_OPTS=-XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45
aws.threads=50
DEFAULT_DATA_COUNTRY=US
CRON_INDEX_CLEANUP_THRESHOLD_DAYS=3
CRON_EMPTY_INDEX_CLEANUP_THRESHOLD_DAYS=7
-## AWS DynamoDB configuration
-aws.dynamodb.key=kind
-aws.dynamodb.table.prefix=${ENVIRONMENT}-
-aws.dynamodb.region=${AWS_REGION}
-aws.dynamodb.endpoint=dynamodb.${AWS_REGION}.amazonaws.com
-
-## AWS S3 configuration
-aws.s3.region=${AWS_REGION}
-aws.s3.endpoint=s3.${AWS_REGION}.amazonaws.com
-aws.s3.records.bucket-name=${ENVIRONMENT}-${S3_DATA_BUCKET}
-aws.s3.max-record-threads=2000
-aws.s3.enable-https=true
-
-## AWS SNS configuration
-aws.sns.region=${AWS_REGION}
-aws.sns.arn=arn:aws:sns:${AWS_REGION}:${AWS_ACCOUNT_ID}:${ENVIRONMENT}-${SNS_TOPIC_NAME}
-aws.sns.storage.arn=arn:aws:sns:${AWS_REGION}:${AWS_ACCOUNT_ID}:${SNS_STORAGE_TOPIC_NAME}
-
-## AWS SQS Configuration
-aws.sqs.queue=${ENVIRONMENT}-osdu-indexer-queue
-
# AWS ES configuration
+ELASTIC_HOST=""
+ELASTIC_PORT=0
aws.es.host=${ELASTIC_HOST}
aws.es.port=${ELASTIC_PORT}
aws.es.userNameAndPassword=notused
@@ -41,26 +20,34 @@ aws.es.serviceName=es
GAE_SERVICE=indexer
-# TODO This needs to be changed so it snot hard
-STORAGE_SCHEMA_HOST=https://${STORAGE_HOST}/api/storage/v2/schemas
-STORAGE_QUERY_RECORD_HOST=https://${STORAGE_HOST}/api/storage/v2/query/records
-STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=https://${STORAGE_HOST}/api/storage/v2/query/records:batch
+STORAGE_SCHEMA_HOST=${STORAGE_HOST}/api/storage/v2/schemas
+STORAGE_QUERY_RECORD_HOST=${STORAGE_HOST}/api/storage/v2/query/records
+STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=${STORAGE_HOST}/api/storage/v2/query/records:batch
STORAGE_RECORDS_BATCH_SIZE=20
-INDEXER_QUEUE_HOST=http://sqs.${AWS_REGION}.amazonaws.com/${AWS_ACCOUNT_ID}/${ENVIRONMENT}-osdu-indexer-queue
-
+INDEXER_QUEUE_HOST=""
## AWS ElastiCache configuration
-aws.elasticache.cluster.index.endpoint=${CACHE_CLUSTER_INDEX_ENDPOINT}
-aws.elasticache.cluster.index.port=${CACHE_CLUSTER_INDEX_PORT}
-aws.elasticache.cluster.schema.endpoint=${CACHE_CLUSTER_SCHEMA_ENDPOINT}
-aws.elasticache.cluster.schema.port=${CACHE_CLUSTER_SCHEMA_PORT}
+aws.elasticache.cluster.endpoint=${CACHE_CLUSTER_ENDPOINT}
+aws.elasticache.cluster.port=${CACHE_CLUSTER_PORT}
## Cache Settings
aws.elasticache.cluster.index.expiration=60
aws.elasticache.cluster.schema.expiration=60
-
# Maximum size of cache value
MAX_CACHE_VALUE_SIZE=1000
## AWS Lambda configuration
-aws.lambda.get-groups-function-name=${ENVIRONMENT}-os-entitlements-GroupsFunction
\ No newline at end of file
+aws.lambda.get-groups-function-name=${ENVIRONMENT}-os-entitlements-GroupsFunction
+
+
+## Default DynamoDB Settings
+aws.dynamodb.table.prefix=${ENVIRONMENT}-
+aws.dynamodb.endpoint=dynamodb.${AWS_REGION}.amazonaws.com
+
+aws.ssm=${SSM_ENABLED}
+aws.ssm.prefix=/osdu/${ENVIRONMENT}
+
+aws.elasticsearch.host=${aws.ssm.prefix}/elastic-search/end-point
+aws.elasticsearch.port=${aws.ssm.prefix}/elastic-search/end-point-port
+aws.indexer.sns.topic.arn=${aws.ssm.prefix}/indexer/indexer-sns-topic-arn
+aws.storage.sns.topic.arn=${aws.ssm.prefix}/storage/storage-sns-topic-arn
diff --git a/testing/indexer-test-aws/pom.xml b/testing/indexer-test-aws/pom.xml
index 551f70b808eca63f6e409c470c43e818e0db58b9..d1fa34d89e7d8b903aa0f135ec070080185c2ff8 100644
--- a/testing/indexer-test-aws/pom.xml
+++ b/testing/indexer-test-aws/pom.xml
@@ -62,7 +62,7 @@
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
<artifactId>os-core-lib-aws</artifactId>
- <version>0.0.10</version>
+ <version>0.2.0</version>
</dependency>
<!-- Testing -->
@@ -149,13 +149,4 @@
</dependency>
</dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <version>2.5</version>
- </plugin>
- </plugins>
- </build>
</project>
\ No newline at end of file
diff --git a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
index 83c36a2456d3ecbe1284db5c1ded925d49d731aa..7b1c280c6b9bd96a111c8cfcdfffcefc3c60f09d 100644
--- a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
+++ b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -16,6 +16,7 @@ package org.opengroup.osdu.step_definitions.index.record;
import lombok.extern.java.Log;
import org.opengroup.osdu.common.RecordSteps;
+import org.opengroup.osdu.core.common.model.legal.Legal;
import org.opengroup.osdu.util.AWSHTTPClient;
import cucumber.api.Scenario;
@@ -25,22 +26,53 @@ import cucumber.api.java.en.Given;
import cucumber.api.java.en.Then;
import cucumber.api.java.en.When;
import org.opengroup.osdu.util.ElasticUtilsAws;
+import org.opengroup.osdu.util.LegalTagUtilsAws;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.opengroup.osdu.util.Config.getLegalTag;
+import static org.opengroup.osdu.util.Config.getOtherRelevantDataCountries;
@Log
public class Steps extends RecordSteps {
+ protected LegalTagUtilsAws legalTagUtils;
+ private String legalTagName;
+
public Steps() {
super(new AWSHTTPClient(), new ElasticUtilsAws());
}
@Before
- public void before(Scenario scenario) {
+ public void before(Scenario scenario) throws Exception {
this.scenario = scenario;
this.httpClient = new AWSHTTPClient();
+ legalTagUtils = new LegalTagUtilsAws(this.httpClient);
+ this.legalTagName = this.legalTagUtils.createRandomName();
+ this.legalTagUtils.create(this.legalTagName);
+ }
+
+ @Override
+ public void tearDown() {
+ super.tearDown();
+ this.legalTagUtils.delete(this.legalTagName);
+ }
+
+ @Override
+ protected Legal generateLegalTag() {
+ Legal legal = new Legal();
+ Set<String> legalTags = new HashSet<>();
+ legalTags.add(this.legalTagName);
+ legal.setLegaltags(legalTags);
+ Set<String> otherRelevantCountries = new HashSet<>();
+ otherRelevantCountries.add(getOtherRelevantDataCountries());
+ legal.setOtherRelevantDataCountries(otherRelevantCountries);
+ return legal;
}
@Given("^the schema is created with the following kind$")
- public void the_schema_is_created_with_the_following_kind(DataTable dataTable) {
+ public void the_schema_is_created_with_the_following_kind(DataTable dataTable){
super.the_schema_is_created_with_the_following_kind(dataTable);
}
diff --git a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/ElasticUtilsAws.java b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/ElasticUtilsAws.java
index 77a0440064ca1c7c41194dd3ca45a93613ac6b5f..28d3646d655232dc147719945b02282748e7d498 100644
--- a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/ElasticUtilsAws.java
+++ b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/ElasticUtilsAws.java
@@ -28,6 +28,7 @@ public class ElasticUtilsAws extends ElasticUtils {
@Override
public RestClientBuilder createClientBuilder(String host, String usernameAndPassword, int port) {
+ port = Integer.parseInt(System.getProperty("ELASTIC_PORT", System.getenv("ELASTIC_PORT")));
RestClientBuilder builder = RestClient.builder(new HttpHost(host, port, "https"));
builder.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(REST_CLIENT_CONNECT_TIMEOUT)
.setSocketTimeout(REST_CLIENT_SOCKET_TIMEOUT));
diff --git a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/LegalTagUtilsAws.java b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/LegalTagUtilsAws.java
new file mode 100644
index 0000000000000000000000000000000000000000..71cbc0900edf92ab9c0d76cbaa4f87c969cd3663
--- /dev/null
+++ b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/LegalTagUtilsAws.java
@@ -0,0 +1,85 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.util;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.http.HttpStatus;
+
+import com.google.gson.JsonArray;
+import com.google.gson.JsonObject;
+import com.sun.jersey.api.client.ClientResponse;
+
+import javax.ws.rs.HttpMethod;
+
+public class LegalTagUtilsAws {
+
+ private HTTPClient httpClient;
+ public LegalTagUtilsAws(HTTPClient httpClient) {
+ this.httpClient = httpClient;
+ }
+ public String createRandomName() {
+ return Config.getDataPartitionIdTenant1() + "-" + System.currentTimeMillis();
+ }
+
+ public ClientResponse create(String legalTagName) throws Exception {
+ return this.create("US", legalTagName, "2099-01-25", "Public Domain Data");
+ }
+
+ protected ClientResponse create(String countryOfOrigin, String name, String expDate, String dataType)
+ throws Exception {
+ String body = getBody(countryOfOrigin, name, expDate, dataType);
+ ClientResponse response = this.httpClient.send(HttpMethod.POST, String.format("%s%s",getLegalUrl(), "legaltags"), body, httpClient.getCommonHeader(), httpClient.getAccessToken());
+
+ assertEquals(HttpStatus.SC_CREATED, response.getStatus());
+ Thread.sleep(100);
+ return response;
+ }
+
+ public ClientResponse delete(String legalTagName) {
+ return this.httpClient.send(HttpMethod.DELETE ,getLegalUrl(), "legaltags/" + legalTagName,httpClient.getCommonHeader(), httpClient.getAccessToken());
+ }
+
+ protected static String getLegalUrl() {
+ String legalUrl = System.getProperty("LEGAL_URL", System.getenv("LEGAL_URL"));
+ if (legalUrl == null || legalUrl.contains("-null")) {
+ legalUrl = "https://os-legal-dot-opendes.appspot.com/api/legal/v1/";
+ }
+ return legalUrl;
+ }
+
+ protected static String getBody(String countryOfOrigin, String name, String expDate, String dataType) {
+
+ JsonArray coo = new JsonArray();
+ coo.add(countryOfOrigin);
+
+ JsonObject properties = new JsonObject();
+ properties.add("countryOfOrigin", coo);
+ properties.addProperty("contractId", "A1234");
+ properties.addProperty("expirationDate", expDate);
+ properties.addProperty("dataType", dataType);
+ properties.addProperty("originator", "MyCompany");
+ properties.addProperty("securityClassification", "Public");
+ properties.addProperty("exportClassification", "EAR99");
+ properties.addProperty("personalData", "No Personal Data");
+
+ JsonObject tag = new JsonObject();
+ tag.addProperty("name", name);
+ tag.addProperty("description", "test for " + name);
+ tag.add("properties", properties);
+
+ return tag.toString();
+ }
+}
diff --git a/testing/indexer-test-aws/src/test/resources/logback-test.xml b/testing/indexer-test-aws/src/test/resources/logback-test.xml
new file mode 100644
index 0000000000000000000000000000000000000000..dafd8ae394dc34b62ffa81307a76900db4b66af8
--- /dev/null
+++ b/testing/indexer-test-aws/src/test/resources/logback-test.xml
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<configuration>
+ <include resource="org/springframework/boot/logging/logback/base.xml" />
+ <root level="INFO" />
+ <logger name="org.springframework" level="INFO"/>
+</configuration>
\ No newline at end of file