Commit 3cd40423 authored by ethiraj krishnamanaidu's avatar ethiraj krishnamanaidu
Browse files

ado-codemerge

parent b9c4b229
......@@ -36,4 +36,7 @@ build/
target/*
*/target
/mvn
provider/indexer-gcp/bin/
\ No newline at end of file
provider/indexer-gcp/bin/*
# Environment configuration
*.env
......@@ -4,15 +4,15 @@ variables:
AWS_ENVIRONMENT: dev
GCP_BUILD_SUBDIR: provider/indexer-gcp
GCP_INT_TEST_SUBDIR: testing/indexer-test-gcp
GCP_APPLICATION_NAME: os-indexer
GCP_APPLICATION_NAME: osdu-indexer
GCP_ENVIRONMENT: dev
GCP_PROJECT: opendes
GCP_TENANT_NAME: opendes
GCP_DEPLOY_ENV: p4d
GCP_DOMAIN: cloud.slb-ds.com
GCP_STORAGE_URL: https://os-indexer-dot-opendes.appspot.com/api/storage/v2/
GCP_STORAGE_URL: https://osdu-indexer-dot-opendes.appspot.com/api/storage/v2/
include:
- project: 'osdu/platform/ci-cd-pipelines'
ref: 'master'
file: 'service.gitlab-ci.yml'
file: 'temp-service.gitlab-ci.yml'
......@@ -71,6 +71,22 @@ public class ElasticClientHandler {
String basicEncoded = Base64.getEncoder().encodeToString(clusterSettings.getUserNameAndPassword().getBytes());
String basicAuthenticationHeaderVal = String.format("Basic %s", basicEncoded);
RestClientBuilder builder = createClientBuilder(host, basicAuthenticationHeaderVal, port, protocolScheme, tls);
return new RestHighLevelClient(builder);
} catch (AppException e) {
throw e;
} catch (Exception e) {
throw new AppException(
HttpStatus.SC_INTERNAL_SERVER_ERROR,
"search client error",
"error creating search client",
String.format("Elastic client connection params, cluster: %s, host: %s, port: %s", cluster, host, port),
e);
}
}
public RestClientBuilder createClientBuilder(String host, String basicAuthenticationHeaderVal, int port, String protocolScheme, String tls) {
RestClientBuilder builder = RestClient.builder(new HttpHost(host, port, protocolScheme));
builder.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(REST_CLIENT_CONNECT_TIMEOUT)
.setSocketTimeout(REST_CLIENT_SOCKET_TIMEOUT));
......@@ -80,23 +96,13 @@ public class ElasticClientHandler {
new BasicHeader("client.transport.nodes_sampler_interval", "30s"),
new BasicHeader("client.transport.ping_timeout", "30s"),
new BasicHeader("client.transport.sniff", "false"),
new BasicHeader("request.headers.X-Found-Cluster", cluster),
new BasicHeader("cluster.name", cluster),
new BasicHeader("request.headers.X-Found-Cluster", host),
new BasicHeader("cluster.name", host),
new BasicHeader("xpack.security.transport.ssl.enabled", tls),
new BasicHeader("Authorization", basicAuthenticationHeaderVal),
};
builder.setDefaultHeaders(defaultHeaders);
return new RestHighLevelClient(builder);
} catch (AppException e) {
throw e;
} catch (Exception e) {
throw new AppException(
HttpStatus.SC_INTERNAL_SERVER_ERROR,
"search client error",
"error creating search client",
String.format("Elastic client connection params, cluster: %s, host: %s, port: %s", cluster, host, port),
e);
}
return builder;
}
}
\ No newline at end of file
......@@ -62,6 +62,7 @@
<module>provider/indexer-aws</module>
<module>provider/indexer-azure</module>
<module>provider/indexer-gcp</module>
<module>provider/indexer-ibm</module>
</modules>
</project>
# Copyright © Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##### Sample .env file ###########################################################
#
# Basic use: duplicate this file, and make sure the new copy is also in the root of the AWS
# 'provider' folder, and name it `.env`. Note that on macOS, by default, files starting with
# are considered hidden system files, and are not displayed by default in Finder or the file
# selector (which you will need to use when adding the environment file(s) to the run
# configuration(s). While you can change a setting to show hidden files and folders by
# default, there is also a keyboard shortcut to quickly toggle between hide/show. With either
# Finder as the active application ("Finder" appears next to the Apple logo in the Menu Bar),
# press: command + shift + . (period). You can store configurations for multiple environments
# by adding more duplicates following a naming scheme of your choosing, for example:
# `staging.env`, `uat.env`, or `local.env`.
#
# This requires installing a plugin to your IDE that allows you to use a .env
# file in your repository folder (does NOT get checked into source control;
# only the sample environment configuration (sample.env) should be committed.
#
# Download links for .env file plugins:
# IntelliJ - https://github.com/Ashald/EnvFile
##### Authentication / Secrets #####
# Replace placeholder text with your own AWS secret access keys
# and rename to `.env` - do NOT check-in .env with your credentials! Leave it in .gitignore
AWS_ACCESS_KEY_ID=
AWS_SECRET_KEY=
AWS_ACCOUNT_ID=
#### Urls/Ports #############
STORAGE_HOST=
APPLICATION_PORT=
CACHE_CLUSTER_INDEX_ENDPOINT=
CACHE_CLUSTER_INDEX_PORT=
CACHE_CLUSTER_CURSOR_ENDPOINT=
CACHE_CLUSTER_CURSOR_PORT=
ELASTIC_HOST=
ELASTIC_PORT=
##### Other environment variables ##########################################################
JAVA_HEAP_MEMORY=
SNS_TOPIC_NAME=
SNS_STORAGE_TOPIC_NAME=
ENVIRONMENT=
AWS_REGION=
##### Integration test-specific - these are only used for integration tests, not the app ###
OTHER_RELEVANT_DATA_COUNTRIES=
LEGAL_TAG=
DEFAULT_DATA_PARTITION_ID_TENANT1=
DEFAULT_DATA_PARTITION_ID_TENANT2=
ENTITLEMENTS_DOMAIN=
AWS_COGNITO_CLIENT_ID=
AWS_COGNITO_AUTH_FLOW=
AWS_COGNITO_AUTH_PARAMS_PASSWORD=
AWS_COGNITO_AUTH_PARAMS_USER=
AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS=
ELASTIC_HOST=
DEFAULT_ELASTIC_USER_NAME=
DEFAULT_ELASTIC_PASSWORD=
ELASTIC_PORT=
SEARCH_HOST=
STORAGE_HOST=
INDEXER_HOST=
\ No newline at end of file
......@@ -42,7 +42,7 @@ Parameters:
MaxLength: '64'
AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
Default: os-storage
Default: os-indexer
CacheName:
Description: The name of the cache cluster. Will be prefixed with the environment name.
......
# Copyright © Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
AWSTemplateFormatVersion: 2010-09-09
Description: >-
CloudFormation template for creating the resources used for the ECS cluster the application will
......@@ -161,6 +175,30 @@ Parameters:
MinValue: 256
MaxValue: 131072
DomainName:
Description: >-
The optional custom DNS name for the ECS service's load balancer. If omitted, the site will only be accessible
via the ECS service's Application Load Balancer DNS name. This value is used in the creation and signing of
the service's SSL certificate. Leave blank is not using a custom domain for this deployment.
Type: String
Default: ''
HostedZoneName:
Description: >-
The name of the hosted zone (ex: for indexer.osdu.slb.com, this would likely be osdu.slb.com).
Leave blank is not using a custom domain for this deployment.
Type: String
Default: ''
ElasticsearchDomainName:
Description: The name of the Elasticsearch domain. Will be prefixed with the environment name.
Type: String
MinLength: '1'
MaxLength: '64'
AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
Default: osdu-indexer
Mappings:
# This mapping is for the ECS-optimized edition of the November 13-14, 2019 release of the Amazon Linux 2 AMI
# It will need to be periodically updated as new versions are released by Amazon.
......@@ -200,7 +238,45 @@ Mappings:
sa-east-1:
AMIID: ami-0c947c117562538ee
Conditions:
IncludeCustomDomain: !Not [!Equals [ !Ref DomainName, '' ]]
IsPortStandardSSL:
!Or [!Equals [ !Ref ECSPort, '443' ], !Equals [ !Ref ECSPort, '8443' ]]
IsLoadBalancerHTTPS: !And # HTTPS for ECS requires a custom domain, but CloudFront will still have HTTPS/SSL
- !Condition IncludeCustomDomain
- !Condition IsPortStandardSSL
Resources:
# This sets up a Route 53 record for CloudFront if a custom domain is being used,
# otherwise a default cloudfront.net value will be used instead
CloudFrontDNSName:
Type: AWS::Route53::RecordSetGroup
Condition: IncludeCustomDomain
Properties:
HostedZoneName: !Join ['', [!Ref HostedZoneName, .]] # Route 53 requires a trailing period
RecordSets:
- Name: !Ref DomainName
Type: A
AliasTarget:
# This hosted zone ID is for ALL CloudFront distributions, always, and should be hard-coded
HostedZoneId: Z2FDTNDATAQYW2
DNSName: !GetAtt ECSCloudFrontDistribution.DomainName
# This sets up a Route 53 record for the ECS ALB origin if a custom domain is being used
ECSDNSName:
Type: AWS::Route53::RecordSetGroup
Condition: IncludeCustomDomain
Properties:
HostedZoneName: !Join ['', [!Ref HostedZoneName, .]] # Route 53 requires a trailing period
RecordSets:
- Name: !Join ['.', ['origin', !Ref DomainName]] # prefix the ECS origin record with 'origin.'
Type: A
AliasTarget:
HostedZoneId: !GetAtt ECSALB.CanonicalHostedZoneID # this value comes from the ALB attributes
DNSName: !GetAtt ECSALB.DNSName
EvaluateTargetHealth: true # Route 53 routes traffic to ECS targets based on their health checks
DependsOn: ECSALB
CodeDeployApplication:
Type: AWS::CodeDeploy::Application
Properties:
......@@ -220,11 +296,11 @@ Resources:
AWS:
- !Sub arn:aws:iam::${AWS::AccountId}:root
- Fn::ImportValue:
!Sub "${Environment}-${ApplicationName}-CodeBuildRoleArn"
!Sub "${Environment}-CodeBuildRoleArn"
- Fn::ImportValue:
!Sub "${Environment}-${ApplicationName}-CFNRoleArn"
!Sub "${Environment}-CFNRoleArn"
- Fn::ImportValue:
!Sub "${Environment}-${ApplicationName}-PipelineRoleArn"
!Sub "${Environment}-PipelineRoleArn"
Service:
- codebuild.amazonaws.com
Action:
......@@ -278,6 +354,8 @@ Resources:
Value: '{{resolve:secretsmanager:dev-IndexerServiceIamCredentials:SecretString:secret_key}}'
- Name: ENVIRONMENT
Value: !Ref Environment
- Name: VSTS_FEED_USER
Value: '{{resolve:secretsmanager:dev-VSTSFeedToken:SecretString:vsts_feed_user}}'
- Name: VSTS_FEED_TOKEN
Value: '{{resolve:secretsmanager:dev-VSTSFeedToken:SecretString:vsts_feed_token}}'
- Name: CACHE_CLUSTER_SCHEMA_ENDPOINT
......@@ -304,8 +382,22 @@ Resources:
Value: !Ref 'AWS::AccountId'
- Name: SNS_TOPIC_NAME
Value: !Ref SNSTopicName
- Name: ELASTIC_HOST
Value:
Fn::ImportValue:
!Sub "${Environment}-${ElasticsearchDomainName}-ElasticsearchDomainEndpoint"
- Name: ELASTIC_PORT
Value: '443' # the Elasticsearch port is not configurable on AWS, and is always 80 for HTTP and 443 for HTTPS, so there's no value in using a CFN parameter
- Name: JAVA_HEAP_MEMORY
Value: !Ref ECSMemoryAllocation
- Name: STORAGE_HOST
Value:
Fn::ImportValue:
!Sub "${Environment}-os-storage-EcsCloudFrontDomainName"
- Name: SNS_STORAGE_TOPIC_NAME
Value:
Fn::ImportValue:
!Sub "${Environment}-OSDUStorageSNSTopic"
Volumes:
- Name: docker-volume
......@@ -335,7 +427,16 @@ Resources:
TargetGroupArn: !Ref 'ECSTargetGroup'
LoadBalancerArn: !Ref 'ECSALB'
Port: !Ref ECSPort
Protocol: HTTP
Protocol: !If [IsLoadBalancerHTTPS, HTTPS, HTTP]
LoadBalancerALBListenerCertificate:
Type: AWS::ElasticLoadBalancingV2::ListenerCertificate
Condition: IncludeCustomDomain
Properties:
Certificates:
- Fn::ImportValue:
!Sub "${Environment}-${ApplicationName}-LoadBalancerSSLCertificateArn"
ListenerArn: !Ref 'ALBListener'
ECSALBPrimaryListenerRule:
Type: AWS::ElasticLoadBalancingV2::ListenerRule
......@@ -356,17 +457,92 @@ Resources:
Properties:
HealthCheckIntervalSeconds: 120
HealthCheckPath: /api/indexer/v2/liveness_check
HealthCheckProtocol: HTTP
HealthCheckProtocol: !If [IsLoadBalancerHTTPS, HTTPS, HTTP]
HealthCheckTimeoutSeconds: 5
HealthyThresholdCount: 2
Name: !Sub ECSTargetGroup-${ApplicationName}
Name: !Sub ECSTargetGroup-New-${ApplicationName}
Port: !Ref ECSPort
Protocol: HTTP
Protocol: !If [IsLoadBalancerHTTPS, HTTPS, HTTP]
UnhealthyThresholdCount: 2
VpcId:
Fn::ImportValue:
!Sub "${Environment}-OSDU-VPC"
ECSCloudFrontDistribution:
Type: AWS::CloudFront::Distribution
DependsOn: ECSALB
Properties:
DistributionConfig:
Comment: 'Cloudfront Distribution pointing ALB Origin'
Origins:
- DomainName: !GetAtt 'ECSALB.DNSName'
Id: !Ref 'ECSALB'
CustomOriginConfig:
HTTPPort: !Ref ECSPort # The ports are the same because we'll only ever be accessing the ECS cluster over one protocol, as set in OriginProtocolPolicy below
HTTPSPort: !Ref ECSPort # The ports are the same because we'll only ever be accessing the ECS cluster over one protocol, as set in OriginProtocolPolicy below
OriginProtocolPolicy: !If [IsLoadBalancerHTTPS, https-only, http-only] # this only affects the origin, not CloudFront / the user's request
OriginKeepaliveTimeout: '60'
OriginReadTimeout: '60'
OriginSSLProtocols:
- TLSv1
- TLSv1.1
- TLSv1.2
- SSLv3
Enabled: true
HttpVersion: 'http2'
Aliases:
- Fn::If:
- IncludeCustomDomain
- !Ref DomainName
- !Ref AWS::NoValue
DefaultCacheBehavior:
AllowedMethods:
- GET
- HEAD
- OPTIONS
- PUT
- POST
- PATCH
- DELETE
Compress: true
TargetOriginId: !Ref 'ECSALB'
DefaultTTL: 5
MaxTTL: 30
ForwardedValues:
QueryString: true
Cookies:
Forward: all
Headers:
- Authorization
- Data-Partition-Id
- Content-Type
- Kind
- Limit
- Cursor
ViewerProtocolPolicy: redirect-to-https # CloudFront requests will always be HTTPS, regardless of the origin or the request
ViewerCertificate:
AcmCertificateArn:
Fn::If:
- IncludeCustomDomain
- Fn::ImportValue:
!Sub "${Environment}-${ApplicationName}-LoadBalancerSSLCertificateArn"
- Ref: AWS::NoValue
CloudFrontDefaultCertificate:
Fn::If:
- IncludeCustomDomain
- Ref: AWS::NoValue
- true
SslSupportMethod:
Fn::If:
- IncludeCustomDomain
- sni-only # sni-only is free; 'vip' is the only other option, which allows viewers without Server Name Indication (SNI) support by using dedicated IP addresses, but it costs $600/mo per SSL certificate
- Ref: AWS::NoValue
MinimumProtocolVersion:
Fn::If:
- IncludeCustomDomain
- TLSv1
- Ref: AWS::NoValue # this is not used when using the default CloudFront certificate (which is always TLSv1)
ECSAutoScalingGroup:
Type: AWS::AutoScaling::AutoScalingGroup
Properties:
......@@ -555,6 +731,26 @@ Outputs:
Export:
Name: !Sub ${Environment}-${ApplicationName}-EcsAlbUrl
ECSALBCustomDNSName:
Description: The custom DNS name of the ECS service's ALB origin.
Condition: IncludeCustomDomain
Value: !Join ['.', ['origin', !Ref DomainName]]
Export:
Name: !Sub ${Environment}-${ApplicationName}-EcsAlbCustomDnsName
ECSCloudFrontCustomDNSName:
Description: The custom DNS name of the ECS service's CloudFront Distribution.
Condition: IncludeCustomDomain
Value: !Ref DomainName
Export:
Name: !Sub ${Environment}-${ApplicationName}-EcsCloudFrontCustomDnsName
ECSCloudFrontDomainName:
Description: The custom DNS name of the ECS service's CloudFront Distribution.
Value: !GetAtt ECSCloudFrontDistribution.DomainName
Export:
Name: !Sub ${Environment}-${ApplicationName}-EcsCloudFrontDomainName
TaskDefinitionArn:
Description: The ARN of the Indexer Service ECS task definition.
Value: !Ref 'TaskDefinition'
......
# Copyright © Amazon Web Services
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
AWSTemplateFormatVersion: 2010-09-09
Description: >-
CloudFormation template for creating the network resources used for the ECS cluster the application will
......@@ -38,21 +52,59 @@ Parameters:
ECSPort:
Description: The port that the ECS Service will listen on.
Type: Number
Default: 80
Default: 443
MinValue: 1
MaxValue: 65535
DomainName:
Description: >-
The optional custom DNS name for the service's load balancer. If omitted, the site will only be accessible
via the ECS service's Application Load Balancer DNS name. This value is used in the creation and signing of
the service's SSL certificate. Leave blank for none.
Type: String
Default: ''
AcmCertificateArn:
Description: >-
The Amazon Resource Name (ARN) of an existing AWS Certificate Manager (ACM) certificate.
If omitted, a new SSL certified will be requested/generated (only if the custom domain name
parameter is provided, otherwise the ECS service's ALB will not use SSL/HTTPS).
Type: String
AllowedPattern: "^(|arn:aws:acm:.*)$"
Default: ''
Conditions:
IncludeCustomDomain: !Not [!Equals [ !Ref DomainName, '' ]]
UseExistingACMSSLCertificate: !And
- !Not [!Equals [ !Ref AcmCertificateArn, '' ]]
- !Condition IncludeCustomDomain
ShouldRequestNewSSLCertificate: !And
- !Not [!Condition UseExistingACMSSLCertificate]
- !Condition IncludeCustomDomain
ShouldExportSSLCertificate: !Or
- !Condition IncludeCustomDomain
- !Condition UseExistingACMSSLCertificate
Resources:
# If an existing SSL certificate is not provided, but a custom domain is, request one
LoadBalancerSSLCertificate:
Type: 'AWS::CertificateManager::Certificate'
Condition: ShouldRequestNewSSLCertificate
Properties:
DomainName: !Ref DomainName
SubjectAlternativeNames:
- !Join ['.', ['origin', !Ref DomainName]] #
ECSSecurityGroup:
Type: AWS::EC2::SecurityGroup
Properties:
GroupName: !Sub "${Environment}-${ApplicationName}-sg"
GroupDescription: indexer Service ECS Security Group
GroupDescription: Indexer Service ECS Security Group
VpcId:
Fn::ImportValue:
!Sub "${Environment}-OSDU-VPC"
# Public access to ECS Listening Port
# Public access to the specified ECS Listening Port
ECSSecurityGroupECSListenerInbound:
Type: AWS::EC2::SecurityGroupIngress
Properties:
......@@ -62,37 +114,8 @@ Resources:
ToPort: !Ref ECSPort
CidrIp: 0.0.0.0/0
# Public access to port 443
ECSSecurityGroupHTTPSInbound:
Type: AWS::EC2::SecurityGroupIngress
Properties:
GroupId: !Ref 'ECSSecurityGroup'
IpProtocol: tcp
FromPort: '443'
ToPort: '443'
CidrIp: 0.0.0.0/0
# Public access to port 8080
ECSSecurityGroupHTTPAltInbound:
Type: AWS::EC2::SecurityGroupIngress
Properties:
GroupId: !Ref 'ECSSecurityGroup'
IpProtocol: tcp
FromPort: '8080'
ToPort: '8080'
CidrIp: 0.0.0.0/0
# Public access to port 8443
ECSSecurityGroupHTTPSAltInbound:
Type: AWS::EC2::SecurityGroupIngress
Properties:
GroupId: !Ref 'ECSSecurityGroup'
IpProtocol: tcp
FromPort: '8443'
ToPort: '8443'
CidrIp: 0.0.0.0/0
# SSH access for instances in our VPC's jump box subnet group (coming soon – will be part of the Util CFN)
# SSH access for instances in our VPC's jump box subnet group
# TODO: Update when the jump box is created as a part of the Util CFN, for now it is public
ECSSecurityGroupSSHInbound:
Type: AWS::EC2::SecurityGroupIngress
Properties:
......@@ -102,7 +125,7 @@ Resources:
ToPort: '22'
CidrIp: 0.0.0.0/0
# Open Application Load Balancer port range to itself
# Open Application Load Balancer port range to self-access
ECSSecurityGroupALBports:
Type: AWS::EC2::SecurityGroupIngress
Properties:
......@@ -114,7 +137,14 @@ Resources:
Outputs:
EcsNetworkSecurityGroupId:
Description: The ID of the indexer Service ECS EC2 security group.
Description: The ID of the Indexer Service ECS EC2 security group.
Value: !Ref 'ECSSecurityGroup'
Export:
Name: !Sub ${Environment}-${ApplicationName}-EcsNetworkSecurityGroupId
LoadBalancerSSLCertificateArn:
Condition: ShouldExportSSLCertificate
Description: The ARN of the SSL certificate to be used for both ECS and CloudFront (includes both DNS names).
Value: !If [UseExistingACMSSLCertificate, !Ref AcmCertificateArn, !Ref 'LoadBalancerSSLCertificate']
Export:
Name: !Sub ${Environment}-${ApplicationName}-LoadBalancerSSLCertificateArn
......@@ -28,6 +28,28 @@ Parameters:
ConstraintDescription: Can only be "dev/uat/prod"