Squashed commit of the following

commit aabba696 
Author: Barclay Walsh <barclay.walsh@parivedasolutions.com> 
Date: Tue Nov 19 2019 14:21:04 GMT-0600 (Central Standard Time) 

    Updated Elasticsearch to 6.8.


commit 25ca9eb8 
Author: Barclay Walsh <barclay.walsh@parivedasolutions.com> 
Date: Tue Nov 19 2019 08:54:05 GMT-0600 (Central Standard Time) 

    Changed test index name back to the normal name, now that testing is over.


commit 1833d39b 
Author: Barclay Walsh <barclay.walsh@parivedasolutions.com> 
Date: Tue Nov 19 2019 08:44:30 GMT-0600 (Central Standard Time) 

    Changed the default number of Elasticsearch nodes to 2.


commit 1aebda14 
Author: Barclay Walsh <barclay.walsh@parivedasolutions.com> 
Date: Tue Nov 19 2019 08:17:26 GMT-0600 (Central Standard Time) 

    Added Elasticsearch permissions to the CFNRole.


commit 0fb07525 
Author: Barclay Walsh <barclay.walsh@parivedasolutions.com> 
Date: Tue Nov 19 2019 08:08:30 GMT-0600 (Central Standard Time) 

    Added template for Elasticsearch domain/cluster; added an Elasticsearch stack to the automated pipeline; added Elasticsearch parameters to each environment's parameters file.

provider/indexer-aws/CloudFormation/Automated/elasticsearch.yml

+AWSTemplateFormatVersion: 2010-09-09
+Description: >-
+  CloudFormation template for creating the resources used for the tenant info database for OSDU.
+  It creates the DynamoDB table and the API Gateway endpoints.
+
+Parameters:
+  Environment:
+    Description: An environment name that will be prefixed to resource names.
+    Type: String
+    AllowedValues:
+      - dev
+      - uat
+      - prod
+    ConstraintDescription: Can only be "dev/uat/prod"
+    Default: dev
+
+  Region:
+    Description: The AWS region to deploy the resources to.
+    Type: String
+    Default: us-east-1
+
+  ElasticsearchDomainName:
+    Description: The name of the Elasticsearch domain. Will be prefixed with the environment name.
+    Type: String
+    MinLength: '1'
+    MaxLength: '64'
+    AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
+    ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
+    Default: osdu-indexer
+
+  ElasticsearchNodeInstanceType:
+    Description: The instance type for the main Elasticsearch nodes.
+    ConstraintDescription: Must be a valid instance type from the list of allowed values.
+    Default: t2.medium.elasticsearch
+    AllowedValues:
+      - t2.small.elasticsearch
+      - t2.medium.elasticsearch
+      - m5.large.elasticsearch
+      - m5.xlarge.elasticsearch
+      - m5.2xlarge.elasticsearch
+      - m5.4xlarge.elasticsearch
+      - m5.12xlarge.elasticsearch
+      - c5.large.elasticsearch
+      - c5.xlarge.elasticsearch
+      - c5.2xlarge.elasticsearch
+      - c5.4xlarge.elasticsearch
+      - c5.9xlarge.elasticsearch
+      - c5.18xlarge.elasticsearch
+      - r5.large.elasticsearch
+      - r5.xlarge.elasticsearch
+      - r5.2xlarge.elasticsearch
+      - r5.4xlarge.elasticsearch
+      - r5.12xlarge.elasticsearch
+      - i3.large.elasticsearch
+      - i3.xlarge.elasticsearch
+      - i3.2xlarge.elasticsearch
+      - i3.4xlarge.elasticsearch
+      - i3.8xlarge.elasticsearch
+      - i3.16xlarge.elasticsearch
+    Type: String
+
+  DedicatedMasterInstanceType:
+    Description: >
+      The instance type for the dedicated master nodes. These nodes perform cluster management
+      tasks, but doesn't hold data or respond to data upload requests.
+    ConstraintDescription: Must be a valid instance type from the list of allowed values.
+    Default: t2.medium.elasticsearch
+    AllowedValues:
+      - t2.small.elasticsearch
+      - t2.medium.elasticsearch
+      - m5.large.elasticsearch
+      - m5.xlarge.elasticsearch
+      - m5.2xlarge.elasticsearch
+      - m5.4xlarge.elasticsearch
+      - m5.12xlarge.elasticsearch
+      - c5.large.elasticsearch
+      - c5.xlarge.elasticsearch
+      - c5.2xlarge.elasticsearch
+      - c5.4xlarge.elasticsearch
+      - c5.9xlarge.elasticsearch
+      - c5.18xlarge.elasticsearch
+      - r5.large.elasticsearch
+      - r5.xlarge.elasticsearch
+      - r5.2xlarge.elasticsearch
+      - r5.4xlarge.elasticsearch
+      - r5.12xlarge.elasticsearch
+      - i3.large.elasticsearch
+      - i3.xlarge.elasticsearch
+      - i3.2xlarge.elasticsearch
+      - i3.4xlarge.elasticsearch
+      - i3.8xlarge.elasticsearch
+      - i3.16xlarge.elasticsearch
+    Type: String
+
+  NumberOfElasticsearchNodes:
+    Description: An integer value specifying the number of Elasticsearch primary nodes in the cluster.
+    Type: Number
+    Default: 2
+    MinValue: 1
+    MaxValue: 40
+
+  NumberOfDedicatedMasterNodes:
+    Description: An integer value specifying the number of dedicated master nodes.
+    Type: Number
+    Default: 2
+    MinValue: 2
+    MaxValue: 5
+    
+  ZoneAwarenessEnabled:
+    Description: >
+      When Zone Awareness is enabled, Elasticsearch allocates the nodes and replica
+      index shards that belong to a cluster across multiple AZs in the deployment region.
+    Type: String
+    AllowedValues:
+      - true
+      - false
+    Default: false
+
+  ElasticsearchVersion:
+    Description: >
+      The version of Elasticsearch to deploy on the cluster. Defaults to 6.8. Note
+      that an update requires a full replacement of the Elasticsearch cluster.
+    Type: String
+    AllowedValues:
+      - 1.5
+      - 2.3
+      - 5.1
+      - 5.3
+      - 5.5
+      - 5.6
+      - 6.0
+      - 6.2
+      - 6.3
+      - 6.4
+      - 6.5
+      - 6.6
+      - 6.8
+      - 6.8
+      - 7.1
+    Default: 6.8
+
+  EBSVolumeSize:
+    Description: >
+      The size of the EBS volume (per instance; total cluster size = EBS volume size x Instance count)
+      Maximum size varies by instance type, from 35GiB for t2 instances, up to 12TiB for r5.12xlarge.
+    Type: Number
+    Default: 10
+    MinValue: 10
+    MaxValue: 12000
+
+Resources:
+  ElasticsearchDomain:
+    Type: AWS::Elasticsearch::Domain
+    Properties:
+      DomainName: !Sub ${Environment}-${ElasticsearchDomainName}
+      ElasticsearchVersion: !Ref ElasticsearchVersion
+      ElasticsearchClusterConfig:
+        DedicatedMasterEnabled: "true"
+        InstanceCount: !Ref NumberOfElasticsearchNodes
+        ZoneAwarenessEnabled: !Ref ZoneAwarenessEnabled
+        InstanceType: !Ref ElasticsearchNodeInstanceType
+        DedicatedMasterType: !Ref DedicatedMasterInstanceType
+        DedicatedMasterCount: !Ref NumberOfDedicatedMasterNodes
+      EBSOptions:
+        EBSEnabled: true
+        VolumeSize: !Ref EBSVolumeSize
+        VolumeType: "gp2"
+      NodeToNodeEncryptionOptions:
+        Enabled: false
+      SnapshotOptions:
+        AutomatedSnapshotStartHour: "0"
+      AccessPolicies:
+        Version: "2012-10-17"
+        Statement:
+          - Effect: "Allow"
+            Principal:
+              AWS:
+                - !Sub arn:aws:iam::${AWS::AccountId}:root
+                - Fn::ImportValue:
+                    !Sub "${Environment}-IndexerServiceIamUserArn"
+            Action: "es:*"
+            Resource: !Sub arn:aws:es:us-east-1:846973539254:domain/${Environment}-${ElasticsearchDomainName}/*
+      AdvancedOptions:
+        rest.action.multi.allow_explicit_index: "true"
+      Tags:
+        -
+          Key: "Environment"
+          Value: !Ref Environment
+
+Outputs:
+  # Elasticsearch domain ARN
+  ElasticsearchDomainArn:
+    Description: The ARN of the Elasticsearch domain.
+    Value: !GetAtt ElasticsearchDomain.DomainArn
+    Export:
+      Name: !Sub ${Environment}-${ElasticsearchDomainName}-ElasticsearchDomainArn
+
+  # Elasticsearch domain endpoint
+  ElasticsearchDomainEndpoint:
+    Description: The endpoint URL of the Elasticsearch domain.
+    Value: !GetAtt ElasticsearchDomain.DomainEndpoint
+    Export:
+      Name: !Sub ${Environment}-${ElasticsearchDomainName}-ElasticsearchDomainEndpoint

provider/indexer-aws/CloudFormation/Manual/01-CreateCodePipeline.yml

@@ -339,6 +339,7 @@ Resources:
                   - 'route53:*'
                   - 'route53domains:*'
                   - 'elasticache:*'
+                  - 'es:*'
                 Effect: Allow
                 Resource: '*'
   CodeBuildRole:

provider/indexer-aws/CloudFormation/Master/os-indexer-master.yml

@@ -178,6 +178,136 @@ Parameters:
     MinValue: 1
     MaxValue: 128
 
+  ElasticsearchDomainName:
+    Description: The name of the Elasticsearch domain. Will be prefixed with the environment name.
+    Type: String
+    MinLength: '1'
+    MaxLength: '64'
+    AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
+    ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
+    Default: osdu-indexer
+
+  ElasticsearchNodeInstanceType:
+    Description: The instance type for the main Elasticsearch nodes.
+    ConstraintDescription: Must be a valid instance type from the list of allowed values.
+    Default: t2.medium.elasticsearch
+    AllowedValues:
+      - t2.small.elasticsearch
+      - t2.medium.elasticsearch
+      - m5.large.elasticsearch
+      - m5.xlarge.elasticsearch
+      - m5.2xlarge.elasticsearch
+      - m5.4xlarge.elasticsearch
+      - m5.12xlarge.elasticsearch
+      - c5.large.elasticsearch
+      - c5.xlarge.elasticsearch
+      - c5.2xlarge.elasticsearch
+      - c5.4xlarge.elasticsearch
+      - c5.9xlarge.elasticsearch
+      - c5.18xlarge.elasticsearch
+      - r5.large.elasticsearch
+      - r5.xlarge.elasticsearch
+      - r5.2xlarge.elasticsearch
+      - r5.4xlarge.elasticsearch
+      - r5.12xlarge.elasticsearch
+      - i3.large.elasticsearch
+      - i3.xlarge.elasticsearch
+      - i3.2xlarge.elasticsearch
+      - i3.4xlarge.elasticsearch
+      - i3.8xlarge.elasticsearch
+      - i3.16xlarge.elasticsearch
+    Type: String
+
+  DedicatedMasterInstanceType:
+    Description: >
+      The instance type for the dedicated master nodes. These nodes perform cluster management
+      tasks, but doesn't hold data or respond to data upload requests.
+    ConstraintDescription: Must be a valid instance type from the list of allowed values.
+    Default: t2.medium.elasticsearch
+    AllowedValues:
+      - t2.small.elasticsearch
+      - t2.medium.elasticsearch
+      - m5.large.elasticsearch
+      - m5.xlarge.elasticsearch
+      - m5.2xlarge.elasticsearch
+      - m5.4xlarge.elasticsearch
+      - m5.12xlarge.elasticsearch
+      - c5.large.elasticsearch
+      - c5.xlarge.elasticsearch
+      - c5.2xlarge.elasticsearch
+      - c5.4xlarge.elasticsearch
+      - c5.9xlarge.elasticsearch
+      - c5.18xlarge.elasticsearch
+      - r5.large.elasticsearch
+      - r5.xlarge.elasticsearch
+      - r5.2xlarge.elasticsearch
+      - r5.4xlarge.elasticsearch
+      - r5.12xlarge.elasticsearch
+      - i3.large.elasticsearch
+      - i3.xlarge.elasticsearch
+      - i3.2xlarge.elasticsearch
+      - i3.4xlarge.elasticsearch
+      - i3.8xlarge.elasticsearch
+      - i3.16xlarge.elasticsearch
+    Type: String
+
+  NumberOfElasticsearchNodes:
+    Description: An integer value specifying the number of Elasticsearch primary nodes in the cluster.
+    Type: Number
+    Default: 1
+    MinValue: 1
+    MaxValue: 40
+
+  NumberOfDedicatedMasterNodes:
+    Description: An integer value specifying the number of dedicated master nodes.
+    Type: Number
+    Default: 2
+    MinValue: 2
+    MaxValue: 5
+
+  ZoneAwarenessEnabled:
+    Description: >
+      When Zone Awareness is enabled, Elasticsearch allocates the nodes and replica
+      index shards that belong to a cluster across multiple AZs in the deployment region.
+    Type: String
+    AllowedValues:
+      - true
+      - false
+    Default: false
+
+  ElasticsearchVersion:
+    Description: >
+      The version of Elasticsearch to deploy on the cluster. Defaults to 6.8. Note
+      that an update requires a full replacement of the Elasticsearch cluster.
+    Type: String
+    AllowedValues:
+      - 1.5
+      - 2.3
+      - 5.1
+      - 5.3
+      - 5.5
+      - 5.6
+      - 6.0
+      - 6.2
+      - 6.3
+      - 6.4
+      - 6.5
+      - 6.6
+      - 6.8
+      - 6.8
+      - 7.1
+    Default: 6.8
+
+  EBSVolumeSize:
+    Description: >
+      The size of the EBS volume, in GiB, (per instance; total cluster size =
+      EBS volume size x Instance count). Maximum size varies by instance type, from 35GiB
+      for t2 instances, up to 12TiB for r5.12xlarge.
+    Type: Number
+    Default: 10
+    MinValue: 10
+    MaxValue: 12000
+
 Resources:
 
   #### Shared Resources ################################################################
@@ -229,3 +359,23 @@ Resources:
         CacheEngine: !Ref SchemaCacheEngine
         NodeInstanceType: !Ref SchemaCacheNodeInstanceType
         NumberOfCacheNodes: !Ref SchemaCacheNumberOfCacheNodes
+
+  #### Elasticsearch Resources #########################################################
+  # TODO: Add VPCs to Elasticsearch
+
+  ElasticsearchStack:
+    Type: 'AWS::CloudFormation::Stack'
+    DependsOn: IAMCredentialsStack
+    Properties:
+      TemplateURL: !Join [ '', [ !Ref ChildTemplateBasePath, elasticsearch.yml ] ]
+      Parameters:
+        Environment: !Ref Environment
+        Region: !Ref DeploymentRegion
+        ElasticsearchDomainName: !Ref ElasticsearchDomainName
+        ElasticsearchNodeInstanceType: !Ref ElasticsearchNodeInstanceType
+        DedicatedMasterInstanceType: !Ref DedicatedMasterInstanceType
+        NumberOfElasticsearchNodes: !Ref NumberOfElasticsearchNodes
+        NumberOfDedicatedMasterNodes: !Ref NumberOfDedicatedMasterNodes
+        ZoneAwarenessEnabled: !Ref ZoneAwarenessEnabled
+        ElasticsearchVersion: !Ref ElasticsearchVersion
+        EBSVolumeSize: !Ref EBSVolumeSize

provider/indexer-aws/CloudFormation/Params/dev.template_configuration.json

@@ -14,7 +14,15 @@
     "SchemaCacheName": "indexerschemacache",
     "SchemaCacheEngine": "redis",
     "SchemaCacheNodeInstanceType": "cache.t2.micro",
-    "SchemaCacheNumberOfCacheNodes": "1"
+    "SchemaCacheNumberOfCacheNodes": "1",
+    "ElasticsearchDomainName": "osdu-indexer",
+    "ElasticsearchNodeInstanceType": "t2.medium.elasticsearch",
+    "DedicatedMasterInstanceType": "t2.medium.elasticsearch",
+    "NumberOfElasticsearchNodes": "2",
+    "NumberOfDedicatedMasterNodes": "2",
+    "ZoneAwarenessEnabled": "false",
+    "ElasticsearchVersion": "6.8",
+    "EBSVolumeSize": "10"
   },
   "Tags" : {
     "Environment" : "dev"

provider/indexer-aws/CloudFormation/Params/prod.template_configuration.json

@@ -14,7 +14,15 @@
     "SchemaCacheName": "indexerschemacache",
     "SchemaCacheEngine": "redis",
     "SchemaCacheNodeInstanceType": "cache.t2.micro",
-    "SchemaCacheNumberOfCacheNodes": "1"
+    "SchemaCacheNumberOfCacheNodes": "1",
+    "ElasticsearchDomainName": "osdu-indexer",
+    "ElasticsearchNodeInstanceType": "t2.medium.elasticsearch",
+    "DedicatedMasterInstanceType": "t2.medium.elasticsearch",
+    "NumberOfElasticsearchNodes": "2",
+    "NumberOfDedicatedMasterNodes": "2",
+    "ZoneAwarenessEnabled": "false",
+    "ElasticsearchVersion": "6.8",
+    "EBSVolumeSize": "10"
   },
   "Tags" : {
     "Environment" : "prod"

provider/indexer-aws/CloudFormation/Params/uat.template_configuration.json

@@ -14,7 +14,15 @@
     "SchemaCacheName": "indexerschemacache",
     "SchemaCacheEngine": "redis",
     "SchemaCacheNodeInstanceType": "cache.t2.micro",
-    "SchemaCacheNumberOfCacheNodes": "1"
+    "SchemaCacheNumberOfCacheNodes": "1",
+    "ElasticsearchDomainName": "osdu-indexer",
+    "ElasticsearchNodeInstanceType": "t2.medium.elasticsearch",
+    "DedicatedMasterInstanceType": "t2.medium.elasticsearch",
+    "NumberOfElasticsearchNodes": "2",
+    "NumberOfDedicatedMasterNodes": "2",
+    "ZoneAwarenessEnabled": "false",
+    "ElasticsearchVersion": "6.8",
+    "EBSVolumeSize": "10"
   },
   "Tags" : {
     "Environment" : "uat"