diff --git a/.gitignore b/.gitignore
index 81a6ed9f80ee2d3a7a930b81fa1176651f20c0ae..80fb6f6fb61ee2b16095613fdecf9f835fc68d05 100644
--- a/.gitignore
+++ b/.gitignore
@@ -36,4 +36,7 @@ build/
target/*
*/target
/mvn
-provider/indexer-gcp/bin/
\ No newline at end of file
+provider/indexer-gcp/bin/*
+
+# Environment configuration
+*.env
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7819692f9a39083f11b262b79dff3c97a20a1db3..29aea813863e3270b5412e143f841682193e3689 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -4,15 +4,15 @@ variables:
AWS_ENVIRONMENT: dev
GCP_BUILD_SUBDIR: provider/indexer-gcp
GCP_INT_TEST_SUBDIR: testing/indexer-test-gcp
- GCP_APPLICATION_NAME: os-indexer
+ GCP_APPLICATION_NAME: osdu-indexer
GCP_ENVIRONMENT: dev
GCP_PROJECT: opendes
GCP_TENANT_NAME: opendes
GCP_DEPLOY_ENV: p4d
GCP_DOMAIN: cloud.slb-ds.com
- GCP_STORAGE_URL: https://os-indexer-dot-opendes.appspot.com/api/storage/v2/
+ GCP_STORAGE_URL: https://osdu-indexer-dot-opendes.appspot.com/api/storage/v2/
include:
- project: 'osdu/platform/ci-cd-pipelines'
ref: 'master'
- file: 'service.gitlab-ci.yml'
+ file: 'temp-service.gitlab-ci.yml'
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/ElasticClientHandler.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/ElasticClientHandler.java
index fcc90699042b27fb2c4e67b7c3d4f24e97bf8267..7035abf3ecb9820e4f09b1c7f3ff89966437152a 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/ElasticClientHandler.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/ElasticClientHandler.java
@@ -71,6 +71,22 @@ public class ElasticClientHandler {
String basicEncoded = Base64.getEncoder().encodeToString(clusterSettings.getUserNameAndPassword().getBytes());
String basicAuthenticationHeaderVal = String.format("Basic %s", basicEncoded);
+ RestClientBuilder builder = createClientBuilder(host, basicAuthenticationHeaderVal, port, protocolScheme, tls);
+
+ return new RestHighLevelClient(builder);
+ } catch (AppException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AppException(
+ HttpStatus.SC_INTERNAL_SERVER_ERROR,
+ "search client error",
+ "error creating search client",
+ String.format("Elastic client connection params, cluster: %s, host: %s, port: %s", cluster, host, port),
+ e);
+ }
+ }
+
+ public RestClientBuilder createClientBuilder(String host, String basicAuthenticationHeaderVal, int port, String protocolScheme, String tls) {
RestClientBuilder builder = RestClient.builder(new HttpHost(host, port, protocolScheme));
builder.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(REST_CLIENT_CONNECT_TIMEOUT)
.setSocketTimeout(REST_CLIENT_SOCKET_TIMEOUT));
@@ -80,23 +96,13 @@ public class ElasticClientHandler {
new BasicHeader("client.transport.nodes_sampler_interval", "30s"),
new BasicHeader("client.transport.ping_timeout", "30s"),
new BasicHeader("client.transport.sniff", "false"),
- new BasicHeader("request.headers.X-Found-Cluster", cluster),
- new BasicHeader("cluster.name", cluster),
+ new BasicHeader("request.headers.X-Found-Cluster", host),
+ new BasicHeader("cluster.name", host),
new BasicHeader("xpack.security.transport.ssl.enabled", tls),
new BasicHeader("Authorization", basicAuthenticationHeaderVal),
};
builder.setDefaultHeaders(defaultHeaders);
- return new RestHighLevelClient(builder);
- } catch (AppException e) {
- throw e;
- } catch (Exception e) {
- throw new AppException(
- HttpStatus.SC_INTERNAL_SERVER_ERROR,
- "search client error",
- "error creating search client",
- String.format("Elastic client connection params, cluster: %s, host: %s, port: %s", cluster, host, port),
- e);
- }
+ return builder;
}
}
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index 9e36b2746e2b28283aa939b56560511b11a43bee..0ce2c8068920277504c25f3b6557316db50eeec2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -62,6 +62,7 @@
<module>provider/indexer-aws</module>
<module>provider/indexer-azure</module>
<module>provider/indexer-gcp</module>
+ <module>provider/indexer-ibm</module>
</modules>
</project>
diff --git a/provider/indexer-aws/.env.template b/provider/indexer-aws/.env.template
new file mode 100644
index 0000000000000000000000000000000000000000..b55ecdc872088111f1c3bb74f33a75bfb4af3253
--- /dev/null
+++ b/provider/indexer-aws/.env.template
@@ -0,0 +1,77 @@
+# Copyright © Amazon Web Services
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+##### Sample .env file ###########################################################
+#
+# Basic use: duplicate this file, and make sure the new copy is also in the root of the AWS
+# 'provider' folder, and name it `.env`. Note that on macOS, by default, files starting with
+# are considered hidden system files, and are not displayed by default in Finder or the file
+# selector (which you will need to use when adding the environment file(s) to the run
+# configuration(s). While you can change a setting to show hidden files and folders by
+# default, there is also a keyboard shortcut to quickly toggle between hide/show. With either
+# Finder as the active application ("Finder" appears next to the Apple logo in the Menu Bar),
+# press: command + shift + . (period). You can store configurations for multiple environments
+# by adding more duplicates following a naming scheme of your choosing, for example:
+# `staging.env`, `uat.env`, or `local.env`.
+#
+# This requires installing a plugin to your IDE that allows you to use a .env
+# file in your repository folder (does NOT get checked into source control;
+# only the sample environment configuration (sample.env) should be committed.
+#
+# Download links for .env file plugins:
+# IntelliJ - https://github.com/Ashald/EnvFile
+
+##### Authentication / Secrets #####
+# Replace placeholder text with your own AWS secret access keys
+# and rename to `.env` - do NOT check-in .env with your credentials! Leave it in .gitignore
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_KEY=
+AWS_ACCOUNT_ID=
+
+#### Urls/Ports #############
+STORAGE_HOST=
+
+APPLICATION_PORT=
+CACHE_CLUSTER_INDEX_ENDPOINT=
+CACHE_CLUSTER_INDEX_PORT=
+CACHE_CLUSTER_CURSOR_ENDPOINT=
+CACHE_CLUSTER_CURSOR_PORT=
+ELASTIC_HOST=
+ELASTIC_PORT=
+
+##### Other environment variables ##########################################################
+JAVA_HEAP_MEMORY=
+SNS_TOPIC_NAME=
+SNS_STORAGE_TOPIC_NAME=
+ENVIRONMENT=
+AWS_REGION=
+
+##### Integration test-specific - these are only used for integration tests, not the app ###
+OTHER_RELEVANT_DATA_COUNTRIES=
+LEGAL_TAG=
+DEFAULT_DATA_PARTITION_ID_TENANT1=
+DEFAULT_DATA_PARTITION_ID_TENANT2=
+ENTITLEMENTS_DOMAIN=
+AWS_COGNITO_CLIENT_ID=
+AWS_COGNITO_AUTH_FLOW=
+AWS_COGNITO_AUTH_PARAMS_PASSWORD=
+AWS_COGNITO_AUTH_PARAMS_USER=
+AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS=
+ELASTIC_HOST=
+DEFAULT_ELASTIC_USER_NAME=
+DEFAULT_ELASTIC_PASSWORD=
+ELASTIC_PORT=
+SEARCH_HOST=
+STORAGE_HOST=
+INDEXER_HOST=
\ No newline at end of file
diff --git a/provider/indexer-aws/CloudFormation/Automated/cache.yml b/provider/indexer-aws/CloudFormation/Automated/cache.yml
index dcdc9a87f34a1e391eed3e3a178b79ada9904c0c..207f9815a887994d91731ced4215c00403587394 100644
--- a/provider/indexer-aws/CloudFormation/Automated/cache.yml
+++ b/provider/indexer-aws/CloudFormation/Automated/cache.yml
@@ -42,7 +42,7 @@ Parameters:
MaxLength: '64'
AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: os-storage
+ Default: os-indexer
CacheName:
Description: The name of the cache cluster. Will be prefixed with the environment name.
diff --git a/provider/indexer-aws/CloudFormation/Automated/ecs-cluster.yml b/provider/indexer-aws/CloudFormation/Automated/ecs-cluster.yml
index 3e2787b66b85926b0153ad194a33191ff29b8f0b..3a357d30c1edaedc60a366f266552ee354850c38 100644
--- a/provider/indexer-aws/CloudFormation/Automated/ecs-cluster.yml
+++ b/provider/indexer-aws/CloudFormation/Automated/ecs-cluster.yml
@@ -1,3 +1,17 @@
+# Copyright © Amazon Web Services
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
AWSTemplateFormatVersion: 2010-09-09
Description: >-
CloudFormation template for creating the resources used for the ECS cluster the application will
@@ -161,6 +175,30 @@ Parameters:
MinValue: 256
MaxValue: 131072
+ DomainName:
+ Description: >-
+ The optional custom DNS name for the ECS service's load balancer. If omitted, the site will only be accessible
+ via the ECS service's Application Load Balancer DNS name. This value is used in the creation and signing of
+ the service's SSL certificate. Leave blank is not using a custom domain for this deployment.
+ Type: String
+ Default: ''
+
+ HostedZoneName:
+ Description: >-
+ The name of the hosted zone (ex: for indexer.osdu.slb.com, this would likely be osdu.slb.com).
+ Leave blank is not using a custom domain for this deployment.
+ Type: String
+ Default: ''
+
+ ElasticsearchDomainName:
+ Description: The name of the Elasticsearch domain. Will be prefixed with the environment name.
+ Type: String
+ MinLength: '1'
+ MaxLength: '64'
+ AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
+ ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
+ Default: osdu-indexer
+
Mappings:
# This mapping is for the ECS-optimized edition of the November 13-14, 2019 release of the Amazon Linux 2 AMI
# It will need to be periodically updated as new versions are released by Amazon.
@@ -200,7 +238,45 @@ Mappings:
sa-east-1:
AMIID: ami-0c947c117562538ee
+Conditions:
+ IncludeCustomDomain: !Not [!Equals [ !Ref DomainName, '' ]]
+ IsPortStandardSSL:
+ !Or [!Equals [ !Ref ECSPort, '443' ], !Equals [ !Ref ECSPort, '8443' ]]
+ IsLoadBalancerHTTPS: !And # HTTPS for ECS requires a custom domain, but CloudFront will still have HTTPS/SSL
+ - !Condition IncludeCustomDomain
+ - !Condition IsPortStandardSSL
+
Resources:
+ # This sets up a Route 53 record for CloudFront if a custom domain is being used,
+ # otherwise a default cloudfront.net value will be used instead
+ CloudFrontDNSName:
+ Type: AWS::Route53::RecordSetGroup
+ Condition: IncludeCustomDomain
+ Properties:
+ HostedZoneName: !Join ['', [!Ref HostedZoneName, .]] # Route 53 requires a trailing period
+ RecordSets:
+ - Name: !Ref DomainName
+ Type: A
+ AliasTarget:
+ # This hosted zone ID is for ALL CloudFront distributions, always, and should be hard-coded
+ HostedZoneId: Z2FDTNDATAQYW2
+ DNSName: !GetAtt ECSCloudFrontDistribution.DomainName
+
+ # This sets up a Route 53 record for the ECS ALB origin if a custom domain is being used
+ ECSDNSName:
+ Type: AWS::Route53::RecordSetGroup
+ Condition: IncludeCustomDomain
+ Properties:
+ HostedZoneName: !Join ['', [!Ref HostedZoneName, .]] # Route 53 requires a trailing period
+ RecordSets:
+ - Name: !Join ['.', ['origin', !Ref DomainName]] # prefix the ECS origin record with 'origin.'
+ Type: A
+ AliasTarget:
+ HostedZoneId: !GetAtt ECSALB.CanonicalHostedZoneID # this value comes from the ALB attributes
+ DNSName: !GetAtt ECSALB.DNSName
+ EvaluateTargetHealth: true # Route 53 routes traffic to ECS targets based on their health checks
+ DependsOn: ECSALB
+
CodeDeployApplication:
Type: AWS::CodeDeploy::Application
Properties:
@@ -220,11 +296,11 @@ Resources:
AWS:
- !Sub arn:aws:iam::${AWS::AccountId}:root
- Fn::ImportValue:
- !Sub "${Environment}-${ApplicationName}-CodeBuildRoleArn"
+ !Sub "${Environment}-CodeBuildRoleArn"
- Fn::ImportValue:
- !Sub "${Environment}-${ApplicationName}-CFNRoleArn"
+ !Sub "${Environment}-CFNRoleArn"
- Fn::ImportValue:
- !Sub "${Environment}-${ApplicationName}-PipelineRoleArn"
+ !Sub "${Environment}-PipelineRoleArn"
Service:
- codebuild.amazonaws.com
Action:
@@ -278,6 +354,8 @@ Resources:
Value: '{{resolve:secretsmanager:dev-IndexerServiceIamCredentials:SecretString:secret_key}}'
- Name: ENVIRONMENT
Value: !Ref Environment
+ - Name: VSTS_FEED_USER
+ Value: '{{resolve:secretsmanager:dev-VSTSFeedToken:SecretString:vsts_feed_user}}'
- Name: VSTS_FEED_TOKEN
Value: '{{resolve:secretsmanager:dev-VSTSFeedToken:SecretString:vsts_feed_token}}'
- Name: CACHE_CLUSTER_SCHEMA_ENDPOINT
@@ -304,8 +382,22 @@ Resources:
Value: !Ref 'AWS::AccountId'
- Name: SNS_TOPIC_NAME
Value: !Ref SNSTopicName
+ - Name: ELASTIC_HOST
+ Value:
+ Fn::ImportValue:
+ !Sub "${Environment}-${ElasticsearchDomainName}-ElasticsearchDomainEndpoint"
+ - Name: ELASTIC_PORT
+ Value: '443' # the Elasticsearch port is not configurable on AWS, and is always 80 for HTTP and 443 for HTTPS, so there's no value in using a CFN parameter
- Name: JAVA_HEAP_MEMORY
Value: !Ref ECSMemoryAllocation
+ - Name: STORAGE_HOST
+ Value:
+ Fn::ImportValue:
+ !Sub "${Environment}-os-storage-EcsCloudFrontDomainName"
+ - Name: SNS_STORAGE_TOPIC_NAME
+ Value:
+ Fn::ImportValue:
+ !Sub "${Environment}-OSDUStorageSNSTopic"
Volumes:
- Name: docker-volume
@@ -335,7 +427,16 @@ Resources:
TargetGroupArn: !Ref 'ECSTargetGroup'
LoadBalancerArn: !Ref 'ECSALB'
Port: !Ref ECSPort
- Protocol: HTTP
+ Protocol: !If [IsLoadBalancerHTTPS, HTTPS, HTTP]
+
+ LoadBalancerALBListenerCertificate:
+ Type: AWS::ElasticLoadBalancingV2::ListenerCertificate
+ Condition: IncludeCustomDomain
+ Properties:
+ Certificates:
+ - Fn::ImportValue:
+ !Sub "${Environment}-${ApplicationName}-LoadBalancerSSLCertificateArn"
+ ListenerArn: !Ref 'ALBListener'
ECSALBPrimaryListenerRule:
Type: AWS::ElasticLoadBalancingV2::ListenerRule
@@ -356,17 +457,92 @@ Resources:
Properties:
HealthCheckIntervalSeconds: 120
HealthCheckPath: /api/indexer/v2/liveness_check
- HealthCheckProtocol: HTTP
+ HealthCheckProtocol: !If [IsLoadBalancerHTTPS, HTTPS, HTTP]
HealthCheckTimeoutSeconds: 5
HealthyThresholdCount: 2
- Name: !Sub ECSTargetGroup-${ApplicationName}
+ Name: !Sub ECSTargetGroup-New-${ApplicationName}
Port: !Ref ECSPort
- Protocol: HTTP
+ Protocol: !If [IsLoadBalancerHTTPS, HTTPS, HTTP]
UnhealthyThresholdCount: 2
VpcId:
Fn::ImportValue:
!Sub "${Environment}-OSDU-VPC"
+ ECSCloudFrontDistribution:
+ Type: AWS::CloudFront::Distribution
+ DependsOn: ECSALB
+ Properties:
+ DistributionConfig:
+ Comment: 'Cloudfront Distribution pointing ALB Origin'
+ Origins:
+ - DomainName: !GetAtt 'ECSALB.DNSName'
+ Id: !Ref 'ECSALB'
+ CustomOriginConfig:
+ HTTPPort: !Ref ECSPort # The ports are the same because we'll only ever be accessing the ECS cluster over one protocol, as set in OriginProtocolPolicy below
+ HTTPSPort: !Ref ECSPort # The ports are the same because we'll only ever be accessing the ECS cluster over one protocol, as set in OriginProtocolPolicy below
+ OriginProtocolPolicy: !If [IsLoadBalancerHTTPS, https-only, http-only] # this only affects the origin, not CloudFront / the user's request
+ OriginKeepaliveTimeout: '60'
+ OriginReadTimeout: '60'
+ OriginSSLProtocols:
+ - TLSv1
+ - TLSv1.1
+ - TLSv1.2
+ - SSLv3
+ Enabled: true
+ HttpVersion: 'http2'
+ Aliases:
+ - Fn::If:
+ - IncludeCustomDomain
+ - !Ref DomainName
+ - !Ref AWS::NoValue
+ DefaultCacheBehavior:
+ AllowedMethods:
+ - GET
+ - HEAD
+ - OPTIONS
+ - PUT
+ - POST
+ - PATCH
+ - DELETE
+ Compress: true
+ TargetOriginId: !Ref 'ECSALB'
+ DefaultTTL: 5
+ MaxTTL: 30
+ ForwardedValues:
+ QueryString: true
+ Cookies:
+ Forward: all
+ Headers:
+ - Authorization
+ - Data-Partition-Id
+ - Content-Type
+ - Kind
+ - Limit
+ - Cursor
+ ViewerProtocolPolicy: redirect-to-https # CloudFront requests will always be HTTPS, regardless of the origin or the request
+ ViewerCertificate:
+ AcmCertificateArn:
+ Fn::If:
+ - IncludeCustomDomain
+ - Fn::ImportValue:
+ !Sub "${Environment}-${ApplicationName}-LoadBalancerSSLCertificateArn"
+ - Ref: AWS::NoValue
+ CloudFrontDefaultCertificate:
+ Fn::If:
+ - IncludeCustomDomain
+ - Ref: AWS::NoValue
+ - true
+ SslSupportMethod:
+ Fn::If:
+ - IncludeCustomDomain
+ - sni-only # sni-only is free; 'vip' is the only other option, which allows viewers without Server Name Indication (SNI) support by using dedicated IP addresses, but it costs $600/mo per SSL certificate
+ - Ref: AWS::NoValue
+ MinimumProtocolVersion:
+ Fn::If:
+ - IncludeCustomDomain
+ - TLSv1
+ - Ref: AWS::NoValue # this is not used when using the default CloudFront certificate (which is always TLSv1)
+
ECSAutoScalingGroup:
Type: AWS::AutoScaling::AutoScalingGroup
Properties:
@@ -555,6 +731,26 @@ Outputs:
Export:
Name: !Sub ${Environment}-${ApplicationName}-EcsAlbUrl
+ ECSALBCustomDNSName:
+ Description: The custom DNS name of the ECS service's ALB origin.
+ Condition: IncludeCustomDomain
+ Value: !Join ['.', ['origin', !Ref DomainName]]
+ Export:
+ Name: !Sub ${Environment}-${ApplicationName}-EcsAlbCustomDnsName
+
+ ECSCloudFrontCustomDNSName:
+ Description: The custom DNS name of the ECS service's CloudFront Distribution.
+ Condition: IncludeCustomDomain
+ Value: !Ref DomainName
+ Export:
+ Name: !Sub ${Environment}-${ApplicationName}-EcsCloudFrontCustomDnsName
+
+ ECSCloudFrontDomainName:
+ Description: The custom DNS name of the ECS service's CloudFront Distribution.
+ Value: !GetAtt ECSCloudFrontDistribution.DomainName
+ Export:
+ Name: !Sub ${Environment}-${ApplicationName}-EcsCloudFrontDomainName
+
TaskDefinitionArn:
Description: The ARN of the Indexer Service ECS task definition.
Value: !Ref 'TaskDefinition'
diff --git a/provider/indexer-aws/CloudFormation/Automated/ecs-network.yml b/provider/indexer-aws/CloudFormation/Automated/ecs-network.yml
index cd3baa306cf870d1663747520c8823e542543214..0fa408776e7b9a4040531b58afcafc4550e8f239 100644
--- a/provider/indexer-aws/CloudFormation/Automated/ecs-network.yml
+++ b/provider/indexer-aws/CloudFormation/Automated/ecs-network.yml
@@ -1,3 +1,17 @@
+# Copyright © Amazon Web Services
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
AWSTemplateFormatVersion: 2010-09-09
Description: >-
CloudFormation template for creating the network resources used for the ECS cluster the application will
@@ -38,21 +52,59 @@ Parameters:
ECSPort:
Description: The port that the ECS Service will listen on.
Type: Number
- Default: 80
+ Default: 443
MinValue: 1
MaxValue: 65535
+ DomainName:
+ Description: >-
+ The optional custom DNS name for the service's load balancer. If omitted, the site will only be accessible
+ via the ECS service's Application Load Balancer DNS name. This value is used in the creation and signing of
+ the service's SSL certificate. Leave blank for none.
+ Type: String
+ Default: ''
+
+ AcmCertificateArn:
+ Description: >-
+ The Amazon Resource Name (ARN) of an existing AWS Certificate Manager (ACM) certificate.
+ If omitted, a new SSL certified will be requested/generated (only if the custom domain name
+ parameter is provided, otherwise the ECS service's ALB will not use SSL/HTTPS).
+ Type: String
+ AllowedPattern: "^(|arn:aws:acm:.*)$"
+ Default: ''
+
+Conditions:
+ IncludeCustomDomain: !Not [!Equals [ !Ref DomainName, '' ]]
+ UseExistingACMSSLCertificate: !And
+ - !Not [!Equals [ !Ref AcmCertificateArn, '' ]]
+ - !Condition IncludeCustomDomain
+ ShouldRequestNewSSLCertificate: !And
+ - !Not [!Condition UseExistingACMSSLCertificate]
+ - !Condition IncludeCustomDomain
+ ShouldExportSSLCertificate: !Or
+ - !Condition IncludeCustomDomain
+ - !Condition UseExistingACMSSLCertificate
+
Resources:
+ # If an existing SSL certificate is not provided, but a custom domain is, request one
+ LoadBalancerSSLCertificate:
+ Type: 'AWS::CertificateManager::Certificate'
+ Condition: ShouldRequestNewSSLCertificate
+ Properties:
+ DomainName: !Ref DomainName
+ SubjectAlternativeNames:
+ - !Join ['.', ['origin', !Ref DomainName]] #
+
ECSSecurityGroup:
Type: AWS::EC2::SecurityGroup
Properties:
GroupName: !Sub "${Environment}-${ApplicationName}-sg"
- GroupDescription: indexer Service ECS Security Group
+ GroupDescription: Indexer Service ECS Security Group
VpcId:
Fn::ImportValue:
!Sub "${Environment}-OSDU-VPC"
- # Public access to ECS Listening Port
+ # Public access to the specified ECS Listening Port
ECSSecurityGroupECSListenerInbound:
Type: AWS::EC2::SecurityGroupIngress
Properties:
@@ -62,37 +114,8 @@ Resources:
ToPort: !Ref ECSPort
CidrIp: 0.0.0.0/0
- # Public access to port 443
- ECSSecurityGroupHTTPSInbound:
- Type: AWS::EC2::SecurityGroupIngress
- Properties:
- GroupId: !Ref 'ECSSecurityGroup'
- IpProtocol: tcp
- FromPort: '443'
- ToPort: '443'
- CidrIp: 0.0.0.0/0
-
- # Public access to port 8080
- ECSSecurityGroupHTTPAltInbound:
- Type: AWS::EC2::SecurityGroupIngress
- Properties:
- GroupId: !Ref 'ECSSecurityGroup'
- IpProtocol: tcp
- FromPort: '8080'
- ToPort: '8080'
- CidrIp: 0.0.0.0/0
-
- # Public access to port 8443
- ECSSecurityGroupHTTPSAltInbound:
- Type: AWS::EC2::SecurityGroupIngress
- Properties:
- GroupId: !Ref 'ECSSecurityGroup'
- IpProtocol: tcp
- FromPort: '8443'
- ToPort: '8443'
- CidrIp: 0.0.0.0/0
-
- # SSH access for instances in our VPC's jump box subnet group (coming soon – will be part of the Util CFN)
+ # SSH access for instances in our VPC's jump box subnet group
+ # TODO: Update when the jump box is created as a part of the Util CFN, for now it is public
ECSSecurityGroupSSHInbound:
Type: AWS::EC2::SecurityGroupIngress
Properties:
@@ -102,7 +125,7 @@ Resources:
ToPort: '22'
CidrIp: 0.0.0.0/0
- # Open Application Load Balancer port range to itself
+ # Open Application Load Balancer port range to self-access
ECSSecurityGroupALBports:
Type: AWS::EC2::SecurityGroupIngress
Properties:
@@ -114,7 +137,14 @@ Resources:
Outputs:
EcsNetworkSecurityGroupId:
- Description: The ID of the indexer Service ECS EC2 security group.
+ Description: The ID of the Indexer Service ECS EC2 security group.
Value: !Ref 'ECSSecurityGroup'
Export:
Name: !Sub ${Environment}-${ApplicationName}-EcsNetworkSecurityGroupId
+
+ LoadBalancerSSLCertificateArn:
+ Condition: ShouldExportSSLCertificate
+ Description: The ARN of the SSL certificate to be used for both ECS and CloudFront (includes both DNS names).
+ Value: !If [UseExistingACMSSLCertificate, !Ref AcmCertificateArn, !Ref 'LoadBalancerSSLCertificate']
+ Export:
+ Name: !Sub ${Environment}-${ApplicationName}-LoadBalancerSSLCertificateArn
diff --git a/provider/indexer-aws/CloudFormation/Automated/elasticsearch.yml b/provider/indexer-aws/CloudFormation/Automated/elasticsearch.yml
index 00ef39a89b1c2496f654c32a620fe485a662569e..7a18783ae484ffcb6b248c98e131db4d28ee63a6 100644
--- a/provider/indexer-aws/CloudFormation/Automated/elasticsearch.yml
+++ b/provider/indexer-aws/CloudFormation/Automated/elasticsearch.yml
@@ -28,6 +28,28 @@ Parameters:
ConstraintDescription: Can only be "dev/uat/prod"
Default: dev
+ ApplicationName:
+ Description: >
+ The name of the application, which will be used to generate the ECS cluster name.
+ It will be prefixed with the environment name.
+ Type: String
+ MinLength: '1'
+ MaxLength: '64'
+ AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
+ ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
+ Default: os-indexer
+
+ SearchApplicationName:
+ Description: >
+ The name of the application, which will be used to generate the ECS cluster name.
+ It will be prefixed with the environment name.
+ Type: String
+ MinLength: '1'
+ MaxLength: '64'
+ AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
+ ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
+ Default: os-search
+
Region:
Description: The AWS region to deploy the resources to.
Type: String
@@ -183,36 +205,26 @@ Resources:
Enabled: false
SnapshotOptions:
AutomatedSnapshotStartHour: "0"
-# AccessPolicies:
-# - Version: "2012-10-17"
-# Statement:
-# - Effect: "Allow"
-# Principal:
-# AWS:
-# - !Sub arn:aws:iam::${AWS::AccountId}:root
-# - Fn::ImportValue:
-# !Sub "${Environment}-IndexerServiceIamUserArn"
-# # TODO: need to create cognito user and identity pool and link it to principal for dynamic creation
-# - "arn:aws:iam::888733619319:role/Cognito_osduelasticsearchAuth_Role"
-# Action:
-# - "es:*"
-# - 'es:ESHttp*'
-# - 'cognito-identity:*'
-# - 'cognito-idp:*'
-# - 'sts:AssumeRole'
-# Resource: !Sub arn:aws:es:us-east-1:846973539254:domain/${Environment}-${ElasticsearchDomainName}/*
-# - "Version": "2012-10-17"
-# "Statement":
-# - "Effect": "Allow"
-# "Action":
-# - "iam:PassRole"
-# "Resource": "arn:aws:iam::888733619319:role/service-role/CognitoAccessForAmazonES"
+ AccessPolicies:
+ Version: 2012-10-17
+ Statement:
+ - Effect: "Allow"
+ Principal: "*"
+ Resource: '*'
+ Action: "*"
AdvancedOptions:
rest.action.multi.allow_explicit_index: "true"
Tags:
-
Key: "Environment"
Value: !Ref Environment
+ VPCOptions:
+ SubnetIds:
+ - Fn::ImportValue:
+ !Sub "${Environment}-OSDU-PrivateSubnet-AZ1"
+ SecurityGroupIds:
+ - Fn::ImportValue:
+ !Sub "${Environment}-${ApplicationName}-EcsNetworkSecurityGroupId"
Outputs:
# Elasticsearch domain ARN
diff --git a/provider/indexer-aws/CloudFormation/JarDeploy/CodePipeline-JarDeploy.yml b/provider/indexer-aws/CloudFormation/JarDeploy/CodePipeline-JarDeploy.yml
new file mode 100644
index 0000000000000000000000000000000000000000..f75d29cd5944fc25edfcf213bac567194319af4e
--- /dev/null
+++ b/provider/indexer-aws/CloudFormation/JarDeploy/CodePipeline-JarDeploy.yml
@@ -0,0 +1,249 @@
+# Copyright © Amazon Web Services
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+AWSTemplateFormatVersion: 2010-09-09
+
+Description: >
+ This CloudFormation script creates the deployment pipeline for OSDU's indexer service. The CodePipeline
+ should automatically trigger whenever commits are made on the tracked branch. The start and end
+ of the CodePipeline should trigger a SNS alert to keep track of when the deployment has started
+ and when it finishes.
+
+Parameters:
+ Environment:
+ Description: Environment Name. Defaults to 'dev'. Can only be dev/uat/prod.
+ Type: String
+ AllowedValues:
+ - dev
+ - uat
+ - prod
+ Default: dev
+
+ DeploymentRegion:
+ Description: The AWS region to deploy the application to. The default is us-east-1.
+ Type: String
+ Default: us-east-1
+
+ SNSNotificationEmail:
+ Description: The email address to send SNS notifications about the build to.
+ Type: String
+ Default: barclay.walsh@parivedasolutions.com
+
+ CodeCommitRepositoryName:
+ Description: The name of the Code Commit Repository that the CodePipeline source is connected to.
+ Type: String
+ Default: os-indexer
+
+ JarServiceBase:
+ Description: The name of the service base path for the JAR files (e.g. 'indexer').
+ Type: String
+ Default: indexer
+
+ CodeCommitBranchName:
+ Description: The name of the Code Commit branch that the CodePipeline source is connected to.
+ Type: String
+ Default: dev
+
+Resources:
+ ArtifactStoreBucket:
+ Type: AWS::S3::Bucket
+ DeletionPolicy: Delete
+ Properties:
+ VersioningConfiguration:
+ Status: Enabled
+
+ ArtifactStoreBucketPolicy:
+ Type: AWS::S3::BucketPolicy
+ Properties:
+ Bucket: !Ref ArtifactStoreBucket
+ PolicyDocument:
+ Statement:
+ - Action:
+ - s3:*
+ Effect: Allow
+ Resource:
+ - !Sub arn:aws:s3:::${ArtifactStoreBucket}
+ - !Sub arn:aws:s3:::${ArtifactStoreBucket}/*
+ Principal:
+ AWS:
+ - !Sub arn:aws:iam::${AWS::AccountId}:root
+ - !ImportValue
+ 'Fn::Sub': '${Environment}-CodeBuildRoleArn'
+ - !ImportValue
+ 'Fn::Sub': '${Environment}-PipelineRoleArn'
+ - !ImportValue
+ 'Fn::Sub': '${Environment}-CFNRoleArn'
+
+ CachingBucket:
+ Type: AWS::S3::Bucket
+ DeletionPolicy: Delete
+ Properties:
+ VersioningConfiguration:
+ Status: Enabled
+
+ CachingBucketPolicy:
+ Type: AWS::S3::BucketPolicy
+ Properties:
+ Bucket: !Ref CachingBucket
+ PolicyDocument:
+ Statement:
+ - Action:
+ - s3:*
+ Effect: Allow
+ Resource:
+ - !Sub arn:aws:s3:::${CachingBucket}
+ - !Sub arn:aws:s3:::${CachingBucket}/*
+ Principal:
+ AWS:
+ - !Sub arn:aws:iam::${AWS::AccountId}:root
+ - !ImportValue
+ 'Fn::Sub': '${Environment}-CodeBuildRoleArn'
+ - !ImportValue
+ 'Fn::Sub': '${Environment}-PipelineRoleArn'
+ - !ImportValue
+ 'Fn::Sub': '${Environment}-CFNRoleArn'
+
+ SNSCodePipelineDeploymentFailed:
+ Type: AWS::SNS::Topic
+ Properties:
+ Subscription:
+ - Endpoint: !Ref SNSNotificationEmail
+ Protocol: email
+ TopicName: !Sub '${Environment}-OS-Indexer-Deployment-CodePipeline-JarDeploy-Failed'
+
+ EventRuleCodePipelineFailed:
+ Type: AWS::Events::Rule
+ Properties:
+ Description: Triggered whenever the CodePipeline deployment stage has failed.
+ EventPattern:
+ source:
+ - "aws.codepipeline"
+ detail-type:
+ - "CodePipeline Stage Execution State Change"
+ detail:
+ state:
+ - "FAILED"
+ pipeline:
+ - !Sub '${Environment}-OSDU-OS-Indexer-CodePipeline-JarDeploy'
+
+ Name: !Sub ${Environment}-CodePipelineEventRule-${CodeCommitRepositoryName}-JarDeploy
+ Targets:
+ -
+ Arn:
+ !Ref SNSCodePipelineDeploymentFailed
+ Id: "Deployment-CodePipeline-JarDeploy-Failed"
+ InputTransformer:
+ InputPathsMap:
+ pipeline : "$.detail.pipeline"
+ InputTemplate: '"The Pipeline <pipeline> has failed."'
+
+ Pipeline:
+ Type: AWS::CodePipeline::Pipeline
+ Properties:
+ ArtifactStore:
+ Location: !Ref ArtifactStoreBucket
+ Type: S3
+ Name: !Sub '${Environment}-OSDU-OS-Indexer-CodePipeline-JarDeploy'
+ RoleArn: !ImportValue
+ 'Fn::Sub': '${Environment}-PipelineRoleArn'
+ Stages:
+ - Name: Source
+ Actions:
+ - Name: Source
+ ActionTypeId:
+ Category: Source
+ Owner: AWS
+ Provider: CodeCommit
+ Version: '1'
+ Configuration:
+ BranchName: !Ref CodeCommitBranchName
+ RepositoryName: !Ref CodeCommitRepositoryName
+ OutputArtifacts:
+ - Name: Source
+ RunOrder: '1'
+
+ - Name: CodeBuild
+ Actions:
+ - Name: Jar-CodeBuild
+ ActionTypeId:
+ Category: Build
+ Owner: AWS
+ Provider: CodeBuild
+ Version: '1'
+ InputArtifacts:
+ - Name: Source
+ OutputArtifacts:
+ - Name: Jar-CodeBuild
+ Configuration:
+ ProjectName: !Ref JarCodeBuild
+ RunOrder: '2'
+
+ JarCodeBuild:
+ Type: AWS::CodeBuild::Project
+ Properties:
+ Name: !Sub ${Environment}-jar-codebuild-${CodeCommitRepositoryName}
+ Description: CodeBuild commands which run after the CloudFormation deployment.
+ ServiceRole: !ImportValue
+ 'Fn::Sub': '${Environment}-CodeBuildRoleArn'
+ Artifacts:
+ Type: S3
+ Location: !Ref ArtifactStoreBucket
+ Name: !Sub ${Environment}-jar-codebuild
+ Environment:
+ Type: LINUX_CONTAINER
+ ComputeType: BUILD_GENERAL1_SMALL
+ Image: aws/codebuild/standard:2.0
+ EnvironmentVariables:
+ - Name: ENVIRONMENT
+ Type: PLAINTEXT
+ Value: !Ref Environment
+ - Name: AWS_ACCOUNT_ID
+ Type: PLAINTEXT
+ Value: !Ref AWS::AccountId
+ - Name: AWS_REGION
+ Type: PLAINTEXT
+ Value: !Ref DeploymentRegion
+ - Name: APPLICATION_NAME
+ Type: PLAINTEXT
+ Value: !Ref CodeCommitRepositoryName
+ - Name: JAR_SERVICE_BASE
+ Type: PLAINTEXT
+ Value: !Ref JarServiceBase
+ - Name: M2_REPO_S3_BUCKET
+ Type: PLAINTEXT
+ Value: !Sub "${Environment}-${AWS::AccountId}-persistent-maven-m2-bucket"
+ - Name: JAR_DEPLOY_S3_BUCKET
+ Type: PLAINTEXT
+ Value: !Sub ${Environment}-${AWS::AccountId}-osdu-jar-deploy
+ PrivilegedMode: true
+ Source:
+ BuildSpec: ./provider/indexer-aws/buildspec-jar-deploy.yml
+ Location: !Sub https://git-codecommit.${AWS::Region}.amazonaws.com/v1/repos/${CodeCommitRepositoryName}
+ Type: CODECOMMIT
+ Cache:
+ Type: S3
+ Location: !Sub ${CachingBucket}/${Environment}
+ TimeoutInMinutes: 15
+ VpcConfig:
+ SecurityGroupIds:
+ - Fn::ImportValue:
+ !Sub "${Environment}-OSDU-CodeBuildSecurityGroup"
+ Subnets:
+ - Fn::ImportValue:
+ !Sub "${Environment}-OSDU-PrivateSubnet-AZ1"
+ - Fn::ImportValue:
+ !Sub "${Environment}-OSDU-PrivateSubnet-AZ2"
+ VpcId:
+ Fn::ImportValue:
+ !Sub "${Environment}-OSDU-VPC"
diff --git a/provider/indexer-aws/CloudFormation/Manual/01-CreateCodePipeline.yml b/provider/indexer-aws/CloudFormation/Manual/01-CreateCodePipeline.yml
index cfaad181934f70d70b9ce9213c9ab0f6b3175c1f..0109633f0071acd48c8520bf240dd962068ead11 100644
--- a/provider/indexer-aws/CloudFormation/Manual/01-CreateCodePipeline.yml
+++ b/provider/indexer-aws/CloudFormation/Manual/01-CreateCodePipeline.yml
@@ -84,35 +84,6 @@ Parameters:
Default: indexerIndexCache
Resources:
- S3BucketCloudFormation:
- Type: 'AWS::S3::Bucket'
- DeletionPolicy: Delete
- Properties:
- BucketName: !Sub '${Environment}-os-indexer-cloudformation-scripts'
- CloudFormationS3BucketPolicy:
- Type: 'AWS::S3::BucketPolicy'
- Properties:
- Bucket: !Ref S3BucketCloudFormation
- PolicyDocument:
- Statement:
- - Action:
- - 's3:*'
- Effect: Allow
- Resource:
- - !Sub 'arn:aws:s3:::${S3BucketCloudFormation}'
- - !Sub 'arn:aws:s3:::${S3BucketCloudFormation}/*'
- Principal:
- AWS:
- - !Sub 'arn:aws:iam::${AWS::AccountId}:root'
- - !GetAtt
- - CodeBuildRole
- - Arn
- - !GetAtt
- - PipelineRole
- - Arn
- - !GetAtt
- - CFNRole
- - Arn
ArtifactStoreBucket:
Type: 'AWS::S3::Bucket'
DeletionPolicy: Delete
@@ -134,15 +105,12 @@ Resources:
Principal:
AWS:
- !Sub 'arn:aws:iam::${AWS::AccountId}:root'
- - !GetAtt
- - CodeBuildRole
- - Arn
- - !GetAtt
- - PipelineRole
- - Arn
- - !GetAtt
- - CFNRole
- - Arn
+ - !ImportValue
+ 'Fn::Sub': '${Environment}-CodeBuildRoleArn'
+ - !ImportValue
+ 'Fn::Sub': '${Environment}-PipelineRoleArn'
+ - !ImportValue
+ 'Fn::Sub': '${Environment}-CFNRoleArn'
CachingBucket:
Type: AWS::S3::Bucket
DeletionPolicy: Delete
@@ -165,9 +133,12 @@ Resources:
Principal:
AWS:
- !Sub arn:aws:iam::${AWS::AccountId}:root
- - !GetAtt [CodeBuildRole,Arn]
- - !GetAtt [PipelineRole,Arn]
- - !GetAtt [CFNRole,Arn]
+ - !ImportValue
+ 'Fn::Sub': '${Environment}-CodeBuildRoleArn'
+ - !ImportValue
+ 'Fn::Sub': '${Environment}-PipelineRoleArn'
+ - !ImportValue
+ 'Fn::Sub': '${Environment}-CFNRoleArn'
SNSCodePipelineDeploymentFailed:
Type: 'AWS::SNS::Topic'
@@ -205,9 +176,8 @@ Resources:
Location: !Ref ArtifactStoreBucket
Type: S3
Name: !Sub '${Environment}-OSDU-OS-Indexer-CodePipeline'
- RoleArn: !GetAtt
- - PipelineRole
- - Arn
+ RoleArn: !ImportValue
+ 'Fn::Sub': '${Environment}-PipelineRoleArn'
Stages:
- Name: Source
Actions:
@@ -251,9 +221,8 @@ Resources:
Configuration:
ActionMode: CREATE_UPDATE
Capabilities: CAPABILITY_NAMED_IAM
- RoleArn: !GetAtt
- - CFNRole
- - Arn
+ RoleArn: !ImportValue
+ 'Fn::Sub': '${Environment}-CFNRoleArn'
StackName: !Sub '${Environment}-${MasterStackName}'
TemplatePath: !Sub 'Source::${MasterTemplateName}'
TemplateConfiguration: !Sub >-
@@ -280,10 +249,8 @@ Resources:
Properties:
Name: !Sub '${Environment}-pre-deployment-codebuild-${CodeCommitRepositoryName}'
Description: CodeBuild commands which run prior to the CloudFormation deployment.
- ServiceRole:
- 'Fn::GetAtt':
- - CodeBuildRole
- - Arn
+ ServiceRole: !ImportValue
+ 'Fn::Sub': '${Environment}-CodeBuildRoleArn'
Artifacts:
Type: S3
Location: !Ref ArtifactStoreBucket
@@ -303,8 +270,11 @@ Resources:
Type: PLAINTEXT
Value: !Ref DeploymentRegion
- Name: CFN_S3_BUCKET
+ Value: !ImportValue
+ 'Fn::Sub': '${Environment}-S3BucketCloudFormation'
+ - Name: APPLICATION_NAME
Type: PLAINTEXT
- Value: !Sub '${Environment}-os-indexer-cloudformation-scripts'
+ Value: !Ref CodeCommitRepositoryName
PrivilegedMode: false
Source:
BuildSpec: ./provider/indexer-aws/buildspec-pre-deploy.yml
@@ -318,8 +288,8 @@ Resources:
Properties:
Name: !Sub ${Environment}-post-deployment-codebuild-${CodeCommitRepositoryName}
Description: CodeBuild commands which run after the CloudFormation deployment.
- ServiceRole:
- Fn::GetAtt: [ CodeBuildRole, Arn ]
+ ServiceRole: !ImportValue
+ 'Fn::Sub': '${Environment}-CodeBuildRoleArn'
Artifacts:
Type: S3
Location: !Ref ArtifactStoreBucket
@@ -338,18 +308,12 @@ Resources:
- Name: AWS_REGION
Type: PLAINTEXT
Value: !Ref DeploymentRegion
+ - Name: VSTS_FEED_USER
+ Type: PLAINTEXT
+ Value: '{{resolve:secretsmanager:dev-VSTSFeedToken:SecretString:vsts_feed_user}}'
- Name: VSTS_FEED_TOKEN
Type: PLAINTEXT
Value: '{{resolve:secretsmanager:dev-VSTSFeedToken:SecretString:vsts_feed_token}}'
- - Name: S3_MAVEN_REPOSITORY
- Type: PLAINTEXT
- Value: !Sub ${Environment}-osdu-local-maven-repository
- - Name: AWS_SECRET_ACCESS_KEY_MAVEN
- Type: PLAINTEXT
- Value: '{{resolve:secretsmanager:dev-MavenS3RepositoryIamCredentials:SecretString:secret_key}}'
- - Name: AWS_ACCESS_KEY_ID_MAVEN
- Type: PLAINTEXT
- Value: '{{resolve:secretsmanager:dev-MavenS3RepositoryIamCredentials:SecretString:access_key}}'
- Name: IMAGE_TAG
Type: PLAINTEXT
Value: latest
@@ -368,6 +332,9 @@ Resources:
- Name: APPLICATION_NAME
Type: PLAINTEXT
Value: !Ref CodeCommitRepositoryName
+ - Name: M2_REPO_S3_BUCKET
+ Type: PLAINTEXT
+ Value: !Sub "${Environment}-${AWS::AccountId}-persistent-maven-m2-bucket"
PrivilegedMode: true
Source:
BuildSpec: ./provider/indexer-aws/buildspec-post-deploy.yml
@@ -388,293 +355,4 @@ Resources:
!Sub "${Environment}-OSDU-PrivateSubnet-AZ2"
VpcId:
Fn::ImportValue:
- !Sub "${Environment}-OSDU-VPC"
-
- CFNRole:
- Type: 'AWS::IAM::Role'
- Properties:
- AssumeRolePolicyDocument:
- Statement:
- - Action:
- - 'sts:AssumeRole'
- Effect: Allow
- Principal:
- Service:
- - cloudformation.amazonaws.com
- Version: 2012-10-17
- Path: /
- Policies:
- - PolicyName: !Sub 'CloudFormationRole-${CodeCommitRepositoryName}'
- PolicyDocument:
- Version: 2012-10-17
- Statement:
- - Action:
- - 's3:*'
- - 'ec2:*'
- - 'apigateway:*'
- - 'cloudwatch:*'
- - 'events:*'
- - 'logs:*'
- - 'xray:*'
- - 'lambda:*'
- - 'rds:*'
- - 'codepipeline:*'
- - 'codecommit:*'
- - 'cloudformation:*'
- - 'ecr:*'
- - 'dynamodb:*'
- - 'application-autoscaling:*'
- - 'autoscaling:*'
- - 'states:*'
- - 'iam:CreateUser'
- - 'iam:UpdateUser'
- - 'iam:DeleteUser'
- - 'iam:CreateAccessKey'
- - 'iam:UpdateAccessKey'
- - 'iam:DeleteAccessKey'
- - 'iam:Delete*'
- - 'iam:List*'
- - 'iam:Get*'
- - 'iam:Put*'
- - 'iam:CreateServiceSpecificCredential'
- - 'iam:DeactivateMFADevice'
- - 'iam:GenerateServiceLastAccessedDetails'
- - 'iam:UpdateOpenIDConnectProviderThumbprint'
- - 'iam:PutRolePolicy'
- - 'iam:AddRoleToInstanceProfile'
- - 'iam:SimulateCustomPolicy'
- - 'iam:UploadSSHPublicKey'
- - 'iam:UpdateServiceSpecificCredential'
- - 'iam:RemoveClientIDFromOpenIDConnectProvider'
- - 'iam:UpdateRoleDescription'
- - 'iam:UpdateServerCertificate'
- - 'iam:CreateInstanceProfile'
- - 'iam:GenerateCredentialReport'
- - 'iam:UntagRole'
- - 'iam:PutRolePermissionsBoundary'
- - 'iam:TagRole'
- - 'iam:ResetServiceSpecificCredential'
- - 'iam:PassRole'
- - 'iam:EnableMFADevice'
- - 'iam:ResyncMFADevice'
- - 'iam:UpdateSAMLProvider'
- - 'iam:CreatePolicy'
- - 'iam:CreateServiceLinkedRole'
- - 'iam:UpdateRole'
- - 'iam:AddClientIDToOpenIDConnectProvider'
- - 'iam:SetDefaultPolicyVersion'
- - 'iam:UpdateAssumeRolePolicy'
- - 'iam:RemoveRoleFromInstanceProfile'
- - 'iam:CreateRole'
- - 'iam:AttachRolePolicy'
- - 'iam:CreateLoginProfile'
- - 'iam:DetachRolePolicy'
- - 'iam:AttachUserPolicy'
- - 'iam:DetachUserPolicy'
- - 'iam:SimulatePrincipalPolicy'
- - 'iam:CreateAccountAlias'
- - 'iam:ChangePassword'
- - 'iam:UpdateLoginProfile'
- - 'iam:UpdateAccessKey'
- - 'iam:UpdateSSHPublicKey'
- - 'iam:UpdateAccountPasswordPolicy'
- - 'iam:CreateSAMLProvider'
- - 'iam:CreateVirtualMFADevice'
- - 'iam:CreateAccessKey'
- - 'iam:AddUserToGroup'
- - 'iam:RemoveUserFromGroup'
- - 'iam:CreatePolicyVersion'
- - 'iam:UploadSigningCertificate'
- - 'iam:TagUser'
- - 'iam:CreateOpenIDConnectProvider'
- - 'iam:UploadServerCertificate'
- - 'iam:UntagUser'
- - 'iam:UpdateSigningCertificate'
- - 'sns:*'
- - 'sqs:*'
- - 'secretsmanager:*'
- - 'acm:*'
- - 'kms:*'
- - 'cloudfront:*'
- - 'route53:*'
- - 'route53domains:*'
- - 'elasticache:*'
- - 'es:*'
- - 'ecr:*'
- - 'codedeploy:*'
- - 'elasticloadbalancing:*'
- - 'ecs:*'
- - 'servicediscovery:CreatePrivateDnsNamespace'
- - 'servicediscovery:CreateService'
- - 'servicediscovery:GetNamespace'
- - 'servicediscovery:GetOperation'
- - 'servicediscovery:GetService'
- - 'servicediscovery:ListNamespaces'
- - 'servicediscovery:ListServices'
- - 'servicediscovery:UpdateService'
- - 'servicediscovery:DeleteService'
- Effect: Allow
- Resource: '*'
- CodeBuildRole:
- Type: 'AWS::IAM::Role'
- Properties:
- RoleName: !Sub 'CodeBuildRole-${CodeCommitRepositoryName}'
- AssumeRolePolicyDocument:
- Version: 2012-10-17
- Statement:
- - Effect: Allow
- Principal:
- Service:
- - codebuild.amazonaws.com
- Action:
- - 'sts:AssumeRole'
- Path: /service-role/
- Policies:
- - PolicyName: !Sub 'CodeBuildNestedCFNAccessPolicy-${CodeCommitRepositoryName}'
- PolicyDocument:
- Version: 2012-10-17
- Statement:
- - Effect: Allow
- Action:
- - 'cloudformation:Get*'
- - 'cloudformation:Describe*'
- - 'cloudformation:List*'
- Resource:
- - '*'
- - Effect: Allow
- Action:
- - 'codebuild:StartBuild'
- Resource:
- - 'Fn::Sub': >-
- arn:aws:codebuild:${AWS::Region}:${AWS::AccountId}:project/*
- - Effect: Allow
- Action:
- - 'codecommit:ListBranches'
- - 'codecommit:ListRepositories'
- - 'codecommit:BatchGetRepositories'
- - 'codecommit:Get*'
- - 'codecommit:GitPull'
- Resource:
- - !Sub >-
- arn:aws:codecommit:${AWS::Region}:${AWS::AccountId}:${CodeCommitRepositoryName}
- - Effect: Allow
- Action:
- - 'ec2:*'
- - 'cloudformation:ValidateTemplate'
- - 'elasticloadbalancing:Describe*'
- - 'autoscaling:Describe*'
- - 'iam:Get*'
- - 'iam:List*'
- - 'logs:Describe*'
- - 'logs:Get*'
- - 'tag:Get*'
- - "ecr:*"
- - "codedeploy:*"
- - "ecs:*"
- Resource:
- - '*'
- - Effect: Allow
- Action:
- - 'logs:CreateLogGroup'
- - 'logs:CreateLogStream'
- - 'logs:PutLogEvents'
- Resource:
- - 'Fn::Sub': >-
- arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/codebuild/*
- - Effect: Allow
- Action:
- - 's3:*'
- Resource: '*'
- - Effect: Allow
- Action:
- - 'lambda:UpdateFunctionCode'
- - 'lambda:UpdateFunctionConfiguration'
- - 'lambda:PublishLayerVersion'
- - 'lambda:GetLayerVersion'
- Resource: '*'
- - Effect: Allow
- Action:
- - 'apigateway:GET'
- - 'apigateway:POST'
- Resource: '*'
- PipelineRole:
- Type: 'AWS::IAM::Role'
- Properties:
- AssumeRolePolicyDocument:
- Statement:
- - Action:
- - 'sts:AssumeRole'
- Effect: Allow
- Principal:
- Service:
- - codepipeline.amazonaws.com
- Version: 2012-10-17
- Path: /
- Policies:
- - PolicyName: !Sub 'CodePipelineAccess-${CodeCommitRepositoryName}'
- PolicyDocument:
- Version: 2012-10-17
- Statement:
- - Action:
- - 's3:*'
- - 'cloudformation:CreateStack'
- - 'cloudformation:DescribeStacks'
- - 'cloudformation:DeleteStack'
- - 'cloudformation:UpdateStack'
- - 'cloudformation:CreateChangeSet'
- - 'cloudformation:ExecuteChangeSet'
- - 'cloudformation:DeleteChangeSet'
- - 'cloudformation:DescribeChangeSet'
- - 'cloudformation:SetStackPolicy'
- - 'cloudformation:ValidateTemplate'
- - 'iam:PassRole'
- - 'sns:Publish'
- - 'lambda:ListFunctions'
- - 'lambda:InvokeFunction'
- - 'ec2:Describe*'
- - 'ec2:Get*'
- - 'ec2:Search*'
- - 'ec2:*Vpc*'
- - 'ec2:*Gateway'
- - 'ec2:*Tags'
- - 'ec2:*Subnet*'
- - 'ec2:*Route*'
- - 'ec2:*SecurityGroup'
- - 'ec2:allocate*'
- - 'ec2:release*'
- Effect: Allow
- Resource: '*'
- - Action:
- - 'codecommit:GetUploadArchiveStatus'
- - 'codecommit:CancelUploadArchive'
- - 'codecommit:GetBranch'
- - 'codecommit:GetCommit'
- - 'codecommit:GetUploadStatus'
- - 'codecommit:UploadArchive'
- Effect: Allow
- Resource: '*'
- - Effect: Allow
- Action:
- - 'codebuild:*'
- Resource:
- - 'Fn::Sub': >-
- arn:aws:codebuild:${AWS::Region}:${AWS::AccountId}:project/*
-
-Outputs:
- CodeBuildRoleArn:
- Description: The ARN of the role used by the CodeBuild projects.
- Value: !GetAtt CodeBuildRole.Arn
- Export:
- Name: !Sub ${Environment}-${CodeCommitRepositoryName}-CodeBuildRoleArn
-
- CFNRoleArn:
- Description: The ARN of the role used by CloudFormation templates run from the automated pipeline.
- Value: !GetAtt CFNRole.Arn
- Export:
- Name: !Sub ${Environment}-${CodeCommitRepositoryName}-CFNRoleArn
-
- PipelineRoleArn:
- Description: The ARN of the role used by the application's CodePipeline.
- Value: !GetAtt PipelineRole.Arn
- Export:
- Name: !Sub ${Environment}-${CodeCommitRepositoryName}-PipelineRoleArn
+ !Sub "${Environment}-OSDU-VPC"
\ No newline at end of file
diff --git a/provider/indexer-aws/CloudFormation/Master/os-indexer-master.yml b/provider/indexer-aws/CloudFormation/Master/os-indexer-master.yml
index 17dd9c69dfcd27cc4c4ab2249d49028251770815..c4a49e0b6ab7580627d96052056f23df5e745680 100644
--- a/provider/indexer-aws/CloudFormation/Master/os-indexer-master.yml
+++ b/provider/indexer-aws/CloudFormation/Master/os-indexer-master.yml
@@ -15,6 +15,18 @@
AWSTemplateFormatVersion: 2010-09-09
Description: Creates all AWS resources used by OSDU's Indexer Service. Requires having previously setup the CodeCommit repository, as well as the CodePipeline (manual template).
Parameters:
+
+ VersionNumber:
+ Description: Version Number for the pom to deploy the jar and Docker Image deployment in the Dockerfile
+ Type: String
+ Default: '0.0.1'
+
+ ServiceName:
+ Description: >-
+ Service name for jar deployment in the Dockerfile
+ Type: String
+ Default: 'indexer'
+
Environment:
Description: The name of the environment.
Type: String
@@ -30,17 +42,9 @@ Parameters:
Type: String
Default: us-east-1
- ChildTemplateBasePath:
- Description: >-
- The base path for where child CloudFormation templates are located – can be relative or absolute, e.g.
- https://s3.amazonaws.com/dev-osdu-cloudformation-scripts/Automated/
- Type: String
- AllowedPattern: '^https:\/\/s3.amazonaws.com\/.*\/$'
- Default: https://s3.amazonaws.com/dev-osdu-cloudformation-scripts/Automated/
-
ApplicationName:
Description: >
- The name of the application, should be equal to the repository name.
+ The name of the indexer application, should be equal to the repository name.
Type: String
MinLength: '1'
MaxLength: '64'
@@ -48,15 +52,15 @@ Parameters:
ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
Default: os-indexer
- StorageApplicationName:
+ SearchApplicationName:
Description: >
- The name of the storage application
+ The name of the Search Service application (ex: os-search). Should be the same as the Search Service repo name.
Type: String
MinLength: '1'
MaxLength: '64'
AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
- Default: osdu-storage
+ Default: os-search
KeyName:
Description: >
@@ -443,6 +447,30 @@ Parameters:
MinValue: 256
MaxValue: 131072
+ DomainName:
+ Description: >-
+ The optional custom DNS name for the ECS service's load balancer. If omitted, the site will only be accessible
+ via the ECS service's Application Load Balancer DNS name. This value is used in the creation and signing of
+ the service's SSL certificate. Leave blank is not using a custom domain for this deployment.
+ Type: String
+ Default: ''
+
+ HostedZoneName:
+ Description: >-
+ The name of the hosted zone (ex: for indexer.osdu.slb.com, this would likely be osdu.slb.com).
+ Leave blank is not using a custom domain for this deployment.
+ Type: String
+ Default: ''
+
+ AcmCertificateArn:
+ Description: >-
+ The Amazon Resource Name (ARN) of an existing AWS Certificate Manager (ACM) certificate.
+ If omitted, a new SSL certified will be requested/generated (only if the custom domain name
+ parameter is provided, otherwise the ECS service's ALB will not use SSL/HTTPS).
+ Type: String
+ AllowedPattern: "^(|arn:aws:acm:.*)$"
+ Default: ''
+
Resources:
#### Shared Resources ################################################################
@@ -450,7 +478,11 @@ Resources:
IAMCredentialsStack:
Type: 'AWS::CloudFormation::Stack'
Properties:
- TemplateURL: !Join [ '', [ !Ref ChildTemplateBasePath, iam-credentials.yml ] ]
+ TemplateURL: !Sub
+ - https://s3.amazonaws.com/${CloudFormationS3Bucket}/${ApplicationName}/Automated/${CFNTemplateFilename}
+ - CloudFormationS3Bucket: !ImportValue
+ 'Fn::Sub': '${Environment}-S3BucketCloudFormation'
+ CFNTemplateFilename: iam-credentials.yml
Parameters:
Environment: !Ref Environment
Region: !Ref DeploymentRegion
@@ -460,7 +492,11 @@ Resources:
MessageBusSNSStack:
Type: 'AWS::CloudFormation::Stack'
Properties:
- TemplateURL: !Join ['', [!Ref ChildTemplateBasePath, sns-topic.yml ] ]
+ TemplateURL: !Sub
+ - https://s3.amazonaws.com/${CloudFormationS3Bucket}/${ApplicationName}/Automated/${CFNTemplateFilename}
+ - CloudFormationS3Bucket: !ImportValue
+ 'Fn::Sub': '${Environment}-S3BucketCloudFormation'
+ CFNTemplateFilename: sns-topic.yml
Parameters:
Environment: !Ref Environment
Region: !Ref DeploymentRegion
@@ -473,18 +509,28 @@ Resources:
Type: 'AWS::CloudFormation::Stack'
DependsOn: IAMCredentialsStack
Properties:
- TemplateURL: !Join [ '', [ !Ref ChildTemplateBasePath, ecs-network.yml ] ]
+ TemplateURL: !Sub
+ - https://s3.amazonaws.com/${CloudFormationS3Bucket}/${ApplicationName}/Automated/${CFNTemplateFilename}
+ - CloudFormationS3Bucket: !ImportValue
+ 'Fn::Sub': '${Environment}-S3BucketCloudFormation'
+ CFNTemplateFilename: ecs-network.yml
Parameters:
Environment: !Ref Environment
Region: !Ref DeploymentRegion
ApplicationName: !Ref ApplicationName
ECSPort: !Ref ECSPort
+ DomainName: !Ref DomainName
+ AcmCertificateArn: !Ref AcmCertificateArn
ECSClusterStack:
Type: 'AWS::CloudFormation::Stack'
DependsOn: [SchemaCacheStack, IndexCacheStack]
Properties:
- TemplateURL: !Join [ '', [ !Ref ChildTemplateBasePath, ecs-cluster.yml ] ]
+ TemplateURL: !Sub
+ - https://s3.amazonaws.com/${CloudFormationS3Bucket}/${ApplicationName}/Automated/${CFNTemplateFilename}
+ - CloudFormationS3Bucket: !ImportValue
+ 'Fn::Sub': '${Environment}-S3BucketCloudFormation'
+ CFNTemplateFilename: ecs-cluster.yml
Parameters:
Environment: !Ref Environment
Region: !Ref DeploymentRegion
@@ -498,6 +544,9 @@ Resources:
ECSPort: !Ref ECSPort
SNSTopicName: !Ref SNSTopicName
ECSMemoryAllocation: !Ref ECSMemoryAllocation
+ DomainName: !Ref DomainName
+ HostedZoneName: !Ref HostedZoneName
+ ElasticsearchDomainName: !Ref ElasticsearchDomainName
#### Caching Resources ###############################################################
@@ -505,7 +554,11 @@ Resources:
Type: 'AWS::CloudFormation::Stack'
DependsOn: ECSNetworkStack
Properties:
- TemplateURL: !Join [ '', [ !Ref ChildTemplateBasePath, cache.yml ] ]
+ TemplateURL: !Sub
+ - https://s3.amazonaws.com/${CloudFormationS3Bucket}/${ApplicationName}/Automated/${CFNTemplateFilename}
+ - CloudFormationS3Bucket: !ImportValue
+ 'Fn::Sub': '${Environment}-S3BucketCloudFormation'
+ CFNTemplateFilename: cache.yml
Parameters:
Environment: !Ref Environment
Region: !Ref DeploymentRegion
@@ -519,7 +572,11 @@ Resources:
Type: 'AWS::CloudFormation::Stack'
DependsOn: ECSNetworkStack
Properties:
- TemplateURL: !Join [ '', [ !Ref ChildTemplateBasePath, cache.yml ] ]
+ TemplateURL: !Sub
+ - https://s3.amazonaws.com/${CloudFormationS3Bucket}/${ApplicationName}/Automated/${CFNTemplateFilename}
+ - CloudFormationS3Bucket: !ImportValue
+ 'Fn::Sub': '${Environment}-S3BucketCloudFormation'
+ CFNTemplateFilename: cache.yml
Parameters:
Environment: !Ref Environment
Region: !Ref DeploymentRegion
@@ -530,13 +587,16 @@ Resources:
NumberOfCacheNodes: !Ref SchemaCacheNumberOfCacheNodes
#### Elasticsearch Resources #########################################################
- # TODO: Add VPCs to Elasticsearch
ElasticsearchStack:
Type: 'AWS::CloudFormation::Stack'
- DependsOn: IAMCredentialsStack
+ DependsOn: [IAMCredentialsStack, ECSNetworkStack]
Properties:
- TemplateURL: !Join [ '', [ !Ref ChildTemplateBasePath, elasticsearch.yml ] ]
+ TemplateURL: !Sub
+ - https://s3.amazonaws.com/${CloudFormationS3Bucket}/${ApplicationName}/Automated/${CFNTemplateFilename}
+ - CloudFormationS3Bucket: !ImportValue
+ 'Fn::Sub': '${Environment}-S3BucketCloudFormation'
+ CFNTemplateFilename: elasticsearch.yml
Parameters:
Environment: !Ref Environment
Region: !Ref DeploymentRegion
@@ -548,3 +608,18 @@ Resources:
ZoneAwarenessEnabled: !Ref ZoneAwarenessEnabled
ElasticsearchVersion: !Ref ElasticsearchVersion
EBSVolumeSize: !Ref EBSVolumeSize
+ ApplicationName: !Ref ApplicationName
+ SearchApplicationName: !Ref SearchApplicationName
+
+Outputs:
+ JarVersionNumber:
+ Description: The service name associated with the JAR package for the Dockerfile.
+ Value: !Ref 'VersionNumber'
+ Export:
+ Name: !Sub ${Environment}-${ApplicationName}-JarVersionNumber
+
+ JarServiceName:
+ Description: The service name associated with the JAR package for the Dockerfile.
+ Value: !Ref 'ServiceName'
+ Export:
+ Name: !Sub ${Environment}-${ApplicationName}-JarServiceName
diff --git a/provider/indexer-aws/CloudFormation/Params/dev.template_configuration.json b/provider/indexer-aws/CloudFormation/Params/dev.template_configuration.json
index c24b72c65cdf1c366e0c85096177c12f30107a3c..f33fd69a62180cc5c08e9962a46aae62705cabf5 100644
--- a/provider/indexer-aws/CloudFormation/Params/dev.template_configuration.json
+++ b/provider/indexer-aws/CloudFormation/Params/dev.template_configuration.json
@@ -2,8 +2,8 @@
"Parameters" : {
"Environment" : "dev",
"DeploymentRegion" : "us-east-1",
- "ChildTemplateBasePath" : "https://s3.amazonaws.com/dev-os-indexer-cloudformation-scripts/Automated/",
"ApplicationName" : "os-indexer",
+ "SearchApplicationName" : "os-search",
"KeyName": "indexer-ecs-keypair",
"DesiredCapacity": "2",
"MinSize": "0",
@@ -24,14 +24,19 @@
"ElasticsearchDomainName": "osdu-indexer",
"ElasticsearchNodeInstanceType": "t2.medium.elasticsearch",
"DedicatedMasterInstanceType": "t2.medium.elasticsearch",
- "NumberOfElasticsearchNodes": "2",
- "NumberOfDedicatedMasterNodes": "2",
+ "NumberOfElasticsearchNodes": "4",
+ "NumberOfDedicatedMasterNodes": "3",
"ZoneAwarenessEnabled": "false",
"ElasticsearchVersion": "6.8",
"EBSVolumeSize": "10",
- "ECSPort": "80",
+ "ECSPort": "443",
"ECSCPUAllocation": "1024",
- "ECSMemoryAllocation": "3072"
+ "ECSMemoryAllocation": "3072",
+ "DomainName": "",
+ "HostedZoneName": "",
+ "AcmCertificateArn": "",
+ "ServiceName": "indexer",
+ "VersionNumber": "1.0.5-SNAPSHOT"
},
"Tags" : {
"Environment" : "dev"
diff --git a/provider/indexer-aws/CloudFormation/Params/prod.template_configuration.json b/provider/indexer-aws/CloudFormation/Params/prod.template_configuration.json
index 013cc5a2bd6a96f28185a4b08291e228f18f9948..cf2353331086afe91f04f7abcaa4c9736a4a5d84 100644
--- a/provider/indexer-aws/CloudFormation/Params/prod.template_configuration.json
+++ b/provider/indexer-aws/CloudFormation/Params/prod.template_configuration.json
@@ -2,8 +2,8 @@
"Parameters" : {
"Environment" : "prod",
"DeploymentRegion" : "us-east-1",
- "ChildTemplateBasePath" : "https://s3.amazonaws.com/prod-os-indexer-cloudformation-scripts/Automated/",
"ApplicationName" : "os-indexer",
+ "SearchApplicationName" : "os-search",
"KeyName": "indexer-ecs-keypair",
"DesiredCapacity": "2",
"MinSize": "0",
@@ -29,9 +29,14 @@
"ZoneAwarenessEnabled": "false",
"ElasticsearchVersion": "6.8",
"EBSVolumeSize": "10",
- "ECSPort": "80",
+ "ECSPort": "443",
"ECSCPUAllocation": "1024",
- "ECSMemoryAllocation": "3072"
+ "ECSMemoryAllocation": "3072",
+ "DomainName": "",
+ "HostedZoneName": "",
+ "AcmCertificateArn": "",
+ "ServiceName": "indexer",
+ "VersionNumber": "1.0.5-SNAPSHOT"
},
"Tags" : {
"Environment" : "prod"
diff --git a/provider/indexer-aws/CloudFormation/Params/uat.template_configuration.json b/provider/indexer-aws/CloudFormation/Params/uat.template_configuration.json
index c5e5a3af82545b5617f743aae1485117413132d2..10d7331ed0253013feff3eaf96e468aca3b80394 100644
--- a/provider/indexer-aws/CloudFormation/Params/uat.template_configuration.json
+++ b/provider/indexer-aws/CloudFormation/Params/uat.template_configuration.json
@@ -2,8 +2,8 @@
"Parameters" : {
"Environment" : "uat",
"DeploymentRegion" : "us-east-1",
- "ChildTemplateBasePath" : "https://s3.amazonaws.com/uat-os-indexer-cloudformation-scripts/Automated/",
"ApplicationName" : "os-indexer",
+ "SearchApplicationName" : "os-search",
"KeyName": "indexer-ecs-keypair",
"DesiredCapacity": "2",
"MinSize": "0",
@@ -29,9 +29,14 @@
"ZoneAwarenessEnabled": "false",
"ElasticsearchVersion": "6.8",
"EBSVolumeSize": "10",
- "ECSPort": "80",
+ "ECSPort": "443",
"ECSCPUAllocation": "1024",
- "ECSMemoryAllocation": "3072"
+ "ECSMemoryAllocation": "3072",
+ "DomainName": "",
+ "HostedZoneName": "",
+ "AcmCertificateArn": "",
+ "ServiceName": "indexer",
+ "VersionNumber": "1.0.5-SNAPSHOT"
},
"Tags" : {
"Environment" : "uat"
diff --git a/provider/indexer-aws/Dockerfile b/provider/indexer-aws/Dockerfile
index bb98f5ab9611074293ed6f060512463bfb068597..07b7281a7d160013df35a5a150fdcaa8f5bf2c53 100644
--- a/provider/indexer-aws/Dockerfile
+++ b/provider/indexer-aws/Dockerfile
@@ -1,10 +1,26 @@
+# Copyright © Amazon Web Services
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
FROM amazoncorretto:8
-ARG JAR_VERSION
-ENV JAR_FILE=indexer-aws-${JAR_VERSION}-spring-boot.jar
+ARG versionNumber
+ARG service
+ENV serviceName=${service}-aws
+ENV awsJar=${serviceName}-${versionNumber}-spring-boot.jar
WORKDIR /
-COPY provider/indexer-aws/target/$JAR_FILE $JAR_FILE
+COPY provider/${serviceName}/target/${awsJar} ${awsJar}
EXPOSE 8080
-CMD java -jar $JAR_FILE
+CMD ["sh","-c", " java -jar ${awsJar}"]
diff --git a/provider/indexer-aws/buildspec-jar-deploy.yml b/provider/indexer-aws/buildspec-jar-deploy.yml
new file mode 100644
index 0000000000000000000000000000000000000000..546f75409fd9e37722dd85a7298633e2f3b0932b
--- /dev/null
+++ b/provider/indexer-aws/buildspec-jar-deploy.yml
@@ -0,0 +1,59 @@
+# Copyright © Amazon Web Services
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: 0.2
+
+phases:
+ install:
+ runtime-versions:
+ java: openjdk8
+ commands:
+ - echo Entered the install phase...
+ - apt-get update -y
+ - apt-get install -y maven
+ - java -version
+ - mvn clean # .m2 is not created until the first Maven command
+ - cp ./provider/indexer-aws/maven/settings.xml /root/.m2/settings.xml # copy the AWS-specific settings.xml to the CodeBuild instance's .m2 folder
+ - cat /root/.m2/settings.xml
+ - java -version
+ - export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
+ - echo $JAVA_HOME
+ - mvn -version
+ - echo "Look below for M2 bucket name:"
+ - echo $M2_REPO_S3_BUCKET
+ - aws s3 sync s3://$M2_REPO_S3_BUCKET /root/.m2 # copy previous state of the shared libraries' .m2 folder from S3 to local
+ - nohup /usr/local/bin/dockerd --host=unix:///var/run/docker.sock --host=tcp://127.0.0.1:2375 --storage-driver=overlay2& # start the Docker Daemon
+ - timeout 15 sh -c "until docker info; do echo .; sleep 1; done" # wait for Docker to be ready before proceeding to the build steps
+ build:
+ commands:
+ - echo os-indexer Java build started on `date`...
+ - java -version
+ - export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
+ - mvn -version
+ - echo All environment variables
+ - printenv
+ - mvn clean test -P indexer-core,indexer-aws
+ - echo ...os-indexer Java build completed on `date`.
+ - echo os-indexer beginning packaging to jar...
+ - mvn clean install -P indexer-core,indexer-aws -Ddeployment.environment=$ENVIRONMENT
+ - echo Uploading os-indexer JAR to S3...
+ - aws s3 cp provider/$JAR_SERVICE_BASE-aws/target s3://$JAR_DEPLOY_S3_BUCKET/$JAR_SERVICE_BASE-aws --recursive --exclude "*" --include "*.jar" # build and push the JAR(s) to S3
+
+cache:
+ paths:
+ - '/root/.m2/**/*'
+
+artifacts:
+ files:
+ - '**/*'
diff --git a/provider/indexer-aws/buildspec-post-deploy.yml b/provider/indexer-aws/buildspec-post-deploy.yml
index 02fd6e3a518ef177e7140ac5a164f94a60d26c46..5a576fa4c687e91a904ee66939f01d99df4b0981 100644
--- a/provider/indexer-aws/buildspec-post-deploy.yml
+++ b/provider/indexer-aws/buildspec-post-deploy.yml
@@ -30,6 +30,9 @@ phases:
- export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
- echo $JAVA_HOME
- mvn -version
+ - echo "Look below for M2 bucket name:"
+ - echo $M2_REPO_S3_BUCKET
+ - aws s3 sync s3://$M2_REPO_S3_BUCKET /root/.m2 # copy previous state of the shared libraries' .m2 folder from S3 to local
- nohup /usr/local/bin/dockerd --host=unix:///var/run/docker.sock --host=tcp://127.0.0.1:2375 --storage-driver=overlay2& # start the Docker Daemon
- timeout 15 sh -c "until docker info; do echo .; sleep 1; done" # wait for Docker to be ready before proceeding to the build steps
pre_build:
@@ -47,28 +50,24 @@ phases:
- java -version
- export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
- mvn -version
- - echo Look below for repository bucket
- - echo $S3_MAVEN_REPOSITORY
- echo Setting environment variables from CloudFormation Exports... # use the AWS CLI commands to query for the CloudFormation export values created in the previous step and set the required environment variables
- echo Environment - $ENVIRONMENT
- echo SchemaCacheName - $SCHEMA_CACHE_NAME
- echo IndexCacheName - $INDEX_CACHE_NAME
- echo AWSRegion - $AWS_REGION
- - export CACHE_CLUSTER_SCHEMA_ENDPOINT=$(aws cloudformation list-exports --query "Exports[?Name=='$ENVIRONMENT-$SCHEMA_CACHE_NAME-RedisEndpointAddress'].[Value]" --output text --region $AWS_REGION)
- - export CACHE_CLUSTER_SCHEMA_PORT=$(aws cloudformation list-exports --query "Exports[?Name=='$ENVIRONMENT-$SCHEMA_CACHE_NAME-RedisEndpointPort'].[Value]" --output text --region $AWS_REGION)
- - export CACHE_CLUSTER_INDEX_ENDPOINT=$(aws cloudformation list-exports --query "Exports[?Name=='$ENVIRONMENT-$INDEX_CACHE_NAME-RedisEndpointAddress'].[Value]" --output text --region $AWS_REGION)
- - export CACHE_CLUSTER_INDEX_PORT=$(aws cloudformation list-exports --query "Exports[?Name=='$ENVIRONMENT-$INDEX_CACHE_NAME-RedisEndpointPort'].[Value]" --output text --region $AWS_REGION)
- - export S3_DATA_BUCKET=$(aws cloudformation list-exports --query "Exports[?Name=='$ENVIRONMENT-S3BucketDataStorage'].[Value]" --output text --region $AWS_REGION)
- - export SNS_TOPIC_NAME=$(aws cloudformation list-exports --query "Exports[?Name=='$ENVIRONMENT-OSDUIndexerSNSTopic'].[Value]" --output text --region $AWS_REGION)
+ - export VERSIONNUMBER=$(aws cloudformation list-exports --query "Exports[?Name=='$ENVIRONMENT-$APPLICATION_NAME-JarVersionNumber'].[Value]" --output text --region $AWS_REGION)
+ - echo VERSIONNUMBER - $VERSIONNUMBER
+ - export SERVICE=$(aws cloudformation list-exports --query "Exports[?Name=='$ENVIRONMENT-$APPLICATION_NAME-JarServiceName'].[Value]" --output text --region $AWS_REGION)
+ - echo SERVICE - $SERVICE
- echo ...finished setting environment variables!
- echo All environment variables
- printenv
- - mvn clean test -pl indexer-core,provider/indexer-aws -Ddeployment.environment=$ENVIRONMENT -Ddeployment.repositorybucket=$S3_MAVEN_REPOSITORY -Daws.accessKeyId=$AWS_ACCESS_KEY_ID -Daws.secretKey=$AWS_SECRET_KEY -Dazure.devops.token=$VSTS_FEED_TOKEN -DCACHE_CLUSTER_SCHEMA_ENDPOINT=$CACHE_CLUSTER_SCHEMA_ENDPOINT -DCACHE_CLUSTER_SCHEMA_PORT=$CACHE_CLUSTER_SCHEMA_PORT -DCACHE_CLUSTER_GROUP_ENDPOINT=$CACHE_CLUSTER_SCHEMA_ENDPOINT -DCACHE_CLUSTER_SCHEMA_PORT=$CACHE_CLUSTER_SCHEMA_PORT -DCACHE_CLUSTER_INDEX_ENDPOINT=$CACHE_CLUSTER_INDEX_ENDPOINT -DCACHE_CLUSTER_INDEX_PORT=$CACHE_CLUSTER_INDEX_PORT -DS3_DATA_BUCKET=$S3_DATA_BUCKET -DSNS_TOPIC_NAME=$SNS_TOPIC_NAME -DAWS_ACCOUNT_ID=$AWS_ACCOUNT_ID -DAWS_REGION=$AWS_REGION -DENVIRONMENT=$ENVIRONMENT
+ - mvn clean test -P indexer-core,indexer-aws
- echo ...os-indexer Java build completed on `date`.
- echo os-indexer beginning packaging to jar...
- - mvn clean deploy -pl indexer-core,provider/indexer-aws -Ddeployment.environment=$ENVIRONMENT -Ddeployment.repositorybucket=$S3_MAVEN_REPOSITORY -Daws.accessKeyId=$AWS_ACCESS_KEY_ID_MAVEN -Daws.secretKey=$AWS_SECRET_ACCESS_KEY_MAVEN -Dazure.devops.token=$VSTS_FEED_TOKEN
+ - mvn clean install -P indexer-core,indexer-aws -Ddeployment.environment=$ENVIRONMENT -Dversion.number=$VERSIONNUMBER
- echo os-indexer Docker image build started on `date`...
- - docker build -f provider/indexer-aws/Dockerfile -t $REPOSITORY_URI:latest .
+ - docker build -f provider/indexer-aws/Dockerfile -t $REPOSITORY_URI:latest --build-arg versionNumber=$VERSIONNUMBER --build-arg service=$SERVICE .
- docker tag $REPOSITORY_URI:latest $REPOSITORY_URI:$IMAGE_TAG
- echo ...os-indexer Docker image build completed on `date`.
- echo Pushing the Docker image to ECR...
diff --git a/provider/indexer-aws/buildspec-pre-deploy.yml b/provider/indexer-aws/buildspec-pre-deploy.yml
index baf44c6068f3ec612bf4965e95c8f094e9ec8aae..14697d9c882a1905a8205f06328611e2d3c478ef 100644
--- a/provider/indexer-aws/buildspec-pre-deploy.yml
+++ b/provider/indexer-aws/buildspec-pre-deploy.yml
@@ -15,14 +15,6 @@
version: 0.2
phases:
-# pre-build:
-# commands:
-# - echo Logging in to Amazon ECR...
-# - aws --version
-# - $(aws ecr get-login --region $AWS_DEFAULT_REGION --no-include-email)
-# - REPOSITORY_URI=012345678910.dkr.ecr.us-east-1.amazonaws.com/hello-world
-# - COMMIT_HASH=$(echo $CODEBUILD_RESOLVED_SOURCE_VERSION | cut -c 1-7)
-# - IMAGE_TAG=build-$(echo $CODEBUILD_BUILD_ID | awk -F":" '{print $2}')
install:
runtime-versions:
java: openjdk8
@@ -41,11 +33,11 @@ phases:
- mvn -version
build:
commands:
- - echo Starting 'Copying CloudFormation scripts to S3://$CFN_S3_BUCKET'
+ - echo Starting 'Copying CloudFormation scripts to S3://$CFN_S3_BUCKET/$APPLICATION_NAME'
- pwd
- ls
- - aws s3 cp ./provider/indexer-aws/CloudFormation "s3://$CFN_S3_BUCKET" --exclude "*" --include "*.yml" --recursive --debug
- - echo Ending 'Ending CloudFormation scripts to S3://$CFN_S3_BUCKET'
+ - aws s3 cp ./provider/indexer-aws/CloudFormation "s3://$CFN_S3_BUCKET/$APPLICATION_NAME" --exclude "*" --include "*.yml" --recursive --debug
+ - echo Ending 'Ending CloudFormation scripts to S3://$CFN_S3_BUCKET/$APPLICATION_NAME'
# - echo os-indexer build started on `date`...
- pwd
- ls -R -la
diff --git a/provider/indexer-aws/maven/settings.xml b/provider/indexer-aws/maven/settings.xml
index 54c4a7227d6191761d3b091148e6182d45d129cf..adba9a7602b0887236967efc41b49ccad08a7ca1 100644
--- a/provider/indexer-aws/maven/settings.xml
+++ b/provider/indexer-aws/maven/settings.xml
@@ -4,10 +4,12 @@
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
<servers>
+ <!-- Azure DevOps Artifact Store -->
<server>
<id>os-core</id>
- <username>slb-des-ext-collaboration</username>
- <password>${VSTS_FEED_TOKEN}</password>
+ <username>${azure.devops.username}</username>
+ <!-- The generated token was last updated on 01/13/2020 and expires on or before 01/12/2021 -->
+ <password>${azure.devops.token}</password>
</server>
</servers>
diff --git a/provider/indexer-aws/pom.xml b/provider/indexer-aws/pom.xml
index 9a2981be01f8c46def71e2b20aba0de0715c3986..aa763a675d577afe9e3380606336703d0a09a48c 100644
--- a/provider/indexer-aws/pom.xml
+++ b/provider/indexer-aws/pom.xml
@@ -27,11 +27,12 @@
<artifactId>indexer-aws</artifactId>
<description>Storage service on AWS</description>
<packaging>jar</packaging>
+ <version>${version.number}</version>
<properties>
<aws.version>1.11.637</aws.version>
<deployment.environment>dev</deployment.environment>
- <deployment.repositorybucket>osdu-local-maven-repository</deployment.repositorybucket>
+ <version.number>1.0.4-SNAPSHOT</version.number>
</properties>
<dependencies>
@@ -48,8 +49,8 @@
</dependency>
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
- <artifactId>aws-osdu-util</artifactId>
- <version>0.0.9</version>
+ <artifactId>os-core-lib-aws</artifactId>
+ <version>0.0.10</version>
</dependency>
<!-- AWS managed packages -->
@@ -58,11 +59,6 @@
<artifactId>aws-java-sdk-core</artifactId>
<version>1.11.651</version>
</dependency>
- <dependency>
- <groupId>com.github.awslabs</groupId>
- <artifactId>aws-request-signing-apache-interceptor</artifactId>
- <version>deb7941</version>
- </dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
@@ -177,13 +173,6 @@
</executions>
</plugin>
</plugins>
- <extensions>
- <extension>
- <groupId>com.github.seahen</groupId>
- <artifactId>maven-s3-wagon</artifactId>
- <version>1.3.0</version>
- </extension>
- </extensions>
</build>
<repositories>
<repository>
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/persistence/ElasticRepositoryImpl.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/persistence/ElasticRepositoryImpl.java
index 22b0d8f5c5ca21fc079d1409889d9c5f2673f269..70e64ca21d70497fc21121729350d5b16f2a6a44 100644
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/persistence/ElasticRepositoryImpl.java
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/persistence/ElasticRepositoryImpl.java
@@ -28,7 +28,8 @@ public class ElasticRepositoryImpl implements IElasticRepository {
@Value("${aws.es.host}")
String host;
- int port = 8080;
+ @Value("${aws.es.port}")
+ int port;
String userNameAndPassword = "testing";
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/service/ElasticClientHandlerAws.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/service/ElasticClientHandlerAws.java
index 6a2498d0b89e2e85d9dd676039284605cf177fa4..3c7383a80aaa9aaf1a7fa8943673fa86dc0f371b 100644
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/service/ElasticClientHandlerAws.java
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/service/ElasticClientHandlerAws.java
@@ -14,15 +14,12 @@
package org.opengroup.osdu.indexer.aws.service;
-import com.amazonaws.auth.AWS4Signer;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.http.AWSRequestSigningApacheInterceptor;
+import org.apache.http.Header;
import org.apache.http.HttpHost;
-import org.apache.http.HttpRequestInterceptor;
+import org.apache.http.message.BasicHeader;
import org.elasticsearch.client.RestClient;
-import org.elasticsearch.client.RestHighLevelClient;
-import org.opengroup.osdu.core.aws.iam.IAMConfig;
import org.opengroup.osdu.indexer.util.ElasticClientHandler;
+import org.elasticsearch.client.RestClientBuilder;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
@@ -34,31 +31,38 @@ import javax.inject.Inject;
@Component
public class ElasticClientHandlerAws extends ElasticClientHandler {
- @Value("${aws.es.serviceName}")
- private String serviceName;
-
- @Value("${aws.region}")
- private String region;
-
- @Value("${aws.es.host}")
- String host;
+ private static final int REST_CLIENT_CONNECT_TIMEOUT = 60000;
+ private static final int REST_CLIENT_SOCKET_TIMEOUT = 60000;
+ private static final int REST_CLIENT_RETRY_TIMEOUT = 60000;
public ElasticClientHandlerAws() {
}
@Override
- public RestHighLevelClient createRestClient() {
+ public RestClientBuilder createClientBuilder(String host, String basicAuthenticationHeaderVal, int port, String protocolScheme, String tls) {
- return esClient();
- }
+ RestClientBuilder builder = RestClient.builder(new HttpHost(host, port, protocolScheme));
+ builder.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder
+ .setConnectTimeout(REST_CLIENT_CONNECT_TIMEOUT)
+ .setSocketTimeout(REST_CLIENT_SOCKET_TIMEOUT));
+ builder.setMaxRetryTimeoutMillis(REST_CLIENT_RETRY_TIMEOUT);
- private RestHighLevelClient esClient() {
- AWSCredentialsProvider credentials = new IAMConfig().amazonAWSCredentials();
- AWS4Signer signer = new AWS4Signer();
- signer.setServiceName(serviceName);
- signer.setRegionName(region);
- HttpRequestInterceptor interceptor = new AWSRequestSigningApacheInterceptor(serviceName, signer, credentials);
- return new RestHighLevelClient(RestClient.builder(HttpHost.create(host)).setHttpClientConfigCallback(configCallBack -> configCallBack.addInterceptorLast(interceptor)));
+ if(isLocalHost(host)) {
+ builder.setHttpClientConfigCallback(httpAsyncClientBuilder -> httpAsyncClientBuilder.setSSLHostnameVerifier((s, sslSession) -> true));
+ }
+ Header[] defaultHeaders = new Header[]{
+ new BasicHeader("client.transport.nodes_sampler_interval", "30s"),
+ new BasicHeader("client.transport.ping_timeout", "30s"),
+ new BasicHeader("client.transport.sniff", "false"),
+ new BasicHeader("request.headers.X-Found-Cluster", host),
+ new BasicHeader("cluster.name", host),
+ new BasicHeader("xpack.security.transport.ssl.enabled", tls)
+ };
+ builder.setDefaultHeaders(defaultHeaders);
+ return builder;
+ }
+ private boolean isLocalHost(String uri) {
+ return (uri.contains("localhost") || uri.contains("127.0.0.1"));
}
}
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java
index 76607c97a21bd574e3f4f59d0b1f477c1543e799..bffecf18d7765fe7c52acaddc7dab246c64e13ec 100644
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java
@@ -14,33 +14,46 @@
package org.opengroup.osdu.indexer.aws.util;
+import com.amazonaws.services.sns.AmazonSNS;
+import com.amazonaws.services.sns.model.MessageAttributeValue;
+import com.amazonaws.services.sns.model.PublishRequest;
import com.amazonaws.services.sqs.AmazonSQS;
import com.amazonaws.services.sqs.model.SendMessageRequest;
+import com.google.gson.Gson;
+import org.opengroup.osdu.core.aws.sns.AmazonSNSConfig;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.aws.sqs.AmazonSQSConfig;
+import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
import javax.inject.Inject;
+import java.util.HashMap;
+import java.util.Map;
@Primary
@Component
public class IndexerQueueTaskBuilderAws extends IndexerQueueTaskBuilder {
- private AmazonSQS sqsClient;
+ private AmazonSNS snsClient;
@Value("${aws.region}")
private String region;
- @Value("${aws.sqs.queue}")
- private String queueName;
+ @Value("${aws.sns.storage.arn}")
+ private String amazonSNSTopic;
+
+ private String retryString = "retry";
+
+ private Gson gson;
@Inject
public void init() {
- AmazonSQSConfig sqsConfig = new AmazonSQSConfig(region);
- sqsClient = sqsConfig.AmazonSQS();
+ AmazonSNSConfig config = new AmazonSNSConfig(region);
+ snsClient = config.AmazonSNS();
+ gson =new Gson();
}
@Override
@@ -54,8 +67,40 @@ public class IndexerQueueTaskBuilderAws extends IndexerQueueTaskBuilder {
}
private void createTask(String payload, DpsHeaders headers) {
- String queueUrl = sqsClient.getQueueUrl(queueName).getQueueUrl();
- SendMessageRequest messageRequest = new SendMessageRequest().withQueueUrl(queueUrl).withMessageBody(payload);
- sqsClient.sendMessage(messageRequest);
+
+ Map<String, MessageAttributeValue> messageAttributes = new HashMap<>();
+ messageAttributes.put(DpsHeaders.ACCOUNT_ID, new MessageAttributeValue()
+ .withDataType("String")
+ .withStringValue(headers.getPartitionIdWithFallbackToAccountId()));
+ messageAttributes.put(DpsHeaders.DATA_PARTITION_ID, new MessageAttributeValue()
+ .withDataType("String")
+ .withStringValue(headers.getPartitionIdWithFallbackToAccountId()));
+ headers.addCorrelationIdIfMissing();
+ messageAttributes.put(DpsHeaders.CORRELATION_ID, new MessageAttributeValue()
+ .withDataType("String")
+ .withStringValue(headers.getCorrelationId()));
+ messageAttributes.put(DpsHeaders.USER_EMAIL, new MessageAttributeValue()
+ .withDataType("String")
+ .withStringValue(headers.getUserEmail()));
+ messageAttributes.put(DpsHeaders.AUTHORIZATION, new MessageAttributeValue()
+ .withDataType("String")
+ .withStringValue(headers.getAuthorization()));
+
+ RecordChangedMessages message = gson.fromJson(payload, RecordChangedMessages.class);
+ int retryCount;
+ if(message.getAttributes().containsKey(retryString)){
+ retryCount = Integer.parseInt(message.getAttributes().get(retryString));
+ retryCount++;
+ } else {
+ retryCount = 1;
+ }
+ messageAttributes.put(retryString, new MessageAttributeValue()
+ .withDataType("String")
+ .withStringValue(String.valueOf(retryCount)));
+
+ PublishRequest publishRequest = new PublishRequest(amazonSNSTopic, message.getData())
+ .withMessageAttributes(messageAttributes);
+
+ snsClient.publish(publishRequest);
}
}
diff --git a/provider/indexer-aws/src/main/resources/application.properties b/provider/indexer-aws/src/main/resources/application.properties
index a295c1956dba6ff653c0b84dce89f3697480431b..3ddc087e9e18032a061f0a2d5a8384df645d9a69 100644
--- a/provider/indexer-aws/src/main/resources/application.properties
+++ b/provider/indexer-aws/src/main/resources/application.properties
@@ -27,33 +27,24 @@ aws.s3.enable-https=true
## AWS SNS configuration
aws.sns.region=${AWS_REGION}
aws.sns.arn=arn:aws:sns:${AWS_REGION}:${AWS_ACCOUNT_ID}:${ENVIRONMENT}-${SNS_TOPIC_NAME}
-aws.sns.topic-name=${ENVIRONMENT}-${SNS_TOPIC_NAME}
+aws.sns.storage.arn=arn:aws:sns:${AWS_REGION}:${AWS_ACCOUNT_ID}:${SNS_STORAGE_TOPIC_NAME}
## AWS SQS Configuration
aws.sqs.queue=${ENVIRONMENT}-osdu-indexer-queue
-#Spring Configuration
-spring.security.user.name=opendes@byoc.local
-spring.security.user.password=123
-spring.security.user.roles=service.storage.admin
-
-
# AWS ES configuration
-#aws.es.host=https://search-${ENVIRONMENT}-osdu-indexer-i5bpf2gv4iv6ha2xi7rook2rga.${AWS_REGION}.es.amazonaws.com
-aws.es.host=https://search-dev-ado-test-y5m2xdyqha2npp36kh42vl3n7m.us-east-1.es.amazonaws.com
-aws.es.port=-1
+aws.es.host=${ELASTIC_HOST}
+aws.es.port=${ELASTIC_PORT}
aws.es.userNameAndPassword=notused
aws.region=${AWS_REGION}
aws.es.serviceName=es
GAE_SERVICE=indexer
-#STORAGE_SCHEMA_HOST=http://ECSALB-os-storage-1575155422.${AWS_REGION}.elb.amazonaws.com/api/storage/v2/schemas
-STORAGE_SCHEMA_HOST=http://localhost:8080/api/storage/v2/schemas
-#STORAGE_QUERY_RECORD_HOST=http://ECSALB-os-storage-1575155422.${AWS_REGION}.elb.amazonaws.com/api/storage/v2/query/records
-STORAGE_QUERY_RECORD_HOST=http://localhost:8080/api/storage/v2/query/records
-#STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=http://ECSALB-os-storage-1575155422.${AWS_REGION}.elb.amazonaws.com/api/storage/v2/query/records:batch
-STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=http://localhost:8080/api/storage/v2/query/records:batch
+# TODO This needs to be changed so it snot hard
+STORAGE_SCHEMA_HOST=https://${STORAGE_HOST}/api/storage/v2/schemas
+STORAGE_QUERY_RECORD_HOST=https://${STORAGE_HOST}/api/storage/v2/query/records
+STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=https://${STORAGE_HOST}/api/storage/v2/query/records:batch
STORAGE_RECORDS_BATCH_SIZE=20
INDEXER_QUEUE_HOST=http://sqs.${AWS_REGION}.amazonaws.com/${AWS_ACCOUNT_ID}/${ENVIRONMENT}-osdu-indexer-queue
@@ -72,4 +63,4 @@ aws.elasticache.cluster.schema.expiration=60
MAX_CACHE_VALUE_SIZE=1000
## AWS Lambda configuration
-aws.lambda.get-groups-function-name=${ENVIRONMENT}-os-entitlements-GroupsFunction-1DTM5841SUIFO
\ No newline at end of file
+aws.lambda.get-groups-function-name=${ENVIRONMENT}-os-entitlements-GroupsFunction
\ No newline at end of file
diff --git a/provider/indexer-ibm/azure-build.yml b/provider/indexer-ibm/azure-build.yml
new file mode 100644
index 0000000000000000000000000000000000000000..705d5f2168a7f1c6a077d21d9c69066ed673f303
--- /dev/null
+++ b/provider/indexer-ibm/azure-build.yml
@@ -0,0 +1,55 @@
+# Maven
+# Build your Java project and run tests with Apache Maven.
+# Add steps that analyze code, save build artifacts, deploy, and more:
+# https://docs.microsoft.com/azure/devops/pipelines/languages/java
+
+trigger:
+ branches:
+ include:
+ - master
+ paths:
+ exclude:
+ - README.md
+ - .gitignore
+
+pool:
+ name: Hosted Ubuntu 1604
+ demands: maven
+
+steps:
+- task: Maven@3
+ displayName: 'build, test, code coverage'
+ inputs:
+ mavenPomFile: 'pom.xml'
+ options: '--settings ./indexer-core/maven/settings.xml -DVSTS_FEED_TOKEN=$(VSTS_FEED_TOKEN)'
+ testResultsFiles: '**/*/TEST-*.xml'
+ codeCoverageToolOption: JaCoCo
+ goals: 'install'
+
+- task: Maven@3
+ displayName: 'build, test, code coverage'
+ inputs:
+ mavenPomFile: 'pom.xml'
+ options: '--settings ./provider/indexer-azure/maven/settings.xml -DVSTS_FEED_TOKEN=$(VSTS_FEED_TOKEN) -P indexer-azure'
+ testResultsFiles: '**/*/TEST-*.xml'
+ codeCoverageToolOption: JaCoCo
+ goals: 'package'
+
+- task: CopyFiles@2
+ displayName: 'Copy Azure artifacts for maven deploy to: $(build.artifactstagingdirectory)'
+ inputs:
+ SourceFolder:
+ Contents: |
+ pom.xml
+ provider/indexer-azure/maven/settings.xml
+ provider/indexer-azure/pom.xml
+ provider/indexer-azure/target/*-spring-boot.jar
+ TargetFolder: '$(build.artifactstagingdirectory)'
+
+- task: PublishBuildArtifacts@1
+ displayName: 'Publish Artifact: drop'
+ inputs:
+ PathtoPublish: '$(build.artifactstagingdirectory)'
+ ArtifactName: 'drop'
+ publishLocation: 'Container'
+ condition: succeededOrFailed()
diff --git a/provider/indexer-ibm/pom.xml b/provider/indexer-ibm/pom.xml
new file mode 100644
index 0000000000000000000000000000000000000000..09f44294727c8e808e2a0feb2297e2013f051ca4
--- /dev/null
+++ b/provider/indexer-ibm/pom.xml
@@ -0,0 +1,193 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Copyright 2019 IBM Corp. All Rights Reserved.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.opengroup.osdu.indexer</groupId>
+ <artifactId>indexer-service</artifactId>
+ <version>1.0.4-SNAPSHOT</version>
+ <relativePath>../..</relativePath>
+ </parent>
+
+ <artifactId>indexer-ibm</artifactId>
+ <name>indexer-ibm</name>
+ <description>Indexer Service IBM</description>
+ <packaging>jar</packaging>
+
+ <dependencies>
+
+ <!-- OSDU core service dependencies -->
+ <dependency>
+ <groupId>org.opengroup.osdu.indexer</groupId>
+ <artifactId>indexer-core</artifactId>
+ <version>1.0.4-SNAPSHOT</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.opengroup.osdu</groupId>
+ <artifactId>os-core-lib-ibm</artifactId>
+ <version>0.0.13-SNAPSHOT</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-security</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.security.oauth</groupId>
+ <artifactId>spring-security-oauth2</artifactId>
+ <version>2.3.6.RELEASE</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-jwt</artifactId>
+ <version>1.0.10.RELEASE</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-oauth2-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.data</groupId>
+ <artifactId>spring-data-commons</artifactId>
+ <version>2.2.0.RELEASE</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch</artifactId>
+ <version>6.6.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-client</artifactId>
+ <version>6.6.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-high-level-client</artifactId>
+ <version>6.6.2</version>
+ </dependency>
+
+ <!-- Test Dependencies -->
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.auth0</groupId>
+ <artifactId>java-jwt</artifactId>
+ <version>3.8.1</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.12</version>
+ <scope>test</scope>
+ </dependency>
+ <!-- https://mvnrepository.com/artifact/org.powermock/powermock-api-mockito2 -->
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-mockito2</artifactId>
+ <version>2.0.2</version>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/org.powermock/powermock-module-junit4 -->
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <version>2.0.2</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <version>3.0.0</version>
+ <scope>test</scope>
+ </dependency>
+ <!-- <dependency>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>cobertura-maven-plugin</artifactId>
+ <version>2.7</version>
+ <scope>test</scope>
+ </dependency> -->
+
+ </dependencies>
+
+ <repositories>
+ <repository>
+ <id>${gitlab-server}</id>
+ <url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</url>
+ </repository>
+ </repositories>
+
+ <distributionManagement>
+ <repository>
+ <id>${gitlab-server}</id>
+ <url>https://community.opengroup.org/api/v4/projects/25/packages/maven</url>
+ </repository>
+ <snapshotRepository>
+ <id>${gitlab-server}</id>
+ <url>https://community.opengroup.org/api/v4/projects/25/packages/maven</url>
+ </snapshotRepository>
+ </distributionManagement>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>repackage</goal>
+ </goals>
+ <configuration>
+ <classifier>spring-boot</classifier>
+ <mainClass>
+ org.opengroup.osdu.indexer.ibm.IndexerIBMApplication
+ </mainClass>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.4.2</version>
+ <configuration>
+ <useSystemClassLoader>false</useSystemClassLoader>
+ <threadCount>1</threadCount>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+
+</project>
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/IndexerIBMApplication.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/IndexerIBMApplication.java
new file mode 100644
index 0000000000000000000000000000000000000000..b7ec63076f86c413b8636f353863fccd502877a9
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/IndexerIBMApplication.java
@@ -0,0 +1,43 @@
+// Copyright 2020 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm;
+
+import org.opengroup.osdu.indexer.IndexerApplication;
+import org.opengroup.osdu.indexer.ServerletInitializer;
+import org.opengroup.osdu.indexer.service.ElasticSettingServiceImpl;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.FilterType;
+
+
+@SpringBootApplication
+@ComponentScan(
+ basePackages = {"org.opengroup.osdu"},
+ excludeFilters = {
+ @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value=IndexerApplication.class),
+ @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value=ServerletInitializer.class),
+ @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value=ElasticSettingServiceImpl.class),
+ }
+ )
+public class IndexerIBMApplication {
+
+ public static void main(String[] args) throws Exception {
+
+ SpringApplication.run(IndexerIBMApplication.class, args);
+
+ }
+
+}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/AttributesCache.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/AttributesCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..e1d8f1162004471449950e1d2ca8c07dd403052d
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/AttributesCache.java
@@ -0,0 +1,55 @@
+// Copyright 2020 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.cache;
+
+import org.opengroup.osdu.core.common.cache.VmCache;
+import org.opengroup.osdu.core.common.provider.interfaces.IAttributesCache;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import java.util.Set;
+
+@Component
+public class AttributesCache implements IAttributesCache<String,Set> {
+
+ private VmCache<String, Set> cache;
+
+ public AttributesCache(@Value("${INDEX_CACHE_EXPIRATION}") final String INDEX_CACHE_EXPIRATION,
+ @Value("${MAX_CACHE_VALUE_SIZE}") final String MAX_CACHE_VALUE_SIZE) {
+ cache = new VmCache<>(Integer.parseInt(INDEX_CACHE_EXPIRATION) * 60,
+ Integer.parseInt(MAX_CACHE_VALUE_SIZE));
+ }
+
+ @Override
+ public void put(String key, Set value) {
+ this.cache.put(key, value);
+ }
+
+ @Override
+ public Set get(String key) {
+ return this.cache.get(key);
+ }
+
+ @Override
+ public void delete(String key) {
+ this.cache.delete(key);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+
+}
\ No newline at end of file
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/ElasticCredentialsCache.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/ElasticCredentialsCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..339051e58dacf7ee0a0c36b16fde5b2484ad0cab
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/ElasticCredentialsCache.java
@@ -0,0 +1,60 @@
+// Copyright 2020 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.cache;
+
+import org.opengroup.osdu.core.common.cache.VmCache;
+import org.opengroup.osdu.core.common.model.search.ClusterSettings;
+import org.opengroup.osdu.core.common.provider.interfaces.IElasticCredentialsCache;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+@Component
+public class ElasticCredentialsCache implements IElasticCredentialsCache<String, ClusterSettings>, AutoCloseable {
+
+ private VmCache<String, ClusterSettings> cache;
+
+ public ElasticCredentialsCache(@Value("${ELASTIC_CACHE_EXPIRATION}") final String ELASTIC_CACHE_EXPIRATION,
+ @Value("${MAX_CACHE_VALUE_SIZE}") final String MAX_CACHE_VALUE_SIZE) {
+
+ cache = new VmCache<>(Integer.parseInt(ELASTIC_CACHE_EXPIRATION) * 60,
+ Integer.parseInt(MAX_CACHE_VALUE_SIZE));
+
+ }
+
+ @Override
+ public void close() throws Exception {
+ //this.cache.close();
+ }
+
+ @Override
+ public void put(String s, ClusterSettings o) {
+ this.cache.put(s,o);
+ }
+
+ @Override
+ public ClusterSettings get(String s) {
+ return this.cache.get(s);
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(s);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/IndexCache.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/IndexCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..019f5ae06cd4c3c560d23be11629b293622e8296
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/IndexCache.java
@@ -0,0 +1,51 @@
+// Copyright 2020 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.cache;
+
+import org.opengroup.osdu.core.common.cache.VmCache;
+import org.opengroup.osdu.core.common.provider.interfaces.IIndexCache;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+@Component
+public class IndexCache implements IIndexCache<String, Boolean> {
+ private VmCache<String, Boolean> cache;
+
+ public IndexCache(@Value("${INDEX_CACHE_EXPIRATION}") final String INDEX_CACHE_EXPIRATION,
+ @Value("${MAX_CACHE_VALUE_SIZE}") final String MAX_CACHE_VALUE_SIZE) {
+ cache = new VmCache<>(Integer.parseInt(INDEX_CACHE_EXPIRATION) * 60,
+ Integer.parseInt(MAX_CACHE_VALUE_SIZE));
+ }
+
+ @Override
+ public void put(String s, Boolean o) {
+ this.cache.put(s, o);
+ }
+
+ @Override
+ public Boolean get(String s) {
+ return this.cache.get(s);
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(s);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/JwtCache.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/JwtCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..3c6a37d24cd8321e0c9d3c17031c9943ec7f7178
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/JwtCache.java
@@ -0,0 +1,53 @@
+// Copyright 2020 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.cache;
+
+import org.opengroup.osdu.core.common.cache.VmCache;
+import org.opengroup.osdu.core.common.model.search.IdToken;
+import org.opengroup.osdu.core.common.provider.interfaces.IJwtCache;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+@Component
+public class JwtCache implements IJwtCache<String, IdToken> {
+ private VmCache<String, IdToken> cache;
+
+ // Azure service account id_token can be requested only for 1 hr
+ private final static int EXPIRED_AFTER = 59;
+
+ public JwtCache(@Value("${MAX_CACHE_VALUE_SIZE}") final String MAX_CACHE_VALUE_SIZE) {
+ cache = new VmCache<>(EXPIRED_AFTER * 60, Integer.parseInt(MAX_CACHE_VALUE_SIZE));
+ }
+
+ @Override
+ public void put(String s, IdToken o) {
+ this.cache.put(s, o);
+ }
+
+ @Override
+ public IdToken get(String s) {
+ return this.cache.get(s);
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(s);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/KindsCache.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/KindsCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..9ab87c833f5bd6f3a431e331aa262f394d9fa057
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/KindsCache.java
@@ -0,0 +1,53 @@
+// Copyright 2020 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.cache;
+
+import org.opengroup.osdu.core.common.cache.VmCache;
+import org.opengroup.osdu.core.common.provider.interfaces.IKindsCache;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import java.util.Set;
+
+@Component
+public class KindsCache implements IKindsCache<String, Set> {
+ private VmCache<String, Set> cache;
+
+ public KindsCache(@Value("${KINDS_CACHE_EXPIRATION}") final String KINDS_CACHE_EXPIRATION,
+ @Value("${MAX_CACHE_VALUE_SIZE}") final String MAX_CACHE_VALUE_SIZE) {
+ cache = new VmCache<>(Integer.parseInt(KINDS_CACHE_EXPIRATION) * 60,
+ Integer.parseInt(MAX_CACHE_VALUE_SIZE));
+ }
+
+ @Override
+ public void put(String s, Set o) {
+ this.cache.put(s, o);
+ }
+
+ @Override
+ public Set get(String s) {
+ return this.cache.get(s);
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(s);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/SchemaCache.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/SchemaCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..83bef463a83f7cfe8c38014e8a927a6f3ca97dca
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/cache/SchemaCache.java
@@ -0,0 +1,51 @@
+// Copyright 2020 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.cache;
+
+import org.opengroup.osdu.core.common.cache.VmCache;
+import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+@Component
+public class SchemaCache implements ISchemaCache<String, String> {
+ private VmCache<String, String> cache;
+
+ public SchemaCache(@Value("${SCHEMA_CACHE_EXPIRATION}") final String SCHEMA_CACHE_EXPIRATION,
+ @Value("${MAX_CACHE_VALUE_SIZE}") final String MAX_CACHE_VALUE_SIZE) {
+ cache = new VmCache<>(Integer.parseInt(SCHEMA_CACHE_EXPIRATION) * 60,
+ Integer.parseInt(MAX_CACHE_VALUE_SIZE));
+ }
+
+ @Override
+ public void put(String s, String o) {
+ this.cache.put(s, o);
+ }
+
+ @Override
+ public String get(String s) {
+ return this.cache.get(s);
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(s);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/di/TenantInfoService.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/di/TenantInfoService.java
new file mode 100644
index 0000000000000000000000000000000000000000..c7d8128726195b31c9905814f73845e26f33a5f7
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/di/TenantInfoService.java
@@ -0,0 +1,39 @@
+// Copyright 2020 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.di;
+
+import javax.inject.Inject;
+
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
+import org.opengroup.osdu.core.common.multitenancy.ITenantInfoService;
+import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+@RequestScope
+@Component
+public class TenantInfoService implements ITenantInfoService {
+ @Inject
+ private ITenantFactory tenantFactory;
+
+ @Inject
+ private DpsHeaders headers;
+
+ @Override
+ public TenantInfo getTenantInfo() {
+ return tenantFactory.getTenantInfo(headers.getPartitionId());
+ }
+}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/model/ElasticSettingSchema.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/model/ElasticSettingSchema.java
new file mode 100644
index 0000000000000000000000000000000000000000..e504cbddb3a3870becf68235b9eb36ebdd6b1b8a
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/model/ElasticSettingSchema.java
@@ -0,0 +1,40 @@
+// Copyright 2020 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.model;
+
+import javax.validation.constraints.NotEmpty;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class ElasticSettingSchema {
+
+ @NotEmpty
+ private String host;
+
+ @NotEmpty
+ private String port;
+
+ @NotEmpty
+ private String usernameAndPassword;
+
+ @NotEmpty
+ private boolean isHttps;
+
+}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/persistence/ElasticRepositoryIBM.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/persistence/ElasticRepositoryIBM.java
new file mode 100644
index 0000000000000000000000000000000000000000..1a7b0835612c4c5b8ea62379015b75f5c42eff78
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/persistence/ElasticRepositoryIBM.java
@@ -0,0 +1,81 @@
+// Copyright 2020 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.persistence;
+
+import javax.inject.Inject;
+
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.search.ClusterSettings;
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
+import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository;
+import org.opengroup.osdu.core.common.search.Preconditions;
+import org.opengroup.osdu.indexer.ibm.model.ElasticSettingSchema;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+@Component
+public class ElasticRepositoryIBM implements IElasticRepository {
+
+ @Value("${ELASTIC_DATASTORE_KIND}")
+ private String ELASTIC_DATASTORE_KIND;
+
+ @Value("${ELASTIC_DATASTORE_ID}")
+ private String ELASTIC_DATASTORE_ID;
+
+ @Inject
+ private ISchemaRepository schemaRepository;
+
+ @Value("${ELASTIC_HOST}")
+ private String ELASTIC_HOST;
+ @Value("${ELASTIC_PORT:443}")
+ private String ELASTIC_PORT;
+ @Value("${ELASTIC_USER_PASSWORD}")
+ private String ELASTIC_USER_PASSWORD;
+
+ @Inject
+ private JaxRsDpsLog log;
+
+ @Override
+ public ClusterSettings getElasticClusterSettings(TenantInfo tenantInfo) {
+
+ if(tenantInfo == null)
+ throw new AppException(HttpStatus.SC_NOT_FOUND, "TenantInfo is null", "");
+
+ String settingId = tenantInfo.getName().concat("-").concat(ELASTIC_DATASTORE_ID);
+ ElasticSettingSchema schema = this.schemaRepository.get(settingId);
+
+ if (schema == null) {
+ // if creds not in the db, use default from env
+ log.warning(settingId + " credentials not found at database.");
+ return new ClusterSettings(ELASTIC_HOST, Integer.parseInt(ELASTIC_PORT), ELASTIC_USER_PASSWORD, true, false);
+ //throw new AppException(HttpStatus.SC_NOT_FOUND, "Elastic setting not found", "The requested cluster setting was not found in CosmosDB.", String.format("Elastic setting with key: '%s' does not exist in CosmostDB.", ELASTIC_DATASTORE_KIND));
+ }
+
+ String host = schema.getHost();
+ String portString = schema.getPort();
+ String usernameAndPassword = schema.getUsernameAndPassword();
+
+ Preconditions.checkNotNullOrEmpty(host, "host cannot be null");
+ Preconditions.checkNotNullOrEmpty(portString, "port cannot be null");
+ Preconditions.checkNotNullOrEmpty(usernameAndPassword, "configuration cannot be null");
+
+ int port = Integer.parseInt(portString);
+
+ return new ClusterSettings(host, port, usernameAndPassword, schema.isHttps(), schema.isHttps());
+
+ }
+}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/persistence/ElasticSettingSchemaRepositoryImpl.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/persistence/ElasticSettingSchemaRepositoryImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..cde9a4b88acca4c11180810143b61735e472bafa
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/persistence/ElasticSettingSchemaRepositoryImpl.java
@@ -0,0 +1,88 @@
+// Copyright 2020 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.persistence;
+
+import javax.annotation.PostConstruct;
+import javax.inject.Inject;
+
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.ibm.auth.ServiceCredentials;
+import org.opengroup.osdu.core.ibm.cloudant.IBMCloudantClientFactory;
+import org.opengroup.osdu.indexer.ibm.model.ElasticSettingSchema;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Repository;
+
+import com.cloudant.client.api.Database;
+
+
+@Repository
+public class ElasticSettingSchemaRepositoryImpl implements ISchemaRepository {
+
+ @Value("${ibm.db.url}")
+ private String dbUrl;
+ @Value("${ibm.db.apikey:#{null}}")
+ private String apiKey;
+ @Value("${ibm.db.user:#{null}}")
+ private String dbUser;
+ @Value("${ibm.db.password:#{null}}")
+ private String dbPassword;
+
+ @Value("${ibm.env.prefix:local-dev}")
+ private String dbNamePrefix;
+
+ private IBMCloudantClientFactory cloudantFactory;
+ private Database db;
+
+ @Inject
+ private JaxRsDpsLog logger;
+
+ @PostConstruct
+ public void init(){
+ try {
+ if (apiKey != null) {
+ cloudantFactory = new IBMCloudantClientFactory(new ServiceCredentials(dbUrl, apiKey));
+ } else {
+ cloudantFactory = new IBMCloudantClientFactory(new ServiceCredentials(dbUrl, dbUser, dbPassword));
+ } db = cloudantFactory.getDatabase(dbNamePrefix, "SearchSettings");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ public void add(ElasticSettingSchema schema, String id) {
+ ElasticSettingsDoc sd = new ElasticSettingsDoc();
+ sd.setId(id);
+ sd.setSettingSchema(schema);
+ db.save(sd);
+ }
+
+ @Override
+ public ElasticSettingSchema get(String id) {
+ if (db.contains(id)) {
+ ElasticSettingsDoc sd = db.find(ElasticSettingsDoc.class, id);
+ ElasticSettingSchema newSchema = new ElasticSettingSchema();
+ newSchema.setPort(sd.getSettingSchema().getPort());
+ newSchema.setHost(sd.getSettingSchema().getHost());
+ newSchema.setUsernameAndPassword(sd.getSettingSchema().getUsernameAndPassword());
+ newSchema.setHttps(sd.getSettingSchema().isHttps());
+ return newSchema;
+ } else {
+ logger.error(ElasticSettingsDoc.class + " with id " + id + " was not found in the database.");
+ return null;
+ }
+ }
+
+}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/persistence/ElasticSettingsDoc.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/persistence/ElasticSettingsDoc.java
new file mode 100644
index 0000000000000000000000000000000000000000..fe417507f5f8a76e05eb8a9c3e49ebf867a254a2
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/persistence/ElasticSettingsDoc.java
@@ -0,0 +1,40 @@
+// Copyright 2020 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.persistence;
+
+import org.opengroup.osdu.indexer.ibm.model.ElasticSettingSchema;
+import org.springframework.data.annotation.Id;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class ElasticSettingsDoc {
+
+ public static final String DB_NAME = "SearchSettings"; //collection name
+
+ @Id
+ private String _id;
+ private String _rev;
+ private ElasticSettingSchema settingSchema;
+
+ public void setId(String id) {
+ this._id = id;
+ }
+
+}
\ No newline at end of file
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/persistence/ISchemaRepository.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/persistence/ISchemaRepository.java
new file mode 100644
index 0000000000000000000000000000000000000000..114aa4fa554d1f76f91edd7c4f22f61d2996058f
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/persistence/ISchemaRepository.java
@@ -0,0 +1,29 @@
+// Copyright 2020 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.persistence;
+
+import org.opengroup.osdu.indexer.ibm.model.ElasticSettingSchema;
+
+public interface ISchemaRepository {
+ String SCHEMA_KIND = "IndexerSchema";
+
+ String SCHEMA = "schema";
+ String KIND = "KIND";
+
+ void add(ElasticSettingSchema schema, String id);
+
+ ElasticSettingSchema get(String id);
+}
+
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/publish/PublisherImpl.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/publish/PublisherImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..809568c8c082923ee1e3e03bf3756e772a307c91
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/publish/PublisherImpl.java
@@ -0,0 +1,102 @@
+// Copyright 2020 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.publish;
+
+
+import java.lang.reflect.Type;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.inject.Inject;
+
+import org.elasticsearch.common.Strings;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.indexer.JobStatus;
+import org.opengroup.osdu.core.common.model.indexer.RecordStatus;
+import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
+import org.opengroup.osdu.core.ibm.messagebus.IMessageFactory;
+import org.opengroup.osdu.indexer.provider.interfaces.IPublisher;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+import com.google.common.reflect.TypeToken;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonElement;
+
+@Component
+@RequestScope
+public class PublisherImpl implements IPublisher {
+
+ @Inject
+ IMessageFactory mq;
+
+ @Inject
+ private JaxRsDpsLog logger;
+
+
+ @Override
+ public void publishStatusChangedTagsToTopic(DpsHeaders headers, JobStatus indexerBatchStatus) throws Exception {
+
+ String tenant = headers.getPartitionId();
+ if (Strings.isNullOrEmpty(tenant))
+ tenant = headers.getAccountId();
+
+ Map<String, String> message = new HashMap<>();
+ message.put(tenant, headers.getPartitionIdWithFallbackToAccountId());
+ headers.addCorrelationIdIfMissing();
+ message.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
+
+ RecordChangedMessages recordChangedMessages = getRecordChangedMessage(headers, indexerBatchStatus);
+ message.put("data", recordChangedMessages.toString());
+
+ try {
+ logger.info("Indexer publishes message " + headers.getCorrelationId());
+ mq.sendMessage(IMessageFactory.INDEXER_QUEUE_NAME, message.toString());
+ }
+ catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ }
+ }
+
+ private RecordChangedMessages getRecordChangedMessage(DpsHeaders headers, JobStatus indexerBatchStatus) {
+
+ Gson gson = new GsonBuilder().create();
+ Map<String, String> attributesMap = new HashMap<>();
+ Type listType = new TypeToken<List<RecordStatus>>() {}.getType();
+
+ JsonElement statusChangedTagsJson = gson.toJsonTree(indexerBatchStatus.getStatusesList(), listType);
+ String statusChangedTagsData = (statusChangedTagsJson.toString());
+
+ String tenant = headers.getPartitionId();
+ // This code it to provide backward compatibility to slb-account-id
+ if (!Strings.isNullOrEmpty(tenant)) {
+ attributesMap.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
+ } else {
+ attributesMap.put(DpsHeaders.ACCOUNT_ID, headers.getPartitionIdWithFallbackToAccountId());
+ }
+ headers.addCorrelationIdIfMissing();
+ attributesMap.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
+
+ RecordChangedMessages recordChangedMessages = new RecordChangedMessages();
+ // statusChangedTagsData is not ByteString but String
+ recordChangedMessages.setData(statusChangedTagsData);
+ recordChangedMessages.setAttributes(attributesMap);
+
+ return recordChangedMessages;
+ }
+}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/security/SecurityConfig.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/security/SecurityConfig.java
new file mode 100644
index 0000000000000000000000000000000000000000..2cf274beac0c1dd57bb2926fcd2b6fbe98fda4e4
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/security/SecurityConfig.java
@@ -0,0 +1,42 @@
+// Copyright 2020 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.security;
+
+import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity;
+import org.springframework.security.config.annotation.web.builders.HttpSecurity;
+import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
+import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter;
+
+@EnableWebSecurity
+@EnableGlobalMethodSecurity(prePostEnabled = true)
+public class SecurityConfig extends WebSecurityConfigurerAdapter {
+
+ @Override
+ protected void configure(HttpSecurity http) throws Exception {
+ http
+ .csrf().disable()
+ .authorizeRequests()
+ .antMatchers("/", "/index.html",
+ "/index-worker", "/_dps/task-handlers", "/_dps/task-handlers/**",
+ "/reindex",
+ "/v2/api-docs",
+ "/swagger-resources/**",
+ "/configuration/security",
+ "/swagger",
+ "/swagger-ui.html",
+ "/webjars/**").permitAll()
+ .anyRequest().anonymous();
+ }
+}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/security/WhoamiController.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/security/WhoamiController.java
new file mode 100644
index 0000000000000000000000000000000000000000..dd8b395e89b135ff31d2e81671463213521b31c4
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/security/WhoamiController.java
@@ -0,0 +1,36 @@
+// Copyright 2020 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.security;
+
+import org.springframework.security.core.Authentication;
+import org.springframework.security.core.context.SecurityContextHolder;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.ResponseBody;
+
+@Controller
+public class WhoamiController {
+ @RequestMapping(value = { "/", "/whoami" })
+ @ResponseBody
+ public String whoami() {
+ final Authentication auth = SecurityContextHolder.getContext().getAuthentication();
+
+ String userName = auth.getName();
+ String roles = String.valueOf(auth.getAuthorities());
+ String details = String.valueOf(auth.getPrincipal());
+
+ return "user: " + userName + "<BR>" + "roles: " + roles + "<BR>" + "details: " + details;
+ }
+}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/DpsHeadersIBMQueue.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/DpsHeadersIBMQueue.java
new file mode 100644
index 0000000000000000000000000000000000000000..3fa378301ed518deb8c470dd853f7a3308f6ae6b
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/DpsHeadersIBMQueue.java
@@ -0,0 +1,57 @@
+// Copyright 2020 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.util;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import javax.inject.Inject;
+import javax.servlet.http.HttpServletRequest;
+
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Primary;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+@Component
+@RequestScope
+@Primary
+public class DpsHeadersIBMQueue extends DpsHeaders {
+
+ @Value("${indexer.queue.key}")
+ private String queueKey;
+
+ //TODO this should be moved to Azure client-lib
+ public static final String INDEXER_QUEUE_KEY = "x-functions-key";
+
+ @Inject
+ public DpsHeadersIBMQueue(HttpServletRequest request) {
+
+ Map<String, String> headers = Collections
+ .list(request.getHeaderNames())
+ .stream()
+ .collect(Collectors.toMap(h -> h, request::getHeader));
+
+ headers.put(INDEXER_QUEUE_KEY,queueKey);
+
+ this.addFromMap(headers);
+
+ // Add Correlation ID if missing
+ this.addCorrelationIdIfMissing();
+
+ }
+}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/IHeadersInfo.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/IHeadersInfo.java
new file mode 100644
index 0000000000000000000000000000000000000000..722e4c9f59c551cc712f71764377e13ae3252a60
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/IHeadersInfo.java
@@ -0,0 +1,35 @@
+// Copyright 2020 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.util;
+
+import java.util.Map;
+
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+
+public interface IHeadersInfo {
+
+ DpsHeaders getHeaders();
+
+ String getUser();
+
+ String getPartitionId();
+
+ String getPrimaryPartitionId();
+
+ Map<String, String> getHeadersMap();
+
+ DpsHeaders getCoreServiceHeaders(Map<String, String> input);
+
+}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/KeyCloakProvider.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/KeyCloakProvider.java
new file mode 100644
index 0000000000000000000000000000000000000000..04d134da767d8a0020feb230a4e1f0da30c620ad
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/KeyCloakProvider.java
@@ -0,0 +1,110 @@
+// Copyright 2020 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.util;
+
+import java.io.BufferedReader;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.UnsupportedEncodingException;
+import java.net.URL;
+import java.net.URLEncoder;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.net.ssl.HttpsURLConnection;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import com.google.gson.Gson;
+import com.google.gson.JsonObject;
+
+@Component
+public class KeyCloakProvider {
+
+ @Value("${ibm.keycloak.endpoint_url}")
+ private String url;
+
+ @Value("${ibm.keycloak.realm}")
+ private String realm ;
+
+ @Value("${ibm.keycloak.grant_type:password}")
+ private String grantType;
+
+ @Value("${ibm.keycloak.scope:openid}")
+ private String scope;
+
+ @Value("${ibm.keycloak.client_id}")
+ private String clientId;
+
+ @Value("${ibm.keycloak.client_secret}")
+ private String clientSecret;
+
+ public String getToken(String user, String password) throws IOException {
+ String endpoint = String.format("https://%s/auth/realms/%s/protocol/openid-connect/token", url, realm);
+ URL url = new URL(endpoint);
+ HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
+ con.setRequestMethod("POST");
+ con.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
+
+ Map<String, String> parameters = new HashMap<>();
+ parameters.put("grant_type", grantType);
+ parameters.put("client_id", clientId);
+ parameters.put("client_secret", clientSecret);
+ parameters.put("username", user);
+ parameters.put("password", password);
+ parameters.put("scope", scope);
+
+ con.setDoOutput(true);
+ DataOutputStream out = new DataOutputStream(con.getOutputStream());
+ out.writeBytes(getParamsString(parameters));
+ out.flush();
+ out.close();
+
+ BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream()));
+ String inputLine;
+ StringBuffer content = new StringBuffer();
+ while ((inputLine = in.readLine()) != null) {
+ content.append(inputLine);
+ }
+ in.close();
+
+ con.disconnect();
+
+ Gson gson = new Gson();
+ JsonObject jobj = gson.fromJson(content.toString(), JsonObject.class);
+ String token = jobj.get("access_token").getAsString();
+ return token;
+ }
+
+ private static String getParamsString(Map<String, String> params)
+ throws UnsupportedEncodingException {
+ StringBuilder result = new StringBuilder();
+
+ for (Map.Entry<String, String> entry : params.entrySet()) {
+ result.append(URLEncoder.encode(entry.getKey(), "UTF-8"));
+ result.append("=");
+ result.append(URLEncoder.encode(entry.getValue(), "UTF-8"));
+ result.append("&");
+ }
+
+ String resultString = result.toString();
+ return resultString.length() > 0
+ ? resultString.substring(0, resultString.length() - 1)
+ : resultString;
+ }
+
+}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/RequestInfoImpl.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/RequestInfoImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..12fd218f826bc9ea1177e433d305cbc951571605
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/RequestInfoImpl.java
@@ -0,0 +1,123 @@
+// Copyright 2020 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.util;
+
+import static org.opengroup.osdu.core.common.model.http.DpsHeaders.AUTHORIZATION;
+
+import java.util.Map;
+
+import javax.inject.Inject;
+
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.search.DeploymentEnvironment;
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.opengroup.osdu.core.common.search.Preconditions;
+import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Primary;
+import org.springframework.stereotype.Component;
+
+import com.google.common.base.Strings;
+
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+
+@Primary
+
+@Component
+//@RequestScope
+public class RequestInfoImpl implements IRequestInfo {
+
+ @Inject
+ private DpsHeaders headersMap;
+
+ @Inject
+ private JaxRsDpsLog logger;
+
+ @Inject
+ private IServiceAccountJwtClient serviceAccountJwtClient;
+
+ @Inject
+ private TenantInfo tenantInfo;
+
+ @Value("${DEPLOYMENT_ENVIRONMENT}")
+ private String DEPLOYMENT_ENVIRONMENT;
+
+
+ @Override
+ public DpsHeaders getHeaders() {
+ if (headersMap == null) {
+ logger.warning("Headers Map DpsHeaders is null");
+ // throw to prevent null reference exception below
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Invalid Headers", "Headers Map DpsHeaders is null");
+ }
+ DpsHeaders headers = this.getCoreServiceHeaders(headersMap.getHeaders());
+ return headers;
+ }
+
+ @Override
+ public String getPartitionId() {
+ return getHeaders().getPartitionIdWithFallbackToAccountId();
+ }
+
+ @Override
+ public Map<String, String> getHeadersMap() {
+ return getHeaders().getHeaders();
+ }
+
+ @Override
+ public Map<String, String> getHeadersMapWithDwdAuthZ() {
+ return getHeadersWithDwdAuthZ().getHeaders();
+ }
+
+ @Override
+ public DpsHeaders getHeadersWithDwdAuthZ() {
+ this.headersMap.put(AUTHORIZATION, this.checkOrGetAuthorizationHeader());
+ return getHeaders();
+ }
+
+ @Override
+ public boolean isCronRequest() { return false;}
+
+ @Override
+ public boolean isTaskQueueRequest() {
+ //if (!this.dpsHeaders.getHeaders().containsKey(INDEXER_QUEUE_KEY)) return false;
+
+// String queueId = this.headersInfo.getHeadersMap().get(AppEngineHeaders.TASK_QUEUE_NAME);
+// return queueId.endsWith(Constants.INDEXER_QUEUE_IDENTIFIER);
+ return false;
+ }
+
+ public String checkOrGetAuthorizationHeader() {
+ if (DeploymentEnvironment.valueOf(DEPLOYMENT_ENVIRONMENT) == DeploymentEnvironment.LOCAL) {
+ String authHeader = this.headersMap.getAuthorization();
+ if (Strings.isNullOrEmpty(authHeader)) {
+ logger.error(HttpStatus.SC_UNAUTHORIZED + " : Invalid authorization header");
+ throw new AppException(HttpStatus.SC_UNAUTHORIZED, "Invalid authorization header", "Authorization token cannot be empty");
+ }
+ return "Bearer " + authHeader;
+ } else {
+ return "Bearer " + this.serviceAccountJwtClient.getIdToken(tenantInfo.getName());
+ }
+ }
+
+ private DpsHeaders getCoreServiceHeaders(Map<String, String> input) {
+ Preconditions.checkNotNull(input, "input headers cannot be null");
+ DpsHeaders output = DpsHeaders.createFromMap(input);
+ return output;
+ }
+}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/ServiceAccountJwtClientImpl.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/ServiceAccountJwtClientImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..adb2e3cd2fd6236b1e050c9c21d775045e137ef7
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/ServiceAccountJwtClientImpl.java
@@ -0,0 +1,83 @@
+// Copyright 2020 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.util;
+
+import javax.inject.Inject;
+
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.provider.interfaces.IJwtCache;
+import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
+import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+@Component
+@RequestScope
+public class ServiceAccountJwtClientImpl implements IServiceAccountJwtClient {
+
+ @Inject
+ private ITenantFactory tenantInfoServiceProvider;
+
+ @Inject
+ private DpsHeaders dpsHeaders;
+
+ @Inject
+ private IJwtCache cacheService;
+
+ @Inject
+ private JaxRsDpsLog logger;
+
+ @Inject
+ private KeyCloakProvider keyCloack;
+
+ @Value("${ibm.keycloak.useremail}")
+ private String userEmail;
+
+ @Value("${ibm.keycloak.username}")
+ private String userName;
+
+ @Value("${ibm.keycloak.password}")
+ private String userPassword;
+
+ @Override
+ public String getIdToken(String tenantName) {
+ /*this.log.info("Tenant name received for auth token is: " + tenantName);
+ TenantInfo tenant = this.tenantInfoServiceProvider.getTenantInfo(tenantName);
+ if (tenant == null) {
+ this.log.error("Invalid tenant name receiving from azure");
+ throw new AppException(HttpStatus.SC_BAD_REQUEST, "Invalid tenant Name", "Invalid tenant Name from azure");
+ }*/
+ String ACCESS_TOKEN = "";
+ try {
+
+ this.dpsHeaders.put(DpsHeaders.USER_EMAIL, userEmail);
+
+ ACCESS_TOKEN = keyCloack.getToken(userName, userPassword);
+
+ } catch (AppException e) {
+ throw e;
+ } catch (Exception e) {
+ logger.error("Error generating token", e);
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Persistence error", "Error generating token", e);
+ }
+
+ return ACCESS_TOKEN;
+ }
+
+}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceIBMImpl.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceIBMImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..da352d885fa21eae88ce99f1ae83c680c8dd2bdd
--- /dev/null
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceIBMImpl.java
@@ -0,0 +1,61 @@
+// Copyright 2020 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.service;
+
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.core.common.model.search.ClusterSettings;
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository;
+import org.opengroup.osdu.core.common.provider.interfaces.IElasticCredentialsCache;
+import org.opengroup.osdu.core.common.model.indexer.IElasticSettingService;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+import javax.inject.Inject;
+
+@Service
+public class ElasticSettingServiceIBMImpl implements IElasticSettingService {
+
+ @Inject
+ private TenantInfo tenantInfo;
+
+ @Inject
+ private IElasticRepository elasticRepository;
+ @Inject
+ private IElasticCredentialsCache elasticCredentialCache;
+ @Inject
+ private JaxRsDpsLog logger;
+
+ @Override
+ public ClusterSettings getElasticClusterInformation() {
+
+ String cacheKey = String.format("%s-%s", "ibm", tenantInfo.getName());
+ ClusterSettings clusterInfo = (ClusterSettings) this.elasticCredentialCache.get(cacheKey);
+ if (clusterInfo != null) {
+ return clusterInfo;
+ }
+
+ logger.warning(String.format("elastic-credential cache missed for tenant: %s", tenantInfo.getName()));
+
+ clusterInfo = this.elasticRepository.getElasticClusterSettings(tenantInfo);
+ if (clusterInfo == null) {
+ throw new AppException(HttpStatus.SC_NOT_FOUND, "Tenant not found", "No information about the given tenant was found");
+ }
+
+ this.elasticCredentialCache.put(cacheKey, clusterInfo);
+ return clusterInfo;
+ }
+}
diff --git a/provider/indexer-ibm/src/main/resources/application.properties b/provider/indexer-ibm/src/main/resources/application.properties
new file mode 100644
index 0000000000000000000000000000000000000000..242a77daef41824a0a7d59a4076135893f73d7ce
--- /dev/null
+++ b/provider/indexer-ibm/src/main/resources/application.properties
@@ -0,0 +1,64 @@
+server.servlet.contextPath=/api/indexer/v2/
+
+LOG_PREFIX=indexer
+
+logging.level.org.springframework.web=DEBUG
+server.port=8080
+JAVA_HEAP_OPTS=-Xms4096M -Xmx4096M
+JAVA_GC_OPTS=-XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45
+
+DEFAULT_DATA_COUNTRY=US
+
+AUTHORIZE_API=https://entitlements:8080/api/entitlements/v1
+AUTHORIZE_API_KEY=tobeupdated
+LEGALTAG_API=http://legal:8080/api/legal/v1
+
+DEPLOYMENT_ENVIRONMENT=CLOUD
+
+SCHEMA_CACHE_EXPIRATION=60
+INDEX_CACHE_EXPIRATION=60
+ELASTIC_CACHE_EXPIRATION=1440
+CURSOR_CACHE_EXPIRATION=60
+KINDS_CACHE_EXPIRATION=2880
+ATTRIBUTES_CACHE_EXPIRATION=2880
+MAX_CACHE_VALUE_SIZE=1000
+KINDS_REDIS_DATABASE=1
+CRON_INDEX_CLEANUP_THRESHOLD_DAYS=3
+CRON_EMPTY_INDEX_CLEANUP_THRESHOLD_DAYS=7
+
+storage_service_url=http://localhost:8082
+#storage_service_url=https://os-storage-ibm-osdu-r2.osduadev-a1c3eaf78a86806e299f5f3f207556f0-0000.us-south.containers.appdomain.cloud
+STORAGE_SCHEMA_HOST=${storage_service_url}/api/storage/v2/schemas
+STORAGE_QUERY_RECORD_HOST=${storage_service_url}/api/storage/v2/query/records
+STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=${storage_service_url}/api/storage/v2/query/records:batch
+STORAGE_RECORDS_BATCH_SIZE=20
+
+ibm.db.url=REPLACE_ME
+## use apikey or user/password
+ibm.db.user=REPLACE_ME
+ibm.db.password=REPLACE_ME
+
+ibm.tenant.db.url=${ibm.db.url}
+## use apikey or user/password
+ibm.tenant.db.user=${ibm.db.user}
+ibm.tenant.db.password=${ibm.db.password}
+
+ibm.rabbitmq.uri=REPLACE_ME
+
+ibm.keycloak.endpoint_url=keycloak-host.com
+ibm.keycloak.realm=OSDU
+ibm.keycloak.client_id=osdu-login
+ibm.keycloak.client_secret=REPLACE_ME
+ibm.keycloak.username=REPLACE_ME
+ibm.keycloak.password=REPLACE_ME
+ibm.keycloak.useremail=osdu-user@osdu.opengroup.org
+
+#Indexer-Queue-header
+indexer.queue.key=abcd
+
+ELASTIC_DATASTORE_KIND=SearchSettings
+ELASTIC_DATASTORE_ID=indexer-service
+
+ELASTIC_HOST=elasticsearch.com
+ELASTIC_PORT=443
+ELASTIC_USER_PASSWORD=REPLACE_ME:REPLACE_ME
diff --git a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/CronServiceImplTest.java b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/CronServiceImplTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..a51f2d8bf2bb95f8afd53db6cac16d81faa8b17f
--- /dev/null
+++ b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/CronServiceImplTest.java
@@ -0,0 +1,141 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.service;
+
+import com.google.common.collect.Lists;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.indexer.service.CronServiceImpl;
+import org.opengroup.osdu.core.common.model.search.IndexInfo;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.opengroup.osdu.core.common.search.IndicesService;
+import org.opengroup.osdu.core.common.search.Config;
+import org.opengroup.osdu.indexer.util.ElasticClientHandler;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.*;
+import static org.powermock.api.mockito.PowerMockito.mockStatic;
+
+
+@Ignore
+@RunWith(SpringRunner.class)
+@PrepareForTest({RestHighLevelClient.class})
+public class CronServiceImplTest {
+
+ @Mock
+ private RestHighLevelClient restHighLevelClient;
+ @Mock
+ private IndicesService indicesService;
+ @Mock
+ private ElasticClientHandler elasticClientHandler;
+ @Mock
+ private IRequestInfo requestInfo;
+ @Mock
+ private JaxRsDpsLog log;
+ @InjectMocks
+ private CronServiceImpl sut;
+
+ @InjectMocks
+ private DpsHeaders dpsHeaders;
+
+ @Before
+ public void setup() {
+ mockStatic(Config.class);
+
+ when(this.requestInfo.getHeaders()).thenReturn(dpsHeaders);
+
+ when(Config.getIndexCleanupThresholdDays()).thenReturn(3);
+ when(Config.getEmptyIndexCleanupThresholdDays()).thenReturn(7);
+ }
+
+ @Test
+ public void run_cleanup_when_cron_job_runs_with_correct_pattern() throws Exception {
+ final String indexPattern = "tenant1-index-*";
+
+ IndexInfo info = IndexInfo.builder().name("tenant1-index-1.0.0").documentCount("10").creationDate(Long.toString(Instant.now().minus(4, ChronoUnit.DAYS).toEpochMilli())).build();
+
+ when(this.requestInfo.getPartitionId()).thenReturn("tenant1");
+ when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient);
+ when(this.indicesService.getIndexInfo(this.restHighLevelClient, indexPattern)).thenReturn(Lists.newArrayList(info));
+
+ this.sut.cleanupIndices(indexPattern);
+
+ verify(this.indicesService, times(1)).deleteIndex(restHighLevelClient, "tenant1-index-1.0.0");
+ verify(this.indicesService, times(1)).getIndexInfo(restHighLevelClient, indexPattern);
+ }
+
+ @Test(expected = IOException.class)
+ public void run_cleanup_when_cron_job_runs_with_wrong_pattern() throws Exception {
+ IOException exception = new IOException("blah");
+ when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient);
+ when(this.indicesService.getIndexInfo(this.restHighLevelClient, "tenant1-test-*")).thenThrow(exception);
+
+ this.sut.cleanupIndices("tenant1-test-*");
+
+ verify(this.indicesService, times(0)).deleteIndex(any(), any());
+ }
+
+ @Test
+ public void run_cleanup_when_backend_does_not_have_empty_stale_indices() throws Exception {
+ IndexInfo info = IndexInfo.builder().name("tenant1-index-1.0.0").documentCount("10").creationDate(Long.toString(Instant.now().minus(8, ChronoUnit.DAYS).toEpochMilli())).build();
+
+ when(this.requestInfo.getPartitionId()).thenReturn("tenant1");
+ when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient);
+ when(this.indicesService.getIndexInfo(this.restHighLevelClient, null)).thenReturn(Lists.newArrayList(info));
+
+ this.sut.cleanupEmptyStaleIndices();
+
+ verify(this.indicesService, times(0)).deleteIndex(restHighLevelClient, null);
+ verify(this.indicesService, times(1)).getIndexInfo(restHighLevelClient, null);
+ }
+
+ @Test
+ public void run_cleanup_when_backend_have_empty_stale_indices() throws Exception {
+ IndexInfo info = IndexInfo.builder().name("tenant1-index-1.0.0").documentCount("0").creationDate(Long.toString(Instant.now().minus(8, ChronoUnit.DAYS).toEpochMilli())).build();
+
+ when(this.requestInfo.getPartitionId()).thenReturn("tenant1");
+ when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient);
+ when(this.indicesService.getIndexInfo(this.restHighLevelClient, null)).thenReturn(Lists.newArrayList(info));
+
+ this.sut.cleanupEmptyStaleIndices();
+
+ verify(this.indicesService, times(1)).deleteIndex(restHighLevelClient, "tenant1-index-1.0.0");
+ verify(this.indicesService, times(1)).getIndexInfo(restHighLevelClient, null);
+ }
+
+ @Test(expected = IOException.class)
+ public void run_cleanup_when_backend_throws_exception() throws Exception {
+ IOException exception = new IOException("blah");
+ when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient);
+ when(this.indicesService.getIndexInfo(this.restHighLevelClient, null)).thenThrow(exception);
+
+ this.sut.cleanupEmptyStaleIndices();
+
+ verify(this.indicesService, times(0)).deleteIndex(any(), any());
+ }
+}
\ No newline at end of file
diff --git a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexCopyServiceImplTest.java b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexCopyServiceImplTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..2b37cddf112a713c6e08166cce591a0ff5cb923a
--- /dev/null
+++ b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexCopyServiceImplTest.java
@@ -0,0 +1,191 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.service;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import org.apache.http.HttpEntity;
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentMatchers;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.indexer.logging.AuditLogger;
+import org.opengroup.osdu.indexer.service.IndexCopyServiceImpl;
+import org.opengroup.osdu.indexer.service.IndexerMappingService;
+import org.opengroup.osdu.core.common.model.search.ClusterSettings;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.opengroup.osdu.core.common.model.indexer.IElasticSettingService;
+import org.opengroup.osdu.core.common.search.IndicesService;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.indexer.util.ElasticClientHandler;
+import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.io.IOException;
+import java.lang.reflect.Type;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@RunWith(SpringRunner.class)
+@PrepareForTest({RestHighLevelClient.class, Response.class, RestClient.class, HttpEntity.class, EntityUtils.class})
+public class IndexCopyServiceImplTest {
+ private final String correlationId = UUID.randomUUID().toString();
+
+ @Mock
+ private HttpEntity httpEntity;
+ @Mock
+ private HttpEntity httpEntityRequest;
+ @Mock
+ private IRequestInfo requestInfo;
+ @Mock
+ private DpsHeaders dpsHeaders;
+ @Mock
+ private RestClient restClient;
+ @Mock
+ private RestHighLevelClient restHighLevelClient;
+ @Mock
+ private IndicesService indicesService;
+ @Mock
+ private IndexerMappingService mappingService;
+ @Mock
+ private ElasticClientHandler elasticClientHandler;
+ @Mock
+ private ElasticIndexNameResolver elasticIndexNameResolver;
+ @Mock
+ private Response response;
+ @Mock
+ private IElasticSettingService elasticSettingService;
+ @Mock
+ private AuditLogger auditLogger;
+ @Mock
+ private Map<String, String> httpHeaders;
+ @InjectMocks
+ private IndexCopyServiceImpl sut;
+
+ private ClusterSettings commonCluster;
+
+ private Map<String, Object> correctMap;
+
+ @Before
+ public void setup() {
+
+ commonCluster = ClusterSettings.builder().host("commonhost").port(8080).userNameAndPassword("username:pwd").build();
+
+ httpHeaders = new HashMap<>();
+ httpHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
+ httpHeaders.put(DpsHeaders.CORRELATION_ID, correlationId);
+ when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(httpHeaders);
+ when(response.getEntity()).thenReturn(httpEntity);
+
+ Type mapType = new TypeToken<Map<String, Object>>() {}.getType();
+ String afterFormat = "{\"properties\":{\"id\":{\"type\":\"keyword\"}}}";
+ correctMap = new Gson().fromJson(afterFormat, mapType);
+
+ restHighLevelClient = mock(RestHighLevelClient.class);
+
+ }
+
+ @Test(expected = IOException.class)
+ public void should_throwIOException_when_indexMappingNotFound() throws Exception {
+ IOException exception = new IOException("Fail to get mapping for the given index from common cluster.");
+
+ when(this.mappingService.getIndexMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenThrow(exception);
+
+ this.sut.copyIndex("common:metadata:entity:1.0.0");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void should_throwIllegalArgExceptionCopyIndexRequest_copyIndexTest() {
+ try {
+ this.sut.copyIndex(null);
+ } catch (IOException e) {
+ fail("Should not throw IOException but illegalArgumentException.");
+ }
+ }
+
+ @Test
+ public void should_returnIndexMapping_getIndexMappingFromCommonClustertest() {
+ String mappingJson = "{\"common-metadata-entity-1.0.0\":{\"mappings\":{\"entity\":{\"properties\":{\"id\":{\"type\":\"keyword\"}}}}}}";
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ try {
+ when(this.mappingService.getIndexMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(mappingJson);
+ Map<String, Object> resultMap = this.sut.getIndexMappingsFromCommonCluster("test", "test");
+ Assert.assertEquals(resultMap, correctMap);
+ } catch (Exception ignored) {
+ }
+ }
+
+ @Test
+ public void should_returnClusterInfo_getCommonClusterInformationtest() {
+ try {
+ String[] correctCommonCluster = {"https://commonhost:8080", "username", "pwd"};
+
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+
+ when(elasticSettingService.getElasticClusterInformation()).thenReturn(commonCluster);
+
+ String[] resultCommonCluster = this.sut.getCommonClusterInformation();
+ Assert.assertEquals(correctCommonCluster[0], resultCommonCluster[0]);
+ Assert.assertEquals(correctCommonCluster[1], resultCommonCluster[1]);
+ Assert.assertEquals(correctCommonCluster[2], resultCommonCluster[2]);
+ } catch (IOException ignored) {
+ fail("Should not throw this exception " + ignored.getMessage());
+ }
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throwException_failToCreateIndexInTenantCluster_createIndexInTenantClustertest() {
+ try {
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ when(indicesService.createIndex(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(false);
+ this.sut.createIndexInTenantCluster("test", "test", "test", correctMap);
+ } catch (IOException ignored) {
+ fail("Should not throw this exception " + ignored.getMessage());
+ }
+ }
+
+ @Ignore
+ public void should_returnTaskIdResponse_reindexRequestSucceed_reindexInTenantClustertest() {
+ //TODO: fix the null Response from restHighLevelClient.getLowLevelClient().performRequest().
+ try {
+ String[] correctCommonCluster = {"https://commonhost:8080", "username", "pwd"};
+ Request request = new Request("POST", "/_reindex?wait_for_completion=false");
+ request.setEntity(httpEntityRequest);
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ when(indicesService.createIndex(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(false);
+ when(restHighLevelClient.getLowLevelClient()).thenReturn(restClient);
+ when(restClient.performRequest(request)).thenReturn(response);
+ when(response.getEntity()).thenReturn(httpEntity);
+ Assert.assertEquals(httpEntity, this.sut.reindexInTenantCluster("test", "test", correctCommonCluster));
+ } catch (IOException ignored) {
+ }
+ }
+}
diff --git a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerMappingServiceTest.java b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerMappingServiceTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..fe2905d770837c9bdf8d1f9bdb0d81d127667042
--- /dev/null
+++ b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerMappingServiceTest.java
@@ -0,0 +1,316 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.service;
+
+import org.apache.http.StatusLine;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
+import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData;
+import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.*;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.reindex.BulkByScrollResponse;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentMatchers;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
+import org.opengroup.osdu.indexer.service.IndexerMappingServiceImpl;
+import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.indexer.util.ElasticClientHandler;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.io.IOException;
+import java.util.*;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@Ignore
+@RunWith(SpringRunner.class)
+@PrepareForTest({ RestHighLevelClient.class, IndicesClient.class })
+public class IndexerMappingServiceTest {
+
+ private final String kind = "tenant:test:test:1.0.0";
+ private final String index = "tenant-test-test-1.0.0";
+ private final String type = "test";
+ private final String mappingValid = "{\"dynamic\":false,\"properties\":{\"data\":{\"properties\":{\"Location\":{\"type\":\"geo_point\"}}},\"id\":{\"type\":\"keyword\"}}}";
+
+ @Mock
+ private RestClient restClient;
+ @Mock
+ private Response response;
+ @Mock
+ private StatusLine statusLine;
+
+ @InjectMocks
+ private IndexerMappingServiceImpl sut;
+
+ @Mock
+ private ElasticClientHandler elasticClientHandler;
+
+ @InjectMocks
+ private RestHighLevelClient restHighLevelClient;
+
+ @InjectMocks
+ private IndexSchema indexSchema;
+ @InjectMocks
+ private IndicesClient indicesClient;
+
+ @InjectMocks
+ private AcknowledgedResponse mappingResponse;
+
+ @Before
+ public void setup() throws IOException {
+ Map<String, String> dataMapping = new HashMap<>();
+ dataMapping.put("Location", "geo_point");
+ Map<String, Object> metaMapping = new HashMap<>();
+ metaMapping.put(RecordMetaAttribute.ID.getValue(), "keyword");
+ this.indexSchema = IndexSchema.builder().kind(kind).type(type).dataSchema(dataMapping).metaSchema(metaMapping)
+ .build();
+
+ this.indicesClient = PowerMockito.mock(IndicesClient.class);
+ this.restHighLevelClient = PowerMockito.mock(RestHighLevelClient.class);
+
+ when(this.restHighLevelClient.getLowLevelClient()).thenReturn(restClient);
+ when(this.restClient.performRequest(ArgumentMatchers.any())).thenReturn(response);
+ when(this.response.getStatusLine()).thenReturn(statusLine);
+ when(this.statusLine.getStatusCode()).thenReturn(200);
+ }
+
+ @Test
+ public void should_returnValidMapping_givenFalseMerge_createMappingTest() {
+ try {
+ String mapping = this.sut.createMapping(restHighLevelClient, indexSchema, index, false);
+ assertEquals(mappingValid, mapping);
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_returnValidMapping_givenTrueMerge_createMappingTest() {
+ try {
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+
+ String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true);
+ assertEquals(this.mappingValid, mapping);
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_returnValidMapping_givenExistType_createMappingTest() {
+ try {
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+
+ IndexerMappingServiceImpl indexerMappingServiceLocal = PowerMockito.spy(new IndexerMappingServiceImpl());
+ doReturn(false).when(indexerMappingServiceLocal).isTypeExist(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any());
+ String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true);
+ assertEquals(this.mappingValid, mapping);
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_update_indices_field_with_keyword_when_valid_indices() throws Exception {
+ try {
+ Set<String> indices = new HashSet<String>();
+ indices.add("indices 1");
+ GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
+ XContentBuilder builder = XContentFactory.jsonBuilder();
+ builder.startObject();
+ builder.field("any field", new HashMap());
+ builder.endObject();
+ BytesReference bytesReference = BytesReference.bytes(builder);
+ FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
+ Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ mapBuilder.put("data.any field", mappingMetaData);
+ Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ mappingBuilder.put("any index 1", mapBuilder);
+ mappingBuilder.put("any index 2", mapBuilder);
+ Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ mapping.put("indices 1", mappingBuilder);
+ when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ BulkByScrollResponse response = mock(BulkByScrollResponse.class);
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+
+ this.sut.updateIndexMappingForIndicesOfSameType( indices,"any field");
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throw_exception_if_someIndex_is_invalid_andWeIndexfield_with_keyword() throws Exception {
+ try {
+ Set<String> indices = new HashSet<String>();
+ indices.add("invalid 1");
+ GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
+ XContentBuilder builder = XContentFactory.jsonBuilder();
+ builder.startObject();
+ builder.field("any field", new HashMap());
+ builder.endObject();
+ BytesReference bytesReference = BytesReference.bytes(builder);
+ FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
+ Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ mapBuilder.put("data.any field", mappingMetaData);
+ Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ mappingBuilder.put("any index 1", mapBuilder);
+ mappingBuilder.put("any index 2", mapBuilder);
+ Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ mapping.put("indices 1", mappingBuilder);
+ when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ BulkByScrollResponse response = mock(BulkByScrollResponse.class);
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+
+ this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
+ } catch (Exception e) {
+ throw e;
+ }
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throw_exception_if_type_of_index_is_invalid_andWeIndexfield_with_keyword() throws Exception {
+ try {
+ Set<String> indices = new HashSet<String>();
+ indices.add("indices 1");
+ GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
+ XContentBuilder builder = XContentFactory.jsonBuilder();
+ builder.startObject();
+ builder.field("any field", new HashMap());
+ builder.endObject();
+ BytesReference bytesReference = BytesReference.bytes(builder);
+ FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
+ Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ mapBuilder.put("data.any field", mappingMetaData);
+ Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ mappingBuilder.put("any index 1", mapBuilder);
+ mappingBuilder.put("any index 2", mapBuilder);
+ Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ mapping.put("indices 1", mappingBuilder);
+ when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ BulkByScrollResponse response = mock(BulkByScrollResponse.class);
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field invalid");
+ } catch (Exception e) {
+ throw e;
+ }
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throw_exception_if_elastic_search_failedToFetch_andWeIndexfield_with_keyword() throws Exception {
+ try {
+
+ Set<String> indices = new HashSet<String>();
+ indices.add("indices 1");
+ indices.add("indices Invalid");
+ GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenThrow(new ElasticsearchException(""));
+ XContentBuilder builder = XContentFactory.jsonBuilder();
+ builder.startObject();
+ builder.field("any field", new HashMap());
+ builder.endObject();
+ BytesReference bytesReference = BytesReference.bytes(builder);
+ FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
+ Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ mapBuilder.put("data.any field", mappingMetaData);
+ Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ mappingBuilder.put("any index 1", mapBuilder);
+ mappingBuilder.put("any index 2", mapBuilder);
+ Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ mapping.put("indices 1", mappingBuilder);
+ when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ BulkByScrollResponse response = mock(BulkByScrollResponse.class);
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
+ } catch (AppException e) {
+ throw e;
+ }
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throw_exception_when_elastic_failedToIndex_indices_field_with_keyword() {
+ try {
+ Set<String> indices = new HashSet<String>();
+ indices.add("indices 1");
+ indices.add("indices Invalid");
+ GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
+ XContentBuilder builder = XContentFactory.jsonBuilder();
+ builder.startObject();
+ builder.field("any field", new HashMap());
+ builder.endObject();
+ BytesReference bytesReference = BytesReference.bytes(builder);
+ FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
+ Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+ mapBuilder.put("data.any field", mappingMetaData);
+ Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+ mappingBuilder.put("any index 1", mapBuilder);
+ mappingBuilder.put("any index 2", mapBuilder);
+ Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+ mapping.put("indices 1", mappingBuilder);
+ when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ BulkByScrollResponse response = mock(BulkByScrollResponse.class);
+ doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+ when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
+ when(this.indicesClient.putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class))).thenThrow(new ElasticsearchException(""));
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
+ } catch (AppException e) {
+ throw e;
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+}
diff --git a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerSchemaServiceTest.java b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerSchemaServiceTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..25dd2ed446ebc1cb107a0149c139db46705569f2
--- /dev/null
+++ b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerSchemaServiceTest.java
@@ -0,0 +1,329 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.service;
+
+import org.apache.http.HttpStatus;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
+import org.opengroup.osdu.core.common.model.indexer.OperationType;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache;
+import org.opengroup.osdu.indexer.service.IndexSchemaServiceImpl;
+import org.opengroup.osdu.indexer.service.IndexerMappingService;
+import org.opengroup.osdu.indexer.service.StorageService;
+import org.opengroup.osdu.core.common.model.http.RequestStatus;
+import org.opengroup.osdu.core.common.search.IndicesService;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.indexer.util.ElasticClientHandler;
+import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.*;
+import static org.mockito.ArgumentMatchers.*;
+import static org.mockito.Mockito.*;
+import static org.mockito.MockitoAnnotations.initMocks;
+import static org.powermock.api.mockito.PowerMockito.mock;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@Ignore
+@RunWith(SpringRunner.class)
+@PrepareForTest({RestHighLevelClient.class})
+public class IndexerSchemaServiceTest {
+
+ private final String kind = "tenant:test:test:1.0.0";
+ private final String emptySchema = null;
+ private final String someSchema = "{\"kind\":\"tenant:test:test:1.0.0\", \"schema\":[{\"path\":\"test-path\", \"kind\":\"tenant:test:test:1.0.0\"}]}";
+
+ @Mock
+ private JaxRsDpsLog log;
+ @Mock
+ private StorageService storageService;
+ @Mock
+ private ElasticClientHandler elasticClientHandler;
+ @Mock
+ private ElasticIndexNameResolver elasticIndexNameResolver;
+ @Mock
+ private IndexerMappingService mappingService;
+ @Mock
+ private IndicesService indicesService;
+ @Mock
+ private ISchemaCache schemaCache;
+ @InjectMocks
+ private IndexSchemaServiceImpl sut;
+
+ @Before
+ public void setup() {
+ initMocks(this);
+ RestHighLevelClient restHighLevelClient = mock(RestHighLevelClient.class);
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ }
+
+ @Test
+ public void should_returnNull_givenEmptySchema_getIndexerInputSchemaSchemaTest() throws Exception {
+ when(storageService.getStorageSchema(any())).thenReturn(emptySchema);
+
+ IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind);
+
+ Assert.assertNotNull(indexSchema);
+ }
+
+ @Test
+ public void should_returnValidResponse_givenValidSchema_getIndexerInputSchemaTest() throws Exception {
+ when(storageService.getStorageSchema(any())).thenReturn(someSchema);
+
+ IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind);
+
+ Assert.assertEquals(kind, indexSchema.getKind());
+ }
+
+ @Test
+ public void should_returnValidResponse_givenValidSchemaWithCacheHit_getIndexerInputSchemaTest() throws Exception {
+ when(storageService.getStorageSchema(any())).thenReturn(someSchema);
+ when(this.schemaCache.get(kind + "_flattened")).thenReturn(someSchema);
+
+ IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind);
+
+ Assert.assertEquals(kind, indexSchema.getKind());
+ }
+
+ @Test
+ public void should_throw500_givenInvalidSchemaCacheHit_getIndexerInputSchemaTest() {
+ try {
+ String invalidSchema = "{}}";
+ when(storageService.getStorageSchema(any())).thenReturn(invalidSchema);
+
+ this.sut.getIndexerInputSchema(kind);
+ fail("Should throw exception");
+ } catch (AppException e) {
+ Assert.assertEquals(HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getError().getCode());
+ Assert.assertEquals("An error has occurred while normalizing the schema.", e.getError().getMessage());
+ } catch (Exception e) {
+ fail("Should not throw exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_return_basic_schema_when_storage_returns_no_schema() {
+ IndexSchema returnedSchema = this.sut.getIndexerInputSchema(kind);
+
+ assertNotNull(returnedSchema.getDataSchema());
+ assertNotNull(returnedSchema);
+ assertEquals(kind, returnedSchema.getKind());
+ }
+
+ @Test
+ public void should_create_schema_when_storage_returns_valid_schema() throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"startDate\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"endDate\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"type \"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"itemguid\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.create_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(false);
+ when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
+
+ this.sut.processSchemaMessages(schemaMessages);
+
+ verify(this.mappingService, times(1)).getIndexMappingFromRecordSchema(any());
+ verify(this.indicesService, times(1)).createIndex(any(), any(), any(), any(), any());
+ verifyNoMoreInteractions(this.mappingService);
+ }
+
+ @Test
+ public void should_merge_mapping_when_storage_returns_valid_schema() throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"startDate\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.create_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
+
+ this.sut.processSchemaMessages(schemaMessages);
+
+ verify(this.indicesService, times(0)).createIndex(any(), any(), any(), any(), any());
+ verify(this.mappingService, times(1)).createMapping(any(), any(), any(), anyBoolean());
+ verifyNoMoreInteractions(this.mappingService);
+ }
+
+ @Test
+ public void should_throw_mapping_conflict_when_elastic_backend_cannot_process_schema_changes() throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String reason = String.format("Could not create type mapping %s/completion.", kind.replace(":", "-"));
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.create_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
+ when(this.mappingService.createMapping(any(), any(), any(), anyBoolean())).thenThrow(new AppException(HttpStatus.SC_BAD_REQUEST, reason, ""));
+
+ try {
+ this.sut.processSchemaMessages(schemaMessages);
+ } catch (AppException e){
+ assertEquals(e.getError().getCode(), RequestStatus.SCHEMA_CONFLICT);
+ assertEquals(e.getError().getMessage(), "error creating or merging index mapping");
+ assertEquals(e.getError().getReason(), reason);
+ } catch (Exception e) {
+ fail("Should not throw this exception " + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_throw_genericAppException_when_elastic_backend_cannot_process_schema_changes() throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String reason = String.format("Could not create type mapping %s/completion.", kind.replace(":", "-"));
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.create_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
+ when(this.mappingService.createMapping(any(), any(), any(), anyBoolean())).thenThrow(new AppException(HttpStatus.SC_FORBIDDEN, reason, "blah"));
+
+ try {
+ this.sut.processSchemaMessages(schemaMessages);
+ } catch (AppException e){
+ assertEquals(e.getError().getCode(), HttpStatus.SC_FORBIDDEN);
+ assertEquals(e.getError().getMessage(), "blah");
+ assertEquals(e.getError().getReason(), reason);
+ } catch (Exception e) {
+ fail("Should not throw this exception " + e.getMessage());
+ }
+ }
+
+
+ @Test
+ public void should_log_and_do_nothing_when_storage_returns_invalid_schema() throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"" +
+ "}";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.create_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.storageService.getStorageSchema(kind)).thenReturn(storageSchema);
+
+ this.sut.processSchemaMessages(schemaMessages);
+
+ verify(this.log).warning(eq("schema not found for kind: tenant1:avocet:completion:1.0.0"));
+ }
+
+ @Test
+ public void should_invalidateCache_when_purge_schema_and_schema_found_in_cache() throws IOException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.purge_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.schemaCache.get(kind)).thenReturn("schema");
+ when(this.schemaCache.get(kind + "_flattened")).thenReturn("flattened schema");
+
+ this.sut.processSchemaMessages(schemaMessages);
+
+ verify(this.schemaCache, times(2)).get(anyString());
+ verify(this.schemaCache, times(2)).delete(anyString());
+ }
+
+ @Test
+ public void should_log_warning_when_purge_schema_and_schema_not_found_in_cache() throws IOException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ Map<String, OperationType> schemaMessages = new HashMap<>();
+ schemaMessages.put(kind, OperationType.purge_schema);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(false);
+
+ this.sut.processSchemaMessages(schemaMessages);
+
+ verify(this.log).warning(eq(String.format("Kind: %s not found", kind)));
+ }
+}
diff --git a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerServiceTest.java b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerServiceTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..0043a0d898b93ace6e7a49c7f2e67b6f17f41726
--- /dev/null
+++ b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerServiceTest.java
@@ -0,0 +1,337 @@
+package org.opengroup.osdu.indexer.ibm.service;//// Copyright 2017-2019, Schlumberger
+////
+//// Licensed under the Apache License, Version 2.0 (the "License");
+//// you may not use this file except in compliance with the License.
+//// You may obtain a copy of the License at
+////
+//// http://www.apache.org/licenses/LICENSE-2.0
+////
+//// Unless required by applicable law or agreed to in writing, software
+//// distributed under the License is distributed on an "AS IS" BASIS,
+//// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//// See the License for the specific language governing permissions and
+//// limitations under the License.
+//
+//package org.opendes.indexer.service;
+//
+//import com.google.gson.Gson;
+//import com.google.gson.reflect.TypeToken;
+//import org.elasticsearch.action.bulk.BulkItemResponse;
+//import org.elasticsearch.action.bulk.BulkResponse;
+//import org.elasticsearch.client.RequestOptions;
+//import org.elasticsearch.client.RestHighLevelClient;
+//import org.junit.Before;
+//import org.junit.Ignore;
+//import org.junit.Test;
+//import org.junit.runner.RunWith;
+//import org.mockito.InjectMocks;
+//import org.mockito.Mock;
+//import org.mockito.Spy;
+//import org.opendes.client.api.DpsHeaders;
+//import org.opendes.core.logging.JaxRsDpsLog;
+//import org.opendes.core.model.DeploymentEnvironment;
+//import org.opendes.core.model.RecordChangedMessages;
+//import org.opendes.core.service.IndicesService;
+//import org.opendes.core.util.Config;
+//import org.opendes.core.util.ElasticClientHandler;
+//import org.opendes.core.util.ElasticIndexNameResolver;
+//import org.opendes.core.util.HeadersUtil;
+//import org.opendes.indexer.logging.AuditLogger;
+//import org.opendes.indexer.model.*;
+//import org.opendes.indexer.publish.IPublisher;
+//import org.opendes.indexer.util.IRequestInfo;
+//import org.opendes.indexer.util.IndexerQueueTaskBuilder;
+//import org.opendes.indexer.util.JobStatus;
+//import org.opendes.indexer.util.RecordInfo;
+//import org.powermock.core.classloader.annotations.PrepareForTest;
+//import org.springframework.context.annotation.Lazy;
+//import org.springframework.test.context.junit4.SpringRunner;
+//
+//import javax.inject.Inject;
+//import java.io.IOException;
+//import java.lang.reflect.Type;
+//import java.util.*;
+//
+//import static java.util.Collections.singletonList;
+//import static org.junit.Assert.*;
+//import static org.mockito.Matchers.any;
+//import static org.mockito.Mockito.verify;
+//import static org.mockito.Mockito.when;
+//import static org.powermock.api.mockito.PowerMockito.mock;
+//import static org.powermock.api.mockito.PowerMockito.mockStatic;
+//
+//@Ignore
+//@RunWith(SpringRunner.class)
+//@PrepareForTest({RestHighLevelClient.class, BulkResponse.class, StorageAcl.class, HeadersUtil.class, Config.class})
+//public class IndexerServiceTest {
+//
+// private final String pubsubMsg = "[{\"id\":\"tenant1:doc:test1\",\"kind\":\"tenant1:testindexer1:well:1.0.0\",\"op\":\"update\"}," +
+// "{\"id\":\"tenant1:doc:test2\",\"kind\":\"tenant1:testindexer2:well:1.0.0\",\"op\":\"create\"}]";
+// private final String kind1 = "tenant1:testindexer1:well:1.0.0";
+// private final String kind2 = "tenant1:testindexer2:well:1.0.0";
+// private final String recordId1 = "tenant1:doc:test1";
+// private final String recordId2 = "tenant1:doc:test2";
+// private final String failureMassage = "test failure";
+//
+// @Mock
+// private IndexSchemaService indexSchemaService;
+// @Mock
+// private IndicesService indicesService;
+// @Mock
+// private IndexerMappingService indexerMappingService;
+// @Mock
+// private StorageService storageService;
+// @Mock
+// private IPublisher publisherImpl;
+// @Mock
+// private RestHighLevelClient restHighLevelClient;
+// @Mock
+// private ElasticClientHandler elasticClientHandler;
+// @Mock
+// private BulkResponse bulkResponse;
+// @Mock
+// private IRequestInfo requestInfo;
+// @Mock
+// private ElasticIndexNameResolver elasticIndexNameResolver;
+// @Mock
+// private AttributeParsingServiceImpl attributeParsingServiceImpl;
+// @Mock
+// private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
+// @Mock
+// private JaxRsDpsLog log;
+// @Mock
+// private AuditLogger auditLogger;
+// @InjectMocks
+// private IndexerServiceImpl sut;
+// @InjectMocks @Spy
+// private JobStatus jobStatus = new JobStatus();
+//
+// @Inject
+// @Lazy
+// private DpsHeaders dpsHeaders;
+// private RecordChangedMessages recordChangedMessages;
+// private List<RecordInfo> recordInfos;
+//
+// @Before
+// public void setup() throws IOException {
+//
+// mockStatic(StorageAcl.class);
+// mockStatic(Config.class);
+//
+// when(Config.getDeploymentEnvironment()).thenReturn(DeploymentEnvironment.LOCAL);
+// when(Config.getElasticClusterName()).thenReturn("CLUSTER");
+// when(Config.getElasticServerAddress()).thenReturn("testsite");
+//
+// dpsHeaders = new DpsHeaders();
+// dpsHeaders.put(AppEngineHeaders.TASK_QUEUE_RETRY_COUNT, "1");
+// dpsHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
+// when(requestInfo.getHeaders()).thenReturn(dpsHeaders);
+// when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(dpsHeaders.getHeaders());
+//
+// Type listType = new TypeToken<List<RecordInfo>>() {}.getType();
+// recordInfos = (new Gson()).fromJson(pubsubMsg, listType);
+//
+// when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+// when(restHighLevelClient.bulk(any(), any(RequestOptions.class))).thenReturn(bulkResponse);
+//
+// BulkItemResponse[] responses = new BulkItemResponse[]{prepareResponseFail(), prepareResponseSuccess()};
+// when(bulkResponse.getItems()).thenReturn(responses);
+// Map<String, String> attr = new HashMap<>();
+// attr.put(DpsHeaders.ACCOUNT_ID, "slb");
+// recordChangedMessages = RecordChangedMessages.builder().attributes(attr).messageId("xxxx").publishTime("2000-01-02T10:10:44+0000").data("{}").build();
+// when(StorageAcl.flattenAcl(any())).thenReturn(null);
+// }
+//
+// @Test
+// public void should_returnNull_givenEmptyJobSubInfo_processRecordChangedMessageTest() throws Exception {
+// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, new ArrayList<>());
+//
+// assertNull(jobStatus);
+// }
+//
+// @Test
+// public void should_returnValidJobStatus_givenNullSchema_processRecordChangedMessageTest() {
+// try {
+// indexSchemaServiceMock(kind1, null);
+// indexSchemaServiceMock(kind2, null);
+// List<ConversionStatus> conversionStatus = new LinkedList<>();
+// List<Records.Entity> validRecords = new ArrayList<>();
+// Map<String, Object> storageData = new HashMap<>();
+// storageData.put("schema1", "test-value");
+// storageData.put("schema2", "test-value");
+// storageData.put("schema3", "test-value");
+// storageData.put("schema4", "test-value");
+// storageData.put("schema5", "test-value");
+// storageData.put("schema6", "test-value");
+// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build());
+// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build();
+//
+// when(storageService.getStorageRecords(any())).thenReturn(storageRecords);
+// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true);
+//
+// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
+//
+// assertEquals(2, jobStatus.getStatusesList().size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).size());
+// } catch (Exception e) {
+// fail("Should not throw this exception" + e.getMessage());
+// }
+// }
+//
+// @Test
+// public void should_returnValidJobStatus_givenFailedUnitsConversion_processRecordChangedMessageTest() {
+// try {
+// indexSchemaServiceMock(kind1, null);
+// indexSchemaServiceMock(kind2, null);
+// List<ConversionStatus> conversionStatuses = new LinkedList<>();
+// List<String> status=new ArrayList<>();
+// status.add("crs bla bla");
+// ConversionStatus conversionStatus=ConversionStatus.builder().status("ERROR").errors(status).id(recordId2).build();
+// conversionStatuses.add(conversionStatus);
+// List<Records.Entity> validRecords = new ArrayList<>();
+// Map<String, Object> storageData = new HashMap<>();
+// storageData.put("schema1", "test-value");
+// storageData.put("schema2", "test-value");
+// storageData.put("schema3", "test-value");
+// storageData.put("schema4", "test-value");
+// storageData.put("schema5", "test-value");
+// storageData.put("schema6", "test-value");
+// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build());
+// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatuses).build();
+//
+// when(storageService.getStorageRecords(any())).thenReturn(storageRecords);
+// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true);
+//
+// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
+//
+// assertEquals(2, jobStatus.getStatusesList().size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).size());
+// assertTrue(jobStatus.getJobStatusByRecordId(jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).get(0)).getIndexProgress().getTrace().contains("crs bla bla"));
+// } catch (Exception e) {
+// fail("Should not throw this exception" + e.getMessage());
+// }
+// }
+//
+// @Test
+// public void should_returnValidJobStatus_givenNullSchemaForARecord_processRecordChangedMessageTest() {
+// try {
+// List<Records.Entity> validRecords = new ArrayList<>();
+// List<ConversionStatus> conversionStatus = new LinkedList<>();
+// Map<String, Object> storageData = new HashMap<>();
+// storageData.put("schema1", "test-value");
+// storageData.put("schema2", "test-value");
+// storageData.put("schema3", "test-value");
+// storageData.put("schema4", "test-value");
+// storageData.put("schema5", "test-value");
+// storageData.put("schema6", "test-value");
+// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build());
+// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build();
+// when(storageService.getStorageRecords(any())).thenReturn(storageRecords);
+//
+// Map<String, String> schema = createSchema();
+// indexSchemaServiceMock(kind1, schema);
+// indexSchemaServiceMock(kind2, null);
+// when(elasticIndexNameResolver.getIndexNameFromKind(kind2)).thenReturn("tenant1-testindexer2-well-1.0.0");
+// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true);
+// JobStatus jobStatus = sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
+//
+// assertEquals(2, jobStatus.getStatusesList().size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.WARN).size());
+// assertEquals("Indexed Successfully", jobStatus.getStatusesList().get(1).getIndexProgress().getTrace().pop());
+// assertEquals("schema not found", jobStatus.getStatusesList().get(1).getIndexProgress().getTrace().pop());
+// } catch (Exception e) {
+// fail("Should not throw this exception" + e.getMessage());
+// }
+// }
+//
+// @Test
+// public void should_returnValidJobStatus_givenValidCreateAndUpdateRecords_processRecordChangedMessagesTest() {
+// try {
+// Map<String, Object> storageData = new HashMap<>();
+// storageData.put("schema1", "test-value");
+// List<ConversionStatus> conversionStatus = new LinkedList<>();
+// List<Records.Entity> validRecords = new ArrayList<>();
+// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build());
+// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build();
+//
+// when(storageService.getStorageRecords(any())).thenReturn(storageRecords);
+// when(indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true);
+// Map<String, String> schema = createSchema();
+// indexSchemaServiceMock(kind2, schema);
+// indexSchemaServiceMock(kind1, null);
+// JobStatus jobStatus = sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
+//
+// assertEquals(2, jobStatus.getStatusesList().size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.SUCCESS).size());
+// } catch (Exception e) {
+// fail("Should not throw this exception" + e.getMessage());
+// }
+// }
+//
+// @Test
+// public void should_properlyUpdateAuditLogs_givenValidCreateAndUpdateRecords() {
+// try {
+// Map<String, Object> storageData = new HashMap<>();
+// List<ConversionStatus> conversionStatus = new LinkedList<>();
+//
+// storageData.put("schema1", "test-value");
+// List<Records.Entity> validRecords = new ArrayList<>();
+// validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build());
+// Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build();
+//
+// when(this.storageService.getStorageRecords(any())).thenReturn(storageRecords);
+// when(this.indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true);
+// Map<String, String> schema = createSchema();
+// indexSchemaServiceMock(kind2, schema);
+// indexSchemaServiceMock(kind1, null);
+// JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
+//
+// assertEquals(2, jobStatus.getStatusesList().size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
+// assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.SUCCESS).size());
+//
+// verify(this.auditLogger).indexCreateRecordSuccess(singletonList("RecordStatus(id=tenant1:doc:test2, kind=tenant1:testindexer2:well:1.0.0, operationType=create, status=SUCCESS)"));
+// verify(this.auditLogger).indexUpdateRecordFail(singletonList("RecordStatus(id=tenant1:doc:test1, kind=tenant1:testindexer1:well:1.0.0, operationType=update, status=FAIL)"));
+// } catch (Exception e) {
+// fail("Should not throw this exception" + e.getMessage());
+// }
+// }
+//
+// private BulkItemResponse prepareResponseFail() {
+// BulkItemResponse responseFail = mock(BulkItemResponse.class);
+// when(responseFail.isFailed()).thenReturn(true);
+// when(responseFail.getFailureMessage()).thenReturn(failureMassage);
+// when(responseFail.getId()).thenReturn(recordId1);
+// when(responseFail.getFailure()).thenReturn(new BulkItemResponse.Failure("failure index", "failure type", "failure id", new Exception("test failure")));
+// return responseFail;
+// }
+//
+// private BulkItemResponse prepareResponseSuccess() {
+// BulkItemResponse responseSuccess = mock(BulkItemResponse.class);
+// when(responseSuccess.getId()).thenReturn(recordId2);
+// return responseSuccess;
+// }
+//
+// private void indexSchemaServiceMock(String kind, Map<String, String> schema) {
+// if (schema == null) {
+// IndexSchema indexSchema = IndexSchema.builder().kind(kind).dataSchema(null).build();
+// when(indexSchemaService.getIndexerInputSchema(kind)).thenReturn(indexSchema);
+// } else {
+// IndexSchema indexSchema = IndexSchema.builder().kind(kind).dataSchema(schema).build();
+// when(indexSchemaService.getIndexerInputSchema(kind)).thenReturn(indexSchema);
+// }
+// }
+//
+// private Map<String, String> createSchema() {
+// Map<String, String> schema = new HashMap<>();
+// schema.put("schema1", "keyword");
+// schema.put("schema2", "boolean");
+// schema.put("schema3", "date");
+// schema.put("schema6", "object");
+// return schema;
+// }
+//}
diff --git a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/ReindexServiceTest.java b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/ReindexServiceTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..e6a102b7821fa8a58bf38b0a992ce2203d8a2a47
--- /dev/null
+++ b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/ReindexServiceTest.java
@@ -0,0 +1,145 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.service;
+
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentMatchers;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
+import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.indexer.service.ReindexServiceImpl;
+import org.opengroup.osdu.indexer.service.StorageService;
+import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.util.*;
+
+import static org.junit.Assert.fail;
+import static org.mockito.MockitoAnnotations.initMocks;
+import static org.powermock.api.mockito.PowerMockito.mockStatic;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@Ignore
+@RunWith(SpringRunner.class)
+public class ReindexServiceTest {
+
+ private final String cursor = "100";
+
+ private final String correlationId = UUID.randomUUID().toString();
+
+ @Mock
+ private StorageService storageService;
+
+ @Mock
+ private Map<String, String> httpHeaders;
+ @Mock
+ private IRequestInfo requestInfo;
+ @Mock
+ private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
+ @Mock
+ private JaxRsDpsLog log;
+ @InjectMocks
+ private ReindexServiceImpl sut;
+
+ private RecordReindexRequest recordReindexRequest;
+ private RecordQueryResponse recordQueryResponse;
+
+ @Before
+ public void setup() {
+ initMocks(this);
+
+ mockStatic(UUID.class);
+
+ recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor(cursor).build();
+ recordQueryResponse = new RecordQueryResponse();
+
+ httpHeaders = new HashMap<>();
+ httpHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
+ httpHeaders.put(DpsHeaders.CORRELATION_ID, correlationId);
+ DpsHeaders standardHeaders = DpsHeaders.createFromMap(httpHeaders);
+ when(requestInfo.getHeaders()).thenReturn(standardHeaders);
+ when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(httpHeaders);
+ }
+
+ @Test
+ public void should_returnNull_givenNullResponseResult_reIndexRecordsTest() {
+ try {
+ recordQueryResponse.setResults(null);
+ when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
+
+ String response = sut.reindexRecords(recordReindexRequest);
+
+ Assert.assertNull(response);
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_returnNull_givenEmptyResponseResult_reIndexRecordsTest() {
+ try {
+ recordQueryResponse.setResults(new ArrayList<>());
+ when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
+
+ String response = sut.reindexRecords(recordReindexRequest);
+
+ Assert.assertNull(response);
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_returnRecordQueryRequestPayload_givenValidResponseResult_reIndexRecordsTest() {
+ try {
+ recordQueryResponse.setCursor(cursor);
+ List<String> results = new ArrayList<>();
+ results.add("test1");
+ recordQueryResponse.setResults(results);
+ when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
+
+ String taskQueuePayload = sut.reindexRecords(recordReindexRequest);
+
+ Assert.assertEquals("{\"kind\":\"tenant:test:test:1.0.0\",\"cursor\":\"100\"}", taskQueuePayload);
+ } catch (Exception e) {
+ fail("Should not throw exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_returnRecordChangedMessage_givenValidResponseResult_reIndexRecordsTest() {
+ try {
+ List<String> results = new ArrayList<>();
+ results.add("test1");
+ recordQueryResponse.setResults(results);
+ when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
+
+ String taskQueuePayload = sut.reindexRecords(recordReindexRequest);
+
+ Assert.assertEquals(String.format("{\"data\":\"[{\\\"id\\\":\\\"test1\\\",\\\"kind\\\":\\\"tenant:test:test:1.0.0\\\",\\\"op\\\":\\\"create\\\"}]\",\"attributes\":{\"slb-correlation-id\":\"%s\"}}", correlationId), taskQueuePayload);
+ } catch (Exception e) {
+ fail("Should not throw exception" + e.getMessage());
+ }
+ }
+}
diff --git a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/StorageServiceTest.java b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/StorageServiceTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..93e90b30e58f53034af05e8580b945c759f85fe9
--- /dev/null
+++ b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/StorageServiceTest.java
@@ -0,0 +1,211 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.service;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentMatchers;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
+import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
+import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
+import org.opengroup.osdu.core.common.model.indexer.Records;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.indexer.service.StorageServiceImpl;
+import org.opengroup.osdu.core.common.model.indexer.JobStatus;
+import org.opengroup.osdu.core.common.model.http.HttpResponse;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.opengroup.osdu.core.common.http.IUrlFetchService;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.springframework.http.HttpStatus;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.lang.reflect.Type;
+import java.net.URISyntaxException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+
+import static org.junit.Assert.*;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@Ignore
+@RunWith(SpringRunner.class)
+public class StorageServiceTest {
+
+ @Mock
+ private IUrlFetchService urlFetchService;
+ @Mock
+ private JobStatus jobStatus;
+ @Mock
+ private JaxRsDpsLog log;
+ @Mock
+ private IRequestInfo requestInfo;
+ @InjectMocks
+ private StorageServiceImpl sut;
+
+ private List<String> ids;
+
+ @Before
+ public void setup() {
+
+ String recordChangedMessages = "[{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"purge\"}," +
+ "{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\",\"kind\":\"tenant1:testindexer1528919679710:well:1.0.0\",\"op\":\"create\"}]";
+
+ when(this.requestInfo.getHeadersMap()).thenReturn(new HashMap<>());
+
+ Type listType = new TypeToken<List<RecordInfo>>() {}.getType();
+
+ List<RecordInfo> msgs = (new Gson()).fromJson(recordChangedMessages, listType);
+ jobStatus.initialize(msgs);
+ ids = Arrays.asList("tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465", "tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465");
+ }
+
+ @Test
+ public void should_return404_givenNullData_getValidStorageRecordsTest() throws URISyntaxException {
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(null);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ should_return404_getValidStorageRecordsTest();
+ }
+
+ @Test
+ public void should_return404_givenEmptyData_getValidStorageRecordsTest() throws URISyntaxException {
+
+ String emptyDataFromStorage = "{\"records\":[],\"notFound\":[]}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(emptyDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ should_return404_getValidStorageRecordsTest();
+ }
+
+ @Test
+ public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsTest() throws URISyntaxException {
+
+ String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"]}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(httpResponse);
+ Records storageRecords = this.sut.getStorageRecords(ids);
+
+ assertEquals(1, storageRecords.getRecords().size());
+ }
+
+ @Test
+ public void should_returnValidResponse_givenValidRecordQueryRequest_getRecordListByKind() throws Exception {
+
+ RecordReindexRequest recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor("100").build();
+
+ HttpResponse httpResponse = new HttpResponse();
+ httpResponse.setBody(new Gson().toJson(recordReindexRequest, RecordReindexRequest.class));
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ RecordQueryResponse recordQueryResponse = this.sut.getRecordsByKind(recordReindexRequest);
+
+ assertEquals("100", recordQueryResponse.getCursor());
+ assertNull(recordQueryResponse.getResults());
+ }
+
+ @Test
+ public void should_returnValidResponse_givenValidKind_getSchemaByKind() throws Exception {
+
+ String validSchemaFromStorage = "{" +
+ " \"kind\": \"tenant:test:test:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"msg\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"references.entity\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]," +
+ " \"ext\": null" +
+ "}";
+ String kind = "tenant:test:test:1.0.0";
+
+ HttpResponse httpResponse = new HttpResponse();
+ httpResponse.setResponseCode(HttpStatus.OK.value());
+ httpResponse.setBody(validSchemaFromStorage);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ String recordSchemaResponse = this.sut.getStorageSchema(kind);
+
+ assertNotNull(recordSchemaResponse);
+ }
+
+ @Test
+ public void should_returnNullResponse_givenAbsentKind_getSchemaByKind() throws Exception {
+
+ String kind = "tenant:test:test:1.0.0";
+
+ HttpResponse httpResponse = new HttpResponse();
+ httpResponse.setResponseCode(HttpStatus.NOT_FOUND.value());
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(httpResponse);
+
+ String recordSchemaResponse = this.sut.getStorageSchema(kind);
+
+ assertNull(recordSchemaResponse);
+ }
+
+ @Test
+ public void should_returnOneValidRecords_givenValidData_getValidStorageRecordsWithInvalidConversionTest() throws URISyntaxException {
+
+ String validDataFromStorage = "{\"records\":[{\"id\":\"testid\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}],\"notFound\":[\"invalid1\"],\"conversionStatuses\": [{\"id\":\"testid\",\"status\":\"ERROR\",\"errors\":[\"conversion error occured\"] } ]}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ Mockito.when(httpResponse.getBody()).thenReturn(validDataFromStorage);
+
+ when(this.urlFetchService.sendRequest(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(httpResponse);
+ Records storageRecords = this.sut.getStorageRecords(ids);
+
+ assertEquals(1, storageRecords.getRecords().size());
+
+ assertEquals(1, storageRecords.getConversionStatuses().get(0).getErrors().size());
+
+ assertEquals("conversion error occured", storageRecords.getConversionStatuses().get(0).getErrors().get(0));
+ }
+
+ private void should_return404_getValidStorageRecordsTest() {
+ try {
+ this.sut.getStorageRecords(ids);
+ fail("Should throw exception");
+ } catch (AppException e) {
+ assertEquals(HttpStatus.NOT_FOUND, e.getError().getCode());
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+}
diff --git a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/TenantInfoServiceTest.java b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/TenantInfoServiceTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..378d7a8a467e06ab994545a06f56dc295ad906db
--- /dev/null
+++ b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/TenantInfoServiceTest.java
@@ -0,0 +1,92 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.ibm.service;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.Mockito.when;
+
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
+import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
+import org.opengroup.osdu.indexer.ibm.di.TenantInfoService;
+import org.opengroup.osdu.indexer.ibm.util.IHeadersInfo;
+import org.springframework.http.HttpHeaders;
+import org.springframework.test.context.junit4.SpringRunner;
+
+@RunWith(SpringRunner.class)
+public class TenantInfoServiceTest {
+
+ private static final String HEADER_NAME = "ANY_HEADER";
+ private static final String HEADER_VALUE = "ANY_VALUE";
+
+ @Mock
+ private ITenantFactory tenantFactory;
+ @Mock
+ private IHeadersInfo headersInfo;
+
+ @InjectMocks
+ private TenantInfoService sut;
+
+ @Mock
+ private TenantInfo info;
+
+ @Mock
+ private HttpHeaders httpHeaders;
+
+ @InjectMocks
+ private DpsHeaders HEADERS;
+
+ @Before
+ public void setup() {
+ HEADERS.put(HEADER_NAME, HEADER_VALUE);
+ }
+
+ @Ignore
+ @Test
+ public void should_return_validTenant_given_validAccountId() {
+
+ when(this.info.getName()).thenReturn("tenant1");
+ when(tenantFactory.getTenantInfo("tenant1")).thenReturn(info);
+
+ when(this.headersInfo.getHeaders()).thenReturn(HEADERS);
+
+ when(this.headersInfo.getPartitionId()).thenReturn("tenant1");
+
+ when(this.sut.getTenantInfo()).thenReturn(info);
+
+ assertNotNull(this.sut.getTenantInfo());
+ assertEquals("tenant1", this.sut.getTenantInfo().getName());
+ }
+
+ @Ignore
+ @Test(expected = AppException.class)
+ public void should_throwException_given_invalidAccountId() {
+
+ when(this.info.getName()).thenReturn("tenant2");
+ when(tenantFactory.getTenantInfo("tenant1")).thenReturn(null);
+
+ when(this.sut.getTenantInfo()).thenReturn(info);
+
+ assertNotNull(this.sut.getTenantInfo());
+ }
+}
\ No newline at end of file
diff --git a/testing/indexer-test-aws/pom.xml b/testing/indexer-test-aws/pom.xml
index 800ef6ef3cc40354fe5c420e8bcb8d5b3002e835..551f70b808eca63f6e409c470c43e818e0db58b9 100644
--- a/testing/indexer-test-aws/pom.xml
+++ b/testing/indexer-test-aws/pom.xml
@@ -25,6 +25,24 @@
<version>0.0.2-SNAPSHOT</version>
<packaging>jar</packaging>
+ <repositories>
+ <repository>
+ <id>${gitlab-server}</id>
+ <url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</url>
+ </repository>
+ </repositories>
+
+ <distributionManagement>
+ <repository>
+ <id>${gitlab-server}</id>
+ <url>https://community.opengroup.org/api/v4/projects/25/packages/maven</url>
+ </repository>
+ <snapshotRepository>
+ <id>${gitlab-server}</id>
+ <url>https://community.opengroup.org/api/v4/projects/25/packages/maven</url>
+ </snapshotRepository>
+ </distributionManagement>
+
<properties>
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
@@ -43,15 +61,8 @@
<!-- AWS specific packages -->
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
- <artifactId>aws-osdu-util</artifactId>
- <version>0.0.9</version>
- </dependency>
-
- <!-- AWS managed packages -->
- <dependency>
- <groupId>com.github.awslabs</groupId>
- <artifactId>aws-request-signing-apache-interceptor</artifactId>
- <version>deb7941</version>
+ <artifactId>os-core-lib-aws</artifactId>
+ <version>0.0.10</version>
</dependency>
<!-- Testing -->
@@ -138,44 +149,12 @@
</dependency>
</dependencies>
-
- <repositories>
- <repository>
- <id>${gitlab-server}</id>
- <url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</url>
- </repository>
-
- <repository>
- <id>jitpack.io</id>
- <url>https://jitpack.io</url>
- </repository>
- </repositories>
-
- <distributionManagement>
- <repository>
- <id>${gitlab-server}</id>
- <url>https://community.opengroup.org/api/v4/projects/25/packages/maven</url>
- </repository>
- <snapshotRepository>
- <id>${gitlab-server}</id>
- <url>https://community.opengroup.org/api/v4/projects/25/packages/maven</url>
- </snapshotRepository>
- </distributionManagement>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.5</version>
- <configuration>
- <systemPropertyVariables>
- <AWS_COGNITO_CLIENT_ID>3rmgmg8mup281ttc1mbut1pimc</AWS_COGNITO_CLIENT_ID>
- <AWS_COGNITO_AUTH_FLOW>USER_PASSWORD_AUTH</AWS_COGNITO_AUTH_FLOW>
- <AWS_COGNITO_AUTH_PARAMS_USER>test-user-with-access@testing.com</AWS_COGNITO_AUTH_PARAMS_USER>
- <AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS>test-user-without-access@testing.com</AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS>
- <AWS_COGNITO_AUTH_PARAMS_PASSWORD>Password123*</AWS_COGNITO_AUTH_PARAMS_PASSWORD>
- </systemPropertyVariables>
- </configuration>
</plugin>
</plugins>
</build>
diff --git a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/ElasticUtilsAws.java b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/ElasticUtilsAws.java
index fff624c682d59709622511e86e0ce499191f8ff1..77a0440064ca1c7c41194dd3ca45a93613ac6b5f 100644
--- a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/ElasticUtilsAws.java
+++ b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/ElasticUtilsAws.java
@@ -14,24 +14,36 @@
package org.opengroup.osdu.util;
-import com.amazonaws.auth.AWS4Signer;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.http.AWSRequestSigningApacheInterceptor;
+import org.apache.http.Header;
import org.apache.http.HttpHost;
-import org.apache.http.HttpRequestInterceptor;
+import org.apache.http.message.BasicHeader;
import org.elasticsearch.client.RestClient;
-import org.elasticsearch.client.RestHighLevelClient;
-import org.opengroup.osdu.core.aws.iam.IAMConfig;
+import org.elasticsearch.client.RestClientBuilder;
public class ElasticUtilsAws extends ElasticUtils {
+ private static final int REST_CLIENT_CONNECT_TIMEOUT = 60000;
+ private static final int REST_CLIENT_SOCKET_TIMEOUT = 60000;
+ private static final int REST_CLIENT_RETRY_TIMEOUT = 60000;
+
@Override
- protected RestHighLevelClient createClient(String username, String password, String host) {
- AWSCredentialsProvider credentials = new IAMConfig().amazonAWSCredentials();
- AWS4Signer signer = new AWS4Signer();
- signer.setServiceName(username);
- signer.setRegionName(password);
- HttpRequestInterceptor interceptor = new AWSRequestSigningApacheInterceptor(username, signer, credentials);
- return new RestHighLevelClient(RestClient.builder(HttpHost.create(host)).setHttpClientConfigCallback(configCallBack -> configCallBack.addInterceptorLast(interceptor)));
+ public RestClientBuilder createClientBuilder(String host, String usernameAndPassword, int port) {
+ RestClientBuilder builder = RestClient.builder(new HttpHost(host, port, "https"));
+ builder.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(REST_CLIENT_CONNECT_TIMEOUT)
+ .setSocketTimeout(REST_CLIENT_SOCKET_TIMEOUT));
+ builder.setMaxRetryTimeoutMillis(REST_CLIENT_RETRY_TIMEOUT);
+ builder.setHttpClientConfigCallback(httpAsyncClientBuilder -> httpAsyncClientBuilder.setSSLHostnameVerifier((s, sslSession) -> true));
+
+ Header[] defaultHeaders = new Header[]{
+ new BasicHeader("client.transport.nodes_sampler_interval", "30s"),
+ new BasicHeader("client.transport.ping_timeout", "30s"),
+ new BasicHeader("client.transport.sniff", "false"),
+ new BasicHeader("request.headers.X-Found-Cluster", Config.getElasticHost()),
+ new BasicHeader("cluster.name", Config.getElasticHost()),
+ new BasicHeader("xpack.security.transport.ssl.enabled", Boolean.toString(true))
+ };
+
+ builder.setDefaultHeaders(defaultHeaders);
+ return builder;
}
}
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/Config.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/Config.java
index c76cb4307f74d17257d0c540c8dd5f16be784421..f015548d59648765ae22ea33a698299622a13b70 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/Config.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/Config.java
@@ -22,6 +22,10 @@ public class Config {
private static final String DEFAULT_ENTITLEMENTS_DOMAIN = "";
+ public static int getPort() {
+ return Integer.parseInt(getEnvironmentVariableOrDefaultValue("ELASTIC_PORT", String.valueOf(PORT)));
+ }
+
public static String getOtherRelevantDataCountries() {
return getEnvironmentVariableOrDefaultValue("OTHER_RELEVANT_DATA_COUNTRIES", DEFAULT_OTHER_RELEVANT_DATA_COUNTRIES);
}
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
index fa8e8f716b3031a86a0d417b3feeb3ebbbf9fdf8..230f44d2e13228dac9e0d8a39ff110570220103c 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
@@ -257,12 +257,24 @@ public class ElasticUtils {
}
- protected RestHighLevelClient createClient(String username, String password, String host) {
+ private RestHighLevelClient createClient(String username, String password, String host) {
RestHighLevelClient restHighLevelClient;
int port = Config.PORT;
try {
String rawString = String.format("%s:%s", username, password);
+
+ RestClientBuilder builder = createClientBuilder(host,rawString, port);
+
+ restHighLevelClient = new RestHighLevelClient(builder);
+
+ } catch (Exception e) {
+ throw new AssertionError("Setup elastic error");
+ }
+ return restHighLevelClient;
+ }
+
+ public RestClientBuilder createClientBuilder(String host, String usernameAndPassword, int port) {
RestClientBuilder builder = RestClient.builder(new HttpHost(host, port, "https"));
builder.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(REST_CLIENT_CONNECT_TIMEOUT)
.setSocketTimeout(REST_CLIENT_SOCKET_TIMEOUT));
@@ -275,15 +287,12 @@ public class ElasticUtils {
new BasicHeader("request.headers.X-Found-Cluster", Config.getElasticHost()),
new BasicHeader("cluster.name", Config.getElasticHost()),
new BasicHeader("xpack.security.transport.ssl.enabled", Boolean.toString(true)),
- new BasicHeader("Authorization", String.format("Basic %s", Base64.getEncoder().encodeToString(rawString.getBytes()))),
+ new BasicHeader("Authorization", String.format("Basic %s", Base64.getEncoder().encodeToString(usernameAndPassword.getBytes()))),
};
builder.setDefaultHeaders(defaultHeaders);
- restHighLevelClient = new RestHighLevelClient(builder);
-
- } catch (Exception e) {
- throw new AssertionError("Setup elastic error");
- }
- return restHighLevelClient;
+ return builder;
}
+
+
}
\ No newline at end of file
diff --git a/testing/indexer-test-ibm/pom.xml b/testing/indexer-test-ibm/pom.xml
new file mode 100644
index 0000000000000000000000000000000000000000..4ff1ccc6d0bb0f83c9fbba32ef04c3191a328990
--- /dev/null
+++ b/testing/indexer-test-ibm/pom.xml
@@ -0,0 +1,208 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>org.opengroup.osdu.indexer</groupId>
+ <artifactId>indexer-test-ibm</artifactId>
+ <version>0.0.2</version>
+ <packaging>jar</packaging>
+
+ <properties>
+ <maven.compiler.target>1.8</maven.compiler.target>
+ <maven.compiler.source>1.8</maven.compiler.source>
+ <cucumber.version>1.2.5</cucumber.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.google.api-client</groupId>
+ <artifactId>google-api-client</artifactId>
+ <version>1.28.0</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.opengroup.osdu.indexer</groupId>
+ <artifactId>indexer-test-core</artifactId>
+ <version>0.0.2-SNAPSHOT</version>
+ </dependency>
+
+ <!-- Cucumber -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.12</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>info.cukes</groupId>
+ <artifactId>cucumber-java</artifactId>
+ <version>${cucumber.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>info.cukes</groupId>
+ <artifactId>cucumber-junit</artifactId>
+ <version>${cucumber.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- Gson: Java to Json conversion -->
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>2.8.5</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.jaxrs</groupId>
+ <artifactId>jackson-jaxrs-json-provider</artifactId>
+ <version>2.9.9</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.glassfish</groupId>
+ <artifactId>javax.json</artifactId>
+ <version>1.1.4</version>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-client</artifactId>
+ <version>1.19.4</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ <version>1.18.2</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>2.6</version>
+ </dependency>
+
+ <!--Elasticsearch-->
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch</artifactId>
+ <version>6.6.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-client</artifactId>
+ <version>6.6.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-high-level-client</artifactId>
+ <version>6.6.2</version>
+ </dependency>
+
+ <!--Logging-->
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-to-slf4j</artifactId>
+ <version>2.11.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-jdk14</artifactId>
+ <version>1.8.0-beta4</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>27.1-jre</version>
+ </dependency>
+ </dependencies>
+
+ <repositories>
+ <repository>
+ <id>${gitlab-server}</id>
+ <url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</url>
+ </repository>
+ </repositories>
+
+ <distributionManagement>
+ <repository>
+ <id>${gitlab-server}</id>
+ <url>https://community.opengroup.org/api/v4/projects/25/packages/maven</url>
+ </repository>
+ <snapshotRepository>
+ <id>${gitlab-server}</id>
+ <url>https://community.opengroup.org/api/v4/projects/25/packages/maven</url>
+ </snapshotRepository>
+ </distributionManagement>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.22.2</version>
+ <executions>
+ <execution>
+ <phase>integration-test</phase>
+ <goals>
+ <goal>test</goal>
+ </goals>
+ <configuration>
+ <excludes>
+ <exclude>none</exclude>
+ </excludes>
+ <includes>
+ <include>**/index</include>
+ </includes>
+ </configuration>
+ </execution>
+ </executions>
+ <configuration>
+ <trimStackTrace>false</trimStackTrace>
+ <systemPropertyVariables>
+
+ <KEYCLOAK_URL>keycloak-osdu-r2.osduadev-a1c3eaf78a86806e299f5f3f207556f0-0000.us-south.containers.appdomain.cloud</KEYCLOAK_URL>
+ <KEYCLOAK_REALM>OSDU</KEYCLOAK_REALM>
+ <KEYCLOAK_CLIENT_ID>osdu-login</KEYCLOAK_CLIENT_ID>
+ <AUTH_USER_ACCESS>osdu-user</AUTH_USER_ACCESS>
+ <AUTH_USER_ACCESS_PASSWORD>CHANGE_ME</AUTH_USER_ACCESS_PASSWORD>
+
+ <ENTITLEMENTS_DOMAIN>ibm.com</ENTITLEMENTS_DOMAIN>
+ <OTHER_RELEVANT_DATA_COUNTRIES>US</OTHER_RELEVANT_DATA_COUNTRIES>
+ <!-- must be a valid, existing tag -->
+ <LEGAL_TAG>opendes-public-usa-dataset</LEGAL_TAG>
+
+ <!-- tenant1 MUSTG be opendes to work -->
+ <DEFAULT_DATA_PARTITION_ID_TENANT1>opendes</DEFAULT_DATA_PARTITION_ID_TENANT1>
+ <DEFAULT_DATA_PARTITION_ID_TENANT2>tenant2</DEFAULT_DATA_PARTITION_ID_TENANT2>
+
+ <!-- test against OpenShift -->
+ <STORAGE_HOST>https://storage.osdu-r2-dev-a1c3eaf78a86806e299f5f3f207556f0-0000.us-south.containers.appdomain.cloud/api/storage/v2/</STORAGE_HOST>
+
+ <!-- test locally -->
+ <!--
+ <STORAGE_HOST>http://localhost:8082/api/storage/v2/</STORAGE_HOST>
+ -->
+ <ELASTIC_HOST>CHANGE_ME.us-south.containers.appdomain.cloud</ELASTIC_HOST>
+ <ELASTIC_USER_NAME>CHANGE_ME</ELASTIC_USER_NAME>
+ <ELASTIC_PASSWORD>CHANGE_ME</ELASTIC_PASSWORD>
+ <ELASTIC_PORT>443</ELASTIC_PORT>
+
+
+ </systemPropertyVariables>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..4978ddfccad197628432d4ccb8b5985b4af5017e
--- /dev/null
+++ b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java
@@ -0,0 +1,13 @@
+package org.opengroup.osdu.step_definitions.index.record;
+
+import cucumber.api.CucumberOptions;
+import cucumber.api.junit.Cucumber;
+import org.junit.runner.RunWith;
+
+@RunWith(Cucumber.class)
+@CucumberOptions(
+ features = "classpath:features/indexrecord/IndexRecord.feature",
+ glue = {"classpath:org.opengroup.osdu.step_definitions/index/record"},
+ plugin = {"pretty", "junit:target/cucumber-reports/TEST-indexrecord.xml"})
+public class RunTest {
+}
\ No newline at end of file
diff --git a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
new file mode 100644
index 0000000000000000000000000000000000000000..126934d41fd3689a45218c6c78fe575741c33aa3
--- /dev/null
+++ b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -0,0 +1,53 @@
+package org.opengroup.osdu.step_definitions.index.record;
+
+import org.opengroup.osdu.common.RecordSteps;
+import org.opengroup.osdu.util.ElasticUtilsIBM;
+import org.opengroup.osdu.util.IBMHTTPClient;
+
+import cucumber.api.DataTable;
+import cucumber.api.Scenario;
+import cucumber.api.java.Before;
+import cucumber.api.java.en.Given;
+import cucumber.api.java.en.Then;
+import cucumber.api.java.en.When;
+import lombok.extern.java.Log;
+
+@Log
+public class Steps extends RecordSteps {
+
+ public Steps() {
+ super(new IBMHTTPClient(), new ElasticUtilsIBM());
+ }
+
+ @Before
+ public void before(Scenario scenario) {
+ this.scenario = scenario;
+ this.httpClient = new IBMHTTPClient();
+ }
+
+ @Given("^the schema is created with the following kind$")
+ public void the_schema_is_created_with_the_following_kind(DataTable dataTable) {
+ super.the_schema_is_created_with_the_following_kind(dataTable);
+ }
+
+ @When("^I ingest records with the \"(.*?)\" with \"(.*?)\" for a given \"(.*?)\"$")
+ public void i_ingest_records_with_the_for_a_given(String record, String dataGroup, String kind) {
+ super.i_ingest_records_with_the_for_a_given(record, dataGroup, kind);
+ }
+
+ @Then("^I should get the (\\d+) documents for the \"([^\"]*)\" in the Elastic Search$")
+ public void i_should_get_the_documents_for_the_in_the_Elastic_Search(int expectedCount, String index) throws Throwable {
+ super.i_should_get_the_documents_for_the_in_the_Elastic_Search(expectedCount, index);
+ }
+
+ @Then("^I should get the elastic \"(.*?)\" for the \"([^\"]*)\" and \"([^\"]*)\" in the Elastic Search$")
+ public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String type, String index) throws Throwable {
+ super.i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(expectedMapping, type, index);
+ }
+
+ @Then("^I should get the (\\d+) documents for the \"([^\"]*)\" in the Elastic Search with out \"(.*?)\"$")
+ public void iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(int expectedCount, String index, String skippedAttributes) throws Throwable {
+ super.iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(expectedCount, index, skippedAttributes);
+ }
+
+}
\ No newline at end of file
diff --git a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/util/ElasticUtilsIBM.java b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/util/ElasticUtilsIBM.java
new file mode 100644
index 0000000000000000000000000000000000000000..5174b7c932b7629c6f67747b3c45525ee2cb0812
--- /dev/null
+++ b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/util/ElasticUtilsIBM.java
@@ -0,0 +1,106 @@
+// Copyright 2020 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.util;
+
+import java.security.SecureRandom;
+import java.security.cert.X509Certificate;
+import java.util.Base64;
+
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
+
+import org.apache.http.Header;
+import org.apache.http.HttpHost;
+import org.apache.http.conn.ssl.SSLSocketFactory;
+import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
+import org.apache.http.message.BasicHeader;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestClientBuilder;
+import org.elasticsearch.client.RestClientBuilder.HttpClientConfigCallback;
+import org.elasticsearch.client.RestHighLevelClient;
+
+import lombok.extern.java.Log;
+
+/**
+ * All util methods to use elastic apis for tests
+ * It should be used only in the Setup or TearDown phase of the test
+ */
+@Log
+public class ElasticUtilsIBM extends ElasticUtils {
+
+ private static final int REST_CLIENT_CONNECT_TIMEOUT = 5000;
+ private static final int REST_CLIENT_SOCKET_TIMEOUT = 60000;
+ private static final int REST_CLIENT_RETRY_TIMEOUT = 60000;
+
+ public ElasticUtilsIBM() {
+ super();
+ }
+
+ protected RestHighLevelClient createClient(String username, String password, String host) {
+
+ RestHighLevelClient restHighLevelClient;
+ int port = Config.getPort();
+ try {
+ String rawString = String.format("%s:%s", username, password);
+ RestClientBuilder builder = RestClient.builder(new HttpHost(host, port, "https"));
+ builder.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(REST_CLIENT_CONNECT_TIMEOUT)
+ .setSocketTimeout(REST_CLIENT_SOCKET_TIMEOUT));
+ builder.setMaxRetryTimeoutMillis(REST_CLIENT_RETRY_TIMEOUT);
+
+ Header[] defaultHeaders = new Header[]{
+ new BasicHeader("client.transport.nodes_sampler_interval", "30s"),
+ new BasicHeader("client.transport.ping_timeout", "30s"),
+ new BasicHeader("client.transport.sniff", "false"),
+ new BasicHeader("request.headers.X-Found-Cluster", Config.getElasticHost()),
+ new BasicHeader("cluster.name", Config.getElasticHost()),
+ new BasicHeader("xpack.security.transport.ssl.enabled", Boolean.toString(true)),
+ new BasicHeader("Authorization", String.format("Basic %s", Base64.getEncoder().encodeToString(rawString.getBytes()))),
+ };
+
+
+ SSLContext sslContext = SSLContext.getInstance("SSL");
+ // set up a TrustManager that trusts everything
+ sslContext.init(null, new TrustManager[]{new X509TrustManager() {
+ public X509Certificate[] getAcceptedIssuers() {
+ return null;
+ }
+
+ public void checkClientTrusted(X509Certificate[] certs, String authType) {
+ }
+
+ public void checkServerTrusted(X509Certificate[] certs, String authType) {
+ }
+ }}, new SecureRandom());
+
+ builder.setHttpClientConfigCallback(new HttpClientConfigCallback() {
+
+ public HttpAsyncClientBuilder customizeHttpClient(
+ HttpAsyncClientBuilder httpClientBuilder) {
+ return httpClientBuilder
+ .setHostnameVerifier(SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER)
+ .setSSLContext(sslContext);
+ }
+ })
+ .setDefaultHeaders(defaultHeaders);
+
+ restHighLevelClient = new RestHighLevelClient(builder);
+
+ } catch (Exception e) {
+ throw new AssertionError("Setup elastic error");
+ }
+ return restHighLevelClient;
+ }
+}
\ No newline at end of file
diff --git a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/util/IBMHTTPClient.java b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/util/IBMHTTPClient.java
new file mode 100644
index 0000000000000000000000000000000000000000..c313c3833f037039a61f939ad31af6a04870f031
--- /dev/null
+++ b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/util/IBMHTTPClient.java
@@ -0,0 +1,39 @@
+// Copyright 2020 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.util;
+
+import lombok.ToString;
+import lombok.extern.java.Log;
+
+import com.google.common.base.Strings;
+
+@Log
+@ToString
+public class IBMHTTPClient extends HTTPClient {
+ private static String token = null;
+
+ @Override
+ public synchronized String getAccessToken() {
+ if(Strings.isNullOrEmpty(token)) {
+ try {
+ token = "Bearer " + IdentityUtilIBM.getAccessToken();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ return token;
+ }
+
+}
\ No newline at end of file
diff --git a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/util/IdentityUtilIBM.java b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/util/IdentityUtilIBM.java
new file mode 100644
index 0000000000000000000000000000000000000000..1f2fdb06f2a0a28de33481be957bfc1a875716f3
--- /dev/null
+++ b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/util/IdentityUtilIBM.java
@@ -0,0 +1,32 @@
+// Copyright 2020 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.util;
+
+import java.io.IOException;
+
+public class IdentityUtilIBM {
+
+ public static String getAccessToken(){
+ try {
+ String user = System.getProperty("AUTH_USER_ACCESS");
+ String pass = System.getProperty("AUTH_USER_ACCESS_PASSWORD");
+ return KeyCloakProvider.getToken(user, pass);
+ } catch (IOException e) {
+ e.printStackTrace();
+ return null;
+ }
+ }
+
+}
diff --git a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/util/KeyCloakProvider.java b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/util/KeyCloakProvider.java
new file mode 100644
index 0000000000000000000000000000000000000000..d890d9a99b609e25dbe777c5f14fec9aa8f1846b
--- /dev/null
+++ b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/util/KeyCloakProvider.java
@@ -0,0 +1,144 @@
+// Copyright 2020 IBM Corp. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.util;
+
+import java.io.BufferedReader;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.UnsupportedEncodingException;
+import java.net.URL;
+import java.net.URLEncoder;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.X509Certificate;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLSession;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
+
+import com.google.gson.Gson;
+import com.google.gson.JsonObject;
+
+public class KeyCloakProvider {
+
+ private static String url;
+ private static String realm;
+ private static String client_id;
+ private static String grant_type = "password";
+
+ static {
+ disableSslVerification();
+ url = System.getProperty("KEYCLOAK_URL");
+ realm = System.getProperty("KEYCLOAK_REALM");
+ client_id = System.getProperty("KEYCLOAK_CLIENT_ID");
+ }
+
+ private static void disableSslVerification() {
+ try
+ {
+ // Create a trust manager that does not validate certificate chains
+ TrustManager[] trustAllCerts = new TrustManager[] {new X509TrustManager() {
+ public java.security.cert.X509Certificate[] getAcceptedIssuers() {
+ return null;
+ }
+ public void checkClientTrusted(X509Certificate[] certs, String authType) {
+ }
+ public void checkServerTrusted(X509Certificate[] certs, String authType) {
+ }
+ }
+ };
+
+ // Install the all-trusting trust manager
+ SSLContext sc = SSLContext.getInstance("SSL");
+ sc.init(null, trustAllCerts, new java.security.SecureRandom());
+ HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory());
+
+ // Create all-trusting host name verifier
+ HostnameVerifier allHostsValid = new HostnameVerifier() {
+ public boolean verify(String hostname, SSLSession session) {
+ return true;
+ }
+ };
+
+ // Install the all-trusting host verifier
+ HttpsURLConnection.setDefaultHostnameVerifier(allHostsValid);
+ } catch (NoSuchAlgorithmException e) {
+ e.printStackTrace();
+ } catch (KeyManagementException e) {
+ e.printStackTrace();
+ }
+ }
+
+
+ public static String getToken(String user, String pwd) throws IOException {
+ String token_endpoint = String.format("https://%s/auth/realms/%s/protocol/openid-connect/token", url, realm);
+ URL url = new URL(token_endpoint);
+ HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
+ con.setRequestMethod("POST");
+ con.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
+
+ Map<String, String> parameters = new HashMap<>();
+ parameters.put("grant_type", grant_type);
+ parameters.put("client_id", client_id);
+ parameters.put("username", user);
+ parameters.put("password", pwd);
+
+ con.setDoOutput(true);
+ DataOutputStream out = new DataOutputStream(con.getOutputStream());
+ out.writeBytes(getParamsString(parameters));
+ out.flush();
+ out.close();
+
+ BufferedReader in = new BufferedReader(
+ new InputStreamReader(con.getInputStream()));
+ String inputLine;
+ StringBuffer content = new StringBuffer();
+ while ((inputLine = in.readLine()) != null) {
+ content.append(inputLine);
+ }
+ in.close();
+
+ con.disconnect();
+
+ Gson gson = new Gson();
+ JsonObject jobj = gson.fromJson(content.toString(), JsonObject.class);
+ String token = jobj.get("access_token").getAsString();
+ return token;
+ }
+
+ private static String getParamsString(Map<String, String> params)
+ throws UnsupportedEncodingException {
+ StringBuilder result = new StringBuilder();
+
+ for (Map.Entry<String, String> entry : params.entrySet()) {
+ result.append(URLEncoder.encode(entry.getKey(), "UTF-8"));
+ result.append("=");
+ result.append(URLEncoder.encode(entry.getValue(), "UTF-8"));
+ result.append("&");
+ }
+
+ String resultString = result.toString();
+ return resultString.length() > 0
+ ? resultString.substring(0, resultString.length() - 1)
+ : resultString;
+ }
+
+}
diff --git a/testing/pom.xml b/testing/pom.xml
new file mode 100644
index 0000000000000000000000000000000000000000..1bac11ececef04b5fad0f21269dd96c5b4f59a97
--- /dev/null
+++ b/testing/pom.xml
@@ -0,0 +1,58 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Copyright 2017-2019, The Open Group
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>org.opengroup.osdu</groupId>
+ <artifactId>indexer-test</artifactId>
+ <version>0.0.5-SNAPSHOT</version>
+ <description>Indexer Service Integration Test Root Project</description>
+ <packaging>pom</packaging>
+
+ <licenses>
+ <license>
+ <name>Apache License, Version 2.0</name>
+ <url>https://www.apache.org/licenses/LICENSE-2.0.txt</url>
+ <distribution>repo</distribution>
+ </license>
+ </licenses>
+
+ <modules>
+ <module>indexer-test-core</module>
+ <module>indexer-test-aws</module>
+ <module>indexer-test-azure</module>
+ <module>indexer-test-gcp</module>
+ </modules>
+
+ <repositories>
+ <repository>
+ <id>${gitlab-server}</id>
+ <url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</url>
+ </repository>
+ </repositories>
+
+ <distributionManagement>
+ <repository>
+ <id>${gitlab-server}</id>
+ <url>https://community.opengroup.org/api/v4/projects/19/packages/maven</url>
+ </repository>
+ <snapshotRepository>
+ <id>${gitlab-server}</id>
+ <url>https://community.opengroup.org/api/v4/projects/19/packages/maven</url>
+ </snapshotRepository>
+ </distributionManagement>
+</project>