diff --git a/indexer-core/pom.xml b/indexer-core/pom.xml
index dc27ed2b5d7e57c989eef455a6128ceb9d84ad7d..1e60e821881c2bbd2d4c983d1309c4c8a9685427 100644
--- a/indexer-core/pom.xml
+++ b/indexer-core/pom.xml
@@ -198,23 +198,18 @@
</dependency>
</dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-maven-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>repackage</goal>
- </goals>
- <configuration>
- <classifier>spring-boot</classifier>
- <mainClass>org.opengroup.osdu.indexer.IndexerApplication</mainClass>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
+ <build>
+ <defaultGoal>test</defaultGoal>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.4.2</version>
+ <configuration>
+ <useSystemClassLoader>false</useSystemClassLoader>
+ <threadCount>1</threadCount>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
</project>
diff --git a/pom.xml b/pom.xml
index 1c48d6d26a6d804298e0c092e76a2692c8eae22d..7e99f7f610cbb79be5fb388f3a34411032c2fb66 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1,5 +1,7 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
@@ -41,10 +43,23 @@
</snapshots>
</repository>
</repositories>
-
+
+ <distributionManagement>
+ <repository>
+ <id>dev-azure-com-slb-des-ext-collaboration-os-core</id>
+ <url>https://pkgs.dev.azure.com/slb-des-ext-collaboration/_packaging/os-core/maven/v1</url>
+ <releases>
+ <enabled>true</enabled>
+ </releases>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ </repository>
+ </distributionManagement>
+
<profiles>
<profile>
- <id>indexer-core</id>
+ <id>root</id>
<activation>
<!-- this profile is active by default -->
<activeByDefault>true</activeByDefault>
@@ -78,18 +93,6 @@
<module>provider/indexer-gcp</module>
</modules>
</profile>
- <profile>
- <id>indexer-aws</id>
- <activation>
- <property>
- <name>provider</name>
- <value>indexer-aws</value>
- </property>
- </activation>
- <modules>
- <module>provider/indexer-aws</module>
- </modules>
- </profile>
</profiles>
</project>
diff --git a/provider/indexer-aws/CloudFormation/Automated/elasticsearch.yml b/provider/indexer-aws/CloudFormation/Automated/elasticsearch.yml
new file mode 100644
index 0000000000000000000000000000000000000000..9a2844823459eb232b9745549aab384c0d4168b6
--- /dev/null
+++ b/provider/indexer-aws/CloudFormation/Automated/elasticsearch.yml
@@ -0,0 +1,203 @@
+AWSTemplateFormatVersion: 2010-09-09
+Description: >-
+ CloudFormation template for creating the resources used for the tenant info database for OSDU.
+ It creates the DynamoDB table and the API Gateway endpoints.
+
+Parameters:
+ Environment:
+ Description: An environment name that will be prefixed to resource names.
+ Type: String
+ AllowedValues:
+ - dev
+ - uat
+ - prod
+ ConstraintDescription: Can only be "dev/uat/prod"
+ Default: dev
+
+ Region:
+ Description: The AWS region to deploy the resources to.
+ Type: String
+ Default: us-east-1
+
+ ElasticsearchDomainName:
+ Description: The name of the Elasticsearch domain. Will be prefixed with the environment name.
+ Type: String
+ MinLength: '1'
+ MaxLength: '64'
+ AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
+ ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
+ Default: osdu-indexer
+
+ ElasticsearchNodeInstanceType:
+ Description: The instance type for the main Elasticsearch nodes.
+ ConstraintDescription: Must be a valid instance type from the list of allowed values.
+ Default: t2.medium.elasticsearch
+ AllowedValues:
+ - t2.small.elasticsearch
+ - t2.medium.elasticsearch
+ - m5.large.elasticsearch
+ - m5.xlarge.elasticsearch
+ - m5.2xlarge.elasticsearch
+ - m5.4xlarge.elasticsearch
+ - m5.12xlarge.elasticsearch
+ - c5.large.elasticsearch
+ - c5.xlarge.elasticsearch
+ - c5.2xlarge.elasticsearch
+ - c5.4xlarge.elasticsearch
+ - c5.9xlarge.elasticsearch
+ - c5.18xlarge.elasticsearch
+ - r5.large.elasticsearch
+ - r5.xlarge.elasticsearch
+ - r5.2xlarge.elasticsearch
+ - r5.4xlarge.elasticsearch
+ - r5.12xlarge.elasticsearch
+ - i3.large.elasticsearch
+ - i3.xlarge.elasticsearch
+ - i3.2xlarge.elasticsearch
+ - i3.4xlarge.elasticsearch
+ - i3.8xlarge.elasticsearch
+ - i3.16xlarge.elasticsearch
+ Type: String
+
+ DedicatedMasterInstanceType:
+ Description: >
+ The instance type for the dedicated master nodes. These nodes perform cluster management
+ tasks, but doesn't hold data or respond to data upload requests.
+ ConstraintDescription: Must be a valid instance type from the list of allowed values.
+ Default: t2.medium.elasticsearch
+ AllowedValues:
+ - t2.small.elasticsearch
+ - t2.medium.elasticsearch
+ - m5.large.elasticsearch
+ - m5.xlarge.elasticsearch
+ - m5.2xlarge.elasticsearch
+ - m5.4xlarge.elasticsearch
+ - m5.12xlarge.elasticsearch
+ - c5.large.elasticsearch
+ - c5.xlarge.elasticsearch
+ - c5.2xlarge.elasticsearch
+ - c5.4xlarge.elasticsearch
+ - c5.9xlarge.elasticsearch
+ - c5.18xlarge.elasticsearch
+ - r5.large.elasticsearch
+ - r5.xlarge.elasticsearch
+ - r5.2xlarge.elasticsearch
+ - r5.4xlarge.elasticsearch
+ - r5.12xlarge.elasticsearch
+ - i3.large.elasticsearch
+ - i3.xlarge.elasticsearch
+ - i3.2xlarge.elasticsearch
+ - i3.4xlarge.elasticsearch
+ - i3.8xlarge.elasticsearch
+ - i3.16xlarge.elasticsearch
+ Type: String
+
+ NumberOfElasticsearchNodes:
+ Description: An integer value specifying the number of Elasticsearch primary nodes in the cluster.
+ Type: Number
+ Default: 2
+ MinValue: 1
+ MaxValue: 40
+
+ NumberOfDedicatedMasterNodes:
+ Description: An integer value specifying the number of dedicated master nodes.
+ Type: Number
+ Default: 2
+ MinValue: 2
+ MaxValue: 5
+
+ ZoneAwarenessEnabled:
+ Description: >
+ When Zone Awareness is enabled, Elasticsearch allocates the nodes and replica
+ index shards that belong to a cluster across multiple AZs in the deployment region.
+ Type: String
+ AllowedValues:
+ - true
+ - false
+ Default: false
+
+ ElasticsearchVersion:
+ Description: >
+ The version of Elasticsearch to deploy on the cluster. Defaults to 6.8. Note
+ that an update requires a full replacement of the Elasticsearch cluster.
+ Type: String
+ AllowedValues:
+ - 1.5
+ - 2.3
+ - 5.1
+ - 5.3
+ - 5.5
+ - 5.6
+ - 6.0
+ - 6.2
+ - 6.3
+ - 6.4
+ - 6.5
+ - 6.6
+ - 6.8
+ - 6.8
+ - 7.1
+ Default: 6.8
+
+ EBSVolumeSize:
+ Description: >
+ The size of the EBS volume (per instance; total cluster size = EBS volume size x Instance count)
+ Maximum size varies by instance type, from 35GiB for t2 instances, up to 12TiB for r5.12xlarge.
+ Type: Number
+ Default: 10
+ MinValue: 10
+ MaxValue: 12000
+
+Resources:
+ ElasticsearchDomain:
+ Type: AWS::Elasticsearch::Domain
+ Properties:
+ DomainName: !Sub ${Environment}-${ElasticsearchDomainName}
+ ElasticsearchVersion: !Ref ElasticsearchVersion
+ ElasticsearchClusterConfig:
+ DedicatedMasterEnabled: "true"
+ InstanceCount: !Ref NumberOfElasticsearchNodes
+ ZoneAwarenessEnabled: !Ref ZoneAwarenessEnabled
+ InstanceType: !Ref ElasticsearchNodeInstanceType
+ DedicatedMasterType: !Ref DedicatedMasterInstanceType
+ DedicatedMasterCount: !Ref NumberOfDedicatedMasterNodes
+ EBSOptions:
+ EBSEnabled: true
+ VolumeSize: !Ref EBSVolumeSize
+ VolumeType: "gp2"
+ NodeToNodeEncryptionOptions:
+ Enabled: false
+ SnapshotOptions:
+ AutomatedSnapshotStartHour: "0"
+ AccessPolicies:
+ Version: "2012-10-17"
+ Statement:
+ - Effect: "Allow"
+ Principal:
+ AWS:
+ - !Sub arn:aws:iam::${AWS::AccountId}:root
+ - Fn::ImportValue:
+ !Sub "${Environment}-IndexerServiceIamUserArn"
+ Action: "es:*"
+ Resource: !Sub arn:aws:es:us-east-1:846973539254:domain/${Environment}-${ElasticsearchDomainName}/*
+ AdvancedOptions:
+ rest.action.multi.allow_explicit_index: "true"
+ Tags:
+ -
+ Key: "Environment"
+ Value: !Ref Environment
+
+Outputs:
+ # Elasticsearch domain ARN
+ ElasticsearchDomainArn:
+ Description: The ARN of the Elasticsearch domain.
+ Value: !GetAtt ElasticsearchDomain.DomainArn
+ Export:
+ Name: !Sub ${Environment}-${ElasticsearchDomainName}-ElasticsearchDomainArn
+
+ # Elasticsearch domain endpoint
+ ElasticsearchDomainEndpoint:
+ Description: The endpoint URL of the Elasticsearch domain.
+ Value: !GetAtt ElasticsearchDomain.DomainEndpoint
+ Export:
+ Name: !Sub ${Environment}-${ElasticsearchDomainName}-ElasticsearchDomainEndpoint
diff --git a/provider/indexer-aws/CloudFormation/Manual/01-CreateCodePipeline.yml b/provider/indexer-aws/CloudFormation/Manual/01-CreateCodePipeline.yml
index 51202fa863b000cc7d8f2311750949ca380a87d2..f2d18f5db8b69b352779010c6b835dd7fff52343 100644
--- a/provider/indexer-aws/CloudFormation/Manual/01-CreateCodePipeline.yml
+++ b/provider/indexer-aws/CloudFormation/Manual/01-CreateCodePipeline.yml
@@ -339,6 +339,7 @@ Resources:
- 'route53:*'
- 'route53domains:*'
- 'elasticache:*'
+ - 'es:*'
Effect: Allow
Resource: '*'
CodeBuildRole:
diff --git a/provider/indexer-aws/CloudFormation/Master/os-indexer-master.yml b/provider/indexer-aws/CloudFormation/Master/os-indexer-master.yml
index 9ad149d92b352be65609436789a0e319ffad50a0..e2863fbbb560a3e3782566293cdecb5469f24587 100644
--- a/provider/indexer-aws/CloudFormation/Master/os-indexer-master.yml
+++ b/provider/indexer-aws/CloudFormation/Master/os-indexer-master.yml
@@ -178,6 +178,136 @@ Parameters:
MinValue: 1
MaxValue: 128
+ ElasticsearchDomainName:
+ Description: The name of the Elasticsearch domain. Will be prefixed with the environment name.
+ Type: String
+ MinLength: '1'
+ MaxLength: '64'
+ AllowedPattern: "^[a-zA-Z]+[0-9a-zA-Z_-]*$"
+ ConstraintDescription: Must start with a letter. Only numbers, letters, -, and _ accepted. Max. length 64 characters.
+ Default: osdu-indexer
+
+ ElasticsearchNodeInstanceType:
+ Description: The instance type for the main Elasticsearch nodes.
+ ConstraintDescription: Must be a valid instance type from the list of allowed values.
+ Default: t2.medium.elasticsearch
+ AllowedValues:
+ - t2.small.elasticsearch
+ - t2.medium.elasticsearch
+ - m5.large.elasticsearch
+ - m5.xlarge.elasticsearch
+ - m5.2xlarge.elasticsearch
+ - m5.4xlarge.elasticsearch
+ - m5.12xlarge.elasticsearch
+ - c5.large.elasticsearch
+ - c5.xlarge.elasticsearch
+ - c5.2xlarge.elasticsearch
+ - c5.4xlarge.elasticsearch
+ - c5.9xlarge.elasticsearch
+ - c5.18xlarge.elasticsearch
+ - r5.large.elasticsearch
+ - r5.xlarge.elasticsearch
+ - r5.2xlarge.elasticsearch
+ - r5.4xlarge.elasticsearch
+ - r5.12xlarge.elasticsearch
+ - i3.large.elasticsearch
+ - i3.xlarge.elasticsearch
+ - i3.2xlarge.elasticsearch
+ - i3.4xlarge.elasticsearch
+ - i3.8xlarge.elasticsearch
+ - i3.16xlarge.elasticsearch
+ Type: String
+
+ DedicatedMasterInstanceType:
+ Description: >
+ The instance type for the dedicated master nodes. These nodes perform cluster management
+ tasks, but doesn't hold data or respond to data upload requests.
+ ConstraintDescription: Must be a valid instance type from the list of allowed values.
+ Default: t2.medium.elasticsearch
+ AllowedValues:
+ - t2.small.elasticsearch
+ - t2.medium.elasticsearch
+ - m5.large.elasticsearch
+ - m5.xlarge.elasticsearch
+ - m5.2xlarge.elasticsearch
+ - m5.4xlarge.elasticsearch
+ - m5.12xlarge.elasticsearch
+ - c5.large.elasticsearch
+ - c5.xlarge.elasticsearch
+ - c5.2xlarge.elasticsearch
+ - c5.4xlarge.elasticsearch
+ - c5.9xlarge.elasticsearch
+ - c5.18xlarge.elasticsearch
+ - r5.large.elasticsearch
+ - r5.xlarge.elasticsearch
+ - r5.2xlarge.elasticsearch
+ - r5.4xlarge.elasticsearch
+ - r5.12xlarge.elasticsearch
+ - i3.large.elasticsearch
+ - i3.xlarge.elasticsearch
+ - i3.2xlarge.elasticsearch
+ - i3.4xlarge.elasticsearch
+ - i3.8xlarge.elasticsearch
+ - i3.16xlarge.elasticsearch
+ Type: String
+
+ NumberOfElasticsearchNodes:
+ Description: An integer value specifying the number of Elasticsearch primary nodes in the cluster.
+ Type: Number
+ Default: 1
+ MinValue: 1
+ MaxValue: 40
+
+ NumberOfDedicatedMasterNodes:
+ Description: An integer value specifying the number of dedicated master nodes.
+ Type: Number
+ Default: 2
+ MinValue: 2
+ MaxValue: 5
+
+ ZoneAwarenessEnabled:
+ Description: >
+ When Zone Awareness is enabled, Elasticsearch allocates the nodes and replica
+ index shards that belong to a cluster across multiple AZs in the deployment region.
+ Type: String
+ AllowedValues:
+ - true
+ - false
+ Default: false
+
+ ElasticsearchVersion:
+ Description: >
+ The version of Elasticsearch to deploy on the cluster. Defaults to 6.8. Note
+ that an update requires a full replacement of the Elasticsearch cluster.
+ Type: String
+ AllowedValues:
+ - 1.5
+ - 2.3
+ - 5.1
+ - 5.3
+ - 5.5
+ - 5.6
+ - 6.0
+ - 6.2
+ - 6.3
+ - 6.4
+ - 6.5
+ - 6.6
+ - 6.8
+ - 6.8
+ - 7.1
+ Default: 6.8
+
+ EBSVolumeSize:
+ Description: >
+ The size of the EBS volume, in GiB, (per instance; total cluster size =
+ EBS volume size x Instance count). Maximum size varies by instance type, from 35GiB
+ for t2 instances, up to 12TiB for r5.12xlarge.
+ Type: Number
+ Default: 10
+ MinValue: 10
+ MaxValue: 12000
+
Resources:
#### Shared Resources ################################################################
@@ -229,3 +359,23 @@ Resources:
CacheEngine: !Ref SchemaCacheEngine
NodeInstanceType: !Ref SchemaCacheNodeInstanceType
NumberOfCacheNodes: !Ref SchemaCacheNumberOfCacheNodes
+
+ #### Elasticsearch Resources #########################################################
+ # TODO: Add VPCs to Elasticsearch
+
+ ElasticsearchStack:
+ Type: 'AWS::CloudFormation::Stack'
+ DependsOn: IAMCredentialsStack
+ Properties:
+ TemplateURL: !Join [ '', [ !Ref ChildTemplateBasePath, elasticsearch.yml ] ]
+ Parameters:
+ Environment: !Ref Environment
+ Region: !Ref DeploymentRegion
+ ElasticsearchDomainName: !Ref ElasticsearchDomainName
+ ElasticsearchNodeInstanceType: !Ref ElasticsearchNodeInstanceType
+ DedicatedMasterInstanceType: !Ref DedicatedMasterInstanceType
+ NumberOfElasticsearchNodes: !Ref NumberOfElasticsearchNodes
+ NumberOfDedicatedMasterNodes: !Ref NumberOfDedicatedMasterNodes
+ ZoneAwarenessEnabled: !Ref ZoneAwarenessEnabled
+ ElasticsearchVersion: !Ref ElasticsearchVersion
+ EBSVolumeSize: !Ref EBSVolumeSize
diff --git a/provider/indexer-aws/CloudFormation/Params/dev.template_configuration.json b/provider/indexer-aws/CloudFormation/Params/dev.template_configuration.json
index c512766c915e7e4d57b336348006369f4dce91ef..09ee0fdaabe6c5a750bc90c399aba2d5b69dff6b 100644
--- a/provider/indexer-aws/CloudFormation/Params/dev.template_configuration.json
+++ b/provider/indexer-aws/CloudFormation/Params/dev.template_configuration.json
@@ -14,7 +14,15 @@
"SchemaCacheName": "indexerschemacache",
"SchemaCacheEngine": "redis",
"SchemaCacheNodeInstanceType": "cache.t2.micro",
- "SchemaCacheNumberOfCacheNodes": "1"
+ "SchemaCacheNumberOfCacheNodes": "1",
+ "ElasticsearchDomainName": "osdu-indexer",
+ "ElasticsearchNodeInstanceType": "t2.medium.elasticsearch",
+ "DedicatedMasterInstanceType": "t2.medium.elasticsearch",
+ "NumberOfElasticsearchNodes": "2",
+ "NumberOfDedicatedMasterNodes": "2",
+ "ZoneAwarenessEnabled": "false",
+ "ElasticsearchVersion": "6.8",
+ "EBSVolumeSize": "10"
},
"Tags" : {
"Environment" : "dev"
diff --git a/provider/indexer-aws/CloudFormation/Params/prod.template_configuration.json b/provider/indexer-aws/CloudFormation/Params/prod.template_configuration.json
index 4dbe4a646e84b3b47967892dae7e2a9f12748e87..3966818a4b4e0348d00164667bb2e66a8ea8481a 100644
--- a/provider/indexer-aws/CloudFormation/Params/prod.template_configuration.json
+++ b/provider/indexer-aws/CloudFormation/Params/prod.template_configuration.json
@@ -14,7 +14,15 @@
"SchemaCacheName": "indexerschemacache",
"SchemaCacheEngine": "redis",
"SchemaCacheNodeInstanceType": "cache.t2.micro",
- "SchemaCacheNumberOfCacheNodes": "1"
+ "SchemaCacheNumberOfCacheNodes": "1",
+ "ElasticsearchDomainName": "osdu-indexer",
+ "ElasticsearchNodeInstanceType": "t2.medium.elasticsearch",
+ "DedicatedMasterInstanceType": "t2.medium.elasticsearch",
+ "NumberOfElasticsearchNodes": "2",
+ "NumberOfDedicatedMasterNodes": "2",
+ "ZoneAwarenessEnabled": "false",
+ "ElasticsearchVersion": "6.8",
+ "EBSVolumeSize": "10"
},
"Tags" : {
"Environment" : "prod"
diff --git a/provider/indexer-aws/CloudFormation/Params/uat.template_configuration.json b/provider/indexer-aws/CloudFormation/Params/uat.template_configuration.json
index 3f9b350b9c9c1745b4e6142845ca0c85374a8f5c..f47971307d4c2a7535b9c12298c4f93061472ab7 100644
--- a/provider/indexer-aws/CloudFormation/Params/uat.template_configuration.json
+++ b/provider/indexer-aws/CloudFormation/Params/uat.template_configuration.json
@@ -14,7 +14,15 @@
"SchemaCacheName": "indexerschemacache",
"SchemaCacheEngine": "redis",
"SchemaCacheNodeInstanceType": "cache.t2.micro",
- "SchemaCacheNumberOfCacheNodes": "1"
+ "SchemaCacheNumberOfCacheNodes": "1",
+ "ElasticsearchDomainName": "osdu-indexer",
+ "ElasticsearchNodeInstanceType": "t2.medium.elasticsearch",
+ "DedicatedMasterInstanceType": "t2.medium.elasticsearch",
+ "NumberOfElasticsearchNodes": "2",
+ "NumberOfDedicatedMasterNodes": "2",
+ "ZoneAwarenessEnabled": "false",
+ "ElasticsearchVersion": "6.8",
+ "EBSVolumeSize": "10"
},
"Tags" : {
"Environment" : "uat"
diff --git a/provider/indexer-aws/indexer-queue-aws/src/main/java/org/opengroup/osdu/indexerqueue/aws/api/IndexerQueue.java b/provider/indexer-aws/indexer-queue-aws/src/main/java/org/opengroup/osdu/indexerqueue/aws/api/IndexerQueue.java
index 4fd7d5b21d230d8dff73ec1a3ef377858b34743a..071e44569b6cc8297a0e45c1bc073d753c7ac8e8 100644
--- a/provider/indexer-aws/indexer-queue-aws/src/main/java/org/opengroup/osdu/indexerqueue/aws/api/IndexerQueue.java
+++ b/provider/indexer-aws/indexer-queue-aws/src/main/java/org/opengroup/osdu/indexerqueue/aws/api/IndexerQueue.java
@@ -1,6 +1,6 @@
package org.opengroup.osdu.indexerqueue.aws.api;
// TODO: rename package
-import com.amazonaws.services.sqs.AmazonSQS;
+import com.amazonaws.services.sqs.AmazonSQSConfig;
import com.amazonaws.services.sqs.model.*;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -32,7 +32,8 @@ public class IndexerQueue {
Arguments arguments = new Arguments();
log.debug(String.format("Connecting to the SQS Queue: %s", arguments.queueName));
- AmazonSQS sqsClient = SQSBuilder.generateSqsClient(arguments.region);
+ AmazonSQSConfig sqsConfig = new AmazonSQSConfig(arguments.region);
+ sqsClient = sqsConfig.AmazonSQS();
log.debug(String.format("Creating a thread pool with %s threads", arguments.maxIndexThreads));
ThreadPoolExecutor executorPool = (ThreadPoolExecutor) Executors.newFixedThreadPool(arguments.maxIndexThreads);
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/publish/PublisherImpl.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/publish/PublisherImpl.java
index a2cbef3e4715b20fe98e6b6445a1f30e80ac3052..d82ac766f0fec97c1b63a447f2dcec29ed052a5a 100644
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/publish/PublisherImpl.java
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/publish/PublisherImpl.java
@@ -4,8 +4,8 @@ import com.amazonaws.services.sns.model.MessageAttributeValue;
import com.amazonaws.services.sns.model.PublishRequest;
import com.amazonaws.services.sns.AmazonSNS;
import org.opengroup.osdu.core.api.DpsHeaders;
+import org.opengroup.osdu.core.aws.sns.AmazonSNSConfig;
import org.opengroup.osdu.core.aws.sns.PublishRequestBuilder;
-import org.opengroup.osdu.core.aws.sns.SNSBuilder;
import org.opengroup.osdu.indexer.provider.interfaces.IPublisher;
import org.opengroup.osdu.indexer.util.JobStatus;
import org.springframework.beans.factory.annotation.Value;
@@ -28,7 +28,8 @@ public class PublisherImpl implements IPublisher {
@Inject
public void init(){
- snsClient = SNSBuilder.generateAmazonSNSClient(amazonSNSRegion);
+ AmazonSNSConfig snsConfig = new AmazonSNSConfig(amazonSNSRegion);
+ snsClient = snsConfig.AmazonSNS();
}
public void publishStatusChangedTagsToTopic(DpsHeaders headers, JobStatus indexerBatchStatus) throws Exception
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java
index 42b13513596a38da4db6497e3f4b3def6a2ba2d6..daaaab0fde6c8957fef460284a8981e288abdcd8 100644
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java
@@ -3,7 +3,7 @@ package org.opengroup.osdu.indexer.aws.util;
import com.amazonaws.services.sqs.AmazonSQS;
import com.amazonaws.services.sqs.model.SendMessageRequest;
import org.opengroup.osdu.core.api.DpsHeaders;
-import org.opengroup.osdu.core.aws.sqs.SQSBuilder;
+import org.opengroup.osdu.core.aws.sqs.AmazonSQSConfig;
import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Primary;
@@ -25,7 +25,8 @@ public class IndexerQueueTaskBuilderAws extends IndexerQueueTaskBuilder {
@Inject
public void init() {
- sqsClient = SQSBuilder.generateSqsClient(region);
+ AmazonSQSConfig sqsConfig = new AmazonSQSConfig(region);
+ sqsClient = sqsConfig.AmazonSQS();
}
@Override
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/HTTPClient.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/HTTPClient.java
index 9dc094eb06da2b35d31547c9ae67b8ec36c364b4..8906865a34a90b9a4bf519e89440aba760a6e1c1 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/HTTPClient.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/HTTPClient.java
@@ -3,7 +3,6 @@ package org.opengroup.osdu.util;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.api.client.filter.HTTPBasicAuthFilter;
import lombok.ToString;
import lombok.extern.java.Log;
@@ -51,11 +50,7 @@ public abstract class HTTPClient {
HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory());
} catch (Exception ignored) {
}
-
- Client client = Client.create();
- client.addFilter(new HTTPBasicAuthFilter("opendes@byoc.local", "123"));
-
- return client;
+ return Client.create();
}
public ClientResponse send(String httpMethod, String url, String payLoad, Map<String, String> headers, String token) {
@@ -95,4 +90,4 @@ public abstract class HTTPClient {
currentHeaders.put("data-partition-id", value);
return currentHeaders;
}
-}
+}
\ No newline at end of file
diff --git a/testing/indexer-test-core/src/main/resources/features/indexrecord/IndexRecord.feature b/testing/indexer-test-core/src/main/resources/features/indexrecord/IndexRecord.feature
index 12dff87200c9f6eaf99e04990e4f0b9260df09d1..e0fffb72205939a4bb8913395f027fe80145ef0f 100644
--- a/testing/indexer-test-core/src/main/resources/features/indexrecord/IndexRecord.feature
+++ b/testing/indexer-test-core/src/main/resources/features/indexrecord/IndexRecord.feature
@@ -15,8 +15,8 @@ Feature: Indexing of the documents
Examples:
| kind | recordFile | number | index | type | acl | mapping |
- | "tenant1:testindex<timestamp>:well:1.0.0" | "index_records_1" | 5 | "tenant1-testindex<timestamp>-well-1.0.0" | "well" | "data.domain@byoc" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
- | "tenant1:testindex<timestamp>:well:3.0.0" | "index_records_1" | 5 | "tenant1-testindex<timestamp>-well-3.0.0" | "well" | "data.domain@byoc" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
+ | "tenant1:testindex<timestamp>:well:1.0.0" | "index_records_1" | 5 | "tenant1-testindex<timestamp>-well-1.0.0" | "well" | "data.default.viewers@opendes" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
+ | "tenant1:testindex<timestamp>:well:3.0.0" | "index_records_1" | 5 | "tenant1-testindex<timestamp>-well-3.0.0" | "well" | "data.default.viewers@opendes" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
Scenario Outline: Ingest the record and Index in the Elastic Search with bad attribute
When I ingest records with the <recordFile> with <acl> for a given <kind>
@@ -24,4 +24,4 @@ Feature: Indexing of the documents
Examples:
| kind | recordFile | number | index | skippedAttribute | acl |
- | "tenant1:testindex<timestamp>:well:2.0.0" | "index_records_2" | 4 | "tenant1-testindex<timestamp>-well-2.0.0" | "data.Location" | "data.domain@byoc" |
\ No newline at end of file
+ | "tenant1:testindex<timestamp>:well:2.0.0" | "index_records_2" | 4 | "tenant1-testindex<timestamp>-well-2.0.0" | "data.Location" | "data.default.viewers@opendes" |
\ No newline at end of file