Skip to content
Snippets Groups Projects
Commit 6e435f55 authored by Matt Wise's avatar Matt Wise
Browse files

Merge branch 'aws-integration' into 'master' 

EKS Deploy

See merge request !191
parents e6727dd4 b8d56625
No related branches found
No related tags found
1 merge request!191EKS Deploy
Pipeline #62263 failed
Stage: .pre
Stage: build
Stage: containerize
Stage: scan
Stage: deploy
Stage: integration
Stage: attribution
Stage: publish
Hide whitespace changes
Inline Side-by-side
Showing
with 168 additions and 101 deletions
.gitlab-ci.yml
+ 2
− 0
View file @ 6e435f55
......@@ -3,6 +3,8 @@ variables:
AWS_TEST_SUBDIR: testing/indexer-test-aws
AWS_SERVICE: indexer
AWS_ENVIRONMENT: dev
AWS_DEPLOY_TARGET: EKS
AWS_EKS_DEPLOYMENT_NAME: os-indexer
GCP_BUILD_SUBDIR: provider/indexer-gcp
GCP_INT_TEST_SUBDIR: testing/indexer-test-gcp
......
provider/indexer-aws/build-aws/buildspec.yaml
+ 10
− 2
View file @ 6e435f55
......@@ -21,6 +21,11 @@ env:
secrets-manager:
DOCKER_USERNAME: /osdu/devops/docker_credentials:username
DOCKER_PASSWORD: /osdu/devops/docker_credentials:password
SONAR_USERNAME: /osdu/devops/sonar_credentials:username
SONAR_PASSWORD: /osdu/devops/sonar_credentials:password
parameter-store:
SONAR_URL: /osdu/devops/sonar_url
phases:
install:
......@@ -59,8 +64,7 @@ phases:
- printenv
- echo "Building primary service assemblies..."
- mvn -ntp -B test install -pl indexer-core,provider/indexer-aws -Ddeployment.environment=prod
- mvn -ntp -B test install sonar:sonar -pl .,indexer-core,provider/indexer-aws -Ddeployment.environment=prod -Dsonar.login=${SONAR_USERNAME} -Dsonar.password=${SONAR_PASSWORD} -Dsonar.branch.name=${BRANCH_NAME}
# - echo "Copying assemblies to dist..."
# - cp ./provider/indexer-aws/target/*spring-boot.jar ${OUTPUT_DIR}/bin # copy aws jars
......@@ -84,6 +88,10 @@ phases:
python provider/indexer-aws/build-aws/build-info.py --branch ${CODEBUILD_SOURCE_VERSION} --commit ${CODEBUILD_RESOLVED_SOURCE_VERSION} \
--buildid ${CODEBUILD_BUILD_ID} --buildnumber ${CODEBUILD_BUILD_NUMBER} --reponame ${REPO_NAME} --outdir ${OUTPUT_DIR} \
--artifact ${ECR_IMAGE}
post_build:
commands:
- cp provider/indexer-aws/target/dependency-check-report.html ${OUTPUT_DIR}
reports:
SurefireReports: # CodeBuild will create a report group called "SurefireReports".
files: #Store all of the files
......
provider/indexer-aws/maven/settings.xml
+ 11
− 0
View file @ 6e435f55
......@@ -54,6 +54,17 @@ limitations under the License.
<azure.devops.token>no-default</azure.devops.token>
</properties>
</profile>
<profile>
<id>sonar</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<sonar.host.url>
${env.SONAR_URL}
</sonar.host.url>
</properties>
</profile>
</profiles>
<servers>
......
provider/indexer-aws/pom.xml
+ 17
− 5
View file @ 6e435f55
......@@ -24,12 +24,12 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>indexer-aws</artifactId>
<description>Storage service on AWS</description>
<description>Indexer service on AWS</description>
<packaging>jar</packaging>
<version>0.11.0-SNAPSHOT</version>
<properties>
<aws.version>1.11.637</aws.version>
<aws.version>1.11.1018</aws.version>
<deployment.environment>dev</deployment.environment>
</properties>
......@@ -47,19 +47,19 @@
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
<artifactId>os-core-lib-aws</artifactId>
<version>0.3.16</version>
<version>0.11.0-SNAPSHOT</version>
</dependency>
<!-- AWS managed packages -->
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-core</artifactId>
<version>1.11.651</version>
<version>${aws.version}</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>1.11.327</version>
<version>${aws.version}</version>
</dependency>
<!-- Third party Apache 2.0 license packages -->
......@@ -161,6 +161,18 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.owasp</groupId>
<artifactId>dependency-check-maven</artifactId>
<version>6.2.2</version>
<executions>
<execution>
<goals>
<goal>check</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/IndexCacheImpl.java
+ 46
− 9
View file @ 6e435f55
......@@ -14,27 +14,64 @@
package org.opengroup.osdu.indexer.aws.cache;
import com.fasterxml.jackson.core.JsonProcessingException;
import org.opengroup.osdu.core.common.cache.ICache;
import org.opengroup.osdu.core.common.cache.RedisCache;
import org.opengroup.osdu.core.common.cache.VmCache;
import org.opengroup.osdu.core.aws.cache.DummyCache;
import org.opengroup.osdu.core.aws.ssm.K8sLocalParameterProvider;
import org.opengroup.osdu.core.aws.ssm.K8sParameterNotFoundException;
import org.opengroup.osdu.core.common.provider.interfaces.IIndexCache;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.util.Map;
@Component
public class IndexCacheImpl implements IIndexCache<String, Boolean>, AutoCloseable {
private RedisCache<String, Boolean> cache;
public IndexCacheImpl(@Value("${aws.elasticache.cluster.endpoint}") final String REDIS_SEARCH_HOST,
@Value("${aws.elasticache.cluster.port}") final String REDIS_SEARCH_PORT,
@Value("${aws.elasticache.cluster.key}") final String REDIS_SEARCH_KEY,
@Value("${aws.elasticache.cluster.index.expiration}") final String INDEX_CACHE_EXPIRATION) {
cache = new RedisCache<>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), REDIS_SEARCH_KEY,
Integer.parseInt(INDEX_CACHE_EXPIRATION) * 60, String.class, Boolean.class);
private ICache<String, Boolean> cache;
private Boolean local;
@Value("${aws.elasticache.cluster.endpoint}")
String REDIS_SEARCH_HOST;
@Value("${aws.elasticache.cluster.port}")
String REDIS_SEARCH_PORT;
@Value("${aws.elasticache.cluster.key}")
String REDIS_SEARCH_KEY;
@Value("${aws.elasticache.cluster.index.expiration}")
String INDEX_CACHE_EXPIRATION;
public IndexCacheImpl() throws K8sParameterNotFoundException, JsonProcessingException {
int expTimeSeconds = 60 * 60;
K8sLocalParameterProvider provider = new K8sLocalParameterProvider();
if (provider.getLocalMode()){
if (Boolean.parseBoolean(System.getenv("DISABLE_CACHE"))){
cache = new DummyCache<>();
}else{
cache = new VmCache<String,Boolean>(expTimeSeconds, 10);
}
}else {
String host = provider.getParameterAsStringOrDefault("CACHE_CLUSTER_ENDPOINT", REDIS_SEARCH_HOST);
int port = Integer.parseInt(provider.getParameterAsStringOrDefault("CACHE_CLUSTER_PORT", REDIS_SEARCH_PORT));
Map<String, String > credential =provider.getCredentialsAsMap("CACHE_CLUSTER_KEY");
String password;
if (credential !=null){
password = credential.get("token");
}else{
password = REDIS_SEARCH_KEY;
}
cache = new RedisCache<String, Boolean>(host, port, password, expTimeSeconds, String.class,Boolean.class);
}
local = cache instanceof AutoCloseable;
}
@Override
public void close() throws Exception {
this.cache.close();
if (local){
}else{
((AutoCloseable)this.cache).close();
}
}
@Override
......
provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/SchemaCacheImpl.java
+ 46
− 10
View file @ 6e435f55
......@@ -14,27 +14,63 @@
package org.opengroup.osdu.indexer.aws.cache;
import com.fasterxml.jackson.core.JsonProcessingException;
import org.opengroup.osdu.core.aws.cache.DummyCache;
import org.opengroup.osdu.core.aws.ssm.K8sLocalParameterProvider;
import org.opengroup.osdu.core.aws.ssm.K8sParameterNotFoundException;
import org.opengroup.osdu.core.common.cache.ICache;
import org.opengroup.osdu.core.common.cache.RedisCache;
import org.opengroup.osdu.core.common.cache.VmCache;
import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.util.Map;
@Component
public class SchemaCacheImpl implements ISchemaCache<String, String>, AutoCloseable {
private RedisCache<String, String> cache;
public SchemaCacheImpl(@Value("${aws.elasticache.cluster.endpoint}") final String REDIS_SEARCH_HOST,
@Value("${aws.elasticache.cluster.port}") final String REDIS_SEARCH_PORT,
@Value("${aws.elasticache.cluster.key}") final String REDIS_SEARCH_KEY,
@Value("${aws.elasticache.cluster.schema.expiration}") final String SCHEMA_CACHE_EXPIRATION) {
cache = new RedisCache<>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), REDIS_SEARCH_KEY,
Integer.parseInt(SCHEMA_CACHE_EXPIRATION) * 60, String.class, String.class);
@Value("${aws.elasticache.cluster.endpoint}")
String REDIS_SEARCH_HOST;
@Value("${aws.elasticache.cluster.port}")
String REDIS_SEARCH_PORT;
@Value("${aws.elasticache.cluster.key}")
String REDIS_SEARCH_KEY;
@Value("${aws.elasticache.cluster.schema.expiration}")
String SCHEMA_CACHE_EXPIRATION;
private ICache<String, String> cache;
private Boolean local = false;
public SchemaCacheImpl() throws K8sParameterNotFoundException, JsonProcessingException {
int expTimeSeconds = 60 * 60;
K8sLocalParameterProvider provider = new K8sLocalParameterProvider();
if (provider.getLocalMode()){
if (Boolean.parseBoolean(System.getenv("DISABLE_CACHE"))){
cache = new DummyCache<>();
}else{
cache = new VmCache<String,String>(expTimeSeconds, 10);
}
}else {
String host = provider.getParameterAsStringOrDefault("CACHE_CLUSTER_ENDPOINT", REDIS_SEARCH_HOST);
int port = Integer.parseInt(provider.getParameterAsStringOrDefault("CACHE_CLUSTER_PORT", REDIS_SEARCH_PORT));
Map<String, String > credential =provider.getCredentialsAsMap("CACHE_CLUSTER_KEY");
String password;
if (credential !=null){
password = credential.get("token");
}else{
password = REDIS_SEARCH_KEY;
}
cache = new RedisCache<>(host, port, password, expTimeSeconds, String.class, String.class);
}
local = cache instanceof AutoCloseable;
}
@Override
public void close() throws Exception {
this.cache.close();
if (this.local){
// do nothing, this is using local dummy cache
}else {
// cast to redis cache so it can be closed
((AutoCloseable)this.cache).close();
}
}
@Override
......
provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/persistence/ElasticRepositoryImpl.java
+ 14
− 28
View file @ 6e435f55
......@@ -14,16 +14,18 @@
package org.opengroup.osdu.indexer.aws.persistence;
import org.opengroup.osdu.core.aws.secrets.SecretsManager;
import org.opengroup.osdu.core.aws.ssm.ParameterStorePropertySource;
import org.opengroup.osdu.core.aws.ssm.SSMConfig;
import com.fasterxml.jackson.core.JsonProcessingException;
import org.opengroup.osdu.core.aws.ssm.K8sParameterNotFoundException;
import org.opengroup.osdu.core.common.model.search.ClusterSettings;
import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository;
import org.opengroup.osdu.core.aws.ssm.K8sLocalParameterProvider;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import java.util.Map;
@Component
public class ElasticRepositoryImpl implements IElasticRepository {
......@@ -45,36 +47,20 @@ public class ElasticRepositoryImpl implements IElasticRepository {
String usernameAndPassword;
@Value("${aws.elasticsearch.port}")
String portParameter;
@Value("${aws.elasticsearch.host}")
String hostParameter;
@Value("${aws.elasticsearch.credentials.secret}")
String elasticCredentialsSecret;
@Value("${aws.region}")
private String amazonRegion;
@Value("${aws.ssm}")
String ssmEnabledString;
private ParameterStorePropertySource ssm;
@PostConstruct
private void postConstruct() {
if( Boolean.parseBoolean(ssmEnabledString)) {
SSMConfig ssmConfig = new SSMConfig();
ssm = ssmConfig.amazonSSM();
host = ssm.getProperty(hostParameter).toString();
port = Integer.parseInt(ssm.getProperty(portParameter).toString());
private void postConstruct() throws K8sParameterNotFoundException, JsonProcessingException {
K8sLocalParameterProvider provider = new K8sLocalParameterProvider();
host = provider.getParameterAsStringOrDefault("elasticsearch_host", host);
port = Integer.parseInt(provider.getParameterAsStringOrDefault("elasticsearch_port", String.valueOf(port)));
Map<String, String> val = provider.getCredentialsAsMap("elasticsearch_credentials");
if (val != null){
username = val.get("username");
password = val.get("password");
}
SecretsManager sm = new SecretsManager();
username = sm.getSecret(elasticCredentialsSecret,amazonRegion,"username");
password = sm.getSecret(elasticCredentialsSecret,amazonRegion,"password");
//elastic expects username:password format
usernameAndPassword = String.format("%s:%s", username, password);
}
......
provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/publish/PublisherImpl.java
+ 6
− 14
View file @ 6e435f55
......@@ -17,8 +17,8 @@ package org.opengroup.osdu.indexer.aws.publish;
import com.amazonaws.services.sns.model.MessageAttributeValue;
import com.amazonaws.services.sns.model.PublishRequest;
import com.amazonaws.services.sns.AmazonSNS;
import org.opengroup.osdu.core.aws.ssm.ParameterStorePropertySource;
import org.opengroup.osdu.core.aws.ssm.SSMConfig;
import org.opengroup.osdu.core.aws.ssm.K8sParameterNotFoundException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.aws.sns.AmazonSNSConfig;
import org.opengroup.osdu.core.aws.sns.PublishRequestBuilder;
......@@ -26,7 +26,7 @@ import org.opengroup.osdu.indexer.provider.interfaces.IPublisher;
import org.opengroup.osdu.core.common.model.indexer.JobStatus;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.opengroup.osdu.core.aws.ssm.K8sLocalParameterProvider;
import javax.inject.Inject;
import java.util.HashMap;
import java.util.Map;
......@@ -35,25 +35,17 @@ import java.util.Map;
public class PublisherImpl implements IPublisher {
AmazonSNS snsClient;
private ParameterStorePropertySource ssm;
private String amazonSNSTopic;
@Value("${aws.region}")
private String amazonSNSRegion;
@Value("${aws.indexer.sns.topic.arn}")
private String parameter;
@Inject
public void init(){
public void init() throws K8sParameterNotFoundException {
AmazonSNSConfig snsConfig = new AmazonSNSConfig(amazonSNSRegion);
snsClient = snsConfig.AmazonSNS();
SSMConfig ssmConfig = new SSMConfig();
ssm = ssmConfig.amazonSSM();
amazonSNSTopic = ssm.getProperty(parameter).toString();
K8sLocalParameterProvider provider = new K8sLocalParameterProvider();
amazonSNSTopic = provider.getParameterAsString("indexer-sns-topic-arn");
}
public void publishStatusChangedTagsToTopic(DpsHeaders headers, JobStatus indexerBatchStatus) throws Exception
......
provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java
+ 6
− 14
View file @ 6e435f55
......@@ -19,8 +19,8 @@ import org.opengroup.osdu.core.aws.sqs.AmazonSQSConfig;
import com.amazonaws.services.sqs.model.MessageAttributeValue;
import com.amazonaws.services.sqs.model.SendMessageRequest;
import com.google.gson.Gson;
import org.opengroup.osdu.core.aws.ssm.ParameterStorePropertySource;
import org.opengroup.osdu.core.aws.ssm.SSMConfig;
import org.opengroup.osdu.core.aws.ssm.K8sLocalParameterProvider;
import org.opengroup.osdu.core.aws.ssm.K8sParameterNotFoundException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
......@@ -41,8 +41,6 @@ public class IndexerQueueTaskBuilderAws extends IndexerQueueTaskBuilder {
private AmazonSQS sqsClient;
private ParameterStorePropertySource ssm;
private String storageQueue;
private String dlq;
private final String retryString = "retry";
......@@ -52,20 +50,14 @@ public class IndexerQueueTaskBuilderAws extends IndexerQueueTaskBuilder {
@Value("${aws.region}")
private String region;
@Value("${aws.storage.sqs.queue.url}")
String sqsStorageQueueParameter;
@Value("${aws.indexer.sqs.dlq.url}")
String deadLetterQueueParameter;
@Inject
public void init() {
public void init() throws K8sParameterNotFoundException {
AmazonSQSConfig config = new AmazonSQSConfig(region);
sqsClient = config.AmazonSQS();
gson =new Gson();
SSMConfig ssmConfig = new SSMConfig();
ssm = ssmConfig.amazonSSM();
storageQueue = ssm.getProperty(sqsStorageQueueParameter).toString();
dlq = ssm.getProperty(deadLetterQueueParameter).toString();
K8sLocalParameterProvider provider = new K8sLocalParameterProvider();
storageQueue = provider.getParameterAsString("storage-sqs-url");
dlq = provider.getParameterAsString("indexer-deadletter-queue-sqs-url");
}
@Override
......
provider/indexer-aws/src/main/resources/application.properties
+ 10
− 19
View file @ 6e435f55
......@@ -22,20 +22,19 @@ aws.es.certificate.disableTrust=${ELASTIC_DISABLE_CERTIFICATE_TRUST:false}
GAE_SERVICE=indexer
#reusing STORAGE_HOST variable here as the base url to point to schema service
SCHEMA_HOST=${STORAGE_HOST}/api/schema-service/v1/schema
#reusing STORAGE_BASE_URL variable here as the base url to point to schema service
SCHEMA_HOST=${SCHEMA_BASE_URL}/api/schema-service/v1/schema
STORAGE_SCHEMA_HOST=${STORAGE_HOST}/api/storage/v2/schemas
STORAGE_QUERY_RECORD_HOST=${STORAGE_HOST}/api/storage/v2/query/records
STORAGE_QUERY_KINDS_HOST=${STORAGE_HOST}/api/storage/v2/query/kinds
STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=${STORAGE_HOST}/api/storage/v2/query/records:batch
STORAGE_SCHEMA_HOST=${STORAGE_BASE_URL}/api/storage/v2/schemas
STORAGE_QUERY_RECORD_HOST=${STORAGE_BASE_URL}/api/storage/v2/query/records
STORAGE_QUERY_KINDS_HOST=${STORAGE_BASE_URL}/api/storage/v2/query/kinds
STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=${STORAGE_BASE_URL}/api/storage/v2/query/records:batch
STORAGE_RECORDS_BATCH_SIZE=20
INDEXER_QUEUE_HOST=""
## AWS ElastiCache configuration
aws.elasticache.cluster.endpoint=${CACHE_CLUSTER_ENDPOINT}
aws.elasticache.cluster.port=${CACHE_CLUSTER_PORT}
aws.elasticache.cluster.key=${CACHE_CLUSTER_KEY}
aws.elasticache.cluster.endpoint=${CACHE_CLUSTER_ENDPOINT:null}
aws.elasticache.cluster.port=${CACHE_CLUSTER_PORT:null}
aws.elasticache.cluster.key=${CACHE_CLUSTER_KEY:null}
## Cache Settings
aws.elasticache.cluster.index.expiration=60
aws.elasticache.cluster.schema.expiration=60
......@@ -53,15 +52,7 @@ aws.dynamodb.endpoint=dynamodb.${AWS_REGION}.amazonaws.com
aws.ssm=${SSM_ENABLED}
aws.ssm.prefix=/osdu/${ENVIRONMENT}
aws.elasticsearch.host=${aws.ssm.prefix}/elasticsearch/end-point
aws.elasticsearch.port=${aws.ssm.prefix}/elasticsearch/end-point-port
aws.elasticsearch.credentials.secret=${aws.ssm.prefix}/elasticsearch/credentials
aws.indexer.sns.topic.arn=${aws.ssm.prefix}/indexer/indexer-sns-topic-arn
aws.storage.sns.topic.arn=${aws.ssm.prefix}/storage/storage-sns-topic-arn
aws.storage.sqs.queue.url=${aws.ssm.prefix}/storage/storage-sqs-url
aws.indexer.sqs.dlq.url=${aws.ssm.prefix}/indexer-queue/indexer-deadletter-queue-sqs-url
aws.parameter.prefix=/osdu/${ENVIRONMENT}
server.ssl.enabled=${SSL_ENABLED:true}
server.ssl.key-store-type=PKCS12
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment