Commit ae97de8c authored by Matt Wise's avatar Matt Wise
Browse files

Merge branch 'aws-integration' into 'master'

EKS Deploy

See merge request !155
parents 78ee7ec5 a0554110
Pipeline #61823 failed with stages
in 43 minutes and 28 seconds
......@@ -3,6 +3,8 @@ variables:
AWS_TEST_SUBDIR: testing/integration-tests/search-test-aws
AWS_SERVICE: search
AWS_ENVIRONMENT: dev
AWS_DEPLOY_TARGET: EKS
AWS_EKS_DEPLOYMENT_NAME: os-search
GCP_BUILD_SUBDIR: provider/search-gcp
GCP_INT_TEST_SUBDIR: testing/integration-tests/search-test-gcp
......
......@@ -21,6 +21,11 @@ env:
secrets-manager:
DOCKER_USERNAME: /osdu/devops/docker_credentials:username
DOCKER_PASSWORD: /osdu/devops/docker_credentials:password
SONAR_USERNAME: /osdu/devops/sonar_credentials:username
SONAR_PASSWORD: /osdu/devops/sonar_credentials:password
parameter-store:
SONAR_URL: /osdu/devops/sonar_url
phases:
install:
......@@ -59,7 +64,7 @@ phases:
- printenv
- echo "Building primary service assemblies..."
- mvn -ntp -B test install -pl search-core,provider/search-aws -Ddeployment.environment=prod
- mvn -ntp -B test install sonar:sonar -pl .,search-core,provider/search-aws -Ddeployment.environment=prod -Dsonar.login=${SONAR_USERNAME} -Dsonar.password=${SONAR_PASSWORD} -Dsonar.branch.name=${BRANCH_NAME}
# Suspended until further notice
# - echo "Copying assemblies to dist..."
......@@ -84,6 +89,10 @@ phases:
python provider/search-aws/build-aws/build-info.py --branch ${CODEBUILD_SOURCE_VERSION} --commit ${CODEBUILD_RESOLVED_SOURCE_VERSION} \
--buildid ${CODEBUILD_BUILD_ID} --buildnumber ${CODEBUILD_BUILD_NUMBER} --reponame ${REPO_NAME} --outdir ${OUTPUT_DIR} \
--artifact ${ECR_IMAGE}
post_build:
commands:
- cp provider/search-aws/target/dependency-check-report.html ${OUTPUT_DIR}
reports:
SurefireReports: # CodeBuild will create a report group called "SurefireReports".
files: #Store all of the files
......
......@@ -27,10 +27,6 @@ limitations under the License.
<activeByDefault>true</activeByDefault>
</activation>
<repositories>
<repository>
<id>aws-osdu-dev-maven</id>
<url>${env.AWS_OSDU_DEV_MAVEN_URL}</url>
</repository>
<repository>
<id>gitlab-os-core-common-maven</id>
<url>https://community.opengroup.org/api/v4/projects/67/packages/maven</url>
......@@ -39,6 +35,10 @@ limitations under the License.
<id>gitlab-os-core-lib-aws-maven</id>
<url>https://community.opengroup.org/api/v4/projects/68/packages/maven</url>
</repository>
<repository>
<id>aws-osdu-dev-maven</id>
<url>${env.AWS_OSDU_DEV_MAVEN_URL}</url>
</repository>
</repositories>
</profile>
<profile>
......@@ -54,6 +54,17 @@ limitations under the License.
<azure.devops.token>no-default</azure.devops.token>
</properties>
</profile>
<profile>
<id>sonar</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<sonar.host.url>
${env.SONAR_URL}
</sonar.host.url>
</properties>
</profile>
</profiles>
<servers>
......
......@@ -36,7 +36,7 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<failOnMissingWebXml>false</failOnMissingWebXml>
<project.main.basedir>${project.parent.basedir}</project.main.basedir>
<aws.version>1.11.637</aws.version>
<aws.version>1.11.1018</aws.version>
<deployment.environment>dev</deployment.environment>
<version.number>0.0.4-SNAPSHOT</version.number>
<osdu.oscorecommon.version>0.11.0-SNAPSHOT</osdu.oscorecommon.version>
......@@ -52,7 +52,7 @@
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
<artifactId>os-core-lib-aws</artifactId>
<version>0.3.17</version>
<version>0.11.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
......@@ -165,6 +165,18 @@
<threadCount>1</threadCount>
</configuration>
</plugin>
<plugin>
<groupId>org.owasp</groupId>
<artifactId>dependency-check-maven</artifactId>
<version>6.2.2</version>
<executions>
<execution>
<goals>
<goal>check</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
......
......@@ -14,30 +14,70 @@
package org.opengroup.osdu.search.provider.aws.cache;
import com.fasterxml.jackson.core.JsonProcessingException;
import org.opengroup.osdu.core.aws.cache.DummyCache;
import org.opengroup.osdu.core.aws.ssm.K8sLocalParameterProvider;
import org.opengroup.osdu.core.aws.ssm.K8sParameterNotFoundException;
import org.opengroup.osdu.core.common.cache.ICache;
import org.opengroup.osdu.core.common.cache.RedisCache;
import org.opengroup.osdu.core.common.cache.VmCache;
import org.opengroup.osdu.search.cache.CursorCache;
import org.opengroup.osdu.core.common.model.search.CursorSettings;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.util.Map;
@Component
public class CursorCacheImpl implements CursorCache {
@Value ("${aws.elasticache.cluster.cursor.endpoint}")
String REDIS_SEARCH_HOST;
@Value("${aws.elasticache.cluster.cursor.port}")
String REDIS_SEARCH_PORT;
@Value("${aws.elasticache.cluster.cursor.key}")
String REDIS_SEARCH_KEY;
@Value("${aws.elasticache.cluster.cursor.expiration}")
String INDEX_CACHE_EXPIRATION;
private ICache<String, CursorSettings> cache;
private Boolean local;
private RedisCache<String, CursorSettings> cache;
public void close() throws Exception {
if (this.local){
// local dummy cache, no need to close
}else{
((AutoCloseable)this.cache).close();
}
}
/**
* Initializes a Cursor Cache with Redis connection parameters specified in the application
* properties file.
* @param REDIS_SEARCH_HOST - the hostname of the Cursor Cache Redis cluster.
* @param REDIS_SEARCH_PORT - the port of the Cursor Cache Redis cluster.
*
* @param INDEX_CACHE_EXPIRATION - the expiration time for the Cursor Cache Redis cluster.
*/
public CursorCacheImpl(@Value("${aws.elasticache.cluster.cursor.endpoint}") final String REDIS_SEARCH_HOST,
@Value("${aws.elasticache.cluster.cursor.port}") final String REDIS_SEARCH_PORT,
@Value("${aws.elasticache.cluster.cursor.key}") final String REDIS_SEARCH_KEY,
@Value("${aws.elasticache.cluster.cursor.expiration}") final String INDEX_CACHE_EXPIRATION) {
cache = new RedisCache<String, CursorSettings>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), REDIS_SEARCH_KEY,
Integer.parseInt(INDEX_CACHE_EXPIRATION) * 60, String.class, CursorSettings.class);
public CursorCacheImpl() throws K8sParameterNotFoundException, JsonProcessingException {
int expTimeSeconds = 60*60;
K8sLocalParameterProvider provider = new K8sLocalParameterProvider();
if (provider.getLocalMode()){
if (Boolean.parseBoolean(System.getenv("DISABLE_CACHE"))){
cache = new DummyCache();
}else {
this.cache = new VmCache<>(expTimeSeconds, 10);
}
}else {
String host = provider.getParameterAsStringOrDefault("CACHE_CLUSTER_ENDPOINT", REDIS_SEARCH_HOST);
int port = Integer.parseInt(provider.getParameterAsStringOrDefault("CACHE_CLUSTER_PORT", REDIS_SEARCH_PORT));
Map<String, String > credential =provider.getCredentialsAsMap("CACHE_CLUSTER_KEY");
String password;
if (credential !=null){
password = credential.get("token");
}else{
password = REDIS_SEARCH_KEY;
}
cache = new RedisCache(host, port, password, expTimeSeconds, String.class, CursorSettings.class);
}
local = cache instanceof AutoCloseable;
}
/**
......
......@@ -14,7 +14,13 @@
package org.opengroup.osdu.search.provider.aws.cache;
import com.fasterxml.jackson.core.JsonProcessingException;
import org.opengroup.osdu.core.aws.cache.DummyCache;
import org.opengroup.osdu.core.aws.ssm.K8sLocalParameterProvider;
import org.opengroup.osdu.core.aws.ssm.K8sParameterNotFoundException;
import org.opengroup.osdu.core.common.cache.ICache;
import org.opengroup.osdu.core.common.cache.RedisCache;
import org.opengroup.osdu.core.common.cache.VmCache;
import org.opengroup.osdu.core.common.model.search.CursorSettings;
import org.opengroup.osdu.search.cache.IFieldTypeMappingCache;
import org.springframework.beans.factory.annotation.Value;
......@@ -24,22 +30,45 @@ import java.util.Map;
@Component
public class FieldTypeMappingCacheImpl implements IFieldTypeMappingCache {
private RedisCache<String, Map> cache;
@Value("${aws.elasticache.cluster.cursor.endpoint}")
String REDIS_SEARCH_HOST;
@Value("${aws.elasticache.cluster.cursor.port}")
String REDIS_SEARCH_PORT;
@Value("${aws.elasticache.cluster.cursor.key}")
String REDIS_SEARCH_KEY;
@Value("${aws.elasticache.cluster.cursor.expiration}")
String INDEX_CACHE_EXPIRATION;
private ICache<String,Map> cache;
private Boolean local;
/**
* Initializes a Cursor Cache with Redis connection parameters specified in the application
* properties file.
*
* @param REDIS_SEARCH_HOST - the hostname of the Cursor Cache Redis cluster.
* @param REDIS_SEARCH_PORT - the port of the Cursor Cache Redis cluster.
*/
public FieldTypeMappingCacheImpl(@Value("${aws.elasticache.cluster.cursor.endpoint}") final String REDIS_SEARCH_HOST,
@Value("${aws.elasticache.cluster.cursor.port}") final String REDIS_SEARCH_PORT,
@Value("${aws.elasticache.cluster.cursor.key}") final String REDIS_SEARCH_KEY,
@Value("${aws.elasticache.cluster.cursor.expiration}") final String INDEX_CACHE_EXPIRATION) {
cache = new RedisCache<String, Map>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), REDIS_SEARCH_KEY,
Integer.parseInt(INDEX_CACHE_EXPIRATION) * 60, String.class, Map.class);
public FieldTypeMappingCacheImpl() throws K8sParameterNotFoundException, JsonProcessingException {
int expTimeSeconds = 60 * 60;
K8sLocalParameterProvider provider = new K8sLocalParameterProvider();
if (provider.getLocalMode()){
if (Boolean.parseBoolean(System.getenv("DISABLE_CACHE"))){
cache = new DummyCache();
}else{
this.cache = new VmCache<>(expTimeSeconds, 10);
}
}else {
String host = provider.getParameterAsStringOrDefault("CACHE_CLUSTER_ENDPOINT", REDIS_SEARCH_HOST);
int port = Integer.parseInt(provider.getParameterAsStringOrDefault("CACHE_CLUSTER_PORT", REDIS_SEARCH_PORT));
Map<String, String > credential =provider.getCredentialsAsMap("CACHE_CLUSTER_KEY");
String password;
if (credential !=null){
password = credential.get("token");
}else{
password = REDIS_SEARCH_KEY;
}
cache = new RedisCache(host, port, password, expTimeSeconds, String.class, Map.class);
}
local = cache instanceof AutoCloseable;
}
/**
......
......@@ -14,27 +14,63 @@
package org.opengroup.osdu.search.provider.aws.cache;
import com.fasterxml.jackson.core.JsonProcessingException;
import org.opengroup.osdu.core.aws.cache.DummyCache;
import org.opengroup.osdu.core.aws.ssm.K8sLocalParameterProvider;
import org.opengroup.osdu.core.aws.ssm.K8sParameterNotFoundException;
import org.opengroup.osdu.core.common.cache.ICache;
import org.opengroup.osdu.core.common.cache.RedisCache;
import org.opengroup.osdu.core.common.cache.VmCache;
import org.opengroup.osdu.core.common.model.search.CursorSettings;
import org.opengroup.osdu.core.common.provider.interfaces.IIndexCache;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.util.Map;
@Component
public class IndexCacheImpl implements IIndexCache<String, Boolean>, AutoCloseable {
private RedisCache<String, Boolean> cache;
public IndexCacheImpl(@Value("${aws.elasticache.cluster.index.endpoint}") final String REDIS_SEARCH_HOST,
@Value("${aws.elasticache.cluster.index.port}") final String REDIS_SEARCH_PORT,
@Value("${aws.elasticache.cluster.index.key}") final String REDIS_SEARCH_KEY,
@Value("${aws.elasticache.cluster.index.expiration}") final String INDEX_CACHE_EXPIRATION) {
cache = new RedisCache<>(REDIS_SEARCH_HOST, Integer.parseInt(REDIS_SEARCH_PORT), REDIS_SEARCH_KEY,
Integer.parseInt(INDEX_CACHE_EXPIRATION) * 60, String.class, Boolean.class);
@Value ("${aws.elasticache.cluster.cursor.endpoint}")
String REDIS_SEARCH_HOST;
@Value("${aws.elasticache.cluster.cursor.port}")
String REDIS_SEARCH_PORT;
@Value("${aws.elasticache.cluster.cursor.key}")
String REDIS_SEARCH_KEY;
@Value("${aws.elasticache.cluster.cursor.expiration}")
String INDEX_CACHE_EXPIRATION;
private ICache<String, Boolean> cache;
private Boolean local;
public IndexCacheImpl(@Value("${aws.elasticache.cluster.index.expiration}") final String INDEX_CACHE_EXPIRATION) throws K8sParameterNotFoundException, JsonProcessingException {
int expTimeSeconds = 60 * 60;
K8sLocalParameterProvider provider = new K8sLocalParameterProvider();
if (provider.getLocalMode()){
if (Boolean.parseBoolean(System.getenv("DISABLE_CACHE"))){
cache = new DummyCache();
}else{
this.cache = new VmCache<>(expTimeSeconds, 10);
}
}else {
String host = provider.getParameterAsStringOrDefault("CACHE_CLUSTER_ENDPOINT", REDIS_SEARCH_HOST);
int port = Integer.parseInt(provider.getParameterAsStringOrDefault("CACHE_CLUSTER_PORT", REDIS_SEARCH_PORT));
Map<String, String > credential =provider.getCredentialsAsMap("CACHE_CLUSTER_KEY");
String password;
if (credential !=null){
password = credential.get("token");
}else{
password = REDIS_SEARCH_KEY;
}
cache = new RedisCache(host, port, password, expTimeSeconds, String.class, Boolean.class);
}
local = cache instanceof AutoCloseable;
}
@Override
public void close() throws Exception {
this.cache.close();
if (this.local){
// local dummy cache, no need to close
}else{
((AutoCloseable)this.cache).close();
}
}
@Override
......
......@@ -13,21 +13,14 @@
// limitations under the License.
package org.opengroup.osdu.search.provider.aws.persistence;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.Setter;
import org.opengroup.osdu.core.aws.ssm.K8sLocalParameterProvider;
import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
import javax.annotation.PostConstruct;
import org.opengroup.osdu.core.aws.ssm.ParameterStorePropertySource;
import org.opengroup.osdu.core.aws.ssm.SSMConfig;
import org.opengroup.osdu.core.common.model.search.ClusterSettings;
import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.opengroup.osdu.core.aws.secrets.SecretsManager;
import java.util.Map;
@Component
public class ElasticRepositoryImpl implements IElasticRepository {
......@@ -49,39 +42,21 @@ public class ElasticRepositoryImpl implements IElasticRepository {
String usernameAndPassword;
@Value("${aws.elasticsearch.port}")
String portParameter;
@Value("${aws.elasticsearch.host}")
String hostParameter;
@Value("${aws.elasticsearch.credentials.secret}")
String elasticCredentialsSecret;
@Value("${aws.region}")
private String amazonRegion;
@Value("${aws.ssm}")
String ssmEnabledString;
private ParameterStorePropertySource ssm;
@PostConstruct
private void postConstruct() {
if( Boolean.parseBoolean(ssmEnabledString)) {
SSMConfig ssmConfig = new SSMConfig();
ssm = ssmConfig.amazonSSM();
host = ssm.getProperty(hostParameter).toString();
port = Integer.parseInt(ssm.getProperty(portParameter).toString());
private void postConstruct() throws Exception{
K8sLocalParameterProvider provider = new K8sLocalParameterProvider();
host = provider.getParameterAsStringOrDefault("elasticsearch_host", host);
port = Integer.parseInt(provider.getParameterAsStringOrDefault("elasticsearch_port", String.valueOf(port)));
Map<String, String>val = provider.getCredentialsAsMap("elasticsearch_credentials");
if (val != null){
username = val.get("username");
password = val.get("password");
}
SecretsManager sm = new SecretsManager();
username = sm.getSecret(elasticCredentialsSecret,amazonRegion,"username");
password = sm.getSecret(elasticCredentialsSecret,amazonRegion,"password");
//elastic expects username:password format
usernameAndPassword = String.format("%s:%s", username, password);
}
......
......@@ -14,9 +14,7 @@
package org.opengroup.osdu.search.provider.aws.service;
import org.opengroup.osdu.core.aws.secrets.SecretsManager;
import org.opengroup.osdu.core.aws.ssm.ParameterStorePropertySource;
import org.opengroup.osdu.core.aws.ssm.SSMConfig;
import org.opengroup.osdu.core.aws.ssm.K8sLocalParameterProvider;
import org.opengroup.osdu.core.common.model.search.ClusterSettings;
import org.opengroup.osdu.core.common.model.indexer.IElasticSettingService;
import org.springframework.beans.factory.annotation.Value;
......@@ -24,6 +22,7 @@ import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import java.util.Map;
@Primary
@Component
......@@ -43,42 +42,23 @@ public class ElasticSettingServiceImpl implements IElasticSettingService {
@Value("${aws.es.password}")
String password;
String usernameAndPassword;
@Value("${aws.elasticsearch.port}")
String portParameter;
@Value("${aws.elasticsearch.host}")
String hostParameter;
@Value("${aws.elasticsearch.credentials.secret}")
String elasticCredentialsSecret;
@Value("${aws.region}")
private String amazonRegion;
@Value("${aws.ssm}")
String ssmEnabledString;
private ParameterStorePropertySource ssm;
@PostConstruct
private void postConstruct() {
if( Boolean.parseBoolean(ssmEnabledString)) {
SSMConfig ssmConfig = new SSMConfig();
ssm = ssmConfig.amazonSSM();
host = ssm.getProperty(hostParameter).toString();
port = Integer.parseInt(ssm.getProperty(portParameter).toString());
private void postConstruct() throws Exception {
K8sLocalParameterProvider provider = new K8sLocalParameterProvider();
host = provider.getParameterAsStringOrDefault("elasticsearch_host", host);
port = Integer.parseInt(provider.getParameterAsStringOrDefault("elasticsearch_port", String.valueOf(port)));
Map<String, String>val = provider.getCredentialsAsMap("elasticsearch_credentials");
if (val != null){
username = val.get("username");
password = val.get("password");
}
SecretsManager sm = new SecretsManager();
username = sm.getSecret(elasticCredentialsSecret,amazonRegion,"username");
password = sm.getSecret(elasticCredentialsSecret,amazonRegion,"password");
//elastic expects username:password format
usernameAndPassword = String.format("%s:%s", username, password);
}
......
// Copyright © 2020 Amazon Web Services
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.search.provider.aws.util;
import com.amazonaws.services.simplesystemsmanagement.AWSSimpleSystemsManagement;
import com.amazonaws.services.simplesystemsmanagement.AWSSimpleSystemsManagementClientBuilder;
import com.amazonaws.services.simplesystemsmanagement.model.GetParameterRequest;
import com.amazonaws.services.simplesystemsmanagement.model.GetParameterResult;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.Setter;
import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.opengroup.osdu.core.aws.entitlements.ServicePrincipal;
import org.opengroup.osdu.core.aws.iam.IAMConfig;
import org.opengroup.osdu.core.aws.secrets.SecretsManager;
import com.amazonaws.auth.AWSCredentialsProvider;
import javax.annotation.PostConstruct;
@Component
public class ServiceAccountJwtClientImpl implements IServiceAccountJwtClient {
@Value("${aws.region}")
@Getter()
@Setter(AccessLevel.PROTECTED)
public String amazonRegion;
@Value("${aws.ssm}")
@Getter()
@Setter(AccessLevel.PROTECTED)
public Boolean ssmEnabled;
@Value("${aws.environment}")
@Getter()
@Setter(AccessLevel.PROTECTED)
public String environment;
private String awsOauthCustomScope;
String client_credentials_secret;
String client_credentials_clientid;
ServicePrincipal sp;
private AWSCredentialsProvider amazonAWSCredentials;
private AWSSimpleSystemsManagement ssmManager;
@PostConstruct
public void init() {
if (ssmEnabled) {
SecretsManager sm = new SecretsManager();
String oauth_token_url = "/osdu/" + environment + "/oauth-token-uri";
String oauth_custom_scope = "/osdu/" + environment + "/oauth-custom-scope";
String client_credentials_client_id = "/osdu/" + environment + "/client-credentials-client-id";
String client_secret_key = "client_credentials_client_secret";
String client_secret_secretName = "/osdu/" + environment + "/client_credentials_secret";
amazonAWSCredentials = IAMConfig.amazonAWSCredentials();
ssmManager = AWSSimpleSystemsManagementClientBuilder.standard()
.withCredentials(amazonAWSCredentials)
.withRegion(amazonRegion)
.build();