Merge branch 'aws-integration' into 'master'

AWS Support ECK/Elasticsearch on EKS

See merge request !145

provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/persistence/ElasticRepositoryImpl.java

@@ -33,7 +33,17 @@ public class ElasticRepositoryImpl implements IElasticRepository {
     @Value("${aws.es.port}")
     int port;
 
-    String userNameAndPassword = "testing";
+    @Value("${aws.es.isHttps}")
+    boolean isHttps;
+
+    @Value("${aws.es.username}")
+    String username;
+
+    @Value("${aws.es.password}")
+    String password;
+
+    String usernameAndPassword;
+
 
     @Value("${aws.elasticsearch.port}")
     String portParameter;
@@ -41,6 +51,12 @@ public class ElasticRepositoryImpl implements IElasticRepository {
     @Value("${aws.elasticsearch.host}")
     String hostParameter;
 
+    @Value("${aws.elasticsearch.username}")
+    String usernameParameter;
+
+    @Value("${aws.elasticsearch.password}")
+    String passwordParameter;
+
     @Value("${aws.ssm}")
     String ssmEnabledString;
 
@@ -52,12 +68,24 @@ public class ElasticRepositoryImpl implements IElasticRepository {
             SSMConfig ssmConfig = new SSMConfig();
             ssm = ssmConfig.amazonSSM();
             host = ssm.getProperty(hostParameter).toString();
-            port = Integer.parseInt(ssm.getProperty(portParameter).toString());
+            port = Integer.parseInt(ssm.getProperty(portParameter).toString());            
+            username = ssm.getProperty(usernameParameter).toString();            
+            password = ssm.getProperty(passwordParameter).toString();            
         }
+        
+        //elastic expects username:password format
+        usernameAndPassword = String.format("%s:%s", username, password);
     }
 
     @Override
     public ClusterSettings getElasticClusterSettings(TenantInfo tenantInfo) {
-        return new ClusterSettings(host, port, userNameAndPassword);
+        ClusterSettings settings = new ClusterSettings(host, port, usernameAndPassword);
+        
+        if (!isHttps) {
+            settings.setHttps(false);
+            settings.setTls(false);
+        }
+
+        return settings;
     }
 }

provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/service/ElasticClientHandlerAws.java

@@ -16,7 +16,9 @@ package org.opengroup.osdu.indexer.aws.service;
 
 import org.apache.http.Header;
 import org.apache.http.HttpHost;
+import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
 import org.apache.http.message.BasicHeader;
+import org.apache.http.ssl.SSLContextBuilder;
 import org.elasticsearch.client.RestClient;
 import org.opengroup.osdu.indexer.util.ElasticClientHandler;
 import org.elasticsearch.client.RestClientBuilder;
@@ -24,17 +26,30 @@ import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Primary;
 import org.springframework.stereotype.Component;
 
+import lombok.extern.java.Log;
+
+import java.security.KeyManagementException;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+
 import javax.inject.Inject;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
 // TODO: Elastic Client Handler should be designed to allow cloud providers to implement their own handler if not we have to inherited
 // SPI needs to be refactored
 @Primary
 @Component
+@Log
 public class ElasticClientHandlerAws extends ElasticClientHandler {
 
     private static final int REST_CLIENT_CONNECT_TIMEOUT = 60000;
     private static final int REST_CLIENT_SOCKET_TIMEOUT = 60000;
     private static final int REST_CLIENT_RETRY_TIMEOUT = 60000;
 
+    @Value("${aws.es.certificate.disableTrust:false}")
+    // @Value("#{new Boolean('${aws.es.certificate.disableTrust:false}')}")
+    private Boolean disableSslCertificateTrust;
+
     public ElasticClientHandlerAws() {
     }
 
@@ -46,8 +61,23 @@ public class ElasticClientHandlerAws extends ElasticClientHandler {
                 .setConnectTimeout(REST_CLIENT_CONNECT_TIMEOUT)
                 .setSocketTimeout(REST_CLIENT_SOCKET_TIMEOUT));        
 
-        if(isLocalHost(host)) {
-            builder.setHttpClientConfigCallback(httpAsyncClientBuilder -> httpAsyncClientBuilder.setSSLHostnameVerifier((s, sslSession) -> true));
+        if(isLocalHost(host) || disableSslCertificateTrust) {            
+
+            SSLContext sslContext;            
+            try {
+                sslContext = SSLContext.getInstance("TLS");
+                sslContext.init(null, new TrustManager[]{ UnsafeX509ExtendedTrustManager.INSTANCE }, null);
+                builder.setHttpClientConfigCallback(httpClientBuilder -> 
+                httpClientBuilder.setSSLContext(sslContext)
+                                .setSSLHostnameVerifier((s, session) -> true));
+            } catch (NoSuchAlgorithmException e) {
+                // TODO Auto-generated catch block
+                e.printStackTrace();
+            } catch (KeyManagementException e) {
+                // TODO Auto-generated catch block
+                e.printStackTrace();
+            }
+
         }
         Header[] defaultHeaders = new Header[]{
                 new BasicHeader("client.transport.nodes_sampler_interval", "30s"),
@@ -55,7 +85,8 @@ public class ElasticClientHandlerAws extends ElasticClientHandler {
                 new BasicHeader("client.transport.sniff", "false"),
                 new BasicHeader("request.headers.X-Found-Cluster", host),
                 new BasicHeader("cluster.name", host),
-                new BasicHeader("xpack.security.transport.ssl.enabled", tls)
+                new BasicHeader("xpack.security.transport.ssl.enabled", tls),
+                new BasicHeader("Authorization", basicAuthenticationHeaderVal),
         };
         builder.setDefaultHeaders(defaultHeaders);
         return builder;

provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/service/UnsafeX509ExtendedTrustManager.java

+package org.opengroup.osdu.indexer.aws.service;
+
+import javax.net.ssl.SSLEngine;
+import javax.net.ssl.X509ExtendedTrustManager;
+
+import java.net.Socket;
+import java.security.cert.X509Certificate;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * An insecure {@link UnsafeX509ExtendedTrustManager TrustManager} that trusts all X.509 certificates without any verification.
+ * <p>
+ * <strong>NOTE:</strong>
+ * Never use this {@link UnsafeX509ExtendedTrustManager} in production.
+ * It is purely for testing purposes, and thus it is very insecure.
+ * </p>
+ * <br>
+ * Suppressed warning: java:S4830 - "Server certificates should be verified during SSL/TLS connections"
+ *                                  This TrustManager doesn't validate certificates and should not be used at production.
+ *                                  It is just meant to be used for testing purposes and it is designed not to verify server certificates.
+ */
+class UnsafeX509ExtendedTrustManager extends X509ExtendedTrustManager {
+
+    public static final UnsafeX509ExtendedTrustManager INSTANCE = new UnsafeX509ExtendedTrustManager();
+    private static final Logger LOGGER = LoggerFactory.getLogger(UnsafeX509ExtendedTrustManager.class);
+    private static final X509Certificate[] EMPTY_X509_CERTIFICATES = new X509Certificate[0];
+    private static final String CLIENT_CERTIFICATE_LOG_MESSAGE = "Accepting a client certificate: [{}]";
+    private static final String SERVER_CERTIFICATE_LOG_MESSAGE = "Accepting a server certificate: [{}]";
+
+    private UnsafeX509ExtendedTrustManager() {}
+
+    @Override
+    public void checkClientTrusted(X509Certificate[] x509Certificates, String authType) {
+        if (LOGGER.isDebugEnabled()) {
+            LOGGER.debug(CLIENT_CERTIFICATE_LOG_MESSAGE, x509Certificates[0].getSubjectDN());
+        }
+    }
+
+    @Override
+    public void checkClientTrusted(X509Certificate[] x509Certificates, String authType, Socket socket) {
+        if (LOGGER.isDebugEnabled()) {
+            LOGGER.debug(CLIENT_CERTIFICATE_LOG_MESSAGE, x509Certificates[0].getSubjectDN());
+        }
+    }
+
+    @Override
+    public void checkClientTrusted(X509Certificate[] x509Certificates, String authType, SSLEngine sslEngine) {
+        if (LOGGER.isDebugEnabled()) {
+            LOGGER.debug(CLIENT_CERTIFICATE_LOG_MESSAGE, x509Certificates[0].getSubjectDN());
+        }
+    }
+
+    @Override
+    public void checkServerTrusted(X509Certificate[] x509Certificates, String authType) {
+        if (LOGGER.isDebugEnabled()) {
+            LOGGER.debug(SERVER_CERTIFICATE_LOG_MESSAGE, x509Certificates[0].getSubjectDN());
+        }
+    }
+
+    @Override
+    public void checkServerTrusted(X509Certificate[] x509Certificates, String authType, Socket socket) {
+        if (LOGGER.isDebugEnabled()) {
+            LOGGER.debug(SERVER_CERTIFICATE_LOG_MESSAGE, x509Certificates[0].getSubjectDN());
+        }
+    }
+
+    @Override
+    public void checkServerTrusted(X509Certificate[] x509Certificates, String authType, SSLEngine sslEngine) {
+        if (LOGGER.isDebugEnabled()) {
+            LOGGER.debug(SERVER_CERTIFICATE_LOG_MESSAGE, x509Certificates[0].getSubjectDN());
+        }
+    }
+
+    @Override
+    public X509Certificate[] getAcceptedIssuers() {
+        return EMPTY_X509_CERTIFICATES;
+    }
+
+}
\ No newline at end of file

provider/indexer-aws/src/main/resources/application.properties

@@ -11,14 +11,14 @@ CRON_INDEX_CLEANUP_THRESHOLD_DAYS=3
 CRON_EMPTY_INDEX_CLEANUP_THRESHOLD_DAYS=7
 
 # AWS ES configuration
-# DO NOT COMMENT THESE OUT THEY ARE PLACE HOLDERS
-ELASTIC_HOST=""
-ELASTIC_PORT=0
-aws.es.host=${ELASTIC_HOST}
-aws.es.port=${ELASTIC_PORT}
-aws.es.userNameAndPassword=notused
+aws.es.host=${ELASTIC_HOST:}
+aws.es.port=${ELASTIC_PORT:0}
+aws.es.isHttps=${ELASTIC_HTTPS:true}
+aws.es.username=${ELASTIC_USERNAME:empty}
+aws.es.password=${ELASTIC_PASSWORD:empty}
 aws.region=${AWS_REGION}
 aws.es.serviceName=es
+aws.es.certificate.disableTrust=${ELASTIC_DISABLE_CERTIFICATE_TRUST:false}
 
 GAE_SERVICE=indexer
 
@@ -53,8 +53,10 @@ aws.dynamodb.endpoint=dynamodb.${AWS_REGION}.amazonaws.com
 aws.ssm=${SSM_ENABLED}
 aws.ssm.prefix=/osdu/${ENVIRONMENT}
 
-aws.elasticsearch.host=${aws.ssm.prefix}/elastic-search/end-point
-aws.elasticsearch.port=${aws.ssm.prefix}/elastic-search/end-point-port
+aws.elasticsearch.host=${aws.ssm.prefix}/elasticsearch/end-point
+aws.elasticsearch.port=${aws.ssm.prefix}/elasticsearch/end-point-port
+aws.elasticsearch.username=${aws.ssm.prefix}/elasticsearch/username
+aws.elasticsearch.password=${aws.ssm.prefix}/elasticsearch/password
 aws.indexer.sns.topic.arn=${aws.ssm.prefix}/indexer/indexer-sns-topic-arn
 aws.storage.sns.topic.arn=${aws.ssm.prefix}/storage/storage-sns-topic-arn
 

testing/indexer-test-aws/build-aws/run-tests.sh

@@ -28,10 +28,15 @@ export ENTITLEMENTS_DOMAIN=testing.com
 export OTHER_RELEVANT_DATA_COUNTRIES=US
 export STORAGE_HOST=$STORAGE_URL
 export HOST=$SCHEMA_URL
+export ELASTIC_HOST=$ELASTIC_HOST
+export ELASTIC_PORT=$ELASTIC_PORT
+export ELASTIC_PASSWORD=$ELASTIC_PASSWORD
+export ELASTIC_USER_NAME=$ELASTIC_USERNAME
 
 #### RUN INTEGRATION TEST #########################################################################
 
 mvn -ntp test -f "$SCRIPT_SOURCE_DIR"/../pom.xml -Dcucumber.options="--plugin junit:target/junit-report.xml"
+# mvn -Dmaven.surefire.debug test -f "$SCRIPT_SOURCE_DIR"/../pom.xml -Dcucumber.options="--plugin junit:target/junit-report.xml"
 TEST_EXIT_CODE=$?
 
 #### COPY TEST REPORTS #########################################################################

testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/ElasticUtilsAws.java

@@ -14,6 +14,13 @@
 
 package org.opengroup.osdu.util;
 
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.Base64;
+
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+
 import org.apache.http.Header;
 import org.apache.http.HttpHost;
 import org.apache.http.message.BasicHeader;
@@ -32,7 +39,30 @@ public class ElasticUtilsAws extends ElasticUtils {
         RestClientBuilder builder = RestClient.builder(new HttpHost(host, port, "https"));
         builder.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(REST_CLIENT_CONNECT_TIMEOUT)
                 .setSocketTimeout(REST_CLIENT_SOCKET_TIMEOUT));
-        builder.setHttpClientConfigCallback(httpAsyncClientBuilder -> httpAsyncClientBuilder.setSSLHostnameVerifier((s, sslSession) -> true));
+        
+       
+        //dont enforce CA/cert validity for tests
+        SSLContext sslContext;            
+        try {
+            sslContext = SSLContext.getInstance("TLS");
+            sslContext.init(null, new TrustManager[]{ UnsafeX509ExtendedTrustManager.INSTANCE }, null);
+            builder.setHttpClientConfigCallback(httpClientBuilder -> 
+            httpClientBuilder.setSSLContext(sslContext)
+                            .setSSLHostnameVerifier((s, session) -> true));
+        } catch (NoSuchAlgorithmException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        } catch (KeyManagementException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
+
+
+        String basicEncoded = Base64
+                            .getEncoder().encodeToString(usernameAndPassword.getBytes());
+        String basicAuthenticationHeaderVal = String.format("Basic %s", basicEncoded);
+
+    
 
         Header[] defaultHeaders = new Header[]{
                 new BasicHeader("client.transport.nodes_sampler_interval", "30s"),
@@ -40,7 +70,8 @@ public class ElasticUtilsAws extends ElasticUtils {
                 new BasicHeader("client.transport.sniff", "false"),
                 new BasicHeader("request.headers.X-Found-Cluster", Config.getElasticHost()),
                 new BasicHeader("cluster.name", Config.getElasticHost()),
-                new BasicHeader("xpack.security.transport.ssl.enabled", Boolean.toString(true))
+                new BasicHeader("xpack.security.transport.ssl.enabled", Boolean.toString(true)),
+                new BasicHeader("Authorization", basicAuthenticationHeaderVal),
         };
 
         builder.setDefaultHeaders(defaultHeaders);

testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/UnsafeX509ExtendedTrustManager.java

+package org.opengroup.osdu.util;
+
+import javax.net.ssl.SSLEngine;
+import javax.net.ssl.X509ExtendedTrustManager;
+
+import java.net.Socket;
+import java.security.cert.X509Certificate;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * An insecure {@link UnsafeX509ExtendedTrustManager TrustManager} that trusts all X.509 certificates without any verification.
+ * <p>
+ * <strong>NOTE:</strong>
+ * Never use this {@link UnsafeX509ExtendedTrustManager} in production.
+ * It is purely for testing purposes, and thus it is very insecure.
+ * </p>
+ * <br>
+ * Suppressed warning: java:S4830 - "Server certificates should be verified during SSL/TLS connections"
+ *                                  This TrustManager doesn't validate certificates and should not be used at production.
+ *                                  It is just meant to be used for testing purposes and it is designed not to verify server certificates.
+ */
+class UnsafeX509ExtendedTrustManager extends X509ExtendedTrustManager {
+
+    public static final UnsafeX509ExtendedTrustManager INSTANCE = new UnsafeX509ExtendedTrustManager();
+    private static final Logger LOGGER = LoggerFactory.getLogger(UnsafeX509ExtendedTrustManager.class);
+    private static final X509Certificate[] EMPTY_X509_CERTIFICATES = new X509Certificate[0];
+    private static final String CLIENT_CERTIFICATE_LOG_MESSAGE = "Accepting a client certificate: [{}]";
+    private static final String SERVER_CERTIFICATE_LOG_MESSAGE = "Accepting a server certificate: [{}]";
+
+    private UnsafeX509ExtendedTrustManager() {}
+
+    @Override
+    public void checkClientTrusted(X509Certificate[] x509Certificates, String authType) {
+        if (LOGGER.isDebugEnabled()) {
+            LOGGER.debug(CLIENT_CERTIFICATE_LOG_MESSAGE, x509Certificates[0].getSubjectDN());
+        }
+    }
+
+    @Override
+    public void checkClientTrusted(X509Certificate[] x509Certificates, String authType, Socket socket) {
+        if (LOGGER.isDebugEnabled()) {
+            LOGGER.debug(CLIENT_CERTIFICATE_LOG_MESSAGE, x509Certificates[0].getSubjectDN());
+        }
+    }
+
+    @Override
+    public void checkClientTrusted(X509Certificate[] x509Certificates, String authType, SSLEngine sslEngine) {
+        if (LOGGER.isDebugEnabled()) {
+            LOGGER.debug(CLIENT_CERTIFICATE_LOG_MESSAGE, x509Certificates[0].getSubjectDN());
+        }
+    }
+
+    @Override
+    public void checkServerTrusted(X509Certificate[] x509Certificates, String authType) {
+        if (LOGGER.isDebugEnabled()) {
+            LOGGER.debug(SERVER_CERTIFICATE_LOG_MESSAGE, x509Certificates[0].getSubjectDN());
+        }
+    }
+
+    @Override
+    public void checkServerTrusted(X509Certificate[] x509Certificates, String authType, Socket socket) {
+        if (LOGGER.isDebugEnabled()) {
+            LOGGER.debug(SERVER_CERTIFICATE_LOG_MESSAGE, x509Certificates[0].getSubjectDN());
+        }
+    }
+
+    @Override
+    public void checkServerTrusted(X509Certificate[] x509Certificates, String authType, SSLEngine sslEngine) {
+        if (LOGGER.isDebugEnabled()) {
+            LOGGER.debug(SERVER_CERTIFICATE_LOG_MESSAGE, x509Certificates[0].getSubjectDN());
+        }
+    }
+
+    @Override
+    public X509Certificate[] getAcceptedIssuers() {
+        return EMPTY_X509_CERTIFICATES;
+    }
+
+}
\ No newline at end of file