Merge branch 'integration-master' into 'feature/GONRG-539_creds_for_storage'

# Conflicts:
#   provider/indexer-gcp/src/main/resources/application-dev.properties

provider/indexer-gcp/src/main/appengine/app.yaml

@@ -39,3 +39,4 @@ env_variables:
   LEGAL_HOSTNAME: "LEGAL_HOSTNAME_VAR"
   REGION: "REGION_VAR"
   SPRING_PROFILES_ACTIVE: 'ENVIRONMENT'
+  SECURITY_HTTPS_CERTIFICATE_TRUST: 'SECURITY_HTTPS_CERTIFICATE_TRUST_VAR'

provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/ElasticClientHandlerGcp.java

+package org.opengroup.osdu.indexer.util;
+
+import java.security.KeyManagementException;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.util.Base64;
+import javax.net.ssl.SSLContext;
+import lombok.extern.java.Log;
+import org.apache.http.Header;
+import org.apache.http.HttpHost;
+import org.apache.http.HttpStatus;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
+import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
+import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
+import org.apache.http.message.BasicHeader;
+import org.apache.http.ssl.SSLContextBuilder;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestClientBuilder;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.indexer.IElasticSettingService;
+import org.opengroup.osdu.core.common.model.search.ClusterSettings;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Primary;
+import org.springframework.stereotype.Component;
+
+@Component
+@Primary
+@Log
+public class ElasticClientHandlerGcp extends ElasticClientHandler {
+
+  // Elastic cluster Rest client settings
+  private static final int CLOUD_REST_CLIENT_PORT = 9243;
+  private static final int REST_CLIENT_CONNECT_TIMEOUT = 60000;
+  private static final int REST_CLIENT_SOCKET_TIMEOUT = 60000;
+  private static final int REST_CLIENT_RETRY_TIMEOUT = 60000;
+
+  @Value("#{new Boolean('${security.https.certificate.trust}')}")
+  private Boolean securityHttpsCertificateTrust;
+
+  @Autowired
+  private IElasticSettingService elasticSettingService;
+
+  public RestHighLevelClient createRestClient() {
+    return getCloudRestClient(elasticSettingService.getElasticClusterInformation());
+  }
+
+  // TODO: Remove this temporary implementation when ECE CCS is utilized
+  public RestHighLevelClient createRestClient(final ClusterSettings clusterSettings) {
+    return getCloudRestClient(clusterSettings);
+  }
+
+  private RestHighLevelClient getCloudRestClient(final ClusterSettings clusterSettings) {
+
+    String cluster = null;
+    String host = null;
+    int port = CLOUD_REST_CLIENT_PORT;
+    String protocolScheme = "https";
+    String tls = "true";
+
+    try {
+      cluster = clusterSettings.getHost();
+      host = clusterSettings.getHost();
+      port = clusterSettings.getPort();
+      if (!clusterSettings.isHttps()) {
+        protocolScheme = "http";
+      }
+
+      if (!clusterSettings.isTls()) {
+        tls = "false";
+      }
+      String basicEncoded = Base64
+          .getEncoder().encodeToString(clusterSettings.getUserNameAndPassword().getBytes());
+      String basicAuthenticationHeaderVal = String.format("Basic %s", basicEncoded);
+
+      RestClientBuilder builder = createClientBuilder(host, basicAuthenticationHeaderVal, port,
+          protocolScheme, tls);
+
+      return new RestHighLevelClient(builder);
+    } catch (AppException e) {
+      throw e;
+    } catch (Exception e) {
+      throw new AppException(
+          HttpStatus.SC_INTERNAL_SERVER_ERROR,
+          "search client error",
+          "error creating search client",
+          String
+              .format("Elastic client connection params, cluster: %s, host: %s, port: %s", cluster,
+                  host, port),
+          e);
+    }
+  }
+
+  public RestClientBuilder createClientBuilder(String host, String basicAuthenticationHeaderVal,
+      int port, String protocolScheme, String tls) {
+    RestClientBuilder builder = RestClient.builder(new HttpHost(host, port, protocolScheme));
+    builder.setRequestConfigCallback(
+        requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(REST_CLIENT_CONNECT_TIMEOUT)
+            .setSocketTimeout(REST_CLIENT_SOCKET_TIMEOUT));
+    builder.setMaxRetryTimeoutMillis(REST_CLIENT_RETRY_TIMEOUT);
+
+    Header[] defaultHeaders = new Header[]{
+        new BasicHeader("client.transport.nodes_sampler_interval", "30s"),
+        new BasicHeader("client.transport.ping_timeout", "30s"),
+        new BasicHeader("client.transport.sniff", "false"),
+        new BasicHeader("request.headers.X-Found-Cluster", host),
+        new BasicHeader("cluster.name", host),
+        new BasicHeader("xpack.security.transport.ssl.enabled", tls),
+        new BasicHeader("Authorization", basicAuthenticationHeaderVal),
+    };
+    log.info(String.format(
+        "Elastic client connection uses protocolScheme = %s with a flag "
+            + "'security.https.certificate.trust' = %s",
+        protocolScheme, securityHttpsCertificateTrust));
+    if ("https".equals(protocolScheme) && securityHttpsCertificateTrust) {
+      log.warning("Elastic client connection uses TrustSelfSignedStrategy()");
+      SSLContext sslContext = createSSLContext();
+      builder.setHttpClientConfigCallback(httpClientBuilder ->
+      {
+        HttpAsyncClientBuilder httpAsyncClientBuilder = httpClientBuilder.setSSLContext(sslContext)
+            .setSSLHostnameVerifier(
+                NoopHostnameVerifier.INSTANCE);
+        return httpAsyncClientBuilder;
+      });
+    }
+
+    builder.setDefaultHeaders(defaultHeaders);
+    return builder;
+  }
+
+  private SSLContext createSSLContext() {
+    SSLContextBuilder sslContextBuilder = new SSLContextBuilder();
+    try {
+      sslContextBuilder.loadTrustMaterial(null, new TrustSelfSignedStrategy());
+      return sslContextBuilder.build();
+    } catch (NoSuchAlgorithmException e) {
+      log.severe(e.getMessage());
+    } catch (KeyStoreException e) {
+      log.severe(e.getMessage());
+    } catch (KeyManagementException e) {
+      log.severe(e.getMessage());
+    }
+    return null;
+  }
+}

provider/indexer-gcp/src/main/resources/application-dev.properties

@@ -15,7 +15,9 @@ LEGALTAG_API=https://os-legal-dot-opendes.appspot.com/api/legal/v1
 CRS_API=https://crs-converter-gae-dot-opendes.appspot.com/api/crs/v1
 
 ## use below values for gcp: opendes
-REDIS_GROUP_HOST=127.0.0.1
-REDIS_STORAGE_HOST=127.0.0.1
-REDIS_SEARCH_HOST=127.0.0.1
-GOOGLE_AUDIENCES=245464679631-ktfdfpl147m1mjpbutl00b3cmffissgq.apps.googleusercontent.com
\ No newline at end of file
+REDIS_GROUP_HOST=10.0.16.28
+REDIS_SEARCH_HOST=10.0.16.20
+
+GOOGLE_AUDIENCES=245464679631-ktfdfpl147m1mjpbutl00b3cmffissgq.apps.googleusercontent.com
+
+security.https.certificate.trust=${SECURITY_HTTPS_CERTIFICATE_TRUST}
\ No newline at end of file

provider/indexer-gcp/src/main/resources/application-kuber.properties

@@ -22,3 +22,5 @@ GOOGLE_AUDIENCES=${GOOGLE_AUDIENCES}
 
 DEPLOYMENT_ENVIRONMENT=CLOUD
 disable.appengine.log.factory=true
+
+security.https.certificate.trust=${SECURITY_HTTPS_CERTIFICATE_TRUST}
\ No newline at end of file

provider/indexer-gcp/src/main/resources/application-testing.properties

@@ -19,3 +19,5 @@ REDIS_GROUP_HOST=10.253.209.196
 REDIS_SEARCH_HOST=10.118.2.140
 
 GOOGLE_AUDIENCES=833591776864-oobhqvmtdg9rpreubjvn44m5f8revglk.apps.googleusercontent.com
+
+security.https.certificate.trust=false
\ No newline at end of file