diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 726e9af5cd8e2492aa1014a506889326d78b9cf9..f03a7d4989795cac0acbcbbb43dd28911d1e05bd 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,6 +14,14 @@ variables:
GCP_DOMAIN: cloud.slb-ds.com
GCP_STORAGE_URL: https://osdu-indexer-dot-opendes.appspot.com/api/storage/v2/
+ OSDU_GCP_BUILD_SUBDIR: provider/indexer-gcp
+ OSDU_GCP_INT_TEST_SUBDIR: testing/indexer-test-gcp
+ OSDU_GCP_APPLICATION_NAME: os-indexer
+ OSDU_GCP_PROJECT: nice-etching-277309
+ OSDU_GCP_TENANT_NAME: osdu
+ OSDU_GCP_STORAGE_SCHEMA_HOST: https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/schemas
+ OSDU_SECURITY_HTTPS_CERTIFICATE_TRUST: 'true'
+
IBM_BUILD_SUBDIR: provider/indexer-ibm
IBM_INT_TEST_SUBDIR: testing/indexer-test-ibm
@@ -46,5 +54,10 @@ include:
- project: "osdu/platform/ci-cd-pipelines"
file: "publishing/pages.yml"
+ - project: 'osdu/platform/ci-cd-pipelines'
+ ref: "master"
+ file: 'cloud-providers/osdu-gcp.yml'
+
+
aws-test-java:
tags: ['aws-internal-test']
diff --git a/NOTICE b/NOTICE
index e5e6ed91f54ee9456702ada24d4cb07c7a40fccf..e87d8d2c9ca99afbefd080659f757b1a731f782d 100644
--- a/NOTICE
+++ b/NOTICE
@@ -875,10 +875,10 @@ The following software have components provided under the terms of this license:
- Microsoft Azure client library for Identity (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure client library for KeyVault Secrets (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure common module for Storage (from https://github.com/Azure/azure-sdk-for-java)
+- Mockito (from http://www.mockito.org)
- Mockito (from http://mockito.org)
- Mockito (from http://mockito.org)
- Mockito (from http://mockito.org)
-- Mockito (from http://www.mockito.org)
- Netty/Codec/HTTP (from )
- Netty/Common (from )
- Plexus :: Default Container (from )
diff --git a/README.md b/README.md
index 596f3c7eaf6b21437d830faf88a560c2f166b2a8..5e081a10b1a16179aad7aadafe2b55fa0e5314c9 100644
--- a/README.md
+++ b/README.md
@@ -4,3 +4,8 @@
The [os-indexer-azure README.md](./provider/indexer-azure/README.md) has all the information needed to get started
running the `os-indexer` Azure implementation
+
+## GCP Implementation
+
+All documentation for the GCP implementation of `os-indexer` lives [here](./provider/indexer-gcp/README.md)
+
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java
index 4fb392ded7f7900b1df3b966c207506ab56072d2..9b287fd5952dc47ff6ea19317a653d50359225dc 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java
@@ -202,7 +202,7 @@ public class StorageServiceImpl implements StorageService {
FetchServiceHttpRequest request = FetchServiceHttpRequest.builder()
.httpMethod(HttpMethods.GET)
.headers(this.requestInfo.getHeadersMap())
- .url(Config.getStorageQueryRecordHostUrl())
+ .url(STORAGE_QUERY_RECORD_HOST)
.queryParams(queryParams)
.build();
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/ElasticClientHandler.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/ElasticClientHandler.java
index 1007967178a2d2c4859ebb8f3f50c88d29c80afc..7c40de07302ec2c533f400f5e3b329dabe2060f4 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/ElasticClientHandler.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/ElasticClientHandler.java
@@ -1,108 +1,152 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.util;
-
-import org.apache.http.Header;
-import org.apache.http.HttpHost;
-import org.apache.http.HttpStatus;
-import org.apache.http.message.BasicHeader;
-import org.elasticsearch.client.RestClient;
-import org.elasticsearch.client.RestClientBuilder;
-import org.elasticsearch.client.RestHighLevelClient;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.search.ClusterSettings;
-import org.opengroup.osdu.core.common.model.indexer.IElasticSettingService;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-import java.util.Base64;
-
-@Component
-public class ElasticClientHandler {
-
- // Elastic cluster Rest client settings
- private static final int CLOUD_REST_CLIENT_PORT = 9243;
- private static final int REST_CLIENT_CONNECT_TIMEOUT = 60000;
- private static final int REST_CLIENT_SOCKET_TIMEOUT = 60000;
- private static final int REST_CLIENT_RETRY_TIMEOUT = 60000;
-
- @Autowired
- private IElasticSettingService elasticSettingService;
-
- public RestHighLevelClient createRestClient() {
- return getCloudRestClient(elasticSettingService.getElasticClusterInformation());
- }
- // TODO: Remove this temporary implementation when ECE CCS is utilized
- public RestHighLevelClient createRestClient(final ClusterSettings clusterSettings) {
- return getCloudRestClient(clusterSettings);
- }
-
- private RestHighLevelClient getCloudRestClient(final ClusterSettings clusterSettings) {
-
- String cluster = null;
- String host = null;
- int port = CLOUD_REST_CLIENT_PORT;
- String protocolScheme = "https";
- String tls = "true";
-
- try {
- cluster = clusterSettings.getHost();
- host = clusterSettings.getHost();
- port = clusterSettings.getPort();
- if(!clusterSettings.isHttps()){
- protocolScheme = "http";
- }
-
- if(!clusterSettings.isTls()){
- tls = "false";
- }
- String basicEncoded = Base64.getEncoder().encodeToString(clusterSettings.getUserNameAndPassword().getBytes());
- String basicAuthenticationHeaderVal = String.format("Basic %s", basicEncoded);
-
- RestClientBuilder builder = createClientBuilder(host, basicAuthenticationHeaderVal, port, protocolScheme, tls);
-
- return new RestHighLevelClient(builder);
- } catch (AppException e) {
- throw e;
- } catch (Exception e) {
- throw new AppException(
- HttpStatus.SC_INTERNAL_SERVER_ERROR,
- "search client error",
- "error creating search client",
- String.format("Elastic client connection params, cluster: %s, host: %s, port: %s", cluster, host, port),
- e);
- }
- }
-
- public RestClientBuilder createClientBuilder(String host, String basicAuthenticationHeaderVal, int port, String protocolScheme, String tls) {
- RestClientBuilder builder = RestClient.builder(new HttpHost(host, port, protocolScheme));
- builder.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(REST_CLIENT_CONNECT_TIMEOUT)
- .setSocketTimeout(REST_CLIENT_SOCKET_TIMEOUT));
- builder.setMaxRetryTimeoutMillis(REST_CLIENT_RETRY_TIMEOUT);
-
- Header[] defaultHeaders = new Header[]{
- new BasicHeader("client.transport.nodes_sampler_interval", "30s"),
- new BasicHeader("client.transport.ping_timeout", "30s"),
- new BasicHeader("client.transport.sniff", "false"),
- new BasicHeader("request.headers.X-Found-Cluster", host),
- new BasicHeader("cluster.name", host),
- new BasicHeader("xpack.security.transport.ssl.enabled", tls),
- new BasicHeader("Authorization", basicAuthenticationHeaderVal),
- };
-
- builder.setDefaultHeaders(defaultHeaders);
- return builder;
- }
+package org.opengroup.osdu.indexer.util;
+
+import java.security.KeyManagementException;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.util.Base64;
+import javax.net.ssl.SSLContext;
+import lombok.extern.java.Log;
+import org.apache.http.Header;
+import org.apache.http.HttpHost;
+import org.apache.http.HttpStatus;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
+import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
+import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
+import org.apache.http.message.BasicHeader;
+import org.apache.http.ssl.SSLContextBuilder;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestClientBuilder;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.indexer.IElasticSettingService;
+import org.opengroup.osdu.core.common.model.search.ClusterSettings;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+@Component
+@Log
+public class ElasticClientHandler {
+
+ // Elastic cluster Rest client settings
+ private static final int CLOUD_REST_CLIENT_PORT = 9243;
+ private static final int REST_CLIENT_CONNECT_TIMEOUT = 60000;
+ private static final int REST_CLIENT_SOCKET_TIMEOUT = 60000;
+ private static final int REST_CLIENT_RETRY_TIMEOUT = 60000;
+
+ @Value("#{new Boolean('${security.https.certificate.trust:false}')}")
+ private Boolean isSecurityHttpsCertificateTrust;
+
+ @Autowired
+ private IElasticSettingService elasticSettingService;
+
+ public RestHighLevelClient createRestClient() {
+ return getCloudRestClient(elasticSettingService.getElasticClusterInformation());
+ }
+
+ // TODO: Remove this temporary implementation when ECE CCS is utilized
+ public RestHighLevelClient createRestClient(final ClusterSettings clusterSettings) {
+ return getCloudRestClient(clusterSettings);
+ }
+
+ private RestHighLevelClient getCloudRestClient(final ClusterSettings clusterSettings) {
+
+ String cluster = null;
+ String host = null;
+ int port = CLOUD_REST_CLIENT_PORT;
+ String protocolScheme = "https";
+ String tls = "true";
+
+ try {
+ cluster = clusterSettings.getHost();
+ host = clusterSettings.getHost();
+ port = clusterSettings.getPort();
+ if (!clusterSettings.isHttps()) {
+ protocolScheme = "http";
+ }
+
+ if (!clusterSettings.isTls()) {
+ tls = "false";
+ }
+ String basicEncoded = Base64
+ .getEncoder().encodeToString(clusterSettings.getUserNameAndPassword().getBytes());
+ String basicAuthenticationHeaderVal = String.format("Basic %s", basicEncoded);
+
+ RestClientBuilder builder = createClientBuilder(host, basicAuthenticationHeaderVal, port,
+ protocolScheme, tls);
+
+ return new RestHighLevelClient(builder);
+ } catch (AppException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AppException(
+ HttpStatus.SC_INTERNAL_SERVER_ERROR,
+ "search client error",
+ "error creating search client",
+ String
+ .format("Elastic client connection params, cluster: %s, host: %s, port: %s", cluster,
+ host, port),
+ e);
+ }
+ }
+
+ public RestClientBuilder createClientBuilder(String host, String basicAuthenticationHeaderVal,
+ int port, String protocolScheme, String tls) {
+ RestClientBuilder builder = RestClient.builder(new HttpHost(host, port, protocolScheme));
+ builder.setRequestConfigCallback(
+ requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(REST_CLIENT_CONNECT_TIMEOUT)
+ .setSocketTimeout(REST_CLIENT_SOCKET_TIMEOUT));
+ builder.setMaxRetryTimeoutMillis(REST_CLIENT_RETRY_TIMEOUT);
+
+ Header[] defaultHeaders = new Header[]{
+ new BasicHeader("client.transport.nodes_sampler_interval", "30s"),
+ new BasicHeader("client.transport.ping_timeout", "30s"),
+ new BasicHeader("client.transport.sniff", "false"),
+ new BasicHeader("request.headers.X-Found-Cluster", host),
+ new BasicHeader("cluster.name", host),
+ new BasicHeader("xpack.security.transport.ssl.enabled", tls),
+ new BasicHeader("Authorization", basicAuthenticationHeaderVal),
+ };
+ log.info(String.format(
+ "Elastic client connection uses protocolScheme = %s with a flag "
+ + "'security.https.certificate.trust' = %s",
+ protocolScheme, isSecurityHttpsCertificateTrust));
+ if ("https".equals(protocolScheme) && isSecurityHttpsCertificateTrust) {
+ log.warning("Elastic client connection uses TrustSelfSignedStrategy()");
+ SSLContext sslContext = createSSLContext();
+ builder.setHttpClientConfigCallback(httpClientBuilder ->
+ {
+ HttpAsyncClientBuilder httpAsyncClientBuilder = httpClientBuilder.setSSLContext(sslContext)
+ .setSSLHostnameVerifier(
+ NoopHostnameVerifier.INSTANCE);
+ return httpAsyncClientBuilder;
+ });
+ }
+
+ builder.setDefaultHeaders(defaultHeaders);
+ return builder;
+ }
+
+ private SSLContext createSSLContext() {
+ SSLContextBuilder sslContextBuilder = new SSLContextBuilder();
+ try {
+ sslContextBuilder.loadTrustMaterial(null, new TrustSelfSignedStrategy());
+ return sslContextBuilder.build();
+ } catch (NoSuchAlgorithmException e) {
+ log.severe(e.getMessage());
+ } catch (KeyStoreException e) {
+ log.severe(e.getMessage());
+ } catch (KeyManagementException e) {
+ log.severe(e.getMessage());
+ }
+ return null;
+ }
+
+ public Boolean isSecurityHttpsCertificateTrust() {
+ return isSecurityHttpsCertificateTrust;
+ }
+
+ public void setSecurityHttpsCertificateTrust(Boolean isSecurityHttpsCertificateTrust) {
+ this.isSecurityHttpsCertificateTrust = isSecurityHttpsCertificateTrust;
+ }
}
\ No newline at end of file
diff --git a/indexer-core/src/test/resources/application.properties b/indexer-core/src/test/resources/application.properties
new file mode 100644
index 0000000000000000000000000000000000000000..50f201247aa58eb42e6ede1d631cd2ce2127a6a8
--- /dev/null
+++ b/indexer-core/src/test/resources/application.properties
@@ -0,0 +1 @@
+security.https.certificate.trust=false
\ No newline at end of file
diff --git a/lombok.config b/lombok.config
new file mode 100644
index 0000000000000000000000000000000000000000..a23edb413fc5d2e329b180ce25ef307f0c408874
--- /dev/null
+++ b/lombok.config
@@ -0,0 +1,2 @@
+config.stopBubbling = true
+lombok.addLombokGeneratedAnnotation = true
\ No newline at end of file
diff --git a/provider/indexer-gcp/README.md b/provider/indexer-gcp/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..23c482574d2515459a8f962b197687472e1bba52
--- /dev/null
+++ b/provider/indexer-gcp/README.md
@@ -0,0 +1,228 @@
+# Indexer Service
+os-indexer-gcp is a [Spring Boot](https://spring.io/projects/spring-boot) service that is responsible for indexing Records that enable the `os-search` service to execute OSDU R2 domain searches against Elasticsearch.
+
+## Getting Started
+These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system.
+
+### Prerequisites
+Pre-requisites
+
+* GCloud SDK with java (latest version)
+* JDK 8
+* Lombok 1.16 or later
+* Maven
+
+### Installation
+In order to run the service locally or remotely, you will need to have the following environment variables defined.
+
+| name | value | description | sensitive? | source |
+| --- | --- | --- | --- | --- |
+| `LOG_PREFIX` | `service` | Logging prefix | no | - |
+| `SERVER_SERVLET_CONTEXPATH` | `/api/indexer/v2` | Servlet context path | no | - |
+| `AUTHORIZE_API` | ex `https://entitlements.com/entitlements/v1` | Entitlements API endpoint | no | output of infrastructure deployment |
+| `ENTITLEMENTS_HOST` | ex `https://entitlements.com/entitlements/v1` | Entitlements API endpoint | no | output of infrastructure deployment |
+| `LEGALTAG_API` | ex `https://legal.com/api/legal/v1` | Legal API endpoint | no | output of infrastructure deployment |
+| `INDEXER_HOST` | ex `os-indexer-dot-opendes.appspot.com` | Indexer Host | no | output of infrastructure deployment |
+| `INDEXER_QUEUE_HOST` | ex `https://os-indexer-queue-dot-opendes.appspot.com/_dps/task-handlers/enqueue` | Indexer-Queue API endpoint | no | output of infrastructure deployment |
+| `CRS_API` | ex `https://crs-converter-gae-dot-opendes.appspot.com/api/crs/v1` | CRS API endpoint | no | https://console.cloud.google.com/memorystore/redis/instances |
+| `STORAGE_HOSTNAME` | ex `os-storage-dot-opendes.appspot.com` | Storage Host | no | output of infrastructure deployment |
+| `STORAGE_SCHEMA_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/schemas` | Storage API endpoint 'schemas' | no | https://console.cloud.google.com/apis/credentials |
+| `STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records:batch` | Storage API endpoint 'records' | no | https://console.cloud.google.com/iam-admin/serviceaccounts |
+| `REDIS_SEARCH_HOST` | ex `127.0.0.1` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances |
+| `REDIS_GROUP_HOST` | ex `127.0.0.1` | Redis host for groups | no | https://console.cloud.google.com/memorystore/redis/instances |
+| `REDIS_SEARCH_PORT` | ex `6379` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances |
+| `GOOGLE_CLOUD_PROJECT` | ex `opendes` | Google Cloud Project Id| no | output of infrastructure deployment |
+| `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials |
+| `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
+| `security.https.certificate.trust` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment |
+
+
+### Run Locally
+Check that maven is installed:
+
+```bash
+$ mvn --version
+Apache Maven 3.6.0
+Maven home: /usr/share/maven
+Java version: 1.8.0_212, vendor: AdoptOpenJDK, runtime: /usr/lib/jvm/jdk8u212-b04/jre
+...
+```
+
+You may need to configure access to the remote maven repository that holds the OSDU dependencies. This file should live within `~/.mvn/community-maven.settings.xml`:
+
+```bash
+$ cat ~/.m2/settings.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
+ <servers>
+ <server>
+ <id>community-maven-via-private-token</id>
+ <!-- Treat this auth token like a password. Do not share it with anyone, including Microsoft support. -->
+ <!-- The generated token expires on or before 11/14/2019 -->
+ <configuration>
+ <httpHeaders>
+ <property>
+ <name>Private-Token</name>
+ <value>${env.COMMUNITY_MAVEN_TOKEN}</value>
+ </property>
+ </httpHeaders>
+ </configuration>
+ </server>
+ </servers>
+</settings>
+```
+
+* Update the Google cloud SDK to the latest version:
+
+```bash
+gcloud components update
+```
+* Set Google Project Id:
+
+```bash
+gcloud config set project <YOUR-PROJECT-ID>
+```
+
+* Perform a basic authentication in the selected project:
+
+```bash
+gcloud auth application-default login
+```
+
+* Navigate to indexer service's root folder and run:
+
+```bash
+mvn jetty:run
+## Testing
+* Navigate to indexer service's root folder and run:
+
+```bash
+mvn clean install
+```
+
+* If you wish to see the coverage report then go to testing/target/site/jacoco-aggregate and open index.html
+
+* If you wish to build the project without running tests
+
+```bash
+mvn clean install -DskipTests
+```
+
+After configuring your environment as specified above, you can follow these steps to build and run the application. These steps should be invoked from the *repository root.*
+
+```bash
+cd provider/indexer-gcp/ && mvn spring-boot:run
+```
+
+## Testing
+Navigate to indexer service's root folder and run all the tests:
+
+```bash
+# build + install integration test core
+$ (cd testing/indexer-test-core/ && mvn clean install)
+```
+
+### Running E2E Tests
+This section describes how to run cloud OSDU E2E tests (testing/integration-tests/indexer-test-gcp).
+
+You will need to have the following environment variables defined.
+
+| name | value | description | sensitive? | source |
+| --- | --- | --- | --- | --- |
+| `ENTITLEMENTS_HOST` | ex `https://entitlements.com/entitlements/v1` | Entitlements API endpoint | no | output of infrastructure deployment |
+| `ELASTIC_PASSWORD` | `********` | Password for Elasticsearch | yes | output of infrastructure deployment |
+| `ELASTIC_USER_NAME` | `********` | User name for Elasticsearch | yes | output of infrastructure deployment |
+| `ELASTIC_HOST` | ex `elastic.domain.com` | Host Elasticsearch | yes | output of infrastructure deployment |
+| `ELASTIC_PORT` | ex `9243` | Port Elasticsearch | yes | output of infrastructure deployment |
+| `GCLOUD_PROJECT` | ex `opendes` | Google Cloud Project Id| no | output of infrastructure deployment |
+| `INDEXER_HOST` | ex `https://os-indexer-dot-opendes.appspot.com/api/indexer/v2/` | Indexer API endpoint | no | output of infrastructure deployment |
+| `DATA_GROUP` | `opendes` | The service account to this group and substitute | no | - |
+| `ENTITLEMENTS_DOMAIN` | ex `opendes-gcp.projects.com` | OSDU R2 to run tests under | no | - |
+| `INTEGRATION_TEST_AUDIENCE` | `********` | client application ID | yes | https://console.cloud.google.com/apis/credentials |
+| `OTHER_RELEVANT_DATA_COUNTRIES` | ex `US` | valid legal tag with a other relevant data countries | no | - |
+| `LEGAL_TAG` | ex `opendes-demo-legaltag` | valid legal tag with a other relevant data countries from `DEFAULT_OTHER_RELEVANT_DATA_COUNTRIES` | no | - |
+| `DEFAULT_DATA_PARTITION_ID_TENANT1` | ex `opendes` | HTTP Header 'Data-Partition-ID' | no | - |
+| `SEARCH_INTEGRATION_TESTER` | `********` | Service account for API calls. Note: this user must have entitlements configured already | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
+| `SEARCH_HOST` | ex `http://localhost:8080/api/search/v2/` | Endpoint of search service | no | - |
+| `STORAGE_HOST` | ex `http://os-storage-dot-opendes.appspot.com/api/storage/v2/schemas` | Storage API endpoint | Storage Host | no | output of infrastructure deployment |
+
+**Entitlements configuration for integration accounts**
+
+| INTEGRATION_TESTER | NO_DATA_ACCESS_TESTER |
+| --- | --- |
+| users<br/>service.entitlements.user<br/>service.search.user<br/>data.test1<br/>data.integration.test<br/>users@{tenant1}@{domain}.com |
+
+Execute following command to build code and run all the integration tests:
+
+```bash
+# Note: this assumes that the environment variables for integration tests as outlined
+# above are already exported in your environment.
+$ (cd testing/indexer-test-gcp/ && mvn clean test)
+```
+
+## Deployment
+
+* Data-Lake Indexer Google Cloud Endpoints on App Engine Flex environment
+ * Edit the app.yaml
+ * Open the [app.yaml](indexer/src/main/appengine/app.yaml) file in editor, and replace the YOUR-PROJECT-ID `GOOGLE_CLOUD_PROJECT` line with Google Cloud Platform project Id. Also update `AUTHORIZE_API`, `CRON_JOB_IP`, `LEGAL_HOSTNAME`, `REGION` and `SECURITY_HTTPS_CERTIFICATE_TRUST` based on your deployment
+
+ * Deploy
+ ```sh
+ mvn appengine:deploy -pl org.opengroup.osdu.indexer:indexer -amd
+ ```
+
+ * If you wish to deploy the search service without running tests
+ ```sh
+ mvn appengine:deploy -pl org.opengroup.osdu.indexer:indexer -amd -DskipTests
+ ```
+
+or
+* Google Documentation: https://cloud.google.com/cloud-build/docs/deploying-builds/deploy-appengine
+
+#### Cloud KMS Setup
+
+Enable cloud KMS on master project
+
+Create king ring and key in the ***master project***
+
+```bash
+ gcloud services enable cloudkms.googleapis.com
+ export KEYRING_NAME="csqp"
+ export CRYPTOKEY_NAME="searchService"
+ gcloud kms keyrings create $KEYRING_NAME --location global
+ gcloud kms keys create $CRYPTOKEY_NAME --location global \
+ --keyring $KEYRING_NAME \
+ --purpose encryption
+```
+
+Add **Cloud KMS CryptoKey Encrypter/Decrypter** role to the **App Engine default service account** of the master project through IAM - Role tab
+
+Add "Cloud KMS Encrypt/Decrypt" role to the "App Engine default service account" of ***master project***
+
+#### Memory Store (Redis Instance) Setup
+
+Create a new Standard tier Redis instance on the ***service project***
+
+The Redis instance must be created under the same region with the App Engine application which needs to access it.
+
+```bash
+ gcloud beta redis instances create redis-cache-search --size=10 --region=<service-deployment-region> --zone=<service-deployment-zone> --tier=STANDARD
+```
+
+## Licence
+Copyright © Google LLC
+Copyright © EPAM Systems
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+[http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0)
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
\ No newline at end of file
diff --git a/provider/indexer-gcp/pom.xml b/provider/indexer-gcp/pom.xml
index 09cc7e5240b1cfef8ffab0eb432888c8c8d9e640..fe4e902e8ea3f5a5b829b90ad822637b4a02d465 100644
--- a/provider/indexer-gcp/pom.xml
+++ b/provider/indexer-gcp/pom.xml
@@ -202,7 +202,26 @@
<useSystemClassLoader>false</useSystemClassLoader>
<threadCount>1</threadCount>
</configuration>
- </plugin>
+ </plugin>
+ <plugin>
+ <groupId>org.jacoco</groupId>
+ <artifactId>jacoco-maven-plugin</artifactId>
+ <version>0.7.7.201606060606</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>prepare-agent</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>report</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>report</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
diff --git a/provider/indexer-gcp/src/main/appengine/app.yaml b/provider/indexer-gcp/src/main/appengine/app.yaml
index 16d9a8ac495f04fa88c5ec438902c621dc08bf96..6d89c2e56b566564d1d4e58b3fd4b7a02ceeceab 100644
--- a/provider/indexer-gcp/src/main/appengine/app.yaml
+++ b/provider/indexer-gcp/src/main/appengine/app.yaml
@@ -39,4 +39,5 @@ env_variables:
LEGAL_HOSTNAME: "LEGAL_HOSTNAME_VAR"
REGION: "REGION_VAR"
SPRING_PROFILES_ACTIVE: 'ENVIRONMENT'
+ SECURITY_HTTPS_CERTIFICATE_TRUST: 'SECURITY_HTTPS_CERTIFICATE_TRUST_VAR'
diff --git a/provider/indexer-gcp/src/main/resources/application-kuber.properties b/provider/indexer-gcp/src/main/resources/application-kuber.properties
index af85274d4030d56204400765876db4eeb3c9d70a..bc2a5f9014fa1b406a008e2cc69f161a8e3a9a8b 100644
--- a/provider/indexer-gcp/src/main/resources/application-kuber.properties
+++ b/provider/indexer-gcp/src/main/resources/application-kuber.properties
@@ -21,4 +21,4 @@ REDIS_SEARCH_HOST=${REDIS_SEARCH_HOST}
GOOGLE_AUDIENCES=${GOOGLE_AUDIENCES}
DEPLOYMENT_ENVIRONMENT=CLOUD
-disable.appengine.log.factory=true
+disable.appengine.log.factory=true
\ No newline at end of file
diff --git a/provider/indexer-gcp/src/main/resources/application-testing.properties b/provider/indexer-gcp/src/main/resources/application-testing.properties
index 5368d291c5653bcf9bd31bfbc9c31af3765e9b1f..fb0c9e2d6ce29c87dca6f58d1afec0b420c1e663 100644
--- a/provider/indexer-gcp/src/main/resources/application-testing.properties
+++ b/provider/indexer-gcp/src/main/resources/application-testing.properties
@@ -18,4 +18,4 @@ CRS_API=https://crs-converter-gae-dot-opendes-evt.appspot.com/api/crs/v1
REDIS_GROUP_HOST=10.253.209.196
REDIS_SEARCH_HOST=10.118.2.140
-GOOGLE_AUDIENCES=833591776864-oobhqvmtdg9rpreubjvn44m5f8revglk.apps.googleusercontent.com
+GOOGLE_AUDIENCES=833591776864-oobhqvmtdg9rpreubjvn44m5f8revglk.apps.googleusercontent.com
\ No newline at end of file
diff --git a/provider/indexer-gcp/src/main/resources/application.properties b/provider/indexer-gcp/src/main/resources/application.properties
index 1ccde084878626a19f762e0fd8abb6a650450447..a0cbce3427ff62ba4d63ffabc17323efc833295b 100644
--- a/provider/indexer-gcp/src/main/resources/application.properties
+++ b/provider/indexer-gcp/src/main/resources/application.properties
@@ -32,3 +32,5 @@ KMS_KEY=searchService
ELASTIC_DATASTORE_KIND=SearchSettings
ELASTIC_DATASTORE_ID=indexer-service
+
+security.https.certificate.trust=false
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/HTTPClient.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/HTTPClient.java
index 28d7aa69c95bbf74a9cb547f549c6b54753aa3c1..1ce98c78a3710d279c9500d01a80e2ccaa0075f2 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/HTTPClient.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/HTTPClient.java
@@ -62,6 +62,7 @@ public abstract class HTTPClient {
Client client = getClient();
client.setReadTimeout(180000);
client.setConnectTimeout(10000);
+ log.info("URL: = " + url);
WebResource webResource = client.resource(url);
response = this.getClientResponse(httpMethod, payLoad, webResource, headers, token);
} catch (Exception e) {
diff --git a/testing/indexer-test-core/src/main/resources/features/indexrecord/IndexRecord.feature b/testing/indexer-test-core/src/main/resources/features/indexrecord/IndexRecord.feature
index 30ee32eaa576dc0a7369c0a29a9110a20c566775..c281ec4a0b174ec98eb781f8e603eba3b3968ca1 100644
--- a/testing/indexer-test-core/src/main/resources/features/indexrecord/IndexRecord.feature
+++ b/testing/indexer-test-core/src/main/resources/features/indexrecord/IndexRecord.feature
@@ -15,8 +15,8 @@ Feature: Indexing of the documents
Examples:
| kind | recordFile | number | index | type | acl | mapping |
- | "tenant1:testindex<timestamp>:well:1.0.0" | "index_records_1" | 5 | "tenant1-testindex<timestamp>-well-1.0.0" | "well" | "data.default.viewers@opendes" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
- | "tenant1:testindex<timestamp>:well:3.0.0" | "index_records_1" | 5 | "tenant1-testindex<timestamp>-well-3.0.0" | "well" | "data.default.viewers@opendes" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
+ | "tenant1:testindex<timestamp>:well:1.0.0" | "index_records_1" | 5 | "tenant1-testindex<timestamp>-well-1.0.0" | "well" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
+ | "tenant1:testindex<timestamp>:well:3.0.0" | "index_records_1" | 5 | "tenant1-testindex<timestamp>-well-3.0.0" | "well" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
Scenario Outline: Ingest the record and Index in the Elastic Search with bad attribute
When I ingest records with the <recordFile> with <acl> for a given <kind>
@@ -24,5 +24,5 @@ Feature: Indexing of the documents
Examples:
| kind | recordFile | number | index | skippedAttribute | acl |
- | "tenant1:testindex<timestamp>:well:2.0.0" | "index_records_2" | 4 | "tenant1-testindex<timestamp>-well-2.0.0" | "data.Location" | "data.default.viewers@opendes" |
- | "tenant1:testindex<timestamp>:well:3.0.0" | "index_records_3" | 7 | "tenant1-testindex<timestamp>-well-3.0.0" | "data.GeoShape" | "data.default.viewers@opendes" |
+ | "tenant1:testindex<timestamp>:well:2.0.0" | "index_records_2" | 4 | "tenant1-testindex<timestamp>-well-2.0.0" | "data.Location" | "data.default.viewers@tenant1" |
+ | "tenant1:testindex<timestamp>:well:3.0.0" | "index_records_3" | 7 | "tenant1-testindex<timestamp>-well-3.0.0" | "data.GeoShape" | "data.default.viewers@tenant1" |