diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 726e9af5cd8e2492aa1014a506889326d78b9cf9..f03a7d4989795cac0acbcbbb43dd28911d1e05bd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,6 +14,14 @@ variables: GCP_DOMAIN: cloud.slb-ds.com GCP_STORAGE_URL: https://osdu-indexer-dot-opendes.appspot.com/api/storage/v2/ + OSDU_GCP_BUILD_SUBDIR: provider/indexer-gcp + OSDU_GCP_INT_TEST_SUBDIR: testing/indexer-test-gcp + OSDU_GCP_APPLICATION_NAME: os-indexer + OSDU_GCP_PROJECT: nice-etching-277309 + OSDU_GCP_TENANT_NAME: osdu + OSDU_GCP_STORAGE_SCHEMA_HOST: https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/schemas + OSDU_SECURITY_HTTPS_CERTIFICATE_TRUST: 'true' + IBM_BUILD_SUBDIR: provider/indexer-ibm IBM_INT_TEST_SUBDIR: testing/indexer-test-ibm @@ -46,5 +54,10 @@ include: - project: "osdu/platform/ci-cd-pipelines" file: "publishing/pages.yml" + - project: 'osdu/platform/ci-cd-pipelines' + ref: "master" + file: 'cloud-providers/osdu-gcp.yml' + + aws-test-java: tags: ['aws-internal-test'] diff --git a/NOTICE b/NOTICE index e5e6ed91f54ee9456702ada24d4cb07c7a40fccf..3483ecfed18e16e6594381f48db4aac3fb0f202b 100644 --- a/NOTICE +++ b/NOTICE @@ -11,7 +11,6 @@ The following software have components provided under the terms of this license: - Cobertura code coverage (from http://cobertura.sourceforge.net) - Plexus :: Default Container (from ) - Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils) -- StAX (from http://stax.codehaus.org/) - oro (from ) ======================================================================== @@ -244,6 +243,10 @@ The following software have components provided under the terms of this license: - Guava: Google Core Libraries for Java (from https://github.com/google/guava.git) - HPPC Collections (from http://labs.carrotsearch.com) - Hibernate Validator Engine (from ) +- IBM COS Java SDK for Amazon S3 (from https://github.com/ibm/ibm-cos-sdk-java) +- IBM COS Java SDK for COS KMS (from https://github.com/ibm/ibm-cos-sdk-java) +- IBM COS SDK For Java (from https://github.com/ibm/ibm-cos-sdk-java) +- IBM COS SDK for Java - Core (from https://github.com/ibm/ibm-cos-sdk-java) - Identity and Access Management (IAM) API v1-rev247-1.23.0 (from ) - J2ObjC Annotations (from https://github.com/google/j2objc/) - J2ObjC Annotations (from https://github.com/google/j2objc/) @@ -336,9 +339,9 @@ The following software have components provided under the terms of this license: - Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Azure Netty HTTP Client Library (from https://github.com/Azure/azure-sdk-for-java) - Mockito (from http://mockito.org) -- Mockito (from http://mockito.org) - Mockito (from http://www.mockito.org) - Mockito (from http://mockito.org) +- Mockito (from http://mockito.org) - Mojo's Maven plugin for Cobertura (from http://mojo.codehaus.org/cobertura-maven-plugin/) - Netty Reactive Streams Implementation (from ) - Netty/Buffer (from http://netty.io/) @@ -368,7 +371,6 @@ The following software have components provided under the terms of this license: - OkHttp URLConnection (from ) - OkHttp URLConnection (from ) - Okio (from ) -- Okio (from ) - OpenCensus (from https://github.com/census-instrumentation/opencensus-java) - OpenCensus (from https://github.com/census-instrumentation/opencensus-java) - OpenCensus (from https://github.com/census-instrumentation/opencensus-java) @@ -385,7 +387,6 @@ The following software have components provided under the terms of this license: - Reactive Streams Netty driver (from https://github.com/reactor/reactor-netty) - Retrofit (from ) - Servlet Specification 2.5 API (from ) -- Simple XML (from http://simple.sourceforge.net) - SnakeYAML (from http://www.snakeyaml.org) - Spatial4J (from http://www.locationtech.org/projects/locationtech.spatial4j) - Spring AOP (from https://github.com/spring-projects/spring-framework) @@ -425,8 +426,6 @@ The following software have components provided under the terms of this license: - Spring Transaction (from https://github.com/spring-projects/spring-framework) - Spring Web (from https://github.com/spring-projects/spring-framework) - Spring Web MVC (from https://github.com/spring-projects/spring-framework) -- StAX (from http://stax.codehaus.org/) -- StAX API (from http://stax.codehaus.org/) - T-Digest (from https://github.com/tdunning/t-digest) - Woodstox (from https://github.com/FasterXML/woodstox) - Xerces2-j (from https://xerces.apache.org/xerces2-j/) @@ -457,6 +456,7 @@ The following software have components provided under the terms of this license: - io.grpc:grpc-protobuf-lite (from https://github.com/grpc/grpc-java) - io.grpc:grpc-stub (from https://github.com/grpc/grpc-java) - ion-java (from https://github.com/amznlabs/ion-java/) +- ion-java (from https://github.com/amznlabs/ion-java/) - jackson-databind (from http://github.com/FasterXML/jackson) - java-cloudant (from https://cloudant.com) - java-cloudant (from https://cloudant.com) @@ -479,7 +479,6 @@ The following software have components provided under the terms of this license: - lettuce (from http://github.com/mp911de/lettuce/wiki) - micrometer-core (from https://github.com/micrometer-metrics/micrometer) - micrometer-registry-azure-monitor (from https://github.com/micrometer-metrics/micrometer) -- minio (from https://github.com/minio/minio-java) - org.xmlunit:xmlunit-core (from http://www.xmlunit.org/) - oro (from ) - parent-join (from https://github.com/elastic/elasticsearch) @@ -542,7 +541,6 @@ The following software have components provided under the terms of this license: - Lucene Common Analyzers (from ) - Plexus :: Default Container (from ) - Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils) -- StAX (from http://stax.codehaus.org/) - Stax2 API (from http://github.com/FasterXML/stax2-api) - jersey-ext-bean-validation (from ) - jersey-spring4 (from ) @@ -616,7 +614,6 @@ CC-BY-3.0 ======================================================================== The following software have components provided under the terms of this license: -- "Java Concurrency in Practice" book annotations (from http://jcip.net/) - FindBugs-jsr305 (from http://findbugs.sourceforge.net/) ======================================================================== @@ -813,7 +810,6 @@ The following software have components provided under the terms of this license: - Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java) -- SpotBugs Annotations (from https://spotbugs.github.io/) - java-getopt (from ) ======================================================================== @@ -825,7 +821,6 @@ The following software have components provided under the terms of this license: - Java Native Access Platform (from https://github.com/java-native-access/jna) - Javassist (from http://www.javassist.org/) - SnakeYAML (from http://www.snakeyaml.org) -- SpotBugs Annotations (from https://spotbugs.github.io/) ======================================================================== LGPL-3.0-only @@ -875,10 +870,10 @@ The following software have components provided under the terms of this license: - Microsoft Azure client library for Identity (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure client library for KeyVault Secrets (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure common module for Storage (from https://github.com/Azure/azure-sdk-for-java) +- Mockito (from http://www.mockito.org) - Mockito (from http://mockito.org) - Mockito (from http://mockito.org) - Mockito (from http://mockito.org) -- Mockito (from http://www.mockito.org) - Netty/Codec/HTTP (from ) - Netty/Common (from ) - Plexus :: Default Container (from ) diff --git a/README.md b/README.md index 596f3c7eaf6b21437d830faf88a560c2f166b2a8..5e081a10b1a16179aad7aadafe2b55fa0e5314c9 100644 --- a/README.md +++ b/README.md @@ -4,3 +4,8 @@ The [os-indexer-azure README.md](./provider/indexer-azure/README.md) has all the information needed to get started running the `os-indexer` Azure implementation + +## GCP Implementation + +All documentation for the GCP implementation of `os-indexer` lives [here](./provider/indexer-gcp/README.md) + diff --git a/devops/Jenkinsfile b/devops/Jenkinsfile new file mode 100644 index 0000000000000000000000000000000000000000..84a24dd3c826eb52acad13826d342fd1d493c3b9 --- /dev/null +++ b/devops/Jenkinsfile @@ -0,0 +1,85 @@ +pipeline { + agent { + kubernetes { + cloud 'openshift' + label 'maven-openjdk18' + yaml """ +spec: + containers: + - name: jnlp + image: quay.io/openshift/origin-jenkins-agent-maven:v4.0.0 + volumeMounts: + - mountPath: "/tmp" + name: "workspace-volume" + readOnly: false + workingDir: "/tmp" + securityContext: + privileged: false + tty: false + resources: + limits: + cpu: 200m + memory: 2Gi + requests: + cpu: 200m + memory: 2Gi + restartPolicy: "Never" +""" + } + } + + environment { + + //Cluster environment variable(CLS_ENV). Like QA, DEV, PERF, PROD etc. + CLS_ENV = "dev" + + //Service variable(CORE_SERVICE). Like indexer, search, delivery, storage, legal etc. + CORE_SERVICE = "indexer" + + //GitHub repo URL credential ID for Environment variable files which saved as Secure text in Jenkins Credential. + GIT_ENV_VAR_PATH_URL = credentials('GitRepo-URL-For-Environment-variables') + + //Personal token variable ID which saved as Secure text in Jenkins Credential. Like: GitHub-PRIVATE-TOKEN. + PRIVATE_TOKEN = credentials('GitHub-PRIVATE-TOKEN') + + def runShell = sh (returnStdout: true, script: "curl --header 'PRIVATE-TOKEN: $PRIVATE_TOKEN' ''$GIT_ENV_VAR_PATH_URL'%2F'$CORE_SERVICE'_'$CLS_ENV'_env.json/raw?ref=master' -s -o env.json") + + } + + stages { + stage('Integration_test') { + environment { + def readContent = readJSON file: 'env.json' + + AUTH_USER_ACCESS = "${readContent['AUTH_USER_ACCESS']}" + AUTH_USER_ACCESS_PASSWORD = "${readContent['AUTH_USER_ACCESS_PASSWORD']}" + DEFAULT_DATA_PARTITION_ID_TENANT1 = "${readContent['DEFAULT_DATA_PARTITION_ID_TENANT1']}" + DEFAULT_DATA_PARTITION_ID_TENANT2 = "${readContent['DEFAULT_DATA_PARTITION_ID_TENANT2']}" + ELASTIC_HOST = "${readContent['ELASTIC_HOST']}" + ELASTIC_PASSWORD = "${readContent['ELASTIC_PASSWORD']}" + ELASTIC_PORT = "${readContent['ELASTIC_PORT']}" + ELASTIC_USER_NAME = "${readContent['ELASTIC_USER_NAME']}" + ENTITLEMENTS_DOMAIN = "${readContent['ENTITLEMENTS_DOMAIN']}" + KEYCLOAK_CLIENT_ID = "${readContent['KEYCLOAK_CLIENT_ID']}" + KEYCLOAK_CLIENT_SECRET = "${readContent['KEYCLOAK_CLIENT_SECRET']}" + KEYCLOAK_REALM = "${readContent['KEYCLOAK_REALM']}" + KEYCLOAK_URL = "${readContent['KEYCLOAK_URL']}" + LEGAL_TAG = "${readContent['LEGAL_TAG']}" + OTHER_RELEVANT_DATA_COUNTRIES = "${readContent['OTHER_RELEVANT_DATA_COUNTRIES']}" + STORAGE_HOST = "${readContent['STORAGE_HOST']}" + SEARCH_HOST = "${readContent['SEARCH_HOST']}" + LEGAL_HOST = "${readContent['LEGAL_HOST']}" + INDEXER_HOST = "${readContent['INDEXER_HOST']}" + ENTITLEMENT_URL = "${readContent['ENTITLEMENT_URL']}" + + } + steps { + script { + sh 'mvn -f testing/indexer-test-ibm/pom.xml test' + } + } + } + } + + +} \ No newline at end of file diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/CleanupIndiciesApi.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/CleanupIndiciesApi.java new file mode 100644 index 0000000000000000000000000000000000000000..347caf52d5057b0dd8e3b67c1e517f9bbe1ccab9 --- /dev/null +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/CleanupIndiciesApi.java @@ -0,0 +1,81 @@ +// Copyright 2017-2019, Schlumberger +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.opengroup.osdu.indexer.api; + +import com.google.common.reflect.TypeToken; +import com.google.gson.Gson; +import com.google.gson.JsonParseException; +import java.lang.reflect.Type; +import java.util.List; +import javax.validation.Valid; +import javax.validation.constraints.NotNull; +import lombok.extern.java.Log; +import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.core.common.model.http.DpsHeaders; +import org.opengroup.osdu.core.common.model.indexer.RecordInfo; +import org.opengroup.osdu.core.common.model.search.RecordChangedMessages; +import org.opengroup.osdu.core.common.model.search.SearchServiceRole; +import org.opengroup.osdu.indexer.SwaggerDoc; +import org.opengroup.osdu.indexer.service.IndexerService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.context.annotation.RequestScope; + +@Log +@RestController +@RequestScope +public class CleanupIndiciesApi { + + @Autowired + private IndexerService indexerService; + + @PostMapping(path = "/index-cleanup", consumes = "application/json") + @PreAuthorize("@authorizationFilter.hasPermission('" + SearchServiceRole.ADMIN + "')") + public ResponseEntity cleanupIndices(@NotNull(message = SwaggerDoc.REQUEST_VALIDATION_NOT_NULL_BODY) + @Valid @RequestBody RecordChangedMessages message) { + if (message == null) { + throw new AppException(HttpStatus.BAD_REQUEST.value(), "Request body is null", + SwaggerDoc.REQUEST_VALIDATION_NOT_NULL_BODY); + } + + if (message.missingAccountId()) { + throw new AppException(HttpStatus.BAD_REQUEST.value(), "Invalid tenant", + String.format("Required header: '%s' not found", DpsHeaders.DATA_PARTITION_ID)); + } + try { + Type listType = new TypeToken<List<RecordInfo>>() { + }.getType(); + List<RecordInfo> recordInfos = new Gson().fromJson(message.getData(), listType); + + if (recordInfos.isEmpty()) { + log.info("none of record-change message can be deserialized"); + return new ResponseEntity(HttpStatus.OK); + } + indexerService.processSchemaMessages(recordInfos); + return new ResponseEntity(HttpStatus.OK); + } catch (AppException e) { + throw e; + } catch (JsonParseException e) { + throw new AppException(HttpStatus.BAD_REQUEST.value(), "Request payload parsing error", "Unable to parse request payload.", e); + } catch (Exception e) { + throw new AppException(HttpStatus.BAD_REQUEST.value(), "Unknown error", "An unknown error has occurred.", e); + } + } +} diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerService.java index 4ae323c7656efb92132a5c64c69564f867ed183c..8feadb3d332414480d82f9082f621ccb86a17521 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerService.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerService.java @@ -14,6 +14,7 @@ package org.opengroup.osdu.indexer.service; +import java.io.IOException; import java.util.List; import org.opengroup.osdu.core.common.model.indexer.JobStatus; @@ -24,4 +25,6 @@ public interface IndexerService { JobStatus processRecordChangedMessages(RecordChangedMessages recordChangedMessages, List<RecordInfo> recordInfos) throws Exception; + void processSchemaMessages(List<RecordInfo> recordInfos) throws IOException; + } \ No newline at end of file diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java index 0fa06829611148c5ed9525dc71a1c4649f995d43..36f91af2405098e188a4113b009e165dfd4500be 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java @@ -153,6 +153,33 @@ public class IndexerServiceImpl implements IndexerService { return jobStatus; } + @Override + public void processSchemaMessages(List<RecordInfo> recordInfos) throws IOException { + Map<String, OperationType> schemaMsgs = RecordInfo.getSchemaMsgs(recordInfos); + if (schemaMsgs != null && !schemaMsgs.isEmpty()) { + try (RestHighLevelClient restClient = elasticClientHandler.createRestClient()) { + schemaMsgs.entrySet().forEach(msg -> { + try { + processSchemaEvents(restClient, msg); + } catch (IOException | ElasticsearchStatusException e) { + throw new AppException(org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR.value(), "unable to process schema delete", e.getMessage()); + } + }); + } + } + } + + private void processSchemaEvents(RestHighLevelClient restClient, + Map.Entry<String, OperationType> msg) throws IOException, ElasticsearchStatusException { + String kind = msg.getKey(); + String index = elasticIndexNameResolver.getIndexNameFromKind(kind); + + boolean indexExist = indicesService.isIndexExist(restClient, index); + if (indexExist && msg.getValue() == OperationType.purge_schema) { + indicesService.deleteIndex(restClient, index); + } + } + private List<String> processUpsertRecords(Map<String, Map<String, OperationType>> upsertRecordMap) throws Exception { // get schema for kind Map<String, IndexSchema> schemas = this.getSchema(upsertRecordMap); diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java index 4fb392ded7f7900b1df3b966c207506ab56072d2..9b287fd5952dc47ff6ea19317a653d50359225dc 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java @@ -202,7 +202,7 @@ public class StorageServiceImpl implements StorageService { FetchServiceHttpRequest request = FetchServiceHttpRequest.builder() .httpMethod(HttpMethods.GET) .headers(this.requestInfo.getHeadersMap()) - .url(Config.getStorageQueryRecordHostUrl()) + .url(STORAGE_QUERY_RECORD_HOST) .queryParams(queryParams) .build(); diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/ElasticClientHandler.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/ElasticClientHandler.java index 1007967178a2d2c4859ebb8f3f50c88d29c80afc..7c40de07302ec2c533f400f5e3b329dabe2060f4 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/ElasticClientHandler.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/ElasticClientHandler.java @@ -1,108 +1,152 @@ -// Copyright 2017-2019, Schlumberger -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.opengroup.osdu.indexer.util; - -import org.apache.http.Header; -import org.apache.http.HttpHost; -import org.apache.http.HttpStatus; -import org.apache.http.message.BasicHeader; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.RestClientBuilder; -import org.elasticsearch.client.RestHighLevelClient; -import org.opengroup.osdu.core.common.model.http.AppException; -import org.opengroup.osdu.core.common.model.search.ClusterSettings; -import org.opengroup.osdu.core.common.model.indexer.IElasticSettingService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import java.util.Base64; - -@Component -public class ElasticClientHandler { - - // Elastic cluster Rest client settings - private static final int CLOUD_REST_CLIENT_PORT = 9243; - private static final int REST_CLIENT_CONNECT_TIMEOUT = 60000; - private static final int REST_CLIENT_SOCKET_TIMEOUT = 60000; - private static final int REST_CLIENT_RETRY_TIMEOUT = 60000; - - @Autowired - private IElasticSettingService elasticSettingService; - - public RestHighLevelClient createRestClient() { - return getCloudRestClient(elasticSettingService.getElasticClusterInformation()); - } - // TODO: Remove this temporary implementation when ECE CCS is utilized - public RestHighLevelClient createRestClient(final ClusterSettings clusterSettings) { - return getCloudRestClient(clusterSettings); - } - - private RestHighLevelClient getCloudRestClient(final ClusterSettings clusterSettings) { - - String cluster = null; - String host = null; - int port = CLOUD_REST_CLIENT_PORT; - String protocolScheme = "https"; - String tls = "true"; - - try { - cluster = clusterSettings.getHost(); - host = clusterSettings.getHost(); - port = clusterSettings.getPort(); - if(!clusterSettings.isHttps()){ - protocolScheme = "http"; - } - - if(!clusterSettings.isTls()){ - tls = "false"; - } - String basicEncoded = Base64.getEncoder().encodeToString(clusterSettings.getUserNameAndPassword().getBytes()); - String basicAuthenticationHeaderVal = String.format("Basic %s", basicEncoded); - - RestClientBuilder builder = createClientBuilder(host, basicAuthenticationHeaderVal, port, protocolScheme, tls); - - return new RestHighLevelClient(builder); - } catch (AppException e) { - throw e; - } catch (Exception e) { - throw new AppException( - HttpStatus.SC_INTERNAL_SERVER_ERROR, - "search client error", - "error creating search client", - String.format("Elastic client connection params, cluster: %s, host: %s, port: %s", cluster, host, port), - e); - } - } - - public RestClientBuilder createClientBuilder(String host, String basicAuthenticationHeaderVal, int port, String protocolScheme, String tls) { - RestClientBuilder builder = RestClient.builder(new HttpHost(host, port, protocolScheme)); - builder.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(REST_CLIENT_CONNECT_TIMEOUT) - .setSocketTimeout(REST_CLIENT_SOCKET_TIMEOUT)); - builder.setMaxRetryTimeoutMillis(REST_CLIENT_RETRY_TIMEOUT); - - Header[] defaultHeaders = new Header[]{ - new BasicHeader("client.transport.nodes_sampler_interval", "30s"), - new BasicHeader("client.transport.ping_timeout", "30s"), - new BasicHeader("client.transport.sniff", "false"), - new BasicHeader("request.headers.X-Found-Cluster", host), - new BasicHeader("cluster.name", host), - new BasicHeader("xpack.security.transport.ssl.enabled", tls), - new BasicHeader("Authorization", basicAuthenticationHeaderVal), - }; - - builder.setDefaultHeaders(defaultHeaders); - return builder; - } +package org.opengroup.osdu.indexer.util; + +import java.security.KeyManagementException; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.util.Base64; +import javax.net.ssl.SSLContext; +import lombok.extern.java.Log; +import org.apache.http.Header; +import org.apache.http.HttpHost; +import org.apache.http.HttpStatus; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.TrustSelfSignedStrategy; +import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; +import org.apache.http.message.BasicHeader; +import org.apache.http.ssl.SSLContextBuilder; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.client.RestHighLevelClient; +import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.core.common.model.indexer.IElasticSettingService; +import org.opengroup.osdu.core.common.model.search.ClusterSettings; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +@Log +public class ElasticClientHandler { + + // Elastic cluster Rest client settings + private static final int CLOUD_REST_CLIENT_PORT = 9243; + private static final int REST_CLIENT_CONNECT_TIMEOUT = 60000; + private static final int REST_CLIENT_SOCKET_TIMEOUT = 60000; + private static final int REST_CLIENT_RETRY_TIMEOUT = 60000; + + @Value("#{new Boolean('${security.https.certificate.trust:false}')}") + private Boolean isSecurityHttpsCertificateTrust; + + @Autowired + private IElasticSettingService elasticSettingService; + + public RestHighLevelClient createRestClient() { + return getCloudRestClient(elasticSettingService.getElasticClusterInformation()); + } + + // TODO: Remove this temporary implementation when ECE CCS is utilized + public RestHighLevelClient createRestClient(final ClusterSettings clusterSettings) { + return getCloudRestClient(clusterSettings); + } + + private RestHighLevelClient getCloudRestClient(final ClusterSettings clusterSettings) { + + String cluster = null; + String host = null; + int port = CLOUD_REST_CLIENT_PORT; + String protocolScheme = "https"; + String tls = "true"; + + try { + cluster = clusterSettings.getHost(); + host = clusterSettings.getHost(); + port = clusterSettings.getPort(); + if (!clusterSettings.isHttps()) { + protocolScheme = "http"; + } + + if (!clusterSettings.isTls()) { + tls = "false"; + } + String basicEncoded = Base64 + .getEncoder().encodeToString(clusterSettings.getUserNameAndPassword().getBytes()); + String basicAuthenticationHeaderVal = String.format("Basic %s", basicEncoded); + + RestClientBuilder builder = createClientBuilder(host, basicAuthenticationHeaderVal, port, + protocolScheme, tls); + + return new RestHighLevelClient(builder); + } catch (AppException e) { + throw e; + } catch (Exception e) { + throw new AppException( + HttpStatus.SC_INTERNAL_SERVER_ERROR, + "search client error", + "error creating search client", + String + .format("Elastic client connection params, cluster: %s, host: %s, port: %s", cluster, + host, port), + e); + } + } + + public RestClientBuilder createClientBuilder(String host, String basicAuthenticationHeaderVal, + int port, String protocolScheme, String tls) { + RestClientBuilder builder = RestClient.builder(new HttpHost(host, port, protocolScheme)); + builder.setRequestConfigCallback( + requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(REST_CLIENT_CONNECT_TIMEOUT) + .setSocketTimeout(REST_CLIENT_SOCKET_TIMEOUT)); + builder.setMaxRetryTimeoutMillis(REST_CLIENT_RETRY_TIMEOUT); + + Header[] defaultHeaders = new Header[]{ + new BasicHeader("client.transport.nodes_sampler_interval", "30s"), + new BasicHeader("client.transport.ping_timeout", "30s"), + new BasicHeader("client.transport.sniff", "false"), + new BasicHeader("request.headers.X-Found-Cluster", host), + new BasicHeader("cluster.name", host), + new BasicHeader("xpack.security.transport.ssl.enabled", tls), + new BasicHeader("Authorization", basicAuthenticationHeaderVal), + }; + log.info(String.format( + "Elastic client connection uses protocolScheme = %s with a flag " + + "'security.https.certificate.trust' = %s", + protocolScheme, isSecurityHttpsCertificateTrust)); + if ("https".equals(protocolScheme) && isSecurityHttpsCertificateTrust) { + log.warning("Elastic client connection uses TrustSelfSignedStrategy()"); + SSLContext sslContext = createSSLContext(); + builder.setHttpClientConfigCallback(httpClientBuilder -> + { + HttpAsyncClientBuilder httpAsyncClientBuilder = httpClientBuilder.setSSLContext(sslContext) + .setSSLHostnameVerifier( + NoopHostnameVerifier.INSTANCE); + return httpAsyncClientBuilder; + }); + } + + builder.setDefaultHeaders(defaultHeaders); + return builder; + } + + private SSLContext createSSLContext() { + SSLContextBuilder sslContextBuilder = new SSLContextBuilder(); + try { + sslContextBuilder.loadTrustMaterial(null, new TrustSelfSignedStrategy()); + return sslContextBuilder.build(); + } catch (NoSuchAlgorithmException e) { + log.severe(e.getMessage()); + } catch (KeyStoreException e) { + log.severe(e.getMessage()); + } catch (KeyManagementException e) { + log.severe(e.getMessage()); + } + return null; + } + + public Boolean isSecurityHttpsCertificateTrust() { + return isSecurityHttpsCertificateTrust; + } + + public void setSecurityHttpsCertificateTrust(Boolean isSecurityHttpsCertificateTrust) { + this.isSecurityHttpsCertificateTrust = isSecurityHttpsCertificateTrust; + } } \ No newline at end of file diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/CleanupIndiciesApiTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/CleanupIndiciesApiTest.java new file mode 100644 index 0000000000000000000000000000000000000000..6a9f2ead5cf965f9e05f0e1d1782d97843ff7deb --- /dev/null +++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/CleanupIndiciesApiTest.java @@ -0,0 +1,84 @@ +package org.opengroup.osdu.indexer.api; + +import static org.junit.Assert.fail; +import static org.mockito.MockitoAnnotations.initMocks; +import com.google.gson.Gson; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.opengroup.osdu.core.common.http.HeadersUtil; +import org.opengroup.osdu.core.common.model.http.AppException; +import org.opengroup.osdu.core.common.model.http.DpsHeaders; +import org.opengroup.osdu.core.common.model.search.RecordChangedMessages; +import org.opengroup.osdu.core.common.search.Config; +import org.opengroup.osdu.indexer.service.IndexerService; +import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({HeadersUtil.class, IndexerQueueTaskBuilder.class, DpsHeaders.class, Config.class}) +public class CleanupIndiciesApiTest { + + private final String messageValid = "{\"data\":\"[{\\\"id\\\":\\\"opendes:welldb:wellbore-d9033ae1-fb15-496c-9ba0-880fd1d2b2cf\\\",\\\"kind\\\":\\\"tenant1:welldb:wellbore:1.0.0\\\",\\\"op\\\":\\\"purge_schema\\\"}]\",\"attributes\":{\"account-id\":\"opendes\",\"correlation-id\":\"b5a281bd-f59d-4db2-9939-b2d85036fc7e\"},\"messageId\":\"75328163778221\",\"publishTime\":\"2018-05-08T21:48:56.131Z\"}"; + private final String messageEmpty = "{}"; + private final String messageWithEmptyData = "{\"data\":\"[]\",\"attributes\":{\"account-id\":\"opendes\",\"correlation-id\":\"b5a281bd-f59d-4db2-9939-b2d85036fc7e\"},\"messageId\":\"75328163778221\",\"publishTime\":\"2018-05-08T21:48:56.131Z\"}"; + private final String messageWithIncorrectJsonFormat = "{\"data\":\"[{}}]\",\"attributes\":{\"account-id\":\"opendes\",\"correlation-id\":\"b5a281bd-f59d-4db2-9939-b2d85036fc7e\"},\"messageId\":\"75328163778221\",\"publishTime\":\"2018-05-08T21:48:56.131Z\"}"; + + @InjectMocks + private CleanupIndiciesApi sut; + + @Mock + private IndexerService indexerService; + + @Before + public void setup() { + initMocks(this); + } + + @Test + public void should_return200_given_validMessage_indexCleanupTest() { + should_return200_indexerWorkerTest(messageValid); + } + + @Test + public void should_return200_given_emptyData_indexCleanupTest() { + should_return200_indexerWorkerTest(messageWithEmptyData); + } + + @Test + public void should_return400_given_emptyMessage_indexCleanupTest() { + should_return400_indexerWorkerTest(messageEmpty, String.format("Required header: '%s' not found", DpsHeaders.DATA_PARTITION_ID)); + } + + @Test + public void should_return400_given_incorrectJsonFormatMessage_indexWorkerTest() { + should_return400_indexerWorkerTest(messageWithIncorrectJsonFormat, "Unable to parse request payload."); + } + + private void should_return200_indexerWorkerTest(String message) { + ResponseEntity response = this.sut.cleanupIndices(createRecordChangedMessage(message)); + Assert.assertEquals(HttpStatus.OK.value(), response.getStatusCodeValue()); + } + + private void should_return400_indexerWorkerTest(String message, String errorMessage) { + try { + this.sut.cleanupIndices(createRecordChangedMessage(message)); + fail("Should throw exception"); + } catch (AppException e) { + Assert.assertEquals(HttpStatus.BAD_REQUEST.value(), e.getError().getCode()); + Assert.assertEquals(errorMessage, e.getError().getMessage()); + } catch (Exception e) { + fail("Should not throw this exception" + e.getMessage()); + } + } + + private RecordChangedMessages createRecordChangedMessage(String message) { + return (new Gson()).fromJson(message, RecordChangedMessages.class); + } +} diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceImplTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceImplTest.java new file mode 100644 index 0000000000000000000000000000000000000000..c3c3d3a7a09e86a35c96fe224397aabf60ccfcf1 --- /dev/null +++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceImplTest.java @@ -0,0 +1,56 @@ +package org.opengroup.osdu.indexer.service; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.MockitoAnnotations.initMocks; +import java.util.ArrayList; +import java.util.List; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.opengroup.osdu.core.common.model.indexer.RecordInfo; +import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver; +import org.opengroup.osdu.core.common.search.IndicesService; +import org.opengroup.osdu.indexer.util.ElasticClientHandler; +import org.springframework.test.context.junit4.SpringRunner; + +@RunWith(SpringRunner.class) +public class IndexerServiceImplTest { + + @InjectMocks + private IndexerServiceImpl indexerService; + + @Mock + private ElasticClientHandler elasticClientHandler; + + @Mock + private ElasticIndexNameResolver elasticIndexNameResolver; + + @Mock + private IndicesService indicesService; + + private List<RecordInfo> recordInfos = new ArrayList<>(); + + @Before + public void setup() { + RecordInfo recordInfo = new RecordInfo(); + recordInfo.setId("opendes:ds:mytest3-d9033ae1-fb15-496c-9ba0-880fd1d2b2qf"); + recordInfo.setKind("opendes:ds:mytest2:1.0.0"); + recordInfo.setOp("purge_schema"); + recordInfos.add(recordInfo); + + initMocks(this); + } + + @Test + public void processSchemaMessagesTest() throws Exception { + indexerService.processSchemaMessages(recordInfos); + + verify(elasticClientHandler, times(1)).createRestClient(); + verify(elasticIndexNameResolver, times(1)).getIndexNameFromKind(any()); + verify(indicesService, times(1)).isIndexExist(any(), any()); + } +} diff --git a/indexer-core/src/test/resources/application.properties b/indexer-core/src/test/resources/application.properties new file mode 100644 index 0000000000000000000000000000000000000000..50f201247aa58eb42e6ede1d631cd2ce2127a6a8 --- /dev/null +++ b/indexer-core/src/test/resources/application.properties @@ -0,0 +1 @@ +security.https.certificate.trust=false \ No newline at end of file diff --git a/lombok.config b/lombok.config new file mode 100644 index 0000000000000000000000000000000000000000..a23edb413fc5d2e329b180ce25ef307f0c408874 --- /dev/null +++ b/lombok.config @@ -0,0 +1,2 @@ +config.stopBubbling = true +lombok.addLombokGeneratedAnnotation = true \ No newline at end of file diff --git a/provider/indexer-gcp/README.md b/provider/indexer-gcp/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23c482574d2515459a8f962b197687472e1bba52 --- /dev/null +++ b/provider/indexer-gcp/README.md @@ -0,0 +1,228 @@ +# Indexer Service +os-indexer-gcp is a [Spring Boot](https://spring.io/projects/spring-boot) service that is responsible for indexing Records that enable the `os-search` service to execute OSDU R2 domain searches against Elasticsearch. + +## Getting Started +These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system. + +### Prerequisites +Pre-requisites + +* GCloud SDK with java (latest version) +* JDK 8 +* Lombok 1.16 or later +* Maven + +### Installation +In order to run the service locally or remotely, you will need to have the following environment variables defined. + +| name | value | description | sensitive? | source | +| --- | --- | --- | --- | --- | +| `LOG_PREFIX` | `service` | Logging prefix | no | - | +| `SERVER_SERVLET_CONTEXPATH` | `/api/indexer/v2` | Servlet context path | no | - | +| `AUTHORIZE_API` | ex `https://entitlements.com/entitlements/v1` | Entitlements API endpoint | no | output of infrastructure deployment | +| `ENTITLEMENTS_HOST` | ex `https://entitlements.com/entitlements/v1` | Entitlements API endpoint | no | output of infrastructure deployment | +| `LEGALTAG_API` | ex `https://legal.com/api/legal/v1` | Legal API endpoint | no | output of infrastructure deployment | +| `INDEXER_HOST` | ex `os-indexer-dot-opendes.appspot.com` | Indexer Host | no | output of infrastructure deployment | +| `INDEXER_QUEUE_HOST` | ex `https://os-indexer-queue-dot-opendes.appspot.com/_dps/task-handlers/enqueue` | Indexer-Queue API endpoint | no | output of infrastructure deployment | +| `CRS_API` | ex `https://crs-converter-gae-dot-opendes.appspot.com/api/crs/v1` | CRS API endpoint | no | https://console.cloud.google.com/memorystore/redis/instances | +| `STORAGE_HOSTNAME` | ex `os-storage-dot-opendes.appspot.com` | Storage Host | no | output of infrastructure deployment | +| `STORAGE_SCHEMA_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/schemas` | Storage API endpoint 'schemas' | no | https://console.cloud.google.com/apis/credentials | +| `STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST` | ex `https://os-storage-dot-opendes.appspot.com/api/storage/v2/query/records:batch` | Storage API endpoint 'records' | no | https://console.cloud.google.com/iam-admin/serviceaccounts | +| `REDIS_SEARCH_HOST` | ex `127.0.0.1` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances | +| `REDIS_GROUP_HOST` | ex `127.0.0.1` | Redis host for groups | no | https://console.cloud.google.com/memorystore/redis/instances | +| `REDIS_SEARCH_PORT` | ex `6379` | Redis host for search | no | https://console.cloud.google.com/memorystore/redis/instances | +| `GOOGLE_CLOUD_PROJECT` | ex `opendes` | Google Cloud Project Id| no | output of infrastructure deployment | +| `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials | +| `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | https://console.cloud.google.com/iam-admin/serviceaccounts | +| `security.https.certificate.trust` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment | + + +### Run Locally +Check that maven is installed: + +```bash +$ mvn --version +Apache Maven 3.6.0 +Maven home: /usr/share/maven +Java version: 1.8.0_212, vendor: AdoptOpenJDK, runtime: /usr/lib/jvm/jdk8u212-b04/jre +... +``` + +You may need to configure access to the remote maven repository that holds the OSDU dependencies. This file should live within `~/.mvn/community-maven.settings.xml`: + +```bash +$ cat ~/.m2/settings.xml +<?xml version="1.0" encoding="UTF-8"?> +<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd"> + <servers> + <server> + <id>community-maven-via-private-token</id> + <!-- Treat this auth token like a password. Do not share it with anyone, including Microsoft support. --> + <!-- The generated token expires on or before 11/14/2019 --> + <configuration> + <httpHeaders> + <property> + <name>Private-Token</name> + <value>${env.COMMUNITY_MAVEN_TOKEN}</value> + </property> + </httpHeaders> + </configuration> + </server> + </servers> +</settings> +``` + +* Update the Google cloud SDK to the latest version: + +```bash +gcloud components update +``` +* Set Google Project Id: + +```bash +gcloud config set project <YOUR-PROJECT-ID> +``` + +* Perform a basic authentication in the selected project: + +```bash +gcloud auth application-default login +``` + +* Navigate to indexer service's root folder and run: + +```bash +mvn jetty:run +## Testing +* Navigate to indexer service's root folder and run: + +```bash +mvn clean install +``` + +* If you wish to see the coverage report then go to testing/target/site/jacoco-aggregate and open index.html + +* If you wish to build the project without running tests + +```bash +mvn clean install -DskipTests +``` + +After configuring your environment as specified above, you can follow these steps to build and run the application. These steps should be invoked from the *repository root.* + +```bash +cd provider/indexer-gcp/ && mvn spring-boot:run +``` + +## Testing +Navigate to indexer service's root folder and run all the tests: + +```bash +# build + install integration test core +$ (cd testing/indexer-test-core/ && mvn clean install) +``` + +### Running E2E Tests +This section describes how to run cloud OSDU E2E tests (testing/integration-tests/indexer-test-gcp). + +You will need to have the following environment variables defined. + +| name | value | description | sensitive? | source | +| --- | --- | --- | --- | --- | +| `ENTITLEMENTS_HOST` | ex `https://entitlements.com/entitlements/v1` | Entitlements API endpoint | no | output of infrastructure deployment | +| `ELASTIC_PASSWORD` | `********` | Password for Elasticsearch | yes | output of infrastructure deployment | +| `ELASTIC_USER_NAME` | `********` | User name for Elasticsearch | yes | output of infrastructure deployment | +| `ELASTIC_HOST` | ex `elastic.domain.com` | Host Elasticsearch | yes | output of infrastructure deployment | +| `ELASTIC_PORT` | ex `9243` | Port Elasticsearch | yes | output of infrastructure deployment | +| `GCLOUD_PROJECT` | ex `opendes` | Google Cloud Project Id| no | output of infrastructure deployment | +| `INDEXER_HOST` | ex `https://os-indexer-dot-opendes.appspot.com/api/indexer/v2/` | Indexer API endpoint | no | output of infrastructure deployment | +| `DATA_GROUP` | `opendes` | The service account to this group and substitute | no | - | +| `ENTITLEMENTS_DOMAIN` | ex `opendes-gcp.projects.com` | OSDU R2 to run tests under | no | - | +| `INTEGRATION_TEST_AUDIENCE` | `********` | client application ID | yes | https://console.cloud.google.com/apis/credentials | +| `OTHER_RELEVANT_DATA_COUNTRIES` | ex `US` | valid legal tag with a other relevant data countries | no | - | +| `LEGAL_TAG` | ex `opendes-demo-legaltag` | valid legal tag with a other relevant data countries from `DEFAULT_OTHER_RELEVANT_DATA_COUNTRIES` | no | - | +| `DEFAULT_DATA_PARTITION_ID_TENANT1` | ex `opendes` | HTTP Header 'Data-Partition-ID' | no | - | +| `SEARCH_INTEGRATION_TESTER` | `********` | Service account for API calls. Note: this user must have entitlements configured already | yes | https://console.cloud.google.com/iam-admin/serviceaccounts | +| `SEARCH_HOST` | ex `http://localhost:8080/api/search/v2/` | Endpoint of search service | no | - | +| `STORAGE_HOST` | ex `http://os-storage-dot-opendes.appspot.com/api/storage/v2/schemas` | Storage API endpoint | Storage Host | no | output of infrastructure deployment | + +**Entitlements configuration for integration accounts** + +| INTEGRATION_TESTER | NO_DATA_ACCESS_TESTER | +| --- | --- | +| users<br/>service.entitlements.user<br/>service.search.user<br/>data.test1<br/>data.integration.test<br/>users@{tenant1}@{domain}.com | + +Execute following command to build code and run all the integration tests: + +```bash +# Note: this assumes that the environment variables for integration tests as outlined +# above are already exported in your environment. +$ (cd testing/indexer-test-gcp/ && mvn clean test) +``` + +## Deployment + +* Data-Lake Indexer Google Cloud Endpoints on App Engine Flex environment + * Edit the app.yaml + * Open the [app.yaml](indexer/src/main/appengine/app.yaml) file in editor, and replace the YOUR-PROJECT-ID `GOOGLE_CLOUD_PROJECT` line with Google Cloud Platform project Id. Also update `AUTHORIZE_API`, `CRON_JOB_IP`, `LEGAL_HOSTNAME`, `REGION` and `SECURITY_HTTPS_CERTIFICATE_TRUST` based on your deployment + + * Deploy + ```sh + mvn appengine:deploy -pl org.opengroup.osdu.indexer:indexer -amd + ``` + + * If you wish to deploy the search service without running tests + ```sh + mvn appengine:deploy -pl org.opengroup.osdu.indexer:indexer -amd -DskipTests + ``` + +or +* Google Documentation: https://cloud.google.com/cloud-build/docs/deploying-builds/deploy-appengine + +#### Cloud KMS Setup + +Enable cloud KMS on master project + +Create king ring and key in the ***master project*** + +```bash + gcloud services enable cloudkms.googleapis.com + export KEYRING_NAME="csqp" + export CRYPTOKEY_NAME="searchService" + gcloud kms keyrings create $KEYRING_NAME --location global + gcloud kms keys create $CRYPTOKEY_NAME --location global \ + --keyring $KEYRING_NAME \ + --purpose encryption +``` + +Add **Cloud KMS CryptoKey Encrypter/Decrypter** role to the **App Engine default service account** of the master project through IAM - Role tab + +Add "Cloud KMS Encrypt/Decrypt" role to the "App Engine default service account" of ***master project*** + +#### Memory Store (Redis Instance) Setup + +Create a new Standard tier Redis instance on the ***service project*** + +The Redis instance must be created under the same region with the App Engine application which needs to access it. + +```bash + gcloud beta redis instances create redis-cache-search --size=10 --region=<service-deployment-region> --zone=<service-deployment-zone> --tier=STANDARD +``` + +## Licence +Copyright © Google LLC +Copyright © EPAM Systems + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +[http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0) + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/provider/indexer-gcp/pom.xml b/provider/indexer-gcp/pom.xml index 09cc7e5240b1cfef8ffab0eb432888c8c8d9e640..fe4e902e8ea3f5a5b829b90ad822637b4a02d465 100644 --- a/provider/indexer-gcp/pom.xml +++ b/provider/indexer-gcp/pom.xml @@ -202,7 +202,26 @@ <useSystemClassLoader>false</useSystemClassLoader> <threadCount>1</threadCount> </configuration> - </plugin> + </plugin> + <plugin> + <groupId>org.jacoco</groupId> + <artifactId>jacoco-maven-plugin</artifactId> + <version>0.7.7.201606060606</version> + <executions> + <execution> + <goals> + <goal>prepare-agent</goal> + </goals> + </execution> + <execution> + <id>report</id> + <phase>prepare-package</phase> + <goals> + <goal>report</goal> + </goals> + </execution> + </executions> + </plugin> </plugins> </build> diff --git a/provider/indexer-gcp/src/main/appengine/app.yaml b/provider/indexer-gcp/src/main/appengine/app.yaml index 16d9a8ac495f04fa88c5ec438902c621dc08bf96..363b4097e3a611b59391832910b92ca518800872 100644 --- a/provider/indexer-gcp/src/main/appengine/app.yaml +++ b/provider/indexer-gcp/src/main/appengine/app.yaml @@ -39,4 +39,4 @@ env_variables: LEGAL_HOSTNAME: "LEGAL_HOSTNAME_VAR" REGION: "REGION_VAR" SPRING_PROFILES_ACTIVE: 'ENVIRONMENT' - + SECURITY_HTTPS_CERTIFICATE_TRUST: 'SECURITY_HTTPS_CERTIFICATE_TRUST_VAR' diff --git a/provider/indexer-gcp/src/main/resources/application-kuber.properties b/provider/indexer-gcp/src/main/resources/application-kuber.properties index af85274d4030d56204400765876db4eeb3c9d70a..bc2a5f9014fa1b406a008e2cc69f161a8e3a9a8b 100644 --- a/provider/indexer-gcp/src/main/resources/application-kuber.properties +++ b/provider/indexer-gcp/src/main/resources/application-kuber.properties @@ -21,4 +21,4 @@ REDIS_SEARCH_HOST=${REDIS_SEARCH_HOST} GOOGLE_AUDIENCES=${GOOGLE_AUDIENCES} DEPLOYMENT_ENVIRONMENT=CLOUD -disable.appengine.log.factory=true +disable.appengine.log.factory=true \ No newline at end of file diff --git a/provider/indexer-gcp/src/main/resources/application-testing.properties b/provider/indexer-gcp/src/main/resources/application-testing.properties index 5368d291c5653bcf9bd31bfbc9c31af3765e9b1f..fb0c9e2d6ce29c87dca6f58d1afec0b420c1e663 100644 --- a/provider/indexer-gcp/src/main/resources/application-testing.properties +++ b/provider/indexer-gcp/src/main/resources/application-testing.properties @@ -18,4 +18,4 @@ CRS_API=https://crs-converter-gae-dot-opendes-evt.appspot.com/api/crs/v1 REDIS_GROUP_HOST=10.253.209.196 REDIS_SEARCH_HOST=10.118.2.140 -GOOGLE_AUDIENCES=833591776864-oobhqvmtdg9rpreubjvn44m5f8revglk.apps.googleusercontent.com +GOOGLE_AUDIENCES=833591776864-oobhqvmtdg9rpreubjvn44m5f8revglk.apps.googleusercontent.com \ No newline at end of file diff --git a/provider/indexer-gcp/src/main/resources/application.properties b/provider/indexer-gcp/src/main/resources/application.properties index 1ccde084878626a19f762e0fd8abb6a650450447..a0cbce3427ff62ba4d63ffabc17323efc833295b 100644 --- a/provider/indexer-gcp/src/main/resources/application.properties +++ b/provider/indexer-gcp/src/main/resources/application.properties @@ -32,3 +32,5 @@ KMS_KEY=searchService ELASTIC_DATASTORE_KIND=SearchSettings ELASTIC_DATASTORE_ID=indexer-service + +security.https.certificate.trust=false diff --git a/provider/indexer-ibm/pom.xml b/provider/indexer-ibm/pom.xml index ae6917c0921fd6311ee88561b8e7048980e46504..07b269a729827466ea59658c95c8e4520e3f0447 100644 --- a/provider/indexer-ibm/pom.xml +++ b/provider/indexer-ibm/pom.xml @@ -33,7 +33,7 @@ <packaging>jar</packaging> <properties> - <os-core-lib-ibm.version>0.0.18</os-core-lib-ibm.version> + <os-core-lib-ibm.version>0.3.8-SNAPSHOT</os-core-lib-ibm.version> </properties> <profiles> diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/security/SecurityConfig.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/security/SecurityConfig.java index 2cf274beac0c1dd57bb2926fcd2b6fbe98fda4e4..561edbc1e74108b0a79bcb5a159aa764c2b5c24c 100644 --- a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/security/SecurityConfig.java +++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/security/SecurityConfig.java @@ -29,6 +29,8 @@ public class SecurityConfig extends WebSecurityConfigurerAdapter { .csrf().disable() .authorizeRequests() .antMatchers("/", "/index.html", + "/liveness_check", + "/readiness_check", "/index-worker", "/_dps/task-handlers", "/_dps/task-handlers/**", "/reindex", "/v2/api-docs", diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/GlobalExceptionMapper.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/GlobalExceptionMapper.java new file mode 100644 index 0000000000000000000000000000000000000000..60e0be562e836d67ff2d1ab0ecf7a4fcf07e4dd0 --- /dev/null +++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/GlobalExceptionMapper.java @@ -0,0 +1,83 @@ +// Copyright 2017-2019, Schlumberger +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.opengroup.osdu.indexer.ibm.util; + +import javax.validation.ValidationException; + +import javassist.NotFoundException; +import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.security.access.AccessDeniedException; +import org.springframework.web.bind.annotation.ControllerAdvice; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler; +import org.opengroup.osdu.core.common.model.http.AppException; + +@Order(Ordered.HIGHEST_PRECEDENCE) +@ControllerAdvice +public class GlobalExceptionMapper extends ResponseEntityExceptionHandler { + + @Autowired + private JaxRsDpsLog logger; + + @ExceptionHandler(AppException.class) + protected ResponseEntity<Object> handleAppException(AppException e) { + return this.getErrorResponse(e); + } + + @ExceptionHandler(ValidationException.class) + protected ResponseEntity<Object> handleValidationException(ValidationException e) { + return this.getErrorResponse( + new AppException(HttpStatus.BAD_REQUEST.value(), "Validation error.", e.getMessage(), e)); + } + + @ExceptionHandler(NotFoundException.class) + protected ResponseEntity<Object> handleNotFoundException(NotFoundException e) { + return this.getErrorResponse( + new AppException(HttpStatus.NOT_FOUND.value(), "Resource not found.", e.getMessage(), e)); + } + + @ExceptionHandler(AccessDeniedException.class) + protected ResponseEntity<Object> handleAccessDeniedException(AccessDeniedException e) { + return this.getErrorResponse( + new AppException(HttpStatus.FORBIDDEN.value(), "Access denied", e.getMessage(), e)); + } + + @ExceptionHandler(Exception.class) + protected ResponseEntity<Object> handleGeneralException(Exception e) { + return this.getErrorResponse( + new AppException(HttpStatus.INTERNAL_SERVER_ERROR.value(), "Server error.", + "An unknown error has occurred.", e)); + } + + private ResponseEntity<Object> getErrorResponse(AppException e) { + + String exceptionMsg = e.getOriginalException() != null + ? e.getOriginalException().getMessage() + : e.getError().getMessage(); + + if (e.getError().getCode() > 499) { + this.logger.error(exceptionMsg, e); + } else { + this.logger.warning(exceptionMsg, e); + } + + return new ResponseEntity<Object>(e.getError(), HttpStatus.resolve(e.getError().getCode())); + } +} \ No newline at end of file diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/IndexerQueueTaskBuilderIbm.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/IndexerQueueTaskBuilderIbm.java new file mode 100644 index 0000000000000000000000000000000000000000..a429d145548efc7905840fae0b40764b492e4816 --- /dev/null +++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/IndexerQueueTaskBuilderIbm.java @@ -0,0 +1,86 @@ +package org.opengroup.osdu.indexer.ibm.util; + +import java.util.Map; + +import javax.inject.Inject; + +import org.opengroup.osdu.core.common.model.http.DpsHeaders; +import org.opengroup.osdu.core.common.model.search.RecordChangedMessages; +import org.opengroup.osdu.core.ibm.messagebus.IMessageFactory; +import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.annotation.Primary; +import org.springframework.stereotype.Component; + +import com.google.gson.Gson; +import com.google.gson.JsonSyntaxException; + +@Primary +@Component +public class IndexerQueueTaskBuilderIbm extends IndexerQueueTaskBuilder { + private static final Logger logger = LoggerFactory.getLogger(IndexerQueueTaskBuilderIbm.class); + + @Inject + IMessageFactory mq; + + private Gson gson; + private final static String RETRY_STRING = "retry"; + private final static String ERROR_CODE = "errorCode"; + private final static String ERROR_MESSAGE = "errorMessage"; + + @Inject + public void init() { + gson = new Gson(); + } + + @Override + public void createWorkerTask(String payload, DpsHeaders headers) { + createTask(payload, headers); + } + + @Override + public void createReIndexTask(String payload, DpsHeaders headers) { + createTask(payload, headers); + } + + private void createTask(String payload, DpsHeaders headers) { + + try { + RecordChangedMessages receivedPayload = gson.fromJson(payload, RecordChangedMessages.class); + + Map<String, String> attributes = receivedPayload.getAttributes(); + int retryCount = 0; + if (attributes.containsKey(RETRY_STRING)) { + retryCount = Integer.parseInt(attributes.get(RETRY_STRING)); + retryCount++; + } else { + retryCount = 1; + } + attributes.put(RETRY_STRING, String.valueOf(retryCount)); + attributes.put(ERROR_CODE, "999"); //error code TBD + attributes.put(ERROR_MESSAGE, "Indexer could not process record"); + receivedPayload.setAttributes(attributes); + + // incase if we need to shift logic from indexer-queue-ibm/subscriber.java + /* + * if(Integer.parseInt(receivedPayload.getAttributes().get(RETRY_STRING))>3) { + * //add DLQ in IMessageFactory + * + * mq.sendMessage("DLQ", gson.toJson(receivedPayload)); } + */ + logger.info("Message send back to queue : " + receivedPayload); + mq.sendMessage(IMessageFactory.DEFAULT_QUEUE_NAME, gson.toJson(receivedPayload)); + } catch (JsonSyntaxException e) { + logger.error("JsonSyntaxException in IndexerQueueTaskBuilderIbm " + e.toString()); + e.printStackTrace(); + } catch (NumberFormatException e) { + logger.error("NumberFormatException in IndexerQueueTaskBuilderIbm " + e.toString()); + e.printStackTrace(); + } catch (Exception e) { + logger.error("Exception in IndexerQueueTaskBuilderIbm " + e.toString()); + e.printStackTrace(); + } + + } +} \ No newline at end of file diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/RequestInfoImpl.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/RequestInfoImpl.java index 12fd218f826bc9ea1177e433d305cbc951571605..6ddf941be1f763557bce5904cd290ff22cd34ef8 100644 --- a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/RequestInfoImpl.java +++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/RequestInfoImpl.java @@ -53,10 +53,14 @@ public class RequestInfoImpl implements IRequestInfo { @Inject private TenantInfo tenantInfo; - + @Value("${DEPLOYMENT_ENVIRONMENT}") private String DEPLOYMENT_ENVIRONMENT; + private static final String INDEXER_API_KEY_HEADER="x-api-key"; + + @Value("${INDEXER_API_KEY}") + private String tokenFromProperty; @Override public DpsHeaders getHeaders() { @@ -65,9 +69,19 @@ public class RequestInfoImpl implements IRequestInfo { // throw to prevent null reference exception below throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Invalid Headers", "Headers Map DpsHeaders is null"); } - DpsHeaders headers = this.getCoreServiceHeaders(headersMap.getHeaders()); - return headers; - } + DpsHeaders headers = this.getCoreServiceHeaders(headersMap.getHeaders()); + + if (headers.getHeaders().containsKey(INDEXER_API_KEY_HEADER)) { + String apiToken = headers.getHeaders().get(INDEXER_API_KEY_HEADER); + if (!apiToken.equals(tokenFromProperty)) { + logger.error("Indexer API Token in header is mismatched"); + throw new AppException(HttpStatus.SC_UNAUTHORIZED, "Indexer API Token in header mismatched.", "Indexer API Token in header mismatched."); + } + } else { + throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Missing Header", "The headers "+ INDEXER_API_KEY_HEADER + " is missing!"); + } + return headers; + } @Override public String getPartitionId() { @@ -95,7 +109,8 @@ public class RequestInfoImpl implements IRequestInfo { @Override public boolean isTaskQueueRequest() { - //if (!this.dpsHeaders.getHeaders().containsKey(INDEXER_QUEUE_KEY)) return false; + //if (!this.dpsHeaders.getHeaders().containsKey(INDEXER_API_KEY_HEADER)) return false; + // String queueId = this.headersInfo.getHeadersMap().get(AppEngineHeaders.TASK_QUEUE_NAME); // return queueId.endsWith(Constants.INDEXER_QUEUE_IDENTIFIER); diff --git a/testing/indexer-test-core/src/main/resources/features/indexrecord/IndexRecord.feature b/testing/indexer-test-core/src/main/resources/features/indexrecord/IndexRecord.feature index 30ee32eaa576dc0a7369c0a29a9110a20c566775..c281ec4a0b174ec98eb781f8e603eba3b3968ca1 100644 --- a/testing/indexer-test-core/src/main/resources/features/indexrecord/IndexRecord.feature +++ b/testing/indexer-test-core/src/main/resources/features/indexrecord/IndexRecord.feature @@ -15,8 +15,8 @@ Feature: Indexing of the documents Examples: | kind | recordFile | number | index | type | acl | mapping | - | "tenant1:testindex<timestamp>:well:1.0.0" | "index_records_1" | 5 | "tenant1-testindex<timestamp>-well-1.0.0" | "well" | "data.default.viewers@opendes" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" | - | "tenant1:testindex<timestamp>:well:3.0.0" | "index_records_1" | 5 | "tenant1-testindex<timestamp>-well-3.0.0" | "well" | "data.default.viewers@opendes" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" | + | "tenant1:testindex<timestamp>:well:1.0.0" | "index_records_1" | 5 | "tenant1-testindex<timestamp>-well-1.0.0" | "well" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" | + | "tenant1:testindex<timestamp>:well:3.0.0" | "index_records_1" | 5 | "tenant1-testindex<timestamp>-well-3.0.0" | "well" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" | Scenario Outline: Ingest the record and Index in the Elastic Search with bad attribute When I ingest records with the <recordFile> with <acl> for a given <kind> @@ -24,5 +24,5 @@ Feature: Indexing of the documents Examples: | kind | recordFile | number | index | skippedAttribute | acl | - | "tenant1:testindex<timestamp>:well:2.0.0" | "index_records_2" | 4 | "tenant1-testindex<timestamp>-well-2.0.0" | "data.Location" | "data.default.viewers@opendes" | - | "tenant1:testindex<timestamp>:well:3.0.0" | "index_records_3" | 7 | "tenant1-testindex<timestamp>-well-3.0.0" | "data.GeoShape" | "data.default.viewers@opendes" | + | "tenant1:testindex<timestamp>:well:2.0.0" | "index_records_2" | 4 | "tenant1-testindex<timestamp>-well-2.0.0" | "data.Location" | "data.default.viewers@tenant1" | + | "tenant1:testindex<timestamp>:well:3.0.0" | "index_records_3" | 7 | "tenant1-testindex<timestamp>-well-3.0.0" | "data.GeoShape" | "data.default.viewers@tenant1" | diff --git a/testing/indexer-test-ibm/pom.xml b/testing/indexer-test-ibm/pom.xml index c9119b69bb64529d6b50e629df7db4bf22b45530..62842b147f03f170690d7817314f7c6e0d8f2b0e 100644 --- a/testing/indexer-test-ibm/pom.xml +++ b/testing/indexer-test-ibm/pom.xml @@ -14,7 +14,7 @@ <maven.compiler.target>1.8</maven.compiler.target> <maven.compiler.source>1.8</maven.compiler.source> <cucumber.version>1.2.5</cucumber.version> - <os-core-lib-ibm.version>0.0.18</os-core-lib-ibm.version> + <os-core-lib-ibm.version>0.3.8-SNAPSHOT</os-core-lib-ibm.version> </properties> <dependencies> diff --git a/testing/indexer-test-ibm/src/main/resources/apikey.feature b/testing/indexer-test-ibm/src/main/resources/apikey.feature new file mode 100644 index 0000000000000000000000000000000000000000..4be8e4eb317aac8c9c6af2d7f847fc0896faa62b --- /dev/null +++ b/testing/indexer-test-ibm/src/main/resources/apikey.feature @@ -0,0 +1,10 @@ +Feature: check the api key in header + This feature check the api key received from headers is matching with configured in environment + + Scenario Outline: compare the api key with configured environment key + When I pass api key + Then compare with key configured in properties file + + Examples: + | apiKey | + | "abcd" | diff --git a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java index 4978ddfccad197628432d4ccb8b5985b4af5017e..897f02c15372583ef985dfc77e2884c4035640fd 100644 --- a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java +++ b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java @@ -6,7 +6,7 @@ import org.junit.runner.RunWith; @RunWith(Cucumber.class) @CucumberOptions( - features = "classpath:features/indexrecord/IndexRecord.feature", + features = {"classpath:features/indexrecord/IndexRecord.feature","classpath:apikey.feature"}, glue = {"classpath:org.opengroup.osdu.step_definitions/index/record"}, plugin = {"pretty", "junit:target/cucumber-reports/TEST-indexrecord.xml"}) public class RunTest { diff --git a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java index 518b2b6f970c5c07b918d7e0ed31aabe819dcdae..3c885b11ca68f15dc3cad6ba15fae1ae7395b514 100644 --- a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java +++ b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java @@ -1,8 +1,25 @@ package org.opengroup.osdu.step_definitions.index.record; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.lang3.StringUtils; import org.opengroup.osdu.common.RecordSteps; +import org.opengroup.osdu.core.common.Constants; +import org.opengroup.osdu.core.common.http.HttpClient; +import org.opengroup.osdu.core.common.http.HttpRequest; +import org.opengroup.osdu.core.common.http.HttpResponse; +import org.opengroup.osdu.core.common.model.http.AppError; +import org.opengroup.osdu.core.common.model.http.DpsHeaders; +import org.opengroup.osdu.core.common.model.search.RecordChangedMessages; +import org.opengroup.osdu.core.ibm.util.Config; import org.opengroup.osdu.util.IBMHTTPClient; +import com.google.gson.Gson; + import cucumber.api.DataTable; import cucumber.api.Scenario; import cucumber.api.java.Before; @@ -48,5 +65,46 @@ public class Steps extends RecordSteps { public void iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(int expectedCount, String index, String skippedAttributes) throws Throwable { super.iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(expectedCount, index, skippedAttributes); } - + + @When("^I pass api key$") + public void i_pass_the_api_key() { + } + + @Then("^compare with key configured in properties file$") + public void compare_with_key_configured_in_propertiesFile() throws Throwable { + + final String CORRELATION_ID = "1234"; + final String OPENDES = "opendes"; + final Gson gson = new Gson(); + + String INDEXER_API_KEY = Config.getEnvironmentVariable("INDEXER_API_KEY"); + String INDEXER_HOST_URL = Config.getEnvironmentVariable("INDEXER_HOST_URL"); + + RecordChangedMessages recordChangeMessage = new RecordChangedMessages(); + + String data = "[{\"id\":\"opendes:doc:1234\",\"kind\":\"opendes:test:test:1.0.0\",\"op\":\"create\"}]"; + + Map<String, String> attributes = new HashMap<>(); + attributes.put("correlation-id", CORRELATION_ID); + attributes.put("data-partition-id", OPENDES); + recordChangeMessage.setAttributes(attributes); + recordChangeMessage.setData(data); + + String url = StringUtils.join(INDEXER_HOST_URL, Constants.WORKER_RELATIVE_URL); + HttpClient httpClient = new HttpClient(); + DpsHeaders dpsHeaders = new DpsHeaders(); + dpsHeaders.put("x-api-key", INDEXER_API_KEY); + dpsHeaders.put("correlation-id", CORRELATION_ID); + dpsHeaders.put("data-partition-id", OPENDES); + + HttpRequest rq = HttpRequest.post(recordChangeMessage).url(url).headers(dpsHeaders.getHeaders()).build(); + HttpResponse result = httpClient.send(rq); + if(result.hasException().equals(false) && result.getResponseCode() == 500) { + assertTrue(result.getResponseCode() == 500); + } else { + AppError error = gson.fromJson(result.getBody(), AppError.class); + assertFalse("Token is mismatched", error.getCode() == 401); + } + + } } \ No newline at end of file