diff --git a/NOTICE b/NOTICE index 878e5e17c130b55f98a6cc98bc813ced6ffe695a..cbd4bf4a82a797ceb9a2f1bec0bd42363cd633ec 100644 --- a/NOTICE +++ b/NOTICE @@ -232,7 +232,6 @@ The following software have components provided under the terms of this license: - Doxia :: XHTML Module (from ) - Elastic JNA Distribution (from https://github.com/java-native-access/jna) - Elasticsearch: 5.0.0-alpha5 (from https://github.com/elastic/elasticsearch) -- Elasticsearch: 5.0.0-alpha5 (from https://github.com/elastic/elasticsearch) - FindBugs-jsr305 (from http://findbugs.sourceforge.net/) - Google APIs Client Library for Java (from ) - Google App Engine extensions to the Google HTTP Client Library for Java. (from ) @@ -313,35 +312,22 @@ The following software have components provided under the terms of this license: - KeePassJava2 :: KDB (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2-kdb) - KeePassJava2 :: KDBX (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2-kdbx) - KeePassJava2 :: Simple (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2-simple) +- Logback Contrib :: JSON :: Classic (from ) +- Logback Contrib :: JSON :: Core (from ) +- Logback Contrib :: Jackson (from ) - Lucene Common Analyzers (from ) -- Lucene Common Analyzers (from ) -- Lucene Core (from ) - Lucene Core (from ) - Lucene Grouping (from ) -- Lucene Grouping (from ) -- Lucene Highlighter (from ) - Lucene Highlighter (from ) - Lucene Join (from ) -- Lucene Join (from ) - Lucene Memory (from ) - Lucene Memory (from ) -- Lucene Memory (from ) -- Lucene Memory (from ) -- Lucene Miscellaneous (from ) - Lucene Miscellaneous (from ) - Lucene Queries (from ) -- Lucene Queries (from ) -- Lucene QueryParsers (from ) - Lucene QueryParsers (from ) - Lucene Sandbox (from ) -- Lucene Sandbox (from ) -- Lucene Spatial (from ) -- Lucene Spatial (from ) -- Lucene Spatial 3D (from ) - Lucene Spatial 3D (from ) - Lucene Spatial Extras (from ) -- Lucene Spatial Extras (from ) -- Lucene Suggest (from ) - Lucene Suggest (from ) - MapStruct Core (from ) - Maven Artifact (from ) @@ -366,10 +352,10 @@ The following software have components provided under the terms of this license: - Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Azure Netty HTTP Client Library (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java) -- Mockito (from http://mockito.org) - Mockito (from http://www.mockito.org) - Mockito (from http://mockito.org) - Mockito (from http://mockito.org) +- Mockito (from http://mockito.org) - Mojo's Maven plugin for Cobertura (from http://mojo.codehaus.org/cobertura-maven-plugin/) - Netty Reactive Streams HTTP support (from ) - Netty Reactive Streams Implementation (from ) @@ -385,11 +371,15 @@ The following software have components provided under the terms of this license: - Netty/TomcatNative [BoringSSL - Static] (from ) - Netty/Transport (from http://netty.io/) - Netty/Transport/Native/Unix/Common (from ) +- Nimbus Content Type (from https://bitbucket.org/connect2id/nimbus-content-type) +- Nimbus JOSE+JWT (from https://bitbucket.org/connect2id/nimbus-jose-jwt) - Nimbus JOSE+JWT (from https://bitbucket.org/connect2id/nimbus-jose-jwt) - Nimbus JOSE+JWT (from https://bitbucket.org/connect2id/nimbus-jose-jwt) - Nimbus LangTag (from https://bitbucket.org/connect2id/nimbus-language-tags) +- Nimbus LangTag (from https://bitbucket.org/connect2id/nimbus-language-tags) - Non-Blocking Reactive Foundation for the JVM (from https://github.com/reactor/reactor) - OAuth 2.0 SDK with OpenID Connect extensions (from https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions) +- OAuth 2.0 SDK with OpenID Connect extensions (from https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions) - OAuth2 for Spring Security (from ) - Objenesis (from http://objenesis.org) - OkHttp (from ) @@ -462,10 +452,8 @@ The following software have components provided under the terms of this license: - Xerces2-j (from https://xerces.apache.org/xerces2-j/) - aalto-xml (from ) - aggs-matrix-stats (from https://github.com/elastic/elasticsearch) -- aggs-matrix-stats (from https://github.com/elastic/elasticsearch) - aws-ssm-java-caching-client (from https://github.com/awslabs/aws-ssm-java-caching-client) - cli (from https://github.com/elastic/elasticsearch) -- cli (from https://github.com/elastic/elasticsearch) - com.google.api.grpc:grpc-google-cloud-pubsub-v1 (from https://github.com/googleapis/googleapis) - com.google.api.grpc:proto-google-cloud-logging-v2 (from https://github.com/googleapis/googleapis) - com.google.api.grpc:proto-google-cloud-pubsub-v1 (from https://github.com/googleapis/googleapis) @@ -475,7 +463,7 @@ The following software have components provided under the terms of this license: - compiler (from http://github.com/spullara/mustache.java) - datastore-v1-proto-client (from ) - elasticsearch-core (from https://github.com/elastic/elasticsearch) -- elasticsearch-core (from https://github.com/elastic/elasticsearch) +- elasticsearch-geo (from https://github.com/elastic/elasticsearch) - error-prone annotations (from ) - error-prone annotations (from ) - io.grpc:grpc-alts (from https://github.com/grpc/grpc-java) @@ -509,14 +497,13 @@ The following software have components provided under the terms of this license: - jersey-spring4 (from ) - jose4j (from https://bitbucket.org/b_c/jose4j/) - lang-mustache (from https://github.com/elastic/elasticsearch) -- lang-mustache (from https://github.com/elastic/elasticsearch) - lettuce (from http://github.com/mp911de/lettuce/wiki) +- mapper-extras (from https://github.com/elastic/elasticsearch) - micrometer-core (from https://github.com/micrometer-metrics/micrometer) - micrometer-registry-azure-monitor (from https://github.com/micrometer-metrics/micrometer) - org.xmlunit:xmlunit-core (from http://www.xmlunit.org/) - oro (from ) - parent-join (from https://github.com/elastic/elasticsearch) -- parent-join (from https://github.com/elastic/elasticsearch) - powermock-api-support (from ) - powermock-core (from http://www.powermock.org) - powermock-module-junit4 (from http://www.powermock.org) @@ -525,14 +512,10 @@ The following software have components provided under the terms of this license: - proto-google-cloud-datastore-v1 (from https://github.com/googleapis/api-client-staging) - proton-j (from ) - rank-eval (from https://github.com/elastic/elasticsearch) -- rank-eval (from https://github.com/elastic/elasticsearch) - rest (from https://github.com/elastic/elasticsearch) -- rest (from https://github.com/elastic/elasticsearch) -- rest-high-level (from https://github.com/elastic/elasticsearch) - rest-high-level (from https://github.com/elastic/elasticsearch) - rxjava (from https://github.com/ReactiveX/RxJava) - secure-sm (from https://github.com/elastic/elasticsearch) -- secure-sm (from https://github.com/elastic/elasticsearch) - spring-security-config (from http://spring.io/spring-security) - spring-security-core (from http://spring.io/spring-security) - spring-security-oauth2-client (from http://spring.io/spring-security) @@ -556,7 +539,6 @@ The following software have components provided under the terms of this license: - tomcat-embed-el (from http://tomcat.apache.org/) - tomcat-embed-websocket (from http://tomcat.apache.org/) - x-content (from https://github.com/elastic/elasticsearch) -- x-content (from https://github.com/elastic/elasticsearch) - xml-apis (from ) ======================================================================== @@ -570,7 +552,7 @@ The following software have components provided under the terms of this license: - GAX (Google Api eXtensions) (from https://github.com/googleapis) - Hamcrest Core (from http://hamcrest.org/) - Lucene Common Analyzers (from ) -- Lucene Common Analyzers (from ) +- Lucene Core (from ) - Plexus :: Default Container (from ) - Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils) - StAX (from http://stax.codehaus.org/) @@ -603,11 +585,8 @@ The following software have components provided under the terms of this license: - JSch (from http://www.jcraft.com/jsch/) - JavaBeans Activation Framework API jar (from ) - Lucene Common Analyzers (from ) -- Lucene Common Analyzers (from ) -- Lucene Core (from ) - Lucene Core (from ) - Lucene Suggest (from ) -- Lucene Suggest (from ) - Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java) @@ -719,6 +698,9 @@ EPL-1.0 The following software have components provided under the terms of this license: - Logback Classic Module (from ) +- Logback Contrib :: JSON :: Classic (from ) +- Logback Contrib :: JSON :: Core (from ) +- Logback Contrib :: Jackson (from ) - Logback Core Module (from ) - Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java) @@ -830,6 +812,9 @@ The following software have components provided under the terms of this license: - Javassist (from http://www.javassist.org/) - Javassist (from http://www.javassist.org/) - Logback Classic Module (from ) +- Logback Contrib :: JSON :: Classic (from ) +- Logback Contrib :: JSON :: Core (from ) +- Logback Contrib :: Jackson (from ) - Logback Core Module (from ) - Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java) @@ -870,16 +855,12 @@ The following software have components provided under the terms of this license: - Bouncy Castle Provider (from http://www.bouncycastle.org/java.html) - Checker Qual (from https://checkerframework.org) - Checker Qual (from https://checkerframework.org) -- Elasticsearch: 5.0.0-alpha5 (from https://github.com/elastic/elasticsearch) - Extensions on Apache Proton-J library (from https://github.com/Azure/qpid-proton-j-extensions) - JOpt Simple (from http://pholser.github.io/jopt-simple) - JUL to SLF4J bridge (from http://www.slf4j.org) - Java Client Runtime for AutoRest (from https://github.com/Azure/autorest-clientruntime-for-java) - Java JWT (from http://www.jwt.io) - Lucene Core (from ) -- Lucene Core (from ) -- Lucene Sandbox (from ) -- Lucene Sandbox (from ) - Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java) @@ -894,9 +875,9 @@ The following software have components provided under the terms of this license: - Microsoft Azure client library for KeyVault Secrets (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure common module for Storage (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure internal Avro module for Storage (from https://github.com/Azure/azure-sdk-for-java) +- Mockito (from http://www.mockito.org) - Mockito (from http://mockito.org) - Mockito (from http://mockito.org) -- Mockito (from http://www.mockito.org) - Mockito (from http://mockito.org) - Netty/Codec/HTTP (from ) - Netty/Common (from ) diff --git a/devops/azure/chart/templates/deployment.yaml b/devops/azure/chart/templates/deployment.yaml index 818f9caceecdbaa7d2492671fa8b8629c1ab6e11..6f8dabb677ca1a52f3f95c310c9b2c18acf2f6a4 100644 --- a/devops/azure/chart/templates/deployment.yaml +++ b/devops/azure/chart/templates/deployment.yaml @@ -100,7 +100,7 @@ spec: - name: STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST value: http://storage/api/storage/v2/query/records:batch - name: STORAGE_QUERY_RECORD_HOST - value: http://storage/api/storage/v2/records + value: http://storage/api/storage/v2/query/records - name: partition_service_endpoint value: http://partition/api/partition/v1 - name: azure_istioauth_enabled diff --git a/devops/azure/release.yaml b/devops/azure/release.yaml index dca21e377c5a3c6d13d9dc59ff9e2f661851d2c2..dad51b35bebd9069044ac08b8848245a29165fb7 100644 --- a/devops/azure/release.yaml +++ b/devops/azure/release.yaml @@ -161,4 +161,4 @@ spec: - name: STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST value: http://storage/api/storage/v2/query/records:batch - name: STORAGE_QUERY_RECORD_HOST - value: http://storage/api/storage/v2/records + value: http://storage/api/storage/v2/query/records diff --git a/provider/indexer-aws/pom.xml b/provider/indexer-aws/pom.xml index d6a11e333aba7bc1168209b4489d02d6648909e3..1ee9eb488b579e264cb5916c96038282b1c26837 100644 --- a/provider/indexer-aws/pom.xml +++ b/provider/indexer-aws/pom.xml @@ -90,12 +90,6 @@ <groupId>org.springframework.security</groupId> <artifactId>spring-security-oauth2-jose</artifactId> </dependency> - <dependency> - <groupId>org.apache.lucene</groupId> - <artifactId>lucene-core</artifactId> - <version>7.6.0</version> - <scope>compile</scope> - </dependency> <dependency> <groupId>javax.inject</groupId> <artifactId>javax.inject</artifactId> @@ -105,17 +99,14 @@ <dependency> <groupId>org.elasticsearch</groupId> <artifactId>elasticsearch</artifactId> - <version>6.6.2</version> </dependency> <dependency> - <groupId>org.elasticsearch.client</groupId> - <artifactId>elasticsearch-rest-client</artifactId> - <version>6.6.2</version> + <groupId>org.elasticsearch.client</groupId> + <artifactId>elasticsearch-rest-client</artifactId> </dependency> <dependency> - <groupId>org.elasticsearch.client</groupId> - <artifactId>elasticsearch-rest-high-level-client</artifactId> - <version>6.6.2</version> + <groupId>org.elasticsearch.client</groupId> + <artifactId>elasticsearch-rest-high-level-client</artifactId> </dependency> <!-- Testing packages --> diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/service/ElasticClientHandlerAws.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/service/ElasticClientHandlerAws.java index 3c7383a80aaa9aaf1a7fa8943673fa86dc0f371b..e6003130aad08cfdb49e57d70cee54f17eca625c 100644 --- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/service/ElasticClientHandlerAws.java +++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/service/ElasticClientHandlerAws.java @@ -44,8 +44,7 @@ public class ElasticClientHandlerAws extends ElasticClientHandler { RestClientBuilder builder = RestClient.builder(new HttpHost(host, port, protocolScheme)); builder.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder .setConnectTimeout(REST_CLIENT_CONNECT_TIMEOUT) - .setSocketTimeout(REST_CLIENT_SOCKET_TIMEOUT)); - builder.setMaxRetryTimeoutMillis(REST_CLIENT_RETRY_TIMEOUT); + .setSocketTimeout(REST_CLIENT_SOCKET_TIMEOUT)); if(isLocalHost(host)) { builder.setHttpClientConfigCallback(httpAsyncClientBuilder -> httpAsyncClientBuilder.setSSLHostnameVerifier((s, sslSession) -> true)); diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java new file mode 100644 index 0000000000000000000000000000000000000000..555dbc3964e0fe7e8308b329e98656310d13d78b --- /dev/null +++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java @@ -0,0 +1,109 @@ +// Copyright © Azure +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.opengroup.osdu.indexer.azure.util; + +import com.google.common.reflect.TypeToken; +import com.google.gson.*; +import com.microsoft.azure.servicebus.Message; +import lombok.extern.java.Log; +import org.opengroup.osdu.azure.servicebus.ITopicClientFactory; +import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; +import org.opengroup.osdu.core.common.model.http.DpsHeaders; +import org.opengroup.osdu.core.common.model.indexer.RecordInfo; +import org.opengroup.osdu.core.common.model.search.RecordChangedMessages; +import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Primary; +import org.springframework.stereotype.Component; +import org.springframework.web.context.annotation.RequestScope; + +import javax.inject.Inject; +import javax.inject.Named; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.lang.reflect.Type; + +@Log +@Component +@RequestScope +@Primary +public class IndexerQueueTaskBuilderAzure extends IndexerQueueTaskBuilder { + + @Autowired + private ITopicClientFactory topicClientFactory; + + @Inject + private JaxRsDpsLog logger; + + @Inject + @Named("SERVICE_BUS_TOPIC") + private String serviceBusTopic; + + @Override + public void createWorkerTask(String payload, Long countdownMillis, DpsHeaders headers) { + createTask(payload, headers); + } + + @Override + public void createReIndexTask(String payload, Long countdownMillis, DpsHeaders headers) { + createTask(payload, headers); + } + + private void createTask(String payload, DpsHeaders headers) { + Gson gson = new Gson(); + RecordChangedMessages receivedPayload = gson.fromJson(payload, RecordChangedMessages.class); + + Message message = new Message(); + Map<String, Object> properties = new HashMap<>(); + + // properties + properties.put(DpsHeaders.ACCOUNT_ID, headers.getPartitionIdWithFallbackToAccountId()); + properties.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId()); + headers.addCorrelationIdIfMissing(); + properties.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId()); + message.setProperties(properties); + + // data + List<RecordInfo> recordInfos = parseRecordsAsJSON(receivedPayload.getData()); + + // add all to body {"message": {"data":[], "id":...}} + JsonObject jo = new JsonObject(); + jo.add("data", gson.toJsonTree(recordInfos)); + jo.addProperty(DpsHeaders.ACCOUNT_ID, headers.getPartitionIdWithFallbackToAccountId()); + jo.addProperty(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId()); + jo.addProperty(DpsHeaders.CORRELATION_ID, headers.getCorrelationId()); + JsonObject jomsg = new JsonObject(); + jomsg.add("message", jo); + + message.setBody(jomsg.toString().getBytes(StandardCharsets.UTF_8)); + message.setContentType("application/json"); + + try { + logger.info("Indexer publishes message to Service Bus " + headers.getCorrelationId()); + topicClientFactory.getClient(headers.getPartitionId(), serviceBusTopic).send(message); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + private List<RecordInfo> parseRecordsAsJSON(String inputPayload) { + Gson gson = new Gson(); + Type type = new TypeToken<List<RecordInfo>>(){}.getType(); + List<RecordInfo> recordInfoList = gson.fromJson(inputPayload, type); + return recordInfoList; + } +} diff --git a/provider/indexer-gcp/README.md b/provider/indexer-gcp/README.md index b4b676171db6246e08ab4819d63580600dd7eb01..c40c11f423195ef35f0f093617f1dbcdcf0d3c75 100644 --- a/provider/indexer-gcp/README.md +++ b/provider/indexer-gcp/README.md @@ -35,7 +35,7 @@ In order to run the service locally or remotely, you will need to have the follo | `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials | | `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | https://console.cloud.google.com/iam-admin/serviceaccounts | | `security.https.certificate.trust` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment | -| `indexer.que.service.mail` | ex `default@iam.gserviceaccount.com` | IndexerQue environment service account mail, required if Indexer Que deployed in cloud task mode, to validate token from it | yes | - | +| `indexer.que.service.mail` | ex `default@iam.gserviceaccount.com` | Indexer Que environment service account mail, required if Indexer Que deployed in cloud task mode, to validate token from it | yes | - | ### Run Locally Check that maven is installed: @@ -154,7 +154,7 @@ You will need to have the following environment variables defined. | INTEGRATION_TESTER | NO_DATA_ACCESS_TESTER | | --- | --- | -| users<br/>service.entitlements.user<br/>service.search.user<br/>data.test1<br/>data.integration.test<br/>users@{tenant1}@{domain}.com | +| users<br/>service.entitlements.user<br/>service.search.user<br/>service.search.admin<br/>data.test1<br/>data.integration.test<br/>users@{tenant1}@{domain}.com | Execute following command to build code and run all the integration tests: @@ -199,9 +199,9 @@ Create king ring and key in the ***master project*** --purpose encryption ``` -Add **Cloud KMS CryptoKey Encrypter/Decrypter** role to the **App Engine default service account** of the master project through IAM - Role tab +Add **Cloud KMS CryptoKey Encrypter/Decrypter** role to the **App Engine default service account** of the ***master project*** through IAM - Role tab -Add "Cloud KMS Encrypt/Decrypt" role to the "App Engine default service account" of ***master project*** +Add **Cloud KMS Encrypt/Decrypt** role to the **App Engine default service account** of ***master project*** #### Memory Store (Redis Instance) Setup diff --git a/provider/indexer-gcp/pom.xml b/provider/indexer-gcp/pom.xml index bd7ee9b85f6dd24949f9ea4610d3d915bfe74bdd..a37a4df8947684574ceb67a8389f5b658fef23c7 100644 --- a/provider/indexer-gcp/pom.xml +++ b/provider/indexer-gcp/pom.xml @@ -25,7 +25,7 @@ <dependency> <groupId>org.opengroup.osdu</groupId> <artifactId>core-lib-gcp</artifactId> - <version>0.3.23</version> + <version>0.6.1-SNAPSHOT</version> </dependency> <dependency> @@ -148,7 +148,16 @@ <artifactId>proto-google-iam-v1</artifactId> <version>0.12.0</version> </dependency> - + <dependency> + <groupId>ch.qos.logback.contrib</groupId> + <artifactId>logback-json-classic</artifactId> + <version>0.1.5</version> + </dependency> + <dependency> + <groupId>ch.qos.logback.contrib</groupId> + <artifactId>logback-jackson</artifactId> + <version>0.1.5</version> + </dependency> </dependencies> <build> diff --git a/provider/indexer-gcp/src/main/resources/application.properties b/provider/indexer-gcp/src/main/resources/application.properties index b3d58b0c9b895a6ca38e8b29143edd258e7d8c8b..c2c74189250e40afae31e9f8c0174370a877c569 100644 --- a/provider/indexer-gcp/src/main/resources/application.properties +++ b/provider/indexer-gcp/src/main/resources/application.properties @@ -1,7 +1,7 @@ LOG_PREFIX=indexer server.servlet.contextPath=/api/indexer/v2 -logging.level.org.springframework.web=DEBUG +logging.level.org.springframework.web=${LOG_LEVEL:DEBUG} server.port=8080 JAVA_OPTS=-Xms3072m -Xmx3072m JAVA_GC_OPTS=-XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45 diff --git a/provider/indexer-gcp/src/main/resources/logback.xml b/provider/indexer-gcp/src/main/resources/logback.xml new file mode 100644 index 0000000000000000000000000000000000000000..3d4fbc039ba4367897cba9a478d4ed44a2059b76 --- /dev/null +++ b/provider/indexer-gcp/src/main/resources/logback.xml @@ -0,0 +1,37 @@ +<?xml version="1.0" encoding="UTF-8"?> +<configuration> + <include resource="org/springframework/boot/logging/logback/defaults.xml"/> + <logger name="org.opengroup.osdu" level="DEBUG"/> + <springProfile name="local"> + <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender"> + <encoder> + <pattern>%yellow([%thread]) %highlight(| %-5level |) %green(%d) %cyan(| %logger{15} |) %highlight(%msg) %n</pattern> + <charset>utf8</charset> + </encoder> + </appender> + <root level="info"> + <appender-ref ref="CONSOLE"/> + </root> + </springProfile> + + <springProfile name="!local"> + <appender name="stdout" class="ch.qos.logback.core.ConsoleAppender"> + <encoder class="ch.qos.logback.core.encoder.LayoutWrappingEncoder"> + <layout class="ch.qos.logback.contrib.json.classic.JsonLayout"> + <timestampFormat>yyyy-MM-dd HH:mm:ss.SSS</timestampFormat> + <timestampFormatTimezoneId>Etc/UTC</timestampFormatTimezoneId> + <appendLineSeparator>true</appendLineSeparator> + + <jsonFormatter class="org.opengroup.osdu.core.gcp.logging.formatter.GoogleJsonFormatter"> + <prettyPrint>false</prettyPrint> + </jsonFormatter> + </layout> + </encoder> + </appender> + + <root level="info"> + <appender-ref ref="stdout"/> + </root> + </springProfile> + +</configuration> diff --git a/provider/indexer-ibm/pom.xml b/provider/indexer-ibm/pom.xml index a8ef4cf4cb9bc3d439b114778adc2cad62fc4fc6..f0a0c5f9249d3faba1e8b92e25fd97aced2ff4e8 100644 --- a/provider/indexer-ibm/pom.xml +++ b/provider/indexer-ibm/pom.xml @@ -89,17 +89,14 @@ <dependency> <groupId>org.elasticsearch</groupId> <artifactId>elasticsearch</artifactId> - <version>6.6.2</version> </dependency> <dependency> <groupId>org.elasticsearch.client</groupId> <artifactId>elasticsearch-rest-client</artifactId> - <version>6.6.2</version> </dependency> <dependency> <groupId>org.elasticsearch.client</groupId> <artifactId>elasticsearch-rest-high-level-client</artifactId> - <version>6.6.2</version> </dependency> <!-- Test Dependencies --> diff --git a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerMappingServiceTest.java b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerMappingServiceTest.java index 7a6447711dda6b83ba8ba520c2480c45a276647f..d6982c113fc059e87626d2f931e4f2288f7fd3fd 100644 --- a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerMappingServiceTest.java +++ b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerMappingServiceTest.java @@ -1,12 +1,12 @@ /* Licensed Materials - Property of IBM */ /* (c) Copyright IBM Corp. 2020. All Rights Reserved.*/ - package org.opengroup.osdu.indexer.ibm.service; import org.apache.http.StatusLine; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse; -import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.action.bulk.BulkItemResponse.Failure; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.*; @@ -106,8 +106,7 @@ public class IndexerMappingServiceTest { public void should_returnValidMapping_givenTrueMerge_createMappingTest() { try { doReturn(this.indicesClient).when(this.restHighLevelClient).indices(); - doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class)); - + doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class)); String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true); assertEquals(this.mappingValid, mapping); } catch (Exception e) { @@ -119,8 +118,7 @@ public class IndexerMappingServiceTest { public void should_returnValidMapping_givenExistType_createMappingTest() { try { doReturn(this.indicesClient).when(this.restHighLevelClient).indices(); - doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class)); - + doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class)); IndexerMappingServiceImpl indexerMappingServiceLocal = PowerMockito.spy(new IndexerMappingServiceImpl()); doReturn(false).when(indexerMappingServiceLocal).isTypeExist(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any()); String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true); @@ -129,7 +127,7 @@ public class IndexerMappingServiceTest { fail("Should not throw this exception" + e.getMessage()); } } - + @Test public void should_update_indices_field_with_keyword_when_valid_indices() throws Exception { try { @@ -137,33 +135,33 @@ public class IndexerMappingServiceTest { indices.add("indices 1"); GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class); doReturn(this.indicesClient).when(this.restHighLevelClient).indices(); - when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse); + when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse); XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject(); builder.field("any field", new HashMap()); builder.endObject(); BytesReference bytesReference = BytesReference.bytes(builder); - FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference); - Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>(); + GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference); + Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>(); mapBuilder.put("data.any field", mappingMetaData); - Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>(); + Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>(); mappingBuilder.put("any index 1", mapBuilder); mappingBuilder.put("any index 2", mapBuilder); - Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>(); + Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>(); mapping.put("indices 1", mappingBuilder); when(getFieldMappingsResponse.mappings()).thenReturn(mapping); - doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class)); + doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class)); BulkByScrollResponse response = mock(BulkByScrollResponse.class); doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class)); when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>()); when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); - + this.sut.updateIndexMappingForIndicesOfSameType( indices,"any field"); } catch (Exception e) { fail("Should not throw this exception" + e.getMessage()); } } - + @Test(expected = AppException.class) public void should_throw_exception_if_someIndex_is_invalid_andWeIndexfield_with_keyword() throws Exception { try { @@ -171,28 +169,28 @@ public class IndexerMappingServiceTest { indices.add("invalid 1"); GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class); doReturn(this.indicesClient).when(this.restHighLevelClient).indices(); - when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse); + when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse); XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject(); builder.field("any field", new HashMap()); builder.endObject(); BytesReference bytesReference = BytesReference.bytes(builder); - FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference); - Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>(); + GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference); + Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>(); mapBuilder.put("data.any field", mappingMetaData); - Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>(); + Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>(); mappingBuilder.put("any index 1", mapBuilder); mappingBuilder.put("any index 2", mapBuilder); - Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>(); + Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>(); mapping.put("indices 1", mappingBuilder); when(getFieldMappingsResponse.mappings()).thenReturn(mapping); - doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class)); + doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class)); BulkByScrollResponse response = mock(BulkByScrollResponse.class); doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class)); when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>()); when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); - - this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field"); + + this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field"); } catch (Exception e) { throw e; } @@ -205,22 +203,22 @@ public class IndexerMappingServiceTest { indices.add("indices 1"); GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class); doReturn(this.indicesClient).when(this.restHighLevelClient).indices(); - when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse); + when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse); XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject(); builder.field("any field", new HashMap()); builder.endObject(); BytesReference bytesReference = BytesReference.bytes(builder); - FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference); - Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>(); + GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference); + Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>(); mapBuilder.put("data.any field", mappingMetaData); - Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>(); + Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>(); mappingBuilder.put("any index 1", mapBuilder); mappingBuilder.put("any index 2", mapBuilder); - Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>(); + Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>(); mapping.put("indices 1", mappingBuilder); when(getFieldMappingsResponse.mappings()).thenReturn(mapping); - doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class)); + doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class)); BulkByScrollResponse response = mock(BulkByScrollResponse.class); doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class)); when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>()); @@ -240,22 +238,22 @@ public class IndexerMappingServiceTest { indices.add("indices Invalid"); GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class); doReturn(this.indicesClient).when(this.restHighLevelClient).indices(); - when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenThrow(new ElasticsearchException("")); + when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenThrow(new ElasticsearchException("")); XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject(); builder.field("any field", new HashMap()); builder.endObject(); BytesReference bytesReference = BytesReference.bytes(builder); - FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference); - Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>(); + GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference); + Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>(); mapBuilder.put("data.any field", mappingMetaData); - Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>(); + Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>(); mappingBuilder.put("any index 1", mapBuilder); mappingBuilder.put("any index 2", mapBuilder); - Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>(); + Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>(); mapping.put("indices 1", mappingBuilder); when(getFieldMappingsResponse.mappings()).thenReturn(mapping); - doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class)); + doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class)); BulkByScrollResponse response = mock(BulkByScrollResponse.class); doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class)); when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>()); @@ -274,26 +272,26 @@ public class IndexerMappingServiceTest { indices.add("indices Invalid"); GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class); doReturn(this.indicesClient).when(this.restHighLevelClient).indices(); - when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse); + when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse); XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject(); builder.field("any field", new HashMap()); builder.endObject(); BytesReference bytesReference = BytesReference.bytes(builder); - FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference); - Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>(); + GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference); + Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>(); mapBuilder.put("data.any field", mappingMetaData); - Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>(); + Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>(); mappingBuilder.put("any index 1", mapBuilder); mappingBuilder.put("any index 2", mapBuilder); - Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>(); + Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>(); mapping.put("indices 1", mappingBuilder); when(getFieldMappingsResponse.mappings()).thenReturn(mapping); - doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class)); + doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class)); BulkByScrollResponse response = mock(BulkByScrollResponse.class); doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class)); when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>()); - when(this.indicesClient.putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class))).thenThrow(new ElasticsearchException("")); + when(this.indicesClient.putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class))).thenThrow(new ElasticsearchException("")); when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient); this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field"); } catch (AppException e) { @@ -303,3 +301,4 @@ public class IndexerMappingServiceTest { } } } + diff --git a/testing/indexer-test-aws/pom.xml b/testing/indexer-test-aws/pom.xml index e8c2f455b1a331a3cff88dbb309e509e59b1cbb5..9335cf4e8d5323f6ad973d3114af68a272a27220 100644 --- a/testing/indexer-test-aws/pom.xml +++ b/testing/indexer-test-aws/pom.xml @@ -73,6 +73,22 @@ </dependency> <!-- third party Apache 2.0 license packages --> + <!--Elasticsearch--> + <dependency> + <groupId>org.elasticsearch</groupId> + <artifactId>elasticsearch</artifactId> + <version>7.8.1</version> + </dependency> + <dependency> + <groupId>org.elasticsearch.client</groupId> + <artifactId>elasticsearch-rest-client</artifactId> + <version>7.8.1</version> + </dependency> + <dependency> + <groupId>org.elasticsearch.client</groupId> + <artifactId>elasticsearch-rest-high-level-client</artifactId> + <version>7.8.1</version> + </dependency> <dependency> <groupId>com.google.code.gson</groupId> <artifactId>gson</artifactId> diff --git a/testing/indexer-test-ibm/pom.xml b/testing/indexer-test-ibm/pom.xml index 1d0ec6267671426190b4f425241b0993597c5f75..f85b8b8ad1a2d3216ac246e5fe580596d3cf6e35 100644 --- a/testing/indexer-test-ibm/pom.xml +++ b/testing/indexer-test-ibm/pom.xml @@ -114,17 +114,17 @@ <dependency> <groupId>org.elasticsearch</groupId> <artifactId>elasticsearch</artifactId> - <version>6.6.2</version> + <version>7.8.1</version> </dependency> <dependency> <groupId>org.elasticsearch.client</groupId> <artifactId>elasticsearch-rest-client</artifactId> - <version>6.6.2</version> + <version>7.8.1</version> </dependency> <dependency> <groupId>org.elasticsearch.client</groupId> <artifactId>elasticsearch-rest-high-level-client</artifactId> - <version>6.6.2</version> + <version>7.8.1</version> </dependency> <!--Logging-->