Merge branch 'master' into enable-keyword

NOTICE

@@ -232,7 +232,6 @@ The following software have components provided under the terms of this license:
 - Doxia :: XHTML Module (from )
 - Elastic JNA Distribution (from https://github.com/java-native-access/jna)
 - Elasticsearch: 5.0.0-alpha5 (from https://github.com/elastic/elasticsearch)
-- Elasticsearch: 5.0.0-alpha5 (from https://github.com/elastic/elasticsearch)
 - FindBugs-jsr305 (from http://findbugs.sourceforge.net/)
 - Google APIs Client Library for Java (from )
 - Google App Engine extensions to the Google HTTP Client Library for Java. (from )
@@ -313,35 +312,22 @@ The following software have components provided under the terms of this license:
 - KeePassJava2 :: KDB (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2-kdb)
 - KeePassJava2 :: KDBX (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2-kdbx)
 - KeePassJava2 :: Simple (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/KeePassJava2-simple)
+- Logback Contrib :: JSON :: Classic (from )
+- Logback Contrib :: JSON :: Core (from )
+- Logback Contrib :: Jackson (from )
 - Lucene Common Analyzers (from )
-- Lucene Common Analyzers (from )
-- Lucene Core (from )
 - Lucene Core (from )
 - Lucene Grouping (from )
-- Lucene Grouping (from )
-- Lucene Highlighter (from )
 - Lucene Highlighter (from )
 - Lucene Join (from )
-- Lucene Join (from )
 - Lucene Memory (from )
 - Lucene Memory (from )
-- Lucene Memory (from )
-- Lucene Memory (from )
-- Lucene Miscellaneous (from )
 - Lucene Miscellaneous (from )
 - Lucene Queries (from )
-- Lucene Queries (from )
-- Lucene QueryParsers (from )
 - Lucene QueryParsers (from )
 - Lucene Sandbox (from )
-- Lucene Sandbox (from )
-- Lucene Spatial (from )
-- Lucene Spatial (from )
-- Lucene Spatial 3D (from )
 - Lucene Spatial 3D (from )
 - Lucene Spatial Extras (from )
-- Lucene Spatial Extras (from )
-- Lucene Suggest (from )
 - Lucene Suggest (from )
 - MapStruct Core (from )
 - Maven Artifact (from )
@@ -366,10 +352,10 @@ The following software have components provided under the terms of this license:
 - Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
 - Microsoft Azure Netty HTTP Client Library (from https://github.com/Azure/azure-sdk-for-java)
 - Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java)
-- Mockito (from http://mockito.org)
 - Mockito (from http://www.mockito.org)
 - Mockito (from http://mockito.org)
 - Mockito (from http://mockito.org)
+- Mockito (from http://mockito.org)
 - Mojo's Maven plugin for Cobertura (from http://mojo.codehaus.org/cobertura-maven-plugin/)
 - Netty Reactive Streams HTTP support (from )
 - Netty Reactive Streams Implementation (from )
@@ -385,11 +371,15 @@ The following software have components provided under the terms of this license:
 - Netty/TomcatNative [BoringSSL - Static] (from )
 - Netty/Transport (from http://netty.io/)
 - Netty/Transport/Native/Unix/Common (from )
+- Nimbus Content Type (from https://bitbucket.org/connect2id/nimbus-content-type)
+- Nimbus JOSE+JWT (from https://bitbucket.org/connect2id/nimbus-jose-jwt)
 - Nimbus JOSE+JWT (from https://bitbucket.org/connect2id/nimbus-jose-jwt)
 - Nimbus JOSE+JWT (from https://bitbucket.org/connect2id/nimbus-jose-jwt)
 - Nimbus LangTag (from https://bitbucket.org/connect2id/nimbus-language-tags)
+- Nimbus LangTag (from https://bitbucket.org/connect2id/nimbus-language-tags)
 - Non-Blocking Reactive Foundation for the JVM (from https://github.com/reactor/reactor)
 - OAuth 2.0 SDK with OpenID Connect extensions (from https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions)
+- OAuth 2.0 SDK with OpenID Connect extensions (from https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions)
 - OAuth2 for Spring Security (from )
 - Objenesis (from http://objenesis.org)
 - OkHttp (from )
@@ -462,10 +452,8 @@ The following software have components provided under the terms of this license:
 - Xerces2-j (from https://xerces.apache.org/xerces2-j/)
 - aalto-xml (from )
 - aggs-matrix-stats (from https://github.com/elastic/elasticsearch)
-- aggs-matrix-stats (from https://github.com/elastic/elasticsearch)
 - aws-ssm-java-caching-client (from https://github.com/awslabs/aws-ssm-java-caching-client)
 - cli (from https://github.com/elastic/elasticsearch)
-- cli (from https://github.com/elastic/elasticsearch)
 - com.google.api.grpc:grpc-google-cloud-pubsub-v1 (from https://github.com/googleapis/googleapis)
 - com.google.api.grpc:proto-google-cloud-logging-v2 (from https://github.com/googleapis/googleapis)
 - com.google.api.grpc:proto-google-cloud-pubsub-v1 (from https://github.com/googleapis/googleapis)
@@ -475,7 +463,7 @@ The following software have components provided under the terms of this license:
 - compiler (from http://github.com/spullara/mustache.java)
 - datastore-v1-proto-client (from )
 - elasticsearch-core (from https://github.com/elastic/elasticsearch)
-- elasticsearch-core (from https://github.com/elastic/elasticsearch)
+- elasticsearch-geo (from https://github.com/elastic/elasticsearch)
 - error-prone annotations (from )
 - error-prone annotations (from )
 - io.grpc:grpc-alts (from https://github.com/grpc/grpc-java)
@@ -509,14 +497,13 @@ The following software have components provided under the terms of this license:
 - jersey-spring4 (from )
 - jose4j (from https://bitbucket.org/b_c/jose4j/)
 - lang-mustache (from https://github.com/elastic/elasticsearch)
-- lang-mustache (from https://github.com/elastic/elasticsearch)
 - lettuce (from http://github.com/mp911de/lettuce/wiki)
+- mapper-extras (from https://github.com/elastic/elasticsearch)
 - micrometer-core (from https://github.com/micrometer-metrics/micrometer)
 - micrometer-registry-azure-monitor (from https://github.com/micrometer-metrics/micrometer)
 - org.xmlunit:xmlunit-core (from http://www.xmlunit.org/)
 - oro (from )
 - parent-join (from https://github.com/elastic/elasticsearch)
-- parent-join (from https://github.com/elastic/elasticsearch)
 - powermock-api-support (from )
 - powermock-core (from http://www.powermock.org)
 - powermock-module-junit4 (from http://www.powermock.org)
@@ -525,14 +512,10 @@ The following software have components provided under the terms of this license:
 - proto-google-cloud-datastore-v1 (from https://github.com/googleapis/api-client-staging)
 - proton-j (from )
 - rank-eval (from https://github.com/elastic/elasticsearch)
-- rank-eval (from https://github.com/elastic/elasticsearch)
 - rest (from https://github.com/elastic/elasticsearch)
-- rest (from https://github.com/elastic/elasticsearch)
-- rest-high-level (from https://github.com/elastic/elasticsearch)
 - rest-high-level (from https://github.com/elastic/elasticsearch)
 - rxjava (from https://github.com/ReactiveX/RxJava)
 - secure-sm (from https://github.com/elastic/elasticsearch)
-- secure-sm (from https://github.com/elastic/elasticsearch)
 - spring-security-config (from http://spring.io/spring-security)
 - spring-security-core (from http://spring.io/spring-security)
 - spring-security-oauth2-client (from http://spring.io/spring-security)
@@ -556,7 +539,6 @@ The following software have components provided under the terms of this license:
 - tomcat-embed-el (from http://tomcat.apache.org/)
 - tomcat-embed-websocket (from http://tomcat.apache.org/)
 - x-content (from https://github.com/elastic/elasticsearch)
-- x-content (from https://github.com/elastic/elasticsearch)
 - xml-apis (from )
 
 ========================================================================
@@ -570,7 +552,7 @@ The following software have components provided under the terms of this license:
 - GAX (Google Api eXtensions) (from https://github.com/googleapis)
 - Hamcrest Core (from http://hamcrest.org/)
 - Lucene Common Analyzers (from )
-- Lucene Common Analyzers (from )
+- Lucene Core (from )
 - Plexus :: Default Container (from )
 - Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils)
 - StAX (from http://stax.codehaus.org/)
@@ -603,11 +585,8 @@ The following software have components provided under the terms of this license:
 - JSch (from http://www.jcraft.com/jsch/)
 - JavaBeans Activation Framework API jar (from )
 - Lucene Common Analyzers (from )
-- Lucene Common Analyzers (from )
-- Lucene Core (from )
 - Lucene Core (from )
 - Lucene Suggest (from )
-- Lucene Suggest (from )
 - Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java)
 - Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java)
 - Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java)
@@ -719,6 +698,9 @@ EPL-1.0
 The following software have components provided under the terms of this license:
 
 - Logback Classic Module (from )
+- Logback Contrib :: JSON :: Classic (from )
+- Logback Contrib :: JSON :: Core (from )
+- Logback Contrib :: Jackson (from )
 - Logback Core Module (from )
 - Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java)
 - Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java)
@@ -830,6 +812,9 @@ The following software have components provided under the terms of this license:
 - Javassist (from http://www.javassist.org/)
 - Javassist (from http://www.javassist.org/)
 - Logback Classic Module (from )
+- Logback Contrib :: JSON :: Classic (from )
+- Logback Contrib :: JSON :: Core (from )
+- Logback Contrib :: Jackson (from )
 - Logback Core Module (from )
 - Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java)
 - Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java)
@@ -870,16 +855,12 @@ The following software have components provided under the terms of this license:
 - Bouncy Castle Provider (from http://www.bouncycastle.org/java.html)
 - Checker Qual (from https://checkerframework.org)
 - Checker Qual (from https://checkerframework.org)
-- Elasticsearch: 5.0.0-alpha5 (from https://github.com/elastic/elasticsearch)
 - Extensions on Apache Proton-J library (from https://github.com/Azure/qpid-proton-j-extensions)
 - JOpt Simple (from http://pholser.github.io/jopt-simple)
 - JUL to SLF4J bridge (from http://www.slf4j.org)
 - Java Client Runtime for AutoRest (from https://github.com/Azure/autorest-clientruntime-for-java)
 - Java JWT (from http://www.jwt.io)
 - Lucene Core (from )
-- Lucene Core (from )
-- Lucene Sandbox (from )
-- Lucene Sandbox (from )
 - Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java)
 - Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java)
 - Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java)
@@ -894,9 +875,9 @@ The following software have components provided under the terms of this license:
 - Microsoft Azure client library for KeyVault Secrets (from https://github.com/Azure/azure-sdk-for-java)
 - Microsoft Azure common module for Storage (from https://github.com/Azure/azure-sdk-for-java)
 - Microsoft Azure internal Avro module for Storage (from https://github.com/Azure/azure-sdk-for-java)
+- Mockito (from http://www.mockito.org)
 - Mockito (from http://mockito.org)
 - Mockito (from http://mockito.org)
-- Mockito (from http://www.mockito.org)
 - Mockito (from http://mockito.org)
 - Netty/Codec/HTTP (from )
 - Netty/Common (from )

devops/azure/chart/templates/deployment.yaml

@@ -100,7 +100,7 @@ spec:
         - name: STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST
           value: http://storage/api/storage/v2/query/records:batch
         - name: STORAGE_QUERY_RECORD_HOST
-          value: http://storage/api/storage/v2/records
+          value: http://storage/api/storage/v2/query/records
         - name: partition_service_endpoint
           value: http://partition/api/partition/v1
         - name: azure_istioauth_enabled

devops/azure/release.yaml

@@ -161,4 +161,4 @@ spec:
         - name: STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST
           value: http://storage/api/storage/v2/query/records:batch
         - name: STORAGE_QUERY_RECORD_HOST
-          value: http://storage/api/storage/v2/records
+          value: http://storage/api/storage/v2/query/records

provider/indexer-aws/pom.xml

@@ -90,12 +90,6 @@
       <groupId>org.springframework.security</groupId>
       <artifactId>spring-security-oauth2-jose</artifactId>
     </dependency>
-    <dependency>
-      <groupId>org.apache.lucene</groupId>
-      <artifactId>lucene-core</artifactId>
-      <version>7.6.0</version>
-      <scope>compile</scope>
-    </dependency>
     <dependency>
       <groupId>javax.inject</groupId>
       <artifactId>javax.inject</artifactId>
@@ -105,17 +99,14 @@
     <dependency>
       <groupId>org.elasticsearch</groupId>
       <artifactId>elasticsearch</artifactId>
-      <version>6.6.2</version>
     </dependency>
     <dependency>
-      <groupId>org.elasticsearch.client</groupId>
-      <artifactId>elasticsearch-rest-client</artifactId>
-      <version>6.6.2</version>
+        <groupId>org.elasticsearch.client</groupId>
+        <artifactId>elasticsearch-rest-client</artifactId>
     </dependency>
     <dependency>
-      <groupId>org.elasticsearch.client</groupId>
-      <artifactId>elasticsearch-rest-high-level-client</artifactId>
-      <version>6.6.2</version>
+        <groupId>org.elasticsearch.client</groupId>
+        <artifactId>elasticsearch-rest-high-level-client</artifactId>
     </dependency>
 
     <!-- Testing packages -->

provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/service/ElasticClientHandlerAws.java

@@ -44,8 +44,7 @@ public class ElasticClientHandlerAws extends ElasticClientHandler {
         RestClientBuilder builder = RestClient.builder(new HttpHost(host, port, protocolScheme));
         builder.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder
                 .setConnectTimeout(REST_CLIENT_CONNECT_TIMEOUT)
-                .setSocketTimeout(REST_CLIENT_SOCKET_TIMEOUT));
-        builder.setMaxRetryTimeoutMillis(REST_CLIENT_RETRY_TIMEOUT);
+                .setSocketTimeout(REST_CLIENT_SOCKET_TIMEOUT));        
 
         if(isLocalHost(host)) {
             builder.setHttpClientConfigCallback(httpAsyncClientBuilder -> httpAsyncClientBuilder.setSSLHostnameVerifier((s, sslSession) -> true));

provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java

+// Copyright © Azure
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.azure.util;
+
+import com.google.common.reflect.TypeToken;
+import com.google.gson.*;
+import com.microsoft.azure.servicebus.Message;
+import lombok.extern.java.Log;
+import org.opengroup.osdu.azure.servicebus.ITopicClientFactory;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
+import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
+import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Primary;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+import javax.inject.Inject;
+import javax.inject.Named;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.lang.reflect.Type;
+
+@Log
+@Component
+@RequestScope
+@Primary
+public class IndexerQueueTaskBuilderAzure extends IndexerQueueTaskBuilder {
+
+    @Autowired
+    private ITopicClientFactory topicClientFactory;
+
+    @Inject
+    private JaxRsDpsLog logger;
+
+    @Inject
+    @Named("SERVICE_BUS_TOPIC")
+    private String serviceBusTopic;
+
+    @Override
+    public void createWorkerTask(String payload, Long countdownMillis, DpsHeaders headers) {
+        createTask(payload, headers);
+    }
+
+    @Override
+    public void createReIndexTask(String payload, Long countdownMillis, DpsHeaders headers) {
+        createTask(payload, headers);
+    }
+
+    private void createTask(String payload, DpsHeaders headers) {
+        Gson gson = new Gson();
+        RecordChangedMessages receivedPayload = gson.fromJson(payload, RecordChangedMessages.class);
+
+        Message message = new Message();
+        Map<String, Object> properties = new HashMap<>();
+
+        // properties
+        properties.put(DpsHeaders.ACCOUNT_ID, headers.getPartitionIdWithFallbackToAccountId());
+        properties.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
+        headers.addCorrelationIdIfMissing();
+        properties.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
+        message.setProperties(properties);
+
+        // data
+        List<RecordInfo> recordInfos = parseRecordsAsJSON(receivedPayload.getData());
+
+        // add all to body {"message": {"data":[], "id":...}}
+        JsonObject jo = new JsonObject();
+        jo.add("data", gson.toJsonTree(recordInfos));
+        jo.addProperty(DpsHeaders.ACCOUNT_ID, headers.getPartitionIdWithFallbackToAccountId());
+        jo.addProperty(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
+        jo.addProperty(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
+        JsonObject jomsg = new JsonObject();
+        jomsg.add("message", jo);
+
+        message.setBody(jomsg.toString().getBytes(StandardCharsets.UTF_8));
+        message.setContentType("application/json");
+
+        try {
+            logger.info("Indexer publishes message to Service Bus " + headers.getCorrelationId());
+            topicClientFactory.getClient(headers.getPartitionId(), serviceBusTopic).send(message);
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+    }
+
+    private List<RecordInfo> parseRecordsAsJSON(String inputPayload) {
+        Gson gson = new Gson();
+        Type type = new TypeToken<List<RecordInfo>>(){}.getType();
+        List<RecordInfo> recordInfoList = gson.fromJson(inputPayload, type);
+        return  recordInfoList;
+    }
+}

provider/indexer-gcp/README.md

@@ -35,7 +35,7 @@ In order to run the service locally or remotely, you will need to have the follo
 | `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials |
 | `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | https://console.cloud.google.com/iam-admin/serviceaccounts |
 | `security.https.certificate.trust` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment |
-| `indexer.que.service.mail` | ex `default@iam.gserviceaccount.com` | IndexerQue environment service account mail, required if Indexer Que deployed in cloud task mode, to validate token from it | yes | - |
+| `indexer.que.service.mail` | ex `default@iam.gserviceaccount.com` | Indexer Que environment service account mail, required if Indexer Que deployed in cloud task mode, to validate token from it | yes | - |
 
 ### Run Locally
 Check that maven is installed:
@@ -154,7 +154,7 @@ You will need to have the following environment variables defined.
 
 | INTEGRATION_TESTER | NO_DATA_ACCESS_TESTER | 
 | ---  | ---   |
-| users<br/>service.entitlements.user<br/>service.search.user<br/>data.test1<br/>data.integration.test<br/>users@{tenant1}@{domain}.com |
+| users<br/>service.entitlements.user<br/>service.search.user<br/>service.search.admin<br/>data.test1<br/>data.integration.test<br/>users@{tenant1}@{domain}.com |
 
 Execute following command to build code and run all the integration tests:
 
@@ -199,9 +199,9 @@ Create king ring and key in the ***master project***
     		--purpose encryption
 ```
 
-Add **Cloud KMS CryptoKey Encrypter/Decrypter** role to the **App Engine default service account** of the master project through IAM - Role tab
+Add **Cloud KMS CryptoKey Encrypter/Decrypter** role to the **App Engine default service account** of the ***master project*** through IAM - Role tab
 
-Add "Cloud KMS Encrypt/Decrypt" role to the "App Engine default service account" of ***master project***
+Add **Cloud KMS Encrypt/Decrypt** role to the **App Engine default service account** of ***master project***
 
 #### Memory Store (Redis Instance) Setup
 

provider/indexer-gcp/pom.xml

@@ -25,7 +25,7 @@
         <dependency>
             <groupId>org.opengroup.osdu</groupId>
             <artifactId>core-lib-gcp</artifactId>
-            <version>0.3.23</version>
+            <version>0.6.1-SNAPSHOT</version>
         </dependency>
 
         <dependency>
@@ -148,7 +148,16 @@
             <artifactId>proto-google-iam-v1</artifactId>
             <version>0.12.0</version>
         </dependency>
-
+        <dependency>
+            <groupId>ch.qos.logback.contrib</groupId>
+            <artifactId>logback-json-classic</artifactId>
+            <version>0.1.5</version>
+        </dependency>
+        <dependency>
+            <groupId>ch.qos.logback.contrib</groupId>
+            <artifactId>logback-jackson</artifactId>
+            <version>0.1.5</version>
+        </dependency>
     </dependencies>
 
     <build>

provider/indexer-gcp/src/main/resources/application.properties

 LOG_PREFIX=indexer
 
 server.servlet.contextPath=/api/indexer/v2
-logging.level.org.springframework.web=DEBUG
+logging.level.org.springframework.web=${LOG_LEVEL:DEBUG}
 server.port=8080
 JAVA_OPTS=-Xms3072m -Xmx3072m
 JAVA_GC_OPTS=-XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45

provider/indexer-gcp/src/main/resources/logback.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<configuration>
+  <include resource="org/springframework/boot/logging/logback/defaults.xml"/>
+  <logger name="org.opengroup.osdu" level="DEBUG"/>
+  <springProfile name="local">
+    <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+      <encoder>
+        <pattern>%yellow([%thread]) %highlight(| %-5level |) %green(%d) %cyan(| %logger{15} |) %highlight(%msg) %n</pattern>
+        <charset>utf8</charset>
+      </encoder>
+    </appender>
+    <root level="info">
+      <appender-ref ref="CONSOLE"/>
+    </root>
+  </springProfile>
+
+  <springProfile name="!local">
+    <appender name="stdout" class="ch.qos.logback.core.ConsoleAppender">
+      <encoder class="ch.qos.logback.core.encoder.LayoutWrappingEncoder">
+        <layout class="ch.qos.logback.contrib.json.classic.JsonLayout">
+          <timestampFormat>yyyy-MM-dd HH:mm:ss.SSS</timestampFormat>
+          <timestampFormatTimezoneId>Etc/UTC</timestampFormatTimezoneId>
+          <appendLineSeparator>true</appendLineSeparator>
+
+          <jsonFormatter class="org.opengroup.osdu.core.gcp.logging.formatter.GoogleJsonFormatter">
+            <prettyPrint>false</prettyPrint>
+          </jsonFormatter>
+        </layout>
+      </encoder>
+    </appender>
+
+    <root level="info">
+      <appender-ref ref="stdout"/>
+    </root>
+  </springProfile>
+
+</configuration>

provider/indexer-ibm/pom.xml

@@ -89,17 +89,14 @@
         <dependency>
             <groupId>org.elasticsearch</groupId>
             <artifactId>elasticsearch</artifactId>
-            <version>6.6.2</version>
         </dependency>
         <dependency>
             <groupId>org.elasticsearch.client</groupId>
             <artifactId>elasticsearch-rest-client</artifactId>
-            <version>6.6.2</version>
         </dependency>
         <dependency>
             <groupId>org.elasticsearch.client</groupId>
             <artifactId>elasticsearch-rest-high-level-client</artifactId>
-            <version>6.6.2</version>
         </dependency>
 
         <!-- Test Dependencies -->

provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerMappingServiceTest.java

 /* Licensed Materials - Property of IBM              */		
 /* (c) Copyright IBM Corp. 2020. All Rights Reserved.*/
-
 package org.opengroup.osdu.indexer.ibm.service;
 
 import org.apache.http.StatusLine;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest;
 import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
-import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.*;
@@ -106,8 +106,7 @@ public class IndexerMappingServiceTest {
 	public void should_returnValidMapping_givenTrueMerge_createMappingTest() {
 		try {
 			doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
-			doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
-
+			doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
 			String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true);
 			assertEquals(this.mappingValid, mapping);
 		} catch (Exception e) {
@@ -119,8 +118,7 @@ public class IndexerMappingServiceTest {
 	public void should_returnValidMapping_givenExistType_createMappingTest() {
 		try {
 			doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
-			doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
-
+			doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
 			IndexerMappingServiceImpl indexerMappingServiceLocal = PowerMockito.spy(new IndexerMappingServiceImpl());
 			doReturn(false).when(indexerMappingServiceLocal).isTypeExist(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any());
 			String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true);
@@ -129,7 +127,7 @@ public class IndexerMappingServiceTest {
 			fail("Should not throw this exception" + e.getMessage());
 		}
 	}
-
+	
 	@Test
 	public void should_update_indices_field_with_keyword_when_valid_indices() throws Exception {
 		try {
@@ -137,33 +135,33 @@ public class IndexerMappingServiceTest {
 			indices.add("indices 1");
 			GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
 			doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
-			when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
+			when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
 			XContentBuilder builder = XContentFactory.jsonBuilder();
 			builder.startObject();
 			builder.field("any field", new HashMap());
 			builder.endObject();
 			BytesReference bytesReference = BytesReference.bytes(builder);
-			FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
-			Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+			GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+			Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
 			mapBuilder.put("data.any field", mappingMetaData);
-			Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+			Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
 			mappingBuilder.put("any index 1", mapBuilder);
 			mappingBuilder.put("any index 2", mapBuilder);
-			Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+			Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
 			mapping.put("indices 1", mappingBuilder);
 			when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
-			doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+			doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
 			BulkByScrollResponse response = mock(BulkByScrollResponse.class);
 			doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
 			when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
 			when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
-			
+
 			this.sut.updateIndexMappingForIndicesOfSameType( indices,"any field");
 		} catch (Exception e) {
 			fail("Should not throw this exception" + e.getMessage());
 		}
 	}
-
+	
 	@Test(expected = AppException.class)
 	public void should_throw_exception_if_someIndex_is_invalid_andWeIndexfield_with_keyword() throws Exception {
 		try {
@@ -171,28 +169,28 @@ public class IndexerMappingServiceTest {
 			indices.add("invalid 1");
 			GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
 			doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
-			when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
+			when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
 			XContentBuilder builder = XContentFactory.jsonBuilder();
 			builder.startObject();
 			builder.field("any field", new HashMap());
 			builder.endObject();
 			BytesReference bytesReference = BytesReference.bytes(builder);
-			FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
-			Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+			GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+			Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
 			mapBuilder.put("data.any field", mappingMetaData);
-			Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+			Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
 			mappingBuilder.put("any index 1", mapBuilder);
 			mappingBuilder.put("any index 2", mapBuilder);
-			Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+			Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
 			mapping.put("indices 1", mappingBuilder);
 			when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
-			doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+			doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
 			BulkByScrollResponse response = mock(BulkByScrollResponse.class);
 			doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
 			when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
 			when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
-			
-			this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");			
+
+			this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
 		} catch (Exception e) {
 			throw e;
 		}
@@ -205,22 +203,22 @@ public class IndexerMappingServiceTest {
 			indices.add("indices 1");			
 			GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
 			doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
-			when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
+			when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
 			XContentBuilder builder = XContentFactory.jsonBuilder();
 			builder.startObject();
 			builder.field("any field", new HashMap());
 			builder.endObject();
 			BytesReference bytesReference = BytesReference.bytes(builder);
-			FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
-			Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+			GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+			Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
 			mapBuilder.put("data.any field", mappingMetaData);
-			Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+			Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
 			mappingBuilder.put("any index 1", mapBuilder);
 			mappingBuilder.put("any index 2", mapBuilder);
-			Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+			Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
 			mapping.put("indices 1", mappingBuilder);
 			when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
-			doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+			doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
 			BulkByScrollResponse response = mock(BulkByScrollResponse.class);
 			doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
 			when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
@@ -240,22 +238,22 @@ public class IndexerMappingServiceTest {
 			indices.add("indices Invalid");
 			GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
 			doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
-			when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenThrow(new ElasticsearchException(""));
+			when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenThrow(new ElasticsearchException(""));
 			XContentBuilder builder = XContentFactory.jsonBuilder();
 			builder.startObject();
 			builder.field("any field", new HashMap());
 			builder.endObject();
 			BytesReference bytesReference = BytesReference.bytes(builder);
-			FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
-			Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+			GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+			Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
 			mapBuilder.put("data.any field", mappingMetaData);
-			Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+			Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
 			mappingBuilder.put("any index 1", mapBuilder);
 			mappingBuilder.put("any index 2", mapBuilder);
-			Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+			Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
 			mapping.put("indices 1", mappingBuilder);
 			when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
-			doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+			doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
 			BulkByScrollResponse response = mock(BulkByScrollResponse.class);
 			doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
 			when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
@@ -274,26 +272,26 @@ public class IndexerMappingServiceTest {
 			indices.add("indices Invalid");
 			GetFieldMappingsResponse getFieldMappingsResponse = mock(GetFieldMappingsResponse.class);
 			doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
-			when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
+			when(this.indicesClient.getFieldMapping(ArgumentMatchers.any(GetFieldMappingsRequest.class), ArgumentMatchers.any())).thenReturn(getFieldMappingsResponse);
 			XContentBuilder builder = XContentFactory.jsonBuilder();
 			builder.startObject();
 			builder.field("any field", new HashMap());
 			builder.endObject();
 			BytesReference bytesReference = BytesReference.bytes(builder);
-			FieldMappingMetaData mappingMetaData = new FieldMappingMetaData(index, bytesReference);
-			Map<String, FieldMappingMetaData> mapBuilder = new HashMap<>();
+			GetFieldMappingsResponse.FieldMappingMetadata mappingMetaData = new GetFieldMappingsResponse.FieldMappingMetadata(index, bytesReference);
+			Map<String, GetFieldMappingsResponse.FieldMappingMetadata> mapBuilder = new HashMap<>();
 			mapBuilder.put("data.any field", mappingMetaData);
-			Map<String, Map<String, FieldMappingMetaData>> mappingBuilder = new HashMap<>();
+			Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> mappingBuilder = new HashMap<>();
 			mappingBuilder.put("any index 1", mapBuilder);
 			mappingBuilder.put("any index 2", mapBuilder);
-			Map<String, Map<String, Map<String, FieldMappingMetaData>>> mapping = new HashMap<>();
+			Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> mapping = new HashMap<>();
 			mapping.put("indices 1", mappingBuilder);
 			when(getFieldMappingsResponse.mappings()).thenReturn(mapping);
-			doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
+			doReturn(mappingResponse).when(this.indicesClient).putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class));
 			BulkByScrollResponse response = mock(BulkByScrollResponse.class);
 			doReturn(response).when(this.restHighLevelClient).updateByQuery(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class));
 			when(response.getBulkFailures()).thenReturn(new ArrayList<Failure>());
-			when(this.indicesClient.putMapping(ArgumentMatchers.any(), ArgumentMatchers.any(RequestOptions.class))).thenThrow(new ElasticsearchException(""));
+			when(this.indicesClient.putMapping(ArgumentMatchers.any(PutMappingRequest.class), ArgumentMatchers.any(RequestOptions.class))).thenThrow(new ElasticsearchException(""));
 			when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
 			this.sut.updateIndexMappingForIndicesOfSameType(indices,"any field");
 		} catch (AppException e) {
@@ -303,3 +301,4 @@ public class IndexerMappingServiceTest {
 		}
 	}
 }
+

testing/indexer-test-aws/pom.xml

@@ -73,6 +73,22 @@
         </dependency>
 
         <!-- third party Apache 2.0 license packages -->
+        <!--Elasticsearch-->
+        <dependency>
+            <groupId>org.elasticsearch</groupId>
+            <artifactId>elasticsearch</artifactId>
+            <version>7.8.1</version>
+        </dependency>
+        <dependency>
+            <groupId>org.elasticsearch.client</groupId>
+            <artifactId>elasticsearch-rest-client</artifactId>
+            <version>7.8.1</version>
+        </dependency>
+        <dependency>
+            <groupId>org.elasticsearch.client</groupId>
+            <artifactId>elasticsearch-rest-high-level-client</artifactId>
+            <version>7.8.1</version>
+        </dependency>
         <dependency>
             <groupId>com.google.code.gson</groupId>
             <artifactId>gson</artifactId>

testing/indexer-test-ibm/pom.xml

@@ -114,17 +114,17 @@
         <dependency>
             <groupId>org.elasticsearch</groupId>
             <artifactId>elasticsearch</artifactId>
-            <version>6.6.2</version>
+            <version>7.8.1</version>
         </dependency>
         <dependency>
             <groupId>org.elasticsearch.client</groupId>
             <artifactId>elasticsearch-rest-client</artifactId>
-            <version>6.6.2</version>
+            <version>7.8.1</version>
         </dependency>
         <dependency>
             <groupId>org.elasticsearch.client</groupId>
             <artifactId>elasticsearch-rest-high-level-client</artifactId>
-            <version>6.6.2</version>
+            <version>7.8.1</version>
         </dependency>
 
         <!--Logging-->