Merge branch 'azure-debug-reindex' into 'master'

Azure: Fixing reindex

See merge request !126

NOTICE

@@ -180,6 +180,7 @@ The following software have components provided under the terms of this license:
 - Apache Commons BeanUtils (from http://commons.apache.org/proper/commons-beanutils/)
 - Apache Commons CLI (from http://commons.apache.org/proper/commons-cli/)
 - Apache Commons Codec (from http://commons.apache.org/proper/commons-codec/)
+- Apache Commons Collections (from http://commons.apache.org/proper/commons-collections/)
 - Apache Commons Lang (from http://commons.apache.org/proper/commons-lang/)
 - Apache Commons Logging (from http://commons.apache.org/proper/commons-logging/)
 - Apache Commons Validator (from http://commons.apache.org/proper/commons-validator/)
@@ -311,6 +312,7 @@ The following software have components provided under the terms of this license:
 - Java Native Access (from https://github.com/java-native-access/jna)
 - Java Native Access Platform (from https://github.com/java-native-access/jna)
 - Java Servlet API (from http://servlet-spec.java.net)
+- Java UUID Generator (from http://wiki.fasterxml.com/JugHome)
 - Javassist (from http://www.javassist.org/)
 - Javassist (from http://www.javassist.org/)
 - Jetty Server (from )
@@ -376,12 +378,13 @@ The following software have components provided under the terms of this license:
 - Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java)
 - Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java)
 - Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
+- Microsoft Azure Java Core Library (from https://github.com/Azure/azure-sdk-for-java)
 - Microsoft Azure Netty HTTP Client Library (from https://github.com/Azure/azure-sdk-for-java)
 - Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java)
 - Mockito (from http://mockito.org)
 - Mockito (from http://mockito.org)
-- Mockito (from http://www.mockito.org)
 - Mockito (from http://mockito.org)
+- Mockito (from http://www.mockito.org)
 - Mojo's Maven plugin for Cobertura (from http://mojo.codehaus.org/cobertura-maven-plugin/)
 - MongoDB Driver (from http://www.mongodb.org)
 - MongoDB Java Driver Core (from http://www.mongodb.org)
@@ -858,6 +861,13 @@ The following software have components provided under the terms of this license:
 - Project Lombok (from https://projectlombok.org)
 - javax.ws.rs-api (from http://jax-rs-spec.java.net)
 
+========================================================================
+JSON
+========================================================================
+The following software have components provided under the terms of this license:
+
+- JSON in Java (from https://github.com/douglascrockford/JSON-java)
+
 ========================================================================
 LGPL-2.1-only
 ========================================================================
@@ -909,6 +919,7 @@ The following software have components provided under the terms of this license:
 - Animal Sniffer Annotations (from )
 - Azure AD Spring Security Integration Spring Boot Starter (from https://github.com/Microsoft/azure-spring-boot)
 - Azure Java Client Authentication Library for AutoRest (from https://github.com/Azure/autorest-clientruntime-for-java)
+- Azure Java Client Runtime for ARM (from https://github.com/Azure/autorest-clientruntime-for-java)
 - Azure Java Client Runtime for AutoRest (from https://github.com/Azure/autorest-clientruntime-for-java)
 - Azure Metrics Spring Boot Starter (from https://github.com/Microsoft/azure-spring-boot)
 - Azure Spring Boot AutoConfigure (from https://github.com/Microsoft/azure-spring-boot)
@@ -932,10 +943,13 @@ The following software have components provided under the terms of this license:
 - Microsoft Azure Java Core Library (from https://github.com/Azure/azure-sdk-for-java)
 - Microsoft Azure Netty HTTP Client Library (from https://github.com/Azure/azure-sdk-for-java)
 - Microsoft Azure SDK annotations (from https://github.com/Microsoft/java-api-annotations)
+- Microsoft Azure SDK for EventGrid Management (from https://github.com/Azure/azure-sdk-for-java)
 - Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java)
 - Microsoft Azure SDK for Service Bus (from https://github.com/Azure/azure-sdk-for-java)
+- Microsoft Azure SDK for eventgrid (from https://github.com/Azure/azure-sdk-for-java)
 - Microsoft Azure client library for Blob Storage (from https://github.com/Azure/azure-sdk-for-java)
 - Microsoft Azure client library for Identity (from https://github.com/Azure/azure-sdk-for-java)
+- Microsoft Azure client library for KeyVault Keys (from https://github.com/Azure/azure-sdk-for-java)
 - Microsoft Azure client library for KeyVault Secrets (from https://github.com/Azure/azure-sdk-for-java)
 - Microsoft Azure common module for Storage (from https://github.com/Azure/azure-sdk-for-java)
 - Microsoft Azure internal Avro module for Storage (from https://github.com/Azure/azure-sdk-for-java)
@@ -952,6 +966,8 @@ The following software have components provided under the terms of this license:
 - Spongy Castle (from http://rtyley.github.io/spongycastle/)
 - Spring Data for Azure Cosmos DB SQL API (from https://github.com/Azure/azure-sdk-for-java/tree/master/sdk/cosmos/azure-spring-data-cosmos)
 - adal4j (from https://github.com/AzureAD/azure-activedirectory-library-for-java)
+- azure-documentdb (from https://azure.microsoft.com/en-us/services/cosmos-db/)
+- documentdb-bulkexecutor (from http://azure.microsoft.com/en-us/services/documentdb/)
 - micrometer-core (from https://github.com/micrometer-metrics/micrometer)
 - msal4j (from https://github.com/AzureAD/microsoft-authentication-library-for-java)
 - msal4j-persistence-extension (from https://github.com/AzureAD/microsoft-authentication-extensions-for-java)
@@ -1048,6 +1064,7 @@ The following software have components provided under the terms of this license:
 - Joda-Time (from http://www.joda.org/joda-time/)
 - LatencyUtils (from http://latencyutils.github.io/LatencyUtils/)
 - Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java)
+- Microsoft Azure SDK for EventGrid Management (from https://github.com/Azure/azure-sdk-for-java)
 - Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java)
 - Microsoft Azure client library for Blob Storage (from https://github.com/Azure/azure-sdk-for-java)
 - Project Lombok (from https://projectlombok.org)
@@ -1056,6 +1073,7 @@ The following software have components provided under the terms of this license:
 - Spring Security JWT Library (from http://github.com/spring-projects/spring-security-oauth)
 - Spring Web (from https://github.com/spring-projects/spring-framework)
 - StAX API (from http://stax.codehaus.org/)
+- azure-documentdb (from https://azure.microsoft.com/en-us/services/cosmos-db/)
 - msal4j (from https://github.com/AzureAD/microsoft-authentication-library-for-java)
 - reactive-streams (from http://www.reactive-streams.org/)
 - xml-apis (from )

devops/azure/chart/templates/deployment.yaml

@@ -87,6 +87,8 @@ spec:
           value: osdu-db
         - name: servicebus_topic_name
           value: indexing-progress
+        - name: reindex_topic_name
+          value: recordstopic
         - name: entitlements_service_endpoint
           value: http://entitlements/api/entitlements/v2
         - name: entitlements_service_api_key

provider/indexer-azure/pom.xml

@@ -41,7 +41,7 @@
         <nimbus-jose-jwt.version>8.2</nimbus-jose-jwt.version>
         <indexer-core.version>0.8.0-SNAPSHOT</indexer-core.version>
         <spring-security-jwt.version>1.1.1.RELEASE</spring-security-jwt.version>
-        <osdu.corelibazure.version>0.0.42</osdu.corelibazure.version>
+        <osdu.corelibazure.version>0.6.2</osdu.corelibazure.version>
         <reactor-netty.version>0.9.12.RELEASE</reactor-netty.version>
         <java-jwt.version>3.8.1</java-jwt.version>
         <powermock.version>2.0.2</powermock.version>

provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/IndexerAzureApplication.java

@@ -18,12 +18,14 @@ import org.opengroup.osdu.indexer.IndexerApplication;
 import org.opengroup.osdu.indexer.ServerletInitializer;
 import org.opengroup.osdu.indexer.service.ElasticSettingServiceImpl;
 import org.springframework.boot.SpringApplication;
+import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
 import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.FilterType;
 
 
-@SpringBootApplication
+@SpringBootApplication(exclude = { SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class })
 @ComponentScan(
         basePackages = {"org.opengroup.osdu"},
         excludeFilters = {

provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/config/AzureBootstrapConfig.java

@@ -36,18 +36,15 @@ public class AzureBootstrapConfig {
     @Value("${azure.servicebus.topic-name}")
     private String serviceBusTopicName;
 
+    @Value("${azure.servicebus.reindex.topic-name}")
+    private String serviceBusReindexTopicName;
+
     @Value("${ELASTIC_CACHE_EXPIRATION}")
     private Integer elasticCacheExpiration;
 
     @Value("${MAX_CACHE_VALUE_SIZE}")
     private Integer maxCacheValueSize;
 
-    @Value("${AUTHORIZE_API_KEY}")
-    private String entitlementsAPIKey;
-
-    @Value("${AUTHORIZE_API}")
-    private String entitlementsAPIEndpoint;
-
     @Bean
     @Named("KEY_VAULT_URL")
     public String getKeyVaultURL() {
@@ -60,6 +57,12 @@ public class AzureBootstrapConfig {
         return serviceBusTopicName;
     }
 
+    @Bean
+    @Named("SERVICE_BUS_REINDEX_TOPIC")
+    public String serviceBusReindexTopicName() {
+        return serviceBusReindexTopicName;
+    }
+
     @Bean
     @Named("ELASTIC_CACHE_EXPIRATION")
     public Integer getElasticCacheExpiration() {
@@ -92,15 +95,4 @@ public class AzureBootstrapConfig {
         return String.format(urlFormat, tenant);
     }
 
-    @Autowired
-    private HttpResponseBodyMapper httpResponseBodyMapper;
-
-    @Bean
-    public IEntitlementsFactory entitlementsFactory() {
-        EntitlementsAPIConfig apiConfig = EntitlementsAPIConfig.builder()
-                .apiKey(entitlementsAPIKey)
-                .rootUrl(entitlementsAPIEndpoint)
-                .build();
-        return new EntitlementsFactory(apiConfig, httpResponseBodyMapper);
-    }
 }

provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/di/EntitlementsClientFactory.java

+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.azure.di;
+
+import org.opengroup.osdu.core.common.entitlements.EntitlementsAPIConfig;
+import org.opengroup.osdu.core.common.entitlements.EntitlementsFactory;
+import org.opengroup.osdu.core.common.entitlements.IEntitlementsFactory;
+import org.opengroup.osdu.core.common.http.json.HttpResponseBodyMapper;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.beans.factory.config.AbstractFactoryBean;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+@Component
+@RequestScope
+@Lazy
+public class EntitlementsClientFactory extends AbstractFactoryBean<IEntitlementsFactory> {
+
+    @Value("${AUTHORIZE_API}")
+    private String authorizeApi;
+
+    @Value("${AUTHORIZE_API_KEY:}")
+    private String authorizeApiKey;
+
+    @Autowired
+    private HttpResponseBodyMapper mapper;
+
+    @Override
+    protected IEntitlementsFactory createInstance() throws Exception {
+
+        return new EntitlementsFactory(EntitlementsAPIConfig
+                .builder()
+                .rootUrl(authorizeApi)
+                .apiKey(authorizeApiKey)
+                .build(),
+                mapper);
+    }
+
+    @Override
+    public Class<?> getObjectType() {
+        return IEntitlementsFactory.class;
+    }
+}

provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/DpsHeadersAzureQueue.java

@@ -33,9 +33,6 @@ import org.springframework.web.context.annotation.RequestScope;
 @Primary
 public class DpsHeadersAzureQueue extends DpsHeaders {
 
-    @Value("${indexer.queue.key}")
-    private String queueKey;
-
     //ToDo this should be moved to Azure client-lib
     public static final String INDEXER_QUEUE_KEY = "x-functions-key";
 
@@ -47,7 +44,8 @@ public class DpsHeadersAzureQueue extends DpsHeaders {
                 .stream()
                 .collect(Collectors.toMap(h -> h, request::getHeader));
 
-        headers.put(INDEXER_QUEUE_KEY,queueKey);
+        // TODO: Figure out if this header is actually required to be set for anything
+        headers.put(INDEXER_QUEUE_KEY, "NOT_USED");
 
         this.addFromMap(headers);
 

provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java

@@ -42,7 +42,6 @@ import javax.inject.Named;
 import java.lang.reflect.Type;
 import java.nio.charset.StandardCharsets;
 import java.util.stream.Collectors;
-import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -64,8 +63,8 @@ public class IndexerQueueTaskBuilderAzure extends IndexerQueueTaskBuilder {
     private JaxRsDpsLog logger;
 
     @Inject
-    @Named("SERVICE_BUS_TOPIC")
-    private String serviceBusTopic;
+    @Named("SERVICE_BUS_REINDEX_TOPIC")
+    private String serviceBusReindexTopicName;
 
     @Inject
     private StorageService storageService;
@@ -153,7 +152,7 @@ public class IndexerQueueTaskBuilderAzure extends IndexerQueueTaskBuilder {
 
         try {
             logger.info("Indexer publishes message to Service Bus " + headers.getCorrelationId());
-            topicClientFactory.getClient(headers.getPartitionId(), serviceBusTopic).send(message);
+            topicClientFactory.getClient(headers.getPartitionId(), serviceBusReindexTopicName).send(message);
         } catch (Exception e) {
             logger.error(e.getMessage(), e);
         }

provider/indexer-azure/src/main/resources/application.properties

@@ -63,6 +63,7 @@ azure.cosmosdb.database=${cosmosdb_database}
 
 #AzureServiceBusconfiguration
 azure.servicebus.topic-name=${servicebus_topic_name}
+azure.servicebus.reindex.topic-name=${reindex_topic_name}
 
 #Indexer-Queue-header
 indexer.queue.key=abcd