diff --git a/NOTICE b/NOTICE index a9257a173b8c7e34a8734ecb7d43f08e6b7e4e01..6b22e74c2e0fad3759f5f580c6e8c2c67c6b8cb6 100644 --- a/NOTICE +++ b/NOTICE @@ -180,6 +180,7 @@ The following software have components provided under the terms of this license: - Apache Commons BeanUtils (from http://commons.apache.org/proper/commons-beanutils/) - Apache Commons CLI (from http://commons.apache.org/proper/commons-cli/) - Apache Commons Codec (from http://commons.apache.org/proper/commons-codec/) +- Apache Commons Collections (from http://commons.apache.org/proper/commons-collections/) - Apache Commons Lang (from http://commons.apache.org/proper/commons-lang/) - Apache Commons Logging (from http://commons.apache.org/proper/commons-logging/) - Apache Commons Validator (from http://commons.apache.org/proper/commons-validator/) @@ -311,6 +312,7 @@ The following software have components provided under the terms of this license: - Java Native Access (from https://github.com/java-native-access/jna) - Java Native Access Platform (from https://github.com/java-native-access/jna) - Java Servlet API (from http://servlet-spec.java.net) +- Java UUID Generator (from http://wiki.fasterxml.com/JugHome) - Javassist (from http://www.javassist.org/) - Javassist (from http://www.javassist.org/) - Jetty Server (from ) @@ -376,12 +378,13 @@ The following software have components provided under the terms of this license: - Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java) +- Microsoft Azure Java Core Library (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure Netty HTTP Client Library (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java) - Mockito (from http://mockito.org) - Mockito (from http://mockito.org) -- Mockito (from http://www.mockito.org) - Mockito (from http://mockito.org) +- Mockito (from http://www.mockito.org) - Mojo's Maven plugin for Cobertura (from http://mojo.codehaus.org/cobertura-maven-plugin/) - MongoDB Driver (from http://www.mongodb.org) - MongoDB Java Driver Core (from http://www.mongodb.org) @@ -858,6 +861,13 @@ The following software have components provided under the terms of this license: - Project Lombok (from https://projectlombok.org) - javax.ws.rs-api (from http://jax-rs-spec.java.net) +======================================================================== +JSON +======================================================================== +The following software have components provided under the terms of this license: + +- JSON in Java (from https://github.com/douglascrockford/JSON-java) + ======================================================================== LGPL-2.1-only ======================================================================== @@ -909,6 +919,7 @@ The following software have components provided under the terms of this license: - Animal Sniffer Annotations (from ) - Azure AD Spring Security Integration Spring Boot Starter (from https://github.com/Microsoft/azure-spring-boot) - Azure Java Client Authentication Library for AutoRest (from https://github.com/Azure/autorest-clientruntime-for-java) +- Azure Java Client Runtime for ARM (from https://github.com/Azure/autorest-clientruntime-for-java) - Azure Java Client Runtime for AutoRest (from https://github.com/Azure/autorest-clientruntime-for-java) - Azure Metrics Spring Boot Starter (from https://github.com/Microsoft/azure-spring-boot) - Azure Spring Boot AutoConfigure (from https://github.com/Microsoft/azure-spring-boot) @@ -932,10 +943,13 @@ The following software have components provided under the terms of this license: - Microsoft Azure Java Core Library (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure Netty HTTP Client Library (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure SDK annotations (from https://github.com/Microsoft/java-api-annotations) +- Microsoft Azure SDK for EventGrid Management (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure SDK for Service Bus (from https://github.com/Azure/azure-sdk-for-java) +- Microsoft Azure SDK for eventgrid (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure client library for Blob Storage (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure client library for Identity (from https://github.com/Azure/azure-sdk-for-java) +- Microsoft Azure client library for KeyVault Keys (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure client library for KeyVault Secrets (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure common module for Storage (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure internal Avro module for Storage (from https://github.com/Azure/azure-sdk-for-java) @@ -952,6 +966,8 @@ The following software have components provided under the terms of this license: - Spongy Castle (from http://rtyley.github.io/spongycastle/) - Spring Data for Azure Cosmos DB SQL API (from https://github.com/Azure/azure-sdk-for-java/tree/master/sdk/cosmos/azure-spring-data-cosmos) - adal4j (from https://github.com/AzureAD/azure-activedirectory-library-for-java) +- azure-documentdb (from https://azure.microsoft.com/en-us/services/cosmos-db/) +- documentdb-bulkexecutor (from http://azure.microsoft.com/en-us/services/documentdb/) - micrometer-core (from https://github.com/micrometer-metrics/micrometer) - msal4j (from https://github.com/AzureAD/microsoft-authentication-library-for-java) - msal4j-persistence-extension (from https://github.com/AzureAD/microsoft-authentication-extensions-for-java) @@ -1048,6 +1064,7 @@ The following software have components provided under the terms of this license: - Joda-Time (from http://www.joda.org/joda-time/) - LatencyUtils (from http://latencyutils.github.io/LatencyUtils/) - Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java) +- Microsoft Azure SDK for EventGrid Management (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure client library for Blob Storage (from https://github.com/Azure/azure-sdk-for-java) - Project Lombok (from https://projectlombok.org) @@ -1056,6 +1073,7 @@ The following software have components provided under the terms of this license: - Spring Security JWT Library (from http://github.com/spring-projects/spring-security-oauth) - Spring Web (from https://github.com/spring-projects/spring-framework) - StAX API (from http://stax.codehaus.org/) +- azure-documentdb (from https://azure.microsoft.com/en-us/services/cosmos-db/) - msal4j (from https://github.com/AzureAD/microsoft-authentication-library-for-java) - reactive-streams (from http://www.reactive-streams.org/) - xml-apis (from ) diff --git a/devops/azure/chart/templates/deployment.yaml b/devops/azure/chart/templates/deployment.yaml index 6ef26c036f8cb921e6ad534eef37cbc74127934a..20fac08cff7a776c18f4daacf8b23b4815b3ef90 100644 --- a/devops/azure/chart/templates/deployment.yaml +++ b/devops/azure/chart/templates/deployment.yaml @@ -87,6 +87,8 @@ spec: value: osdu-db - name: servicebus_topic_name value: indexing-progress + - name: reindex_topic_name + value: recordstopic - name: entitlements_service_endpoint value: http://entitlements/api/entitlements/v2 - name: entitlements_service_api_key diff --git a/provider/indexer-azure/pom.xml b/provider/indexer-azure/pom.xml index 2a3ed197aed841909573439960267407d54d92ce..88c81b927e5cce9085b9dd747731fc3d93630845 100644 --- a/provider/indexer-azure/pom.xml +++ b/provider/indexer-azure/pom.xml @@ -41,7 +41,7 @@ <nimbus-jose-jwt.version>8.2</nimbus-jose-jwt.version> <indexer-core.version>0.8.0-SNAPSHOT</indexer-core.version> <spring-security-jwt.version>1.1.1.RELEASE</spring-security-jwt.version> - <osdu.corelibazure.version>0.0.42</osdu.corelibazure.version> + <osdu.corelibazure.version>0.6.2</osdu.corelibazure.version> <reactor-netty.version>0.9.12.RELEASE</reactor-netty.version> <java-jwt.version>3.8.1</java-jwt.version> <powermock.version>2.0.2</powermock.version> diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/IndexerAzureApplication.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/IndexerAzureApplication.java index 51886b1357c66e14699d82955c757203da00eba0..f0ed79107e68a6d71bb98296734c237de5a62c4a 100644 --- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/IndexerAzureApplication.java +++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/IndexerAzureApplication.java @@ -18,12 +18,14 @@ import org.opengroup.osdu.indexer.IndexerApplication; import org.opengroup.osdu.indexer.ServerletInitializer; import org.opengroup.osdu.indexer.service.ElasticSettingServiceImpl; import org.springframework.boot.SpringApplication; +import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.FilterType; -@SpringBootApplication +@SpringBootApplication(exclude = { SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class }) @ComponentScan( basePackages = {"org.opengroup.osdu"}, excludeFilters = { diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/config/AzureBootstrapConfig.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/config/AzureBootstrapConfig.java index 7be75c3be0c99a9f076f36f422b303e948aa035a..50184634c1f45e4990658b1a8ad0499e43f1ef6e 100644 --- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/config/AzureBootstrapConfig.java +++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/config/AzureBootstrapConfig.java @@ -36,18 +36,15 @@ public class AzureBootstrapConfig { @Value("${azure.servicebus.topic-name}") private String serviceBusTopicName; + @Value("${azure.servicebus.reindex.topic-name}") + private String serviceBusReindexTopicName; + @Value("${ELASTIC_CACHE_EXPIRATION}") private Integer elasticCacheExpiration; @Value("${MAX_CACHE_VALUE_SIZE}") private Integer maxCacheValueSize; - @Value("${AUTHORIZE_API_KEY}") - private String entitlementsAPIKey; - - @Value("${AUTHORIZE_API}") - private String entitlementsAPIEndpoint; - @Bean @Named("KEY_VAULT_URL") public String getKeyVaultURL() { @@ -60,6 +57,12 @@ public class AzureBootstrapConfig { return serviceBusTopicName; } + @Bean + @Named("SERVICE_BUS_REINDEX_TOPIC") + public String serviceBusReindexTopicName() { + return serviceBusReindexTopicName; + } + @Bean @Named("ELASTIC_CACHE_EXPIRATION") public Integer getElasticCacheExpiration() { @@ -92,15 +95,4 @@ public class AzureBootstrapConfig { return String.format(urlFormat, tenant); } - @Autowired - private HttpResponseBodyMapper httpResponseBodyMapper; - - @Bean - public IEntitlementsFactory entitlementsFactory() { - EntitlementsAPIConfig apiConfig = EntitlementsAPIConfig.builder() - .apiKey(entitlementsAPIKey) - .rootUrl(entitlementsAPIEndpoint) - .build(); - return new EntitlementsFactory(apiConfig, httpResponseBodyMapper); - } } diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/di/EntitlementsClientFactory.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/di/EntitlementsClientFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..e83c404f2e9a3327e232281cc3800c71e45216cb --- /dev/null +++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/di/EntitlementsClientFactory.java @@ -0,0 +1,57 @@ +// Copyright 2017-2019, Schlumberger +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.opengroup.osdu.indexer.azure.di; + +import org.opengroup.osdu.core.common.entitlements.EntitlementsAPIConfig; +import org.opengroup.osdu.core.common.entitlements.EntitlementsFactory; +import org.opengroup.osdu.core.common.entitlements.IEntitlementsFactory; +import org.opengroup.osdu.core.common.http.json.HttpResponseBodyMapper; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.beans.factory.config.AbstractFactoryBean; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; +import org.springframework.web.context.annotation.RequestScope; + +@Component +@RequestScope +@Lazy +public class EntitlementsClientFactory extends AbstractFactoryBean<IEntitlementsFactory> { + + @Value("${AUTHORIZE_API}") + private String authorizeApi; + + @Value("${AUTHORIZE_API_KEY:}") + private String authorizeApiKey; + + @Autowired + private HttpResponseBodyMapper mapper; + + @Override + protected IEntitlementsFactory createInstance() throws Exception { + + return new EntitlementsFactory(EntitlementsAPIConfig + .builder() + .rootUrl(authorizeApi) + .apiKey(authorizeApiKey) + .build(), + mapper); + } + + @Override + public Class<?> getObjectType() { + return IEntitlementsFactory.class; + } +} diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/DpsHeadersAzureQueue.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/DpsHeadersAzureQueue.java index 2335abb334b8a1960f1028bea2d250285dde97d0..241451e83016a059d60aea5a96fadaa00ae0315b 100644 --- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/DpsHeadersAzureQueue.java +++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/DpsHeadersAzureQueue.java @@ -33,9 +33,6 @@ import org.springframework.web.context.annotation.RequestScope; @Primary public class DpsHeadersAzureQueue extends DpsHeaders { - @Value("${indexer.queue.key}") - private String queueKey; - //ToDo this should be moved to Azure client-lib public static final String INDEXER_QUEUE_KEY = "x-functions-key"; @@ -47,7 +44,8 @@ public class DpsHeadersAzureQueue extends DpsHeaders { .stream() .collect(Collectors.toMap(h -> h, request::getHeader)); - headers.put(INDEXER_QUEUE_KEY,queueKey); + // TODO: Figure out if this header is actually required to be set for anything + headers.put(INDEXER_QUEUE_KEY, "NOT_USED"); this.addFromMap(headers); diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java index 7210b4aaa06bb72b607ae31353835ca4353a19d1..4428beb160c4d352ebc24b5cf6fff74d9db5d799 100644 --- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java +++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java @@ -42,7 +42,6 @@ import javax.inject.Named; import java.lang.reflect.Type; import java.nio.charset.StandardCharsets; import java.util.stream.Collectors; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -64,8 +63,8 @@ public class IndexerQueueTaskBuilderAzure extends IndexerQueueTaskBuilder { private JaxRsDpsLog logger; @Inject - @Named("SERVICE_BUS_TOPIC") - private String serviceBusTopic; + @Named("SERVICE_BUS_REINDEX_TOPIC") + private String serviceBusReindexTopicName; @Inject private StorageService storageService; @@ -153,7 +152,7 @@ public class IndexerQueueTaskBuilderAzure extends IndexerQueueTaskBuilder { try { logger.info("Indexer publishes message to Service Bus " + headers.getCorrelationId()); - topicClientFactory.getClient(headers.getPartitionId(), serviceBusTopic).send(message); + topicClientFactory.getClient(headers.getPartitionId(), serviceBusReindexTopicName).send(message); } catch (Exception e) { logger.error(e.getMessage(), e); } diff --git a/provider/indexer-azure/src/main/resources/application.properties b/provider/indexer-azure/src/main/resources/application.properties index 619120f5eeab512b3e0f3316e27156e70b6b9aef..897271df395ef070a6f4971f6cfa824d36436e28 100644 --- a/provider/indexer-azure/src/main/resources/application.properties +++ b/provider/indexer-azure/src/main/resources/application.properties @@ -63,6 +63,7 @@ azure.cosmosdb.database=${cosmosdb_database} #AzureServiceBusconfiguration azure.servicebus.topic-name=${servicebus_topic_name} +azure.servicebus.reindex.topic-name=${reindex_topic_name} #Indexer-Queue-header indexer.queue.key=abcd