From e914de37be11b5a26b71762870345a1189888384 Mon Sep 17 00:00:00 2001
From: "Nicholas.Iodice" <Nicholas.Iodice@microsoft.com>
Date: Fri, 21 Feb 2020 04:40:02 +0000
Subject: [PATCH] Merged PR 992: Updating from master; fixing any azure
problems
---
devops/service-pipeline.yml | 33 ++++
.../src/main/resources/application.properties | 26 ---
provider/indexer-azure/README.md | 17 ++
provider/indexer-azure/azure-build.yml | 16 +-
provider/indexer-azure/maven/settings.xml | 7 +
provider/indexer-azure/pom.xml | 16 +-
.../azure/IndexerAzureApplication.java | 1 -
.../osdu/indexer/azure/api/AADController.java | 42 -----
.../azure/cache/ElasticCredentialsCache.java | 39 -----
.../{model => config}/AADConfiguration.java | 2 +-
.../azure/config/AzureBootstrapConfig.java | 57 ++++++
.../indexer/azure/di/TenantInfoService.java | 25 +++
.../ElasticRepositoryCosmosDB.java | 69 --------
.../ElasticSettingSchemaRepositoryImpl.java | 50 ------
.../azure/persistence/ElasticSettingsDoc.java | 37 ----
.../azure/persistence/ISchemaRepository.java | 30 ----
...rityConfig.java => AADSecurityConfig.java} | 4 +-
.../util/ServiceAccountJwtClientImpl.java | 18 +-
.../src/main/resources/application.properties | 84 +++------
.../src/main/resources/logback-spring.xml | 42 -----
.../service/ElasticSettingServiceTest.java | 112 ------------
testing/indexer-test-azure/pom.xml | 165 ++++++++++++++++++
.../index/record/RunTest.java | 39 ++---
.../step_definitions/index/record/Steps.java | 67 +++++++
.../opengroup/osdu/util/AzureHTTPClient.java | 29 ++-
.../osdu/util/AzureServicePrincipal.java | 83 +++++++++
.../org/opengroup/osdu/util/JwtTokenUtil.java | 30 ++++
.../org.mockito.plugins.MockMaker | 1 +
28 files changed, 569 insertions(+), 572 deletions(-)
create mode 100644 devops/service-pipeline.yml
delete mode 100644 indexer-core/src/main/resources/application.properties
create mode 100644 provider/indexer-azure/README.md
delete mode 100644 provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/api/AADController.java
delete mode 100644 provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/ElasticCredentialsCache.java
rename provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/{model => config}/AADConfiguration.java (95%)
create mode 100644 provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/config/AzureBootstrapConfig.java
create mode 100644 provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/di/TenantInfoService.java
delete mode 100644 provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/persistence/ElasticRepositoryCosmosDB.java
delete mode 100644 provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/persistence/ElasticSettingSchemaRepositoryImpl.java
delete mode 100644 provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/persistence/ElasticSettingsDoc.java
delete mode 100644 provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/persistence/ISchemaRepository.java
rename provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/security/{BasicAuthSecurityConfig.java => AADSecurityConfig.java} (94%)
delete mode 100644 provider/indexer-azure/src/main/resources/logback-spring.xml
delete mode 100644 provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/ElasticSettingServiceTest.java
create mode 100644 testing/indexer-test-azure/pom.xml
rename provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/model/ElasticSettingSchema.java => testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java (56%)
create mode 100644 testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
rename provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/ServletInitializer.java => testing/indexer-test-azure/src/test/java/org/opengroup/osdu/util/AzureHTTPClient.java (55%)
create mode 100644 testing/indexer-test-azure/src/test/java/org/opengroup/osdu/util/AzureServicePrincipal.java
create mode 100644 testing/indexer-test-azure/src/test/java/org/opengroup/osdu/util/JwtTokenUtil.java
create mode 100644 testing/indexer-test-azure/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker
diff --git a/devops/service-pipeline.yml b/devops/service-pipeline.yml
new file mode 100644
index 000000000..32636f7e9
--- /dev/null
+++ b/devops/service-pipeline.yml
@@ -0,0 +1,33 @@
+trigger:
+ batch: true
+ branches:
+ include:
+ - master
+ paths:
+ exclude:
+ - /**/*.md
+ - .gitignore
+ - images/
+
+pr:
+ branches:
+ include:
+ - '*'
+ paths:
+ exclude:
+ - /**/*.md
+ - .gitignore
+ - images/
+
+resources:
+ repositories:
+ - repository: infrastructure-templates
+ type: git
+ name: open-data-ecosystem/infrastructure-templates
+
+variables:
+- group: 'Azure Common Secrets'
+- group: 'Azure - Common'
+
+stages:
+- template: devops/service-pipelines/service-pipeline-stages.yml@infrastructure-templates
diff --git a/indexer-core/src/main/resources/application.properties b/indexer-core/src/main/resources/application.properties
deleted file mode 100644
index 022eb7a15..000000000
--- a/indexer-core/src/main/resources/application.properties
+++ /dev/null
@@ -1,26 +0,0 @@
-##server.servlet.contextPath=/api/indexer/v2
-#logging.level.org.springframework.web=DEBUG
-#server.port=8080
-#JAVA_HEAP_OPTS=-Xms4096M -Xmx4096M
-#JAVA_GC_OPTS=-XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45
-#PUBSUB_SEARCH_TOPIC=records-changed
-#AUTHORIZE_API=https://entitlements-dot-opendes.appspot.com/entitlements/v1
-#STORAGE_HOSTNAME=os-storage-dot-opendes.appspot.com
-#GOOGLE_AUDIENCES=245464679631-ktfdfpl147m1mjpbutl00b3cmffissgq.apps.googleusercontent.com
-#
-## must update redis hosts at runtime appropriate to deployment environment
-#REDIS_GROUP_HOST=REDIS-GROUP-HOST
-#REDIS_SEARCH_HOST=REDIS-SEARCH-HOST
-#
-## use 127.0.0.1 for local run/debug mode
-##REDIS_GROUP_HOST=127.0.0.1
-##REDIS_STORAGE_HOST=127.0.0.1
-#
-## use below values for gcp: opendes
-##REDIS_GROUP_HOST=10.0.16.28
-##REDIS_STORAGE_HOST=10.0.16.12
-#
-#REDIS_GROUP_PORT=6379
-#REDIS_SEARCH_PORT=6379
-#DEFAULT_DATA_COUNTRY=US
-
diff --git a/provider/indexer-azure/README.md b/provider/indexer-azure/README.md
new file mode 100644
index 000000000..5386eaab5
--- /dev/null
+++ b/provider/indexer-azure/README.md
@@ -0,0 +1,17 @@
+# Indexer-azure
+
+This document is a WIP, but it is intended to describe the steps needed to run this service both locally and in Azure.
+
+## Environment Configuration
+
+TODO:
+
+## Environment Configuration - Local Execution
+
+Due to the usage of [Managed Identity](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/overview), you will need the following environment variables defined for running the azure implementation locally:
+
+| variable name | value |
+| --- | --- |
+| `AZURE_CLIENT_ID` | service principal's app id |
+| `AZURE_TENANT_ID` | id of the principal's Azure Active Directory tenant |
+| `AZURE_CLIENT_SECRET` | one of the service principal's client secrets |
diff --git a/provider/indexer-azure/azure-build.yml b/provider/indexer-azure/azure-build.yml
index 705d5f216..cd450529e 100644
--- a/provider/indexer-azure/azure-build.yml
+++ b/provider/indexer-azure/azure-build.yml
@@ -13,7 +13,7 @@ trigger:
- .gitignore
pool:
- name: Hosted Ubuntu 1604
+ name: dps-build
demands: maven
steps:
@@ -21,7 +21,7 @@ steps:
displayName: 'build, test, code coverage'
inputs:
mavenPomFile: 'pom.xml'
- options: '--settings ./indexer-core/maven/settings.xml -DVSTS_FEED_TOKEN=$(VSTS_FEED_TOKEN)'
+ options: '--settings ./indexer-core/maven/settings.xml -DVSTS_FEED_TOKEN=$(VSTS_FEED_TOKEN) -U'
testResultsFiles: '**/*/TEST-*.xml'
codeCoverageToolOption: JaCoCo
goals: 'install'
@@ -30,7 +30,7 @@ steps:
displayName: 'build, test, code coverage'
inputs:
mavenPomFile: 'pom.xml'
- options: '--settings ./provider/indexer-azure/maven/settings.xml -DVSTS_FEED_TOKEN=$(VSTS_FEED_TOKEN) -P indexer-azure'
+ options: '--settings ./provider/indexer-azure/maven/settings.xml -DVSTS_FEED_TOKEN=$(VSTS_FEED_TOKEN) -U -P indexer-azure'
testResultsFiles: '**/*/TEST-*.xml'
codeCoverageToolOption: JaCoCo
goals: 'package'
@@ -45,7 +45,15 @@ steps:
provider/indexer-azure/pom.xml
provider/indexer-azure/target/*-spring-boot.jar
TargetFolder: '$(build.artifactstagingdirectory)'
-
+
+- task: ArchiveFiles@2
+ inputs:
+ rootFolderOrFile: 'testing'
+ includeRootFolder: true
+ archiveType: 'zip'
+ archiveFile: '$(Build.ArtifactStagingDirectory)/indexer-integration-tests.zip'
+ replaceExistingArchive: true
+
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: drop'
inputs:
diff --git a/provider/indexer-azure/maven/settings.xml b/provider/indexer-azure/maven/settings.xml
index 855478978..d1af8b94a 100644
--- a/provider/indexer-azure/maven/settings.xml
+++ b/provider/indexer-azure/maven/settings.xml
@@ -19,6 +19,13 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
<servers>
+ <server>
+ <id>os-core</id>
+ <username>slb-des-ext-collaboration</username>
+ <!-- Treat this auth token like a password. Do not share it with anyone, including Microsoft support. -->
+ <!-- The generated token expires on or before 11/14/2019 -->
+ <password>${VSTS_FEED_TOKEN}</password>
+ </server>
<server>
<id>azure-auth</id>
<configuration>
diff --git a/provider/indexer-azure/pom.xml b/provider/indexer-azure/pom.xml
index dcf945891..e64f5cfe7 100644
--- a/provider/indexer-azure/pom.xml
+++ b/provider/indexer-azure/pom.xml
@@ -80,16 +80,6 @@
<artifactId>azure-cosmosdb-spring-boot-starter</artifactId>
<version>${azure.version}</version>
</dependency>
- <dependency>
- <groupId>com.microsoft.azure</groupId>
- <artifactId>azure-storage-spring-boot-starter</artifactId>
- <version>${azure.version}</version>
- </dependency>
- <dependency>
- <groupId>com.microsoft.azure</groupId>
- <artifactId>azure-servicebus-spring-boot-starter</artifactId>
- <version>${azure.version}</version>
- </dependency>
<dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>applicationinsights-logging-logback</artifactId>
@@ -106,6 +96,12 @@
<version>1.0.4-SNAPSHOT</version>
</dependency>
+ <dependency>
+ <groupId>org.opengroup.osdu</groupId>
+ <artifactId>core-lib-azure</artifactId>
+ <version>0.0.6-SNAPSHOT</version>
+ </dependency>
+
<dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>msal4j</artifactId>
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/IndexerAzureApplication.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/IndexerAzureApplication.java
index 8cef929b8..20646be69 100644
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/IndexerAzureApplication.java
+++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/IndexerAzureApplication.java
@@ -17,7 +17,6 @@ package org.opengroup.osdu.indexer.azure;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.ComponentScan;
-import org.springframework.context.annotation.Configuration;
@SpringBootApplication
@ComponentScan({"org.opengroup.osdu"})
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/api/AADController.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/api/AADController.java
deleted file mode 100644
index 22e44cef3..000000000
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/api/AADController.java
+++ /dev/null
@@ -1,42 +0,0 @@
-// Copyright © Microsoft Corporation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.azure.api;
-
-import org.opengroup.osdu.indexer.azure.util.ServiceAccountJwtClientImpl;
-
-import org.springframework.http.*;
-import org.springframework.stereotype.Controller;
-import org.springframework.web.bind.annotation.PostMapping;
-import org.springframework.web.bind.annotation.RequestMapping;
-
-import javax.inject.Inject;
-
-@Controller
-public class AADController {
-
- @Inject
- ServiceAccountJwtClientImpl service;
-
-
- @RequestMapping("/obo_api")
- @PostMapping
- public ResponseEntity<String> callOboApi() throws Throwable {
-
- String token = service.getIdToken("common");
-
- return new ResponseEntity<String>(token, HttpStatus.OK);
- }
-}
-
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/ElasticCredentialsCache.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/ElasticCredentialsCache.java
deleted file mode 100644
index ccca6407e..000000000
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/cache/ElasticCredentialsCache.java
+++ /dev/null
@@ -1,39 +0,0 @@
-package org.opengroup.osdu.indexer.azure.cache;
-
-import org.opengroup.osdu.core.common.cache.VmCache;
-import org.opengroup.osdu.core.common.model.search.ClusterSettings;
-import org.opengroup.osdu.core.common.provider.interfaces.IElasticCredentialsCache;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Component;
-
-@Component
-public class ElasticCredentialsCache implements IElasticCredentialsCache<String, ClusterSettings> {
-
- private VmCache<String, ClusterSettings> cache;
-
- public ElasticCredentialsCache(@Value("${ELASTIC_CACHE_EXPIRATION}") final String ELASTIC_CACHE_EXPIRATION,
- @Value("${MAX_CACHE_VALUE_SIZE}") final String MAX_CACHE_VALUE_SIZE) {
- cache = new VmCache<>(Integer.parseInt(ELASTIC_CACHE_EXPIRATION) * 60,
- Integer.parseInt(MAX_CACHE_VALUE_SIZE));
- }
-
- @Override
- public void put(String s, ClusterSettings o) {
- this.cache.put(s,o);
- }
-
- @Override
- public ClusterSettings get(String s) {
- return this.cache.get(s);
- }
-
- @Override
- public void delete(String s) {
- this.cache.delete(s);
- }
-
- @Override
- public void clearAll() {
- this.cache.clearAll();
- }
-}
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/model/AADConfiguration.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/config/AADConfiguration.java
similarity index 95%
rename from provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/model/AADConfiguration.java
rename to provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/config/AADConfiguration.java
index 3b311a933..28c206055 100644
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/model/AADConfiguration.java
+++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/config/AADConfiguration.java
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package org.opengroup.osdu.indexer.azure.model;
+package org.opengroup.osdu.indexer.azure.config;
import lombok.Getter;
import lombok.Setter;
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/config/AzureBootstrapConfig.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/config/AzureBootstrapConfig.java
new file mode 100644
index 000000000..100a6c964
--- /dev/null
+++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/config/AzureBootstrapConfig.java
@@ -0,0 +1,57 @@
+package org.opengroup.osdu.indexer.azure.config;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.stereotype.Component;
+
+import javax.inject.Named;
+
+@Component
+public class AzureBootstrapConfig {
+
+ @Value("${azure.keyvault.url}")
+ private String keyVaultURL;
+
+ @Value("${azure.servicebus.topic-name}")
+ private String serviceBusTopicName;
+
+ @Value("${azure.servicebus.namespace-name}")
+ private String serviceBusNamespaceName;
+
+ @Value("${ELASTIC_CACHE_EXPIRATION}")
+ private Integer elasticCacheExpiration;
+
+ @Value("${MAX_CACHE_VALUE_SIZE}")
+ private Integer maxCacheValueSize;
+
+ @Bean
+ @Named("KEY_VAULT_URL")
+ public String getKeyVaultURL(){
+ return keyVaultURL;
+ }
+
+ @Bean
+ @Named("SERVICE_BUS_NAMESPACE")
+ public String serviceBusNamespaceName() {
+ return serviceBusNamespaceName;
+ }
+
+ @Bean
+ @Named("SERVICE_BUS_TOPIC")
+ public String serviceBusTopicName() {
+ return serviceBusTopicName;
+ }
+
+ @Bean
+ @Named("ELASTIC_CACHE_EXPIRATION")
+ public Integer getElasticCacheExpiration() {
+ return elasticCacheExpiration;
+ }
+
+ @Bean
+ @Named("MAX_CACHE_VALUE_SIZE")
+ public Integer getMaxCacheValueSize() {
+ return maxCacheValueSize;
+ }
+
+}
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/di/TenantInfoService.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/di/TenantInfoService.java
new file mode 100644
index 000000000..0cf37ded1
--- /dev/null
+++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/di/TenantInfoService.java
@@ -0,0 +1,25 @@
+package org.opengroup.osdu.indexer.azure.di;
+
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
+import org.opengroup.osdu.core.common.multitenancy.ITenantInfoService;
+import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+import javax.inject.Inject;
+
+@RequestScope
+@Component
+public class TenantInfoService implements ITenantInfoService {
+ @Inject
+ private ITenantFactory tenantFactory;
+
+ @Inject
+ private DpsHeaders headers;
+
+ @Override
+ public TenantInfo getTenantInfo() {
+ return tenantFactory.getTenantInfo(headers.getPartitionId());
+ }
+}
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/persistence/ElasticRepositoryCosmosDB.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/persistence/ElasticRepositoryCosmosDB.java
deleted file mode 100644
index 8a7703e12..000000000
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/persistence/ElasticRepositoryCosmosDB.java
+++ /dev/null
@@ -1,69 +0,0 @@
-// Copyright © Microsoft Corporation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.azure.persistence;
-
-import org.apache.http.HttpStatus;
-
-import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
-import org.opengroup.osdu.indexer.azure.model.ElasticSettingSchema;
-import org.opengroup.osdu.core.common.model.search.ClusterSettings;
-import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.search.Preconditions;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Component;
-import javax.inject.Inject;
-
-@Component
-public class ElasticRepositoryCosmosDB implements IElasticRepository {
-
- @Inject
- private CosmosDBElasticSettings cosmosDB;
- @Value("${ELASTIC_DATASTORE_KIND}")
- private String ELASTIC_DATASTORE_KIND;
-
- @Value("${ELASTIC_DATASTORE_ID}")
- private String ELASTIC_DATASTORE_ID;
-
- @Inject
- private ISchemaRepository schemaRepository;
-
-
- @Override
- public ClusterSettings getElasticClusterSettings(TenantInfo tenantInfo) {
-
- if(tenantInfo == null)
- throw new AppException(HttpStatus.SC_NOT_FOUND, "TenantInfo is null", "");
-
- String settingId = tenantInfo.getName().concat("-").concat(ELASTIC_DATASTORE_ID);
- ElasticSettingSchema schema = this.schemaRepository.get(settingId);
-
- if (schema == null) {
- throw new AppException(HttpStatus.SC_NOT_FOUND, "Elastic setting not found", "The requested cluster setting was not found in CosmosDB.", String.format("Elastic setting with key: '%s' does not exist in CosmostDB.", ELASTIC_DATASTORE_KIND));
- }
-
- String host = schema.getHost();
- String portString = schema.getPort();
- String usernameAndPassword = schema.getUsernameAndPassword();
-
- Preconditions.checkNotNullOrEmpty(host, "host cannot be null");
- Preconditions.checkNotNullOrEmpty(portString, "port cannot be null");
- Preconditions.checkNotNullOrEmpty(usernameAndPassword, "configuration cannot be null");
-
- int port = Integer.parseInt(portString);
-
- return new ClusterSettings(host, port, usernameAndPassword, schema.isHttps(), schema.isHttps());
- }
-}
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/persistence/ElasticSettingSchemaRepositoryImpl.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/persistence/ElasticSettingSchemaRepositoryImpl.java
deleted file mode 100644
index c826cb800..000000000
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/persistence/ElasticSettingSchemaRepositoryImpl.java
+++ /dev/null
@@ -1,50 +0,0 @@
-// Copyright © Microsoft Corporation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.azure.persistence;
-
-import org.opengroup.osdu.indexer.azure.model.ElasticSettingSchema;
-import org.springframework.stereotype.Repository;
-
-import javax.inject.Inject;
-import java.util.Optional;
-
-@Repository
-public class ElasticSettingSchemaRepositoryImpl implements ISchemaRepository {
-
- @Inject
- private CosmosDBElasticSettings db;
-
- @Override
- public void add(ElasticSettingSchema schema, String id) {
- ElasticSettingsDoc sd = new ElasticSettingsDoc();
- sd.setId(id);
- sd.setSettingSchema(schema);
- db.save(sd);
- }
-
- @Override
- public ElasticSettingSchema get(String id) {
- Optional<ElasticSettingsDoc> sd = db.findById(id);
- if (!sd.isPresent())
- return null;
-
- ElasticSettingSchema newSchema = new ElasticSettingSchema();
- newSchema.setPort(sd.get().getSettingSchema().getPort());
- newSchema.setHost(sd.get().getSettingSchema().getHost());
- newSchema.setUsernameAndPassword(sd.get().getSettingSchema().getUsernameAndPassword());
- newSchema.setHttps(sd.get().getSettingSchema().isHttps());
- return newSchema;
- }
-}
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/persistence/ElasticSettingsDoc.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/persistence/ElasticSettingsDoc.java
deleted file mode 100644
index 991384dac..000000000
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/persistence/ElasticSettingsDoc.java
+++ /dev/null
@@ -1,37 +0,0 @@
-// Copyright © Microsoft Corporation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.azure.persistence;
-
-import com.microsoft.azure.spring.data.cosmosdb.core.mapping.Document;
-import com.microsoft.azure.spring.data.cosmosdb.core.mapping.PartitionKey;
-import com.microsoft.azure.spring.data.cosmosdb.repository.DocumentDbRepository;
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-import org.opengroup.osdu.indexer.azure.model.ElasticSettingSchema;
-import org.springframework.data.annotation.Id;
-
-@Data
-@AllArgsConstructor
-@NoArgsConstructor
-@Document(collection = "SearchSettings") //collection name
-public class ElasticSettingsDoc {
- @PartitionKey
- @Id
- private String id;
- private ElasticSettingSchema settingSchema;
-}
-
-interface CosmosDBElasticSettings extends DocumentDbRepository<ElasticSettingsDoc, String> {}
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/persistence/ISchemaRepository.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/persistence/ISchemaRepository.java
deleted file mode 100644
index 92f62ec35..000000000
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/persistence/ISchemaRepository.java
+++ /dev/null
@@ -1,30 +0,0 @@
-
-// Copyright © Microsoft Corporation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.azure.persistence;
-
-import org.opengroup.osdu.indexer.azure.model.ElasticSettingSchema;
-
-public interface ISchemaRepository {
- String SCHEMA_KIND = "IndexerSchema";
-
- String SCHEMA = "schema";
- String KIND = "KIND";
-
- void add(ElasticSettingSchema schema, String id);
-
- ElasticSettingSchema get(String id);
-}
-
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/security/BasicAuthSecurityConfig.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/security/AADSecurityConfig.java
similarity index 94%
rename from provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/security/BasicAuthSecurityConfig.java
rename to provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/security/AADSecurityConfig.java
index 6cc8d7cab..d5eaffce1 100644
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/security/BasicAuthSecurityConfig.java
+++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/security/AADSecurityConfig.java
@@ -26,7 +26,7 @@ import javax.inject.Inject;
@EnableWebSecurity
@EnableGlobalMethodSecurity(prePostEnabled = true)
-public class BasicAuthSecurityConfig extends WebSecurityConfigurerAdapter {
+public class AADSecurityConfig extends WebSecurityConfigurerAdapter {
@Inject
private AADAppRoleStatelessAuthenticationFilter appRoleAuthFilter;
@@ -35,7 +35,7 @@ public class BasicAuthSecurityConfig extends WebSecurityConfigurerAdapter {
http.csrf().disable().sessionManagement().sessionCreationPolicy(SessionCreationPolicy.NEVER)
.and()
.authorizeRequests()
- .antMatchers("/", "/index.html","/obo_api",
+ .antMatchers("/", "/index.html",
"/index-worker", "/_dps/task-handlers", "/_dps/task-handlers/**",
"/reindex",
"/v2/api-docs",
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/ServiceAccountJwtClientImpl.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/ServiceAccountJwtClientImpl.java
index 8e67bc9fd..bbc6a1179 100644
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/ServiceAccountJwtClientImpl.java
+++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/ServiceAccountJwtClientImpl.java
@@ -20,22 +20,20 @@ import com.microsoft.aad.adal4j.AuthenticationContext;
import com.microsoft.aad.adal4j.AuthenticationResult;
import com.microsoft.aad.adal4j.ClientCredential;
import org.apache.http.HttpStatus;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
-import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.indexer.azure.model.AADConfiguration;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.search.IdToken;
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
import org.opengroup.osdu.core.common.provider.interfaces.IJwtCache;
+import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient;
+import org.opengroup.osdu.indexer.azure.config.AADConfiguration;
import org.springframework.stereotype.Component;
import org.springframework.web.context.annotation.RequestScope;
import javax.inject.Inject;
-import java.net.MalformedURLException;
-import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
@@ -93,10 +91,6 @@ public class ServiceAccountJwtClientImpl implements IServiceAccountJwtClient {
} catch (InterruptedException e) {
e.printStackTrace();
- } catch (ExecutionException e) {
- e.printStackTrace();
- } catch (MalformedURLException e) {
- e.printStackTrace();
} finally {
service.shutdown();
}
diff --git a/provider/indexer-azure/src/main/resources/application.properties b/provider/indexer-azure/src/main/resources/application.properties
index 9c21787fe..0a0b64900 100644
--- a/provider/indexer-azure/src/main/resources/application.properties
+++ b/provider/indexer-azure/src/main/resources/application.properties
@@ -1,88 +1,50 @@
LOG_PREFIX=indexer
-
-#server.servlet.contextPath=/api/indexer/v2/
-logging.level.org.springframework.web=DEBUG
-server.port=8080
-JAVA_HEAP_OPTS=-Xms4096M -Xmx4096M
-JAVA_GC_OPTS=-XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45
-
+REGION=centralus
+logging.level.org.opendes.osdu=DEBUG
+JAVA_OPTS=-Dserver.port=80
DEFAULT_DATA_COUNTRY=US
-AUTHORIZE_API=https://opendesproxy.azurewebsites.net/entitlements/v1
-AUTHORIZE_API_KEY=dummy
-LEGALTAG_API=https://opendesproxy.azurewebsites.net/legal/v1
-LEGALTAG_API_KEY=dummy
-CRS_API=https://opendesproxy.azurewebsites.net/crs/v1
-CRS_API_KEY=dummy
+AUTHORIZE_API=${entitlements_service_endpoint}
+AUTHORIZE_API_KEY=${entitlements_service_api_key}
+LEGALTAG_API=${legal_service_endpoint}
+
-#Default Cache Settings
SCHEMA_CACHE_EXPIRATION=60
INDEX_CACHE_EXPIRATION=60
ELASTIC_CACHE_EXPIRATION=1440
CURSOR_CACHE_EXPIRATION=60
-# Kinds Cache expiration 2*24*60
KINDS_CACHE_EXPIRATION=2880
-# Attributes Cache expiration 2*24*60
ATTRIBUTES_CACHE_EXPIRATION=2880
-# Maximum size of cache value
MAX_CACHE_VALUE_SIZE=1000
-
KINDS_REDIS_DATABASE=1
CRON_INDEX_CLEANUP_THRESHOLD_DAYS=3
CRON_EMPTY_INDEX_CLEANUP_THRESHOLD_DAYS=7
-STORAGE_SCHEMA_HOST=https://opendesmvp.azurewebsites.net/schemas
-STORAGE_QUERY_RECORD_HOST=https://opendesmvp.azurewebsites.net/query/records
-STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=https://opendesmvp.azurewebsites.net/query/records:batch
+storage_service_url=${storage_service_endpoint}
+STORAGE_SCHEMA_HOST=${storage_service_url}/schemas
+STORAGE_QUERY_RECORD_HOST=${storage_service_url}/query/records
+STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=${storage_service_url}/query/records:batch
STORAGE_RECORDS_BATCH_SIZE=20
-INDEXER_QUEUE_HOST=https://requeuefunction-cd1.azurewebsites.net/api/re-enqueue
-
-#MSAL AAD
-aad.authority=https://login.microsoftonline.com/1668106e-2ae0-456a-bb7d-64b52104db99/oauth2/token
-aad.clientId=b70451c6-936f-4430-8680-0985817ac751
-aad.secretKey=b5GZtbS.bB7iSzLTb+Nb-AOb9G@1@LZo
-aad.oboApi=api://ffb32b1a-c6fe-4ed0-819b-6ca483de3640
-
-spring.security.oauth2.client.registration.azure.client-id=b70451c6-936f-4430-8680-0985817ac751
-spring.security.oauth2.client.registration.azure.client-secret=b5GZtbS.bB7iSzLTb+Nb-AOb9G@1@LZo
-spring.security.oauth2.client.registration.azure.client-name=Azure
-
+INDEXER_QUEUE_HOST=http://127.0.0.1:9000
-#spring.security.user.name=opendes@byoc.local
-#spring.security.user.password=123
-#spring.security.user.roles=service.indexer.admin
-
-
-# Azure AD configuration
-azure.activedirectory.client-id=ffb32b1a-c6fe-4ed0-819b-6ca483de3640
-azure.activedirectory.tenant-id=1668106e-2ae0-456a-bb7d-64b52104db99
-azure.activedirectory.AppIdUri=api://${azure.activedirectory.client-id}
+#AzureADconfiguration
azure.activedirectory.session-stateless=true
+azure.activedirectory.client-id=${aad_client_id}
-#azure.activedirectory.active-directory-groups=ADMIN, VIEWER
-
-# Azure CosmosDB configuration
-azure.cosmosdb.uri=https://opendescosmosdb.documents.azure.com:443/
-azure.cosmosdb.key=AQK0VbnMGn76h9Ypw5aoirKMwoAYJBRY2XENn1dAYypoea83weVyjYcGaRDeEX7HqXIVggaDuFAhQ4Y1Y9lKDQ==
-azure.cosmosdb.database=opendesdb
-
+azure.cosmosdb.uri=${cosmosdb_account}
+azure.cosmosdb.key=${cosmosdb_key}
+azure.cosmosdb.database=${cosmosdb_database}
-# Azure Service Bus configuration
-azure.servicebus.connection-string=Endpoint=sb://pliuopendes.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=km8Nscc0gf299Ck6npmM3D14VU5Tx1lJYRdlHcExIvY=
-azure.servicebus.topic-name=recordstopic
+#AzureServiceBusconfiguration
+azure.servicebus.namespace-name=${servicebus_namespace_name}
+azure.servicebus.topic-name=${servicebus_topic_name}
#Indexer-Queue-header
indexer.queue.key=abcd
-
-REDIS_GROUP_HOST=127.0.0.1
-REDIS_GROUP_PORT=6379
REDIS_SEARCH_HOST=localhost
REDIS_SEARCH_PORT=6379
-ELASTIC_DATASTORE_KIND=SearchSettings
-ELASTIC_DATASTORE_ID=indexer-service
-
-GAE_SERVICE=indexer
-
-DEPLOYMENT_ENVIRONMENT=CLOUD
\ No newline at end of file
+azure.keyvault.url=${KEYVAULT_URI}
+azure.application-insights.instrumentation-key=${appinsights_key}
+spring.application.name=indexer-azure
\ No newline at end of file
diff --git a/provider/indexer-azure/src/main/resources/logback-spring.xml b/provider/indexer-azure/src/main/resources/logback-spring.xml
deleted file mode 100644
index b6cddae16..000000000
--- a/provider/indexer-azure/src/main/resources/logback-spring.xml
+++ /dev/null
@@ -1,42 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Copyright © Microsoft Corporation
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-
-<configuration>
-
- <appender name="Console"
- class="ch.qos.logback.core.ConsoleAppender">
- <layout class="ch.qos.logback.classic.PatternLayout">
- <Pattern>
- %black(%d{ISO8601}) %highlight(%-5level) [%blue(%t)] %yellow(%C{1.}): %msg%n%throwable
- </Pattern>
- </layout>
- </appender>
-
- <appender name="aiAppender"
- class="com.microsoft.applicationinsights.logback.ApplicationInsightsAppender">
- <instrumentationKey>${appinsights_key}</instrumentationKey>
- </appender>
-
- <!-- servicebus throws exceptions when idle -->
- <logger name="com.microsoft.azure.servicebus.primitives.RequestResponseLink" level="error" />
-
- <root level="warn">
- <appender-ref ref="Console" />
- <appender-ref ref="aiAppender" />
- </root>
-
-</configuration>
\ No newline at end of file
diff --git a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/ElasticSettingServiceTest.java b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/ElasticSettingServiceTest.java
deleted file mode 100644
index ca3198ffa..000000000
--- a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/ElasticSettingServiceTest.java
+++ /dev/null
@@ -1,112 +0,0 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.azure.service;
-
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.indexer.service.ElasticSettingServiceImpl;
-import org.opengroup.osdu.core.common.model.search.ClusterSettings;
-import org.opengroup.osdu.core.common.provider.interfaces.IElasticCredentialsCache;
-import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository;
-import org.opengroup.osdu.core.common.multitenancy.ITenantInfoService;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.springframework.test.context.junit4.SpringRunner;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.mockito.Mockito.when;
-
-@RunWith(SpringRunner.class)
-public class ElasticSettingServiceTest {
-
- @Mock
- private ITenantInfoService tenantInfoService;
- @Mock
- private IElasticRepository elasticRepository;
- @Mock
- private IElasticCredentialsCache elasticCredentialCache;
- @Mock
- private TenantInfo tenantInfo;
- @InjectMocks
- private ElasticSettingServiceImpl sut;
- @Mock
- private ClusterSettings clusterSettings;
- @Mock
- private DpsHeaders dpsHeaders;
-
- @Mock
- private JaxRsDpsLog log;
-
-
- public String GAE_SERVICE = "indexer";
-
- private final String host = "db5c51c1.us-central1.gcp.cloud.es.io";
- private final int port = 9243;
- private final String credentials = "name:password";
-
- String cacheKey = "";
-
-
- @Before
- public void setup() {
- when(tenantInfo.getName()).thenReturn("tenant1");
- when(this.dpsHeaders.getPartitionId()).thenReturn("tenant1");
- when(this.tenantInfoService.getTenantInfo()).thenReturn(tenantInfo);
- sut.GAE_SERVICE = "indexer";
- clusterSettings = ClusterSettings.builder().host(host).port(port).userNameAndPassword(credentials).build();
- cacheKey = String.format("%s-%s", GAE_SERVICE, tenantInfo.getName());
- }
-
- @Test
- public void should_getValid_clusterSettings_fromCache() {
-
- when(this.elasticCredentialCache.get(cacheKey)).thenReturn(clusterSettings);
-
- ClusterSettings response = this.sut.getElasticClusterInformation();
- assertNotNull(response);
- assertEquals(response.getHost(), host);
- assertEquals(response.getPort(), port);
- assertEquals(response.getUserNameAndPassword(), credentials);
- }
-
- @Test
- public void should_getValid_clusterSettings_fromCosmosDB() {
-
- when(this.elasticCredentialCache.get(cacheKey)).thenReturn(clusterSettings);
-
- when(this.elasticRepository.getElasticClusterSettings(tenantInfo)).thenReturn(clusterSettings);
-
- ClusterSettings response = this.sut.getElasticClusterInformation();
- assertNotNull(response);
- assertEquals(response.getHost(), host);
- assertEquals(response.getPort(), port);
- assertEquals(response.getUserNameAndPassword(), credentials);
- }
-
- @Test(expected = AppException.class)
- public void should_throwAppException_when_tenantClusterInfo_not_found() throws AppException {
-
- when(this.elasticRepository.getElasticClusterSettings(tenantInfo)).thenReturn(null);
-
- this.sut.getElasticClusterInformation();
-
- }
-}
diff --git a/testing/indexer-test-azure/pom.xml b/testing/indexer-test-azure/pom.xml
new file mode 100644
index 000000000..1abb9f8f4
--- /dev/null
+++ b/testing/indexer-test-azure/pom.xml
@@ -0,0 +1,165 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Copyright © Amazon Web Services
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>org.opengroup.osdu.indexer</groupId>
+ <artifactId>indexer-test-azure</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+ <packaging>jar</packaging>
+
+ <properties>
+ <maven.compiler.target>1.8</maven.compiler.target>
+ <maven.compiler.source>1.8</maven.compiler.source>
+ <cucumber.version>1.2.5</cucumber.version>
+ <azure.keyvault.version>4.1.0</azure.keyvault.version>
+ <junit.jupiter.version>5.6.0</junit.jupiter.version>
+ </properties>
+
+ <dependencies>
+
+ <!-- Internal packages -->
+ <dependency>
+ <groupId>org.opengroup.osdu.indexer</groupId>
+ <artifactId>indexer-test-core</artifactId>
+ <version>0.0.2-SNAPSHOT</version>
+ </dependency>
+
+ <!-- Azure dependencies -->
+ <dependency>
+ <groupId>org.opengroup.osdu</groupId>
+ <artifactId>core-lib-azure</artifactId>
+ <version>0.0.6-SNAPSHOT</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.azure</groupId>
+ <artifactId>azure-security-keyvault-secrets</artifactId>
+ <version>${azure.keyvault.version}</version>
+ </dependency>
+
+ <!-- Testing -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.12</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>info.cukes</groupId>
+ <artifactId>cucumber-java</artifactId>
+ <version>${cucumber.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>info.cukes</groupId>
+ <artifactId>cucumber-junit</artifactId>
+ <version>${cucumber.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter</artifactId>
+ <version>${junit.jupiter.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-junit-jupiter</artifactId>
+ <version>2.23.0</version>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- third party Apache 2.0 license packages -->
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>2.8.5</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.jaxrs</groupId>
+ <artifactId>jackson-jaxrs-json-provider</artifactId>
+ <version>2.9.9</version>
+ </dependency>
+ <dependency>
+ <groupId>org.glassfish</groupId>
+ <artifactId>javax.json</artifactId>
+ <version>1.1.4</version>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-client</artifactId>
+ <version>1.19.4</version>
+ </dependency>
+ <dependency>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ <version>1.18.2</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>2.6</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>27.1-jre</version>
+ </dependency>
+
+ <!--Elasticsearch-->
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch</artifactId>
+ <version>6.6.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-client</artifactId>
+ <version>6.6.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-high-level-client</artifactId>
+ <version>6.6.2</version>
+ </dependency>
+
+ <!--Logging-->
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-to-slf4j</artifactId>
+ <version>2.13.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ <version>2.13.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <version>3.0.0</version>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+</project>
\ No newline at end of file
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/model/ElasticSettingSchema.java b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java
similarity index 56%
rename from provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/model/ElasticSettingSchema.java
rename to testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java
index 0351c0846..ac4cc160e 100644
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/model/ElasticSettingSchema.java
+++ b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java
@@ -12,29 +12,16 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package org.opengroup.osdu.indexer.azure.model;
-
-import javax.validation.constraints.NotEmpty;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@AllArgsConstructor
-@NoArgsConstructor
-public class ElasticSettingSchema {
-
- @NotEmpty
- private String host;
-
- @NotEmpty
- private String port;
-
- @NotEmpty
- private String usernameAndPassword;
-
- @NotEmpty
- private boolean isHttps;
-
-}
+package org.opengroup.osdu.step_definitions.index.record;
+
+import cucumber.api.CucumberOptions;
+import cucumber.api.junit.Cucumber;
+import org.junit.runner.RunWith;
+
+@RunWith(Cucumber.class)
+@CucumberOptions(
+ features = "classpath:features/indexrecord/IndexRecord.feature",
+ glue = {"classpath:org.opengroup.osdu.step_definitions/index/record"},
+ plugin = {"pretty", "junit:target/cucumber-reports/TEST-indexrecord.xml"})
+public class RunTest {
+}
\ No newline at end of file
diff --git a/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
new file mode 100644
index 000000000..dfff25eca
--- /dev/null
+++ b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -0,0 +1,67 @@
+// Copyright © Microsoft Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.step_definitions.index.record;
+
+import lombok.extern.java.Log;
+
+import cucumber.api.Scenario;
+import cucumber.api.java.Before;
+import cucumber.api.DataTable;
+import cucumber.api.java.en.Given;
+import cucumber.api.java.en.Then;
+import cucumber.api.java.en.When;
+import org.opengroup.osdu.common.RecordSteps;
+import org.opengroup.osdu.util.AzureHTTPClient;
+import org.opengroup.osdu.util.ElasticUtils;
+
+@Log
+public class Steps extends RecordSteps {
+
+ public Steps() {
+ super(new AzureHTTPClient(), new ElasticUtils());
+ }
+
+ @Before
+ public void before(Scenario scenario) {
+ this.scenario = scenario;
+ this.httpClient = new AzureHTTPClient();
+ }
+
+ @Given("^the schema is created with the following kind$")
+ public void the_schema_is_created_with_the_following_kind(DataTable dataTable) {
+ super.the_schema_is_created_with_the_following_kind(dataTable);
+ }
+
+ @When("^I ingest records with the \"(.*?)\" with \"(.*?)\" for a given \"(.*?)\"$")
+ public void i_ingest_records_with_the_for_a_given(String record, String dataGroup, String kind) {
+ super.i_ingest_records_with_the_for_a_given(record, dataGroup, kind);
+ }
+
+ @Then("^I should get the (\\d+) documents for the \"([^\"]*)\" in the Elastic Search$")
+ public void i_should_get_the_documents_for_the_in_the_Elastic_Search(int expectedCount, String index) throws Throwable {
+ super.i_should_get_the_documents_for_the_in_the_Elastic_Search(expectedCount, index);
+ }
+
+ @Then("^I should get the elastic \"(.*?)\" for the \"([^\"]*)\" and \"([^\"]*)\" in the Elastic Search$")
+ public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String type, String index) throws Throwable {
+ super.i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(expectedMapping, type, index);
+ }
+
+ @Then("^I should get the (\\d+) documents for the \"([^\"]*)\" in the Elastic Search with out \"(.*?)\"$")
+ public void iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(int expectedCount, String index, String skippedAttributes) throws Throwable {
+ super.iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(expectedCount, index, skippedAttributes);
+ }
+
+}
\ No newline at end of file
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/ServletInitializer.java b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/util/AzureHTTPClient.java
similarity index 55%
rename from provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/ServletInitializer.java
rename to testing/indexer-test-azure/src/test/java/org/opengroup/osdu/util/AzureHTTPClient.java
index 0a6086fb6..d6c57ae84 100644
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/ServletInitializer.java
+++ b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/util/AzureHTTPClient.java
@@ -12,15 +12,28 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package org.opengroup.osdu.indexer.azure;
+package org.opengroup.osdu.util;
-import org.springframework.boot.builder.SpringApplicationBuilder;
-import org.springframework.boot.web.servlet.support.SpringBootServletInitializer;
+import lombok.ToString;
+import lombok.extern.java.Log;
+
+import java.io.IOException;
+
+@Log
+@ToString
+public class AzureHTTPClient extends HTTPClient {
+
+ private static String token = null;
-public class ServletInitializer extends SpringBootServletInitializer {
@Override
- protected SpringApplicationBuilder configure(SpringApplicationBuilder application) {
- return application.sources(IndexerAzureApplication.class);
+ public synchronized String getAccessToken() {
+ if(token == null) {
+ try {
+ token = "Bearer " + JwtTokenUtil.getAccessToken();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ return token;
}
-}
-
+}
\ No newline at end of file
diff --git a/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/util/AzureServicePrincipal.java b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/util/AzureServicePrincipal.java
new file mode 100644
index 000000000..57ef5389c
--- /dev/null
+++ b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/util/AzureServicePrincipal.java
@@ -0,0 +1,83 @@
+// Copyright © Microsoft Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.util;
+
+import com.google.gson.Gson;
+import com.google.gson.JsonObject;
+
+import java.io.BufferedReader;
+import java.io.DataOutputStream;
+import java.io.InputStreamReader;
+import java.io.UnsupportedEncodingException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.net.URLEncoder;
+import java.util.HashMap;
+import java.util.Map;
+
+public class AzureServicePrincipal {
+ public static String getIdToken(String sp_id, String sp_secret, String tenant_id, String app_resource_id) throws Exception {
+ String aad_endpoint = String.format("https://login.microsoftonline.com/%s/oauth2/token", tenant_id);
+ URL url = new URL(aad_endpoint);
+ HttpURLConnection con = (HttpURLConnection) url.openConnection();
+ con.setRequestMethod("POST");
+ con.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
+
+ Map<String, String> parameters = new HashMap<>();
+ parameters.put("grant_type", "client_credentials");
+ parameters.put("client_id", sp_id);
+ parameters.put("client_secret", sp_secret);
+ parameters.put("resource", app_resource_id);
+
+ con.setDoOutput(true);
+ DataOutputStream out = new DataOutputStream(con.getOutputStream());
+ out.writeBytes(getParamsString(parameters));
+ out.flush();
+ out.close();
+
+ BufferedReader in = new BufferedReader(
+ new InputStreamReader(con.getInputStream()));
+ String inputLine;
+ StringBuffer content = new StringBuffer();
+ while ((inputLine = in.readLine()) != null) {
+ content.append(inputLine);
+ }
+ in.close();
+
+ con.disconnect();
+
+ Gson gson = new Gson();
+ JsonObject jobj = gson.fromJson(content.toString(), JsonObject.class);
+ String token = jobj.get("access_token").getAsString();
+ return token;
+ }
+
+ private static String getParamsString(Map<String, String> params)
+ throws UnsupportedEncodingException {
+ StringBuilder result = new StringBuilder();
+
+ for (Map.Entry<String, String> entry : params.entrySet()) {
+ result.append(URLEncoder.encode(entry.getKey(), "UTF-8"));
+ result.append("=");
+ result.append(URLEncoder.encode(entry.getValue(), "UTF-8"));
+ result.append("&");
+ }
+
+ String resultString = result.toString();
+ return resultString.length() > 0
+ ? resultString.substring(0, resultString.length() - 1)
+ : resultString;
+ }
+}
\ No newline at end of file
diff --git a/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/util/JwtTokenUtil.java b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/util/JwtTokenUtil.java
new file mode 100644
index 000000000..b1a799920
--- /dev/null
+++ b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/util/JwtTokenUtil.java
@@ -0,0 +1,30 @@
+// Copyright © Microsoft Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.util;
+
+class JwtTokenUtil {
+ static public String getAccessToken() throws Exception {
+
+ String token = new String();
+
+ String sp_id = System.getProperty("INTEGRATION_TESTER", System.getenv("INTEGRATION_TESTER"));
+ String sp_secret = System.getProperty("AZURE_TESTER_SERVICEPRINCIPAL_SECRET", System.getenv("AZURE_TESTER_SERVICEPRINCIPAL_SECRET"));
+ String tenant_id = System.getProperty("AZURE_AD_TENANT_ID", System.getenv("AZURE_AD_TENANT_ID"));
+ String app_resource_id = System.getProperty("AZURE_AD_APP_RESOURCE_ID", System.getenv("AZURE_AD_APP_RESOURCE_ID"));
+ token = AzureServicePrincipal.getIdToken(sp_id, sp_secret, tenant_id, app_resource_id);
+
+ return token;
+ }
+}
diff --git a/testing/indexer-test-azure/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/testing/indexer-test-azure/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker
new file mode 100644
index 000000000..ca6ee9cea
--- /dev/null
+++ b/testing/indexer-test-azure/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker
@@ -0,0 +1 @@
+mock-maker-inline
\ No newline at end of file
--
GitLab