diff --git a/indexer-core/pom.xml b/indexer-core/pom.xml
index 9242903c9118122c639d169c8ac4c5c62fdd9140..919ad21103a071122b569c0ca33c2215a6ecaff6 100644
--- a/indexer-core/pom.xml
+++ b/indexer-core/pom.xml
@@ -1,16 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-service</artifactId>
- <version>1.0.4-SNAPSHOT</version>
+ <version>0.6.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>indexer-core</artifactId>
- <version>1.0.6-SNAPSHOT</version>
+ <version>0.6.0-SNAPSHOT</version>
<name>indexer-core</name>
<description>Indexer Service Core</description>
<packaging>jar</packaging>
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java
index 0a6f6ba7a14c54fbf7b34f5e4acadf755a75109a..4f6a1f6669731c9a3602c3e73e79dc935387359d 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java
@@ -54,4 +54,11 @@ public class ReindexApi {
this.auditLogger.getReindex(singletonList(recordReindexRequest.getKind()));
return new ResponseEntity<>(org.springframework.http.HttpStatus.OK);
}
+
+ @PreAuthorize("@authorizationFilter.hasPermission('" + SearchServiceRole.ADMIN + "')")
+ @PatchMapping
+ public ResponseEntity<String> fullReindex(@RequestParam(value = "force_clean", defaultValue = "false") boolean forceClean) throws IOException {
+ this.reIndexService.fullReindex(forceClean);
+ return new ResponseEntity<>(org.springframework.http.HttpStatus.OK);
+ }
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/config/IndexerConfigurationProperties.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/config/IndexerConfigurationProperties.java
index 13ed75f28a0ba814cd3af9b01e3a166a5b05ae7d..1123e5ce21712f4ceac5a47cc78b322f4ecca5a9 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/config/IndexerConfigurationProperties.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/config/IndexerConfigurationProperties.java
@@ -43,6 +43,7 @@ public class IndexerConfigurationProperties {
private String environment;
private String indexerHost;
private String searchHost;
+ private String storageQueryKindsHost;
private String storageQueryRecordForConversionHost;
private String storageQueryRecordHost;
private Integer storageRecordsBatchSize;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexService.java
index 39486f1c7498260e7eda5fd83b9de379f79b1d4f..6569743e49fd39a381eec56123e14ea7b96c7232 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexService.java
@@ -20,4 +20,6 @@ import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
public interface ReindexService {
String reindexRecords(RecordReindexRequest recordReindexRequest, boolean forceClean);
+
+ void fullReindex(boolean forceClean);
}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java
index 013fcb6928d9f3a7a8d8ea27123b31bdd93d3832..0f3d5c4f18db650813ca096878925dc49a153921 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java
@@ -16,6 +16,7 @@ package org.opengroup.osdu.indexer.service;
import com.google.common.base.Strings;
import com.google.gson.Gson;
+import java.util.Objects;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.http.AppException;
@@ -100,4 +101,26 @@ public class ReindexServiceImpl implements ReindexService {
throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Unknown error", "An unknown error has occurred.", e);
}
}
+
+ @Override
+ public void fullReindex(boolean forceClean) {
+ List<String> allKinds = null;
+ try {
+ allKinds = storageService.getAllKinds();
+ } catch (Exception e) {
+ jaxRsDpsLog.error("storage service all kinds request failed",e);
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "storage service cannot respond with all kinds", "an unknown error has occurred.", e);
+ }
+ if (Objects.isNull(allKinds)){
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "storage service cannot respond with all kinds", "full reindex failed");
+ }
+ for (String kind : allKinds) {
+ try {
+ reindexRecords(new RecordReindexRequest(kind, ""), forceClean);
+ } catch (Exception e) {
+ jaxRsDpsLog.warning(String.format("kind: %s cannot be re-indexed", kind));
+ continue;
+ }
+ }
+ }
}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageService.java
index ab8e9901294893255b0f00a7632ee0145c37f1fa..bfbb5c6d5a0faf2cf1b62ec031697141f0f649ef 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageService.java
@@ -30,4 +30,6 @@ public interface StorageService {
RecordQueryResponse getRecordsByKind(RecordReindexRequest request) throws URISyntaxException;
String getStorageSchema(String kind) throws URISyntaxException, UnsupportedEncodingException;
+
+ List<String> getAllKinds() throws URISyntaxException;
}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java
index b8a3bf8aabbc4340309cbdaeb5cd29db22c60da1..a390e617ea31b9bb9f399deed3b9cc2b921ea48d 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java
@@ -20,6 +20,9 @@ import com.google.common.collect.Lists;
import com.google.common.reflect.TypeToken;
import com.google.gson.Gson;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
import org.opengroup.osdu.core.common.http.FetchServiceHttpRequest;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.http.AppException;
@@ -209,4 +212,18 @@ public class StorageServiceImpl implements StorageService {
HttpResponse response = this.urlFetchService.sendRequest(request);
return response.getResponseCode() != HttpStatus.SC_OK ? null : response.getBody();
}
+
+ @Override
+ public List<String> getAllKinds() throws URISyntaxException {
+ String url = configurationProperties.getStorageQueryKindsHost();
+ FetchServiceHttpRequest request = FetchServiceHttpRequest.builder()
+ .httpMethod(HttpMethods.GET)
+ .headers(this.requestInfo.getHeadersMap())
+ .url(url)
+ .build();
+ HttpResponse response = this.urlFetchService.sendRequest(request);
+ JsonObject asJsonObject = new JsonParser().parse(response.getBody()).getAsJsonObject();
+ JsonElement results = asJsonObject.get("results");
+ return response.getResponseCode() != HttpStatus.SC_OK ? null : this.gson.fromJson(results,List.class);
+ }
}
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index a85a2d468a67dec3503db2bae8cb53a4be71f1aa..def52c1db676f2e41dad2e018833da0e40dac63b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1,20 +1,18 @@
<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.1.18.RELEASE</version>
- <relativePath/>
+ <relativePath />
</parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-service</artifactId>
<packaging>pom</packaging>
- <version>1.0.4-SNAPSHOT</version>
+ <version>0.6.0-SNAPSHOT</version>
<description>Indexer Service</description>
<properties>
diff --git a/provider/indexer-aws/pom.xml b/provider/indexer-aws/pom.xml
index b8e13f22e2f628b220360494943833840cdcca1a..d6a11e333aba7bc1168209b4489d02d6648909e3 100644
--- a/provider/indexer-aws/pom.xml
+++ b/provider/indexer-aws/pom.xml
@@ -14,12 +14,11 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-service</artifactId>
- <version>1.0.4-SNAPSHOT</version>
+ <version>0.6.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
@@ -27,7 +26,7 @@
<artifactId>indexer-aws</artifactId>
<description>Storage service on AWS</description>
<packaging>jar</packaging>
- <version>${version.number}</version>
+ <version>0.6.0-SNAPSHOT</version>
<properties>
<aws.version>1.11.637</aws.version>
@@ -44,7 +43,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-core</artifactId>
- <version>1.0.6-SNAPSHOT</version>
+ <version>0.6.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
diff --git a/provider/indexer-aws/src/main/resources/application.properties b/provider/indexer-aws/src/main/resources/application.properties
index 77f80a34f9d23307e59202daf13fd6f047bd5ace..d29cc7a7661161f50377c9577cb7c4f418771c70 100644
--- a/provider/indexer-aws/src/main/resources/application.properties
+++ b/provider/indexer-aws/src/main/resources/application.properties
@@ -27,6 +27,7 @@ SCHEMA_HOST=${STORAGE_HOST}/api/schema-service/v1/schema
STORAGE_SCHEMA_HOST=${STORAGE_HOST}/api/storage/v2/schemas
STORAGE_QUERY_RECORD_HOST=${STORAGE_HOST}/api/storage/v2/query/records
+STORAGE_QUERY_KINDS_HOST=${STORAGE_HOST}/api/storage/v2/query/kinds
STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=${STORAGE_HOST}/api/storage/v2/query/records:batch
STORAGE_RECORDS_BATCH_SIZE=20
INDEXER_QUEUE_HOST=""
diff --git a/provider/indexer-azure/pom.xml b/provider/indexer-azure/pom.xml
index 8a5f33513d0b6db572b7410418684c8afea0927f..f67a7fcbb3ab51a12eafb5120b8a56ffb4e64a16 100644
--- a/provider/indexer-azure/pom.xml
+++ b/provider/indexer-azure/pom.xml
@@ -15,33 +15,31 @@
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-service</artifactId>
- <version>1.0.4-SNAPSHOT</version>
+ <version>0.6.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>indexer-azure</artifactId>
- <version>1.0.4-SNAPSHOT</version>
+ <version>0.6.0-SNAPSHOT</version>
<name>indexer-azure</name>
<description>Indexer Service Azure</description>
<packaging>jar</packaging>
<properties>
<azure.version>2.1.7</azure.version>
- <azure.appservice.resourcegroup></azure.appservice.resourcegroup>
- <azure.appservice.plan></azure.appservice.plan>
- <azure.appservice.appname></azure.appservice.appname>
- <azure.appservice.subscription></azure.appservice.subscription>
+ <azure.appservice.resourcegroup />
+ <azure.appservice.plan />
+ <azure.appservice.appname />
+ <azure.appservice.subscription />
<log4j.version>2.11.2</log4j.version>
<nimbus-jose-jwt.version>8.2</nimbus-jose-jwt.version>
- <indexer-core.version>1.0.6-SNAPSHOT</indexer-core.version>
+ <indexer-core.version>0.6.0-SNAPSHOT</indexer-core.version>
<spring-security-jwt.version>1.1.1.RELEASE</spring-security-jwt.version>
<osdu.corelibazure.version>0.0.42</osdu.corelibazure.version>
<osdu.oscorecommon.version>0.3.12</osdu.oscorecommon.version>
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/config/AzureBootstrapConfig.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/config/AzureBootstrapConfig.java
index 09f69ecde0f5d292026c9220ae1c7b89adc184fe..570b932a29f49f8ba849488ed55d9e2f61d440ce 100644
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/config/AzureBootstrapConfig.java
+++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/config/AzureBootstrapConfig.java
@@ -16,6 +16,9 @@ package org.opengroup.osdu.indexer.azure.config;
import com.azure.security.keyvault.secrets.SecretClient;
import org.opengroup.osdu.azure.KeyVaultFacade;
+import org.opengroup.osdu.core.common.entitlements.EntitlementsAPIConfig;
+import org.opengroup.osdu.core.common.entitlements.EntitlementsFactory;
+import org.opengroup.osdu.core.common.entitlements.IEntitlementsFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Component;
@@ -37,6 +40,12 @@ public class AzureBootstrapConfig {
@Value("${MAX_CACHE_VALUE_SIZE}")
private Integer maxCacheValueSize;
+ @Value("${AUTHORIZE_API_KEY}")
+ private String entitlementsAPIKey;
+
+ @Value("${AUTHORIZE_API}")
+ private String entitlementsAPIEndpoint;
+
@Bean
@Named("KEY_VAULT_URL")
public String getKeyVaultURL() {
@@ -80,4 +89,13 @@ public class AzureBootstrapConfig {
String tenant = KeyVaultFacade.getSecretWithValidation(sc, "app-dev-sp-tenant-id");
return String.format(urlFormat, tenant);
}
+
+ @Bean
+ public IEntitlementsFactory entitlementsFactory() {
+ EntitlementsAPIConfig apiConfig = EntitlementsAPIConfig.builder()
+ .apiKey(entitlementsAPIKey)
+ .rootUrl(entitlementsAPIEndpoint)
+ .build();
+ return new EntitlementsFactory(apiConfig);
+ }
}
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/di/EntitlementsFactoryAzure.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/di/EntitlementsFactoryAzure.java
deleted file mode 100644
index 63e268a87015d69021db40de5daac634c027bdea..0000000000000000000000000000000000000000
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/di/EntitlementsFactoryAzure.java
+++ /dev/null
@@ -1,30 +0,0 @@
-// Copyright © Microsoft Corporation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.azure.di;
-
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.entitlements.IEntitlementsFactory;
-import org.opengroup.osdu.core.common.entitlements.IEntitlementsService;
-import org.springframework.context.annotation.Primary;
-import org.springframework.stereotype.Component;
-
-@Component
-@Primary
-public class EntitlementsFactoryAzure implements IEntitlementsFactory {
- @Override
- public IEntitlementsService create(DpsHeaders dpsHeaders) {
- return new EntitlementsServiceAzure(dpsHeaders);
- }
-}
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/di/EntitlementsServiceAzure.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/di/EntitlementsServiceAzure.java
deleted file mode 100644
index 05370bd83eed7eba1c519ad48570149fb06cab6b..0000000000000000000000000000000000000000
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/di/EntitlementsServiceAzure.java
+++ /dev/null
@@ -1,110 +0,0 @@
-// Copyright © Microsoft Corporation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.azure.di;
-
-import com.microsoft.azure.spring.autoconfigure.aad.UserPrincipal;
-import org.apache.http.HttpStatus;
-
-import org.opengroup.osdu.core.common.model.entitlements.*;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.entitlements.IEntitlementsService;
-import org.opengroup.osdu.core.common.http.HttpResponse;
-import org.springframework.security.core.Authentication;
-import org.springframework.security.core.GrantedAuthority;
-import org.springframework.security.core.context.SecurityContextHolder;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-public class EntitlementsServiceAzure implements IEntitlementsService {
- //service principals don't have UPN or email
- static final String INTEGRATION_TEST_ADMIN_ROLE = "data.test.admin@opendes.onmicrosoft.com";
- public static final String PREFIX = "ROLE_";
- DpsHeaders headers;
-
- public EntitlementsServiceAzure(DpsHeaders headers){
- this.headers = headers;
- }
-
- @Override
- public MemberInfo addMember(GroupEmail groupEmail, MemberInfo memberInfo) throws EntitlementsException {
- return null;
- }
-
- @Override
- public Members getMembers(GroupEmail groupEmail, GetMembers getMembers) throws EntitlementsException {
- return null;
- }
-
- @Override
- public Groups getGroups() throws EntitlementsException {
- final Authentication auth = SecurityContextHolder.getContext().getAuthentication();
- final UserPrincipal current = (UserPrincipal) auth.getPrincipal();
- String email = current.getUpn();
-
- List<GroupInfo> giList = new ArrayList();
- Collection<? extends GrantedAuthority> authorities = auth.getAuthorities();
- for(GrantedAuthority authority : authorities)
- {
- GroupInfo gi = new GroupInfo();
- String role = authority.getAuthority();
- if (role.startsWith(PREFIX)){
- role = role.substring(PREFIX.length());
- }
- gi.setName(role);
- if ((email == null || email.isEmpty()) && role.equalsIgnoreCase(INTEGRATION_TEST_ADMIN_ROLE)) {
- email = INTEGRATION_TEST_ADMIN_ROLE;
- gi.setEmail(email);
- giList.add(0, gi);
- }
- else {
- gi.setEmail(email);
- giList.add(gi);
- }
- }
- if (giList.size() > 0)
- {
- Groups groups = new Groups();
- groups.setGroups(giList);
- groups.setDesId(email);
- return groups;
- }
-
- HttpResponse response = new HttpResponse();
- response.setResponseCode(HttpStatus.SC_INTERNAL_SERVER_ERROR);
- throw new EntitlementsException("no authorities found", response);
- }
-
- @Override
- public GroupInfo createGroup(CreateGroup createGroup) throws EntitlementsException {
- return null;
- }
-
- @Override
- public void deleteMember(String s, String s1) throws EntitlementsException {
-
- }
-
- @Override
- public Groups authorizeAny(String... strings) throws EntitlementsException {
- return null;
- }
-
- @Override
- public void authenticate() throws EntitlementsException {
-
- }
-}
diff --git a/provider/indexer-azure/src/main/resources/application.properties b/provider/indexer-azure/src/main/resources/application.properties
index a334813afb2b5cccfe9127b0cf95a9730f9e4f81..619120f5eeab512b3e0f3316e27156e70b6b9aef 100644
--- a/provider/indexer-azure/src/main/resources/application.properties
+++ b/provider/indexer-azure/src/main/resources/application.properties
@@ -41,6 +41,7 @@ SCHEMA_HOST=${schema_service_url}/schema
storage_service_url=${storage_service_endpoint}
STORAGE_SCHEMA_HOST=${storage_service_url}/schemas
STORAGE_QUERY_RECORD_HOST=${storage_service_url}/query/records
+STORAGE_QUERY_KINDS_HOST=${storage_service_url}/query/kinds
STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=${storage_service_url}/query/records:batch
STORAGE_RECORDS_BATCH_SIZE=20
diff --git a/provider/indexer-gcp/cloudbuild/Dockerfile.cloudbuild b/provider/indexer-gcp/cloudbuild/Dockerfile.cloudbuild
index 59c9b5dafcf6824308be6cf80357a887ac75bd36..65a9b4197c4d29aaa0aae6c8dd8d6abd7d6c2dac 100644
--- a/provider/indexer-gcp/cloudbuild/Dockerfile.cloudbuild
+++ b/provider/indexer-gcp/cloudbuild/Dockerfile.cloudbuild
@@ -7,7 +7,7 @@ ENV PROVIDER_NAME $PROVIDER_NAME
ARG PORT
ENV PORT $PORT
# Copy the jar to the production image from the builder stage.
-COPY provider/indexer-${PROVIDER_NAME}/target/indexer-${PROVIDER_NAME}-*-SNAPSHOT-spring-boot.jar indexer-${PROVIDER_NAME}.jar
+COPY provider/indexer-${PROVIDER_NAME}/target/indexer-${PROVIDER_NAME}-*-spring-boot.jar indexer-${PROVIDER_NAME}.jar
# Run the web service on container startup.
CMD java -Djava.security.egd=indexer:/dev/./urandom -Dserver.port=${PORT} -jar /app/indexer-${PROVIDER_NAME}.jar
diff --git a/provider/indexer-gcp/pom.xml b/provider/indexer-gcp/pom.xml
index 2b91fa1322e2b54eb010e91ef1af984f70de7ed7..1152119b412b5d75a638f68ef0c7c40766f340b0 100644
--- a/provider/indexer-gcp/pom.xml
+++ b/provider/indexer-gcp/pom.xml
@@ -1,18 +1,16 @@
<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-service</artifactId>
- <version>1.0.4-SNAPSHOT</version>
+ <version>0.6.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>indexer-gcp</artifactId>
- <version>1.0.4-SNAPSHOT</version>
+ <version>0.6.0-SNAPSHOT</version>
<name>indexer-gcp</name>
<description>Indexer Service GCP App Engine</description>
<packaging>jar</packaging>
@@ -21,7 +19,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-core</artifactId>
- <version>1.0.6-SNAPSHOT</version>
+ <version>0.6.0-SNAPSHOT</version>
</dependency>
<dependency>
diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/AppExceptionHandler.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/AppExceptionHandler.java
new file mode 100644
index 0000000000000000000000000000000000000000..0e8293b1a59abd37ebdcd59bf3f2610a7c39e813
--- /dev/null
+++ b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/util/AppExceptionHandler.java
@@ -0,0 +1,34 @@
+package org.opengroup.osdu.indexer.util;
+
+import java.util.Objects;
+import lombok.extern.slf4j.Slf4j;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.ControllerAdvice;
+import org.springframework.web.bind.annotation.ExceptionHandler;
+
+@ControllerAdvice
+@Slf4j
+public class AppExceptionHandler {
+
+ @ExceptionHandler(AppException.class)
+ public ResponseEntity<Object> handleAppExceptions(AppException e) {
+ return this.getErrorResponse(e);
+ }
+
+ private ResponseEntity<Object> getErrorResponse(AppException e) {
+
+ String exceptionMsg = Objects.nonNull(e.getOriginalException())
+ ? e.getOriginalException().getMessage()
+ : e.getError().getMessage();
+
+ if (e.getError().getCode() > 499) {
+ log.error(exceptionMsg, e.getOriginalException());
+ } else {
+ log.warn(exceptionMsg, e.getOriginalException());
+ }
+
+ return new ResponseEntity<>(e.getError(), HttpStatus.resolve(e.getError().getCode()));
+ }
+}
diff --git a/provider/indexer-gcp/src/main/resources/application.properties b/provider/indexer-gcp/src/main/resources/application.properties
index 06ba9d9417769af49ad5b9d364fa5cdecbb398da..b3d58b0c9b895a6ca38e8b29143edd258e7d8c8b 100644
--- a/provider/indexer-gcp/src/main/resources/application.properties
+++ b/provider/indexer-gcp/src/main/resources/application.properties
@@ -40,5 +40,6 @@ elastic-datastore-id=indexer-service
security.https.certificate.trust=false
indexer.que.service.mail=default@iam.gserviceaccount.com
-
SCHEMA_HOST=${HOST}/api/schema-service/v1/schema
+storage-query-kinds-host=https://${STORAGE_HOSTNAME}/api/storage/v2/query/kinds
+
diff --git a/provider/indexer-ibm/pom.xml b/provider/indexer-ibm/pom.xml
index af9c027d811b553ce5bb7ca4767ca486d6ea8c21..a8ef4cf4cb9bc3d439b114778adc2cad62fc4fc6 100644
--- a/provider/indexer-ibm/pom.xml
+++ b/provider/indexer-ibm/pom.xml
@@ -15,15 +15,13 @@
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-service</artifactId>
- <version>1.0.4-SNAPSHOT</version>
+ <version>0.6.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
@@ -55,7 +53,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-core</artifactId>
- <version>1.0.6-SNAPSHOT</version>
+ <version>0.6.0-SNAPSHOT</version>
</dependency>
<dependency>
diff --git a/provider/indexer-ibm/src/main/resources/application.properties b/provider/indexer-ibm/src/main/resources/application.properties
index def41568078ca252d54986a5eebde71a2fbc62d8..38a394adc7fc3c8230acc4a554212832c3e8861f 100644
--- a/provider/indexer-ibm/src/main/resources/application.properties
+++ b/provider/indexer-ibm/src/main/resources/application.properties
@@ -32,6 +32,7 @@ storage_service_url=http://localhost:8082
#storage_service_url=https://os-storage-ibm-osdu-r2.osduadev-a1c3eaf78a86806e299f5f3f207556f0-0000.us-south.containers.appdomain.cloud
STORAGE_SCHEMA_HOST=${storage_service_url}/api/storage/v2/schemas
STORAGE_QUERY_RECORD_HOST=${storage_service_url}/api/storage/v2/query/records
+STORAGE_QUERY_KINDS_HOST=${storage_service_url}/api/storage/v2/query/kinds
STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=${storage_service_url}/api/storage/v2/query/records:batch
STORAGE_RECORDS_BATCH_SIZE=20
diff --git a/provider/indexer-reference/pom.xml b/provider/indexer-reference/pom.xml
index 99d01082c723b55da8f969a693d301daf9378570..64212d38f47c19bfb2de3205a4b1fbf5617d84d5 100644
--- a/provider/indexer-reference/pom.xml
+++ b/provider/indexer-reference/pom.xml
@@ -16,20 +16,18 @@
~ limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-service</artifactId>
- <version>1.0.4-SNAPSHOT</version>
+ <version>0.6.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>indexer-reference</artifactId>
- <version>0.0.1-SNAPSHOT</version>
+ <version>0.6.0-SNAPSHOT</version>
<name>indexer-reference</name>
<description>Indexer Service GCP Anthos</description>
<packaging>jar</packaging>
@@ -38,7 +36,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-core</artifactId>
- <version>1.0.6-SNAPSHOT</version>
+ <version>0.6.0-SNAPSHOT</version>
</dependency>
<dependency>
diff --git a/testing/indexer-test-aws/pom.xml b/testing/indexer-test-aws/pom.xml
index e31c28157f62c209cf96815321dd459f1d8e7419..ca8036cc484ebc53d8c37303078b57ffb3c4b8fc 100644
--- a/testing/indexer-test-aws/pom.xml
+++ b/testing/indexer-test-aws/pom.xml
@@ -14,22 +14,20 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>indexer-test</artifactId>
- <version>0.0.5-SNAPSHOT</version>
+ <version>0.6.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-aws</artifactId>
- <version>0.0.2-SNAPSHOT</version>
+ <version>0.6.0-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
@@ -44,7 +42,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
- <version>0.0.2-SNAPSHOT</version>
+ <version>0.6.0-SNAPSHOT</version>
</dependency>
<!-- AWS specific packages -->
diff --git a/testing/indexer-test-azure/pom.xml b/testing/indexer-test-azure/pom.xml
index 40ac0301b4b7a76790fca2ba4682fcc93437b309..69273b80281a7825581b8a87a85aa2b1475ae232 100644
--- a/testing/indexer-test-azure/pom.xml
+++ b/testing/indexer-test-azure/pom.xml
@@ -14,22 +14,20 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>indexer-test</artifactId>
- <version>0.0.5-SNAPSHOT</version>
+ <version>0.6.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-azure</artifactId>
- <version>0.0.1-SNAPSHOT</version>
+ <version>0.6.0-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
@@ -46,7 +44,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
- <version>0.0.2-SNAPSHOT</version>
+ <version>0.6.0-SNAPSHOT</version>
</dependency>
<!-- Azure dependencies -->
diff --git a/testing/indexer-test-azure/src/test/resources/logback-test.xml b/testing/indexer-test-azure/src/test/resources/logback-test.xml
new file mode 100644
index 0000000000000000000000000000000000000000..dafd8ae394dc34b62ffa81307a76900db4b66af8
--- /dev/null
+++ b/testing/indexer-test-azure/src/test/resources/logback-test.xml
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<configuration>
+ <include resource="org/springframework/boot/logging/logback/base.xml" />
+ <root level="INFO" />
+ <logger name="org.springframework" level="INFO"/>
+</configuration>
\ No newline at end of file
diff --git a/testing/indexer-test-core/pom.xml b/testing/indexer-test-core/pom.xml
index d61bc6a5bb57e7d89dbeb5b8a13a4db14719daaf..46e5f7eabf80cc978a238595036c366805b66573 100644
--- a/testing/indexer-test-core/pom.xml
+++ b/testing/indexer-test-core/pom.xml
@@ -1,19 +1,17 @@
<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>indexer-test</artifactId>
- <version>0.0.5-SNAPSHOT</version>
+ <version>0.6.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
- <version>0.0.2-SNAPSHOT</version>
+ <version>0.6.0-SNAPSHOT</version>
<properties>
<maven.compiler.target>1.8</maven.compiler.target>
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/CleanupIndiciesSteps.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/CleanupIndiciesSteps.java
index d15a6668d90540a862167070018333599feb1143..d348fd9d2a48fb4daaad6099f4609fea20882329 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/CleanupIndiciesSteps.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/CleanupIndiciesSteps.java
@@ -87,8 +87,8 @@ public class CleanupIndiciesSteps extends TestsBase {
records = new Gson().fromJson(fileContent, new TypeToken<List<Map<String, Object>>>() {}.getType());
for (Map<String, Object> testRecord : records) {
- testRecord.put("id", generateActualName(testRecord.get("id").toString(), timeStamp));
testRecord.put("kind", actualKind);
+ testRecord.put("id", generateActualId(testRecord.get("id").toString(), timeStamp, actualKind));
testRecord.put("legal", generateLegalTag());
String[] x_acl = {generateActualName(dataGroup,timeStamp)+"."+getEntitlementsDomain()};
Acl acl = Acl.builder().viewers(x_acl).owners(x_acl).build();
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
index 3a013e1530b093e9c779890c5a12b230257028b4..948cad79655e474cd4c0639ff4305c3b82d4fd35 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
@@ -97,8 +97,8 @@ public class RecordSteps extends TestsBase {
records = new Gson().fromJson(fileContent, new TypeToken<List<Map<String, Object>>>() {}.getType());
for (Map<String, Object> testRecord : records) {
- testRecord.put("id", generateRecordId(testRecord));
testRecord.put("kind", actualKind);
+ testRecord.put("id", generateRecordId(testRecord));
testRecord.put("legal", generateLegalTag());
String[] x_acl = {generateActualName(dataGroup,timeStamp)+"."+getEntitlementsDomain()};
Acl acl = Acl.builder().viewers(x_acl).owners(x_acl).build();
@@ -114,7 +114,7 @@ public class RecordSteps extends TestsBase {
}
protected String generateRecordId(Map<String, Object> testRecord) {
- return generateActualName(testRecord.get("id").toString(), timeStamp);
+ return generateActualId(testRecord.get("id").toString(), timeStamp, testRecord.get("kind").toString());
}
public void i_should_get_the_documents_for_the_in_the_Elastic_Search(int expectedCount, String index) throws Throwable {
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/SchemaServiceRecordSteps.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/SchemaServiceRecordSteps.java
index fb7a03e1e3866ae034e6b4cd032d57ba530ac09e..993c26b25b545bcac741b7de314e31b6bd8ed984 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/SchemaServiceRecordSteps.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/SchemaServiceRecordSteps.java
@@ -39,7 +39,7 @@ public class SchemaServiceRecordSteps extends RecordSteps {
@Override
protected String generateRecordId(Map<String, Object> testRecord) {
- return generateActualNameWithoutTs(testRecord.get("id").toString());
+ return generateActualIdWithoutTs(testRecord.get("id").toString(), testRecord.get("kind").toString());
}
@Override
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/TestsBase.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/TestsBase.java
index 53f9e877ae2389457a187d9adc58675dad358d3d..e4d6322f715941c1bc18d5fc82ee9b33ce9e2c97 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/TestsBase.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/TestsBase.java
@@ -130,6 +130,17 @@ public abstract class TestsBase {
return rawName.replaceAll("<timestamp>", timeStamp);
}
+ protected String generateActualId(String rawName, String timeStamp, String kind) {
+
+ rawName = generateActualName(rawName, timeStamp);
+
+ String kindSubType = kind.split(":")[2];
+
+ return rawName.replaceAll("<kindSubType>", kindSubType);
+
+ }
+
+
public String generateActualNameWithoutTs(String rawName) {
for (Map.Entry<String, String> tenant : tenantMap.entrySet()) {
rawName = rawName.replaceAll(tenant.getKey(), tenant.getValue());
@@ -137,6 +148,14 @@ public abstract class TestsBase {
return rawName.replaceAll("<timestamp>", "");
}
+ public String generateActualIdWithoutTs(String rawName, String kind) {
+ rawName = generateActualNameWithoutTs(rawName);
+
+ String kindSubType = kind.split(":")[2];
+
+ return rawName.replaceAll("<kindSubType>", kindSubType);
+ }
+
protected Legal generateLegalTag() {
Legal legal = new Legal();
Set<String> legalTags = new HashSet<>();
diff --git a/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature b/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
index 096f997f790e98f69559e074a549a77cb340d488..4152a53252458bd809ae9ffe5da447cf302060d7 100644
--- a/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
+++ b/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
@@ -14,9 +14,9 @@ Feature: Indexing of the documents
Then I should get the elastic <mapping> for the <type> and <index> in the Elastic Search
Examples:
- | kind | recordFile | number | index | type | acl | mapping |
- | "tenant1:indexer-int-test:sample-schema-1:1.0.4" | "index_records_schema_1" | 5 | "tenant1-indexer-int-test-sample-schema-1-1.0.4" | "sample-schema-1" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
- | "tenant1:indexer-int-test:sample-schema-3:1.0.4" | "index_records_schema_1" | 5 | "tenant1-indexer-int-test-sample-schema-3-1.0.4" | "sample-schema-3" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
+ | kind | recordFile | number | index | type | acl | mapping |
+ | "tenant1:indexer-int-test:sample-schema-1:1.0.4" | "index_records_1" | 5 | "tenant1-indexer-int-test-sample-schema-1-1.0.4" | "sample-schema-1" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
+ | "tenant1:indexer-int-test:sample-schema-3:1.0.4" | "index_records_1" | 5 | "tenant1-indexer-int-test-sample-schema-3-1.0.4" | "sample-schema-3" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text"},"Country":{"type":"text"},"County":{"type":"text"},"EmptyAttribute":{"type":"text"},"Established":{"type":"date"},"Field":{"type":"text"},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text"},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text"},"WellName":{"type":"text"},"WellStatus":{"type":"text"},"WellType":{"type":"text"},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
Scenario Outline: Ingest the record and Index in the Elastic Search with bad attribute
When I ingest records with the <recordFile> with <acl> for a given <kind>
diff --git a/testing/indexer-test-core/src/main/resources/testData/index_records_1.json b/testing/indexer-test-core/src/main/resources/testData/index_records_1.json
index 0d9f3c17ef24167affb538fc7db995317a77c348..81613de69df0524fa6d608ffc8eff55d08cfe1c3 100644
--- a/testing/indexer-test-core/src/main/resources/testData/index_records_1.json
+++ b/testing/indexer-test-core/src/main/resources/testData/index_records_1.json
@@ -1,6 +1,6 @@
[
{
- "id": "tenant1:ihs:testIngest2<timestamp>",
+ "id": "tenant1:<kindSubType>:testIngest2<timestamp>",
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
@@ -23,7 +23,7 @@
}
},
{
- "id": "tenant1:ihs:testIngest3<timestamp>",
+ "id": "tenant1:<kindSubType>:testIngest3<timestamp>",
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
@@ -46,7 +46,7 @@
}
},
{
- "id": "tenant1:ihs:testIngest4<timestamp>",
+ "id": "tenant1:<kindSubType>:testIngest4<timestamp>",
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
@@ -68,7 +68,7 @@
}
},
{
- "id": "tenant1:ihs:testIngest5<timestamp>",
+ "id": "tenant1:<kindSubType>:testIngest5<timestamp>",
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
@@ -90,7 +90,7 @@
}
},
{
- "id": "tenant1:ihs:testIngest6<timestamp>",
+ "id": "tenant1:<kindSubType>:testIngest6<timestamp>",
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
diff --git a/testing/indexer-test-core/src/main/resources/testData/index_records_2.json b/testing/indexer-test-core/src/main/resources/testData/index_records_2.json
index 9c35fcd6e58bcd19e01eef2b858ce19b7efcea71..ed11325951b0bc1194eb08ca0a9323358917bee2 100644
--- a/testing/indexer-test-core/src/main/resources/testData/index_records_2.json
+++ b/testing/indexer-test-core/src/main/resources/testData/index_records_2.json
@@ -1,6 +1,6 @@
[
{
- "id": "tenant1:ihs:testIngest7<timestamp>",
+ "id": "tenant1:<kindSubType>:testIngest7<timestamp>",
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
@@ -22,7 +22,7 @@
}
},
{
- "id": "tenant1:ihs:testIngest8<timestamp>",
+ "id": "tenant1:<kindSubType>:testIngest8<timestamp>",
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
@@ -44,7 +44,7 @@
}
},
{
- "id": "tenant1:ihs:testIngest9<timestamp>",
+ "id": "tenant1:<kindSubType>:testIngest9<timestamp>",
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
@@ -66,7 +66,7 @@
}
},
{
- "id": "tenant1:ihs:testIngest10<timestamp>",
+ "id": "tenant1:<kindSubType>:testIngest10<timestamp>",
"data": {
"Field": "OSDU OFFICE - 2",
"Location": {
@@ -88,7 +88,7 @@
}
},
{
- "id": "tenant1:ihs:testIngest11<timestamp>",
+ "id": "tenant1:<kindSubType>:testIngest11<timestamp>",
"data": {
"Field": 1234,
"Location": {
diff --git a/testing/indexer-test-core/src/main/resources/testData/index_records_3.json b/testing/indexer-test-core/src/main/resources/testData/index_records_3.json
index 84c5fcc11fcf96a68dba9c4d90ab06d9efa60c62..983e7bdb99dbe39246cc4ece07a9b3d434bddb5d 100644
--- a/testing/indexer-test-core/src/main/resources/testData/index_records_3.json
+++ b/testing/indexer-test-core/src/main/resources/testData/index_records_3.json
@@ -1,6 +1,6 @@
[
{
- "id": "tenant1:ihs:testIngest51<timestamp>",
+ "id": "tenant1:<kindSubType>:testIngest51<timestamp>",
"data": {
"WellName": "Data Platform Services - 51",
"GeoShape": {
@@ -13,7 +13,7 @@
}
},
{
- "id": "tenant1:ihs:testIngest52<timestamp>",
+ "id": "tenant1:<kindSubType>:testIngest52<timestamp>",
"data": {
"WellName": "Data Platform Services - 52",
"GeoShape": {
@@ -32,7 +32,7 @@
}
},
{
- "id": "tenant1:ihs:testIngest53<timestamp>",
+ "id": "tenant1:<kindSubType>:testIngest53<timestamp>",
"data": {
"WellName": "Data Platform Services - 53",
"GeoShape": {
@@ -147,7 +147,7 @@
}
},
{
- "id": "tenant1:ihs:testIngest54<timestamp>",
+ "id": "tenant1:<kindSubType>:testIngest54<timestamp>",
"data": {
"WellName": "Data Platform Services - 54",
"GeoShape": {
@@ -238,7 +238,7 @@
}
},
{
- "id": "tenant1:ihs:testIngest55<timestamp>",
+ "id": "tenant1:<kindSubType>:testIngest55<timestamp>",
"data": {
"WellName": "Data Platform Services - 55",
"GeoShape": {
@@ -271,7 +271,7 @@
}
},
{
- "id": "tenant1:ihs:testIngest56<timestamp>",
+ "id": "tenant1:<kindSubType>:testIngest56<timestamp>",
"data": {
"WellName": "Data Platform Services - 56",
"GeoShape": {
@@ -330,7 +330,7 @@
}
},
{
- "id": "tenant1:ihs:testIngest57<timestamp>",
+ "id": "tenant1:<kindSubType>:testIngest57<timestamp>",
"data": {
"WellName": "Data Platform Services - 57",
"GeoShape": {
diff --git a/testing/indexer-test-core/src/main/resources/testData/records_1.json b/testing/indexer-test-core/src/main/resources/testData/records_1.json
index 14d4b3c84ec863530213737e76cd8494501e825f..33c85b80d024481be7ae416ba9bf6865fef2aa4c 100644
--- a/testing/indexer-test-core/src/main/resources/testData/records_1.json
+++ b/testing/indexer-test-core/src/main/resources/testData/records_1.json
@@ -1,6 +1,6 @@
[
{
- "id": "test:well:1.0.0:1",
+ "id": "tenant1:well:1.0.0:1",
"kind": "<Kind>",
"namespace": "tenant1:testdatasource",
"type": "well",
@@ -38,7 +38,7 @@
}
},
{
- "id": "test:well:1.0.0:2",
+ "id": "tenant1:well:1.0.0:2",
"kind": "<Kind>",
"namespace": "tenant1:testdatasource",
"type": "well",
@@ -77,7 +77,7 @@
}
},
{
- "id": "test:well:1.0.0:3",
+ "id": "tenant1:well:1.0.0:3",
"kind": "<Kind>",
"namespace": "tenant1:testdatasource",
"type": "well",
diff --git a/testing/indexer-test-core/src/main/resources/testData/records_2.json b/testing/indexer-test-core/src/main/resources/testData/records_2.json
index 12f9ac646e579619b094aa19e571a863b8e23cac..9227d617e772aeb1d1fd0196b3540d04e9ca62ae 100644
--- a/testing/indexer-test-core/src/main/resources/testData/records_2.json
+++ b/testing/indexer-test-core/src/main/resources/testData/records_2.json
@@ -1,6 +1,6 @@
[
{
- "id": "test:well:2.0.0:1",
+ "id": "tenant1:well:2.0.0:1",
"kind": "<kind>",
"acl": "{viewers=[<Data-Group>], owners=[<Data-Group>]}",
"x-acl": [
@@ -37,7 +37,7 @@
}
},
{
- "id": "test:well:2.0.0:2",
+ "id": "tenant1:well:2.0.0:2",
"kind": "<kind>",
"acl": "{viewers=[<Data-Group>], owners=[<Data-Group>]}",
"x-acl": [
@@ -74,7 +74,7 @@
}
},
{
- "id": "test:well:2.0.0:3",
+ "id": "tenant1:well:2.0.0:3",
"kind": "<kind>",
"acl": "{viewers=[<Data-Group>], owners=[<Data-Group>]}",
"x-acl": [
diff --git a/testing/indexer-test-core/src/main/resources/testData/records_3.json b/testing/indexer-test-core/src/main/resources/testData/records_3.json
index a159d885c7a108d19c6831502192c93667323f2e..697644cc7d20c7a768d3cc55183a9d4a788e99cb 100644
--- a/testing/indexer-test-core/src/main/resources/testData/records_3.json
+++ b/testing/indexer-test-core/src/main/resources/testData/records_3.json
@@ -1,6 +1,6 @@
[
{
- "id": "test:well:1.0.0:1",
+ "id": "tenant1:well:1.0.0:1",
"kind": "<Kind>",
"namespace": "tenant1:testdatasource",
"type": "well",
@@ -39,7 +39,7 @@
}
},
{
- "id": "test:well:1.0.0:2",
+ "id": "tenant1:well:1.0.0:2",
"kind": "<Kind>",
"namespace": "tenant1:testdatasource",
"type": "well",
@@ -79,7 +79,7 @@
}
},
{
- "id": "test:well:1.0.0:3",
+ "id": "tenant1:well:1.0.0:3",
"kind": "<Kind>",
"namespace": "tenant1:testdatasource",
"type": "well",
diff --git a/testing/indexer-test-gcp/pom.xml b/testing/indexer-test-gcp/pom.xml
index 15ad0ca19399f6b6bb07b4eb4c9f41ad9abb0cb9..1ed50ebbbfac1e941fb09e41a8db562cbdcdd899 100644
--- a/testing/indexer-test-gcp/pom.xml
+++ b/testing/indexer-test-gcp/pom.xml
@@ -1,20 +1,18 @@
<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>indexer-test</artifactId>
- <version>0.0.5-SNAPSHOT</version>
+ <version>0.6.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-gcp</artifactId>
- <version>0.0.2-SNAPSHOT</version>
+ <version>0.6.0-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
@@ -39,7 +37,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
- <version>0.0.2-SNAPSHOT</version>
+ <version>0.6.0-SNAPSHOT</version>
</dependency>
<!-- Cucumber -->
diff --git a/testing/indexer-test-ibm/pom.xml b/testing/indexer-test-ibm/pom.xml
index edfe569b62f6f59ee25dd422170682f03ab2c68e..1d0ec6267671426190b4f425241b0993597c5f75 100644
--- a/testing/indexer-test-ibm/pom.xml
+++ b/testing/indexer-test-ibm/pom.xml
@@ -1,14 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>indexer-test</artifactId>
- <version>0.0.5-SNAPSHOT</version>
+ <version>0.6.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -40,7 +38,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
- <version>0.0.2-SNAPSHOT</version>
+ <version>0.6.0-SNAPSHOT</version>
</dependency>
<dependency>
diff --git a/testing/pom.xml b/testing/pom.xml
index 742fcdfc699312d137db006d325881da06ed2768..bcbc486166b0193225b59372060485413ea33eaa 100644
--- a/testing/pom.xml
+++ b/testing/pom.xml
@@ -14,12 +14,11 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.opengroup.osdu</groupId>
<artifactId>indexer-test</artifactId>
- <version>0.0.5-SNAPSHOT</version>
+ <version>0.6.0-SNAPSHOT</version>
<description>Indexer Service Integration Test Root Project</description>
<properties>
<spring.version>5.1.19.RELEASE</spring.version>