diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingService.java
index a411ffa08401af8239697c126d0972197d722e59..efbda282b271ebef8814a293c4751b4ecc81953e 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingService.java
@@ -29,4 +29,6 @@ public interface IndexerMappingService extends IMappingService {
Map<String, Object> getIndexMappingFromRecordSchema(IndexSchema schema);
void updateIndexMappingForIndicesOfSameType(Set<String> indices, String fieldName) throws Exception;
+
+ void syncIndexMappingIfRequired(RestHighLevelClient restClient, String index) throws Exception;
}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
index 79ad1bc07b45587ba94a4afec20da1cb6b2f711f..1bfda7e9797a8ed1105d9006d2784432d9139a07 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
@@ -15,12 +15,8 @@
package org.opengroup.osdu.indexer.service;
import com.google.gson.Gson;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-import javax.inject.Inject;
+import com.google.gson.reflect.TypeToken;
+import com.lambdaworks.redis.RedisException;
import org.apache.http.HttpStatus;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
@@ -39,11 +35,19 @@ import org.opengroup.osdu.core.common.Constants;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
+import org.opengroup.osdu.core.common.provider.interfaces.IIndexCache;
+import org.opengroup.osdu.core.common.search.IMappingService;
import org.opengroup.osdu.core.common.search.Preconditions;
import org.opengroup.osdu.indexer.util.ElasticClientHandler;
import org.opengroup.osdu.indexer.util.TypeMapper;
+import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
+import javax.inject.Inject;
+import java.io.IOException;
+import java.lang.reflect.Type;
+import java.util.*;
+
@Service
public class IndexerMappingServiceImpl extends MappingServiceImpl implements IndexerMappingService {
@@ -51,7 +55,12 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
private JaxRsDpsLog log;
@Inject
private ElasticClientHandler elasticClientHandler;
- private TimeValue REQUEST_TIMEOUT = TimeValue.timeValueMinutes(1);
+ @Autowired
+ private IIndexCache indexCache;
+ @Autowired
+ private IMappingService mappingService;
+
+ private static TimeValue REQUEST_TIMEOUT = TimeValue.timeValueMinutes(1);
/**
@@ -149,6 +158,53 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
}
}
+ @Override
+ public void syncIndexMappingIfRequired(RestHighLevelClient restClient, String index) throws Exception {
+ final String cacheKey = String.format("metaAttributeMappingSynced-%s", index);
+
+ try {
+ Boolean mappingSynced = (Boolean) this.indexCache.get(cacheKey);
+ if (mappingSynced != null && mappingSynced) return;
+ } catch (RedisException ex) {
+ //In case the format of cache changes then clean the cache
+ this.indexCache.delete(cacheKey);
+ }
+
+ String jsonResponse = this.mappingService.getIndexMapping(restClient, index);
+ Type type = new TypeToken<Map<String, Object>>() {}.getType();
+ Map<String, Object> mappings = new Gson().fromJson(jsonResponse, type);
+
+ if (mappings == null || mappings.isEmpty()) return;
+
+ Map<String, Object> props = (Map<String, Object>) mappings.get("properties");
+
+ if (props == null || props.isEmpty()) return;
+
+ List<String> missing = new ArrayList<>();
+ for (String attribute : TypeMapper.getMetaAttributesKeys()) {
+ if (props.containsKey(attribute)) continue;
+ missing.add(attribute);
+ }
+
+ if (missing.isEmpty()) {
+ this.indexCache.put(cacheKey, true);
+ return;
+ }
+
+ Map<String, Object> properties = new HashMap<>();
+ for (String attribute : missing) {
+ properties.put(attribute, TypeMapper.getMetaAttributeIndexerMapping(attribute));
+ }
+
+ Map<String, Object> documentMapping = new HashMap<>();
+ documentMapping.put(Constants.PROPERTIES, properties);
+
+ String mapping = new Gson().toJson(documentMapping, Map.class);
+ this.createMappingWithJson(restClient, index, "_doc", mapping, true);
+
+ this.indexCache.put(cacheKey, true);
+ }
+
private boolean updateMappingToEnableKeywordIndexingForField(RestHighLevelClient client, Set<String> indicesSet, String fieldName) throws IOException {
String[] indices = indicesSet.toArray(new String[indicesSet.size()]);
Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>>> indexMappingMap = getIndexFieldMap(new String[]{"data."+fieldName}, client, indices);
@@ -186,7 +242,7 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error", "Error getting indices.", String.format("Failed to get indices error: %s", Arrays.toString(indices)));
}
}
-
+
private boolean updateMappingForAllIndicesOfSameTypeToEnableKeywordIndexingForField(
RestHighLevelClient client, String index, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetadata>> indexMapping, String fieldName) throws IOException {
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
index fed2822af6d993b407e4d7502f0685d35e42a184..36d0bcdcf6b4a4fb3af176a6b742a1035e6cd1f5 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
@@ -365,8 +365,11 @@ public class IndexerServiceImpl implements IndexerService {
for (IndexSchema schema : schemas) {
String index = this.elasticIndexNameResolver.getIndexNameFromKind(schema.getKind());
- // check if index exist
- if (this.indicesService.isIndexExist(restClient, index)) continue;
+ // check if index exist and sync meta attribute schema if required
+ if (this.indicesService.isIndexExist(restClient, index)) {
+ this.mappingService.syncIndexMappingIfRequired(restClient, index);
+ continue;
+ }
// create index
Map<String, Object> mapping = this.mappingService.getIndexMappingFromRecordSchema(schema);
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java
index ec68f6cccbdc88ea0f8a605c90f301fba53e56bc..bd3a90009edc62e105323b2586678083cc094267 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java
@@ -62,7 +62,7 @@ public class IndicesServiceImpl implements IndicesService {
@Autowired
private ElasticIndexNameResolver elasticIndexNameResolver;
@Autowired
- private IIndexCache indicesExistCache;
+ private IIndexCache indexCache;
@Autowired
private JaxRsDpsLog log;
@@ -99,14 +99,14 @@ public class IndicesServiceImpl implements IndicesService {
CreateIndexResponse response = client.indices().create(request, RequestOptions.DEFAULT);
// cache the index status
boolean indexStatus = response.isAcknowledged() && response.isShardsAcknowledged();
- if (indexStatus) this.indicesExistCache.put(index, true);
+ if (indexStatus) this.indexCache.put(index, true);
return indexStatus;
} catch (ElasticsearchStatusException e) {
if (e.status() == RestStatus.BAD_REQUEST && (e.getMessage().contains("resource_already_exists_exception"))) {
log.info("Index already exists. Ignoring error...");
// cache the index status
- this.indicesExistCache.put(index, true);
+ this.indexCache.put(index, true);
return true;
}
throw e;
@@ -123,15 +123,15 @@ public class IndicesServiceImpl implements IndicesService {
public boolean isIndexExist(RestHighLevelClient client, String index) throws IOException {
try {
try {
- Boolean isIndexExist = (Boolean) this.indicesExistCache.get(index);
+ Boolean isIndexExist = (Boolean) this.indexCache.get(index);
if (isIndexExist != null && isIndexExist) return true;
} catch (RedisException ex) {
//In case the format of cache changes then clean the cache
- this.indicesExistCache.delete(index);
+ this.indexCache.delete(index);
}
GetIndexRequest request = new GetIndexRequest(index);
boolean exists = client.indices().exists(request, RequestOptions.DEFAULT);
- if (exists) this.indicesExistCache.put(index, true);
+ if (exists) this.indexCache.put(index, true);
return exists;
} catch (ElasticsearchException exception) {
if (exception.status() == RestStatus.NOT_FOUND) return false;
@@ -152,7 +152,7 @@ public class IndicesServiceImpl implements IndicesService {
public boolean deleteIndex(RestHighLevelClient client, String index) throws ElasticsearchException, IOException, AppException {
boolean responseStatus = removeIndexInElasticsearch(client, index);
if (responseStatus) {
- this.indicesExistCache.delete(index);
+ this.clearCacheOnIndexDeletion(index);
}
return responseStatus;
}
@@ -225,4 +225,10 @@ public class IndicesServiceImpl implements IndicesService {
final Type typeOf = new TypeToken<List<IndexInfo>>() {}.getType();
return new Gson().fromJson(str, typeOf);
}
+
+ private void clearCacheOnIndexDeletion(String index) {
+ final String syncCacheKey = String.format("metaAttributeMappingSynced-%s", index);
+ this.indexCache.delete(index);
+ this.indexCache.delete(syncCacheKey);
+ }
}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/MappingServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/MappingServiceImpl.java
index 3f067f74ef2e7cf7762f196d428fd7b4f889fed2..b9a6180cc63273dd3cc459bed7c001291a99830b 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/MappingServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/MappingServiceImpl.java
@@ -19,10 +19,11 @@ import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.GetMappingsRequest;
import org.elasticsearch.client.indices.GetMappingsResponse;
+import org.elasticsearch.common.unit.TimeValue;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
-import org.opengroup.osdu.core.common.search.IndicesService;
import org.opengroup.osdu.core.common.search.IMappingService;
+import org.opengroup.osdu.core.common.search.IndicesService;
import org.opengroup.osdu.core.common.search.Preconditions;
import org.opengroup.osdu.indexer.util.ElasticClientHandler;
import org.springframework.beans.factory.annotation.Autowired;
@@ -43,7 +44,7 @@ public class MappingServiceImpl implements IMappingService {
@Autowired
private ElasticIndexNameResolver elasticIndexNameResolver;
-// private TimeValue REQUEST_TIMEOUT = TimeValue.timeValueMinutes(1);
+ private static TimeValue REQUEST_TIMEOUT = TimeValue.timeValueMinutes(1);
/*
* Get index schema
@@ -82,10 +83,9 @@ public class MappingServiceImpl implements IMappingService {
try {
GetMappingsRequest request = new GetMappingsRequest();
request.indices(index);
- // TODO: enable this once server is migrated > v6.6.2
- // request.masterNodeTimeout(REQUEST_TIMEOUT);
+ request.setTimeout(REQUEST_TIMEOUT);
GetMappingsResponse response = client.indices().getMapping(request, RequestOptions.DEFAULT);
- return response.toString();
+ return response.mappings().get(index).getSourceAsMap().toString();
} catch (IOException e) {
throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Unknown error", String.format("Error retrieving mapping for kind %s", this.elasticIndexNameResolver.getKindFromIndexName(index)), e);
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java
index ef3cc74bbd03cd6bd673d1b5912adebd356a7603..bd48202110fae6ebb501e6b9de7a2e7ce61e1b63 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java
@@ -14,8 +14,10 @@
package org.opengroup.osdu.indexer.util;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
+import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.opengroup.osdu.core.common.Constants;
import org.opengroup.osdu.core.common.model.entitlements.AclRole;
@@ -89,6 +91,10 @@ public class TypeMapper {
return metaAttributeIndexerType.getOrDefault(attribute.getValue(), null);
}
+ public static List<String> getMetaAttributesKeys() {
+ return new ArrayList<>(metaAttributeIndexerType.keySet());
+ }
+
public static Object getMetaAttributeIndexerMapping(String key) {
if (key.equals(RecordMetaAttribute.ACL.getValue())
|| key.equals(RecordMetaAttribute.LEGAL.getValue()) || key.equals(RecordMetaAttribute.ANCESTRY.getValue()) || key.equals(RecordMetaAttribute.INDEX_STATUS.getValue())) {
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java
index 6254bbc238503ba36abea7d26869a2c2797e6766..a7dca6b6897f5900a9fd6200146a7afc52774ec0 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java
@@ -27,6 +27,8 @@ import org.mockito.Mock;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
+import org.opengroup.osdu.core.common.provider.interfaces.IIndexCache;
+import org.opengroup.osdu.core.common.search.IMappingService;
import org.opengroup.osdu.indexer.util.ElasticClientHandler;
import org.opengroup.osdu.indexer.util.TypeMapper;
import org.powermock.api.mockito.PowerMockito;
@@ -41,6 +43,9 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.MockitoAnnotations.initMocks;
import static org.powermock.api.mockito.PowerMockito.spy;
import static org.powermock.api.mockito.PowerMockito.when;
@@ -65,6 +70,10 @@ public class IndexerMappingServiceTest {
private JaxRsDpsLog log;
@Mock
private ElasticClientHandler elasticClientHandler;
+ @Mock
+ private IIndexCache indexCache;
+ @Mock
+ private IMappingService mappingService;
@InjectMocks
private IndexerMappingServiceImpl sut;
@@ -163,4 +172,43 @@ public class IndexerMappingServiceTest {
fail("Should not throw this exception" + e.getMessage());
}
}
+
+ @Test
+ public void should_returnCachedStatus_givenUpdatedIndex() throws Exception {
+ final String cacheKey = String.format("metaAttributeMappingSynced-%s", index);
+ when(this.indexCache.get(cacheKey)).thenReturn(true);
+
+ this.sut.syncIndexMappingIfRequired(restHighLevelClient, index);
+
+ verifyNoMoreInteractions(this.mappingService);
+ }
+
+ @Test
+ public void should_applyNoUpdate_givenUpdateIndex() throws Exception {
+ final String cacheKey = String.format("metaAttributeMappingSynced-%s", index);
+ final String mapping = "{\"dynamic\":\"false\",\"properties\":{\"acl\":{\"properties\":{\"owners\":{\"type\":\"keyword\"},\"viewers\":{\"type\":\"keyword\"}}},\"ancestry\":{\"properties\":{\"parents\":{\"type\":\"keyword\"}}},\"authority\":{\"type\":\"constant_keyword\",\"value\":\"opendes\"},\"createTime\":{\"type\":\"date\"},\"createUser\":{\"type\":\"keyword\"},\"data\":{\"properties\":{\"message\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"null_value\":\"null\",\"ignore_above\":256}}}}},\"id\":{\"type\":\"keyword\"},\"index\":{\"properties\":{\"lastUpdateTime\":{\"type\":\"date\"},\"statusCode\":{\"type\":\"integer\"},\"trace\":{\"type\":\"text\"}}},\"kind\":{\"type\":\"keyword\"},\"legal\":{\"properties\":{\"legaltags\":{\"type\":\"keyword\"},\"otherRelevantDataCountries\":{\"type\":\"keyword\"},\"status\":{\"type\":\"keyword\"}}},\"modifyTime\":{\"type\":\"date\"},\"modifyUser\":{\"type\":\"keyword\"},\"namespace\":{\"type\":\"keyword\"},\"source\":{\"type\":\"constant_keyword\",\"value\":\"test\"},\"tags\":{\"type\":\"flattened\"},\"type\":{\"type\":\"keyword\"},\"version\":{\"type\":\"long\"},\"x-acl\":{\"type\":\"keyword\"}}}";
+ when(this.mappingService.getIndexMapping(restHighLevelClient, index)).thenReturn(mapping);
+
+ this.sut.syncIndexMappingIfRequired(restHighLevelClient, index);
+
+ verify(this.indexCache, times(1)).get(cacheKey);
+ verify(this.indexCache, times(1)).put(cacheKey, true);
+ }
+
+ @Test
+ public void should_applyUpdate_givenExistingIndex() throws Exception {
+ final String cacheKey = String.format("metaAttributeMappingSynced-%s", index);
+ final String mapping = "{\"dynamic\":\"false\",\"properties\":{\"acl\":{\"properties\":{\"owners\":{\"type\":\"keyword\"},\"viewers\":{\"type\":\"keyword\"}}},\"ancestry\":{\"properties\":{\"parents\":{\"type\":\"keyword\"}}},\"data\":{\"properties\":{\"message\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"null_value\":\"null\",\"ignore_above\":256}}}}},\"id\":{\"type\":\"keyword\"},\"index\":{\"properties\":{\"lastUpdateTime\":{\"type\":\"date\"},\"statusCode\":{\"type\":\"integer\"},\"trace\":{\"type\":\"text\"}}},\"kind\":{\"type\":\"keyword\"},\"legal\":{\"properties\":{\"legaltags\":{\"type\":\"keyword\"},\"otherRelevantDataCountries\":{\"type\":\"keyword\"},\"status\":{\"type\":\"keyword\"}}},\"namespace\":{\"type\":\"keyword\"},\"tags\":{\"type\":\"flattened\"},\"type\":{\"type\":\"keyword\"},\"version\":{\"type\":\"long\"},\"x-acl\":{\"type\":\"keyword\"}}}";
+ when(this.mappingService.getIndexMapping(restHighLevelClient, index)).thenReturn(mapping);
+
+ AcknowledgedResponse mappingResponse = new AcknowledgedResponse(true);
+ doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ doReturn(mappingResponse).when(this.indicesClient).putMapping(any(PutMappingRequest.class), any(RequestOptions.class));
+
+ this.sut.syncIndexMappingIfRequired(restHighLevelClient, index);
+
+ verify(this.indexCache, times(1)).get(cacheKey);
+ verify(this.indexCache, times(1)).put(cacheKey, true);
+ verify(this.indicesClient, times(1)).putMapping(any(PutMappingRequest.class), any(RequestOptions.class));
+ }
}
\ No newline at end of file
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/TypeMapperTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/TypeMapperTest.java
index 6880026132541a16c108b2b90efa14c6a0c13395..6dbc2206dd4a482c3d93731ded079c0ececf8451 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/TypeMapperTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/TypeMapperTest.java
@@ -5,7 +5,10 @@ import org.opengroup.osdu.core.common.model.indexer.ElasticType;
import org.opengroup.osdu.core.common.model.indexer.StorageType;
import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
+import java.util.List;
+
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
public class TypeMapperTest {
@@ -38,4 +41,14 @@ public class TypeMapperTest {
assertEquals(ElasticType.DATE.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.CREATE_TIME));
assertEquals(ElasticType.DATE.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.MODIFY_TIME));
}
+
+ @Test
+ public void validate_meta_attributes() {
+ List<String> keys = TypeMapper.getMetaAttributesKeys();
+
+ String[] meta = new String[] {"id", "kind", "authority", "source", "namespace", "type", "version", "acl", "tags", "legal", "ancestry", "createUser", "modifyUser", "createTime", "modifyTime", "index"};
+ for(String attributeKey : meta) {
+ assertTrue(keys.contains(attributeKey));
+ }
+ }
}
diff --git a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
index 01691b4f8b53f1d0db827844231553a969da8529..898fe91989c407fd4ced01549bd5726bc58dc8fd 100644
--- a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
+++ b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -80,6 +80,11 @@ public class Steps extends SchemaServiceRecordSteps {
super.i_ingest_records_with_the_for_a_given(record, dataGroup, kind);
}
+ @When("^I create index with \"(.*?)\" for a given \"(.*?)\" and \"(.*?)\"$")
+ public void i_create_index_with_mapping_file_for_a_given_kind(String mappingFile, String index, String kind) throws Throwable {
+ super.i_create_index_with_mapping_file_for_a_given_kind(mappingFile, index, kind);
+ }
+
@Then("^I should get the (\\d+) documents for the \"([^\"]*)\" in the Elastic Search$")
public void i_should_get_the_documents_for_the_in_the_Elastic_Search(int expectedCount, String index) throws Throwable {
super.i_should_get_the_documents_for_the_in_the_Elastic_Search(expectedCount, index);
diff --git a/testing/indexer-test-azure/pom.xml b/testing/indexer-test-azure/pom.xml
index 8c615f6736f2a036c24e9bc171614292639e5340..e7f45c911a2ef5da6119561b7fa36a83dbe3d530 100644
--- a/testing/indexer-test-azure/pom.xml
+++ b/testing/indexer-test-azure/pom.xml
@@ -36,7 +36,7 @@
<cucumber.version>1.2.5</cucumber.version>
<junit.jupiter.version>5.6.0</junit.jupiter.version>
<elasticsearch.version>7.8.1</elasticsearch.version>
- <os-core-common.version>0.11.0</os-core-common.version>
+ <os-core-common.version>0.12.0-rc1</os-core-common.version>
</properties>
<dependencies>
diff --git a/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
index 5c0f44c22eff61886aba89ecb1693b7a73856999..41354dfe5492f750dad653e0a1a1510db5e11ec7 100644
--- a/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
+++ b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -48,6 +48,11 @@ public class Steps extends SchemaServiceRecordSteps {
super.i_ingest_records_with_the_for_a_given(record, dataGroup, kind);
}
+ @When("^I create index with \"(.*?)\" for a given \"(.*?)\" and \"(.*?)\"$")
+ public void i_create_index_with_mapping_file_for_a_given_kind(String mappingFile, String index, String kind) throws Throwable {
+ super.i_create_index_with_mapping_file_for_a_given_kind(mappingFile, index, kind);
+ }
+
@Then("^I should get the (\\d+) documents for the \"([^\"]*)\" in the Elastic Search$")
public void i_should_get_the_documents_for_the_in_the_Elastic_Search(int expectedCount, String index) throws Throwable {
super.i_should_get_the_documents_for_the_in_the_Elastic_Search(expectedCount, index);
diff --git a/testing/indexer-test-core/pom.xml b/testing/indexer-test-core/pom.xml
index e35d1b692f4442f1078e3c7e8625bdad7cd49674..808fab5f5312b64e6083a900189a9ca490560dbf 100644
--- a/testing/indexer-test-core/pom.xml
+++ b/testing/indexer-test-core/pom.xml
@@ -17,7 +17,7 @@
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
<cucumber.version>1.2.5</cucumber.version>
- <os-core-common.version>0.11.0</os-core-common.version>
+ <os-core-common.version>0.12.0-rc1</os-core-common.version>
</properties>
<dependencies>
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
index d78985d0d2992c41bb3707cf4ee92414045d58bd..969c9c6ece88f270bed27a8debc1b40e8d36a798 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
@@ -152,6 +152,7 @@ public class RecordSteps extends TestsBase {
public void i_can_validate_indexed_attributes(String index, String kind) throws Throwable {
String authority = tenantMap.get(kind.substring(0, kind.indexOf(":")));
index = generateActualName(index, timeStamp);
+ long numOfIndexedDocuments = createIndex(index);
List<Map<String, Object>> hits = elasticUtils.fetchRecordsByAttribute(index, "authority", authority);
assertTrue(hits.size() > 0);
@@ -218,6 +219,14 @@ public class RecordSteps extends TestsBase {
assertEquals(expectedRecordData.getData().get(objectField),actualRecordData.getData().get(objectField));
}
+ public void i_create_index_with_mapping_file_for_a_given_kind(String mappingFile, String index, String kind) throws Throwable {
+ String actualKind = generateActualName(kind, timeStamp);
+ TestIndex testIndex = getInputIndexMap().get(actualKind);
+ testIndex.setMappingFile(mappingFile);
+ this.getInputIndexMap().put(actualKind, testIndex);
+ testIndex.addIndex();
+ }
+
private long createIndex(String index) throws InterruptedException, IOException {
long numOfIndexedDocuments = 0;
int iterator;
diff --git a/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature b/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
index 0888f9528b60901a73d69862039479726a0648a7..c7b17f6bb6f01f1c3ff732ed6ead34f943e178ba 100644
--- a/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
+++ b/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
@@ -9,6 +9,7 @@ Feature: Indexing of the documents
| tenant1:indexer:test-data--Integration:3.0.1 | tenant1-indexer-test-data--integration-3.0.1 | index_records_3 |
| tenant1:wks:master-data--Wellbore:2.0.3 | tenant1-wks-master-data--wellbore-2.0.3 | r3-index_record_wks_master |
| tenant1:wks:ArraysOfObjectsTestCollection:4.0.0 | tenant1-wks-arraysofobjectstestcollection-4.0.0 | r3-index_record_arrayofobjects |
+ | tenant1:indexer:test-mapping--Sync:1.0.0 | tenant1-indexer-test-mapping--sync-1.0.0 | index_record_sync_mapping |
Scenario Outline: Ingest the record and Index in the Elastic Search
When I ingest records with the <recordFile> with <acl> for a given <kind>
@@ -17,7 +18,7 @@ Feature: Indexing of the documents
Then I can validate indexed meta attributes for the <index> and given <kind>
Examples:
- | kind | recordFile | number | index | acl | mapping |
+ | kind | recordFile | number | index | acl | mapping |
| "tenant1:indexer:test-data--Integration:1.0.1" | "index_records_schema_1" | 5 | "tenant1-indexer-test-data--integration-1.0.1" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Country":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"County":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"EmptyAttribute":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Established":{"type":"date"},"Field":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellName":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellStatus":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellType":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"authority":{"type":"constant_keyword","value":"<authority-id>"},"createTime":{"type":"date"},"createUser":{"type":"keyword"},"modifyTime":{"type":"date"},"modifyUser":{"type":"keyword"},"source":{"type":"constant_keyword","value":"<source-id>"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
| "tenant1:indexer:test-data--Integration:3.0.1" | "index_records_schema_3" | 7 | "tenant1-indexer-test-data--integration-3.0.1" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Country":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"County":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"EmptyAttribute":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Established":{"type":"date"},"Field":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellName":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellStatus":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellType":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"authority":{"type":"constant_keyword","value":"<authority-id>"},"createTime":{"type":"date"},"createUser":{"type":"keyword"},"modifyTime":{"type":"date"},"modifyUser":{"type":"keyword"},"source":{"type":"constant_keyword","value":"<source-id>"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
@@ -54,4 +55,13 @@ Feature: Indexing of the documents
Examples:
| kind | recordFile | number | index | acl | path | first_nested_field | first_nested_value | second_nested_field | second_nested_value | flattened_inner_field | flattened_inner_value | object_inner_field |
- | "tenant1:wks:ArraysOfObjectsTestCollection:4.0.0" | "r3-index_record_arrayofobjects" | 1 | "tenant1-wks-arraysofobjectstestcollection-4.0.0" | "data.default.viewers@tenant1" | "data.NestedTest" | "data.NestedTest.NumberTest" | 12345 | "data.NestedTest.StringTest" | "test string" | "data.FlattenedTest.StringTest" | "test string" | "ObjectTest" |
\ No newline at end of file
+ | "tenant1:wks:ArraysOfObjectsTestCollection:4.0.0" | "r3-index_record_arrayofobjects" | 1 | "tenant1-wks-arraysofobjectstestcollection-4.0.0" | "data.default.viewers@tenant1" | "data.NestedTest" | "data.NestedTest.NumberTest" | 12345 | "data.NestedTest.StringTest" | "test string" | "data.FlattenedTest.StringTest" | "test string" | "ObjectTest" |
+
+ Scenario Outline: Synchronize meta attribute mapping on existing indexed kind
+ When I create index with <mappingFile> for a given <index> and <kind>
+ And I ingest records with the <recordFile> with <acl> for a given <kind>
+ Then I can validate indexed meta attributes for the <index> and given <kind>
+
+ Examples:
+ | kind | index | recordFile | mappingFile | acl |
+ | "tenant1:indexer:test-mapping--Sync:1.0.0" | "tenant1-indexer-test-mapping--sync-1.0.0" | "index_record_sync_mapping" | "index_record_sync_mapping" | "data.default.viewers@tenant1" |
\ No newline at end of file
diff --git a/testing/indexer-test-core/src/main/resources/testData/index_record_sync_mapping.json b/testing/indexer-test-core/src/main/resources/testData/index_record_sync_mapping.json
new file mode 100644
index 0000000000000000000000000000000000000000..a3c05640cb17a84a09f49b2fe0e0ea82410aaab9
--- /dev/null
+++ b/testing/indexer-test-core/src/main/resources/testData/index_record_sync_mapping.json
@@ -0,0 +1,10 @@
+[
+ {
+ "id": "tenant1:<kindSubType>:testMappingSync<timestamp>",
+ "data": {
+ "FullName": "The Open Group",
+ "Address": "Burlington, MA",
+ "Phone": "781-564-9200"
+ }
+ }
+]
\ No newline at end of file
diff --git a/testing/indexer-test-core/src/main/resources/testData/index_record_sync_mapping.mapping b/testing/indexer-test-core/src/main/resources/testData/index_record_sync_mapping.mapping
new file mode 100644
index 0000000000000000000000000000000000000000..e75c9ffe9841b391b140519de3500cdaacf3911c
--- /dev/null
+++ b/testing/indexer-test-core/src/main/resources/testData/index_record_sync_mapping.mapping
@@ -0,0 +1,103 @@
+{
+ "dynamic": "false",
+ "properties": {
+ "acl": {
+ "properties": {
+ "owners": {
+ "type": "keyword"
+ },
+ "viewers": {
+ "type": "keyword"
+ }
+ }
+ },
+ "ancestry": {
+ "properties": {
+ "parents": {
+ "type": "keyword"
+ }
+ }
+ },
+ "data": {
+ "properties": {
+ "FullName": {
+ "type": "text",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "null_value": "null",
+ "ignore_above": 256
+ }
+ }
+ },
+ "Address": {
+ "type": "text",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "null_value": "null",
+ "ignore_above": 256
+ }
+ }
+ },
+ "Phone": {
+ "type": "text",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "null_value": "null",
+ "ignore_above": 256
+ }
+ }
+ }
+ }
+ },
+ "id": {
+ "type": "keyword"
+ },
+ "index": {
+ "properties": {
+ "lastUpdateTime": {
+ "type": "date"
+ },
+ "statusCode": {
+ "type": "integer"
+ },
+ "trace": {
+ "type": "text"
+ }
+ }
+ },
+ "kind": {
+ "type": "keyword"
+ },
+ "legal": {
+ "properties": {
+ "legaltags": {
+ "type": "keyword"
+ },
+ "otherRelevantDataCountries": {
+ "type": "keyword"
+ },
+ "status": {
+ "type": "keyword"
+ }
+ }
+ },
+ "namespace": {
+ "type": "keyword"
+ },
+ "tags": {
+ "type": "flattened"
+ },
+ "type": {
+ "type": "keyword"
+ },
+ "version": {
+ "type": "long"
+ },
+ "x-acl": {
+ "type": "keyword"
+ }
+ }
+}
\ No newline at end of file
diff --git a/testing/indexer-test-core/src/main/resources/testData/index_record_sync_mapping.schema.json b/testing/indexer-test-core/src/main/resources/testData/index_record_sync_mapping.schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..d1b9a4e6e5f46946bb2ad8d0af29d917064c689e
--- /dev/null
+++ b/testing/indexer-test-core/src/main/resources/testData/index_record_sync_mapping.schema.json
@@ -0,0 +1,35 @@
+{
+ "schemaInfo": {
+ "schemaIdentity": {
+ "authority": "tenant1",
+ "source": "indexer",
+ "entityType": "test-mapping--Sync",
+ "schemaVersionMajor": "1",
+ "schemaVersionMinor": "0",
+ "schemaVersionPatch": "0"
+ },
+ "status": "DEVELOPMENT"
+ },
+ "schema": {
+ "properties": {
+ "data": {
+ "allOf": [
+ {
+ "type": "object",
+ "properties": {
+ "FullName": {
+ "type": "string"
+ },
+ "Address": {
+ "type": "string"
+ },
+ "Phone": {
+ "type": "string"
+ }
+ }
+ }
+ ]
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
index 0409e9a21c866f5e3e81aad5af72ae01e56b88c0..991818c68099e24165b3d35cbb017e25a65323f0 100644
--- a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
+++ b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -36,6 +36,11 @@ public class Steps extends SchemaServiceRecordSteps {
super.i_ingest_records_with_the_for_a_given(record, dataGroup, kind);
}
+ @When("^I create index with \"(.*?)\" for a given \"(.*?)\" and \"(.*?)\"$")
+ public void i_create_index_with_mapping_file_for_a_given_kind(String mappingFile, String index, String kind) throws Throwable {
+ super.i_create_index_with_mapping_file_for_a_given_kind(mappingFile, index, kind);
+ }
+
@Then("^I should get the (\\d+) documents for the \"([^\"]*)\" in the Elastic Search$")
public void i_should_get_the_documents_for_the_in_the_Elastic_Search(int expectedCount, String index) throws Throwable {
super.i_should_get_the_documents_for_the_in_the_Elastic_Search(expectedCount, index);
diff --git a/testing/indexer-test-ibm/pom.xml b/testing/indexer-test-ibm/pom.xml
index 36aabbea47d42d2e760904e3cc9ac2109b9d1e62..b0ad772a3f3e2ada3fdad4496782e6206d880946 100644
--- a/testing/indexer-test-ibm/pom.xml
+++ b/testing/indexer-test-ibm/pom.xml
@@ -44,7 +44,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
- <version>0.3.8</version>
+ <version>0.12.0-rc1</version>
</dependency>
<dependency>
diff --git a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
index 3ec2f0033a5cabb1eba90983c1fc3c4692094e53..c6c8662a1f6951d90cfafe2075c1565f74b286df 100644
--- a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
+++ b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -53,6 +53,11 @@ public class Steps extends SchemaServiceRecordSteps {
super.i_ingest_records_with_the_for_a_given(record, dataGroup, kind);
}
+ @When("^I create index with \"(.*?)\" for a given \"(.*?)\" and \"(.*?)\"$")
+ public void i_create_index_with_mapping_file_for_a_given_kind(String mappingFile, String index, String kind) throws Throwable {
+ super.i_create_index_with_mapping_file_for_a_given_kind(mappingFile, index, kind);
+ }
+
@Then("^I should get the (\\d+) documents for the \"([^\"]*)\" in the Elastic Search$")
public void i_should_get_the_documents_for_the_in_the_Elastic_Search(int expectedCount, String index) throws Throwable {
super.i_should_get_the_documents_for_the_in_the_Elastic_Search(expectedCount, index);
@@ -138,7 +143,7 @@ public class Steps extends SchemaServiceRecordSteps {
HttpRequest rq = HttpRequest.post(recordChangeMessage).url(url).headers(dpsHeaders.getHeaders()).build();
HttpResponse result = httpClient.send(rq);
- if(result.hasException().equals(false) && result.getResponseCode() == 500) {
+ if(result.hasException() == false && result.getResponseCode() == 500) {
assertTrue(result.getResponseCode() == 500);
} else {
AppError error = gson.fromJson(result.getBody(), AppError.class);