From 1e01b74ba005976d609727b359ef9001235d8ae4 Mon Sep 17 00:00:00 2001
From: ZMai <zmai@slb.com>
Date: Mon, 25 Sep 2023 21:50:37 -0500
Subject: [PATCH] Enhance augmenter to support flexible matching
---
.../indexproperty/PropertyConfiguration.java | 12 +-
.../indexproperty/PropertyConfigurations.java | 4 +-
.../model/indexproperty/RelatedCondition.java | 38 ++-
.../indexproperty/RelatedObjectsSpec.java | 8 +-
.../model/indexproperty/ValueExtraction.java | 4 +-
.../PropertyConfigurationsServiceImpl.java | 267 ++++++++++--------
.../indexer/service/SearchServiceImpl.java | 7 +-
.../osdu/indexer/util/PropertyUtil.java | 47 +++
.../indexproperty/RelatedObjectsSpecTest.java | 4 +-
.../indexproperty/ValueExtractionTest.java | 14 +-
...PropertyConfigurationsServiceImplTest.java | 4 +-
.../osdu/indexer/util/PropertyUtilTest.java | 49 ++++
.../well_configuration_record.json | 12 +-
...wks_IndexPropertyPathConfiguration_v1.json | 10 +-
14 files changed, 332 insertions(+), 148 deletions(-)
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/PropertyConfiguration.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/PropertyConfiguration.java
index de4e53f7a..289fd6252 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/PropertyConfiguration.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/PropertyConfiguration.java
@@ -19,6 +19,7 @@ import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data;
import lombok.ToString;
+import org.opengroup.osdu.indexer.util.PropertyUtil;
import java.util.List;
@@ -41,6 +42,10 @@ public class PropertyConfiguration {
@JsonProperty("Paths")
private List<PropertyPath> paths;
+ public String getExtendedPropertyName() {
+ return PropertyUtil.removeDataPrefix(this.name);
+ }
+
public boolean isExtractFirstMatch() {
return EXTRACT_FIRST_MATCH_POLICY.equalsIgnoreCase(policy);
}
@@ -54,7 +59,12 @@ public class PropertyConfiguration {
return hasValidPath && (isExtractFirstMatch() || isExtractAllMatches());
}
- public String getRelatedObjectKind() {
+ /**
+ * Though one extended property can be mapped to a property from different kinds of source objects, we assume
+ * that all those source properties should have the same schema.
+ * @return
+ */
+ public String getFirstRelatedObjectKind() {
if(paths != null) {
for (PropertyPath path : paths) {
if (path.isValid() && path.hasValidRelatedObjectsSpec()) {
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/PropertyConfigurations.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/PropertyConfigurations.java
index 6fbb00c77..3c0318dc9 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/PropertyConfigurations.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/PropertyConfigurations.java
@@ -53,7 +53,9 @@ public class PropertyConfigurations {
Set<String> relatedObjectKinds = new HashSet<>();
for(PropertyConfiguration configuration : configurations) {
- String relatedObjectKind = configuration.getRelatedObjectKind();
+ // In the same configuration, we can use any related object kind to
+ // resolve the schema of the extended property in this configuration
+ String relatedObjectKind = configuration.getFirstRelatedObjectKind();
if(!Strings.isNullOrEmpty(relatedObjectKind)) {
relatedObjectKinds.add(relatedObjectKind);
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/RelatedCondition.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/RelatedCondition.java
index ba9683e26..5c087d5b4 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/RelatedCondition.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/RelatedCondition.java
@@ -19,6 +19,7 @@ import com.fasterxml.jackson.annotation.JsonInclude;
import com.google.api.client.util.Strings;
import lombok.Data;
import lombok.ToString;
+import org.opengroup.osdu.indexer.util.PropertyUtil;
import java.util.List;
@@ -32,27 +33,44 @@ public class RelatedCondition {
protected List<String> relatedConditionMatches;
+ protected boolean hasCondition() {
+ return !Strings.isNullOrEmpty(relatedConditionProperty) &&
+ relatedConditionMatches != null &&
+ !relatedConditionMatches.isEmpty();
+ }
+
protected boolean hasValidCondition(String property) {
- if(Strings.isNullOrEmpty(property) ||
- Strings.isNullOrEmpty(relatedConditionProperty) ||
- relatedConditionMatches == null ||
- relatedConditionMatches.isEmpty())
+ if(Strings.isNullOrEmpty(property) || !this.hasCondition())
return false;
- if(property.indexOf(ARRAY_SYMBOL + "." ) <= 0 || property.endsWith(ARRAY_SYMBOL) ||
- relatedConditionProperty.indexOf(ARRAY_SYMBOL + "." ) <= 0 || relatedConditionProperty.endsWith(ARRAY_SYMBOL))
+ if((property.endsWith(ARRAY_SYMBOL) || relatedConditionProperty.endsWith(ARRAY_SYMBOL)) ||
+ (property.contains(ARRAY_SYMBOL) && !relatedConditionProperty.contains(ARRAY_SYMBOL)) ||
+ (!property.contains(ARRAY_SYMBOL) && relatedConditionProperty.contains(ARRAY_SYMBOL)))
return false;
- String delimiter = "\\[\\]\\.";
+ // If it is not nested object, it is valid in terms of syntax.
+ if(!property.contains(ARRAY_SYMBOL) && !relatedConditionProperty.contains(ARRAY_SYMBOL))
+ return true;
+
+ property = this.getSubstringWithLastArrayField(
+ PropertyUtil.removeDataPrefix(property));
+ String conditionProperty = this.getSubstringWithLastArrayField(
+ PropertyUtil.removeDataPrefix(relatedConditionProperty));
+
+ String delimiter = "\\.";
String[] propertyParts = property.split(delimiter);
- String[] relatedConditionPropertyParts = relatedConditionProperty.split(delimiter);
- if(propertyParts.length != relatedConditionPropertyParts.length || propertyParts.length < 2)
+ String[] relatedConditionPropertyParts = conditionProperty.split(delimiter);
+ if(propertyParts.length != relatedConditionPropertyParts.length)
return false;
- for(int i = 0; i < propertyParts.length -1; i++) {
+ for(int i = 0; i < propertyParts.length; i++) {
if(!propertyParts[i].equals(relatedConditionPropertyParts[i]))
return false;
}
return true;
}
+
+ private String getSubstringWithLastArrayField(String property) {
+ return property.substring(0, property.lastIndexOf(ARRAY_SYMBOL));
+ }
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/RelatedObjectsSpec.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/RelatedObjectsSpec.java
index 8ff9b57f5..bb9c8e076 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/RelatedObjectsSpec.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/RelatedObjectsSpec.java
@@ -38,8 +38,10 @@ public class RelatedObjectsSpec extends RelatedCondition {
public boolean isParentToChildren() { return PARENT_TO_CHILDREN.equalsIgnoreCase(relationshipDirection); }
public boolean isValid() {
- return !Strings.isNullOrEmpty(relatedObjectID) && !Strings.isNullOrEmpty(relatedObjectKind) &&
- (isChildToParent() || isParentToChildren());
+ return !Strings.isNullOrEmpty(relatedObjectID) &&
+ !Strings.isNullOrEmpty(relatedObjectKind) &&
+ (isChildToParent() || isParentToChildren()) &&
+ (!this.hasCondition() || this.hasValidCondition());
}
/**
@@ -48,6 +50,6 @@ public class RelatedObjectsSpec extends RelatedCondition {
* @return
*/
public boolean hasValidCondition() {
- return isValid() && super.hasValidCondition(relatedObjectID);
+ return super.hasValidCondition(relatedObjectID);
}
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/ValueExtraction.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/ValueExtraction.java
index 1ae141d2b..82eca51d8 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/ValueExtraction.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/ValueExtraction.java
@@ -29,7 +29,7 @@ public class ValueExtraction extends RelatedCondition {
private String valuePath;
public boolean isValid() {
- return !Strings.isNullOrEmpty(valuePath);
+ return !Strings.isNullOrEmpty(valuePath) && (!this.hasCondition() || this.hasValidCondition());
}
/**
@@ -38,6 +38,6 @@ public class ValueExtraction extends RelatedCondition {
* @return
*/
public boolean hasValidCondition() {
- return isValid() && super.hasValidCondition(valuePath);
+ return super.hasValidCondition(valuePath);
}
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/PropertyConfigurationsServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/PropertyConfigurationsServiceImpl.java
index 91b58d88e..b97465eb4 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/PropertyConfigurationsServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/PropertyConfigurationsServiceImpl.java
@@ -19,6 +19,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Strings;
import com.google.gson.Gson;
+import org.apache.commons.collections.CollectionUtils;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.indexer.OperationType;
@@ -39,10 +40,8 @@ import org.opengroup.osdu.indexer.util.PropertyUtil;
import org.springframework.stereotype.Component;
import javax.inject.Inject;
-import java.io.UnsupportedEncodingException;
import java.net.URISyntaxException;
import java.util.*;
-import java.util.stream.Collectors;
@Component
@@ -64,9 +63,15 @@ public class PropertyConfigurationsServiceImpl implements PropertyConfigurations
private static final String ARRAY_SYMBOL = "[]";
private static final String SCHEMA_NESTED_KIND = "nested";
+ private static final String STRING_KIND = "string";
+
+ private static final String STRING_ARRAY_KIND = "[]string";
+
+ private static final PropertyConfigurations EMPTY_CONFIGURATIONS = new PropertyConfigurations();
+ private static final String SEARCH_GENERAL_ERROR = "Failed to call search service.";
+
private final Gson gson = new Gson();
private final ObjectMapper objectMapper = new ObjectMapper();
- private final PropertyConfigurations EMPTY_CONFIGURATIONS = new PropertyConfigurations();
@Inject
private IndexerConfigurationProperties configurationProperties;
@@ -170,12 +175,13 @@ public class PropertyConfigurationsServiceImpl implements PropertyConfigurations
@Override
public Map<String, Object> getExtendedProperties(String objectId, Map<String, Object> originalDataMap, PropertyConfigurations propertyConfigurations) {
+ // Get all data maps of the related objects in one query in order to improve the performance.
+ Map<String, Map<String, Object>> idObjectDataMap = getRelatedObjectsData(originalDataMap, propertyConfigurations);
+
Set<String> associatedIdentities = new HashSet<>();
Map<String, Object> extendedDataMap = new HashMap<>();
-
- Map<String, Map<String, Object>> idObjectDataMap = getRelatedObjectsData(originalDataMap, propertyConfigurations);
- for (PropertyConfiguration configuration : propertyConfigurations.getConfigurations().stream().filter(c -> c.isValid()).collect(Collectors.toList())) {
- String extendedPropertyName = configuration.getName();
+ for (PropertyConfiguration configuration : propertyConfigurations.getConfigurations().stream().filter(c -> c.isValid()).toList()) {
+ String extendedPropertyName = configuration.getExtendedPropertyName();
if (originalDataMap.containsKey(extendedPropertyName) && originalDataMap.get(extendedPropertyName) != null) {
// If the original record already has the property, then we should not override.
// For example, if the trajectory record already SpatialLocation value, then it should not be overridden by the SpatialLocation of the well bore.
@@ -183,7 +189,7 @@ public class PropertyConfigurationsServiceImpl implements PropertyConfigurations
}
Map<String, Object> allPropertyValues = new HashMap<>();
- for (PropertyPath path : configuration.getPaths().stream().filter(p -> p.hasValidValueExtraction()).collect(Collectors.toList())) {
+ for (PropertyPath path : configuration.getPaths().stream().filter(p -> p.hasValidValueExtraction()).toList()) {
if (path.hasValidRelatedObjectsSpec()) {
RelatedObjectsSpec relatedObjectsSpec = path.getRelatedObjectsSpec();
if (relatedObjectsSpec.isChildToParent()) {
@@ -206,11 +212,11 @@ public class PropertyConfigurationsServiceImpl implements PropertyConfigurations
}
} else {
List<SearchRecord> childrenRecords = searchChildrenRecords(relatedObjectsSpec.getRelatedObjectKind(), relatedObjectsSpec.getRelatedObjectID(), objectId);
- for (SearchRecord record : childrenRecords) {
- // If the child record is in the cache, that means the record was updated very recently.
- // In this case, use the cache's record instead of the record from search result
- RecordData cachedRecordData = this.relatedObjectCache.get(record.getId());
- Map<String, Object> childDataMap = (cachedRecordData != null)? cachedRecordData.getData() : record.getData();
+ for (SearchRecord searchRecord : childrenRecords) {
+ // If the child record is in the cache, that means the searchRecord was updated very recently.
+ // In this case, use the cache's record instead of the searchRecord from search result
+ RecordData cachedRecordData = this.relatedObjectCache.get(searchRecord.getId());
+ Map<String, Object> childDataMap = (cachedRecordData != null)? cachedRecordData.getData() : searchRecord.getData();
Map<String, Object> propertyValues = getExtendedPropertyValues(extendedPropertyName, childDataMap, path.getValueExtraction(), configuration.isExtractFirstMatch());
if (allPropertyValues.isEmpty() && configuration.isExtractFirstMatch()) {
allPropertyValues = propertyValues;
@@ -246,10 +252,10 @@ public class PropertyConfigurationsServiceImpl implements PropertyConfigurations
public List<SchemaItem> getExtendedSchemaItems(Schema originalSchema, Map<String, Schema> relatedObjectKindSchemas, PropertyConfigurations propertyConfigurations) {
List<SchemaItem> extendedSchemaItems = new ArrayList<>();
boolean hasChildToParentRelationship = false;
- for (PropertyConfiguration configuration : propertyConfigurations.getConfigurations().stream().filter(c -> c.isValid()).collect(Collectors.toList())) {
+ for (PropertyConfiguration configuration : propertyConfigurations.getConfigurations().stream().filter(c -> c.isValid()).toList()) {
Schema schema = null;
PropertyPath propertyPath = null;
- for (PropertyPath path : configuration.getPaths().stream().filter(p -> p.hasValidRelatedObjectsSpec()).collect(Collectors.toList())) {
+ for (PropertyPath path : configuration.getPaths().stream().filter(p -> p.hasValidRelatedObjectsSpec()).toList()) {
RelatedObjectsSpec relatedObjectsSpec = path.getRelatedObjectsSpec();
if (relatedObjectsSpec.isChildToParent()) {
hasChildToParentRelationship = true;
@@ -370,6 +376,10 @@ public class PropertyConfigurationsServiceImpl implements PropertyConfigurations
return extendedSchemaItem;
}
+ private String createIdsQuery(List<String> ids) {
+ return String.format("id: (%s)", createIdsFilter(ids));
+ }
+
private String createIdsFilter(List<String> ids) {
StringBuilder idsBuilder = new StringBuilder();
for (String id : ids) {
@@ -420,8 +430,8 @@ public class PropertyConfigurationsServiceImpl implements PropertyConfigurations
private Map<String, Map<String, Object>> getRelatedObjectsData(Map<String, Object> originalDataMap, PropertyConfigurations propertyConfigurations) {
Map<String, Map<String, Object>> idData = new HashMap<>();
Map<String, Set<String>> kindIds = new HashMap<>();
- for (PropertyConfiguration configuration : propertyConfigurations.getConfigurations().stream().filter(c -> c.isValid()).collect(Collectors.toList())) {
- for (PropertyPath path : configuration.getPaths().stream().filter(p -> p.hasValidValueExtraction()).collect(Collectors.toList())) {
+ for (PropertyConfiguration configuration : propertyConfigurations.getConfigurations().stream().filter(c -> c.isValid()).toList()) {
+ for (PropertyPath path : configuration.getPaths().stream().filter(p -> p.hasValidValueExtraction()).toList()) {
if (path.hasValidRelatedObjectsSpec()) {
RelatedObjectsSpec relatedObjectsSpec = path.getRelatedObjectsSpec();
List<String> relatedObjectIds = getRelatedObjectIds(originalDataMap, relatedObjectsSpec);
@@ -441,7 +451,7 @@ public class PropertyConfigurationsServiceImpl implements PropertyConfigurations
for (String recordId : entry.getValue()) {
String id = PropertyUtil.removeIdPostfix(recordId);
RecordData recordData = relatedObjectCache.get(id);
- Map<String, Object> data = (recordData != null)? recordData.getData() : null;;
+ Map<String, Object> data = (recordData != null)? recordData.getData() : null;
if (data != null) {
idData.put(id, data);
} else {
@@ -450,7 +460,7 @@ public class PropertyConfigurationsServiceImpl implements PropertyConfigurations
}
}
}
- if (kindsToSearch.size() > 0) {
+ if (!kindsToSearch.isEmpty()) {
List<SearchRecord> records = searchRelatedRecords(kindsToSearch, idsToSearch);
for (SearchRecord searchRecord : records) {
Map<String, Object> data = searchRecord.getData();
@@ -511,34 +521,35 @@ public class PropertyConfigurationsServiceImpl implements PropertyConfigurations
private List<SchemaItem> getExtendedSchemaItems(Schema schema, PropertyConfiguration configuration, PropertyPath propertyPath) {
String relatedPropertyPath = PropertyUtil.removeDataPrefix(propertyPath.getValueExtraction().getValuePath());
+ List<SchemaItem> extendedSchemaItems;
if (relatedPropertyPath.contains(ARRAY_SYMBOL)) { // Nested
- List<SchemaItem> extendedSchemaItems = new ArrayList<>();
extendedSchemaItems = cloneExtendedSchemaItemsFromNestedSchema(Arrays.asList(schema.getSchema()), configuration, relatedPropertyPath);
- if (extendedSchemaItems.isEmpty()) {
- // It is possible that the format of the source property is not defined
- // In this case, we assume that the format of property is string in order to make its value(s) searchable
- SchemaItem extendedSchemaItem = new SchemaItem();
- extendedSchemaItem.setPath(configuration.getName());
- if (configuration.isExtractFirstMatch()) {
- extendedSchemaItem.setKind("string");
- } else {
- extendedSchemaItem.setKind("[]string");
- }
- extendedSchemaItems.add(extendedSchemaItem);
- }
- return extendedSchemaItems;
} else {// Flatten
- List<SchemaItem> schemaItems = Arrays.asList(schema.getSchema());
- return cloneExtendedSchemaItems(schemaItems, configuration, relatedPropertyPath);
+ extendedSchemaItems = cloneExtendedSchemaItems(Arrays.asList(schema.getSchema()), configuration, relatedPropertyPath);
+ }
+
+ if (extendedSchemaItems.isEmpty()) {
+ // It is possible that the format (or schema) of the source property is not defined.
+ // In this case, we assume that the format of property is string in order to make its value(s) searchable
+ SchemaItem extendedSchemaItem = new SchemaItem();
+ extendedSchemaItem.setPath(configuration.getExtendedPropertyName());
+ if (configuration.isExtractFirstMatch()) {
+ extendedSchemaItem.setKind(STRING_KIND);
+ } else {
+ extendedSchemaItem.setKind(STRING_ARRAY_KIND);
+ }
+ extendedSchemaItems.add(extendedSchemaItem);
}
+ return extendedSchemaItems;
}
private List<SchemaItem> cloneExtendedSchemaItems(List<SchemaItem> schemaItems, PropertyConfiguration configuration, String relatedPropertyPath) {
List<SchemaItem> extendedSchemaItems = new ArrayList<>();
+ String extendedPropertyName = configuration.getExtendedPropertyName();
for (SchemaItem schemaItem : schemaItems) {
if (PropertyUtil.isPropertyPathMatched(schemaItem.getPath(), relatedPropertyPath)) {
String path = schemaItem.getPath();
- path = path.replace(relatedPropertyPath, configuration.getName());
+ path = path.replace(relatedPropertyPath, extendedPropertyName);
SchemaItem extendedSchemaItem = new SchemaItem();
extendedSchemaItem.setPath(path);
if (configuration.isExtractFirstMatch()) {
@@ -580,8 +591,8 @@ public class PropertyConfigurationsServiceImpl implements PropertyConfigurations
Map<String, Object> propertyValues = getPropertyValues(dataMap, relatedObjectsSpec.getRelatedObjectID(), relatedObjectsSpec, relatedObjectsSpec.hasValidCondition(), false);
List<String> relatedObjectIds = new ArrayList<>();
for (Object value : propertyValues.values()) {
- if (value instanceof List) {
- for (Object obj : (List) value) {
+ if (value instanceof List<? extends Object> values) {
+ for (Object obj : values) {
relatedObjectIds.add(obj.toString());
}
} else {
@@ -605,78 +616,113 @@ public class PropertyConfigurationsServiceImpl implements PropertyConfigurations
String conditionProperty = null;
List<String> conditionMatches = null;
if (hasValidCondition) {
- int idx = relatedCondition.getRelatedConditionProperty().lastIndexOf(NESTED_OBJECT_DELIMITER);
- conditionProperty = relatedCondition.getRelatedConditionProperty().substring(idx + NESTED_OBJECT_DELIMITER.length());
+ conditionProperty = relatedCondition.getRelatedConditionProperty();
conditionMatches = relatedCondition.getRelatedConditionMatches();
}
- List<Object> valueList = getPropertyValuesFromNestedObjects(dataMap, valuePath, conditionProperty, conditionMatches, hasValidCondition, isExtractFirstMatch);
- if (!valueList.isEmpty()) {
- if (isExtractFirstMatch) {
- propertyValues.put(valuePath, valueList.get(0));
- } else {
- propertyValues.put(valuePath, valueList);
- }
- }
+ propertyValues = getPropertyValuesFromNestedObjects(dataMap, valuePath, conditionProperty, conditionMatches, hasValidCondition, isExtractFirstMatch);
} else { // Flatten
- for (Map.Entry<String, Object> entry : dataMap.entrySet()) {
- String key = entry.getKey();
- if ((key.equals(valuePath) || key.startsWith(valuePath + PROPERTY_DELIMITER)) && entry.getValue() != null) {
- if (isExtractFirstMatch) {
- propertyValues.put(key, entry.getValue());
- } else {
+ String conditionProperty = null;
+ List<String> conditionMatches = null;
+ if (hasValidCondition) {
+ conditionProperty = relatedCondition.getRelatedConditionProperty();
+ conditionMatches = relatedCondition.getRelatedConditionMatches();
+ }
+ propertyValues = getPropertyValueOfNoneNestedProperty(dataMap, valuePath, conditionProperty, conditionMatches, hasValidCondition);
+ if(!isExtractFirstMatch) {
+ Map<String, Object> tmpValues = new HashMap<>();
+ for(Map.Entry<String, Object> entry : propertyValues.entrySet()) {
+ if (entry.getValue() instanceof List<? extends Object>) {
+ tmpValues.put(entry.getKey(), entry.getValue());
+ }
+ else {
List<Object> values = new ArrayList<>();
values.add(entry.getValue());
- propertyValues.put(key, values);
+ tmpValues.put(entry.getKey(), values);
}
}
+ propertyValues = tmpValues;
}
}
return propertyValues;
}
- private List<Object> getPropertyValuesFromNestedObjects(Map<String, Object> dataMap, String valuePath, String conditionProperty, List<String> conditionMatches, boolean hasCondition, boolean isExtractFirstMatch) {
- Set<Object> propertyValues = new HashSet<>();
+ private Map<String, Object> getPropertyValuesFromNestedObjects(Map<String, Object> dataMap, String valuePath, String conditionProperty, List<String> conditionMatches, boolean hasCondition, boolean isExtractFirstMatch) {
+ Map<String, Object> propertyValues = new HashMap<>();
if (valuePath.contains(ARRAY_SYMBOL)) {
int idx = valuePath.indexOf(NESTED_OBJECT_DELIMITER);
String prePath = valuePath.substring(0, idx);
String postPath = valuePath.substring(idx + NESTED_OBJECT_DELIMITER.length());
+ if(conditionProperty != null) {
+ idx = conditionProperty.indexOf(NESTED_OBJECT_DELIMITER);
+ if(idx > 0) {
+ conditionProperty = conditionProperty.substring(idx + NESTED_OBJECT_DELIMITER.length());
+ }
+ else {
+ // Should not reach here
+ conditionProperty = null;
+ }
+ }
try {
if (dataMap.containsKey(prePath) && dataMap.get(prePath) != null) {
List<Map<String, Object>> nestedObjects = (List<Map<String, Object>>) dataMap.get(prePath);
for (Map<String, Object> nestedObject : nestedObjects) {
- List<Object> valueList = getPropertyValuesFromNestedObjects(nestedObject, postPath, conditionProperty, conditionMatches, hasCondition, isExtractFirstMatch);
- if (valueList != null && !valueList.isEmpty()) {
- propertyValues.addAll(valueList);
- if (isExtractFirstMatch)
- break;
+ Map<String, Object> subPropertyValues = getPropertyValuesFromNestedObjects(nestedObject, postPath, conditionProperty, conditionMatches, hasCondition, isExtractFirstMatch);
+ for (Map.Entry<String, Object> entry: subPropertyValues.entrySet()) {
+ String key = prePath + ARRAY_SYMBOL + PROPERTY_DELIMITER + entry.getKey();
+ if(isExtractFirstMatch) {
+ propertyValues.put(key, entry.getValue());
+ }
+ else {
+ List<Object> values = propertyValues.containsKey(key)
+ ? (List<Object>)propertyValues.get(key)
+ : new ArrayList<>();
+ if(entry.getValue() instanceof List<? extends Object> valueList) {
+ values.addAll(valueList);
+ }
+ else {
+ values.add(entry.getValue());
+ }
+ propertyValues.put(key, values);
+ }
}
+ if (isExtractFirstMatch)
+ break;
}
}
} catch (Exception ex) {
//Ignore cast exception
}
- } else if (dataMap.containsKey(valuePath) && dataMap.get(valuePath) != null) {
- Object extractPropertyValue = dataMap.get(valuePath);
- if (hasCondition) {
- if (dataMap.containsKey(conditionProperty) && dataMap.get(conditionProperty) != null) {
- String conditionPropertyValue = dataMap.get(conditionProperty).toString();
- if (conditionMatches.contains(conditionPropertyValue) && extractPropertyValue != null) {
- propertyValues.add(extractPropertyValue);
+ } else {
+ propertyValues = getPropertyValueOfNoneNestedProperty(dataMap, valuePath, conditionProperty, conditionMatches, hasCondition);
+ }
+ return propertyValues;
+ }
+
+ private Map<String, Object> getPropertyValueOfNoneNestedProperty(Map<String, Object> dataMap, String valuePath, String conditionProperty, List<String> conditionMatches, boolean hasCondition) {
+ Map<String, Object> propertyValue = PropertyUtil.getValueOfNoneNestedProperty(valuePath, dataMap);
+ if(!propertyValue.isEmpty() && hasCondition) {
+ boolean matched = false;
+ Map<String, Object> conditionPropertyValue = PropertyUtil.getValueOfNoneNestedProperty(conditionProperty, dataMap);
+ if (conditionPropertyValue.containsKey(conditionProperty)) {
+ for(String condition: conditionMatches) {
+ if(PropertyUtil.isMatch(conditionPropertyValue.get(conditionProperty).toString(), condition)) {
+ matched = true;
+ break;
}
}
- } else {
- propertyValues.add(extractPropertyValue);
+ }
+ if(!matched) {
+ // Reset the propertyValue if there is no match
+ propertyValue = new HashMap<>();
}
}
-
- List<Object> propertyValueList = new ArrayList<>(propertyValues);
- Collections.sort(propertyValueList, Comparator.comparing(Object::toString));
- return propertyValueList;
+ return propertyValue;
}
+
private List<String> getChildrenKinds(String parentKind) {
final String parentKindWithMajor = PropertyUtil.getKindWithMajor(parentKind);
ChildrenKinds childrenKinds = childrenKindsCache.get(parentKindWithMajor);
@@ -709,7 +755,7 @@ public class PropertyConfigurationsServiceImpl implements PropertyConfigurations
p.hasValidRelatedObjectsSpec() &&
p.getRelatedObjectsSpec().isParentToChildren() &&
p.getRelatedObjectsSpec().getRelatedObjectKind().contains(childKindWithMajor))
- .collect(Collectors.toList());
+ .toList();
for(PropertyPath propertyPath: matchedPropertyPaths) {
ParentChildRelationshipSpec spec = toParentChildRelationshipSpec(propertyPath, configurations.getCode(), childKindWithMajor);
boolean merged = false;
@@ -748,13 +794,14 @@ public class PropertyConfigurationsServiceImpl implements PropertyConfigurations
private void updateAssociatedParentRecords(String ancestors, String childKind, List<RecordChangeInfo> childRecordChangeInfos) {
ParentChildRelationshipSpecs specs = getParentChildRelatedObjectsSpecs(childKind);
- Set ancestorSet = new HashSet<>(Arrays.asList(ancestors.split(ANCESTRY_KINDS_DELIMITER)));
+ Set<String> ancestorSet = new HashSet<>(Arrays.asList(ancestors.split(ANCESTRY_KINDS_DELIMITER)));
for (ParentChildRelationshipSpec spec : specs.getSpecList()) {
- List childRecordIds = getChildRecordIdsWithExtendedPropertiesChanged(spec, childRecordChangeInfos);
- if (childRecordIds.isEmpty())
- continue;
+ List<String> childRecordIds = getChildRecordIdsWithExtendedPropertiesChanged(spec, childRecordChangeInfos);
- List<String> parentIds = searchUniqueParentIds(childKind, childRecordIds, spec.getParentObjectIdPath());
+ List<String> parentIds = new ArrayList<>();
+ if (!childRecordIds.isEmpty()) {
+ parentIds = searchUniqueParentIds(childKind, childRecordIds, spec.getParentObjectIdPath());
+ }
if (parentIds.isEmpty())
continue;
@@ -820,7 +867,7 @@ public class PropertyConfigurationsServiceImpl implements PropertyConfigurations
if(propertyConfigurations != null) {
for (PropertyConfiguration propertyConfiguration : propertyConfigurations.getConfigurations()) {
for (PropertyPath propertyPath : propertyConfiguration.getPaths().stream().filter(
- p -> p.hasValidValueExtraction() && p.hasValidRelatedObjectsSpec()).collect(Collectors.toList())) {
+ p -> p.hasValidValueExtraction() && p.hasValidRelatedObjectsSpec()).toList()) {
String relatedObjectKind = propertyPath.getRelatedObjectsSpec().getRelatedObjectKind();
String valuePath = PropertyUtil.removeDataPrefix(propertyPath.getValueExtraction().getValuePath());
@@ -828,7 +875,7 @@ public class PropertyConfigurationsServiceImpl implements PropertyConfigurations
RecordChangeInfo parentRecordChangeInfo = parentRecordChangeInfos.stream().filter(info -> {
if (PropertyUtil.hasSameMajorKind(info.getRecordInfo().getKind(), relatedObjectKind)) {
List<String> matchedProperties = info.getUpdatedProperties().stream().filter(
- p -> PropertyUtil.isPropertyPathMatched(p, valuePath) || PropertyUtil.isPropertyPathMatched(valuePath, p)).collect(Collectors.toList());
+ p -> PropertyUtil.isPropertyPathMatched(p, valuePath) || PropertyUtil.isPropertyPathMatched(valuePath, p)).toList();
return !matchedProperties.isEmpty();
}
return false;
@@ -844,13 +891,13 @@ public class PropertyConfigurationsServiceImpl implements PropertyConfigurations
}
private void updateAssociatedChildrenRecords(String ancestors, String parentKind, List<RecordChangeInfo> recordChangeInfos) {
- List<String> processedIds = recordChangeInfos.stream().map(recordChangeInfo -> recordChangeInfo.getRecordInfo().getId()).collect(Collectors.toList());
+ List<String> processedIds = recordChangeInfos.stream().map(recordChangeInfo -> recordChangeInfo.getRecordInfo().getId()).toList();
String query = String.format("data.%s:(%s)", ASSOCIATED_IDENTITIES_PROPERTY, createIdsFilter(processedIds));
List<String> childrenKinds = getChildrenKinds(parentKind);
for (String ancestryKind : ancestors.split(ANCESTRY_KINDS_DELIMITER)) {
// Exclude the kinds in the ancestryKinds to prevent circular chasing
- childrenKinds.removeIf(k -> ancestryKind.contains(k));
+ childrenKinds.removeIf(ancestryKind::contains);
}
if(childrenKinds.isEmpty()) {
return;
@@ -866,18 +913,18 @@ public class PropertyConfigurationsServiceImpl implements PropertyConfigurations
searchRequest.setQuery(query);
searchRequest.setReturnedFields(Arrays.asList("kind", "id", "data." + ASSOCIATED_IDENTITIES_PROPERTY));
List<RecordInfo> recordInfos = new ArrayList<>();
- for (SearchRecord record : searchRecordsWithCursor(searchRequest)) {
- Map<String, Object> data = record.getData();
+ for (SearchRecord searchRecord : searchRecordsWithCursor(searchRequest)) {
+ Map<String, Object> data = searchRecord.getData();
if (!data.containsKey(ASSOCIATED_IDENTITIES_PROPERTY) || data.get(ASSOCIATED_IDENTITIES_PROPERTY) == null)
continue;
List<String> associatedParentIds = (List<String>) data.get(ASSOCIATED_IDENTITIES_PROPERTY);
List<RecordChangeInfo> associatedParentRecordChangeInfos = recordChangeInfos.stream().filter(
- info -> associatedParentIds.contains(info.getRecordInfo().getId())).collect(Collectors.toList());
- if (areExtendedPropertiesChanged(record.getKind(), associatedParentRecordChangeInfos)) {
+ info -> associatedParentIds.contains(info.getRecordInfo().getId())).toList();
+ if (areExtendedPropertiesChanged(searchRecord.getKind(), associatedParentRecordChangeInfos)) {
RecordInfo recordInfo = new RecordInfo();
- recordInfo.setKind(record.getKind());
- recordInfo.setId(record.getId());
+ recordInfo.setKind(searchRecord.getKind());
+ recordInfo.setId(searchRecord.getId());
recordInfo.setOp(OperationType.update.getValue());
recordInfos.add(recordInfo);
@@ -900,7 +947,7 @@ public class PropertyConfigurationsServiceImpl implements PropertyConfigurations
String latestKind = null;
try {
SchemaInfoResponse response = schemaService.getSchemaInfos(kind.getAuthority(), kind.getSource(), kind.getType(), majorVersion, null, null, true);
- if (response != null && response.getSchemaInfos() != null && response.getSchemaInfos().size() > 0) {
+ if (response != null && !CollectionUtils.isEmpty(response.getSchemaInfos())) {
SchemaInfo schemaInfo = response.getSchemaInfos().get(0);
SchemaIdentity schemaIdentity = schemaInfo.getSchemaIdentity();
latestKind = schemaIdentity.getAuthority() + ":" +
@@ -910,9 +957,7 @@ public class PropertyConfigurationsServiceImpl implements PropertyConfigurations
schemaIdentity.getSchemaVersionMinor() + "." +
schemaIdentity.getSchemaVersionPatch();
}
- } catch (URISyntaxException e) {
- jaxRsDpsLog.error("failed to get schema info", e);
- } catch (UnsupportedEncodingException e) {
+ } catch (Exception e) {
jaxRsDpsLog.error("failed to get schema info", e);
}
@@ -984,7 +1029,7 @@ public class PropertyConfigurationsServiceImpl implements PropertyConfigurations
kinds.add(kind);
}
searchRequest.setKind(kinds);
- String query = String.format("id: (%s)", createIdsFilter(relatedObjectIds));
+ String query = createIdsQuery(relatedObjectIds);
searchRequest.setQuery(query);
return searchRecords(searchRequest);
}
@@ -994,16 +1039,16 @@ public class PropertyConfigurationsServiceImpl implements PropertyConfigurations
SearchRequest searchRequest = new SearchRequest();
String kind = PropertyUtil.isConcreteKind(majorKind) ? majorKind : majorKind + "*";
searchRequest.setKind(kind);
- String query = String.format("id: (%s)", createIdsFilter(ids));
+ String query = createIdsQuery(ids);
searchRequest.setReturnedFields(Arrays.asList("kind", "id"));
searchRequest.setQuery(query);
- for (SearchRecord record : searchRecords(searchRequest)) {
- if (kindIds.containsKey(record.getKind())) {
- kindIds.get(record.getKind()).add(record.getId());
+ for (SearchRecord searchRecord : searchRecords(searchRequest)) {
+ if (kindIds.containsKey(searchRecord.getKind())) {
+ kindIds.get(searchRecord.getKind()).add(searchRecord.getId());
} else {
List<String> idList = new ArrayList<>();
- idList.add(record.getId());
- kindIds.put(record.getKind(), idList);
+ idList.add(searchRecord.getId());
+ kindIds.put(searchRecord.getKind(), idList);
}
}
return kindIds;
@@ -1013,13 +1058,13 @@ public class PropertyConfigurationsServiceImpl implements PropertyConfigurations
Set<String> parentIds = new HashSet<>();
SearchRequest searchRequest = new SearchRequest();
searchRequest.setKind(childKind);
- String query = String.format("id: (%s)", createIdsFilter(childRecordIds));
+ String query = createIdsQuery(childRecordIds);
searchRequest.setReturnedFields(Arrays.asList(parentObjectIdPath));
searchRequest.setQuery(query);
parentObjectIdPath = PropertyUtil.removeDataPrefix(parentObjectIdPath);
- for (SearchRecord record : searchRecords(searchRequest)) {
- if (record.getData().containsKey(parentObjectIdPath)) {
- Object id = record.getData().get(parentObjectIdPath);
+ for (SearchRecord searchRecord : searchRecords(searchRequest)) {
+ if (searchRecord.getData().containsKey(parentObjectIdPath)) {
+ Object id = searchRecord.getData().get(parentObjectIdPath);
if (id != null && !parentIds.contains(id)) {
parentIds.add(id.toString());
}
@@ -1045,7 +1090,7 @@ public class PropertyConfigurationsServiceImpl implements PropertyConfigurations
do {
SearchResponse searchResponse = searchService.queryWithCursor(searchRequest);
results = searchResponse.getResults();
- if (results != null && results.size() > 0) {
+ if (!CollectionUtils.isEmpty(results)) {
allRecords.addAll(results);
if (!Strings.isNullOrEmpty(searchResponse.getCursor()) && results.size() == MAX_SEARCH_LIMIT) {
searchRequest.setCursor(searchResponse.getCursor());
@@ -1053,7 +1098,7 @@ public class PropertyConfigurationsServiceImpl implements PropertyConfigurations
}
} while(results != null && results.size() == MAX_SEARCH_LIMIT);
} catch (URISyntaxException e) {
- jaxRsDpsLog.error("Failed to call search service.", e);
+ jaxRsDpsLog.error(SEARCH_GENERAL_ERROR, e);
}
return allRecords;
}
@@ -1068,14 +1113,14 @@ public class PropertyConfigurationsServiceImpl implements PropertyConfigurations
do {
SearchResponse searchResponse = searchService.query(searchRequest);
results = searchResponse.getResults();
- if (results != null && results.size() > 0) {
+ if (!CollectionUtils.isEmpty(results)) {
allRecords.addAll(results);
offset += results.size();
searchRequest.setOffset(offset);
}
} while(results != null && results.size() == MAX_SEARCH_LIMIT);
} catch (URISyntaxException e) {
- jaxRsDpsLog.error("Failed to call search service.", e);
+ jaxRsDpsLog.error(SEARCH_GENERAL_ERROR, e);
}
return allRecords;
}
@@ -1089,7 +1134,7 @@ public class PropertyConfigurationsServiceImpl implements PropertyConfigurations
return results.get(0);
}
} catch (URISyntaxException e) {
- jaxRsDpsLog.error("Failed to call search service.", e);
+ jaxRsDpsLog.error(SEARCH_GENERAL_ERROR, e);
}
return null;
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SearchServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SearchServiceImpl.java
index 51f393449..5f7ca9919 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SearchServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SearchServiceImpl.java
@@ -33,6 +33,7 @@ import java.net.URISyntaxException;
@Component
public class SearchServiceImpl implements SearchService {
+ private static final String ERROR_MESSAGE = "Search service: failed to call the search service.";
private static final String QUERY_PATH = "query";
private static final String QUERY_WITH_CURSOR_PATH = "query_with_cursor";
private final Gson gson = new Gson();
@@ -78,9 +79,9 @@ public class SearchServiceImpl implements SearchService {
return gson.fromJson(response.getBody(), SearchResponse.class);
} else {
if (response != null)
- jaxRsDpsLog.error(String.format("Search service: failed to call the search service: %d", response.getResponseCode()));
+ jaxRsDpsLog.error(ERROR_MESSAGE + String.format(" responseCode = %d", response.getResponseCode()));
else
- jaxRsDpsLog.error(String.format("Search service: failed to call the search service. The response is null."));
+ jaxRsDpsLog.error(String.format(ERROR_MESSAGE + " The response is null."));
return new SearchResponse();
}
}
@@ -88,7 +89,7 @@ public class SearchServiceImpl implements SearchService {
throw ex;
}
catch(Exception ex) {
- jaxRsDpsLog.error(String.format("Search service: failed to call the search service", ex));
+ jaxRsDpsLog.error(ERROR_MESSAGE, ex);
throw new URISyntaxException(ex.getMessage(), "Unexpected exception type: " + ex.getClass().getName());
}
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/PropertyUtil.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/PropertyUtil.java
index 49a7175ee..86771914a 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/PropertyUtil.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/PropertyUtil.java
@@ -21,6 +21,8 @@ import com.google.common.collect.Maps;
import org.opengroup.osdu.indexer.model.Kind;
import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
public class PropertyUtil {
public static final String DATA_VIRTUAL_DEFAULT_LOCATION = "data.VirtualProperties.DefaultLocation";
@@ -42,6 +44,33 @@ public class PropertyUtil {
return !Strings.isNullOrEmpty(propertyPath) && (propertyPath.startsWith(parentPropertyPath + PROPERTY_DELIMITER) || propertyPath.equals(parentPropertyPath));
}
+ public static Map<String, Object> getValueOfNoneNestedProperty(String propertyPath, Map<String, Object> data) {
+ if(Strings.isNullOrEmpty(propertyPath) || propertyPath.contains(ARRAY_SYMBOL) || data == null || data.isEmpty())
+ return new HashMap<>();
+
+ Map<String, Object> values = new HashMap<>();
+ if(data.containsKey(propertyPath)) {
+ values.put(propertyPath, data.get(propertyPath));
+ }
+ else {
+ for (String key : data.keySet()) {
+ if (propertyPath.startsWith(key + PROPERTY_DELIMITER)) {
+ Object v = data.get(key);
+ if (v instanceof Map) {
+ propertyPath = propertyPath.substring((key + PROPERTY_DELIMITER).length());
+ Map<String, Object> subPropertyValues = getValueOfNoneNestedProperty(propertyPath, (Map<String, Object>) v);
+ for (Map.Entry<String, Object> entry: subPropertyValues.entrySet()) {
+ values.put(key + PROPERTY_DELIMITER + entry.getKey(), entry.getValue());
+ }
+ }
+ } else if (key.startsWith(propertyPath + PROPERTY_DELIMITER)) {
+ values.put(key, data.get(key));
+ }
+ }
+ }
+ return values;
+ }
+
public static boolean hasSameMajorKind(String left, String right) {
try {
Kind leftKind = new Kind(left);
@@ -258,4 +287,22 @@ public class PropertyUtil {
return new ArrayList<>(changedProperties);
}
+
+ /*
+ * matchCondition can be either non-empty string or regular expression
+ */
+ public static boolean isMatch(String propertyValue, String matchCondition) {
+ if(Strings.isNullOrEmpty(propertyValue) || Strings.isNullOrEmpty(matchCondition))
+ return false;
+
+ try {
+ Pattern pattern = Pattern.compile(matchCondition);
+ Matcher matcher = pattern.matcher(propertyValue);
+ return matcher.find();
+ }
+ catch(Exception ex) {
+ // If matchCondition is not regex, do string compare
+ return propertyValue.equals(matchCondition);
+ }
+ }
}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/indexproperty/RelatedObjectsSpecTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/indexproperty/RelatedObjectsSpecTest.java
index 63a53c37d..52945410c 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/indexproperty/RelatedObjectsSpecTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/indexproperty/RelatedObjectsSpecTest.java
@@ -108,14 +108,14 @@ public class RelatedObjectsSpecTest {
}
@Test
- public void hasValidCondition_return_false_for_none_nested_property() {
+ public void hasValidCondition_return_true_for_none_nested_property() {
RelatedObjectsSpec spec = new RelatedObjectsSpec();
spec.setRelatedObjectKind("osdu:wks:master-data--GeoPoliticalEntity:1.");
spec.setRelatedObjectID("data.GeoContexts.GeoPoliticalEntityID");
spec.setRelatedConditionProperty("data.GeoContexts.GeoTypeID");
List<String> matches = Arrays.asList("opendes:reference-data--GeoPoliticalEntityType:Country:");
spec.setRelatedConditionMatches(matches);
- Assert.assertFalse(spec.hasValidCondition());
+ Assert.assertTrue(spec.hasValidCondition());
}
}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/indexproperty/ValueExtractionTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/indexproperty/ValueExtractionTest.java
index 4c578c9a4..c1db28ecf 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/indexproperty/ValueExtractionTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/indexproperty/ValueExtractionTest.java
@@ -98,12 +98,22 @@ public class ValueExtractionTest {
}
@Test
- public void hasValidCondition_return_false_for_none_nested_property() {
+ public void hasValidCondition_return_true_for_property_under_same_none_nested_property() {
ValueExtraction valueExtraction = new ValueExtraction();
valueExtraction.setValuePath("data.NameAliases.AliasName");
valueExtraction.setRelatedConditionProperty("data.NameAliases.AliasNameTypeID");
List<String> matches = Arrays.asList("opendes:reference-data--AliasNameType:UniqueIdentifier:","reference-data--AliasNameType:RegulatoryName:");
valueExtraction.setRelatedConditionMatches(matches);
- Assert.assertFalse(valueExtraction.hasValidCondition());
+ Assert.assertTrue(valueExtraction.hasValidCondition());
+ }
+
+ @Test
+ public void hasValidCondition_return_true_for_property_under_different_none_nested_property() {
+ ValueExtraction valueExtraction = new ValueExtraction();
+ valueExtraction.setValuePath("data.NameAliases.AliasName");
+ valueExtraction.setRelatedConditionProperty("data.AliasNameTypeID");
+ List<String> matches = Arrays.asList("opendes:reference-data--AliasNameType:UniqueIdentifier:","reference-data--AliasNameType:RegulatoryName:");
+ valueExtraction.setRelatedConditionMatches(matches);
+ Assert.assertTrue(valueExtraction.hasValidCondition());
}
}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/PropertyConfigurationsServiceImplTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/PropertyConfigurationsServiceImplTest.java
index e68ca9dbf..4c5be0ea9 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/PropertyConfigurationsServiceImplTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/PropertyConfigurationsServiceImplTest.java
@@ -267,8 +267,8 @@ public class PropertyConfigurationsServiceImplTest {
Assert.assertEquals(value, extendedProperties.get(name));
}
else if(value instanceof List) {
- List<String> expectedValues = (List<String>)value;
- List<String> values = (List<String>)extendedProperties.get(name);
+ List<String> expectedValues = ((List<String>)value).stream().sorted().toList();
+ List<String> values = ((List<String>)extendedProperties.get(name)).stream().sorted().toList();
Assert.assertEquals(expectedValues.size(), values.size());
for(int i = 0; i < expectedValues.size(); i++) {
Assert.assertEquals(expectedValues.get(i), values.get(i));
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/PropertyUtilTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/PropertyUtilTest.java
index 683953925..b041ae655 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/PropertyUtilTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/PropertyUtilTest.java
@@ -44,6 +44,41 @@ public class PropertyUtilTest {
Assert.assertFalse(PropertyUtil.isPropertyPathMatched(null, "data.ProjectedBottomHole"));
}
+ @Test
+ public void getValueOfNoneNestedProperty() {
+ Map<String, Object> data = this.getDataMap("wellLog.json");
+
+ Map<String, Object> value = PropertyUtil.getValueOfNoneNestedProperty("SpatialLocation.SpatialGeometryTypeID", data);
+ Assert.assertTrue(value.containsKey("SpatialLocation.SpatialGeometryTypeID"));
+ Assert.assertEquals("opendes:reference-data--SpatialGeometryType:Point:", value.get("SpatialLocation.SpatialGeometryTypeID").toString());
+
+ value = PropertyUtil.getValueOfNoneNestedProperty("SpatialLocation.Wgs84Coordinates.type", data);
+ Assert.assertTrue(value.containsKey("SpatialLocation.Wgs84Coordinates.type"));
+ Assert.assertEquals("geometrycollection", value.get("SpatialLocation.Wgs84Coordinates.type").toString());
+
+ List<String> propertyNamesWithValues = Arrays.asList(
+ "SpatialLocation.SpatialGeometryTypeID",
+ "SpatialLocation.Wgs84Coordinates",
+ "SpatialLocation.IsDecimated");
+ List<String> propertyNamesWithNullValues = Arrays.asList(
+ "SpatialLocation.SpatialParameterTypeID",
+ "SpatialLocation.CoordinateQualityCheckPerformedBy",
+ "SpatialLocation.QualitativeSpatialAccuracyTypeID",
+ "SpatialLocation.QuantitativeAccuracyBandID");
+ value = PropertyUtil.getValueOfNoneNestedProperty("SpatialLocation", data);
+ for(String propertyName : propertyNamesWithValues) {
+ Assert.assertTrue(value.containsKey(propertyName));
+ Assert.assertNotNull(value.get(propertyName));
+ }
+ for(String propertyName : propertyNamesWithNullValues) {
+ Assert.assertTrue(value.containsKey(propertyName));
+ Assert.assertNull(value.get(propertyName));
+ }
+
+ value = PropertyUtil.getValueOfNoneNestedProperty("Curves[].CurveID", data);
+ Assert.assertTrue(value.isEmpty());
+ }
+
@Test
public void hasSameMajorVersion() {
Assert.assertTrue(PropertyUtil.hasSameMajorKind("osdu:wks:master-data--Well:1.0.0", "osdu:wks:master-data--Well:1.0.0"));
@@ -272,6 +307,20 @@ public class PropertyUtilTest {
changedProperties.forEach(p -> Assert.assertTrue(expectedChangedWellLogProperties.contains(p)));
}
+ @Test
+ public void isMatch_for_String() {
+ Assert.assertTrue(PropertyUtil.isMatch("opendes:reference-data--AliasNameType:UniqueIdentifier:", "opendes:reference-data--AliasNameType:UniqueIdentifier:"));
+ Assert.assertFalse(PropertyUtil.isMatch("OPENDES:reference-data--AliasNameType:UniqueIdentifier:", "opendes:reference-data--AliasNameType:UniqueIdentifier:"));
+ }
+
+ @Test
+ public void isMatch_for_Regex() {
+ Assert.assertTrue(PropertyUtil.isMatch("opendes:master-data--Wellbore:F03B95515F91--2B886EAF-83A9-4811-9068", "^[\\w\\-\\.]+:master-data\\-\\-Wellbore:[\\w\\-\\.\\:\\%]+$"));
+ Assert.assertTrue(PropertyUtil.isMatch("opendes:work-product-component--WellLog:8CAFB37C-A674-42F8-ABCD-34BA8E6C1480", "^[\\w\\-\\.]+:work-product-component\\-\\-WellLog:[\\w\\-\\.\\:\\%]+$"));
+ Assert.assertTrue(PropertyUtil.isMatch("opendes:work-product-component--Document:wks-0f2546ca6b45960fa32db", "^[\\w\\-\\.]+:work-product-component\\-\\-Document:[\\w\\-\\.\\:\\%]+$"));
+ }
+
+
private Map<String, Object> getDataMap(String file) {
String jsonText = getJsonFromFile(file);
Type type = new TypeToken<Map<String, Object>>() {}.getType();
diff --git a/indexer-core/src/test/resources/indexproperty/well_configuration_record.json b/indexer-core/src/test/resources/indexproperty/well_configuration_record.json
index 0cf4995d3..a4fae666f 100644
--- a/indexer-core/src/test/resources/indexproperty/well_configuration_record.json
+++ b/indexer-core/src/test/resources/indexproperty/well_configuration_record.json
@@ -21,7 +21,7 @@
"Paths": [{
"RelatedObjectsSpec.RelatedObjectID": "data.GeoContexts[].GeoPoliticalEntityID",
"RelatedObjectsSpec.RelatedConditionMatches": [
- "opendes:reference-data--GeoPoliticalEntityType:Country:"
+ "^[\\w\\-\\.]+:reference-data\\-\\-GeoPoliticalEntityType:Country:$"
],
"ValueExtraction.ValuePath": "data.GeoPoliticalEntityName",
"RelatedObjectsSpec.RelatedObjectKind": "osdu:wks:master-data--GeoPoliticalEntity:1.",
@@ -41,11 +41,11 @@
"RelatedObjectsSpec.RelatedConditionProperty": null,
"RelatedObjectsSpec.RelationshipDirection": null,
"ValueExtraction.RelatedConditionMatches": [
- "opendes:reference-data--AliasNameType:UniqueIdentifier:",
- "opendes:reference-data--AliasNameType:RegulatoryName:",
- "opendes:reference-data--AliasNameType:PreferredName:",
- "opendes:reference-data--AliasNameType:CommonName:",
- "opendes:reference-data--AliasNameType:ShortName:"
+ "^[\\w\\-\\.]+:reference-data--AliasNameType:UniqueIdentifier:$",
+ "^[\\w\\-\\.]+:reference-data--AliasNameType:RegulatoryName:$",
+ "^[\\w\\-\\.]+:reference-data--AliasNameType:PreferredName:$",
+ "^[\\w\\-\\.]+:reference-data--AliasNameType:CommonName:$",
+ "^[\\w\\-\\.]+:reference-data--AliasNameType:ShortName:$"
],
"ValueExtraction.RelatedConditionProperty": "data.NameAliases[].AliasNameTypeID"
}
diff --git a/testing/indexer-test-core/src/main/resources/testData/osdu_wks_IndexPropertyPathConfiguration_v1.json b/testing/indexer-test-core/src/main/resources/testData/osdu_wks_IndexPropertyPathConfiguration_v1.json
index f8ba28eb6..075c70f17 100644
--- a/testing/indexer-test-core/src/main/resources/testData/osdu_wks_IndexPropertyPathConfiguration_v1.json
+++ b/testing/indexer-test-core/src/main/resources/testData/osdu_wks_IndexPropertyPathConfiguration_v1.json
@@ -51,11 +51,11 @@
"Paths": [{
"ValueExtraction": {
"RelatedConditionMatches": [
- "tenant1:reference-data--AliasNameType:UniqueIdentifier:",
- "tenant1:reference-data--AliasNameType:RegulatoryName:",
- "tenant1:reference-data--AliasNameType:PreferredName:",
- "tenant1:reference-data--AliasNameType:CommonName:",
- "tenant1:reference-data--AliasNameType:ShortName:"
+ "^[\\w\\-\\.]+:reference-data\\-\\-AliasNameType:UniqueIdentifier:$",
+ "^[\\w\\-\\.]+:reference-data\\-\\-AliasNameType:RegulatoryName:$",
+ "^[\\w\\-\\.]+:reference-data\\-\\-AliasNameType:PreferredName:$",
+ "^[\\w\\-\\.]+:reference-data\\-\\-AliasNameType:CommonName:$",
+ "^[\\w\\-\\.]+:reference-data\\-\\-AliasNameType:ShortName:$"
],
"RelatedConditionProperty": "data.NameAliases[].AliasNameTypeID",
"ValuePath": "data.NameAliases[].AliasName"
--
GitLab