Enhance augmenter to support flexible matching

indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/PropertyConfiguration.java

@@ -19,6 +19,7 @@ import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import lombok.Data;
 import lombok.ToString;
+import org.opengroup.osdu.indexer.util.PropertyUtil;
 
 import java.util.List;
 
@@ -41,6 +42,10 @@ public class PropertyConfiguration {
     @JsonProperty("Paths")
     private List<PropertyPath> paths;
 
+    public String getExtendedPropertyName() {
+        return PropertyUtil.removeDataPrefix(this.name);
+    }
+
     public boolean isExtractFirstMatch() {
         return EXTRACT_FIRST_MATCH_POLICY.equalsIgnoreCase(policy);
     }
@@ -54,7 +59,12 @@ public class PropertyConfiguration {
         return hasValidPath && (isExtractFirstMatch() || isExtractAllMatches());
     }
 
-    public String getRelatedObjectKind() {
+    /**
+     * Though one extended property can be mapped to a property from different kinds of source objects, we assume
+     * that all those source properties should have the same schema.
+     * @return
+     */
+    public String getFirstRelatedObjectKind() {
         if(paths != null) {
             for (PropertyPath path : paths) {
                 if (path.isValid() && path.hasValidRelatedObjectsSpec()) {

indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/PropertyConfigurations.java

@@ -53,7 +53,9 @@ public class PropertyConfigurations {
 
         Set<String> relatedObjectKinds = new HashSet<>();
         for(PropertyConfiguration configuration : configurations) {
-            String relatedObjectKind = configuration.getRelatedObjectKind();
+            // In the same configuration, we can use any related object kind to
+            // resolve the schema of the extended property in this configuration
+            String relatedObjectKind = configuration.getFirstRelatedObjectKind();
             if(!Strings.isNullOrEmpty(relatedObjectKind)) {
                 relatedObjectKinds.add(relatedObjectKind);
             }

indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/RelatedCondition.java

@@ -19,6 +19,7 @@ import com.fasterxml.jackson.annotation.JsonInclude;
 import com.google.api.client.util.Strings;
 import lombok.Data;
 import lombok.ToString;
+import org.opengroup.osdu.indexer.util.PropertyUtil;
 
 import java.util.List;
 
@@ -32,27 +33,44 @@ public class RelatedCondition {
 
     protected List<String> relatedConditionMatches;
 
+    protected boolean hasCondition() {
+        return !Strings.isNullOrEmpty(relatedConditionProperty) &&
+               relatedConditionMatches != null &&
+               !relatedConditionMatches.isEmpty();
+    }
+
     protected boolean hasValidCondition(String property) {
-        if(Strings.isNullOrEmpty(property) ||
-                Strings.isNullOrEmpty(relatedConditionProperty) ||
-                relatedConditionMatches == null ||
-                relatedConditionMatches.isEmpty())
+        if(Strings.isNullOrEmpty(property) || !this.hasCondition())
             return false;
 
-        if(property.indexOf(ARRAY_SYMBOL + "." ) <= 0 || property.endsWith(ARRAY_SYMBOL) ||
-           relatedConditionProperty.indexOf(ARRAY_SYMBOL + "." ) <= 0 || relatedConditionProperty.endsWith(ARRAY_SYMBOL))
+        if((property.endsWith(ARRAY_SYMBOL) || relatedConditionProperty.endsWith(ARRAY_SYMBOL)) ||
+           (property.contains(ARRAY_SYMBOL) && !relatedConditionProperty.contains(ARRAY_SYMBOL)) ||
+           (!property.contains(ARRAY_SYMBOL) && relatedConditionProperty.contains(ARRAY_SYMBOL)))
             return false;
 
-        String delimiter = "\\[\\]\\.";
+        // If it is not nested object, it is valid in terms of syntax.
+        if(!property.contains(ARRAY_SYMBOL) && !relatedConditionProperty.contains(ARRAY_SYMBOL))
+            return true;
+
+        property = this.getSubstringWithLastArrayField(
+                PropertyUtil.removeDataPrefix(property));
+        String conditionProperty = this.getSubstringWithLastArrayField(
+                PropertyUtil.removeDataPrefix(relatedConditionProperty));
+
+        String delimiter = "\\.";
         String[] propertyParts = property.split(delimiter);
-        String[] relatedConditionPropertyParts = relatedConditionProperty.split(delimiter);
-        if(propertyParts.length != relatedConditionPropertyParts.length || propertyParts.length < 2)
+        String[] relatedConditionPropertyParts = conditionProperty.split(delimiter);
+        if(propertyParts.length != relatedConditionPropertyParts.length)
             return false;
 
-        for(int i = 0; i < propertyParts.length -1; i++) {
+        for(int i = 0; i < propertyParts.length; i++) {
             if(!propertyParts[i].equals(relatedConditionPropertyParts[i]))
                 return false;
         }
         return true;
     }
+
+    private String getSubstringWithLastArrayField(String property) {
+        return property.substring(0, property.lastIndexOf(ARRAY_SYMBOL));
+    }
 }

indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/RelatedObjectsSpec.java

@@ -38,8 +38,10 @@ public class RelatedObjectsSpec extends RelatedCondition {
     public boolean isParentToChildren() { return PARENT_TO_CHILDREN.equalsIgnoreCase(relationshipDirection); }
 
     public boolean isValid() {
-        return !Strings.isNullOrEmpty(relatedObjectID) && !Strings.isNullOrEmpty(relatedObjectKind) &&
-                (isChildToParent() || isParentToChildren());
+        return !Strings.isNullOrEmpty(relatedObjectID) &&
+               !Strings.isNullOrEmpty(relatedObjectKind) &&
+                (isChildToParent() || isParentToChildren()) &&
+               (!this.hasCondition() || this.hasValidCondition());
     }
 
     /**
@@ -48,6 +50,6 @@ public class RelatedObjectsSpec extends RelatedCondition {
      * @return
      */
     public boolean hasValidCondition() {
-        return isValid() && super.hasValidCondition(relatedObjectID);
+        return super.hasValidCondition(relatedObjectID);
     }
 }

indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/ValueExtraction.java

@@ -29,7 +29,7 @@ public class ValueExtraction extends RelatedCondition {
     private String valuePath;
 
     public boolean isValid() {
-        return !Strings.isNullOrEmpty(valuePath);
+        return !Strings.isNullOrEmpty(valuePath) && (!this.hasCondition() || this.hasValidCondition());
     }
 
     /**
@@ -38,6 +38,6 @@ public class ValueExtraction extends RelatedCondition {
      * @return
      */
     public boolean hasValidCondition() {
-        return isValid() && super.hasValidCondition(valuePath);
+        return super.hasValidCondition(valuePath);
     }
 }

indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SearchServiceImpl.java

@@ -33,6 +33,7 @@ import java.net.URISyntaxException;
 
 @Component
 public class SearchServiceImpl implements SearchService {
+    private static final String ERROR_MESSAGE = "Search service: failed to call the search service.";
     private static final String QUERY_PATH = "query";
     private static final String QUERY_WITH_CURSOR_PATH = "query_with_cursor";
     private final Gson gson = new Gson();
@@ -78,9 +79,9 @@ public class SearchServiceImpl implements SearchService {
                 return gson.fromJson(response.getBody(), SearchResponse.class);
             } else {
                 if (response != null)
-                    jaxRsDpsLog.error(String.format("Search service: failed to call the search service: %d", response.getResponseCode()));
+                    jaxRsDpsLog.error(ERROR_MESSAGE + String.format(" responseCode = %d", response.getResponseCode()));
                 else
-                    jaxRsDpsLog.error(String.format("Search service: failed to call the search service. The response is null."));
+                    jaxRsDpsLog.error(String.format(ERROR_MESSAGE + " The response is null."));
                 return new SearchResponse();
             }
         }
@@ -88,7 +89,7 @@ public class SearchServiceImpl implements SearchService {
             throw ex;
         }
         catch(Exception ex) {
-            jaxRsDpsLog.error(String.format("Search service: failed to call the search service", ex));
+            jaxRsDpsLog.error(ERROR_MESSAGE, ex);
             throw new URISyntaxException(ex.getMessage(), "Unexpected exception type: " + ex.getClass().getName());
         }
     }

indexer-core/src/main/java/org/opengroup/osdu/indexer/util/PropertyUtil.java

@@ -21,6 +21,8 @@ import com.google.common.collect.Maps;
 import org.opengroup.osdu.indexer.model.Kind;
 
 import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 public class PropertyUtil {
     public static final String DATA_VIRTUAL_DEFAULT_LOCATION = "data.VirtualProperties.DefaultLocation";
@@ -42,6 +44,33 @@ public class PropertyUtil {
         return !Strings.isNullOrEmpty(propertyPath) && (propertyPath.startsWith(parentPropertyPath + PROPERTY_DELIMITER) || propertyPath.equals(parentPropertyPath));
     }
 
+    public static Map<String, Object> getValueOfNoneNestedProperty(String propertyPath, Map<String, Object> data) {
+        if(Strings.isNullOrEmpty(propertyPath) || propertyPath.contains(ARRAY_SYMBOL) || data == null || data.isEmpty())
+            return new HashMap<>();
+
+        Map<String, Object> values = new HashMap<>();
+        if(data.containsKey(propertyPath)) {
+            values.put(propertyPath, data.get(propertyPath));
+        }
+        else {
+            for (String key : data.keySet()) {
+                if (propertyPath.startsWith(key + PROPERTY_DELIMITER)) {
+                    Object v = data.get(key);
+                    if (v instanceof Map) {
+                        propertyPath = propertyPath.substring((key + PROPERTY_DELIMITER).length());
+                        Map<String, Object> subPropertyValues = getValueOfNoneNestedProperty(propertyPath, (Map<String, Object>) v);
+                        for (Map.Entry<String, Object> entry: subPropertyValues.entrySet()) {
+                            values.put(key + PROPERTY_DELIMITER + entry.getKey(), entry.getValue());
+                        }
+                    }
+                } else if (key.startsWith(propertyPath + PROPERTY_DELIMITER)) {
+                    values.put(key, data.get(key));
+                }
+            }
+        }
+        return values;
+    }
+
     public static boolean hasSameMajorKind(String left, String right) {
         try {
             Kind leftKind = new Kind(left);
@@ -258,4 +287,22 @@ public class PropertyUtil {
 
         return new ArrayList<>(changedProperties);
     }
+
+    /*
+     * matchCondition can be either non-empty string or regular expression
+     */
+    public static boolean isMatch(String propertyValue, String matchCondition) {
+        if(Strings.isNullOrEmpty(propertyValue) || Strings.isNullOrEmpty(matchCondition))
+            return false;
+
+        try {
+            Pattern pattern = Pattern.compile(matchCondition);
+            Matcher matcher = pattern.matcher(propertyValue);
+            return matcher.find();
+        }
+        catch(Exception ex) {
+            // If matchCondition is not regex, do string compare
+            return propertyValue.equals(matchCondition);
+        }
+    }
 }

indexer-core/src/test/java/org/opengroup/osdu/indexer/model/indexproperty/RelatedObjectsSpecTest.java

@@ -108,14 +108,14 @@ public class RelatedObjectsSpecTest {
     }
 
     @Test
-    public void hasValidCondition_return_false_for_none_nested_property() {
+    public void hasValidCondition_return_true_for_none_nested_property() {
         RelatedObjectsSpec spec = new RelatedObjectsSpec();
         spec.setRelatedObjectKind("osdu:wks:master-data--GeoPoliticalEntity:1.");
         spec.setRelatedObjectID("data.GeoContexts.GeoPoliticalEntityID");
         spec.setRelatedConditionProperty("data.GeoContexts.GeoTypeID");
         List<String> matches = Arrays.asList("opendes:reference-data--GeoPoliticalEntityType:Country:");
         spec.setRelatedConditionMatches(matches);
-        Assert.assertFalse(spec.hasValidCondition());
+        Assert.assertTrue(spec.hasValidCondition());
     }
 
 }

indexer-core/src/test/java/org/opengroup/osdu/indexer/model/indexproperty/ValueExtractionTest.java

@@ -98,12 +98,22 @@ public class ValueExtractionTest {
     }
 
     @Test
-    public void hasValidCondition_return_false_for_none_nested_property() {
+    public void hasValidCondition_return_true_for_property_under_same_none_nested_property() {
         ValueExtraction valueExtraction = new ValueExtraction();
         valueExtraction.setValuePath("data.NameAliases.AliasName");
         valueExtraction.setRelatedConditionProperty("data.NameAliases.AliasNameTypeID");
         List<String> matches = Arrays.asList("opendes:reference-data--AliasNameType:UniqueIdentifier:","reference-data--AliasNameType:RegulatoryName:");
         valueExtraction.setRelatedConditionMatches(matches);
-        Assert.assertFalse(valueExtraction.hasValidCondition());
+        Assert.assertTrue(valueExtraction.hasValidCondition());
+    }
+
+    @Test
+    public void hasValidCondition_return_true_for_property_under_different_none_nested_property() {
+        ValueExtraction valueExtraction = new ValueExtraction();
+        valueExtraction.setValuePath("data.NameAliases.AliasName");
+        valueExtraction.setRelatedConditionProperty("data.AliasNameTypeID");
+        List<String> matches = Arrays.asList("opendes:reference-data--AliasNameType:UniqueIdentifier:","reference-data--AliasNameType:RegulatoryName:");
+        valueExtraction.setRelatedConditionMatches(matches);
+        Assert.assertTrue(valueExtraction.hasValidCondition());
     }
 }

indexer-core/src/test/java/org/opengroup/osdu/indexer/service/PropertyConfigurationsServiceImplTest.java

@@ -267,8 +267,8 @@ public class PropertyConfigurationsServiceImplTest {
                 Assert.assertEquals(value, extendedProperties.get(name));
             }
             else if(value instanceof List) {
-                List<String> expectedValues = (List<String>)value;
-                List<String> values = (List<String>)extendedProperties.get(name);
+                List<String> expectedValues = ((List<String>)value).stream().sorted().toList();
+                List<String> values = ((List<String>)extendedProperties.get(name)).stream().sorted().toList();
                 Assert.assertEquals(expectedValues.size(), values.size());
                 for(int i = 0; i < expectedValues.size(); i++) {
                     Assert.assertEquals(expectedValues.get(i), values.get(i));

indexer-core/src/test/java/org/opengroup/osdu/indexer/util/PropertyUtilTest.java

@@ -44,6 +44,41 @@ public class PropertyUtilTest {
         Assert.assertFalse(PropertyUtil.isPropertyPathMatched(null, "data.ProjectedBottomHole"));
     }
 
+    @Test
+    public void getValueOfNoneNestedProperty() {
+        Map<String, Object> data = this.getDataMap("wellLog.json");
+
+        Map<String, Object> value = PropertyUtil.getValueOfNoneNestedProperty("SpatialLocation.SpatialGeometryTypeID", data);
+        Assert.assertTrue(value.containsKey("SpatialLocation.SpatialGeometryTypeID"));
+        Assert.assertEquals("opendes:reference-data--SpatialGeometryType:Point:", value.get("SpatialLocation.SpatialGeometryTypeID").toString());
+
+        value = PropertyUtil.getValueOfNoneNestedProperty("SpatialLocation.Wgs84Coordinates.type", data);
+        Assert.assertTrue(value.containsKey("SpatialLocation.Wgs84Coordinates.type"));
+        Assert.assertEquals("geometrycollection", value.get("SpatialLocation.Wgs84Coordinates.type").toString());
+
+        List<String> propertyNamesWithValues = Arrays.asList(
+                "SpatialLocation.SpatialGeometryTypeID",
+                "SpatialLocation.Wgs84Coordinates",
+                "SpatialLocation.IsDecimated");
+        List<String> propertyNamesWithNullValues = Arrays.asList(
+                "SpatialLocation.SpatialParameterTypeID",
+                "SpatialLocation.CoordinateQualityCheckPerformedBy",
+                "SpatialLocation.QualitativeSpatialAccuracyTypeID",
+                "SpatialLocation.QuantitativeAccuracyBandID");
+        value = PropertyUtil.getValueOfNoneNestedProperty("SpatialLocation", data);
+        for(String propertyName : propertyNamesWithValues) {
+            Assert.assertTrue(value.containsKey(propertyName));
+            Assert.assertNotNull(value.get(propertyName));
+        }
+        for(String propertyName : propertyNamesWithNullValues) {
+            Assert.assertTrue(value.containsKey(propertyName));
+            Assert.assertNull(value.get(propertyName));
+        }
+
+        value = PropertyUtil.getValueOfNoneNestedProperty("Curves[].CurveID", data);
+        Assert.assertTrue(value.isEmpty());
+    }
+
     @Test
     public void hasSameMajorVersion() {
         Assert.assertTrue(PropertyUtil.hasSameMajorKind("osdu:wks:master-data--Well:1.0.0", "osdu:wks:master-data--Well:1.0.0"));
@@ -272,6 +307,20 @@ public class PropertyUtilTest {
         changedProperties.forEach(p -> Assert.assertTrue(expectedChangedWellLogProperties.contains(p)));
     }
 
+    @Test
+    public void isMatch_for_String() {
+        Assert.assertTrue(PropertyUtil.isMatch("opendes:reference-data--AliasNameType:UniqueIdentifier:", "opendes:reference-data--AliasNameType:UniqueIdentifier:"));
+        Assert.assertFalse(PropertyUtil.isMatch("OPENDES:reference-data--AliasNameType:UniqueIdentifier:", "opendes:reference-data--AliasNameType:UniqueIdentifier:"));
+    }
+
+    @Test
+    public void isMatch_for_Regex() {
+        Assert.assertTrue(PropertyUtil.isMatch("opendes:master-data--Wellbore:F03B95515F91--2B886EAF-83A9-4811-9068", "^[\\w\\-\\.]+:master-data\\-\\-Wellbore:[\\w\\-\\.\\:\\%]+$"));
+        Assert.assertTrue(PropertyUtil.isMatch("opendes:work-product-component--WellLog:8CAFB37C-A674-42F8-ABCD-34BA8E6C1480", "^[\\w\\-\\.]+:work-product-component\\-\\-WellLog:[\\w\\-\\.\\:\\%]+$"));
+        Assert.assertTrue(PropertyUtil.isMatch("opendes:work-product-component--Document:wks-0f2546ca6b45960fa32db", "^[\\w\\-\\.]+:work-product-component\\-\\-Document:[\\w\\-\\.\\:\\%]+$"));
+    }
+
+
     private Map<String, Object> getDataMap(String file) {
         String jsonText = getJsonFromFile(file);
         Type type = new TypeToken<Map<String, Object>>() {}.getType();

indexer-core/src/test/resources/indexproperty/well_configuration_record.json

@@ -21,7 +21,7 @@
     "Paths": [{
       "RelatedObjectsSpec.RelatedObjectID": "data.GeoContexts[].GeoPoliticalEntityID",
       "RelatedObjectsSpec.RelatedConditionMatches": [
-        "opendes:reference-data--GeoPoliticalEntityType:Country:"
+        "^[\\w\\-\\.]+:reference-data\\-\\-GeoPoliticalEntityType:Country:$"
       ],
       "ValueExtraction.ValuePath": "data.GeoPoliticalEntityName",
       "RelatedObjectsSpec.RelatedObjectKind": "osdu:wks:master-data--GeoPoliticalEntity:1.",
@@ -41,11 +41,11 @@
       "RelatedObjectsSpec.RelatedConditionProperty": null,
       "RelatedObjectsSpec.RelationshipDirection": null,
       "ValueExtraction.RelatedConditionMatches": [
-        "opendes:reference-data--AliasNameType:UniqueIdentifier:",
-        "opendes:reference-data--AliasNameType:RegulatoryName:",
-        "opendes:reference-data--AliasNameType:PreferredName:",
-        "opendes:reference-data--AliasNameType:CommonName:",
-        "opendes:reference-data--AliasNameType:ShortName:"
+        "^[\\w\\-\\.]+:reference-data--AliasNameType:UniqueIdentifier:$",
+        "^[\\w\\-\\.]+:reference-data--AliasNameType:RegulatoryName:$",
+        "^[\\w\\-\\.]+:reference-data--AliasNameType:PreferredName:$",
+        "^[\\w\\-\\.]+:reference-data--AliasNameType:CommonName:$",
+        "^[\\w\\-\\.]+:reference-data--AliasNameType:ShortName:$"
       ],
       "ValueExtraction.RelatedConditionProperty": "data.NameAliases[].AliasNameTypeID"
     }

testing/indexer-test-core/src/main/resources/testData/osdu_wks_IndexPropertyPathConfiguration_v1.json

@@ -51,11 +51,11 @@
                     "Paths": [{
                             "ValueExtraction": {
                                 "RelatedConditionMatches": [
-                                    "tenant1:reference-data--AliasNameType:UniqueIdentifier:",
-                                    "tenant1:reference-data--AliasNameType:RegulatoryName:",
-                                    "tenant1:reference-data--AliasNameType:PreferredName:",
-                                    "tenant1:reference-data--AliasNameType:CommonName:",
-                                    "tenant1:reference-data--AliasNameType:ShortName:"
+                                    "^[\\w\\-\\.]+:reference-data\\-\\-AliasNameType:UniqueIdentifier:$",
+                                    "^[\\w\\-\\.]+:reference-data\\-\\-AliasNameType:RegulatoryName:$",
+                                    "^[\\w\\-\\.]+:reference-data\\-\\-AliasNameType:PreferredName:$",
+                                    "^[\\w\\-\\.]+:reference-data\\-\\-AliasNameType:CommonName:$",
+                                    "^[\\w\\-\\.]+:reference-data\\-\\-AliasNameType:ShortName:$"
                                 ],
                                 "RelatedConditionProperty": "data.NameAliases[].AliasNameTypeID",
                                 "ValuePath": "data.NameAliases[].AliasName"