Prevent any exception from Augmenter functions to corrupt the normal data...

Prevent any exception from Augmenter functions to corrupt the normal data indexing and log invalid PropertyConfigurations caused by user mistake

(cherry picked from commit e8e1dfe9)

indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/PropertyConfigurations.java

@@ -60,4 +60,40 @@ public class PropertyConfigurations {
         }
         return new ArrayList<>(relatedObjectKinds);
     }
+
+    public boolean hasValidCode() {
+        // It is just basic test to detect mistake
+        if(Strings.isNullOrEmpty(this.code)) {
+            return false;
+        }
+
+        String[] parts = this.code.split(":");
+        if(parts.length != 4) {
+            return false;
+        }
+        // Version must be ended with dot and major version only
+        // e.g. "Code": "osdu:wks:master-data--Well:1."
+        String version = parts[3];
+        return (version.length() > 1 && version.indexOf(".") == version.length() - 1);
+    }
+
+    public boolean hasValidConfigurations() {
+        if(configurations == null || configurations.isEmpty()) {
+            return false;
+        }
+
+        return (configurations.stream().filter(config -> config.isValid()).count() > 0);
+    }
+
+    public boolean hasInvalidConfigurations() {
+        if(configurations == null || configurations.isEmpty()) {
+            return false;
+        }
+
+        return (configurations.stream().filter(config -> !config.isValid()).count() > 0);
+    }
+
+    public boolean isValid() {
+        return hasValidCode() && hasValidConfigurations();
+    }
 }

indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java

@@ -167,10 +167,15 @@ public class IndexSchemaServiceImpl implements IndexSchemaService {
                 return this.getEmptySchema(kind);
             } else {
                 if(augmenterSetting.isEnabled()) {
-                    // Merge schema of the extended properties if needed
-                    PropertyConfigurations propertyConfigurations = propertyConfigurationsService.getPropertyConfigurations(kind);
-                    if (propertyConfigurations != null) {
-                        schema = mergeSchemaFromPropertyConfiguration(schema, propertyConfigurations);
+                    try {
+                        // Merge schema of the extended properties if needed
+                        PropertyConfigurations propertyConfigurations = propertyConfigurationsService.getPropertyConfigurations(kind);
+                        if (propertyConfigurations != null) {
+                            schema = mergeSchemaFromPropertyConfiguration(schema, propertyConfigurations);
+                        }
+                    }
+                    catch(Exception ex) {
+                        log.warning(String.format("Augmenter: Failed to merge schema of the extended properties for kind: '%s'", kind), ex);
                     }
                 }
 

indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java

@@ -157,11 +157,16 @@ public class IndexerServiceImpl implements IndexerService {
                 retryAndEnqueueFailedRecords(recordInfos, retryRecordIds, message);
             }
 
-            if(this.augmenterSetting.isEnabled()) {
-                Map<String, List<String>> upsertKindIds = getUpsertRecordIdsForConfigurationsEnabledKinds(upsertRecordMap, retryRecordIds);
-                Map<String, List<String>> deleteKindIds = getDeleteRecordIdsForConfigurationsEnabledKinds(deleteRecordMap, retryRecordIds);
-                if (!upsertKindIds.isEmpty() || !deleteKindIds.isEmpty()) {
-                    propertyConfigurationsService.updateAssociatedRecords(message, upsertKindIds, deleteKindIds);
+            if (this.augmenterSetting.isEnabled()) {
+                try {
+                    Map<String, List<String>> upsertKindIds = getUpsertRecordIdsForConfigurationsEnabledKinds(upsertRecordMap, retryRecordIds);
+                    Map<String, List<String>> deleteKindIds = getDeleteRecordIdsForConfigurationsEnabledKinds(deleteRecordMap, retryRecordIds);
+                    if (!upsertKindIds.isEmpty() || !deleteKindIds.isEmpty()) {
+                        propertyConfigurationsService.updateAssociatedRecords(message, upsertKindIds, deleteKindIds);
+                    }
+                }
+                catch(Exception ex) {
+                    jaxRsDpsLog.warning("Augmenter: Failed to update associated records", ex);
                 }
             }
         } catch (IOException e) {
@@ -346,14 +351,19 @@ public class IndexerServiceImpl implements IndexerService {
                 }
 
                 if(this.augmenterSetting.isEnabled()) {
-                    if(propertyConfigurationsService.isPropertyConfigurationsEnabled(storageRecord.getKind())) {
-                        PropertyConfigurations propertyConfigurations = propertyConfigurationsService.getPropertyConfigurations(storageRecord.getKind());
-                        if (propertyConfigurations != null) {
-                            // Merge extended properties
-                            dataMap = mergeDataFromPropertyConfiguration(storageRecord.getId(), dataMap, propertyConfigurations);
+                    try {
+                        if (propertyConfigurationsService.isPropertyConfigurationsEnabled(storageRecord.getKind())) {
+                            PropertyConfigurations propertyConfigurations = propertyConfigurationsService.getPropertyConfigurations(storageRecord.getKind());
+                            if (propertyConfigurations != null) {
+                                // Merge extended properties
+                                dataMap = mergeDataFromPropertyConfiguration(storageRecord.getId(), dataMap, propertyConfigurations);
+                            }
+                            // We cache the dataMap in case the update of this object will trigger update of the related objects.
+                            propertyConfigurationsService.cacheDataRecord(storageRecord.getId(), storageRecord.getKind(), dataMap);
                         }
-                        // We cache the dataMap in case the update of this object will trigger update of the related objects.
-                        propertyConfigurationsService.cacheDataRecord(storageRecord.getId(), storageRecord.getKind(), dataMap);
+                    }
+                    catch(Exception ex) {
+                        jaxRsDpsLog.warning(String.format("Augmenter: Failed to merge extended properties of the record with id: '%s' and kind: '%s'", storageRecord.getId(), storageRecord.getKind()), ex);
                     }
                 }
 

indexer-core/src/main/java/org/opengroup/osdu/indexer/service/PropertyConfigurationsServiceImpl.java

@@ -126,6 +126,30 @@ public class PropertyConfigurationsServiceImpl implements PropertyConfigurations
         if (configuration == null) {
             configuration = searchConfigurations(kind);
             if (configuration != null) {
+                if(configuration.isValid()) {
+                    // Log for debug
+                    if(configuration.hasInvalidConfigurations()) {
+                        String msg = String.format("PropertyConfigurations: it has invalid PropertyConfiguration for configurations with name '%s':", configuration.getName());
+                        this.jaxRsDpsLog.debug(msg);
+                    }
+                }
+                else {
+                    // Log for debug
+                    StringBuilder msgBuilder = new StringBuilder();
+                    msgBuilder.append(String.format("PropertyConfigurations: it is invalid for configurations with name '%s':", configuration.getName()));
+                    if(!configuration.hasValidCode()) {
+                        msgBuilder.append(System.lineSeparator());
+                        msgBuilder.append(String.format("The code '%s' is invalid. It should be a valid kind with major version ended with '.'", configuration.getCode()));
+                    }
+                    if(!configuration.hasValidConfigurations()) {
+                        msgBuilder.append(System.lineSeparator());
+                        msgBuilder.append("It does not have any valid PropertyConfiguration");
+                    }
+                    this.jaxRsDpsLog.debug(msgBuilder.toString());
+
+                    configuration = EMPTY_CONFIGURATIONS; // reset
+                }
+
                 propertyConfigurationCache.put(kind, configuration);
             } else {
                 // It is common that a kind does not have extended property. So we need to cache an empty configuration

indexer-core/src/test/java/org/opengroup/osdu/indexer/model/indexproperty/PropertyConfigurationsTest.java

+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model.indexproperty;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.SneakyThrows;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.List;
+
+@RunWith(SpringRunner.class)
+public class PropertyConfigurationsTest {
+    private PropertyConfigurations configurations;
+
+    @Before
+    public void setup() throws JsonProcessingException {
+        String jsonText = getJsonFromFile("well_configuration_record.json");
+        ObjectMapper objectMapper = new ObjectMapper();
+        configurations = objectMapper.readValue(jsonText, PropertyConfigurations.class);
+    }
+
+    @Test
+    public void isValid() {
+        Assert.assertTrue(configurations.hasValidCode());
+        Assert.assertTrue(configurations.hasValidConfigurations());
+        Assert.assertTrue(configurations.isValid());
+    }
+
+    @Test
+    public void hasInvalidCode() {
+        String code = configurations.getCode();
+
+        configurations.setCode(code + "0.0");
+        Assert.assertFalse(configurations.hasValidCode());
+        Assert.assertFalse(configurations.isValid());
+
+        configurations.setCode("a:b:1.");
+        Assert.assertFalse(configurations.hasValidCode());
+        Assert.assertFalse(configurations.isValid());
+
+        configurations.setCode("");
+        Assert.assertFalse(configurations.hasValidCode());
+        Assert.assertFalse(configurations.isValid());
+
+        configurations.setCode(null);
+        Assert.assertFalse(configurations.hasValidCode());
+        Assert.assertFalse(configurations.isValid());
+    }
+
+    @Test
+    public void hasNoValidConfigurations() {
+        List<PropertyConfiguration> propertyConfigurations = configurations.getConfigurations();
+
+        configurations.setConfigurations(new ArrayList<>());
+        Assert.assertFalse(configurations.hasValidConfigurations());
+        Assert.assertFalse(configurations.isValid());
+
+        configurations.setConfigurations(null);
+        Assert.assertFalse(configurations.hasValidConfigurations());
+        Assert.assertFalse(configurations.isValid());
+
+        propertyConfigurations.forEach(p -> p.setPolicy(""));
+        configurations.setConfigurations(propertyConfigurations);
+        Assert.assertFalse(configurations.hasValidConfigurations());
+        Assert.assertFalse(configurations.isValid());
+    }
+
+    @Test
+    public void hasPartialValidConfigurations() {
+        List<PropertyConfiguration> propertyConfigurations = configurations.getConfigurations();
+        Assert.assertEquals(2, propertyConfigurations.size());
+
+        propertyConfigurations.get(0).setPolicy("");
+        Assert.assertTrue(configurations.hasValidConfigurations());
+        Assert.assertTrue(configurations.hasInvalidConfigurations());
+        Assert.assertTrue(configurations.isValid());
+    }
+
+    @SneakyThrows
+    private String getJsonFromFile(String file) {
+        InputStream inStream = this.getClass().getResourceAsStream("/indexproperty/" + file);
+        BufferedReader br = new BufferedReader(new InputStreamReader(inStream));
+        StringBuilder stringBuilder = new StringBuilder();
+        String sCurrentLine;
+        while ((sCurrentLine = br.readLine()) != null)
+        {
+            stringBuilder.append(sCurrentLine).append("\n");
+        }
+        return stringBuilder.toString();
+    }
+}