Merge branch 'augmenter_safe_guard' into 'master'

Prevent any exception from Augmenter functions to corrupt the normal indexing

See merge request !583

indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/PropertyConfigurations.java

@@ -60,4 +60,40 @@ public class PropertyConfigurations {
         }
         return new ArrayList<>(relatedObjectKinds);
     }
+
+    public boolean hasValidCode() {
+        // It is just basic test to detect mistake
+        if(Strings.isNullOrEmpty(this.code)) {
+            return false;
+        }
+
+        String[] parts = this.code.split(":");
+        if(parts.length != 4) {
+            return false;
+        }
+        // Version must be ended with dot and major version only
+        // e.g. "Code": "osdu:wks:master-data--Well:1."
+        String version = parts[3];
+        return (version.length() > 1 && version.indexOf(".") == version.length() - 1);
+    }
+
+    public boolean hasValidConfigurations() {
+        if(configurations == null || configurations.isEmpty()) {
+            return false;
+        }
+
+        return configurations.stream().anyMatch(config -> config.isValid());
+    }
+
+    public boolean hasInvalidConfigurations() {
+        if(configurations == null || configurations.isEmpty()) {
+            return false;
+        }
+
+        return configurations.stream().anyMatch(config -> !config.isValid());
+    }
+
+    public boolean isValid() {
+        return hasValidCode() && hasValidConfigurations();
+    }
 }

indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java

@@ -167,10 +167,15 @@ public class IndexSchemaServiceImpl implements IndexSchemaService {
                 return this.getEmptySchema(kind);
             } else {
                 if(augmenterSetting.isEnabled()) {
-                    // Merge schema of the extended properties if needed
-                    PropertyConfigurations propertyConfigurations = propertyConfigurationsService.getPropertyConfigurations(kind);
-                    if (propertyConfigurations != null) {
-                        schema = mergeSchemaFromPropertyConfiguration(schema, propertyConfigurations);
+                    try {
+                        // Merge schema of the extended properties if needed
+                        PropertyConfigurations propertyConfigurations = propertyConfigurationsService.getPropertyConfigurations(kind);
+                        if (propertyConfigurations != null) {
+                            schema = mergeSchemaFromPropertyConfiguration(schema, propertyConfigurations);
+                        }
+                    }
+                    catch(Exception ex) {
+                        log.error(String.format("Augmenter: Failed to merge schema of the extended properties for kind: '%s'", kind), ex);
                     }
                 }
 

indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java

@@ -157,11 +157,16 @@ public class IndexerServiceImpl implements IndexerService {
                 retryAndEnqueueFailedRecords(recordInfos, retryRecordIds, message);
             }
 
-            if(this.augmenterSetting.isEnabled()) {
-                Map<String, List<String>> upsertKindIds = getUpsertRecordIdsForConfigurationsEnabledKinds(upsertRecordMap, retryRecordIds);
-                Map<String, List<String>> deleteKindIds = getDeleteRecordIdsForConfigurationsEnabledKinds(deleteRecordMap, retryRecordIds);
-                if (!upsertKindIds.isEmpty() || !deleteKindIds.isEmpty()) {
-                    propertyConfigurationsService.updateAssociatedRecords(message, upsertKindIds, deleteKindIds);
+            if (this.augmenterSetting.isEnabled()) {
+                try {
+                    Map<String, List<String>> upsertKindIds = getUpsertRecordIdsForConfigurationsEnabledKinds(upsertRecordMap, retryRecordIds);
+                    Map<String, List<String>> deleteKindIds = getDeleteRecordIdsForConfigurationsEnabledKinds(deleteRecordMap, retryRecordIds);
+                    if (!upsertKindIds.isEmpty() || !deleteKindIds.isEmpty()) {
+                        propertyConfigurationsService.updateAssociatedRecords(message, upsertKindIds, deleteKindIds);
+                    }
+                }
+                catch(Exception ex) {
+                    jaxRsDpsLog.error("Augmenter: Failed to update associated records", ex);
                 }
             }
         } catch (IOException e) {
@@ -346,14 +351,19 @@ public class IndexerServiceImpl implements IndexerService {
                 }
 
                 if(this.augmenterSetting.isEnabled()) {
-                    if(propertyConfigurationsService.isPropertyConfigurationsEnabled(storageRecord.getKind())) {
-                        PropertyConfigurations propertyConfigurations = propertyConfigurationsService.getPropertyConfigurations(storageRecord.getKind());
-                        if (propertyConfigurations != null) {
-                            // Merge extended properties
-                            dataMap = mergeDataFromPropertyConfiguration(storageRecord.getId(), dataMap, propertyConfigurations);
+                    try {
+                        if (propertyConfigurationsService.isPropertyConfigurationsEnabled(storageRecord.getKind())) {
+                            PropertyConfigurations propertyConfigurations = propertyConfigurationsService.getPropertyConfigurations(storageRecord.getKind());
+                            if (propertyConfigurations != null) {
+                                // Merge extended properties
+                                dataMap = mergeDataFromPropertyConfiguration(storageRecord.getId(), dataMap, propertyConfigurations);
+                            }
+                            // We cache the dataMap in case the update of this object will trigger update of the related objects.
+                            propertyConfigurationsService.cacheDataRecord(storageRecord.getId(), storageRecord.getKind(), dataMap);
                         }
-                        // We cache the dataMap in case the update of this object will trigger update of the related objects.
-                        propertyConfigurationsService.cacheDataRecord(storageRecord.getId(), storageRecord.getKind(), dataMap);
+                    }
+                    catch(Exception ex) {
+                        jaxRsDpsLog.error(String.format("Augmenter: Failed to merge extended properties of the record with id: '%s' and kind: '%s'", storageRecord.getId(), storageRecord.getKind()), ex);
                     }
                 }
 

indexer-core/src/main/java/org/opengroup/osdu/indexer/service/PropertyConfigurationsServiceImpl.java

@@ -126,6 +126,30 @@ public class PropertyConfigurationsServiceImpl implements PropertyConfigurations
         if (configuration == null) {
             configuration = searchConfigurations(kind);
             if (configuration != null) {
+                if(configuration.isValid()) {
+                    // Log for debug
+                    if(configuration.hasInvalidConfigurations()) {
+                        String msg = String.format("PropertyConfigurations: it has invalid PropertyConfiguration for configurations with name '%s':", configuration.getName());
+                        this.jaxRsDpsLog.warning(msg);
+                    }
+                }
+                else {
+                    // Log for debug
+                    StringBuilder msgBuilder = new StringBuilder();
+                    msgBuilder.append(String.format("PropertyConfigurations: it is invalid for configurations with name '%s':", configuration.getName()));
+                    if(!configuration.hasValidCode()) {
+                        msgBuilder.append(System.lineSeparator());
+                        msgBuilder.append(String.format("The code '%s' is invalid. It should be a valid kind with major version ended with '.'", configuration.getCode()));
+                    }
+                    if(!configuration.hasValidConfigurations()) {
+                        msgBuilder.append(System.lineSeparator());
+                        msgBuilder.append("It does not have any valid PropertyConfiguration");
+                    }
+                    this.jaxRsDpsLog.warning(msgBuilder.toString());
+
+                    configuration = EMPTY_CONFIGURATIONS; // reset
+                }
+
                 propertyConfigurationCache.put(kind, configuration);
             } else {
                 // It is common that a kind does not have extended property. So we need to cache an empty configuration

indexer-core/src/test/java/org/opengroup/osdu/indexer/model/indexproperty/PropertyConfigurationsTest.java

+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model.indexproperty;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.SneakyThrows;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.List;
+
+@RunWith(SpringRunner.class)
+public class PropertyConfigurationsTest {
+    private PropertyConfigurations configurations;
+
+    @Before
+    public void setup() throws JsonProcessingException {
+        String jsonText = getJsonFromFile("well_configuration_record.json");
+        ObjectMapper objectMapper = new ObjectMapper();
+        configurations = objectMapper.readValue(jsonText, PropertyConfigurations.class);
+    }
+
+    @Test
+    public void isValid() {
+        Assert.assertTrue(configurations.hasValidCode());
+        Assert.assertTrue(configurations.hasValidConfigurations());
+        Assert.assertFalse(configurations.hasInvalidConfigurations());
+        Assert.assertTrue(configurations.isValid());
+    }
+
+    @Test
+    public void hasInvalidCode() {
+        String code = configurations.getCode();
+
+        configurations.setCode(code + "0.0");
+        Assert.assertFalse(configurations.hasValidCode());
+        Assert.assertFalse(configurations.isValid());
+
+        configurations.setCode("a:b:1.");
+        Assert.assertFalse(configurations.hasValidCode());
+        Assert.assertFalse(configurations.isValid());
+
+        configurations.setCode("");
+        Assert.assertFalse(configurations.hasValidCode());
+        Assert.assertFalse(configurations.isValid());
+
+        configurations.setCode(null);
+        Assert.assertFalse(configurations.hasValidCode());
+        Assert.assertFalse(configurations.isValid());
+    }
+
+    @Test
+    public void hasNoValidConfigurations() {
+        List<PropertyConfiguration> propertyConfigurations = configurations.getConfigurations();
+
+        configurations.setConfigurations(new ArrayList<>());
+        Assert.assertFalse(configurations.hasValidConfigurations());
+        Assert.assertFalse(configurations.hasInvalidConfigurations());
+        Assert.assertFalse(configurations.isValid());
+
+        configurations.setConfigurations(null);
+        Assert.assertFalse(configurations.hasValidConfigurations());
+        Assert.assertFalse(configurations.hasInvalidConfigurations());
+        Assert.assertFalse(configurations.isValid());
+
+        propertyConfigurations.forEach(p -> p.setPolicy(""));
+        configurations.setConfigurations(propertyConfigurations);
+        Assert.assertFalse(configurations.hasValidConfigurations());
+        Assert.assertTrue(configurations.hasInvalidConfigurations());
+        Assert.assertFalse(configurations.isValid());
+    }
+
+    @Test
+    public void hasPartialValidConfigurations() {
+        List<PropertyConfiguration> propertyConfigurations = configurations.getConfigurations();
+        Assert.assertEquals(2, propertyConfigurations.size());
+
+        propertyConfigurations.get(0).setPolicy("");
+        Assert.assertTrue(configurations.hasValidConfigurations());
+        Assert.assertTrue(configurations.hasInvalidConfigurations());
+        Assert.assertTrue(configurations.isValid());
+    }
+
+    @SneakyThrows
+    private String getJsonFromFile(String file) {
+        InputStream inStream = this.getClass().getResourceAsStream("/indexproperty/" + file);
+        BufferedReader br = new BufferedReader(new InputStreamReader(inStream));
+        StringBuilder stringBuilder = new StringBuilder();
+        String sCurrentLine;
+        while ((sCurrentLine = br.readLine()) != null)
+        {
+            stringBuilder.append(sCurrentLine).append("\n");
+        }
+        return stringBuilder.toString();
+    }
+}