make converter configurable from a propeties file

indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java

@@ -17,6 +17,8 @@ import org.apache.http.HttpStatus;
 import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
 import org.opengroup.osdu.core.common.model.http.AppException;
 import org.opengroup.osdu.core.common.search.Preconditions;
+import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterConfig;
+import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterPropertiesConfig;
 import org.opengroup.osdu.indexer.schema.converter.tags.AllOfItem;
 import org.opengroup.osdu.indexer.schema.converter.tags.Definition;
 import org.opengroup.osdu.indexer.schema.converter.tags.Definitions;
@@ -28,50 +30,24 @@ import java.util.stream.Stream;
 public class PropertiesProcessor {
 
     private JaxRsDpsLog log;
+    private SchemaConverterConfig schemaConverterConfig;
 
     private static final String DEF_PREFIX = "#/definitions/";
 
-    private static final Set<String> SKIP_DEFINITIONS = new HashSet<>(
-            Arrays.asList("AbstractAnyCrsFeatureCollection.1.0.0",
-                          "anyCrsGeoJsonFeatureCollection"));
-
-    private static final Set<String> ARRAY_SUPPORTED_SIMPLE_TYPES = new HashSet<>(
-            Arrays.asList("boolean", "integer", "number", "string"));
-
-    private static final Map<String, String> SPEC_DEFINITION_TYPES = new HashMap<>();
-
-    private static final Map<String, String> PRIMITIVE_TYPES_MAP = new HashMap<>();
-
     private final Definitions definitions;
     private final String pathPrefix;
     private final String pathPrefixWithDot;
 
-    static {
-        SPEC_DEFINITION_TYPES.put("AbstractFeatureCollection.1.0.0", "core:dl:geoshape:1.0.0");
-        SPEC_DEFINITION_TYPES.put("core_dl_geopoint", "core:dl:geopoint:1.0.0");
-        SPEC_DEFINITION_TYPES.put("geoJsonFeatureCollection", "core:dl:geoshape:1.0.0");
-    }
-
-    static {
-        PRIMITIVE_TYPES_MAP.put("boolean", "bool");
-        PRIMITIVE_TYPES_MAP.put("number", "double");
-        PRIMITIVE_TYPES_MAP.put("date-time", "datetime");
-        PRIMITIVE_TYPES_MAP.put("date", "datetime");
-        PRIMITIVE_TYPES_MAP.put("time", "datetime");
-        PRIMITIVE_TYPES_MAP.put("int32", "int");
-        PRIMITIVE_TYPES_MAP.put("integer", "int");
-        PRIMITIVE_TYPES_MAP.put("int64", "long");
-    }
-
-    public PropertiesProcessor(Definitions definitions, JaxRsDpsLog log) {
-        this(definitions, null, log);
+    public PropertiesProcessor(Definitions definitions, JaxRsDpsLog log, SchemaConverterConfig schemaConverterConfig) {
+        this(definitions, null, log, schemaConverterConfig);
     }
 
-    public PropertiesProcessor(Definitions definitions, String pathPrefix, JaxRsDpsLog log) {
+    public PropertiesProcessor(Definitions definitions, String pathPrefix, JaxRsDpsLog log, SchemaConverterConfig schemaConverterConfig) {
         this.log = log;
         this.definitions = definitions;
         this.pathPrefix = pathPrefix;
         this.pathPrefixWithDot = Objects.isNull(pathPrefix)  || pathPrefix.isEmpty() ? "" : pathPrefix + ".";
+        this.schemaConverterConfig = schemaConverterConfig;
     }
 
     public Stream<Map<String, Object>> processItem(AllOfItem allOfItem) {
@@ -93,12 +69,12 @@ public class PropertiesProcessor {
 
         String definitionSubRef = ref.substring(DEF_PREFIX.length());
 
-        if (SKIP_DEFINITIONS.contains(definitionSubRef)) {
+        if (schemaConverterConfig.getSkippedDefinitions().contains(definitionSubRef)) {
             return Stream.empty();
         }
 
-        if (!Objects.isNull(SPEC_DEFINITION_TYPES.get(definitionSubRef))) {
-            return storageSchemaEntry(SPEC_DEFINITION_TYPES.get(definitionSubRef), pathPrefix);
+        if (!Objects.isNull(schemaConverterConfig.getSpecialDefinitionsMap().get(definitionSubRef))) {
+            return storageSchemaEntry(schemaConverterConfig.getSpecialDefinitionsMap().get(definitionSubRef), pathPrefix);
         }
 
         Definition definition = definitions.getDefinition(definitionSubRef);
@@ -120,7 +96,7 @@ public class PropertiesProcessor {
         }
 
         if ("array".equals(entry.getValue().getType())) {
-            if (ARRAY_SUPPORTED_SIMPLE_TYPES.contains(entry.getValue().getItems().getType())) {
+            if (schemaConverterConfig.getSupportedArrayTypes().contains(entry.getValue().getItems().getType())) {
                 return storageSchemaEntry("[]" + getTypeByDefinitionProperty(entry.getValue()), pathPrefixWithDot + entry.getKey());
             }
 
@@ -128,12 +104,13 @@ public class PropertiesProcessor {
         }
 
         if (!Objects.isNull(entry.getValue().getProperties())) {
-            PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey(), log);
+            PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey()
+                    , log, new SchemaConverterPropertiesConfig());
             return entry.getValue().getProperties().entrySet().stream().flatMap(propertiesProcessor::processPropertyEntry);
         }
 
         if (!Objects.isNull(entry.getValue().getRef())) {
-            return new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey(), log)
+            return new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey(), log, new SchemaConverterPropertiesConfig())
                     .processRef(entry.getValue().getRef());
         }
 
@@ -161,8 +138,8 @@ public class PropertiesProcessor {
 
         return !Objects.isNull(pattern) && pattern.startsWith("^srn") ? "link" :
                 !Objects.isNull(itemsPattern) && itemsPattern.startsWith("^srn") ? "link" :
-                !Objects.isNull(format)  ? PRIMITIVE_TYPES_MAP.getOrDefault(format, format) :
-                        !Objects.isNull(itemsType) ? PRIMITIVE_TYPES_MAP.getOrDefault(itemsType, itemsType) :
-                                PRIMITIVE_TYPES_MAP.getOrDefault(type, type);
+                !Objects.isNull(format)  ? schemaConverterConfig.getPrimitiveTypesMap().getOrDefault(format, format) :
+                        !Objects.isNull(itemsType) ? schemaConverterConfig.getPrimitiveTypesMap().getOrDefault(itemsType, itemsType) :
+                                schemaConverterConfig.getPrimitiveTypesMap().getOrDefault(type, type);
     }
 }

indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java

@@ -20,6 +20,7 @@ import org.apache.http.HttpStatus;
 import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
 import org.opengroup.osdu.core.common.model.http.AppException;
 import org.opengroup.osdu.core.common.search.Preconditions;
+import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterConfig;
 import org.opengroup.osdu.indexer.schema.converter.interfaces.SchemaToStorageFormat;
 import org.opengroup.osdu.indexer.schema.converter.tags.PropertiesData;
 import org.opengroup.osdu.indexer.schema.converter.tags.SchemaRoot;
@@ -36,15 +37,16 @@ import java.util.stream.Collectors;
 public class SchemaToStorageFormatImpl implements SchemaToStorageFormat {
 
     private ObjectMapper objectMapper;
-
     private JaxRsDpsLog log;
+    private SchemaConverterConfig schemaConverterConfig;
 
     @Inject
-    public SchemaToStorageFormatImpl(ObjectMapper objectMapper, JaxRsDpsLog log) {
+    public SchemaToStorageFormatImpl(ObjectMapper objectMapper, JaxRsDpsLog log, SchemaConverterConfig schemaConverterConfig) {
         Preconditions.checkNotNull(objectMapper, "objectMapper cannot be null");
 
         this.objectMapper = objectMapper;
         this.log = log;
+        this.schemaConverterConfig = schemaConverterConfig;
     }
 
     @Override
@@ -82,7 +84,7 @@ public class SchemaToStorageFormatImpl implements SchemaToStorageFormat {
         Preconditions.checkNotNull(objectMapper, "schemaServiceSchema cannot be null");
         Preconditions.checkNotNullOrEmpty(kind, "kind cannot be null or empty");
 
-        PropertiesProcessor propertiesProcessor = new PropertiesProcessor(schemaServiceSchema.getDefinitions(), log);
+        PropertiesProcessor propertiesProcessor = new PropertiesProcessor(schemaServiceSchema.getDefinitions(), log, schemaConverterConfig);
 
         final List<Map<String, Object>> storageSchemaItems = new ArrayList<>();
         if (schemaServiceSchema.getProperties() != null) {

indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterConfig.java

+package org.opengroup.osdu.indexer.schema.converter.config;
+
+import java.util.Map;
+import java.util.Set;
+
+/*
+Provides configuration for the schema converter
+ */
+public interface SchemaConverterConfig {
+    Set<String> getSkippedDefinitions();
+    Set<String> getSupportedArrayTypes();
+    Map<String, String> getSpecialDefinitionsMap();
+    Map<String, String> getPrimitiveTypesMap();
+}

indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterPropertiesConfig.java

+package org.opengroup.osdu.indexer.schema.converter.config;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+import java.util.*;
+
+@Configuration
+@ConfigurationProperties(prefix = "schema.converter")
+@Getter
+@Setter
+public class SchemaConverterPropertiesConfig implements SchemaConverterConfig {
+
+    private Set<String> skippedDefinitions = getDefaultSkippedDefinitions();
+    private Set<String> supportedArrayTypes = getDefaultSupportedArrayTypes();
+    private Map<String, String> specialDefinitionsMap = getDefaultSpecialDefinitionsMap();
+    private Map<String, String> primitiveTypesMap = getDefaultPrimitiveTypesMap();
+
+    private Set<String> getDefaultSkippedDefinitions() {
+        return new HashSet<>(Arrays.asList("AbstractAnyCrsFeatureCollection.1.0.0",
+                "anyCrsGeoJsonFeatureCollection"));
+    }
+
+    private Set<String> getDefaultSupportedArrayTypes() {
+        return new HashSet<>(Arrays.asList("boolean", "integer", "number", "string"));
+    }
+
+    private Map<String, String> getDefaultSpecialDefinitionsMap() {
+        Map<String, String> defaultSpecialDefinitions = new HashMap<>();
+
+        defaultSpecialDefinitions.put("AbstractFeatureCollection.1.0.0", "core:dl:geoshape:1.0.0");
+        defaultSpecialDefinitions.put("core_dl_geopoint", "core:dl:geopoint:1.0.0");
+        defaultSpecialDefinitions.put("geoJsonFeatureCollection", "core:dl:geoshape:1.0.0");
+
+        return defaultSpecialDefinitions;
+    }
+
+    private Map<String, String> getDefaultPrimitiveTypesMap() {
+        Map<String, String> defaultPrimitiveTypesMap = new HashMap<>();
+
+        defaultPrimitiveTypesMap.put("boolean", "bool");
+        defaultPrimitiveTypesMap.put("number", "double");
+        defaultPrimitiveTypesMap.put("date-time", "datetime");
+        defaultPrimitiveTypesMap.put("date", "datetime");
+        defaultPrimitiveTypesMap.put("time", "datetime");
+        defaultPrimitiveTypesMap.put("int32", "int");
+        defaultPrimitiveTypesMap.put("integer", "int");
+        defaultPrimitiveTypesMap.put("int64", "long");
+
+        return defaultPrimitiveTypesMap;
+    }
+}

indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessorTest.java

@@ -18,6 +18,7 @@ import org.junit.Test;
 import org.mockito.Mockito;
 import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
 import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterPropertiesConfig;
 import org.opengroup.osdu.indexer.schema.converter.tags.AllOfItem;
 import org.opengroup.osdu.indexer.schema.converter.tags.Definition;
 import org.opengroup.osdu.indexer.schema.converter.tags.Definitions;
@@ -38,14 +39,15 @@ public class PropertiesProcessorTest {
     public void should_fail_on_unknown_reference_definition() {
         JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class);
 
-        new PropertiesProcessor(Mockito.mock(Definitions.class), log).processRef(DEFINITIONS_PREFIX + "unknownDefinition");
+        new PropertiesProcessor(Mockito.mock(Definitions.class), log, new SchemaConverterPropertiesConfig())
+                .processRef(DEFINITIONS_PREFIX + "unknownDefinition");
     }
 
     @Test
     public void should_not_process_special_reference() {
         JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class);
 
-        assertFalse(new PropertiesProcessor(null, log)
+        assertFalse(new PropertiesProcessor(null, log, new SchemaConverterPropertiesConfig())
                 .processRef(DEFINITIONS_PREFIX + "anyCrsGeoJsonFeatureCollection").findAny().isPresent());
     }
 
@@ -53,7 +55,7 @@ public class PropertiesProcessorTest {
     public void should_return_special_type() {
         JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class);
 
-        String res = new PropertiesProcessor(null, PATH, log)
+        String res = new PropertiesProcessor(null, PATH, log, new SchemaConverterPropertiesConfig())
                 .processRef(DEFINITIONS_PREFIX + "core_dl_geopoint").map(Object::toString).reduce("", String::concat);
         assertEquals("{path=" + PATH + ", kind=core:dl:geopoint:1.0.0}", res);
     }
@@ -77,7 +79,7 @@ public class PropertiesProcessorTest {
         String defName = "defName";
         definitions.add(defName, definition);
 
-        String res = new PropertiesProcessor(definitions, PATH, log)
+        String res = new PropertiesProcessor(definitions, PATH, log, new SchemaConverterPropertiesConfig())
                 .processRef(DEFINITIONS_PREFIX + defName).map(Object::toString).reduce("", String::concat);
         assertEquals(res, "{path="+ PATH + "." + propertyName + ", kind=string}");
     }
@@ -95,7 +97,7 @@ public class PropertiesProcessorTest {
         properties.put(PATH, property);
         allOfItem.setProperties(properties);
 
-        String res = new PropertiesProcessor(Mockito.mock(Definitions.class), log)
+        String res = new PropertiesProcessor(Mockito.mock(Definitions.class), log, new SchemaConverterPropertiesConfig())
                 .processItem(allOfItem).map(Object::toString).reduce("", String::concat);
         assertEquals("{path=" + PATH + ", kind=int}", res);
     }

indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImplTest.java

@@ -17,14 +17,12 @@ package org.opengroup.osdu.indexer.schema.converter;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import org.junit.Test;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
 import org.mockito.Mockito;
 import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterPropertiesConfig;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder;
 
-import java.io.File;
 import java.io.IOException;
 import java.net.URISyntaxException;
 import java.nio.charset.StandardCharsets;
@@ -46,7 +44,8 @@ public class SchemaToStorageFormatImplTest {
     private JaxRsDpsLog jaxRsDpsLog = Mockito.mock(JaxRsDpsLog.class);
     
     private SchemaToStorageFormatImpl schemaToStorageFormatImpl
-            = new SchemaToStorageFormatImpl(objectMapper, jaxRsDpsLog);
+            = new SchemaToStorageFormatImpl(objectMapper, jaxRsDpsLog
+                    , new SchemaConverterPropertiesConfig());
 
     @Test
     public void firstSchemaPassed() {

indexer-core/src/test/java/org/opengroup/osdu/indexer/service/impl/SchemaServiceImplTest.java

@@ -44,7 +44,7 @@ public class SchemaServiceImplTest {
     private JaxRsDpsLog jaxRsDpsLog = Mockito.mock(JaxRsDpsLog.class);
 
     @Spy
-    private SchemaToStorageFormatImpl schemaToStorageFormat = new SchemaToStorageFormatImpl(objectMapper, jaxRsDpsLog);
+    private SchemaToStorageFormatImpl schemaToStorageFormat = new SchemaToStorageFormatImpl(objectMapper, jaxRsDpsLog, null);
 
     @Mock
     private IUrlFetchService urlFetchService;

provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/SchemaServiceTest.java

@@ -47,7 +47,7 @@ public class SchemaServiceTest {
     private JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class);
 
     @Spy
-    private SchemaToStorageFormatImpl schemaToStorageFormatImpl = new SchemaToStorageFormatImpl(objectMapper, log);
+    private SchemaToStorageFormatImpl schemaToStorageFormatImpl = new SchemaToStorageFormatImpl(objectMapper, log, null);
     @Mock
     private IUrlFetchService urlFetchService;
     @Mock