From dc232e1c4e2958da86e7221891eab0175317fbf7 Mon Sep 17 00:00:00 2001
From: Sviatoslav Nekhaienko <snekhaienko@slb.com>
Date: Thu, 17 Dec 2020 16:07:12 +0200
Subject: [PATCH] make converter configurable from a propeties file
---
.../schema/converter/PropertiesProcessor.java | 57 ++++++-------------
.../converter/SchemaToStorageFormatImpl.java | 8 ++-
.../config/SchemaConverterConfig.java | 14 +++++
.../SchemaConverterPropertiesConfig.java | 54 ++++++++++++++++++
.../converter/PropertiesProcessorTest.java | 12 ++--
.../SchemaToStorageFormatImplTest.java | 7 +--
.../service/impl/SchemaServiceImplTest.java | 2 +-
.../azure/service/SchemaServiceTest.java | 2 +-
8 files changed, 102 insertions(+), 54 deletions(-)
create mode 100644 indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterConfig.java
create mode 100644 indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterPropertiesConfig.java
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java
index d51c78fc2..6bbe0d0d9 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java
@@ -17,6 +17,8 @@ import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.search.Preconditions;
+import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterConfig;
+import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterPropertiesConfig;
import org.opengroup.osdu.indexer.schema.converter.tags.AllOfItem;
import org.opengroup.osdu.indexer.schema.converter.tags.Definition;
import org.opengroup.osdu.indexer.schema.converter.tags.Definitions;
@@ -28,50 +30,24 @@ import java.util.stream.Stream;
public class PropertiesProcessor {
private JaxRsDpsLog log;
+ private SchemaConverterConfig schemaConverterConfig;
private static final String DEF_PREFIX = "#/definitions/";
- private static final Set<String> SKIP_DEFINITIONS = new HashSet<>(
- Arrays.asList("AbstractAnyCrsFeatureCollection.1.0.0",
- "anyCrsGeoJsonFeatureCollection"));
-
- private static final Set<String> ARRAY_SUPPORTED_SIMPLE_TYPES = new HashSet<>(
- Arrays.asList("boolean", "integer", "number", "string"));
-
- private static final Map<String, String> SPEC_DEFINITION_TYPES = new HashMap<>();
-
- private static final Map<String, String> PRIMITIVE_TYPES_MAP = new HashMap<>();
-
private final Definitions definitions;
private final String pathPrefix;
private final String pathPrefixWithDot;
- static {
- SPEC_DEFINITION_TYPES.put("AbstractFeatureCollection.1.0.0", "core:dl:geoshape:1.0.0");
- SPEC_DEFINITION_TYPES.put("core_dl_geopoint", "core:dl:geopoint:1.0.0");
- SPEC_DEFINITION_TYPES.put("geoJsonFeatureCollection", "core:dl:geoshape:1.0.0");
- }
-
- static {
- PRIMITIVE_TYPES_MAP.put("boolean", "bool");
- PRIMITIVE_TYPES_MAP.put("number", "double");
- PRIMITIVE_TYPES_MAP.put("date-time", "datetime");
- PRIMITIVE_TYPES_MAP.put("date", "datetime");
- PRIMITIVE_TYPES_MAP.put("time", "datetime");
- PRIMITIVE_TYPES_MAP.put("int32", "int");
- PRIMITIVE_TYPES_MAP.put("integer", "int");
- PRIMITIVE_TYPES_MAP.put("int64", "long");
- }
-
- public PropertiesProcessor(Definitions definitions, JaxRsDpsLog log) {
- this(definitions, null, log);
+ public PropertiesProcessor(Definitions definitions, JaxRsDpsLog log, SchemaConverterConfig schemaConverterConfig) {
+ this(definitions, null, log, schemaConverterConfig);
}
- public PropertiesProcessor(Definitions definitions, String pathPrefix, JaxRsDpsLog log) {
+ public PropertiesProcessor(Definitions definitions, String pathPrefix, JaxRsDpsLog log, SchemaConverterConfig schemaConverterConfig) {
this.log = log;
this.definitions = definitions;
this.pathPrefix = pathPrefix;
this.pathPrefixWithDot = Objects.isNull(pathPrefix) || pathPrefix.isEmpty() ? "" : pathPrefix + ".";
+ this.schemaConverterConfig = schemaConverterConfig;
}
public Stream<Map<String, Object>> processItem(AllOfItem allOfItem) {
@@ -93,12 +69,12 @@ public class PropertiesProcessor {
String definitionSubRef = ref.substring(DEF_PREFIX.length());
- if (SKIP_DEFINITIONS.contains(definitionSubRef)) {
+ if (schemaConverterConfig.getSkippedDefinitions().contains(definitionSubRef)) {
return Stream.empty();
}
- if (!Objects.isNull(SPEC_DEFINITION_TYPES.get(definitionSubRef))) {
- return storageSchemaEntry(SPEC_DEFINITION_TYPES.get(definitionSubRef), pathPrefix);
+ if (!Objects.isNull(schemaConverterConfig.getSpecialDefinitionsMap().get(definitionSubRef))) {
+ return storageSchemaEntry(schemaConverterConfig.getSpecialDefinitionsMap().get(definitionSubRef), pathPrefix);
}
Definition definition = definitions.getDefinition(definitionSubRef);
@@ -120,7 +96,7 @@ public class PropertiesProcessor {
}
if ("array".equals(entry.getValue().getType())) {
- if (ARRAY_SUPPORTED_SIMPLE_TYPES.contains(entry.getValue().getItems().getType())) {
+ if (schemaConverterConfig.getSupportedArrayTypes().contains(entry.getValue().getItems().getType())) {
return storageSchemaEntry("[]" + getTypeByDefinitionProperty(entry.getValue()), pathPrefixWithDot + entry.getKey());
}
@@ -128,12 +104,13 @@ public class PropertiesProcessor {
}
if (!Objects.isNull(entry.getValue().getProperties())) {
- PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey(), log);
+ PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey()
+ , log, new SchemaConverterPropertiesConfig());
return entry.getValue().getProperties().entrySet().stream().flatMap(propertiesProcessor::processPropertyEntry);
}
if (!Objects.isNull(entry.getValue().getRef())) {
- return new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey(), log)
+ return new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey(), log, new SchemaConverterPropertiesConfig())
.processRef(entry.getValue().getRef());
}
@@ -161,8 +138,8 @@ public class PropertiesProcessor {
return !Objects.isNull(pattern) && pattern.startsWith("^srn") ? "link" :
!Objects.isNull(itemsPattern) && itemsPattern.startsWith("^srn") ? "link" :
- !Objects.isNull(format) ? PRIMITIVE_TYPES_MAP.getOrDefault(format, format) :
- !Objects.isNull(itemsType) ? PRIMITIVE_TYPES_MAP.getOrDefault(itemsType, itemsType) :
- PRIMITIVE_TYPES_MAP.getOrDefault(type, type);
+ !Objects.isNull(format) ? schemaConverterConfig.getPrimitiveTypesMap().getOrDefault(format, format) :
+ !Objects.isNull(itemsType) ? schemaConverterConfig.getPrimitiveTypesMap().getOrDefault(itemsType, itemsType) :
+ schemaConverterConfig.getPrimitiveTypesMap().getOrDefault(type, type);
}
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java
index 925e15799..98141b5d0 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java
@@ -20,6 +20,7 @@ import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.search.Preconditions;
+import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterConfig;
import org.opengroup.osdu.indexer.schema.converter.interfaces.SchemaToStorageFormat;
import org.opengroup.osdu.indexer.schema.converter.tags.PropertiesData;
import org.opengroup.osdu.indexer.schema.converter.tags.SchemaRoot;
@@ -36,15 +37,16 @@ import java.util.stream.Collectors;
public class SchemaToStorageFormatImpl implements SchemaToStorageFormat {
private ObjectMapper objectMapper;
-
private JaxRsDpsLog log;
+ private SchemaConverterConfig schemaConverterConfig;
@Inject
- public SchemaToStorageFormatImpl(ObjectMapper objectMapper, JaxRsDpsLog log) {
+ public SchemaToStorageFormatImpl(ObjectMapper objectMapper, JaxRsDpsLog log, SchemaConverterConfig schemaConverterConfig) {
Preconditions.checkNotNull(objectMapper, "objectMapper cannot be null");
this.objectMapper = objectMapper;
this.log = log;
+ this.schemaConverterConfig = schemaConverterConfig;
}
@Override
@@ -82,7 +84,7 @@ public class SchemaToStorageFormatImpl implements SchemaToStorageFormat {
Preconditions.checkNotNull(objectMapper, "schemaServiceSchema cannot be null");
Preconditions.checkNotNullOrEmpty(kind, "kind cannot be null or empty");
- PropertiesProcessor propertiesProcessor = new PropertiesProcessor(schemaServiceSchema.getDefinitions(), log);
+ PropertiesProcessor propertiesProcessor = new PropertiesProcessor(schemaServiceSchema.getDefinitions(), log, schemaConverterConfig);
final List<Map<String, Object>> storageSchemaItems = new ArrayList<>();
if (schemaServiceSchema.getProperties() != null) {
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterConfig.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterConfig.java
new file mode 100644
index 000000000..539b2bdfd
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterConfig.java
@@ -0,0 +1,14 @@
+package org.opengroup.osdu.indexer.schema.converter.config;
+
+import java.util.Map;
+import java.util.Set;
+
+/*
+Provides configuration for the schema converter
+ */
+public interface SchemaConverterConfig {
+ Set<String> getSkippedDefinitions();
+ Set<String> getSupportedArrayTypes();
+ Map<String, String> getSpecialDefinitionsMap();
+ Map<String, String> getPrimitiveTypesMap();
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterPropertiesConfig.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterPropertiesConfig.java
new file mode 100644
index 000000000..f530c842b
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterPropertiesConfig.java
@@ -0,0 +1,54 @@
+package org.opengroup.osdu.indexer.schema.converter.config;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+import java.util.*;
+
+@Configuration
+@ConfigurationProperties(prefix = "schema.converter")
+@Getter
+@Setter
+public class SchemaConverterPropertiesConfig implements SchemaConverterConfig {
+
+ private Set<String> skippedDefinitions = getDefaultSkippedDefinitions();
+ private Set<String> supportedArrayTypes = getDefaultSupportedArrayTypes();
+ private Map<String, String> specialDefinitionsMap = getDefaultSpecialDefinitionsMap();
+ private Map<String, String> primitiveTypesMap = getDefaultPrimitiveTypesMap();
+
+ private Set<String> getDefaultSkippedDefinitions() {
+ return new HashSet<>(Arrays.asList("AbstractAnyCrsFeatureCollection.1.0.0",
+ "anyCrsGeoJsonFeatureCollection"));
+ }
+
+ private Set<String> getDefaultSupportedArrayTypes() {
+ return new HashSet<>(Arrays.asList("boolean", "integer", "number", "string"));
+ }
+
+ private Map<String, String> getDefaultSpecialDefinitionsMap() {
+ Map<String, String> defaultSpecialDefinitions = new HashMap<>();
+
+ defaultSpecialDefinitions.put("AbstractFeatureCollection.1.0.0", "core:dl:geoshape:1.0.0");
+ defaultSpecialDefinitions.put("core_dl_geopoint", "core:dl:geopoint:1.0.0");
+ defaultSpecialDefinitions.put("geoJsonFeatureCollection", "core:dl:geoshape:1.0.0");
+
+ return defaultSpecialDefinitions;
+ }
+
+ private Map<String, String> getDefaultPrimitiveTypesMap() {
+ Map<String, String> defaultPrimitiveTypesMap = new HashMap<>();
+
+ defaultPrimitiveTypesMap.put("boolean", "bool");
+ defaultPrimitiveTypesMap.put("number", "double");
+ defaultPrimitiveTypesMap.put("date-time", "datetime");
+ defaultPrimitiveTypesMap.put("date", "datetime");
+ defaultPrimitiveTypesMap.put("time", "datetime");
+ defaultPrimitiveTypesMap.put("int32", "int");
+ defaultPrimitiveTypesMap.put("integer", "int");
+ defaultPrimitiveTypesMap.put("int64", "long");
+
+ return defaultPrimitiveTypesMap;
+ }
+}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessorTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessorTest.java
index ada399a25..600af3b43 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessorTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessorTest.java
@@ -18,6 +18,7 @@ import org.junit.Test;
import org.mockito.Mockito;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterPropertiesConfig;
import org.opengroup.osdu.indexer.schema.converter.tags.AllOfItem;
import org.opengroup.osdu.indexer.schema.converter.tags.Definition;
import org.opengroup.osdu.indexer.schema.converter.tags.Definitions;
@@ -38,14 +39,15 @@ public class PropertiesProcessorTest {
public void should_fail_on_unknown_reference_definition() {
JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class);
- new PropertiesProcessor(Mockito.mock(Definitions.class), log).processRef(DEFINITIONS_PREFIX + "unknownDefinition");
+ new PropertiesProcessor(Mockito.mock(Definitions.class), log, new SchemaConverterPropertiesConfig())
+ .processRef(DEFINITIONS_PREFIX + "unknownDefinition");
}
@Test
public void should_not_process_special_reference() {
JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class);
- assertFalse(new PropertiesProcessor(null, log)
+ assertFalse(new PropertiesProcessor(null, log, new SchemaConverterPropertiesConfig())
.processRef(DEFINITIONS_PREFIX + "anyCrsGeoJsonFeatureCollection").findAny().isPresent());
}
@@ -53,7 +55,7 @@ public class PropertiesProcessorTest {
public void should_return_special_type() {
JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class);
- String res = new PropertiesProcessor(null, PATH, log)
+ String res = new PropertiesProcessor(null, PATH, log, new SchemaConverterPropertiesConfig())
.processRef(DEFINITIONS_PREFIX + "core_dl_geopoint").map(Object::toString).reduce("", String::concat);
assertEquals("{path=" + PATH + ", kind=core:dl:geopoint:1.0.0}", res);
}
@@ -77,7 +79,7 @@ public class PropertiesProcessorTest {
String defName = "defName";
definitions.add(defName, definition);
- String res = new PropertiesProcessor(definitions, PATH, log)
+ String res = new PropertiesProcessor(definitions, PATH, log, new SchemaConverterPropertiesConfig())
.processRef(DEFINITIONS_PREFIX + defName).map(Object::toString).reduce("", String::concat);
assertEquals(res, "{path="+ PATH + "." + propertyName + ", kind=string}");
}
@@ -95,7 +97,7 @@ public class PropertiesProcessorTest {
properties.put(PATH, property);
allOfItem.setProperties(properties);
- String res = new PropertiesProcessor(Mockito.mock(Definitions.class), log)
+ String res = new PropertiesProcessor(Mockito.mock(Definitions.class), log, new SchemaConverterPropertiesConfig())
.processItem(allOfItem).map(Object::toString).reduce("", String::concat);
assertEquals("{path=" + PATH + ", kind=int}", res);
}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImplTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImplTest.java
index 98ed7abda..f43d62617 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImplTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImplTest.java
@@ -17,14 +17,12 @@ package org.opengroup.osdu.indexer.schema.converter;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.Test;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
import org.mockito.Mockito;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterPropertiesConfig;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder;
-import java.io.File;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
@@ -46,7 +44,8 @@ public class SchemaToStorageFormatImplTest {
private JaxRsDpsLog jaxRsDpsLog = Mockito.mock(JaxRsDpsLog.class);
private SchemaToStorageFormatImpl schemaToStorageFormatImpl
- = new SchemaToStorageFormatImpl(objectMapper, jaxRsDpsLog);
+ = new SchemaToStorageFormatImpl(objectMapper, jaxRsDpsLog
+ , new SchemaConverterPropertiesConfig());
@Test
public void firstSchemaPassed() {
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/impl/SchemaServiceImplTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/impl/SchemaServiceImplTest.java
index 92e94444b..b95b152c5 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/impl/SchemaServiceImplTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/impl/SchemaServiceImplTest.java
@@ -44,7 +44,7 @@ public class SchemaServiceImplTest {
private JaxRsDpsLog jaxRsDpsLog = Mockito.mock(JaxRsDpsLog.class);
@Spy
- private SchemaToStorageFormatImpl schemaToStorageFormat = new SchemaToStorageFormatImpl(objectMapper, jaxRsDpsLog);
+ private SchemaToStorageFormatImpl schemaToStorageFormat = new SchemaToStorageFormatImpl(objectMapper, jaxRsDpsLog, null);
@Mock
private IUrlFetchService urlFetchService;
diff --git a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/SchemaServiceTest.java b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/SchemaServiceTest.java
index 6f9011461..d683fe78b 100644
--- a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/SchemaServiceTest.java
+++ b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/SchemaServiceTest.java
@@ -47,7 +47,7 @@ public class SchemaServiceTest {
private JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class);
@Spy
- private SchemaToStorageFormatImpl schemaToStorageFormatImpl = new SchemaToStorageFormatImpl(objectMapper, log);
+ private SchemaToStorageFormatImpl schemaToStorageFormatImpl = new SchemaToStorageFormatImpl(objectMapper, log, null);
@Mock
private IUrlFetchService urlFetchService;
@Mock
--
GitLab