Skip to content

GitLab

Commit dc232e1c authored by Sviatoslav Nekhaienko's avatar Sviatoslav Nekhaienko
Browse files

make converter configurable from a propeties file

parent 4106572a
Pipeline #19567 failed with stages
in 70 minutes and 45 seconds
Hide whitespace changes
Inline Side-by-side
indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java
View file @ dc232e1c
......@@ -17,6 +17,8 @@ import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.search.Preconditions;
import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterConfig;
import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterPropertiesConfig;
import org.opengroup.osdu.indexer.schema.converter.tags.AllOfItem;
import org.opengroup.osdu.indexer.schema.converter.tags.Definition;
import org.opengroup.osdu.indexer.schema.converter.tags.Definitions;
......@@ -28,50 +30,24 @@ import java.util.stream.Stream;
public class PropertiesProcessor {
private JaxRsDpsLog log;
private SchemaConverterConfig schemaConverterConfig;
private static final String DEF_PREFIX = "#/definitions/";
private static final Set<String> SKIP_DEFINITIONS = new HashSet<>(
Arrays.asList("AbstractAnyCrsFeatureCollection.1.0.0",
"anyCrsGeoJsonFeatureCollection"));
private static final Set<String> ARRAY_SUPPORTED_SIMPLE_TYPES = new HashSet<>(
Arrays.asList("boolean", "integer", "number", "string"));
private static final Map<String, String> SPEC_DEFINITION_TYPES = new HashMap<>();
private static final Map<String, String> PRIMITIVE_TYPES_MAP = new HashMap<>();
private final Definitions definitions;
private final String pathPrefix;
private final String pathPrefixWithDot;
static {
SPEC_DEFINITION_TYPES.put("AbstractFeatureCollection.1.0.0", "core:dl:geoshape:1.0.0");
SPEC_DEFINITION_TYPES.put("core_dl_geopoint", "core:dl:geopoint:1.0.0");
SPEC_DEFINITION_TYPES.put("geoJsonFeatureCollection", "core:dl:geoshape:1.0.0");
}
static {
PRIMITIVE_TYPES_MAP.put("boolean", "bool");
PRIMITIVE_TYPES_MAP.put("number", "double");
PRIMITIVE_TYPES_MAP.put("date-time", "datetime");
PRIMITIVE_TYPES_MAP.put("date", "datetime");
PRIMITIVE_TYPES_MAP.put("time", "datetime");
PRIMITIVE_TYPES_MAP.put("int32", "int");
PRIMITIVE_TYPES_MAP.put("integer", "int");
PRIMITIVE_TYPES_MAP.put("int64", "long");
}
public PropertiesProcessor(Definitions definitions, JaxRsDpsLog log) {
this(definitions, null, log);
public PropertiesProcessor(Definitions definitions, JaxRsDpsLog log, SchemaConverterConfig schemaConverterConfig) {
this(definitions, null, log, schemaConverterConfig);
}
public PropertiesProcessor(Definitions definitions, String pathPrefix, JaxRsDpsLog log) {
public PropertiesProcessor(Definitions definitions, String pathPrefix, JaxRsDpsLog log, SchemaConverterConfig schemaConverterConfig) {
this.log = log;
this.definitions = definitions;
this.pathPrefix = pathPrefix;
this.pathPrefixWithDot = Objects.isNull(pathPrefix) || pathPrefix.isEmpty() ? "" : pathPrefix + ".";
this.schemaConverterConfig = schemaConverterConfig;
}
public Stream<Map<String, Object>> processItem(AllOfItem allOfItem) {
......@@ -93,12 +69,12 @@ public class PropertiesProcessor {
String definitionSubRef = ref.substring(DEF_PREFIX.length());
if (SKIP_DEFINITIONS.contains(definitionSubRef)) {
if (schemaConverterConfig.getSkippedDefinitions().contains(definitionSubRef)) {
return Stream.empty();
}
if (!Objects.isNull(SPEC_DEFINITION_TYPES.get(definitionSubRef))) {
return storageSchemaEntry(SPEC_DEFINITION_TYPES.get(definitionSubRef), pathPrefix);
if (!Objects.isNull(schemaConverterConfig.getSpecialDefinitionsMap().get(definitionSubRef))) {
return storageSchemaEntry(schemaConverterConfig.getSpecialDefinitionsMap().get(definitionSubRef), pathPrefix);
}
Definition definition = definitions.getDefinition(definitionSubRef);
......@@ -120,7 +96,7 @@ public class PropertiesProcessor {
}
if ("array".equals(entry.getValue().getType())) {
if (ARRAY_SUPPORTED_SIMPLE_TYPES.contains(entry.getValue().getItems().getType())) {
if (schemaConverterConfig.getSupportedArrayTypes().contains(entry.getValue().getItems().getType())) {
return storageSchemaEntry("[]" + getTypeByDefinitionProperty(entry.getValue()), pathPrefixWithDot + entry.getKey());
}
......@@ -128,12 +104,13 @@ public class PropertiesProcessor {
}
if (!Objects.isNull(entry.getValue().getProperties())) {
PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey(), log);
PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey()
, log, new SchemaConverterPropertiesConfig());
return entry.getValue().getProperties().entrySet().stream().flatMap(propertiesProcessor::processPropertyEntry);
}
if (!Objects.isNull(entry.getValue().getRef())) {
return new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey(), log)
return new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey(), log, new SchemaConverterPropertiesConfig())
.processRef(entry.getValue().getRef());
}
......@@ -161,8 +138,8 @@ public class PropertiesProcessor {
return !Objects.isNull(pattern) && pattern.startsWith("^srn") ? "link" :
!Objects.isNull(itemsPattern) && itemsPattern.startsWith("^srn") ? "link" :
!Objects.isNull(format) ? PRIMITIVE_TYPES_MAP.getOrDefault(format, format) :
!Objects.isNull(itemsType) ? PRIMITIVE_TYPES_MAP.getOrDefault(itemsType, itemsType) :
PRIMITIVE_TYPES_MAP.getOrDefault(type, type);
!Objects.isNull(format) ? schemaConverterConfig.getPrimitiveTypesMap().getOrDefault(format, format) :
!Objects.isNull(itemsType) ? schemaConverterConfig.getPrimitiveTypesMap().getOrDefault(itemsType, itemsType) :
schemaConverterConfig.getPrimitiveTypesMap().getOrDefault(type, type);
}
}
indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java
View file @ dc232e1c
......@@ -20,6 +20,7 @@ import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.search.Preconditions;
import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterConfig;
import org.opengroup.osdu.indexer.schema.converter.interfaces.SchemaToStorageFormat;
import org.opengroup.osdu.indexer.schema.converter.tags.PropertiesData;
import org.opengroup.osdu.indexer.schema.converter.tags.SchemaRoot;
......@@ -36,15 +37,16 @@ import java.util.stream.Collectors;
public class SchemaToStorageFormatImpl implements SchemaToStorageFormat {
private ObjectMapper objectMapper;
private JaxRsDpsLog log;
private SchemaConverterConfig schemaConverterConfig;
@Inject
public SchemaToStorageFormatImpl(ObjectMapper objectMapper, JaxRsDpsLog log) {
public SchemaToStorageFormatImpl(ObjectMapper objectMapper, JaxRsDpsLog log, SchemaConverterConfig schemaConverterConfig) {
Preconditions.checkNotNull(objectMapper, "objectMapper cannot be null");
this.objectMapper = objectMapper;
this.log = log;
this.schemaConverterConfig = schemaConverterConfig;
}
@Override
......@@ -82,7 +84,7 @@ public class SchemaToStorageFormatImpl implements SchemaToStorageFormat {
Preconditions.checkNotNull(objectMapper, "schemaServiceSchema cannot be null");
Preconditions.checkNotNullOrEmpty(kind, "kind cannot be null or empty");
PropertiesProcessor propertiesProcessor = new PropertiesProcessor(schemaServiceSchema.getDefinitions(), log);
PropertiesProcessor propertiesProcessor = new PropertiesProcessor(schemaServiceSchema.getDefinitions(), log, schemaConverterConfig);
final List<Map<String, Object>> storageSchemaItems = new ArrayList<>();
if (schemaServiceSchema.getProperties() != null) {
......
indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterConfig.java 0 → 100644
View file @ dc232e1c
package org.opengroup.osdu.indexer.schema.converter.config;
import java.util.Map;
import java.util.Set;
/*
Provides configuration for the schema converter
*/
public interface SchemaConverterConfig {
Set<String> getSkippedDefinitions();
Set<String> getSupportedArrayTypes();
Map<String, String> getSpecialDefinitionsMap();
Map<String, String> getPrimitiveTypesMap();
}
indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterPropertiesConfig.java 0 → 100644
View file @ dc232e1c
package org.opengroup.osdu.indexer.schema.converter.config;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
import java.util.*;
@Configuration
@ConfigurationProperties(prefix = "schema.converter")
@Getter
@Setter
public class SchemaConverterPropertiesConfig implements SchemaConverterConfig {
private Set<String> skippedDefinitions = getDefaultSkippedDefinitions();
private Set<String> supportedArrayTypes = getDefaultSupportedArrayTypes();
private Map<String, String> specialDefinitionsMap = getDefaultSpecialDefinitionsMap();
private Map<String, String> primitiveTypesMap = getDefaultPrimitiveTypesMap();
private Set<String> getDefaultSkippedDefinitions() {
return new HashSet<>(Arrays.asList("AbstractAnyCrsFeatureCollection.1.0.0",
"anyCrsGeoJsonFeatureCollection"));
}
private Set<String> getDefaultSupportedArrayTypes() {
return new HashSet<>(Arrays.asList("boolean", "integer", "number", "string"));
}
private Map<String, String> getDefaultSpecialDefinitionsMap() {
Map<String, String> defaultSpecialDefinitions = new HashMap<>();
defaultSpecialDefinitions.put("AbstractFeatureCollection.1.0.0", "core:dl:geoshape:1.0.0");
defaultSpecialDefinitions.put("core_dl_geopoint", "core:dl:geopoint:1.0.0");
defaultSpecialDefinitions.put("geoJsonFeatureCollection", "core:dl:geoshape:1.0.0");
return defaultSpecialDefinitions;
}
private Map<String, String> getDefaultPrimitiveTypesMap() {
Map<String, String> defaultPrimitiveTypesMap = new HashMap<>();
defaultPrimitiveTypesMap.put("boolean", "bool");
defaultPrimitiveTypesMap.put("number", "double");
defaultPrimitiveTypesMap.put("date-time", "datetime");
defaultPrimitiveTypesMap.put("date", "datetime");
defaultPrimitiveTypesMap.put("time", "datetime");
defaultPrimitiveTypesMap.put("int32", "int");
defaultPrimitiveTypesMap.put("integer", "int");
defaultPrimitiveTypesMap.put("int64", "long");
return defaultPrimitiveTypesMap;
}
}
indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessorTest.java
View file @ dc232e1c
......@@ -18,6 +18,7 @@ import org.junit.Test;
import org.mockito.Mockito;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterPropertiesConfig;
import org.opengroup.osdu.indexer.schema.converter.tags.AllOfItem;
import org.opengroup.osdu.indexer.schema.converter.tags.Definition;
import org.opengroup.osdu.indexer.schema.converter.tags.Definitions;
......@@ -38,14 +39,15 @@ public class PropertiesProcessorTest {
public void should_fail_on_unknown_reference_definition() {
JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class);
new PropertiesProcessor(Mockito.mock(Definitions.class), log).processRef(DEFINITIONS_PREFIX + "unknownDefinition");
new PropertiesProcessor(Mockito.mock(Definitions.class), log, new SchemaConverterPropertiesConfig())
.processRef(DEFINITIONS_PREFIX + "unknownDefinition");
}
@Test
public void should_not_process_special_reference() {
JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class);
assertFalse(new PropertiesProcessor(null, log)
assertFalse(new PropertiesProcessor(null, log, new SchemaConverterPropertiesConfig())
.processRef(DEFINITIONS_PREFIX + "anyCrsGeoJsonFeatureCollection").findAny().isPresent());
}
......@@ -53,7 +55,7 @@ public class PropertiesProcessorTest {
public void should_return_special_type() {
JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class);
String res = new PropertiesProcessor(null, PATH, log)
String res = new PropertiesProcessor(null, PATH, log, new SchemaConverterPropertiesConfig())
.processRef(DEFINITIONS_PREFIX + "core_dl_geopoint").map(Object::toString).reduce("", String::concat);
assertEquals("{path=" + PATH + ", kind=core:dl:geopoint:1.0.0}", res);
}
......@@ -77,7 +79,7 @@ public class PropertiesProcessorTest {
String defName = "defName";
definitions.add(defName, definition);
String res = new PropertiesProcessor(definitions, PATH, log)
String res = new PropertiesProcessor(definitions, PATH, log, new SchemaConverterPropertiesConfig())
.processRef(DEFINITIONS_PREFIX + defName).map(Object::toString).reduce("", String::concat);
assertEquals(res, "{path="+ PATH + "." + propertyName + ", kind=string}");
}
......@@ -95,7 +97,7 @@ public class PropertiesProcessorTest {
properties.put(PATH, property);
allOfItem.setProperties(properties);
String res = new PropertiesProcessor(Mockito.mock(Definitions.class), log)
String res = new PropertiesProcessor(Mockito.mock(Definitions.class), log, new SchemaConverterPropertiesConfig())
.processItem(allOfItem).map(Object::toString).reduce("", String::concat);
assertEquals("{path=" + PATH + ", kind=int}", res);
}
......
indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImplTest.java
View file @ dc232e1c
......@@ -17,14 +17,12 @@ package org.opengroup.osdu.indexer.schema.converter;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.Test;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterPropertiesConfig;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder;
import java.io.File;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
......@@ -46,7 +44,8 @@ public class SchemaToStorageFormatImplTest {
private JaxRsDpsLog jaxRsDpsLog = Mockito.mock(JaxRsDpsLog.class);
private SchemaToStorageFormatImpl schemaToStorageFormatImpl
= new SchemaToStorageFormatImpl(objectMapper, jaxRsDpsLog);
= new SchemaToStorageFormatImpl(objectMapper, jaxRsDpsLog
, new SchemaConverterPropertiesConfig());
@Test
public void firstSchemaPassed() {
......
indexer-core/src/test/java/org/opengroup/osdu/indexer/service/impl/SchemaServiceImplTest.java
View file @ dc232e1c
......@@ -44,7 +44,7 @@ public class SchemaServiceImplTest {
private JaxRsDpsLog jaxRsDpsLog = Mockito.mock(JaxRsDpsLog.class);
@Spy
private SchemaToStorageFormatImpl schemaToStorageFormat = new SchemaToStorageFormatImpl(objectMapper, jaxRsDpsLog);
private SchemaToStorageFormatImpl schemaToStorageFormat = new SchemaToStorageFormatImpl(objectMapper, jaxRsDpsLog, null);
@Mock
private IUrlFetchService urlFetchService;
......
provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/SchemaServiceTest.java
View file @ dc232e1c
......@@ -47,7 +47,7 @@ public class SchemaServiceTest {
private JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class);
@Spy
private SchemaToStorageFormatImpl schemaToStorageFormatImpl = new SchemaToStorageFormatImpl(objectMapper, log);
private SchemaToStorageFormatImpl schemaToStorageFormatImpl = new SchemaToStorageFormatImpl(objectMapper, log, null);
@Mock
private IUrlFetchService urlFetchService;
@Mock
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment