Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
Menu
Open sidebar
Open Subsurface Data Universe Software
Platform
System
Indexer
Commits
dc232e1c
Commit
dc232e1c
authored
Dec 17, 2020
by
Sviatoslav Nekhaienko
Browse files
make converter configurable from a propeties file
parent
4106572a
Pipeline
#19567
failed with stages
in 70 minutes and 45 seconds
Changes
8
Pipelines
4
Hide whitespace changes
Inline
Side-by-side
indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java
View file @
dc232e1c
...
...
@@ -17,6 +17,8 @@ import org.apache.http.HttpStatus;
import
org.opengroup.osdu.core.common.logging.JaxRsDpsLog
;
import
org.opengroup.osdu.core.common.model.http.AppException
;
import
org.opengroup.osdu.core.common.search.Preconditions
;
import
org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterConfig
;
import
org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterPropertiesConfig
;
import
org.opengroup.osdu.indexer.schema.converter.tags.AllOfItem
;
import
org.opengroup.osdu.indexer.schema.converter.tags.Definition
;
import
org.opengroup.osdu.indexer.schema.converter.tags.Definitions
;
...
...
@@ -28,50 +30,24 @@ import java.util.stream.Stream;
public
class
PropertiesProcessor
{
private
JaxRsDpsLog
log
;
private
SchemaConverterConfig
schemaConverterConfig
;
private
static
final
String
DEF_PREFIX
=
"#/definitions/"
;
private
static
final
Set
<
String
>
SKIP_DEFINITIONS
=
new
HashSet
<>(
Arrays
.
asList
(
"AbstractAnyCrsFeatureCollection.1.0.0"
,
"anyCrsGeoJsonFeatureCollection"
));
private
static
final
Set
<
String
>
ARRAY_SUPPORTED_SIMPLE_TYPES
=
new
HashSet
<>(
Arrays
.
asList
(
"boolean"
,
"integer"
,
"number"
,
"string"
));
private
static
final
Map
<
String
,
String
>
SPEC_DEFINITION_TYPES
=
new
HashMap
<>();
private
static
final
Map
<
String
,
String
>
PRIMITIVE_TYPES_MAP
=
new
HashMap
<>();
private
final
Definitions
definitions
;
private
final
String
pathPrefix
;
private
final
String
pathPrefixWithDot
;
static
{
SPEC_DEFINITION_TYPES
.
put
(
"AbstractFeatureCollection.1.0.0"
,
"core:dl:geoshape:1.0.0"
);
SPEC_DEFINITION_TYPES
.
put
(
"core_dl_geopoint"
,
"core:dl:geopoint:1.0.0"
);
SPEC_DEFINITION_TYPES
.
put
(
"geoJsonFeatureCollection"
,
"core:dl:geoshape:1.0.0"
);
}
static
{
PRIMITIVE_TYPES_MAP
.
put
(
"boolean"
,
"bool"
);
PRIMITIVE_TYPES_MAP
.
put
(
"number"
,
"double"
);
PRIMITIVE_TYPES_MAP
.
put
(
"date-time"
,
"datetime"
);
PRIMITIVE_TYPES_MAP
.
put
(
"date"
,
"datetime"
);
PRIMITIVE_TYPES_MAP
.
put
(
"time"
,
"datetime"
);
PRIMITIVE_TYPES_MAP
.
put
(
"int32"
,
"int"
);
PRIMITIVE_TYPES_MAP
.
put
(
"integer"
,
"int"
);
PRIMITIVE_TYPES_MAP
.
put
(
"int64"
,
"long"
);
}
public
PropertiesProcessor
(
Definitions
definitions
,
JaxRsDpsLog
log
)
{
this
(
definitions
,
null
,
log
);
public
PropertiesProcessor
(
Definitions
definitions
,
JaxRsDpsLog
log
,
SchemaConverterConfig
schemaConverterConfig
)
{
this
(
definitions
,
null
,
log
,
schemaConverterConfig
);
}
public
PropertiesProcessor
(
Definitions
definitions
,
String
pathPrefix
,
JaxRsDpsLog
log
)
{
public
PropertiesProcessor
(
Definitions
definitions
,
String
pathPrefix
,
JaxRsDpsLog
log
,
SchemaConverterConfig
schemaConverterConfig
)
{
this
.
log
=
log
;
this
.
definitions
=
definitions
;
this
.
pathPrefix
=
pathPrefix
;
this
.
pathPrefixWithDot
=
Objects
.
isNull
(
pathPrefix
)
||
pathPrefix
.
isEmpty
()
?
""
:
pathPrefix
+
"."
;
this
.
schemaConverterConfig
=
schemaConverterConfig
;
}
public
Stream
<
Map
<
String
,
Object
>>
processItem
(
AllOfItem
allOfItem
)
{
...
...
@@ -93,12 +69,12 @@ public class PropertiesProcessor {
String
definitionSubRef
=
ref
.
substring
(
DEF_PREFIX
.
length
());
if
(
SKIP_DEFINITIONS
.
contains
(
definitionSubRef
))
{
if
(
schemaConverterConfig
.
getSkippedDefinitions
()
.
contains
(
definitionSubRef
))
{
return
Stream
.
empty
();
}
if
(!
Objects
.
isNull
(
SPEC_DEFINITION_TYPES
.
get
(
definitionSubRef
)))
{
return
storageSchemaEntry
(
SPEC_DEFINITION_TYPES
.
get
(
definitionSubRef
),
pathPrefix
);
if
(!
Objects
.
isNull
(
schemaConverterConfig
.
getSpecialDefinitionsMap
()
.
get
(
definitionSubRef
)))
{
return
storageSchemaEntry
(
schemaConverterConfig
.
getSpecialDefinitionsMap
()
.
get
(
definitionSubRef
),
pathPrefix
);
}
Definition
definition
=
definitions
.
getDefinition
(
definitionSubRef
);
...
...
@@ -120,7 +96,7 @@ public class PropertiesProcessor {
}
if
(
"array"
.
equals
(
entry
.
getValue
().
getType
()))
{
if
(
ARRAY_SUPPORTED_SIMPLE_TYPES
.
contains
(
entry
.
getValue
().
getItems
().
getType
()))
{
if
(
schemaConverterConfig
.
getSupportedArrayTypes
()
.
contains
(
entry
.
getValue
().
getItems
().
getType
()))
{
return
storageSchemaEntry
(
"[]"
+
getTypeByDefinitionProperty
(
entry
.
getValue
()),
pathPrefixWithDot
+
entry
.
getKey
());
}
...
...
@@ -128,12 +104,13 @@ public class PropertiesProcessor {
}
if
(!
Objects
.
isNull
(
entry
.
getValue
().
getProperties
()))
{
PropertiesProcessor
propertiesProcessor
=
new
PropertiesProcessor
(
definitions
,
pathPrefixWithDot
+
entry
.
getKey
(),
log
);
PropertiesProcessor
propertiesProcessor
=
new
PropertiesProcessor
(
definitions
,
pathPrefixWithDot
+
entry
.
getKey
()
,
log
,
new
SchemaConverterPropertiesConfig
());
return
entry
.
getValue
().
getProperties
().
entrySet
().
stream
().
flatMap
(
propertiesProcessor:
:
processPropertyEntry
);
}
if
(!
Objects
.
isNull
(
entry
.
getValue
().
getRef
()))
{
return
new
PropertiesProcessor
(
definitions
,
pathPrefixWithDot
+
entry
.
getKey
(),
log
)
return
new
PropertiesProcessor
(
definitions
,
pathPrefixWithDot
+
entry
.
getKey
(),
log
,
new
SchemaConverterPropertiesConfig
()
)
.
processRef
(
entry
.
getValue
().
getRef
());
}
...
...
@@ -161,8 +138,8 @@ public class PropertiesProcessor {
return
!
Objects
.
isNull
(
pattern
)
&&
pattern
.
startsWith
(
"^srn"
)
?
"link"
:
!
Objects
.
isNull
(
itemsPattern
)
&&
itemsPattern
.
startsWith
(
"^srn"
)
?
"link"
:
!
Objects
.
isNull
(
format
)
?
PRIMITIVE_TYPES_MAP
.
getOrDefault
(
format
,
format
)
:
!
Objects
.
isNull
(
itemsType
)
?
PRIMITIVE_TYPES_MAP
.
getOrDefault
(
itemsType
,
itemsType
)
:
PRIMITIVE_TYPES_MAP
.
getOrDefault
(
type
,
type
);
!
Objects
.
isNull
(
format
)
?
schemaConverterConfig
.
getPrimitiveTypesMap
()
.
getOrDefault
(
format
,
format
)
:
!
Objects
.
isNull
(
itemsType
)
?
schemaConverterConfig
.
getPrimitiveTypesMap
()
.
getOrDefault
(
itemsType
,
itemsType
)
:
schemaConverterConfig
.
getPrimitiveTypesMap
()
.
getOrDefault
(
type
,
type
);
}
}
indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java
View file @
dc232e1c
...
...
@@ -20,6 +20,7 @@ import org.apache.http.HttpStatus;
import
org.opengroup.osdu.core.common.logging.JaxRsDpsLog
;
import
org.opengroup.osdu.core.common.model.http.AppException
;
import
org.opengroup.osdu.core.common.search.Preconditions
;
import
org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterConfig
;
import
org.opengroup.osdu.indexer.schema.converter.interfaces.SchemaToStorageFormat
;
import
org.opengroup.osdu.indexer.schema.converter.tags.PropertiesData
;
import
org.opengroup.osdu.indexer.schema.converter.tags.SchemaRoot
;
...
...
@@ -36,15 +37,16 @@ import java.util.stream.Collectors;
public
class
SchemaToStorageFormatImpl
implements
SchemaToStorageFormat
{
private
ObjectMapper
objectMapper
;
private
JaxRsDpsLog
log
;
private
SchemaConverterConfig
schemaConverterConfig
;
@Inject
public
SchemaToStorageFormatImpl
(
ObjectMapper
objectMapper
,
JaxRsDpsLog
log
)
{
public
SchemaToStorageFormatImpl
(
ObjectMapper
objectMapper
,
JaxRsDpsLog
log
,
SchemaConverterConfig
schemaConverterConfig
)
{
Preconditions
.
checkNotNull
(
objectMapper
,
"objectMapper cannot be null"
);
this
.
objectMapper
=
objectMapper
;
this
.
log
=
log
;
this
.
schemaConverterConfig
=
schemaConverterConfig
;
}
@Override
...
...
@@ -82,7 +84,7 @@ public class SchemaToStorageFormatImpl implements SchemaToStorageFormat {
Preconditions
.
checkNotNull
(
objectMapper
,
"schemaServiceSchema cannot be null"
);
Preconditions
.
checkNotNullOrEmpty
(
kind
,
"kind cannot be null or empty"
);
PropertiesProcessor
propertiesProcessor
=
new
PropertiesProcessor
(
schemaServiceSchema
.
getDefinitions
(),
log
);
PropertiesProcessor
propertiesProcessor
=
new
PropertiesProcessor
(
schemaServiceSchema
.
getDefinitions
(),
log
,
schemaConverterConfig
);
final
List
<
Map
<
String
,
Object
>>
storageSchemaItems
=
new
ArrayList
<>();
if
(
schemaServiceSchema
.
getProperties
()
!=
null
)
{
...
...
indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterConfig.java
0 → 100644
View file @
dc232e1c
package
org.opengroup.osdu.indexer.schema.converter.config
;
import
java.util.Map
;
import
java.util.Set
;
/*
Provides configuration for the schema converter
*/
public
interface
SchemaConverterConfig
{
Set
<
String
>
getSkippedDefinitions
();
Set
<
String
>
getSupportedArrayTypes
();
Map
<
String
,
String
>
getSpecialDefinitionsMap
();
Map
<
String
,
String
>
getPrimitiveTypesMap
();
}
indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterPropertiesConfig.java
0 → 100644
View file @
dc232e1c
package
org.opengroup.osdu.indexer.schema.converter.config
;
import
lombok.Getter
;
import
lombok.Setter
;
import
org.springframework.boot.context.properties.ConfigurationProperties
;
import
org.springframework.context.annotation.Configuration
;
import
java.util.*
;
@Configuration
@ConfigurationProperties
(
prefix
=
"schema.converter"
)
@Getter
@Setter
public
class
SchemaConverterPropertiesConfig
implements
SchemaConverterConfig
{
private
Set
<
String
>
skippedDefinitions
=
getDefaultSkippedDefinitions
();
private
Set
<
String
>
supportedArrayTypes
=
getDefaultSupportedArrayTypes
();
private
Map
<
String
,
String
>
specialDefinitionsMap
=
getDefaultSpecialDefinitionsMap
();
private
Map
<
String
,
String
>
primitiveTypesMap
=
getDefaultPrimitiveTypesMap
();
private
Set
<
String
>
getDefaultSkippedDefinitions
()
{
return
new
HashSet
<>(
Arrays
.
asList
(
"AbstractAnyCrsFeatureCollection.1.0.0"
,
"anyCrsGeoJsonFeatureCollection"
));
}
private
Set
<
String
>
getDefaultSupportedArrayTypes
()
{
return
new
HashSet
<>(
Arrays
.
asList
(
"boolean"
,
"integer"
,
"number"
,
"string"
));
}
private
Map
<
String
,
String
>
getDefaultSpecialDefinitionsMap
()
{
Map
<
String
,
String
>
defaultSpecialDefinitions
=
new
HashMap
<>();
defaultSpecialDefinitions
.
put
(
"AbstractFeatureCollection.1.0.0"
,
"core:dl:geoshape:1.0.0"
);
defaultSpecialDefinitions
.
put
(
"core_dl_geopoint"
,
"core:dl:geopoint:1.0.0"
);
defaultSpecialDefinitions
.
put
(
"geoJsonFeatureCollection"
,
"core:dl:geoshape:1.0.0"
);
return
defaultSpecialDefinitions
;
}
private
Map
<
String
,
String
>
getDefaultPrimitiveTypesMap
()
{
Map
<
String
,
String
>
defaultPrimitiveTypesMap
=
new
HashMap
<>();
defaultPrimitiveTypesMap
.
put
(
"boolean"
,
"bool"
);
defaultPrimitiveTypesMap
.
put
(
"number"
,
"double"
);
defaultPrimitiveTypesMap
.
put
(
"date-time"
,
"datetime"
);
defaultPrimitiveTypesMap
.
put
(
"date"
,
"datetime"
);
defaultPrimitiveTypesMap
.
put
(
"time"
,
"datetime"
);
defaultPrimitiveTypesMap
.
put
(
"int32"
,
"int"
);
defaultPrimitiveTypesMap
.
put
(
"integer"
,
"int"
);
defaultPrimitiveTypesMap
.
put
(
"int64"
,
"long"
);
return
defaultPrimitiveTypesMap
;
}
}
indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessorTest.java
View file @
dc232e1c
...
...
@@ -18,6 +18,7 @@ import org.junit.Test;
import
org.mockito.Mockito
;
import
org.opengroup.osdu.core.common.logging.JaxRsDpsLog
;
import
org.opengroup.osdu.core.common.model.http.AppException
;
import
org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterPropertiesConfig
;
import
org.opengroup.osdu.indexer.schema.converter.tags.AllOfItem
;
import
org.opengroup.osdu.indexer.schema.converter.tags.Definition
;
import
org.opengroup.osdu.indexer.schema.converter.tags.Definitions
;
...
...
@@ -38,14 +39,15 @@ public class PropertiesProcessorTest {
public
void
should_fail_on_unknown_reference_definition
()
{
JaxRsDpsLog
log
=
Mockito
.
mock
(
JaxRsDpsLog
.
class
);
new
PropertiesProcessor
(
Mockito
.
mock
(
Definitions
.
class
),
log
).
processRef
(
DEFINITIONS_PREFIX
+
"unknownDefinition"
);
new
PropertiesProcessor
(
Mockito
.
mock
(
Definitions
.
class
),
log
,
new
SchemaConverterPropertiesConfig
())
.
processRef
(
DEFINITIONS_PREFIX
+
"unknownDefinition"
);
}
@Test
public
void
should_not_process_special_reference
()
{
JaxRsDpsLog
log
=
Mockito
.
mock
(
JaxRsDpsLog
.
class
);
assertFalse
(
new
PropertiesProcessor
(
null
,
log
)
assertFalse
(
new
PropertiesProcessor
(
null
,
log
,
new
SchemaConverterPropertiesConfig
()
)
.
processRef
(
DEFINITIONS_PREFIX
+
"anyCrsGeoJsonFeatureCollection"
).
findAny
().
isPresent
());
}
...
...
@@ -53,7 +55,7 @@ public class PropertiesProcessorTest {
public
void
should_return_special_type
()
{
JaxRsDpsLog
log
=
Mockito
.
mock
(
JaxRsDpsLog
.
class
);
String
res
=
new
PropertiesProcessor
(
null
,
PATH
,
log
)
String
res
=
new
PropertiesProcessor
(
null
,
PATH
,
log
,
new
SchemaConverterPropertiesConfig
()
)
.
processRef
(
DEFINITIONS_PREFIX
+
"core_dl_geopoint"
).
map
(
Object:
:
toString
).
reduce
(
""
,
String:
:
concat
);
assertEquals
(
"{path="
+
PATH
+
", kind=core:dl:geopoint:1.0.0}"
,
res
);
}
...
...
@@ -77,7 +79,7 @@ public class PropertiesProcessorTest {
String
defName
=
"defName"
;
definitions
.
add
(
defName
,
definition
);
String
res
=
new
PropertiesProcessor
(
definitions
,
PATH
,
log
)
String
res
=
new
PropertiesProcessor
(
definitions
,
PATH
,
log
,
new
SchemaConverterPropertiesConfig
()
)
.
processRef
(
DEFINITIONS_PREFIX
+
defName
).
map
(
Object:
:
toString
).
reduce
(
""
,
String:
:
concat
);
assertEquals
(
res
,
"{path="
+
PATH
+
"."
+
propertyName
+
", kind=string}"
);
}
...
...
@@ -95,7 +97,7 @@ public class PropertiesProcessorTest {
properties
.
put
(
PATH
,
property
);
allOfItem
.
setProperties
(
properties
);
String
res
=
new
PropertiesProcessor
(
Mockito
.
mock
(
Definitions
.
class
),
log
)
String
res
=
new
PropertiesProcessor
(
Mockito
.
mock
(
Definitions
.
class
),
log
,
new
SchemaConverterPropertiesConfig
()
)
.
processItem
(
allOfItem
).
map
(
Object:
:
toString
).
reduce
(
""
,
String:
:
concat
);
assertEquals
(
"{path="
+
PATH
+
", kind=int}"
,
res
);
}
...
...
indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImplTest.java
View file @
dc232e1c
...
...
@@ -17,14 +17,12 @@ package org.opengroup.osdu.indexer.schema.converter;
import
com.fasterxml.jackson.core.type.TypeReference
;
import
com.fasterxml.jackson.databind.ObjectMapper
;
import
org.junit.Test
;
import
org.mockito.InjectMocks
;
import
org.mockito.Mock
;
import
org.mockito.Mockito
;
import
org.opengroup.osdu.core.common.logging.JaxRsDpsLog
;
import
org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterPropertiesConfig
;
import
org.springframework.boot.test.context.SpringBootTest
;
import
org.springframework.http.converter.json.Jackson2ObjectMapperBuilder
;
import
java.io.File
;
import
java.io.IOException
;
import
java.net.URISyntaxException
;
import
java.nio.charset.StandardCharsets
;
...
...
@@ -46,7 +44,8 @@ public class SchemaToStorageFormatImplTest {
private
JaxRsDpsLog
jaxRsDpsLog
=
Mockito
.
mock
(
JaxRsDpsLog
.
class
);
private
SchemaToStorageFormatImpl
schemaToStorageFormatImpl
=
new
SchemaToStorageFormatImpl
(
objectMapper
,
jaxRsDpsLog
);
=
new
SchemaToStorageFormatImpl
(
objectMapper
,
jaxRsDpsLog
,
new
SchemaConverterPropertiesConfig
());
@Test
public
void
firstSchemaPassed
()
{
...
...
indexer-core/src/test/java/org/opengroup/osdu/indexer/service/impl/SchemaServiceImplTest.java
View file @
dc232e1c
...
...
@@ -44,7 +44,7 @@ public class SchemaServiceImplTest {
private
JaxRsDpsLog
jaxRsDpsLog
=
Mockito
.
mock
(
JaxRsDpsLog
.
class
);
@Spy
private
SchemaToStorageFormatImpl
schemaToStorageFormat
=
new
SchemaToStorageFormatImpl
(
objectMapper
,
jaxRsDpsLog
);
private
SchemaToStorageFormatImpl
schemaToStorageFormat
=
new
SchemaToStorageFormatImpl
(
objectMapper
,
jaxRsDpsLog
,
null
);
@Mock
private
IUrlFetchService
urlFetchService
;
...
...
provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/SchemaServiceTest.java
View file @
dc232e1c
...
...
@@ -47,7 +47,7 @@ public class SchemaServiceTest {
private
JaxRsDpsLog
log
=
Mockito
.
mock
(
JaxRsDpsLog
.
class
);
@Spy
private
SchemaToStorageFormatImpl
schemaToStorageFormatImpl
=
new
SchemaToStorageFormatImpl
(
objectMapper
,
log
);
private
SchemaToStorageFormatImpl
schemaToStorageFormatImpl
=
new
SchemaToStorageFormatImpl
(
objectMapper
,
log
,
null
);
@Mock
private
IUrlFetchService
urlFetchService
;
@Mock
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment