diff --git a/indexer-core-plus/src/main/resources/application.properties b/indexer-core-plus/src/main/resources/application.properties index 2f1c6c98b8bf64380cb22325a88753578b3571eb..47fdb129c5c7509d50a45ed9e9e09c64e2fec53d 100644 --- a/indexer-core-plus/src/main/resources/application.properties +++ b/indexer-core-plus/src/main/resources/application.properties @@ -63,6 +63,7 @@ rabbitmq-retry-limit=5 # Feature flag settings featureFlag.strategy=dataPartition +featureFlag.mapBooleanToString.enabled=false featureFlag.asIngestedCoordinates.enabled=false featureFlag.keywordLower.enabled=false featureFlag.bagOfWords.enabled=false diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/config/IndexerConfigurationProperties.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/config/IndexerConfigurationProperties.java index 0b7efbc6e977abeb0fdfac645254485f96cbe6f6..a8afb4a134135fccd5b8a6b743af6ae6293f9544 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/config/IndexerConfigurationProperties.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/config/IndexerConfigurationProperties.java @@ -13,6 +13,7 @@ import org.springframework.context.annotation.Configuration; @Getter @Setter public class IndexerConfigurationProperties { + public static final String MAP_BOOL2STRING_FEATURE_NAME = "featureFlag.mapBooleanToString.enabled"; public static final String KEYWORD_LOWER_FEATURE_NAME = "featureFlag.keywordLower.enabled"; public static final String BAG_OF_WORDS_FEATURE_NAME = "featureFlag.bagOfWords.enabled"; public static final String COLLABORATIONS_FEATURE_NAME = "collaborations-enabled"; diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java index 45a13b37f6b127e064f4c0807b5f3f132f1acf9f..0a84da41d75289edaa17d965ee9ce43073dcdc0b 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java @@ -14,6 +14,7 @@ package org.opengroup.osdu.indexer.schema.converter; +import lombok.Getter; import org.apache.http.HttpStatus; import org.opengroup.osdu.core.common.Constants; import org.opengroup.osdu.core.common.model.http.AppException; @@ -36,7 +37,7 @@ import java.util.stream.Stream; public class PropertiesProcessor { - private SchemaConverterConfig schemaConverterConfig; + private final SchemaConverterConfig schemaConverterConfig; private static final String TYPE_KEY = "type"; private static final String DEF_PREFIX = "#/definitions/"; @@ -47,6 +48,7 @@ public class PropertiesProcessor { private final String pathPrefix; private final String pathPrefixWithDot; + @Getter private final List<String> errors = new LinkedList<>(); public PropertiesProcessor(Definitions definitions, SchemaConverterConfig schemaConverterConfig) { @@ -124,10 +126,6 @@ public class PropertiesProcessor { return properties.entrySet().stream().flatMap(this::processPropertyEntry); } - public List<String> getErrors() { - return errors; - } - private String getDefinitionIdentity(String definitionSubRef) { String[] components = definitionSubRef.split(":"); switch (components.length) { @@ -208,7 +206,7 @@ public class PropertiesProcessor { if (Objects.nonNull(entry.getValue().getProperties())) { PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey() - , new SchemaConverterPropertiesConfig()); + , schemaConverterConfig); Stream<Map<String, Object>> result = entry.getValue().getProperties().entrySet().stream().flatMap(propertiesProcessor::processPropertyEntry); errors.addAll(propertiesProcessor.getErrors()); return result; @@ -216,7 +214,7 @@ public class PropertiesProcessor { if (Objects.nonNull(entry.getValue().getRef())) { PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions - , pathPrefixWithDot + entry.getKey(), new SchemaConverterPropertiesConfig()); + , pathPrefixWithDot + entry.getKey(), schemaConverterConfig); Stream<Map<String, Object>> refResult = propertiesProcessor.processRef(entry.getValue().getRef()); errors.addAll(propertiesProcessor.getErrors()); return refResult; @@ -238,7 +236,7 @@ public class PropertiesProcessor { indexHint.getOrDefault(TYPE_KEY, schemaConverterConfig.getDefaultObjectArraysType()); if (schemaConverterConfig.getProcessedArraysTypes().contains(indexingType)) { - PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, new SchemaConverterPropertiesConfig()); + PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, schemaConverterConfig); Stream<Map<String, Object>> propertiesStream = Stream.empty(); @@ -265,7 +263,7 @@ public class PropertiesProcessor { if (Objects.nonNull(entry.getValue().getAllOf())) { PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey() - , new SchemaConverterPropertiesConfig()); + , schemaConverterConfig); ofItems = entry.getValue().getAllOf().stream().flatMap(propertiesProcessor::processItem); errors.addAll(propertiesProcessor.getErrors()); @@ -273,7 +271,7 @@ public class PropertiesProcessor { if (Objects.nonNull(entry.getValue().getAnyOf())) { PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey() - , new SchemaConverterPropertiesConfig()); + , schemaConverterConfig); ofItems = Stream.concat(Optional.ofNullable(ofItems).orElseGet(Stream::empty), entry.getValue().getAnyOf().stream().flatMap(propertiesProcessor::processItem)); @@ -282,7 +280,7 @@ public class PropertiesProcessor { if (Objects.nonNull(entry.getValue().getOneOf())) { PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey() - , new SchemaConverterPropertiesConfig()); + , schemaConverterConfig); ofItems = Stream.concat(Optional.ofNullable(ofItems).orElseGet(Stream::empty), entry.getValue().getOneOf().stream().flatMap(propertiesProcessor::processItem)); diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterPropertiesConfig.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterPropertiesConfig.java index e969f8e31970a105e8ca7becf06b8831baaa5b9d..7cc6a476bf1438bb8660428d0f41d74a936729b4 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterPropertiesConfig.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterPropertiesConfig.java @@ -5,24 +5,51 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; + import lombok.Getter; import lombok.Setter; import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.context.annotation.Configuration; -@Configuration +import org.opengroup.osdu.core.common.feature.IFeatureFlag; +import org.opengroup.osdu.indexer.util.BooleanFeatureFlagClient; +import org.springframework.stereotype.Component; + +import static org.opengroup.osdu.indexer.config.IndexerConfigurationProperties.MAP_BOOL2STRING_FEATURE_NAME; + +@Component @ConfigurationProperties(prefix = "schema.converter") @Getter @Setter public class SchemaConverterPropertiesConfig implements SchemaConverterConfig { - private Set<String> skippedDefinitions = getDefaultSkippedDefinitions(); - private Set<String> supportedArrayTypes = getDefaultSupportedArrayTypes(); - private Map<String, String> specialDefinitionsMap = getDefaultSpecialDefinitionsMap(); - private Map<String, String> primitiveTypesMap = getDefaultPrimitiveTypesMap(); - private Set<String> processedArraysTypes = getDefaultArraysTypesForProcessing(); - private String defaultObjectArraysType = getObjectArraysDefaultType(); + private Set<String> skippedDefinitions; + private Set<String> supportedArrayTypes; + private Map<String, String> specialDefinitionsMap; + private Map<String, String> primitiveTypesMap; + private Set<String> processedArraysTypes; + private String defaultObjectArraysType; + + private final IFeatureFlag featureFlagChecker; + private final BooleanFeatureFlagClient partitionFlagChecker; + + public SchemaConverterPropertiesConfig(IFeatureFlag featureFlagChecker, BooleanFeatureFlagClient partitionFlagChecker) { + this.featureFlagChecker = featureFlagChecker; + this.partitionFlagChecker = partitionFlagChecker; + skippedDefinitions = getDefaultSkippedDefinitions(); + supportedArrayTypes = getDefaultSupportedArrayTypes(); + specialDefinitionsMap = getDefaultSpecialDefinitionsMap(); + processedArraysTypes = getDefaultArraysTypesForProcessing(); + defaultObjectArraysType = getObjectArraysDefaultType(); + } + public void resetToDefault() { + skippedDefinitions = getDefaultSkippedDefinitions(); + supportedArrayTypes = getDefaultSupportedArrayTypes(); + specialDefinitionsMap = getDefaultSpecialDefinitionsMap(); + primitiveTypesMap = getDefaultPrimitiveTypesMap(); + processedArraysTypes = getDefaultArraysTypesForProcessing(); + defaultObjectArraysType = getObjectArraysDefaultType(); + } private Set<String> getDefaultSkippedDefinitions() { return new HashSet<>(Arrays.asList("AbstractAnyCrsFeatureCollection", @@ -46,7 +73,31 @@ public class SchemaConverterPropertiesConfig implements SchemaConverterConfig { private Map<String, String> getDefaultPrimitiveTypesMap() { Map<String, String> defaultPrimitiveTypesMap = new HashMap<>(); - defaultPrimitiveTypesMap.put("boolean", "bool"); + /* + Logic behing feature flag settings + If service-level FF is ON for a given deployment/environment, then the FF is ON for all the data partitions under the deployment/environment + If service-level FF is OFF but it is ON for a given data partition, then the FF is ON for the given data partition + + With this solution, the service providers can decide which level of FF should be turned on. For example, + If there are only few data partitions in a given deployment and there are not many data needed to be indexed, + the service provider can turn on the service-level FF and re-index all the data partitions in one shot. + If there are lots of data partitions or some data partitions have lots of data in a given deployment, step should be as following. + + 1. Turn the service-level FF to be OFF + 2. Turn on the FF via partition service for a given data partition and re-index all the data in the given data partition + 3. Repeat step 2 until all the data partitions have been re-indexed + 4. Turn the service-level FF to be ON and re-deploy the service. So all new data partitions apply the fix + */ + if (this.featureFlagChecker.isFeatureEnabled(MAP_BOOL2STRING_FEATURE_NAME) || partitionFlagChecker.isEnabled(MAP_BOOL2STRING_FEATURE_NAME, false)) { + // in the earlier versions boolean was translated to bool and + // this caused mapping boolean values like text as entry in StorageType entry in map is boolean + // in some places boolean is still presented as bool so here both are normalized to boolean + defaultPrimitiveTypesMap.put("boolean", "boolean"); + defaultPrimitiveTypesMap.put("bool", "boolean"); + } else { + defaultPrimitiveTypesMap.put("boolean", "bool"); + } + defaultPrimitiveTypesMap.put("number", "double"); defaultPrimitiveTypesMap.put("date-time", "datetime"); defaultPrimitiveTypesMap.put("date", "datetime"); @@ -58,6 +109,13 @@ public class SchemaConverterPropertiesConfig implements SchemaConverterConfig { return defaultPrimitiveTypesMap; } + public Map<String, String> getPrimitiveTypesMap() { + if (primitiveTypesMap == null) { + primitiveTypesMap = getDefaultPrimitiveTypesMap(); + } + return primitiveTypesMap; + } + private Set<String> getDefaultArraysTypesForProcessing() { return new HashSet<>(Arrays.asList("nested")); } diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/readme.md b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/readme.md index 17a4ca99a040170dd656acdd655a55d3b6cd3a06..f4f8b2e11dec5b56a9259af0993a60aa23f215ad 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/readme.md +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/readme.md @@ -278,7 +278,7 @@ Following primitive types are converted to the Storage Service Schema types (all "date"->"datetime" "int64"->"long" "number"->"double" -"boolean"->"bool" +"bool"->"boolean" "integer"->"int" ``` diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java index adf0d41d99b59ee4ab9cb014d89530f6b9a4135f..a43a1078fa601122873a0f9d446ce9aa4f8f3541 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java @@ -35,6 +35,7 @@ import org.opengroup.osdu.indexer.util.PropertyUtil; import org.opengroup.osdu.indexer.util.geo.decimator.DecimatedResult; import org.opengroup.osdu.indexer.util.geo.decimator.GeoShapeDecimator; import org.opengroup.osdu.indexer.util.geo.extractor.PointExtractor; +import org.opengroup.osdu.indexer.util.BooleanFeatureFlagClient; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -44,6 +45,7 @@ import java.util.*; import java.util.stream.Collectors; import static org.opengroup.osdu.indexer.model.Constants.AS_INGESTED_COORDINATES_FEATURE_NAME; +import static org.opengroup.osdu.indexer.config.IndexerConfigurationProperties.MAP_BOOL2STRING_FEATURE_NAME; @Component public class StorageIndexerPayloadMapper { @@ -67,7 +69,10 @@ public class StorageIndexerPayloadMapper { private PointExtractor pointExtractor; @Autowired - private IFeatureFlag asIngestedCoordinatesFeatureFlag; + private IFeatureFlag featureFlagChecker; + + @Autowired + private BooleanFeatureFlagClient booleanFeatureFlagClient; public Map<String, Object> mapDataPayload(ArrayList<String> asIngestedCoordinatesPaths, IndexSchema storageSchema, Map<String, Object> storageRecordData, String recordId) { @@ -81,7 +86,7 @@ public class StorageIndexerPayloadMapper { mapDataPayload(storageSchema.getDataSchema(), storageRecordData, recordId, dataCollectorMap); mapVirtualPropertiesPayload(storageSchema, recordId, dataCollectorMap); - if (this.asIngestedCoordinatesFeatureFlag.isFeatureEnabled(AS_INGESTED_COORDINATES_FEATURE_NAME)) { + if (this.featureFlagChecker.isFeatureEnabled(AS_INGESTED_COORDINATES_FEATURE_NAME)) { mapAsIngestedCoordinatesPayload(recordId, asIngestedCoordinatesPaths, storageRecordData, dataCollectorMap); } @@ -115,11 +120,38 @@ public class StorageIndexerPayloadMapper { switch (elasticType) { case KEYWORD: case TEXT: - this.attributeParsingService.tryParseString(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap); + /* + Logic behing feature flag settings + If service-level FF is ON for a given deployment/environment, then the FF is ON for all the data partitions under the deployment/environment + If service-level FF is OFF but it is ON for a given data partition, then the FF is ON for the given data partition + + With this solution, the service providers can decide which level of FF should be turned on. For example, + If there are only few data partitions in a given deployment and there are not many data needed to be indexed, + the service provider can turn on the service-level FF and re-index all the data partitions in one shot. + If there are lots of data partitions or some data partitions have lots of data in a given deployment, step should be as following. + + 1. Turn the service-level FF to be OFF + 2. Turn on the FF via partition service for a given data partition and re-index all the data in the given data partition + 3. Repeat step 2 until all the data partitions have been re-indexed + 4. Turn the service-level FF to be ON and re-deploy the service. So all new data partitions apply the fix + */ + // Feature flag enforces conversion of values to string when index type is string, this is required for features using + // copy_to mechanic like bag of words, however originally index type for boolean values was string so this would convert + // those values to string without changing the index type, changing index type would require migration so it is also + // under this feature flag as not all users will be interested with it + if (this.featureFlagChecker.isFeatureEnabled(MAP_BOOL2STRING_FEATURE_NAME) || booleanFeatureFlagClient.isEnabled(MAP_BOOL2STRING_FEATURE_NAME, false)) { + this.attributeParsingService.tryParseString(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap); + } else { + dataCollectorMap.put(schemaPropertyName, storageRecordValue); + } break; case KEYWORD_ARRAY: case TEXT_ARRAY: - this.attributeParsingService.tryParseValueArray(String.class, recordId, schemaPropertyName, storageRecordValue, dataCollectorMap); + if (this.featureFlagChecker.isFeatureEnabled(MAP_BOOL2STRING_FEATURE_NAME) || booleanFeatureFlagClient.isEnabled(MAP_BOOL2STRING_FEATURE_NAME, false)) { + this.attributeParsingService.tryParseValueArray(String.class, recordId, schemaPropertyName, storageRecordValue, dataCollectorMap); + } else { + dataCollectorMap.put(schemaPropertyName, storageRecordValue); + } break; case INTEGER_ARRAY: this.attributeParsingService.tryParseValueArray(Integer.class, recordId, schemaPropertyName, storageRecordValue, dataCollectorMap); diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessorTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessorTest.java index fdcfeba5de48702c853597f8dcd5dbc5d642c821..5700a71d3566d2947116aa10848ce8fd6dc9b23c 100644 --- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessorTest.java +++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessorTest.java @@ -15,7 +15,9 @@ package org.opengroup.osdu.indexer.schema.converter; import com.google.common.collect.ImmutableMap; +import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; import org.mockito.Mockito; import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterPropertiesConfig; @@ -25,40 +27,70 @@ import org.opengroup.osdu.indexer.schema.converter.tags.Definitions; import org.opengroup.osdu.indexer.schema.converter.tags.Items; import org.opengroup.osdu.indexer.schema.converter.tags.TypeProperty; +import java.io.IOException; import java.util.LinkedHashMap; import java.util.Map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; - +import static org.mockito.Mockito.when; +import static org.mockito.MockitoAnnotations.initMocks; +import static org.opengroup.osdu.indexer.config.IndexerConfigurationProperties.MAP_BOOL2STRING_FEATURE_NAME; +import org.opengroup.osdu.core.common.feature.IFeatureFlag; +import org.opengroup.osdu.indexer.util.BooleanFeatureFlagClient; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.context.annotation.Import; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringRunner; + +@RunWith(SpringRunner.class) +@Import({IFeatureFlag.class, BooleanFeatureFlagClient.class}) +@ContextConfiguration(classes = {PropertiesProcessorTest.class, IFeatureFlag.class, BooleanFeatureFlagClient.class}) +@SpringBootTest(classes = {IFeatureFlag.class, BooleanFeatureFlagClient.class}) public class PropertiesProcessorTest { private static final String PATH = "given_path"; private static final String DEFINITIONS_PREFIX = "#/definitions/"; + @MockBean + private IFeatureFlag featureFlagChecker; + + @MockBean + private BooleanFeatureFlagClient partitionFlagClient; + + private SchemaConverterPropertiesConfig schemaConverterConfig; + + @Before + public void setup() throws IOException { + initMocks(this); + schemaConverterConfig = new SchemaConverterPropertiesConfig(featureFlagChecker, partitionFlagClient); + } + @Test public void should_fail_on_bad_reference_definition() { - PropertiesProcessor propertiesProcessor = new PropertiesProcessor(Mockito.mock(Definitions.class), new SchemaConverterPropertiesConfig()); + PropertiesProcessor propertiesProcessor = new PropertiesProcessor(Mockito.mock(Definitions.class), + schemaConverterConfig); propertiesProcessor.processRef(DEFINITIONS_PREFIX + "unknownDefinition"); assertEquals(1, propertiesProcessor.getErrors().size()); } @Test public void should_fail_on_wrong_definition_format() { - PropertiesProcessor propertiesProcessor = new PropertiesProcessor(Mockito.mock(Definitions.class), new SchemaConverterPropertiesConfig()); + PropertiesProcessor propertiesProcessor = new PropertiesProcessor(Mockito.mock(Definitions.class), schemaConverterConfig); propertiesProcessor.processRef("unknownDefinition"); assertEquals(1, propertiesProcessor.getErrors().size()); } @Test public void should_not_process_special_reference() { - assertFalse(new PropertiesProcessor(null, new SchemaConverterPropertiesConfig()) + assertFalse(new PropertiesProcessor(null, schemaConverterConfig) .processRef(DEFINITIONS_PREFIX + "a:b:anyCrsGeoJsonFeatureCollection:1.0.0").findAny().isPresent()); } @Test public void should_return_special_type() { - String res = new PropertiesProcessor(null, PATH, new SchemaConverterPropertiesConfig()) + String res = new PropertiesProcessor(null, PATH, schemaConverterConfig) .processRef(DEFINITIONS_PREFIX + "a:b:core_dl_geopoint:1.0.0").map(Object::toString).reduce("", String::concat); assertEquals("{path=" + PATH + ", kind=core:dl:geopoint:1.0.0}", res); } @@ -82,7 +114,7 @@ public class PropertiesProcessorTest { String defName = "a:b:defName:1.0.0"; definitions.add(defName, definition); - String res = new PropertiesProcessor(definitions, PATH, new SchemaConverterPropertiesConfig()) + String res = new PropertiesProcessor(definitions, PATH, schemaConverterConfig) .processRef(DEFINITIONS_PREFIX + defName).map(Object::toString).reduce("", String::concat); assertEquals(res, "{path="+ PATH + "." + propertyName + ", kind=string}"); } @@ -100,11 +132,64 @@ public class PropertiesProcessorTest { properties.put(PATH, property); allOfItem.setProperties(properties); - String res = new PropertiesProcessor(Mockito.mock(Definitions.class), new SchemaConverterPropertiesConfig()) + String res = new PropertiesProcessor(Mockito.mock(Definitions.class), schemaConverterConfig) .processItem(allOfItem).map(Object::toString).reduce("", String::concat); assertEquals("{path=" + PATH + ", kind=int}", res); } + @Test + public void should_return_boolean_from_boolean_item_FF_OFF() { + // in the earlier versions boolean was translated to bool and + // this caused mapping boolean values like text as entry in StorageType entry in map is boolean + internal_should_return_boolean_from_boolean_item(false); + } + + @Test + public void should_return_boolean_from_boolean_item_FF_ON() { + // in the earlier versions boolean was translated to bool and + // this caused mapping boolean values like text as entry in StorageType entry in map is boolean + internal_should_return_boolean_from_boolean_item(true); + } + + private void internal_should_return_boolean_from_boolean_item(boolean ffFlag) { + when(this.featureFlagChecker.isFeatureEnabled(MAP_BOOL2STRING_FEATURE_NAME)).thenReturn(ffFlag); + when(schemaConverterConfig.getFeatureFlagChecker().isFeatureEnabled(MAP_BOOL2STRING_FEATURE_NAME)).thenReturn(ffFlag); + schemaConverterConfig.resetToDefault(); + AllOfItem allOfItem = new AllOfItem(); + JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class); + + TypeProperty property = new TypeProperty(); + property.setType("boolean"); + + Map<String, TypeProperty> properties = new LinkedHashMap<>(); + properties.put(PATH, property); + allOfItem.setProperties(properties); + + assertEquals( + "{path="+PATH+", kind="+ (ffFlag?"boolean":"bool")+"}", + new PropertiesProcessor(Mockito.mock(Definitions.class), schemaConverterConfig) + .processItem(allOfItem).map(Object::toString).reduce("", String::concat) + ); + } + + @Test + public void should_return_boolean_from_bool_item() { + // StorageType entry in map is boolean not bool + AllOfItem allOfItem = new AllOfItem(); + JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class); + + TypeProperty property = new TypeProperty(); + property.setType("bool"); + + Map<String, TypeProperty> properties = new LinkedHashMap<>(); + properties.put(PATH, property); + allOfItem.setProperties(properties); + + String res = new PropertiesProcessor(Mockito.mock(Definitions.class), schemaConverterConfig) + .processItem(allOfItem).map(Object::toString).reduce("", String::concat); + assertEquals("{path=" + PATH + ", kind=bool}", res); + } + @Test public void should_return_processed_nested_array_items(){ JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class); @@ -128,7 +213,7 @@ public class PropertiesProcessorTest { AllOfItem allOfItem = new AllOfItem(); allOfItem.setProperties(allOfItemProperties); - String res = new PropertiesProcessor(Mockito.mock(Definitions.class), new SchemaConverterPropertiesConfig()) + String res = new PropertiesProcessor(Mockito.mock(Definitions.class), schemaConverterConfig) .processItem(allOfItem).map(Object::toString).reduce("", String::concat); assertEquals("{path="+ PATH + ", kind=nested, properties=[{path="+ PATH + ", kind=int}]}",res); } @@ -156,7 +241,7 @@ public class PropertiesProcessorTest { AllOfItem allOfItem = new AllOfItem(); allOfItem.setProperties(allOfItemProperties); - String res = new PropertiesProcessor(Mockito.mock(Definitions.class), new SchemaConverterPropertiesConfig()) + String res = new PropertiesProcessor(Mockito.mock(Definitions.class), schemaConverterConfig) .processItem(allOfItem).map(Object::toString).reduce("", String::concat); assertEquals("{path="+ PATH + ", kind=flattened}",res); } @@ -183,7 +268,7 @@ public class PropertiesProcessorTest { AllOfItem allOfItem = new AllOfItem(); allOfItem.setProperties(allOfItemProperties); - String res = new PropertiesProcessor(Mockito.mock(Definitions.class), new SchemaConverterPropertiesConfig()) + String res = new PropertiesProcessor(Mockito.mock(Definitions.class), schemaConverterConfig) .processItem(allOfItem).map(Object::toString).reduce("", String::concat); assertEquals("{path="+ PATH + ", kind=[]object}",res); } @@ -205,8 +290,8 @@ public class PropertiesProcessorTest { AllOfItem allOfItem = new AllOfItem(); allOfItem.setProperties(allOfItemProperties); - String res = new PropertiesProcessor(Mockito.mock(Definitions.class), new SchemaConverterPropertiesConfig()) + String res = new PropertiesProcessor(Mockito.mock(Definitions.class), schemaConverterConfig) .processItem(allOfItem).map(Object::toString).reduce("", String::concat); assertEquals("{path="+ PATH + ", kind=[]int}",res); } -} \ No newline at end of file +} diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImplTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImplTest.java index ea24acdc42810af48f38a03b46f0b7289347c736..80bd8fcfece51977eff26f7b0630ed8ba8e58544 100644 --- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImplTest.java +++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImplTest.java @@ -17,20 +17,26 @@ package org.opengroup.osdu.indexer.schema.converter; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.MockitoAnnotations; +import org.opengroup.osdu.core.common.feature.IFeatureFlag; import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; import org.opengroup.osdu.indexer.cache.partitionsafe.VirtualPropertiesSchemaCache; import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterPropertiesConfig; import org.opengroup.osdu.indexer.schema.converter.exeption.SchemaProcessingException; +import org.opengroup.osdu.indexer.util.BooleanFeatureFlagClient; +import static org.opengroup.osdu.indexer.config.IndexerConfigurationProperties.MAP_BOOL2STRING_FEATURE_NAME; + +import org.springframework.context.annotation.Configuration; import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder; import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.util.ReflectionUtils; import java.io.IOException; +import java.lang.reflect.Field; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -42,10 +48,10 @@ import java.util.Map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.*; @RunWith(SpringRunner.class) +@Configuration public class SchemaToStorageFormatImplTest { private static final String KIND = "KIND_VAL"; @@ -54,17 +60,28 @@ public class SchemaToStorageFormatImplTest { private JaxRsDpsLog jaxRsDpsLog = Mockito.mock(JaxRsDpsLog.class); - @InjectMocks - private SchemaToStorageFormatImpl schemaToStorageFormatImpl - = new SchemaToStorageFormatImpl(objectMapper, jaxRsDpsLog - , new SchemaConverterPropertiesConfig()); + private SchemaToStorageFormatImpl schemaToStorageFormatImpl; - @Mock private VirtualPropertiesSchemaCache virtualPropertiesSchemaCache; + private IFeatureFlag featureFlag; + + private BooleanFeatureFlagClient partitionFlagClient; + + private SchemaConverterPropertiesConfig schemaConverterPropertiesConfig; @Before public void init() { MockitoAnnotations.initMocks(this); + featureFlag = Mockito.mock(IFeatureFlag.class); + partitionFlagClient = Mockito.mock(BooleanFeatureFlagClient.class); + virtualPropertiesSchemaCache = Mockito.mock(VirtualPropertiesSchemaCache.class); + schemaConverterPropertiesConfig = new SchemaConverterPropertiesConfig(featureFlag, partitionFlagClient); + schemaToStorageFormatImpl + = new SchemaToStorageFormatImpl(objectMapper, jaxRsDpsLog, + schemaConverterPropertiesConfig); + Field field = ReflectionUtils.findField(SchemaToStorageFormatImpl.class, "virtualPropertiesSchemaCache"); + field.setAccessible(true); + ReflectionUtils.setField(field, schemaToStorageFormatImpl, virtualPropertiesSchemaCache); } @Test @@ -148,38 +165,75 @@ public class SchemaToStorageFormatImplTest { } @Test - public void virtualProperties() { - testSingleFile("/converter/index-virtual-properties/virtual-properties-schema.json", "osdu:wks:master-data--Wellbore:1.0.0"); + public void virtualProperties_FFoff() { + testSingleFile("/converter/index-virtual-properties/virtual-properties-schema.json", "osdu:wks:master-data--Wellbore:1.0.0", false); + verify(this.virtualPropertiesSchemaCache, times(1)).put(Mockito.anyString(), Mockito.any()); + } + + @Test + public void virtualProperties_FFon() { + testSingleFile("/converter/index-virtual-properties/virtual-properties-schema.json", "osdu:wks:master-data--Wellbore:1.0.0", true); + verify(this.virtualPropertiesSchemaCache, times(1)).put(Mockito.anyString(), Mockito.any()); + } + + @Test + public void unmatchedVirtualProperties_FFon() { + // The actual property "data.Facility" does not exist for "data.VirtualProperties.DefaultName" + testSingleFile("/converter/index-virtual-properties/unmatched-virtual-properties-schema.json", "osdu:wks:master-data--Wellbore:1.0.0", true); verify(this.virtualPropertiesSchemaCache, times(1)).put(Mockito.anyString(), Mockito.any()); } @Test - public void unmatchedVirtualProperties() { + public void unmatchedVirtualProperties_FFoff() { // The actual property "data.Facility" does not exist for "data.VirtualProperties.DefaultName" - testSingleFile("/converter/index-virtual-properties/unmatched-virtual-properties-schema.json", "osdu:wks:master-data--Wellbore:1.0.0"); + testSingleFile("/converter/index-virtual-properties/unmatched-virtual-properties-schema.json", "osdu:wks:master-data--Wellbore:1.0.0", false); verify(this.virtualPropertiesSchemaCache, times(1)).put(Mockito.anyString(), Mockito.any()); } @Test - public void folderPassed() throws URISyntaxException, IOException { + public void folderPassedWithFF() throws URISyntaxException, IOException { + folderPassed(true); + } + + @Test + public void folderPassedFFOff() throws URISyntaxException, IOException { + folderPassed(false); + } + public void folderPassed(boolean map2StringFF) throws URISyntaxException, IOException { String folder = "/converter/R3-json-schema"; Path path = Paths.get(this.getClass().getResource(folder).toURI()); Files.walk(path) .filter(Files::isRegularFile) .filter(f -> f.toString().endsWith(".json")) - .forEach(f -> testSingleFile(f.toString().replaceAll("\\\\", "/").substring(f.toString().replaceAll("\\\\", "/").indexOf(folder)), "osdu:osdu:Wellbore:1.0.0")); + .forEach(f -> testSingleFile( + f.toString().replaceAll("\\\\", "/").substring(f.toString().replaceAll("\\\\", "/").indexOf(folder)), + "osdu:osdu:Wellbore:1.0.0", + map2StringFF + )); } private void testSingleFile(String filename, String kind) { + testSingleFile(filename, kind, false); + } + + private void testSingleFile(String filename, String kind, boolean map2StringFF) { + when(featureFlag.isFeatureEnabled(MAP_BOOL2STRING_FEATURE_NAME)).thenReturn(map2StringFF); + schemaConverterPropertiesConfig.resetToDefault(); String json = getSchemaFromSchemaService(filename); Map<String, Object> converted = schemaToStorageFormatImpl.convertToMap(json, kind); - Map<String, Object> expected = getStorageSchema(filename + ".res"); + String resource = filename + (map2StringFF?".FF":"")+ ".res"; + if (!existsStorageSchema(resource)) resource = filename + ".res"; + Map<String, Object> expected = getStorageSchema(resource); compareSchemas(expected, converted, filename); } + private boolean existsStorageSchema(String s) { + return null != this.getClass().getResource(s); + } + private Map<String, Object> getStorageSchema(String s) { TypeReference<Map<String, Object>> typeRef diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapperTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapperTest.java index ba5696d3302b0935766f9b566f36b6ed6b492da3..6b9267b45592a27e89a7cd658c243915154e9e67 100644 --- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapperTest.java +++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapperTest.java @@ -27,6 +27,7 @@ import org.opengroup.osdu.indexer.util.geo.decimator.GeoShapeDecimator; import org.opengroup.osdu.indexer.util.geo.decimator.GeometryDecimator; import org.opengroup.osdu.indexer.util.geo.extractor.PointExtractor; import org.opengroup.osdu.indexer.util.parser.*; +import org.opengroup.osdu.indexer.util.BooleanFeatureFlagClient; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; @@ -40,6 +41,7 @@ import java.util.*; import static org.junit.Assert.*; import static org.mockito.Mockito.when; +import static org.opengroup.osdu.indexer.config.IndexerConfigurationProperties.MAP_BOOL2STRING_FEATURE_NAME; import static org.opengroup.osdu.indexer.model.Constants.AS_INGESTED_COORDINATES_FEATURE_NAME; @RunWith(SpringRunner.class) @@ -48,7 +50,8 @@ import static org.opengroup.osdu.indexer.model.Constants.AS_INGESTED_COORDINATES GeometryDecimator.class, PointExtractor.class, GeometryConversionService.class, FeatureFlagCache.class, DpsHeaders.class, JobStatus.class, SchemaConverterPropertiesConfig.class, JaxRsDpsLog.class, ServiceAccountJwtClientMock.class, VirtualPropertiesSchemaCacheMock.class, VirtualPropertiesSchemaCache.class, RequestInfoMock.class, - IFeatureFlag.class, StringParser.class}) + IFeatureFlag.class, StringParser.class, BooleanFeatureFlagClient.class} +) public class StorageIndexerPayloadMapperTest { public static final String FIRST_OBJECT_INNER_PROPERTY = "FirstObjectInnerProperty"; @@ -82,7 +85,10 @@ public class StorageIndexerPayloadMapperTest { private VirtualPropertiesSchemaCache virtualPropertiesSchemaCache; @MockBean - protected IFeatureFlag asIngestedCoordinatesFeatureFlag; + protected IFeatureFlag featureFlagChecker; + + @MockBean + protected BooleanFeatureFlagClient partitionFlagChecker; @BeforeClass public static void setUp() { @@ -281,7 +287,7 @@ public class StorageIndexerPayloadMapperTest { @Test public void mapDataPayloadTestAsIngestedCoordinates() { - when(this.asIngestedCoordinatesFeatureFlag.isFeatureEnabled(AS_INGESTED_COORDINATES_FEATURE_NAME)).thenReturn(true); + when(this.featureFlagChecker.isFeatureEnabled(AS_INGESTED_COORDINATES_FEATURE_NAME)).thenReturn(true); ArrayList<String> asIngestedCoordinatesPaths = new ArrayList<>(Arrays.asList("SpatialLocation.AsIngestedCoordinates")); Map<String, Object> storageRecordData = new HashMap<>(); @@ -317,7 +323,7 @@ public class StorageIndexerPayloadMapperTest { @Test public void mapDataPayloadTestAsIngestedCoordinatesGeographicBottomHoleLocationAndSpatialLocation() { - when(this.asIngestedCoordinatesFeatureFlag.isFeatureEnabled(AS_INGESTED_COORDINATES_FEATURE_NAME)).thenReturn(true); + when(this.featureFlagChecker.isFeatureEnabled(AS_INGESTED_COORDINATES_FEATURE_NAME)).thenReturn(true); ArrayList<String> asIngestedCoordinatesPaths = new ArrayList<>(Arrays.asList("GeographicBottomHoleLocation.AsIngestedCoordinates", "SpatialLocation.AsIngestedCoordinates")); Map<String, Object> storageRecordData = new HashMap<>(); @@ -380,7 +386,7 @@ public class StorageIndexerPayloadMapperTest { @Test public void mapDataPayloadTestAsIngestedCoordinatesWithEmptyZCoordinate() { - when(this.asIngestedCoordinatesFeatureFlag.isFeatureEnabled(AS_INGESTED_COORDINATES_FEATURE_NAME)).thenReturn(true); + when(this.featureFlagChecker.isFeatureEnabled(AS_INGESTED_COORDINATES_FEATURE_NAME)).thenReturn(true); ArrayList<String> asIngestedCoordinatesPaths = new ArrayList<>(Arrays.asList("SpatialLocation.AsIngestedCoordinates")); Map<String, Object> storageRecordData = new HashMap<>(); @@ -404,6 +410,92 @@ public class StorageIndexerPayloadMapperTest { assertNull(dataCollectorMap.get("SpatialLocation.AsIngestedCoordinates.FirstPoint.Z")); } + @Test + public void mapDataPayloadTestAutoconversionsBooleanConversionOn() { + when(this.featureFlagChecker.isFeatureEnabled(MAP_BOOL2STRING_FEATURE_NAME)).thenReturn(true); + Map<String, Object> storageRecordLocalData = loadObject("/converter/index-autoconversions/wellStorageRecordData.json", storageRecordData.getClass()); + IndexSchema indexLocalSchema = loadObject("/converter/index-autoconversions/wellStorageSchema.json", IndexSchema.class); + Map<String, Object> dataCollectorMap = payloadMapper.mapDataPayload(emptyAsIngestedCoordinatesPaths, indexLocalSchema, storageRecordLocalData, RECORD_TEST_ID); + + assertTrue(dataCollectorMap.containsKey("SomeKeywordField")); + assertEquals(dataCollectorMap.get("SomeKeywordField"), "42.0"); + assertTrue(dataCollectorMap.containsKey("SomeTextField")); + assertEquals(dataCollectorMap.get("SomeTextField"), "43.0"); + assertTrue(dataCollectorMap.containsKey("SomeOtherTextField")); + assertEquals(dataCollectorMap.get("SomeOtherTextField"), "true"); + assertTrue(dataCollectorMap.containsKey("SomeKeywordArrayField")); + assertEquals(((String[]) dataCollectorMap.get("SomeKeywordArrayField"))[0], "44.0"); + assertTrue(dataCollectorMap.containsKey("SomeTextArrayField")); + assertEquals(((String[]) dataCollectorMap.get("SomeTextArrayField"))[0], "46.1"); + assertTrue(dataCollectorMap.containsKey("SomeIntegerArrayField")); + assertEquals(((Integer[]) dataCollectorMap.get("SomeIntegerArrayField"))[0].intValue(), 48); + assertTrue(dataCollectorMap.containsKey("SomeIntegerField")); + assertEquals(dataCollectorMap.get("SomeIntegerField"), 50); + assertTrue(dataCollectorMap.containsKey("SomeLongArrayField")); + assertEquals(((Long[]) dataCollectorMap.get("SomeLongArrayField"))[0].longValue(), 510000000000001L); + assertTrue(dataCollectorMap.containsKey("SomeLongField")); + assertEquals(dataCollectorMap.get("SomeLongField"), 530000000000001L); + assertTrue(dataCollectorMap.containsKey("SomeFloatArrayField")); + assertEquals(((Float[]) dataCollectorMap.get("SomeFloatArrayField"))[0].floatValue(), 54.11111, 0.0001); + assertTrue(dataCollectorMap.containsKey("SomeFloatField")); + assertEquals((Float) dataCollectorMap.get("SomeFloatField"), 56.11111, 0.0001); + assertTrue(dataCollectorMap.containsKey("SomeDoubleArrayField")); + assertEquals(((Double[]) dataCollectorMap.get("SomeDoubleArrayField"))[0].doubleValue(), 56.11111111111111D, 0.00000000001D); + assertTrue(dataCollectorMap.containsKey("SomeDoubleField")); + assertEquals((Double) dataCollectorMap.get("SomeDoubleField"), 58.11111111111111D, 0.00000000001D); + assertTrue(dataCollectorMap.containsKey("SomeBooleanArrayField")); + assertEquals(((Boolean[]) dataCollectorMap.get("SomeBooleanArrayField"))[0], true); + assertTrue(dataCollectorMap.containsKey("SomeBooleanField")); + assertEquals((Boolean) dataCollectorMap.get("SomeBooleanField"), false); + assertTrue(dataCollectorMap.containsKey("SomeDateArrayField")); + assertEquals(((String[]) dataCollectorMap.get("SomeDateArrayField"))[0], "2024-01-01T00:00:00+0000"); + assertTrue(dataCollectorMap.containsKey("SomeDateField")); + assertEquals(dataCollectorMap.get("SomeDateField"), "2024-01-03T00:00:00+0000"); + } + + @Test + public void mapDataPayloadTestAutoconversionsBooleanConversionOff() { + when(this.featureFlagChecker.isFeatureEnabled(MAP_BOOL2STRING_FEATURE_NAME)).thenReturn(false); + Map<String, Object> storageRecordLocalData = loadObject("/converter/index-autoconversions/wellStorageRecordData.json", storageRecordData.getClass()); + IndexSchema indexLocalSchema = loadObject("/converter/index-autoconversions/wellStorageSchema.json", IndexSchema.class); + Map<String, Object> dataCollectorMap = payloadMapper.mapDataPayload(emptyAsIngestedCoordinatesPaths, indexLocalSchema, storageRecordLocalData, RECORD_TEST_ID); + + assertTrue(dataCollectorMap.containsKey("SomeKeywordField")); + assertEquals(dataCollectorMap.get("SomeKeywordField"), 42.0); + assertTrue(dataCollectorMap.containsKey("SomeTextField")); + assertEquals(dataCollectorMap.get("SomeTextField"), 43.0); + assertTrue(dataCollectorMap.containsKey("SomeOtherTextField")); + assertEquals(dataCollectorMap.get("SomeOtherTextField"), true); + assertTrue(dataCollectorMap.containsKey("SomeKeywordArrayField")); + assertEquals(((List<Double>) dataCollectorMap.get("SomeKeywordArrayField")).get(0), 44.0, 0.0001); + assertTrue(dataCollectorMap.containsKey("SomeTextArrayField")); + assertEquals(((List<Double>) dataCollectorMap.get("SomeTextArrayField")).get(0), 46.1, 0.0001); + assertTrue(dataCollectorMap.containsKey("SomeIntegerArrayField")); + assertEquals(((Integer[]) dataCollectorMap.get("SomeIntegerArrayField"))[0].intValue(), 48); + assertTrue(dataCollectorMap.containsKey("SomeIntegerField")); + assertEquals(dataCollectorMap.get("SomeIntegerField"), 50); + assertTrue(dataCollectorMap.containsKey("SomeLongArrayField")); + assertEquals(((Long[]) dataCollectorMap.get("SomeLongArrayField"))[0].longValue(), 510000000000001L); + assertTrue(dataCollectorMap.containsKey("SomeLongField")); + assertEquals(dataCollectorMap.get("SomeLongField"), 530000000000001L); + assertTrue(dataCollectorMap.containsKey("SomeFloatArrayField")); + assertEquals(((Float[]) dataCollectorMap.get("SomeFloatArrayField"))[0].floatValue(), 54.11111, 0.0001); + assertTrue(dataCollectorMap.containsKey("SomeFloatField")); + assertEquals((Float) dataCollectorMap.get("SomeFloatField"), 56.11111, 0.0001); + assertTrue(dataCollectorMap.containsKey("SomeDoubleArrayField")); + assertEquals(((Double[]) dataCollectorMap.get("SomeDoubleArrayField"))[0].doubleValue(), 56.11111111111111D, 0.00000000001D); + assertTrue(dataCollectorMap.containsKey("SomeDoubleField")); + assertEquals((Double) dataCollectorMap.get("SomeDoubleField"), 58.11111111111111D, 0.00000000001D); + assertTrue(dataCollectorMap.containsKey("SomeBooleanArrayField")); + assertEquals(((Boolean[]) dataCollectorMap.get("SomeBooleanArrayField"))[0], true); + assertTrue(dataCollectorMap.containsKey("SomeBooleanField")); + assertEquals((Boolean) dataCollectorMap.get("SomeBooleanField"), false); + assertTrue(dataCollectorMap.containsKey("SomeDateArrayField")); + assertEquals(((String[]) dataCollectorMap.get("SomeDateArrayField"))[0], "2024-01-01T00:00:00+0000"); + assertTrue(dataCollectorMap.containsKey("SomeDateField")); + assertEquals(dataCollectorMap.get("SomeDateField"), "2024-01-03T00:00:00+0000"); + } + private <T> T loadObject(String file, Class<T> valueType) { String jsonString = readResourceFile(file); return this.gson.fromJson(jsonString, valueType); diff --git a/indexer-core/src/test/resources/converter/R3-json-schema/Generated/file/File.1.0.0.json.FF.res b/indexer-core/src/test/resources/converter/R3-json-schema/Generated/file/File.1.0.0.json.FF.res new file mode 100644 index 0000000000000000000000000000000000000000..c354504c354f1aa77a4768544abc68e2de1b6ea8 --- /dev/null +++ b/indexer-core/src/test/resources/converter/R3-json-schema/Generated/file/File.1.0.0.json.FF.res @@ -0,0 +1,49 @@ +{ + "kind": "osdu:osdu:Wellbore:1.0.0", + "schema": [ + { + "kind": "link", + "path": "SchemaFormatTypeID" + }, + { + "kind": "string", + "path": "PreloadFilePath" + }, + { + "kind": "string", + "path": "FileSource" + }, + { + "kind": "int", + "path": "FileSize" + }, + { + "kind": "link", + "path": "EncodingFormatTypeID" + }, + { + "kind": "string", + "path": "Endian" + }, + { + "kind": "boolean", + "path": "LossyCompressionIndicator" + }, + { + "kind": "link", + "path": "CompressionMethodTypeID" + }, + { + "kind": "double", + "path": "CompressionLevel" + }, + { + "kind": "string", + "path": "Checksum" + }, + { + "kind": "[]object", + "path": "VectorHeaderMapping" + } + ] +} diff --git a/indexer-core/src/test/resources/converter/R3-json-schema/Generated/type/Type.1.0.0.json.FF.res b/indexer-core/src/test/resources/converter/R3-json-schema/Generated/type/Type.1.0.0.json.FF.res new file mode 100644 index 0000000000000000000000000000000000000000..c2e7cffc45bde48605e32b5b27c5eeb9e2c51727 --- /dev/null +++ b/indexer-core/src/test/resources/converter/R3-json-schema/Generated/type/Type.1.0.0.json.FF.res @@ -0,0 +1,41 @@ +{ + "kind": "osdu:osdu:Wellbore:1.0.0", + "schema": [ + { + "kind": "string", + "path": "Description" + }, + { + "kind": "string", + "path": "Schema" + }, + { + "kind": "[]string", + "path": "NaturalKeys" + }, + { + "kind": "string", + "path": "SchemaID" + }, + { + "kind": "string", + "path": "Name" + }, + { + "kind": "string", + "path": "SchemaKind" + }, + { + "kind": "boolean", + "path": "IsReferenceValueType" + }, + { + "kind": "[]link", + "path": "GovernanceAuthorities" + }, + { + "kind": "string", + "path": "GovernanceModel" + } + ] +} diff --git a/indexer-core/src/test/resources/converter/R3-json-schema/Generated/type/Type.1.0.0.json.res b/indexer-core/src/test/resources/converter/R3-json-schema/Generated/type/Type.1.0.0.json.res index a7ba6fd16702e164a45bf8332358280b9756f682..c2817db53cd02719770721c8d82cf2ed2dea7e40 100644 --- a/indexer-core/src/test/resources/converter/R3-json-schema/Generated/type/Type.1.0.0.json.res +++ b/indexer-core/src/test/resources/converter/R3-json-schema/Generated/type/Type.1.0.0.json.res @@ -38,4 +38,4 @@ "path": "GovernanceModel" } ] -} \ No newline at end of file +} diff --git a/indexer-core/src/test/resources/converter/R3-json-schema/Generated/work-product/WorkProduct.1.0.0.json.FF.res b/indexer-core/src/test/resources/converter/R3-json-schema/Generated/work-product/WorkProduct.1.0.0.json.FF.res new file mode 100644 index 0000000000000000000000000000000000000000..ea7b4ff222a6c572f80bfac5600dde8f6b04204e --- /dev/null +++ b/indexer-core/src/test/resources/converter/R3-json-schema/Generated/work-product/WorkProduct.1.0.0.json.FF.res @@ -0,0 +1,53 @@ +{ + "kind": "osdu:osdu:Wellbore:1.0.0", + "schema": [ + { + "kind": "[]link", + "path": "Components" + }, + { + "kind": "boolean", + "path": "IsExtendedLoad" + }, + { + "kind": "boolean", + "path": "IsDiscoverable" + }, + { + "kind": "string", + "path": "Name" + }, + { + "kind": "string", + "path": "Description" + }, + { + "kind": "datetime", + "path": "CreationDateTime" + }, + { + "kind": "[]string", + "path": "Tags" + }, + { + "kind": "string", + "path": "SubmitterName" + }, + { + "kind": "[]string", + "path": "BusinessActivities" + }, + { + "kind": "[]string", + "path": "AuthorIDs" + }, + { + "kind": "[]string", + "path": "Annotations" + }, + { + "kind": "[]object", + "path": "LineageAssertions" + } + ] +} \ No newline at end of file diff --git a/indexer-core/src/test/resources/converter/R3-json-schema/Generated/work-product/WorkProduct.1.0.0.json.res b/indexer-core/src/test/resources/converter/R3-json-schema/Generated/work-product/WorkProduct.1.0.0.json.res index 5852817c878ac55fcc522b6752635389135b40ae..d341d7f8d713b5acda8909853b80649b17aab814 100644 --- a/indexer-core/src/test/resources/converter/R3-json-schema/Generated/work-product/WorkProduct.1.0.0.json.res +++ b/indexer-core/src/test/resources/converter/R3-json-schema/Generated/work-product/WorkProduct.1.0.0.json.res @@ -50,4 +50,4 @@ "path": "LineageAssertions" } ] -} \ No newline at end of file +} diff --git a/indexer-core/src/test/resources/converter/index-autoconversions/wellStorageRecordData.json b/indexer-core/src/test/resources/converter/index-autoconversions/wellStorageRecordData.json new file mode 100644 index 0000000000000000000000000000000000000000..869ff4a1e6ee310f4487518b69fc605a9f8c4c62 --- /dev/null +++ b/indexer-core/src/test/resources/converter/index-autoconversions/wellStorageRecordData.json @@ -0,0 +1,19 @@ +{ + "SomeKeywordField": 42, + "SomeTextField": 43, + "SomeOtherTextField": true, + "SomeKeywordArrayField": [44, 45], + "SomeTextArrayField": [46.1, 47.1], + "SomeIntegerArrayField": ["48", "49"], + "SomeIntegerField": "50", + "SomeLongArrayField": ["510000000000001", "520000000000001"], + "SomeLongField": "530000000000001", + "SomeFloatArrayField": ["54.11111", "55.11111"], + "SomeFloatField": "56.11111", + "SomeDoubleArrayField": ["56.11111111111111", "57.11111111111111"], + "SomeDoubleField": "58.11111111111111", + "SomeBooleanArrayField": ["true", "false"], + "SomeBooleanField": "false", + "SomeDateArrayField": ["2024-01-01", "2024-01-02"], + "SomeDateField": "2024-01-03" +} \ No newline at end of file diff --git a/indexer-core/src/test/resources/converter/index-autoconversions/wellStorageSchema.json b/indexer-core/src/test/resources/converter/index-autoconversions/wellStorageSchema.json new file mode 100644 index 0000000000000000000000000000000000000000..77afcec8db62ae06c59ca1625d3946f0e30e583f --- /dev/null +++ b/indexer-core/src/test/resources/converter/index-autoconversions/wellStorageSchema.json @@ -0,0 +1,32 @@ +{ + "kind": "osdu:wks:master-data--Well:1.2.0", + "type": "master-data--Well", + "dataSchema": { + "SomeKeywordField": "keyword", + "SomeTextField": "text", + "SomeOtherTextField": "text", + "SomeKeywordArrayField": "keyword_array", + "SomeTextArrayField": "text_array", + "SomeIntegerArrayField": "integer_array", + "SomeIntegerField": "integer", + "SomeLongArrayField": "long_array", + "SomeLongField": "long", + "SomeFloatArrayField": "float_array", + "SomeFloatField": "float", + "SomeDoubleArrayField": "double_array", + "SomeDoubleField": "double", + "SomeBooleanArrayField": "boolean_array", + "SomeBooleanField": "boolean", + "SomeDateArrayField": "date_array", + "SomeDateField": "date", + "SomeGeoPointField": "geo_point", + "SomeGeoShapeField": "geo_shape", + "SomeObjectField": { + "type": "nested", + "properties": { + "SomeNestedTextField": "text" + } + }, + "SomeFlattenedField": "flattened" + } +} \ No newline at end of file diff --git a/indexer-core/src/test/resources/converter/index-virtual-properties/unmatched-virtual-properties-schema.json.FF.res b/indexer-core/src/test/resources/converter/index-virtual-properties/unmatched-virtual-properties-schema.json.FF.res new file mode 100644 index 0000000000000000000000000000000000000000..91a70ee6e640d1211c6cc268f23134b9cba6936c --- /dev/null +++ b/indexer-core/src/test/resources/converter/index-virtual-properties/unmatched-virtual-properties-schema.json.FF.res @@ -0,0 +1,340 @@ +{ + "kind" : "osdu:wks:master-data--Wellbore:1.0.0", + "schema" : [ { + "path" : "ResourceHomeRegionID", + "kind" : "string" + }, { + "path" : "ResourceHostRegionIDs", + "kind" : "[]string" + }, { + "path" : "ResourceLifecycleStatus", + "kind" : "string" + }, { + "path" : "ResourceSecurityClassification", + "kind" : "string" + }, { + "path" : "ResourceCurationStatus", + "kind" : "string" + }, { + "path" : "ExistenceKind", + "kind" : "string" + }, { + "path" : "TechnicalAssuranceID", + "kind" : "string" + }, { + "path" : "Source", + "kind" : "string" + }, { + "path" : "NameAliases", + "kind" : "nested", + "properties" : [ { + "path" : "AliasNameTypeID", + "kind" : "string" + }, { + "path" : "EffectiveDateTime", + "kind" : "datetime" + }, { + "path" : "AliasName", + "kind" : "string" + }, { + "path" : "TerminationDateTime", + "kind" : "datetime" + }, { + "path" : "DefinitionOrganisationID", + "kind" : "string" + } ] + }, { + "path" : "SpatialLocation.SpatialParameterTypeID", + "kind" : "string" + }, { + "path" : "SpatialLocation.QuantitativeAccuracyBandID", + "kind" : "string" + }, { + "path" : "SpatialLocation.CoordinateQualityCheckRemarks", + "kind" : "[]string" + }, { + "path" : "SpatialLocation.AppliedOperations", + "kind" : "[]string" + }, { + "path" : "SpatialLocation.QualitativeSpatialAccuracyTypeID", + "kind" : "string" + }, { + "path" : "SpatialLocation.CoordinateQualityCheckPerformedBy", + "kind" : "string" + }, { + "path" : "SpatialLocation.SpatialLocationCoordinatesDate", + "kind" : "datetime" + }, { + "path" : "SpatialLocation.CoordinateQualityCheckDateTime", + "kind" : "datetime" + }, { + "path" : "SpatialLocation.Wgs84Coordinates", + "kind" : "core:dl:geoshape:1.0.0" + }, { + "path" : "SpatialLocation.SpatialGeometryTypeID", + "kind" : "string" + }, { + "path" : "VersionCreationReason", + "kind" : "string" + }, { + "path" : "GeoContexts", + "kind" : "nested", + "properties" : [ { + "path" : "GeoPoliticalEntityID", + "kind" : "string" + }, { + "path" : "GeoTypeID", + "kind" : "string" + }, { + "path" : "BasinID", + "kind" : "string" + }, { + "path" : "GeoTypeID", + "kind" : "string" + }, { + "path" : "FieldID", + "kind" : "string" + }, { + "path" : "PlayID", + "kind" : "string" + }, { + "path" : "GeoTypeID", + "kind" : "string" + }, { + "path" : "ProspectID", + "kind" : "string" + }, { + "path" : "GeoTypeID", + "kind" : "string" + } ] + }, { + "path" : "FacilityStates", + "kind" : "nested", + "properties" : [ { + "path" : "EffectiveDateTime", + "kind" : "datetime" + }, { + "path" : "FacilityStateTypeID", + "kind" : "string" + }, { + "path" : "TerminationDateTime", + "kind" : "datetime" + } ] + }, { + "path" : "FacilityID", + "kind" : "string" + }, { + "path" : "OperatingEnvironmentID", + "kind" : "string" + }, { + "path" : "FacilityNameAliases", + "kind" : "[]object" + }, { + "path" : "FacilityEvents", + "kind" : "nested", + "properties" : [ { + "path" : "EffectiveDateTime", + "kind" : "datetime" + }, { + "path" : "TerminationDateTime", + "kind" : "datetime" + }, { + "path" : "FacilityEventTypeID", + "kind" : "string" + } ] + }, { + "path" : "FacilitySpecifications", + "kind" : "flattened" + }, { + "path" : "DataSourceOrganisationID", + "kind" : "string" + }, { + "path" : "InitialOperatorID", + "kind" : "string" + }, { + "path" : "CurrentOperatorID", + "kind" : "string" + }, { + "path" : "FacilityOperators", + "kind" : "nested", + "properties" : [ { + "path" : "FacilityOperatorID", + "kind" : "string" + }, { + "path" : "EffectiveDateTime", + "kind" : "datetime" + }, { + "path" : "FacilityOperatorOrganisationID", + "kind" : "string" + }, { + "path" : "TerminationDateTime", + "kind" : "datetime" + } ] + }, { + "path" : "FacilityName", + "kind" : "string" + }, { + "path" : "FacilityTypeID", + "kind" : "string" + }, { + "path" : "GeographicBottomHoleLocation.SpatialParameterTypeID", + "kind" : "string" + }, { + "path" : "GeographicBottomHoleLocation.QuantitativeAccuracyBandID", + "kind" : "string" + }, { + "path" : "GeographicBottomHoleLocation.CoordinateQualityCheckRemarks", + "kind" : "[]string" + }, { + "path" : "GeographicBottomHoleLocation.AppliedOperations", + "kind" : "[]string" + }, { + "path" : "GeographicBottomHoleLocation.QualitativeSpatialAccuracyTypeID", + "kind" : "string" + }, { + "path" : "GeographicBottomHoleLocation.CoordinateQualityCheckPerformedBy", + "kind" : "string" + }, { + "path" : "GeographicBottomHoleLocation.SpatialLocationCoordinatesDate", + "kind" : "datetime" + }, { + "path" : "GeographicBottomHoleLocation.CoordinateQualityCheckDateTime", + "kind" : "datetime" + }, { + "path" : "GeographicBottomHoleLocation.Wgs84Coordinates", + "kind" : "core:dl:geoshape:1.0.0" + }, { + "path" : "GeographicBottomHoleLocation.SpatialGeometryTypeID", + "kind" : "string" + }, { + "path" : "DrillingReasons", + "kind" : "[]object" + }, { + "path" : "VerticalMeasurements", + "kind" : "nested", + "properties" : [ { + "path" : "VerticalMeasurementID", + "kind" : "string" + }, { + "path" : "WellboreTVDTrajectoryID", + "kind" : "string" + }, { + "path" : "VerticalCRSID", + "kind" : "string" + }, { + "path" : "VerticalMeasurementSourceID", + "kind" : "string" + }, { + "path" : "VerticalReferenceID", + "kind" : "string" + }, { + "path" : "TerminationDateTime", + "kind" : "datetime" + }, { + "path" : "VerticalMeasurementPathID", + "kind" : "string" + }, { + "path" : "EffectiveDateTime", + "kind" : "datetime" + }, { + "path" : "VerticalMeasurement", + "kind" : "double" + }, { + "path" : "VerticalMeasurementTypeID", + "kind" : "string" + }, { + "path" : "VerticalMeasurementDescription", + "kind" : "string" + }, { + "path" : "VerticalMeasurementUnitOfMeasureID", + "kind" : "string" + } ] + }, { + "path" : "PrimaryMaterialID", + "kind" : "string" + }, { + "path" : "SequenceNumber", + "kind" : "int" + }, { + "path" : "TargetFormation", + "kind" : "string" + }, { + "path" : "KickOffWellbore", + "kind" : "string" + }, { + "path" : "DefaultVerticalMeasurementID", + "kind" : "string" + }, { + "path" : "ProjectedBottomHoleLocation.SpatialParameterTypeID", + "kind" : "string" + }, { + "path" : "ProjectedBottomHoleLocation.QuantitativeAccuracyBandID", + "kind" : "string" + }, { + "path" : "ProjectedBottomHoleLocation.CoordinateQualityCheckRemarks", + "kind" : "[]string" + }, { + "path" : "ProjectedBottomHoleLocation.AppliedOperations", + "kind" : "[]string" + }, { + "path" : "ProjectedBottomHoleLocation.QualitativeSpatialAccuracyTypeID", + "kind" : "string" + }, { + "path" : "ProjectedBottomHoleLocation.CoordinateQualityCheckPerformedBy", + "kind" : "string" + }, { + "path" : "ProjectedBottomHoleLocation.SpatialLocationCoordinatesDate", + "kind" : "datetime" + }, { + "path" : "ProjectedBottomHoleLocation.CoordinateQualityCheckDateTime", + "kind" : "datetime" + }, { + "path" : "ProjectedBottomHoleLocation.Wgs84Coordinates", + "kind" : "core:dl:geoshape:1.0.0" + }, { + "path" : "ProjectedBottomHoleLocation.SpatialGeometryTypeID", + "kind" : "string" + }, { + "path" : "WellID", + "kind" : "string" + }, { + "path" : "TrajectoryTypeID", + "kind" : "string" + }, { + "path" : "DefinitiveTrajectoryID", + "kind" : "string" + }, { + "path" : "VirtualProperties.DefaultLocation.SpatialParameterTypeID", + "kind" : "string" + }, { + "path" : "VirtualProperties.DefaultLocation.QuantitativeAccuracyBandID", + "kind" : "string" + }, { + "path" : "VirtualProperties.DefaultLocation.CoordinateQualityCheckRemarks", + "kind" : "[]string" + }, { + "path" : "VirtualProperties.DefaultLocation.AppliedOperations", + "kind" : "[]string" + }, { + "path" : "VirtualProperties.DefaultLocation.QualitativeSpatialAccuracyTypeID", + "kind" : "string" + }, { + "path" : "VirtualProperties.DefaultLocation.CoordinateQualityCheckPerformedBy", + "kind" : "string" + }, { + "path" : "VirtualProperties.DefaultLocation.SpatialLocationCoordinatesDate", + "kind" : "datetime" + }, { + "path" : "VirtualProperties.DefaultLocation.CoordinateQualityCheckDateTime", + "kind" : "datetime" + }, { + "path" : "VirtualProperties.DefaultLocation.Wgs84Coordinates", + "kind" : "core:dl:geoshape:1.0.0" + }, { + "path" : "VirtualProperties.DefaultLocation.SpatialGeometryTypeID", + "kind" : "string" + }, { + "path" : "VirtualProperties.DefaultLocation.IsDecimated", + "kind" : "boolean" + }] +} diff --git a/indexer-core/src/test/resources/converter/index-virtual-properties/virtual-properties-schema.json.FF.res b/indexer-core/src/test/resources/converter/index-virtual-properties/virtual-properties-schema.json.FF.res new file mode 100644 index 0000000000000000000000000000000000000000..59998ea9e0f535f39dcb51854c8b4f4f01691276 --- /dev/null +++ b/indexer-core/src/test/resources/converter/index-virtual-properties/virtual-properties-schema.json.FF.res @@ -0,0 +1,343 @@ +{ + "kind" : "osdu:wks:master-data--Wellbore:1.0.0", + "schema" : [ { + "path" : "ResourceHomeRegionID", + "kind" : "string" + }, { + "path" : "ResourceHostRegionIDs", + "kind" : "[]string" + }, { + "path" : "ResourceLifecycleStatus", + "kind" : "string" + }, { + "path" : "ResourceSecurityClassification", + "kind" : "string" + }, { + "path" : "ResourceCurationStatus", + "kind" : "string" + }, { + "path" : "ExistenceKind", + "kind" : "string" + }, { + "path" : "TechnicalAssuranceID", + "kind" : "string" + }, { + "path" : "Source", + "kind" : "string" + }, { + "path" : "NameAliases", + "kind" : "nested", + "properties" : [ { + "path" : "AliasNameTypeID", + "kind" : "string" + }, { + "path" : "EffectiveDateTime", + "kind" : "datetime" + }, { + "path" : "AliasName", + "kind" : "string" + }, { + "path" : "TerminationDateTime", + "kind" : "datetime" + }, { + "path" : "DefinitionOrganisationID", + "kind" : "string" + } ] + }, { + "path" : "SpatialLocation.SpatialParameterTypeID", + "kind" : "string" + }, { + "path" : "SpatialLocation.QuantitativeAccuracyBandID", + "kind" : "string" + }, { + "path" : "SpatialLocation.CoordinateQualityCheckRemarks", + "kind" : "[]string" + }, { + "path" : "SpatialLocation.AppliedOperations", + "kind" : "[]string" + }, { + "path" : "SpatialLocation.QualitativeSpatialAccuracyTypeID", + "kind" : "string" + }, { + "path" : "SpatialLocation.CoordinateQualityCheckPerformedBy", + "kind" : "string" + }, { + "path" : "SpatialLocation.SpatialLocationCoordinatesDate", + "kind" : "datetime" + }, { + "path" : "SpatialLocation.CoordinateQualityCheckDateTime", + "kind" : "datetime" + }, { + "path" : "SpatialLocation.Wgs84Coordinates", + "kind" : "core:dl:geoshape:1.0.0" + }, { + "path" : "SpatialLocation.SpatialGeometryTypeID", + "kind" : "string" + }, { + "path" : "VersionCreationReason", + "kind" : "string" + }, { + "path" : "GeoContexts", + "kind" : "nested", + "properties" : [ { + "path" : "GeoPoliticalEntityID", + "kind" : "string" + }, { + "path" : "GeoTypeID", + "kind" : "string" + }, { + "path" : "BasinID", + "kind" : "string" + }, { + "path" : "GeoTypeID", + "kind" : "string" + }, { + "path" : "FieldID", + "kind" : "string" + }, { + "path" : "PlayID", + "kind" : "string" + }, { + "path" : "GeoTypeID", + "kind" : "string" + }, { + "path" : "ProspectID", + "kind" : "string" + }, { + "path" : "GeoTypeID", + "kind" : "string" + } ] + }, { + "path" : "FacilityStates", + "kind" : "nested", + "properties" : [ { + "path" : "EffectiveDateTime", + "kind" : "datetime" + }, { + "path" : "FacilityStateTypeID", + "kind" : "string" + }, { + "path" : "TerminationDateTime", + "kind" : "datetime" + } ] + }, { + "path" : "FacilityID", + "kind" : "string" + }, { + "path" : "OperatingEnvironmentID", + "kind" : "string" + }, { + "path" : "FacilityNameAliases", + "kind" : "[]object" + }, { + "path" : "FacilityEvents", + "kind" : "nested", + "properties" : [ { + "path" : "EffectiveDateTime", + "kind" : "datetime" + }, { + "path" : "TerminationDateTime", + "kind" : "datetime" + }, { + "path" : "FacilityEventTypeID", + "kind" : "string" + } ] + }, { + "path" : "FacilitySpecifications", + "kind" : "flattened" + }, { + "path" : "DataSourceOrganisationID", + "kind" : "string" + }, { + "path" : "InitialOperatorID", + "kind" : "string" + }, { + "path" : "CurrentOperatorID", + "kind" : "string" + }, { + "path" : "FacilityOperators", + "kind" : "nested", + "properties" : [ { + "path" : "FacilityOperatorID", + "kind" : "string" + }, { + "path" : "EffectiveDateTime", + "kind" : "datetime" + }, { + "path" : "FacilityOperatorOrganisationID", + "kind" : "string" + }, { + "path" : "TerminationDateTime", + "kind" : "datetime" + } ] + }, { + "path" : "FacilityName", + "kind" : "string" + }, { + "path" : "FacilityTypeID", + "kind" : "string" + }, { + "path" : "GeographicBottomHoleLocation.SpatialParameterTypeID", + "kind" : "string" + }, { + "path" : "GeographicBottomHoleLocation.QuantitativeAccuracyBandID", + "kind" : "string" + }, { + "path" : "GeographicBottomHoleLocation.CoordinateQualityCheckRemarks", + "kind" : "[]string" + }, { + "path" : "GeographicBottomHoleLocation.AppliedOperations", + "kind" : "[]string" + }, { + "path" : "GeographicBottomHoleLocation.QualitativeSpatialAccuracyTypeID", + "kind" : "string" + }, { + "path" : "GeographicBottomHoleLocation.CoordinateQualityCheckPerformedBy", + "kind" : "string" + }, { + "path" : "GeographicBottomHoleLocation.SpatialLocationCoordinatesDate", + "kind" : "datetime" + }, { + "path" : "GeographicBottomHoleLocation.CoordinateQualityCheckDateTime", + "kind" : "datetime" + }, { + "path" : "GeographicBottomHoleLocation.Wgs84Coordinates", + "kind" : "core:dl:geoshape:1.0.0" + }, { + "path" : "GeographicBottomHoleLocation.SpatialGeometryTypeID", + "kind" : "string" + }, { + "path" : "DrillingReasons", + "kind" : "[]object" + }, { + "path" : "VerticalMeasurements", + "kind" : "nested", + "properties" : [ { + "path" : "VerticalMeasurementID", + "kind" : "string" + }, { + "path" : "WellboreTVDTrajectoryID", + "kind" : "string" + }, { + "path" : "VerticalCRSID", + "kind" : "string" + }, { + "path" : "VerticalMeasurementSourceID", + "kind" : "string" + }, { + "path" : "VerticalReferenceID", + "kind" : "string" + }, { + "path" : "TerminationDateTime", + "kind" : "datetime" + }, { + "path" : "VerticalMeasurementPathID", + "kind" : "string" + }, { + "path" : "EffectiveDateTime", + "kind" : "datetime" + }, { + "path" : "VerticalMeasurement", + "kind" : "double" + }, { + "path" : "VerticalMeasurementTypeID", + "kind" : "string" + }, { + "path" : "VerticalMeasurementDescription", + "kind" : "string" + }, { + "path" : "VerticalMeasurementUnitOfMeasureID", + "kind" : "string" + } ] + }, { + "path" : "PrimaryMaterialID", + "kind" : "string" + }, { + "path" : "SequenceNumber", + "kind" : "int" + }, { + "path" : "TargetFormation", + "kind" : "string" + }, { + "path" : "KickOffWellbore", + "kind" : "string" + }, { + "path" : "DefaultVerticalMeasurementID", + "kind" : "string" + }, { + "path" : "ProjectedBottomHoleLocation.SpatialParameterTypeID", + "kind" : "string" + }, { + "path" : "ProjectedBottomHoleLocation.QuantitativeAccuracyBandID", + "kind" : "string" + }, { + "path" : "ProjectedBottomHoleLocation.CoordinateQualityCheckRemarks", + "kind" : "[]string" + }, { + "path" : "ProjectedBottomHoleLocation.AppliedOperations", + "kind" : "[]string" + }, { + "path" : "ProjectedBottomHoleLocation.QualitativeSpatialAccuracyTypeID", + "kind" : "string" + }, { + "path" : "ProjectedBottomHoleLocation.CoordinateQualityCheckPerformedBy", + "kind" : "string" + }, { + "path" : "ProjectedBottomHoleLocation.SpatialLocationCoordinatesDate", + "kind" : "datetime" + }, { + "path" : "ProjectedBottomHoleLocation.CoordinateQualityCheckDateTime", + "kind" : "datetime" + }, { + "path" : "ProjectedBottomHoleLocation.Wgs84Coordinates", + "kind" : "core:dl:geoshape:1.0.0" + }, { + "path" : "ProjectedBottomHoleLocation.SpatialGeometryTypeID", + "kind" : "string" + }, { + "path" : "WellID", + "kind" : "string" + }, { + "path" : "TrajectoryTypeID", + "kind" : "string" + }, { + "path" : "DefinitiveTrajectoryID", + "kind" : "string" + }, { + "path" : "VirtualProperties.DefaultLocation.SpatialParameterTypeID", + "kind" : "string" + }, { + "path" : "VirtualProperties.DefaultLocation.QuantitativeAccuracyBandID", + "kind" : "string" + }, { + "path" : "VirtualProperties.DefaultLocation.CoordinateQualityCheckRemarks", + "kind" : "[]string" + }, { + "path" : "VirtualProperties.DefaultLocation.AppliedOperations", + "kind" : "[]string" + }, { + "path" : "VirtualProperties.DefaultLocation.QualitativeSpatialAccuracyTypeID", + "kind" : "string" + }, { + "path" : "VirtualProperties.DefaultLocation.CoordinateQualityCheckPerformedBy", + "kind" : "string" + }, { + "path" : "VirtualProperties.DefaultLocation.SpatialLocationCoordinatesDate", + "kind" : "datetime" + }, { + "path" : "VirtualProperties.DefaultLocation.CoordinateQualityCheckDateTime", + "kind" : "datetime" + }, { + "path" : "VirtualProperties.DefaultLocation.Wgs84Coordinates", + "kind" : "core:dl:geoshape:1.0.0" + }, { + "path" : "VirtualProperties.DefaultLocation.SpatialGeometryTypeID", + "kind" : "string" + }, { + "path" : "VirtualProperties.DefaultName", + "kind" : "string" + }, { + "path" : "VirtualProperties.DefaultLocation.IsDecimated", + "kind" : "boolean" + }] +} diff --git a/indexer-core/src/test/resources/converter/wks/slb_wke_wellbore.json.FF.res b/indexer-core/src/test/resources/converter/wks/slb_wke_wellbore.json.FF.res new file mode 100644 index 0000000000000000000000000000000000000000..37d499f5d0a6a1aed03a480bdaea813ffff66726 --- /dev/null +++ b/indexer-core/src/test/resources/converter/wks/slb_wke_wellbore.json.FF.res @@ -0,0 +1,357 @@ +{ + "kind": "slb:wks:wellbore:1.0.6", + "schema": [ + { + "kind": "string", + "path": "airGap.unitKey" + }, + { + "kind": "double", + "path": "airGap.value" + }, + { + "kind": "string", + "path": "block" + }, + { + "kind": "string", + "path": "country" + }, + { + "kind": "string", + "path": "county" + }, + { + "kind": "datetime", + "path": "dateCreated" + }, + { + "kind": "datetime", + "path": "dateModified" + }, + { + "kind": "string", + "path": "drillingDaysTarget.unitKey" + }, + { + "kind": "double", + "path": "drillingDaysTarget.value" + }, + { + "kind": "string", + "path": "elevationReference.elevationFromMsl.unitKey" + }, + { + "kind": "double", + "path": "elevationReference.elevationFromMsl.value" + }, + { + "kind": "string", + "path": "elevationReference.name" + }, + { + "kind": "[]link", + "path": "externalIds" + }, + { + "kind": "string", + "path": "field" + }, + { + "kind": "string", + "path": "formationAtTd" + }, + { + "kind": "string", + "path": "formationProjected" + }, + { + "kind": "boolean", + "path": "hasAchievedTotalDepth" + }, + { + "kind": "boolean", + "path": "isActive" + }, + { + "kind": "string", + "path": "kickOffMd.unitKey" + }, + { + "kind": "double", + "path": "kickOffMd.value" + }, + { + "kind": "string", + "path": "kickOffTvd.unitKey" + }, + { + "kind": "double", + "path": "kickOffTvd.value" + }, + { + "kind": "core:dl:geoshape:1.0.0", + "path": "locationWGS84" + }, + { + "kind": "string", + "path": "name" + }, + { + "kind": "string", + "path": "operator" + }, + { + "kind": "datetime", + "path": "permitDate" + }, + { + "kind": "string", + "path": "permitNumber" + }, + { + "kind": "string", + "path": "plssLocation.aliquotPart" + }, + { + "kind": "string", + "path": "plssLocation.range" + }, + { + "kind": "int", + "path": "plssLocation.section" + }, + { + "kind": "string", + "path": "plssLocation.township" + }, + { + "kind": "string", + "path": "propertyDictionary" + }, + { + "kind": "double", + "path": "relationships.definitiveTimeDepthRelation.confidence" + }, + { + "kind": "link", + "path": "relationships.definitiveTimeDepthRelation.id" + }, + { + "kind": "string", + "path": "relationships.definitiveTimeDepthRelation.name" + }, + { + "kind": "long", + "path": "relationships.definitiveTimeDepthRelation.version" + }, + { + "kind": "double", + "path": "relationships.definitiveTrajectory.confidence" + }, + { + "kind": "link", + "path": "relationships.definitiveTrajectory.id" + }, + { + "kind": "string", + "path": "relationships.definitiveTrajectory.name" + }, + { + "kind": "long", + "path": "relationships.definitiveTrajectory.version" + }, + { + "kind": "double", + "path": "relationships.tieInWellbore.confidence" + }, + { + "kind": "link", + "path": "relationships.tieInWellbore.id" + }, + { + "kind": "string", + "path": "relationships.tieInWellbore.name" + }, + { + "kind": "long", + "path": "relationships.tieInWellbore.version" + }, + { + "kind": "double", + "path": "relationships.well.confidence" + }, + { + "kind": "link", + "path": "relationships.well.id" + }, + { + "kind": "string", + "path": "relationships.well.name" + }, + { + "kind": "long", + "path": "relationships.well.version" + }, + { + "kind": "string", + "path": "shape" + }, + { + "kind": "datetime", + "path": "spudDate" + }, + { + "kind": "string", + "path": "state" + }, + { + "kind": "string", + "path": "totalDepthMd.unitKey" + }, + { + "kind": "double", + "path": "totalDepthMd.value" + }, + { + "kind": "string", + "path": "totalDepthMdDriller.unitKey" + }, + { + "kind": "double", + "path": "totalDepthMdDriller.value" + }, + { + "kind": "string", + "path": "totalDepthMdPlanned.unitKey" + }, + { + "kind": "double", + "path": "totalDepthMdPlanned.value" + }, + { + "kind": "string", + "path": "totalDepthMdSubSeaPlanned.unitKey" + }, + { + "kind": "double", + "path": "totalDepthMdSubSeaPlanned.value" + }, + { + "kind": "string", + "path": "totalDepthProjectedMd.unitKey" + }, + { + "kind": "double", + "path": "totalDepthProjectedMd.value" + }, + { + "kind": "string", + "path": "totalDepthTvd.unitKey" + }, + { + "kind": "double", + "path": "totalDepthTvd.value" + }, + { + "kind": "string", + "path": "totalDepthTvdDriller.unitKey" + }, + { + "kind": "double", + "path": "totalDepthTvdDriller.value" + }, + { + "kind": "string", + "path": "totalDepthTvdPlanned.unitKey" + }, + { + "kind": "double", + "path": "totalDepthTvdPlanned.value" + }, + { + "kind": "string", + "path": "totalDepthTvdSubSeaPlanned.unitKey" + }, + { + "kind": "double", + "path": "totalDepthTvdSubSeaPlanned.value" + }, + { + "kind": "string", + "path": "uwi" + }, + { + "kind": "string", + "path": "wellHeadElevation.unitKey" + }, + { + "kind": "double", + "path": "wellHeadElevation.value" + }, + { + "kind": "string", + "path": "wellHeadGeographic.crsKey" + }, + { + "kind": "string", + "path": "wellHeadGeographic.elevationFromMsl.unitKey" + }, + { + "kind": "double", + "path": "wellHeadGeographic.elevationFromMsl.value" + }, + { + "kind": "double", + "path": "wellHeadGeographic.latitude" + }, + { + "kind": "double", + "path": "wellHeadGeographic.longitude" + }, + { + "kind": "string", + "path": "wellHeadProjected.crsKey" + }, + { + "kind": "string", + "path": "wellHeadProjected.elevationFromMsl.unitKey" + }, + { + "kind": "double", + "path": "wellHeadProjected.elevationFromMsl.value" + }, + { + "kind": "double", + "path": "wellHeadProjected.x" + }, + { + "kind": "double", + "path": "wellHeadProjected.y" + }, + { + "kind": "core:dl:geopoint:1.0.0", + "path": "wellHeadWgs84" + }, + { + "kind": "string", + "path": "wellboreNumberGovernment" + }, + { + "kind": "string", + "path": "wellboreNumberOperator" + }, + { + "kind": "string", + "path": "wellborePurpose" + }, + { + "kind": "string", + "path": "wellboreStatus" + }, + { + "kind": "string", + "path": "wellboreType" + } + ] +} \ No newline at end of file diff --git a/indexer-core/src/test/resources/converter/wks/slb_wke_wellbore.json.res b/indexer-core/src/test/resources/converter/wks/slb_wke_wellbore.json.res index 4b4b9b61d8bdd5cb6ab5575ea72e4bc611ebfbac..8c478801352f5ee9e2e0ef417e3adb1c99ca9f2b 100644 --- a/indexer-core/src/test/resources/converter/wks/slb_wke_wellbore.json.res +++ b/indexer-core/src/test/resources/converter/wks/slb_wke_wellbore.json.res @@ -354,4 +354,4 @@ "path": "wellboreType" } ] -} \ No newline at end of file +} diff --git a/provider/indexer-aws/src/main/resources/application.properties b/provider/indexer-aws/src/main/resources/application.properties index d6180a3d81ad42b702d5136f20baee7f5e0b1df1..10ea1892a39396462aad7adca934312f42dbb9d1 100644 --- a/provider/indexer-aws/src/main/resources/application.properties +++ b/provider/indexer-aws/src/main/resources/application.properties @@ -85,6 +85,7 @@ spring.autoconfigure.exclude=org.springframework.boot.autoconfigure.security.Sec # Feature flag settings featureFlag.strategy=appProperty +featureFlag.mapBooleanToString.enabled=false featureFlag.asIngestedCoordinates.enabled=true featureFlag.keywordLower.enabled=true featureFlag.bagOfWords.enabled=true diff --git a/provider/indexer-azure/src/main/resources/application.properties b/provider/indexer-azure/src/main/resources/application.properties index 08b7c659d222276589ff08c0fff05960a32363b9..e0cefb3db36ed20622a260d91a3250c356b91222 100644 --- a/provider/indexer-azure/src/main/resources/application.properties +++ b/provider/indexer-azure/src/main/resources/application.properties @@ -105,6 +105,7 @@ redis.database=${REDIS_DATABASE} # Feature flag settings featureFlag.strategy=${featureFlag_appProperty:appProperty} +featureFlag.mapBooleanToString.enabled=${featureFlag_mapBooleanToString_enabled:false} featureFlag.asIngestedCoordinates.enabled=${featureFlag_asIngestedCoordinates_enabled:true} featureFlag.keywordLower.enabled=${featureFlag_keywordLower_enabled:true} featureFlag.bagOfWords.enabled=${featureFlag_bagOfWords_enabled:true} @@ -117,4 +118,4 @@ api.server.fullUrl.enabled=${swaggerFullUrlEnabled:true} azure.storage.client.retry.MAX_ATTEMPTS=3 azure.storage.client.retry.INITIAL_DELAY=1000 -elasticsearch.client.cache.size=100 \ No newline at end of file +elasticsearch.client.cache.size=100 diff --git a/provider/indexer-gc/src/main/resources/application.properties b/provider/indexer-gc/src/main/resources/application.properties index f25237671e243c50172c33a8378db9529866b1c6..b4ad62851b2b4433138fdf8269072a10ad86c4f8 100644 --- a/provider/indexer-gc/src/main/resources/application.properties +++ b/provider/indexer-gc/src/main/resources/application.properties @@ -66,6 +66,7 @@ reindex-topic-name=reindex # Feature flag settings featureFlag.strategy=dataPartition featureFlag.xCollaboration.enabled=false +featureFlag.mapBooleanToString.enabled=false featureFlag.asIngestedCoordinates.enabled=true featureFlag.keywordLower.enabled=true featureFlag.bagOfWords.enabled=true diff --git a/provider/indexer-ibm/src/main/resources/application.properties b/provider/indexer-ibm/src/main/resources/application.properties index d0e2d06b30d01a9c6c38093429d23c99eb14aa18..6c205933b175f6b32733b71f4602f2736f6138b4 100644 --- a/provider/indexer-ibm/src/main/resources/application.properties +++ b/provider/indexer-ibm/src/main/resources/application.properties @@ -71,6 +71,7 @@ ELASTIC_USER_PASSWORD=REPLACE_ME:REPLACE_ME # Feature flag settings featureFlag.strategy=appProperty +featureFlag.mapBooleanToString.enabled=false featureFlag.asIngestedCoordinates.enabled=false featureFlag.keywordLower.enabled=false featureFlag.bagOfWords.enabled=false