diff --git a/docs/tutorial/PreviewFeatures.md b/docs/tutorial/PreviewFeatures.md index 16981af3b7af845d163f9288ff738503423b070d..84cb60a0ee68ecd6a49a90899c3049e29277a217 100644 --- a/docs/tutorial/PreviewFeatures.md +++ b/docs/tutorial/PreviewFeatures.md @@ -1,29 +1,3 @@ -## Geoshape Decimation - -In order to improve indexing and search performance for documents with large geometry, the geo-shape of the following -GeoJSON types in the original shape attribute and virtual shape attribute if exists are decimated -by implementing Ramer–Douglas–Peucker algorithm: -- LineString -- MultiLineString -- Polygon -- MultiPolygon - - -The feature is enabled by default for all data partitions. If client does not want the geo-shape to be decimated in their -data partitions, they can disable geo-shape decimation through the Partition Service. -Here is an example to disable this feature by setting the property "indexer-decimation-enabled" in a given data partition: -``` -{ - "indexer-decimation-enabled": { - "sensitive": false, - "value": "false" - } -} -``` - -If the property "indexer-decimation-enabled" is not created or the property value is set to "true" (String type) in the -given data partition, the geo-shape decimation will be enabled. - ## Index extension OSDU Standard index extensions are defined by OSDU Data Definition work-streams with the intent to provide diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java index d949e581fa000842f1d48ce91cec1b08164cecbb..bba17ba3e2748270af062067688ef4378880e433 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java @@ -33,7 +33,6 @@ import org.opengroup.osdu.indexer.schema.converter.tags.VirtualProperty; import org.opengroup.osdu.indexer.util.PropertyUtil; import org.opengroup.osdu.indexer.util.geo.decimator.DecimatedResult; import org.opengroup.osdu.indexer.util.geo.decimator.GeoShapeDecimator; -import org.opengroup.osdu.indexer.util.geo.decimator.GeoShapeDecimationSetting; import org.springframework.stereotype.Component; import javax.inject.Inject; @@ -58,8 +57,6 @@ public class StorageIndexerPayloadMapper { private IVirtualPropertiesSchemaCache virtualPropertiesSchemaCache; @Inject private GeoShapeDecimator decimator; - @Inject - private GeoShapeDecimationSetting decimationSetting; public Map<String, Object> mapDataPayload(IndexSchema storageSchema, Map<String, Object> storageRecordData, String recordId) { @@ -245,7 +242,7 @@ public class StorageIndexerPayloadMapper { // No VirtualProperties.DefaultLocation.Wgs84Coordinates defined, use the default geo-shape property if (originalGeoShapeProperty == null) originalGeoShapeProperty = getDefaultGeoShapeProperty(dataCollectorMap); - if(originalGeoShapeProperty != null && decimationSetting.isDecimationEnabled()) { + if(originalGeoShapeProperty != null) { try { decimateGeoShape(originalGeoShapeProperty, dataCollectorMap); } catch (JsonProcessingException ex) { diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/AugmenterSetting.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/AugmenterSetting.java index d11d41cc38f5b3c7aaf1cd63cfbafd38ca4250eb..84cc039fe58df7d12391640874a4424e0823794e 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/AugmenterSetting.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/AugmenterSetting.java @@ -1,16 +1,34 @@ package org.opengroup.osdu.indexer.util; +import org.opengroup.osdu.core.common.feature.IFeatureFlag; +import org.opengroup.osdu.core.common.model.http.DpsHeaders; +import org.opengroup.osdu.indexer.util.geo.decimator.FeatureFlagCache; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; @Component public class AugmenterSetting { private static final String PROPERTY_NAME = "index-augmenter-enabled"; + @Lazy @Autowired - private BooleanFeatureFlagClient booleanFeatureFlagClient; + private FeatureFlagCache cache; + + @Autowired + private DpsHeaders headers; + + @Autowired + private IFeatureFlag iFeatureFlag; public boolean isEnabled() { - return booleanFeatureFlagClient.isEnabled(PROPERTY_NAME, false); + String dataPartitionId = headers.getPartitionId(); + String cacheKey = String.format("%s-%s", dataPartitionId, PROPERTY_NAME); + if (cache != null && cache.containsKey(cacheKey)) + return cache.get(cacheKey); + + boolean isEnabled = iFeatureFlag.isFeatureEnabled(PROPERTY_NAME); + this.cache.put(cacheKey, isEnabled); + return isEnabled; } } diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/BooleanFeatureFlagClient.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/BooleanFeatureFlagClient.java deleted file mode 100644 index a27ec91cd02d9bb54060e8509ec470512b7bd36d..0000000000000000000000000000000000000000 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/BooleanFeatureFlagClient.java +++ /dev/null @@ -1,81 +0,0 @@ -package org.opengroup.osdu.indexer.util; - -import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; -import org.opengroup.osdu.core.common.model.http.DpsHeaders; -import org.opengroup.osdu.core.common.partition.*; -import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient; -import org.opengroup.osdu.indexer.util.geo.decimator.FeatureFlagCache; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Lazy; -import org.springframework.stereotype.Component; - -@Component -public class BooleanFeatureFlagClient { - private static final String TOKEN_PREFIX = "Bearer "; - - @Lazy - @Autowired - private FeatureFlagCache cache; - - @Autowired - private JaxRsDpsLog logger; - - @Autowired - private DpsHeaders headers; - - @Autowired - private IPartitionFactory factory; - - @Autowired - private IServiceAccountJwtClient tokenService; - - public boolean isEnabled(String featureName, boolean defaultValue) { - String dataPartitionId = headers.getPartitionId(); - String cacheKey = String.format("%s-%s", dataPartitionId, featureName); - if (cache != null && cache.containsKey(cacheKey)) - return cache.get(cacheKey); - - boolean isEnabled = defaultValue; - try { - PartitionInfo partitionInfo = getPartitionInfo(dataPartitionId); - isEnabled = getFeatureValue(partitionInfo, featureName, defaultValue); - } catch (Exception e) { - this.logger.error(String.format("PartitionService: Error getting %s for dataPartition with Id: %s. Turn on the feature flag by default.", featureName, dataPartitionId), e); - } - this.cache.put(cacheKey, isEnabled); - return isEnabled; - } - - private PartitionInfo getPartitionInfo(String dataPartitionId) throws PartitionException { - try { - DpsHeaders partitionHeaders = DpsHeaders.createFromMap(headers.getHeaders()); - partitionHeaders.put(DpsHeaders.AUTHORIZATION, this.getAuthorization(dataPartitionId)); - - IPartitionProvider partitionProvider = this.factory.create(partitionHeaders); - PartitionInfo partitionInfo = partitionProvider.get(dataPartitionId); - return partitionInfo; - } catch (PartitionException e) { - logger.error(String.format("Error getting partition info for data-partition: %s", dataPartitionId), e); - throw e; - } - } - - private String getAuthorization(String dataPartitionId) { - String authorization = this.tokenService.getIdToken(dataPartitionId); - if(!authorization.startsWith(TOKEN_PREFIX)) { - authorization = TOKEN_PREFIX + authorization; - } - return authorization; - } - - private boolean getFeatureValue(PartitionInfo partitionInfo, String featureName, boolean defaultValue) { - if(partitionInfo == null || partitionInfo.getProperties() == null) - return defaultValue; - - if(partitionInfo.getProperties().containsKey(featureName)) { - Property property = partitionInfo.getProperties().get(featureName); - return Boolean.parseBoolean((String)property.getValue()); - } - return defaultValue; - } -} diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/geo/decimator/GeoShapeDecimationSetting.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/geo/decimator/GeoShapeDecimationSetting.java deleted file mode 100644 index ddab10f0180684167776593c5b064816dadd84e5..0000000000000000000000000000000000000000 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/geo/decimator/GeoShapeDecimationSetting.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright © Schlumberger - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opengroup.osdu.indexer.util.geo.decimator; - -import org.opengroup.osdu.indexer.util.BooleanFeatureFlagClient; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -@Component -public class GeoShapeDecimationSetting { - private static final String PROPERTY_NAME = "indexer-decimation-enabled"; - - @Autowired - private BooleanFeatureFlagClient booleanFeatureFlagClient; - - public boolean isDecimationEnabled() { - return booleanFeatureFlagClient.isEnabled(PROPERTY_NAME, true); - } -} diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapperTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapperTest.java index f11d6cb80a2b258d742bf13e26d2e2ee62dabfa8..6b7e5027df1771ad4329a9ed20e5247aaa273b11 100644 --- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapperTest.java +++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapperTest.java @@ -21,7 +21,6 @@ import org.opengroup.osdu.indexer.service.mock.PartitionFactoryMock; import org.opengroup.osdu.indexer.service.mock.PartitionProviderMock; import org.opengroup.osdu.indexer.service.mock.ServiceAccountJwtClientMock; import org.opengroup.osdu.indexer.service.mock.VirtualPropertiesSchemaCacheMock; -import org.opengroup.osdu.indexer.util.BooleanFeatureFlagClient; import org.opengroup.osdu.indexer.util.geo.decimator.*; import org.opengroup.osdu.indexer.util.parser.BooleanParser; import org.opengroup.osdu.indexer.util.parser.DateTimeParser; @@ -32,7 +31,6 @@ import org.springframework.boot.test.context.SpringBootTest; import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder; import org.springframework.test.context.junit4.SpringRunner; -import javax.inject.Inject; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; @@ -47,7 +45,7 @@ import static org.junit.Assert.*; @SpringBootTest(classes = {StorageIndexerPayloadMapper.class, AttributeParsingServiceImpl.class, NumberParser.class, BooleanParser.class, DateTimeParser.class, GeoShapeParser.class, DouglasPeuckerReducer.class, GeoShapeDecimator.class, GeometryDecimator.class, GeometryConversionService.class, FeatureFlagCache.class, - GeoShapeDecimationSetting.class, BooleanFeatureFlagClient.class,DpsHeaders.class, JobStatus.class, SchemaConverterPropertiesConfig.class, JaxRsDpsLog.class, + DpsHeaders.class, JobStatus.class, SchemaConverterPropertiesConfig.class, JaxRsDpsLog.class, PartitionFactoryMock.class, PartitionProviderMock.class, ServiceAccountJwtClientMock.class, VirtualPropertiesSchemaCacheMock.class, }) public class StorageIndexerPayloadMapperTest { @@ -78,9 +76,6 @@ public class StorageIndexerPayloadMapperTest { @Autowired private IVirtualPropertiesSchemaCache virtualPropertiesSchemaCache; - @Inject - private GeoShapeDecimationSetting decimationSetting; - @BeforeClass public static void setUp() { HashMap<String, Object> dataMap = new HashMap<>(); @@ -206,11 +201,6 @@ public class StorageIndexerPayloadMapperTest { assertNull(dataCollectorMap.get("VirtualProperties.DefaultName")); } - @Test - public void geoshape_decimation_is_enabled_by_default() { - assertTrue(decimationSetting.isDecimationEnabled()); - } - @Test public void geoshape_decimation_is_executed_with_virtual_spatial_location() { final String kind = "osdu:wks:master-data--SeismicAcquisitionSurvey:1.0.0"; diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/BooleanFeatureFlagClientTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/BooleanFeatureFlagClientTest.java deleted file mode 100644 index 10e73df399a9ba64b858c121df5e7ad365e2fbcd..0000000000000000000000000000000000000000 --- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/BooleanFeatureFlagClientTest.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright © Schlumberger - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opengroup.osdu.indexer.util; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.opengroup.osdu.core.common.logging.JaxRsDpsLog; -import org.opengroup.osdu.core.common.model.http.DpsHeaders; -import org.opengroup.osdu.core.common.partition.*; -import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient; -import org.opengroup.osdu.indexer.util.geo.decimator.FeatureFlagCache; -import org.springframework.test.context.junit4.SpringRunner; - -import java.util.HashMap; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.powermock.api.mockito.PowerMockito.when; - -@RunWith(SpringRunner.class) -public class BooleanFeatureFlagClientTest { - private static final String PROPERTY_NAME = "indexer-decimation-enabled"; - - @InjectMocks - private BooleanFeatureFlagClient sut; - - @Mock - private FeatureFlagCache cache; - - @Mock - private JaxRsDpsLog logger; - - @Mock - private DpsHeaders headers; - - @Mock - private IPartitionFactory factory; - - @Mock - private IServiceAccountJwtClient tokenService; - - @Mock - IPartitionProvider partitionProvider; - - @Before - public void setup() { - when(this.headers.getPartitionId()).thenReturn("dataPartitionId"); - when(this.headers.getHeaders()).thenReturn(new HashMap()); - when(this.factory.create(any())).thenReturn(partitionProvider); - when(this.tokenService.getIdToken(anyString())).thenReturn("token"); - } - - @Test - public void isDecimationEnabled_return_true() throws PartitionException { - PartitionInfo partitionInfo = new PartitionInfo(); - Property property = new Property(); - property.setSensitive(false); - property.setValue("true"); - partitionInfo.getProperties().put(PROPERTY_NAME, property); - when(this.partitionProvider.get(anyString())).thenReturn(partitionInfo); - - // Default value won't take any effect - boolean enabled = sut.isEnabled(PROPERTY_NAME, true); - Assert.assertTrue(enabled); - - enabled = sut.isEnabled(PROPERTY_NAME, false); - Assert.assertTrue(enabled); - } - - @Test - public void isDecimationEnabled_return_false_when_property_set_to_false() throws PartitionException { - PartitionInfo partitionInfo = new PartitionInfo(); - Property property = new Property(); - property.setSensitive(false); - property.setValue("false"); - partitionInfo.getProperties().put(PROPERTY_NAME, property); - when(this.partitionProvider.get(anyString())).thenReturn(partitionInfo); - - // Default value won't take any effect - boolean enabled = sut.isEnabled(PROPERTY_NAME, true); - Assert.assertFalse(enabled); - - enabled = sut.isEnabled(PROPERTY_NAME, false); - Assert.assertFalse(enabled); - } - - @Test - public void isDecimationEnabled_return_default_value_when_property_does_not_exist() throws PartitionException { - // The feature flag is enabled by default - PartitionInfo partitionInfo = new PartitionInfo(); - when(this.partitionProvider.get(anyString())).thenReturn(partitionInfo); - boolean enabled = sut.isEnabled(PROPERTY_NAME, true);; - Assert.assertTrue(enabled); - - enabled = sut.isEnabled(PROPERTY_NAME, false);; - Assert.assertFalse(enabled); - } - - @Test - public void isDecimationEnabled_return_default_value_when_partitionProvider_throws_exception() throws PartitionException { - // The feature flag is enabled by default - when(this.partitionProvider.get(anyString())).thenThrow(PartitionException.class); - boolean enabled = sut.isEnabled(PROPERTY_NAME, true);; - Assert.assertTrue(enabled); - - enabled = sut.isEnabled(PROPERTY_NAME, false);; - Assert.assertFalse(enabled); - } - -}