diff --git a/NOTICE b/NOTICE index e3b1becbb65b4d8470742746d26fd69293e8c6b1..6c3f19975b3ee7d491dca1595044cd3ca7d4314f 100644 --- a/NOTICE +++ b/NOTICE @@ -397,8 +397,8 @@ The following software have components provided under the terms of this license: - Google APIs Client Library for Java (from https://repo1.maven.org/maven2/com/google/api-client/google-api-client) - Google App Engine extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-appengine) - Google Cloud Core (from https://github.com/googleapis/google-cloud-java, https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-core, https://github.com/googleapis/java-core) -- Google Cloud Core HTTP (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-core-http, https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-core-http, https://github.com/googleapis/java-core) -- Google Cloud Core gRPC (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-core-grpc, https://github.com/googleapis/java-core) +- Google Cloud Core HTTP (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-core-http, https://github.com/googleapis/google-cloud-java, https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-core-http, https://github.com/googleapis/java-core) +- Google Cloud Core gRPC (from https://github.com/googleapis/google-cloud-java, https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-core-grpc, https://github.com/googleapis/java-core) - Google Cloud Datastore (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-datastore, https://github.com/googleapis/java-datastore) - Google Cloud IAM Service Account Credentials (from https://github.com/googleapis/google-cloud-java, https://github.com/googleapis/java-iamcredentials) - Google Cloud Logging (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-logging, https://github.com/googleapis/java-logging) diff --git a/devops/gc/deploy/README.md b/devops/gc/deploy/README.md index 3d49b2f9d9b72cb4bbb27d22c2ee6c321651a3d0..b77aed3c34d7b5dc294439dcd5e0dd970c3831a6 100644 --- a/devops/gc/deploy/README.md +++ b/devops/gc/deploy/README.md @@ -46,8 +46,8 @@ First you need to set variables in **values.yaml** file using any code editor. S | Name | Description | Type | Default |Required | |------|-------------|------|---------|---------| -**data.requestsCpu** | amount of requested CPU | string | 0.1 | yes -**data.requestsMemory** | amount of requested memory| string | 640M | yes +**data.requestsCpu** | amount of requested CPU | string | 480m | yes +**data.requestsMemory** | amount of requested memory| string | 700Mi | yes **data.limitsCpu** | CPU limit | string | 1 | yes **data.limitsMemory** | memory limit | string | 1G | yes **data.image** | service image | string | - | yes @@ -72,9 +72,9 @@ First you need to set variables in **values.yaml** file using any code editor. S | Name | Description | Type | Default |Required | |------|-------------|------|---------|---------| -**istio.proxyCPU** | CPU request for Envoy sidecars | string | 50m | yes +**istio.proxyCPU** | CPU request for Envoy sidecars | string | 130m | yes **istio.proxyCPULimit** | CPU limit for Envoy sidecars | string | 500m | yes -**istio.proxyMemory** | memory request for Envoy sidecars | string | 64Mi | yes +**istio.proxyMemory** | memory request for Envoy sidecars | string | 100Mi | yes **istio.proxyMemoryLimit** | memory limit for Envoy sidecars | string | 512Mi | yes ### Install the helm chart diff --git a/devops/gc/deploy/templates/deploy-redis.yaml b/devops/gc/deploy/templates/deploy-redis.yaml index 0e7dd33349e512ec204668d0d1d452905521cbb6..f310f1a847b80c2af66ef3dc3fa218e7250e661c 100644 --- a/devops/gc/deploy/templates/deploy-redis.yaml +++ b/devops/gc/deploy/templates/deploy-redis.yaml @@ -14,7 +14,8 @@ spec: labels: app: {{ printf "redis-%s" .Values.conf.appName | quote }} annotations: - sidecar.istio.io/proxyCPU: 30m + sidecar.istio.io/proxyCPU: 50m + sidecar.istio.io/proxyMemory: 100Mi spec: containers: - args: @@ -31,5 +32,6 @@ spec: protocol: TCP resources: requests: - memory: 100Mi + cpu: 30m + memory: 50Mi {{ end }} diff --git a/devops/gc/deploy/values.yaml b/devops/gc/deploy/values.yaml index a825cc7f303c17a0a17184a8f9c0b56dfb114d6e..a5f7c271e40d3dfb957915b94612319fc8798ab9 100644 --- a/devops/gc/deploy/values.yaml +++ b/devops/gc/deploy/values.yaml @@ -13,8 +13,8 @@ data: springProfilesActive: "gcp" storageHost: "http://storage" # Deployment - requestsCpu: "75m" - requestsMemory: "640M" + requestsCpu: "480m" + requestsMemory: "700Mi" limitsCpu: "1" limitsMemory: "1G" image: "" @@ -36,7 +36,7 @@ conf: indexerRedisSecretName: "indexer-redis-secret" istio: - proxyCPU: "25m" + proxyCPU: "130m" proxyCPULimit: "200m" - proxyMemory: "64Mi" + proxyMemory: "100Mi" proxyMemoryLimit: "256Mi" diff --git a/docs/tutorial/PreviewFeatures.md b/docs/tutorial/PreviewFeatures.md index eaa06b7194ba3d5cb70bf8f01b8b0ae49995dfd3..307360f59ddab73756195822f622d7ae9948142f 100644 --- a/docs/tutorial/PreviewFeatures.md +++ b/docs/tutorial/PreviewFeatures.md @@ -8,17 +8,18 @@ by implementing Ramer–Douglas–Peucker algorithm: - Polygon - MultiPolygon -In order to reduce the risk when extended evaluation of the solution is still on going, a feature flag that is managed by -the Partition Service is applied to the solution. -Here is an example to enable this feature by setting the property "indexer-decimation-enabled" in a given data partition: + +The feature is enabled by default for all data partitions. If client does not want the geo-shape to be decimated in their +data partitions, they can disable geo-shape decimation through the Partition Service. +Here is an example to disable this feature by setting the property "indexer-decimation-enabled" in a given data partition: ``` { "indexer-decimation-enabled": { "sensitive": false, - "value": "true" + "value": "false" } } ``` -If the property "indexer-decimation-enabled" is not created or the property value is set to "false" (String type) in the -given data partition, the geo-shape decimation will be ignored. +If the property "indexer-decimation-enabled" is not created or the property value is set to "true" (String type) in the +given data partition, the geo-shape decimation will be enabled. diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java index ff381db42c4d0854f8e36eba2d05a9733f0535ec..89c13fcb1fbc6b0da27795b5577a491eac8bf3dd 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java @@ -116,6 +116,10 @@ public class StorageServiceImpl implements StorageService { throw new AppException(HttpStatus.SC_NOT_FOUND, "Invalid request", "Storage service returned empty response"); } + if (response.getResponseCode() == 500) { + throw new AppException(RequestStatus.NO_RETRY, "Server error", String.format("Storage service error: %s", response.getBody())); + } + Records records = null; try { records = this.objectMapper.readValue(bulkStorageData, Records.class); diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/geo/decimator/GeoShapeDecimationSetting.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/geo/decimator/GeoShapeDecimationSetting.java index 492f2d7479f3c2a02d883681fab6216045211b0e..a062c1ac6e645e3e5bdcec4d9c066e0d0230d199 100644 --- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/geo/decimator/GeoShapeDecimationSetting.java +++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/geo/decimator/GeoShapeDecimationSetting.java @@ -51,14 +51,13 @@ public class GeoShapeDecimationSetting { if (cache != null && cache.containsKey(cacheKey)) return cache.get(cacheKey); - boolean decimationEnabled = false; + boolean decimationEnabled = true; try { PartitionInfo partitionInfo = getPartitionInfo(dataPartitionId); decimationEnabled = getDecimationSetting(partitionInfo); } catch (Exception e) { - this.logger.error(String.format("PartitionService: Error getting %s for dataPartition with Id: %s", PROPERTY_NAME, dataPartitionId), e); + this.logger.error(String.format("PartitionService: Error getting %s for dataPartition with Id: %s. Turn on the feature flag by default.", PROPERTY_NAME, dataPartitionId), e); } - this.cache.put(cacheKey, decimationEnabled); return decimationEnabled; } @@ -78,12 +77,12 @@ public class GeoShapeDecimationSetting { private boolean getDecimationSetting(PartitionInfo partitionInfo) { if(partitionInfo == null || partitionInfo.getProperties() == null) - return false; + return true; if(partitionInfo.getProperties().containsKey(PROPERTY_NAME)) { Property property = partitionInfo.getProperties().get(PROPERTY_NAME); return Boolean.parseBoolean((String)property.getValue()); } - return false; + return true; } } diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/geo/decimator/GeoShapeDecimationSettingTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/geo/decimator/GeoShapeDecimationSettingTest.java index 2b4ed8351c66c303b242403c063bcb5d510a0662..82fc83db7ccd0a520eabb90ea3a0f414ca0338c2 100644 --- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/geo/decimator/GeoShapeDecimationSettingTest.java +++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/geo/decimator/GeoShapeDecimationSettingTest.java @@ -91,18 +91,20 @@ public class GeoShapeDecimationSettingTest { } @Test - public void isDecimationEnabled_return_false_when_property_does_not_exist() throws PartitionException { + public void isDecimationEnabled_return_true_when_property_does_not_exist() throws PartitionException { + // The feature flag is enabled by default PartitionInfo partitionInfo = new PartitionInfo(); when(this.partitionProvider.get(anyString())).thenReturn(partitionInfo); boolean enabled = sut.isDecimationEnabled(); - Assert.assertFalse(enabled); + Assert.assertTrue(enabled); } @Test - public void isDecimationEnabled_return_false_when_partitionProvider_throws_exception() throws PartitionException { + public void isDecimationEnabled_return_true_when_partitionProvider_throws_exception() throws PartitionException { + // The feature flag is enabled by default when(this.partitionProvider.get(anyString())).thenThrow(PartitionException.class); boolean enabled = sut.isDecimationEnabled(); - Assert.assertFalse(enabled); + Assert.assertTrue(enabled); } }