From 6966a9e1867defefd244dc67b040d058de9a4dcb Mon Sep 17 00:00:00 2001
From: Arsen Grigoryan <Grigoryan>
Date: Sat, 25 Jun 2022 01:24:41 +0400
Subject: [PATCH] BUG 7154 [osdu][search][indexer] Indexer skipped the entire
record with malformed SpatialLocation
---
.../FeatureCollectionDeserializer.java | 4 +-
.../indexer/util/parser/GeoShapeParser.java | 3 +
.../FeatureCollectionDeserializerTest.java | 202 ------------------
.../util/parser/GeoShapeParserTest.java | 11 +
.../input/invalid_feature_collection.json | 15 ++
5 files changed, 31 insertions(+), 204 deletions(-)
delete mode 100644 indexer-core/src/test/java/org/opengroup/osdu/indexer/model/geojson/jackson/FeatureCollectionDeserializerTest.java
create mode 100644 indexer-core/src/test/resources/geojson/parsing/input/invalid_feature_collection.json
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/geojson/jackson/FeatureCollectionDeserializer.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/geojson/jackson/FeatureCollectionDeserializer.java
index e1b45fc89..390229859 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/geojson/jackson/FeatureCollectionDeserializer.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/geojson/jackson/FeatureCollectionDeserializer.java
@@ -14,12 +14,12 @@
package org.opengroup.osdu.indexer.model.geojson.jackson;
+import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.ObjectCodec;
import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.JsonDeserializer;
import com.fasterxml.jackson.databind.JsonNode;
-import com.google.gson.JsonSyntaxException;
import org.opengroup.osdu.indexer.model.geojson.Feature;
import org.opengroup.osdu.indexer.model.geojson.FeatureCollection;
@@ -42,7 +42,7 @@ public class FeatureCollectionDeserializer extends JsonDeserializer<FeatureColle
JsonNode features = featureCollection.get("features");
if(features == null){
- throw new JsonSyntaxException("Missing feature field in the ");
+ throw new JsonParseException(jsonParser, "Missing feature field in the ");
}
final List<Feature> result = new ArrayList<>();
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/parser/GeoShapeParser.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/parser/GeoShapeParser.java
index 8bd6ec834..3674ca795 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/parser/GeoShapeParser.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/parser/GeoShapeParser.java
@@ -14,6 +14,7 @@
package org.opengroup.osdu.indexer.util.parser;
+import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -42,6 +43,8 @@ public class GeoShapeParser {
});
} catch (InvalidTypeIdException e) {
throw new IllegalArgumentException("must be a valid FeatureCollection");
+ } catch (JsonParseException e){
+ throw new IllegalArgumentException(e.getMessage());
} catch (JsonProcessingException e) {
throw new IllegalArgumentException("unable to parse FeatureCollection");
}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/geojson/jackson/FeatureCollectionDeserializerTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/geojson/jackson/FeatureCollectionDeserializerTest.java
deleted file mode 100644
index fa61f1447..000000000
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/geojson/jackson/FeatureCollectionDeserializerTest.java
+++ /dev/null
@@ -1,202 +0,0 @@
-package org.opengroup.osdu.indexer.model.geojson.jackson;
-
-import com.fasterxml.jackson.core.JsonFactory;
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
-import joptsimple.internal.Strings;
-import lombok.SneakyThrows;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.opengroup.osdu.indexer.model.geojson.Feature;
-import org.opengroup.osdu.indexer.model.geojson.FeatureCollection;
-import org.springframework.test.context.junit4.SpringRunner;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.lang.reflect.Type;
-import java.util.List;
-import java.util.Map;
-
-import static org.hamcrest.CoreMatchers.containsString;
-import static org.junit.Assert.*;
-
-@RunWith(SpringRunner.class)
-public class FeatureCollectionDeserializerTest {
-
- @InjectMocks
- private FeatureCollectionDeserializer featureCollectionDeserializer;
-
- private final ObjectMapper mapper = new ObjectMapper();
- private final JsonFactory factory = mapper.getFactory();
-
- @Test
- public void should_throwException_provided_emptyFeatureJson() {
- String shapeJson = "{}";
-
- this.validateInput(shapeJson, Strings.EMPTY, "feature not included");
- }
-
- @Test
- public void should_throwException_parseInvalidPoint() {
- String shapeJson = getFeatureFromFile("input/invalid_point.json");
- this.validateInput(shapeJson, Strings.EMPTY, "unable to parse FeatureCollection");
- }
-
- @Test
- public void should_throwException_parseInvalidPoint_NaN() {
- String shapeJson = getFeatureFromFile("input/invalid_point_nan.json");
- this.validateInput(shapeJson, Strings.EMPTY, "unable to parse FeatureCollection");
- }
-
- @Test
- public void should_throwException_parseInvalidPoint_missingLatitude() {
- String shapeJson = getFeatureFromFile("input/invalid_point_missing_latitude.json");
- this.validateInput(shapeJson, Strings.EMPTY, "unable to parse FeatureCollection");
- }
-
- @Test
- public void should_throwException_missingMandatoryAttribute() {
- String shapeJson = getFeatureFromFile("input/missing_mandatory_attribute.json");
- this.validateInput(shapeJson, Strings.EMPTY, "must be a valid FeatureCollection");
- }
-
- @Test
- public void should_throwException_parseInvalidShape() {
- String shapeJson = getFeatureFromFile("input/invalid_shape.json");
- this.validateInput(shapeJson, Strings.EMPTY, "must be a valid FeatureCollection");
- }
-
- @Test
- public void should_parseValidPoint() {
- String shapeJson = getFeatureFromFile("input/valid_point.json");
- this.validateInput(shapeJson, shapeJson, Strings.EMPTY);
- }
-
- @Test
- public void should_parseValidMultiPoint() {
- String shapeJson = getFeatureFromFile("input/valid_multi_point.json");
- this.validateInput(shapeJson, shapeJson, Strings.EMPTY);
- }
-
- @Test
- public void should_parseValidLineString() {
- String shapeJson = getFeatureFromFile("input/valid_line_string.json");
- this.validateInput(shapeJson, shapeJson, Strings.EMPTY);
- }
-
- @Test
- public void should_parseValidMultiLineString() {
- String shapeJson = getFeatureFromFile("input/valid_multi_line_string.json");
- this.validateInput(shapeJson, shapeJson, Strings.EMPTY);
- }
-
- @Test
- public void should_parseValidPolygon() {
- String shapeJson = getFeatureFromFile("input/valid_polygon.json");
- this.validateInput(shapeJson, shapeJson, Strings.EMPTY);
- }
-
- @Test
- public void should_throwException_parseInvalidPolygon_malformedLatitude() {
- String shapeJson = getFeatureFromFile("input/invalid_polygon_malformed_latitude.json");
- this.validateInput(shapeJson, Strings.EMPTY, "unable to parse FeatureCollection");
- }
-
- @Test
- public void should_parseValidMultiPolygon() {
- String shapeJson = getFeatureFromFile("input/valid_multi_polygon.json");
- this.validateInput(shapeJson, shapeJson, Strings.EMPTY);
- }
-
- @Test
- public void should_parseValidGeometryCollection() {
- String shapeJson = getFeatureFromFile("input/valid_geometry_collection.json");
- this.validateInput(shapeJson, shapeJson, Strings.EMPTY);
- }
-
- @Test
- public void should_parseValidFeatureCollection() {
- String shapeJson = getFeatureFromFile("input/valid_feature_collection.json");
- this.validateInput(shapeJson, shapeJson, Strings.EMPTY);
- }
-
- @Test
- public void should_parseValidFeatureCollection_withZCoordinate() {
- String shapeJson = getFeatureFromFile("input/valid_feature_collection_with_z_coordinate.json");
- this.validateInput(shapeJson, shapeJson, Strings.EMPTY);
- }
-
- @Test
- public void should_throwException_parseUnsupportedType_feature() {
- String shapeJson = getFeatureFromFile("input/valid_feature_collection_with_z_coordinate.json");
- this.validateInput(shapeJson, shapeJson, Strings.EMPTY);
- }
-
- @Test
- public void shouldParseValidMultiPolygonWithZCoordinates() {
- String shapeJson = getFeatureFromFile("input/multi_polygon_with_z_coordinates.json");
- this.validateInput(shapeJson, shapeJson, Strings.EMPTY);
- }
-
- @Test
- public void shouldParseValidLineStringWithZCoordinates() {
- String shapeJson = getFeatureFromFile("input/line_string_with_z_coordinate.json");
- this.validateInput(shapeJson, shapeJson, Strings.EMPTY);
- }
-
- @Test
- public void shouldParseValidMultiLineStringWithZCoordinates() {
- String shapeJson = getFeatureFromFile("input/valid_milti_line_string_with_z_coordinate.json");
- this.validateInput(shapeJson, shapeJson, Strings.EMPTY);
- }
-
-
- private void validateInput(String shapeJson, String expectedParsedShape, String errorMessage) {
- try {
- Type type = new TypeToken<Map<String, Object>>() {}.getType();
- Map<String, Object> expectedShape = new Gson().fromJson(expectedParsedShape, type);
- FeatureCollection featureCollection = mapper.convertValue(expectedShape, FeatureCollection.class);
-
- JsonParser parser = factory.createParser(shapeJson);
- List<Feature> result = (List<Feature>) callPrivateMethod_ExtractFeature(parser);
-
- assertNotNull(result);
- assertTrue(Strings.isNullOrEmpty(errorMessage));
- assertEquals(featureCollection.getFeatures(), result);
- } catch (IllegalArgumentException e) {
- if (Strings.isNullOrEmpty(errorMessage)) {
- fail(String.format("error parsing valid feature-json %s", shapeJson));
- } else {
- assertThat(String.format("Incorrect error message for feature-json parsing [ %s ]", shapeJson), e.getMessage(), containsString(errorMessage));
- }
- } catch (IOException | InvocationTargetException | NoSuchMethodException | IllegalAccessException e) {
- e.printStackTrace();
- }
- }
-
- @SneakyThrows
- private String getFeatureFromFile(String file) {
- InputStream inStream = this.getClass().getResourceAsStream("/geojson/parsing/" + file);
- assert inStream != null;
- BufferedReader br = new BufferedReader(new InputStreamReader(inStream));
- StringBuilder stringBuilder = new StringBuilder();
- String sCurrentLine;
- while ((sCurrentLine = br.readLine()) != null) {
- stringBuilder.append(sCurrentLine).append("\n");
- }
- return stringBuilder.toString();
- }
-
- public Object callPrivateMethod_ExtractFeature(JsonParser parser) throws NoSuchMethodException, InvocationTargetException, IllegalAccessException, IOException {
- Method method = featureCollectionDeserializer.getClass().getDeclaredMethod("extractFeature", JsonParser.class);
- method.setAccessible(true);
- return method.invoke(featureCollectionDeserializer, parser);
- }
-}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/parser/GeoShapeParserTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/parser/GeoShapeParserTest.java
index 2b47f5ab3..fdec74784 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/parser/GeoShapeParserTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/parser/GeoShapeParserTest.java
@@ -34,6 +34,7 @@ import java.util.function.Function;
import static org.hamcrest.CoreMatchers.containsString;
import static org.junit.Assert.*;
+import static org.junit.jupiter.api.Assertions.assertThrows;
@RunWith(SpringRunner.class)
public class GeoShapeParserTest {
@@ -178,6 +179,16 @@ public class GeoShapeParserTest {
this.validateInput(this.sut::parseGeoJson, shapeJson, expectedParsedShape, Strings.EMPTY);
}
+ @Test
+ public void should_throwException_parseInvalidFeatureCollection() {
+ String shapeJson = getGeoShapeFromFile("input/invalid_feature_collection.json");
+
+ Type type = new TypeToken<Map<String, Object>>() {}.getType();
+ Map<String, Object> shapeObj = new Gson().fromJson(shapeJson, type);
+
+ assertThrows(IllegalArgumentException.class, () -> this.sut.parseGeoJson(shapeObj));
+ }
+
private void validateInput(Function<Map<String, Object>, Map<String, Object>> parser, String shapeJson, String expectedParsedShape, String errorMessage) {
try {
Type type = new TypeToken<Map<String, Object>>() {}.getType();
diff --git a/indexer-core/src/test/resources/geojson/parsing/input/invalid_feature_collection.json b/indexer-core/src/test/resources/geojson/parsing/input/invalid_feature_collection.json
new file mode 100644
index 000000000..5a91d57a7
--- /dev/null
+++ b/indexer-core/src/test/resources/geojson/parsing/input/invalid_feature_collection.json
@@ -0,0 +1,15 @@
+{
+ "type": "FeatureCollection",
+ "featur3es": [
+ {
+ "type": "Feature",
+ "geometry": {
+ "type": "Point",
+ "coordinates": [
+ -105.01621,
+ 39.57422
+ ]
+ }
+ }
+ ]
+}
\ No newline at end of file
--
GitLab