Skip to content
Snippets Groups Projects
Commit 05e6e0e2 authored by Neelesh Thakur's avatar Neelesh Thakur
Browse files

Merge branch 'geo-property-parsing' into 'master' 

don't throw exception on property block (with special character) on geojson

See merge request !364
parents ea844b7e 380aeab5
No related branches found
No related tags found
1 merge request!364don't throw exception on property block (with special character) on geojson
Pipeline #169606 failed
Stage: build
Stage: coverage
Stage: containerize
Stage: scan
Stage: deploy
Stage: integration
Stage: publish
Hide whitespace changes
Inline Side-by-side
Showing
with 85 additions and 27 deletions
indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java
+ 36
− 20
View file @ 05e6e0e2
......@@ -18,14 +18,6 @@ import com.google.common.base.Strings;
import com.google.gson.Gson;
import com.google.gson.JsonSyntaxException;
import com.google.gson.reflect.TypeToken;
import java.lang.reflect.Array;
import java.lang.reflect.Type;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.function.BiFunction;
import javax.inject.Inject;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.model.indexer.ElasticType;
import org.opengroup.osdu.core.common.model.indexer.IndexingStatus;
......@@ -37,6 +29,16 @@ import org.opengroup.osdu.indexer.util.parser.NumberParser;
import org.springframework.stereotype.Service;
import org.springframework.web.context.annotation.RequestScope;
import javax.inject.Inject;
import java.lang.reflect.Array;
import java.lang.reflect.Type;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.BiFunction;
@Service
@RequestScope
public class AttributeParsingServiceImpl implements IAttributeParsingService {
......@@ -54,6 +56,8 @@ public class AttributeParsingServiceImpl implements IAttributeParsingService {
@Inject
private JobStatus jobStatus;
private Gson gson = new Gson();
@Override
public void tryParseValueArray(Class<?> attributeClass, String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
BiFunction<String, Object, ?> parser;
......@@ -158,7 +162,7 @@ public class AttributeParsingServiceImpl implements IAttributeParsingService {
try {
Type type = new TypeToken<Map<String, Double>>() {}.getType();
Map<String, Double> positionMap = new Gson().fromJson(attributeVal.toString(), type);
Map<String, Double> positionMap = this.gson.fromJson(attributeVal.toString(), type);
if (positionMap == null || positionMap.isEmpty()) return;
......@@ -177,8 +181,20 @@ public class AttributeParsingServiceImpl implements IAttributeParsingService {
public void tryParseGeojson(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
try {
Type type = new TypeToken<Map<String, Object>>() {}.getType();
Map<String, Object> geoJsonMap = new Gson().fromJson(attributeVal.toString(), type);
Map<String, Object> geoJsonMap = new HashMap<>();
if (attributeVal instanceof Map) {
try {
geoJsonMap = (Map<String, Object>) attributeVal;
} catch (ClassCastException e) {
String parsingError = String.format("geo-json shape parsing error: %s attribute: %s", e.getMessage(), attributeName);
jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, parsingError, String.format("record-id: %s | %s", recordId, parsingError));
}
}
if (geoJsonMap.isEmpty()) {
Type type = new TypeToken<Map<String, Object>>() {}.getType();
geoJsonMap = this.gson.fromJson(this.gson.toJson(attributeVal, type), type);
}
if (geoJsonMap == null || geoJsonMap.isEmpty()) return;
......@@ -191,19 +207,19 @@ public class AttributeParsingServiceImpl implements IAttributeParsingService {
}
}
@Override
public void tryParseNested(String recordId, String name, Object value, Map<String, Object> dataMap) {
dataMap.put(name,value);
}
@Override
public void tryParseNested(String recordId, String name, Object value, Map<String, Object> dataMap) {
dataMap.put(name, value);
}
@Override
public void tryParseObject(String recordId, String name, Object value, Map<String, Object> dataMap) {
dataMap.put(name,value);
}
@Override
public void tryParseObject(String recordId, String name, Object value, Map<String, Object> dataMap) {
dataMap.put(name, value);
}
@Override
public void tryParseFlattened(String recordId, String name, Object value, Map<String, Object> dataMap) {
dataMap.put(name,value);
dataMap.put(name, value);
}
......
indexer-core/src/test/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImplTest.java
+ 23
− 7
View file @ 05e6e0e2
......@@ -31,11 +31,15 @@ import org.opengroup.osdu.indexer.util.parser.NumberParser;
import org.springframework.test.context.junit4.SpringRunner;
import java.lang.reflect.Type;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.Date;
import static org.junit.Assert.*;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.when;
......@@ -315,8 +319,21 @@ public class AttributeParsingServiceImplTest {
@Test
public void should_returnGeoShape_given_validTreeMap_tryGetGeoShapeTest() {
final String shapeJson = "{\"type\":\"Polygon\",\"coordinates\":[[[100,0],[101,0],[101,1],[100,1],[100,0]]]}";
Map<String, Object> storageData = new HashMap<>();
storageData.put("location", parseJson(shapeJson));
Map<String, Object> storageData = parseJson(shapeJson);
when(this.geoShapeParser.parseGeoJson(storageData)).thenReturn(new HashMap<>());
Map<String, Object> dataMap = new HashMap<>();
this.sut.tryParseGeojson("", "location", storageData, dataMap);
assertFalse(dataMap.isEmpty());
}
@Test
public void should_returnGeoShape_given_validTreeMap_tryGetGeoShapeWithPropertiesTest() {
final String shapeJson = "{\"features\":[{\"geometry\":{\"type\":\"Point\",\"coordinates\":[-105.01621,39.57422]},\"properties\":{\"id\":\"opendes:work-product-component--GenericRepresentation:0be3c0de-7844-4bcb-a17d-83de84cd2eca\",\"uri\":\"wdms:opendes:1188d27c-9132-41ec-b281-502a6245d00c:f597df66-4197-4347-99c2-acb58ce27ef3:0be3c0de-7844-4bcb-a17d-83de84cd2eca\"},\"type\":\"Feature\"}],\"type\":\"FeatureCollection\"}";
Map<String, Object> storageData = parseJson(shapeJson);
when(this.geoShapeParser.parseGeoJson(storageData)).thenReturn(new HashMap<>());
......@@ -330,8 +347,7 @@ public class AttributeParsingServiceImplTest {
@Test
public void should_throwException_given_geoShapeParingFailed() {
final String shapeJson = "{\"type\":\"Polygon\",\"coordinates\":[[[100,NaN],[101,0],[101,1],[100,1],[100,0]]]}";
Map<String, Object> storageData = new HashMap<>();
storageData.put("location", parseJson(shapeJson));
Map<String, Object> storageData = parseJson(shapeJson);
when(this.geoShapeParser.parseGeoJson(any())).thenThrow(new IllegalArgumentException("geo coordinates must be numbers"));
......@@ -374,4 +390,4 @@ public class AttributeParsingServiceImplTest {
*/
void accept(T t, U u, V v, W w, X x);
}
}
\ No newline at end of file
}
indexer-core/src/test/java/org/opengroup/osdu/indexer/util/parser/GeoShapeParserTest.java
+ 7
− 0
View file @ 05e6e0e2
......@@ -89,6 +89,13 @@ public class GeoShapeParserTest {
this.validateInput(this.sut::parseGeoJson, shapeJson, expectedParsedShape, Strings.EMPTY);
}
@Test
public void should_parseValidWithPropertyPoint() {
String shapeJson = getGeoShapeFromFile("input/valid_point_with_property.json");
String expectedParsedShape = getGeoShapeFromFile("expected/valid_point.json");
this.validateInput(this.sut::parseGeoJson, shapeJson, expectedParsedShape, Strings.EMPTY);
}
@Test
public void should_parseValidMultiPoint() {
String shapeJson = getGeoShapeFromFile("input/valid_multi_point.json");
......
indexer-core/src/test/resources/geojson/parsing/input/valid_point_with_property.json 0 → 100644
+ 19
− 0
View file @ 05e6e0e2
{
"features": [
{
"geometry": {
"type": "Point",
"coordinates": [
-105.01621,
39.57422
]
},
"properties": {
"id": "opendes:work-product-component--GenericRepresentation:0be3c0de-7844-4bcb-a17d-83de84cd2eca",
"uri": "wdms:opendes:1188d27c-9132-41ec-b281-502a6245d00c:f597df66-4197-4347-99c2-acb58ce27ef3:0be3c0de-7844-4bcb-a17d-83de84cd2eca"
},
"type": "Feature"
}
],
"type": "FeatureCollection"
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment