From d488c22f240c340e16490b538c962db0789d39a1 Mon Sep 17 00:00:00 2001
From: Neelesh Thakur <NThakur4@slb.com>
Date: Fri, 3 Apr 2020 16:55:05 -0500
Subject: [PATCH] add support for geojson shape indexing
---
indexer-core/pom.xml | 10 ++
.../service/AttributeParsingServiceImpl.java | 53 ++-----
.../service/IAttributeParsingService.java | 3 +-
.../indexer/service/IndexerServiceImpl.java | 2 +-
.../indexer/util/parser/GeoShapeParser.java | 65 ++++++++
.../util/parser/GeoShapeParserTest.java | 141 ++++++++++++++++++
.../features/indexrecord/IndexRecord.feature | 6 +-
.../resources/testData/index_records_3.json | 113 ++++++++++++++
.../resources/testData/index_records_3.schema | 69 +++++++++
9 files changed, 416 insertions(+), 46 deletions(-)
create mode 100644 indexer-core/src/main/java/org/opengroup/osdu/indexer/util/parser/GeoShapeParser.java
create mode 100644 indexer-core/src/test/java/org/opengroup/osdu/indexer/util/parser/GeoShapeParserTest.java
create mode 100644 testing/indexer-test-core/src/main/resources/testData/index_records_3.json
create mode 100644 testing/indexer-test-core/src/main/resources/testData/index_records_3.schema
diff --git a/indexer-core/pom.xml b/indexer-core/pom.xml
index 6012b4efe..097479c09 100644
--- a/indexer-core/pom.xml
+++ b/indexer-core/pom.xml
@@ -78,6 +78,16 @@
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>6.6.2</version>
</dependency>
+ <dependency>
+ <groupId>org.locationtech.spatial4j</groupId>
+ <artifactId>spatial4j</artifactId>
+ <version>0.7</version>
+ </dependency>
+ <dependency>
+ <groupId>org.locationtech.jts.io</groupId>
+ <artifactId>jts-io-common</artifactId>
+ <version>1.15.0</version>
+ </dependency>
<dependency>
<groupId>com.google.guava</groupId>
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java
index 4ed8c46cd..c0d9b2a73 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java
@@ -22,19 +22,14 @@ import com.google.gson.reflect.TypeToken;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.model.indexer.IndexingStatus;
import org.opengroup.osdu.core.common.model.indexer.JobStatus;
-import org.opengroup.osdu.core.common.Constants;
-import org.opengroup.osdu.core.common.model.indexer.ElasticType;
-import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
import org.opengroup.osdu.indexer.util.parser.DateTimeParser;
+import org.opengroup.osdu.indexer.util.parser.GeoShapeParser;
import org.opengroup.osdu.indexer.util.parser.NumberParser;
import org.springframework.stereotype.Service;
import org.springframework.web.context.annotation.RequestScope;
import javax.inject.Inject;
import java.lang.reflect.Type;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
import java.util.Map;
@Service
@@ -42,14 +37,14 @@ import java.util.Map;
public class AttributeParsingServiceImpl implements IAttributeParsingService {
private static final String GEOJSON = "GeoJSON";
- private static final String GEOMETRY_COLLECTION = "geometrycollection";
- private static final String GEOMETRIES = "geometries";
@Inject
private NumberParser numberParser;
@Inject
private DateTimeParser dateTimeParser;
@Inject
+ private GeoShapeParser geoShapeParser;
+ @Inject
private GeometryConversionService geometryConversionService;
@Inject
private JobStatus jobStatus;
@@ -149,45 +144,21 @@ public class AttributeParsingServiceImpl implements IAttributeParsingService {
}
}
- @SuppressWarnings("unchecked")
@Override
- public void tryParseGeojson(String recordId, Map<String, Object> storageRecordData, IndexSchema schemaObj, Map<String, Object> dataMap) {
+ public void tryParseGeojson(String recordId, String attributeName, Map<String, Object> storageRecordData, Map<String, Object> dataMap) {
- final String key = "GeoJSON.features.geometry";
- String[] strArray = key.split("\\.");
try {
- // strArray[0] is GeoJSON
- LinkedTreeMap<String, Object> map = (LinkedTreeMap) storageRecordData.get(strArray[0]);
-
- if (map == null || map.isEmpty()) return;
-
- List<Map<String, Object>> geometries = this.geometryConversionService.getGeoShape(map);
-
- if (geometries == null || geometries.isEmpty()) return;
-
- // Adding the point in geoshape too
- // get the key and get the corresponding object from the storageRecord object
- ArrayList<String> geoPointKeys = schemaObj.getSchemaKeysByValue(ElasticType.GEO_POINT.getValue());
-
- if (geoPointKeys == null || geoPointKeys.isEmpty()) return;
+ Type type = new TypeToken<Map<String, Object>>() {}.getType();
+ Map<String, Object> geoJsonMap = new Gson().fromJson(storageRecordData.get(attributeName).toString(), type);
- LinkedTreeMap<String, Double> innerMap = (LinkedTreeMap) storageRecordData.get(geoPointKeys.get(0));
+ if (geoJsonMap == null || geoJsonMap.isEmpty()) return;
- if (innerMap == null || innerMap.isEmpty()) return;
+ String shape = this.geoShapeParser.parseGeoJson(geoJsonMap);
- Map<String, Object> geopoint = this.geometryConversionService.getGeopointGeometry(innerMap);
-
- if (geopoint == null || geopoint.isEmpty()) return;
-
- geometries.add(geopoint);
-
- Map<String, Object> outerMap = new HashMap<>();
- outerMap.put(Constants.TYPE, GEOMETRY_COLLECTION);
- outerMap.put(GEOMETRIES, geometries);
-
- dataMap.put(DATA_GEOJSON_TAG, outerMap);
- } catch (ClassCastException e) {
- jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, "geo-json parsing error");
+ dataMap.put(attributeName, shape);
+ } catch (JsonSyntaxException | IllegalArgumentException e) {
+ String parsingError = String.format("geo-json shape parsing error: %s attribute: %s", e.getMessage(), attributeName);
+ jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, parsingError, String.format("record-id: %s | %s", recordId, parsingError));
}
}
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java
index edba8e24f..6d7c7d261 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java
@@ -24,6 +24,5 @@ public interface IAttributeParsingService {
void tryParseGeopoint(String recordId, String attributeName, Map<String, Object> storageRecordData, Map<String, Object> dataMap);
- @SuppressWarnings("unchecked")
- void tryParseGeojson(String recordId, Map<String, Object> storageRecordData, IndexSchema schemaObj, Map<String, Object> dataMap);
+ void tryParseGeojson(String recordId, String attributeName, Map<String, Object> storageRecordData, Map<String, Object> dataMap);
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
index ac64f0db7..e794af5fd 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
@@ -347,7 +347,7 @@ public class IndexerServiceImpl implements IndexerService {
this.attributeParsingServiceImpl.tryParseGeopoint(recordId, name, storageRecordData, dataMap);
break;
case GEO_SHAPE:
- this.attributeParsingServiceImpl.tryParseGeojson(recordId, storageRecordData, schemaObj, dataMap);
+ this.attributeParsingServiceImpl.tryParseGeojson(recordId, name, storageRecordData, dataMap);
break;
case NESTED:
case OBJECT:
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/parser/GeoShapeParser.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/parser/GeoShapeParser.java
new file mode 100644
index 000000000..d94bd78df
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/parser/GeoShapeParser.java
@@ -0,0 +1,65 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.util.parser;
+
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.geo.builders.ShapeBuilder;
+import org.elasticsearch.common.geo.parsers.ShapeParser;
+import org.elasticsearch.common.xcontent.*;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.locationtech.spatial4j.exception.InvalidShapeException;
+import org.locationtech.spatial4j.shape.Shape;
+import org.opengroup.osdu.core.common.search.Preconditions;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+import java.io.IOException;
+import java.util.Map;
+
+@Component
+@RequestScope
+public class GeoJsonParser {
+
+ public String parseGeoJson(Map<String, Object> geoShapeObject) {
+
+ Preconditions.checkNotNull(geoShapeObject, "geoShapeObject cannot be null");
+
+ try {
+ // use elasticsearch's ShapeParser to validate shape
+ ShapeBuilder<?, ?> shapeBuilder = getShapeBuilderFromObject(geoShapeObject);
+ Shape shape = shapeBuilder.buildS4J();
+ if (shape == null) {
+ throw new IllegalArgumentException("unable to parse shape");
+ }
+
+ return shapeBuilder.toString();
+ } catch (ElasticsearchParseException | InvalidShapeException | IOException e) {
+ throw new IllegalArgumentException(e);
+ }
+ }
+
+ private ShapeBuilder<?, ?> getShapeBuilderFromObject(Map<String, Object> object) throws IOException {
+ XContentBuilder contentBuilder = JsonXContent.contentBuilder().map(object);
+
+ XContentParser parser = JsonXContent.jsonXContent.createParser(
+ NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
+ XContentHelper.convertToJson(BytesReference.bytes(contentBuilder), false, XContentType.JSON)
+ );
+
+ parser.nextToken();
+ return ShapeParser.parse(parser);
+ }
+}
\ No newline at end of file
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/parser/GeoShapeParserTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/parser/GeoShapeParserTest.java
new file mode 100644
index 000000000..147ce234e
--- /dev/null
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/parser/GeoShapeParserTest.java
@@ -0,0 +1,141 @@
+// Copyright 2017-2019, Schlumberger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.util.parser;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.lang.reflect.Type;
+import java.util.Map;
+
+import static org.junit.Assert.assertNotNull;
+
+@RunWith(SpringRunner.class)
+public class GeoJsonParserTest {
+
+ @InjectMocks
+ private GeoJsonParser sut;
+
+ @Rule
+ public ExpectedException exceptionRule = ExpectedException.none();
+
+ @Test
+ public void should_parseValidPoint() {
+ String shapeJson = "{\"type\":\"Point\",\"coordinates\":[-105.01621,39.57422]}";
+ Type type = new TypeToken<Map<String, Object>>() {}.getType();
+ Map<String, Object> shapeObj = new Gson().fromJson(shapeJson, type);
+
+ String parsedShape = this.sut.parseGeoJson(shapeObj);
+ assertNotNull(parsedShape);
+ }
+
+ @Test
+ public void should_throwException_provided_emptyGeoJson() {
+ String shapeJson = "{}";
+ Type type = new TypeToken<Map<String, Object>>() {}.getType();
+ Map<String, Object> shapeObj = new Gson().fromJson(shapeJson, type);
+
+ exceptionRule.expect(IllegalArgumentException.class);
+ exceptionRule.expectMessage("shape type not included");
+
+ this.sut.parseGeoJson(shapeObj);
+ }
+
+ @Test
+ public void should_throwException_parseInvalidPoint() {
+ String shapeJson = "{\"type\":\"Point\",\"coordinates\":[-205.01621,39.57422]}";
+ Type type = new TypeToken<Map<String, Object>>() {}.getType();
+ Map<String, Object> shapeObj = new Gson().fromJson(shapeJson, type);
+
+ exceptionRule.expect(IllegalArgumentException.class);
+ exceptionRule.expectMessage("Bad X value -205.01621 is not in boundary Rect(minX=-180.0,maxX=180.0,minY=-90.0,maxY=90.0)");
+
+ this.sut.parseGeoJson(shapeObj);
+ }
+
+ @Test
+ public void should_throwException_missingMandatoryAttribute() {
+ String shapeJson = "{\"mistype\":\"Point\",\"coordinates\":[-205.01621,39.57422]}";
+ Type type = new TypeToken<Map<String, Object>>() {}.getType();
+ Map<String, Object> shapeObj = new Gson().fromJson(shapeJson, type);
+
+ exceptionRule.expect(IllegalArgumentException.class);
+ exceptionRule.expectMessage("shape type not included");
+
+ this.sut.parseGeoJson(shapeObj);
+ }
+
+ @Test
+ public void should_throwException_parseInvalidShape() {
+ String shapeJson = "{\"type\":\"InvalidShape\",\"coordinates\":[-105.01621,39.57422]}";
+ Type type = new TypeToken<Map<String, Object>>() {}.getType();
+ Map<String, Object> shapeObj = new Gson().fromJson(shapeJson, type);
+
+ exceptionRule.expect(IllegalArgumentException.class);
+ exceptionRule.expectMessage("unknown geo_shape [InvalidShape]");
+
+ this.sut.parseGeoJson(shapeObj);
+ }
+
+ @Test
+ public void should_parseValidPolygon() {
+ String shapeJson = "{\"type\":\"Polygon\",\"coordinates\":[[[100,0],[101,0],[101,1],[100,1],[100,0]]]}";
+ Type type = new TypeToken<Map<String, Object>>() {}.getType();
+ Map<String, Object> shapeObj = new Gson().fromJson(shapeJson, type);
+
+ String parsedShape = this.sut.parseGeoJson(shapeObj);
+ assertNotNull(parsedShape);
+ }
+
+ @Test
+ public void should_parseValidGeometryCollection() {
+ String shapeJson = "{\"type\":\"GeometryCollection\",\"geometries\":[{\"type\":\"Point\",\"coordinates\":[-80.660805,35.049392]},{\"type\":\"Polygon\",\"coordinates\":[[[-80.664582,35.044965],[-80.663874,35.04428],[-80.662586,35.04558],[-80.663444,35.046036],[-80.664582,35.044965]]]},{\"type\":\"LineString\",\"coordinates\":[[-80.662372,35.059509],[-80.662693,35.059263],[-80.662844,35.05893],[-80.66308,35.058332],[-80.663595,35.057753],[-80.663874,35.057401],[-80.66441,35.057033],[-80.664861,35.056787],[-80.665419,35.056506],[-80.665633,35.056312],[-80.666019,35.055891],[-80.666191,35.055452],[-80.666191,35.055171],[-80.666255,35.05489],[-80.666213,35.054222],[-80.666213,35.053924],[-80.665955,35.052905],[-80.665698,35.052044],[-80.665504,35.051482],[-80.665762,35.050481],[-80.66617,35.049725],[-80.666513,35.049286],[-80.666921,35.048531],[-80.667006,35.048215],[-80.667071,35.047775],[-80.667049,35.047389],[-80.666964,35.046985],[-80.666813,35.046353],[-80.666599,35.045966],[-80.666406,35.045615],[-80.665998,35.045193],[-80.665526,35.044877],[-80.664989,35.044543],[-80.664496,35.044174],[-80.663852,35.043876],[-80.663037,35.043717]]}]}";
+ Type type = new TypeToken<Map<String, Object>>() {}.getType();
+ Map<String, Object> shapeObj = new Gson().fromJson(shapeJson, type);
+
+ String parsedShape = this.sut.parseGeoJson(shapeObj);
+ assertNotNull(parsedShape);
+ }
+
+ @Test
+ public void should_throwException_parseUnsupportedType_featureCollection() {
+ String shapeJson = "{\"type\":\"FeatureCollection\",\"features\":[{\"type\":\"Feature\",\"geometry\":{\"type\":\"Polygon\",\"coordinates\":[[[1.9496737127045984,58.41415669686543],[1.8237672363511042,58.42946998193435],[1.8422735102001124,58.471472136376455],[1.9683241046247606,58.45614207250076],[1.9496737127045984,58.41415669686543]]]}}]}";
+ Type type = new TypeToken<Map<String, Object>>() {}.getType();
+ Map<String, Object> shapeObj = new Gson().fromJson(shapeJson, type);
+
+ exceptionRule.expect(IllegalArgumentException.class);
+ exceptionRule.expectMessage("unknown geo_shape [FeatureCollection]");
+
+ this.sut.parseGeoJson(shapeObj);
+ }
+
+ @Test
+ public void should_throwException_parseUnsupportedType_feature() {
+ String shapeJson = "{\"type\":\"Feature\",\"geometry\":{\"type\":\"Polygon\",\"coordinates\":[[[-80.724878,35.265454],[-80.722646,35.260338],[-80.720329,35.260618],[-80.718698,35.260267],[-80.715093,35.260548],[-80.71681,35.255361],[-80.710887,35.255361],[-80.703248,35.265033],[-80.704793,35.268397],[-80.70857,35.268257],[-80.712518,35.270359],[-80.715179,35.267696],[-80.721359,35.267276],[-80.724878,35.265454]]]},\"properties\":{\"name\":\"Plaza Road Park\"}}";
+ Type type = new TypeToken<Map<String, Object>>() {}.getType();
+ Map<String, Object> shapeObj = new Gson().fromJson(shapeJson, type);
+
+ exceptionRule.expect(IllegalArgumentException.class);
+ exceptionRule.expectMessage("unknown geo_shape [Feature]");
+
+ this.sut.parseGeoJson(shapeObj);
+ }
+}
diff --git a/testing/indexer-test-core/src/main/resources/features/indexrecord/IndexRecord.feature b/testing/indexer-test-core/src/main/resources/features/indexrecord/IndexRecord.feature
index e0fffb722..c53c3b551 100644
--- a/testing/indexer-test-core/src/main/resources/features/indexrecord/IndexRecord.feature
+++ b/testing/indexer-test-core/src/main/resources/features/indexrecord/IndexRecord.feature
@@ -7,6 +7,7 @@ Feature: Indexing of the documents
| tenant1:testindex<timestamp>:well:1.0.0 | tenant1-testindex<timestamp>-well-1.0.0 | index_records_1 |
| tenant1:testindex<timestamp>:well:2.0.0 | tenant1-testindex<timestamp>-well-2.0.0 | index_records_2 |
| tenant1:testindex<timestamp>:well:3.0.0 | tenant1-testindex<timestamp>-well-3.0.0 | index_records_1 |
+ | tenant1:testindex<timestamp>:well:4.0.0 | tenant1-testindex<timestamp>-well-4.0.0 | index_records_3 |
Scenario Outline: Ingest the record and Index in the Elastic Search
When I ingest records with the <recordFile> with <acl> for a given <kind>
@@ -23,5 +24,6 @@ Feature: Indexing of the documents
Then I should get the <number> documents for the <index> in the Elastic Search with out <skippedAttribute>
Examples:
- | kind | recordFile | number | index | skippedAttribute | acl |
- | "tenant1:testindex<timestamp>:well:2.0.0" | "index_records_2" | 4 | "tenant1-testindex<timestamp>-well-2.0.0" | "data.Location" | "data.default.viewers@opendes" |
\ No newline at end of file
+ | kind | recordFile | number | index | skippedAttribute | acl |
+ | "tenant1:testindex<timestamp>:well:2.0.0" | "index_records_2" | 4 | "tenant1-testindex<timestamp>-well-2.0.0" | "data.Location" | "data.default.viewers@opendes" |
+ | "tenant1:testindex<timestamp>:well:4.0.0" | "index_records_3" | 7 | "tenant1-testindex<timestamp>-well-4.0.0" | "data.GeoShape" | "data.default.viewers@opendes" |
\ No newline at end of file
diff --git a/testing/indexer-test-core/src/main/resources/testData/index_records_3.json b/testing/indexer-test-core/src/main/resources/testData/index_records_3.json
new file mode 100644
index 000000000..9c35fcd6e
--- /dev/null
+++ b/testing/indexer-test-core/src/main/resources/testData/index_records_3.json
@@ -0,0 +1,113 @@
+[
+ {
+ "id": "tenant1:ihs:testIngest7<timestamp>",
+ "data": {
+ "Field": "OSDU OFFICE - 2",
+ "Location": {
+ "latitude":32.406402588,
+ "longitude":-86.565592762
+ },
+ "Basin": "Houston",
+ "County": "Harris",
+ "State": "TX",
+ "Country": "USA",
+ "WellStatus": "Under development",
+ "OriginalOperator": "OFFICE2",
+ "WellName": "Data Platform Services",
+ "WellType": "Data Lake Cloud",
+ "EmptyAttribute": "",
+ "Rank": 1,
+ "Score" : 10,
+ "Established": "2000-03-27T23:38:48Z"
+ }
+ },
+ {
+ "id": "tenant1:ihs:testIngest8<timestamp>",
+ "data": {
+ "Field": "OSDU OFFICE - 2",
+ "Location": {
+ "latitude":32.406402588,
+ "longitude":-86.565592762
+ },
+ "Basin": "Houston",
+ "County": "Harris",
+ "State": "TX",
+ "Country": "USA",
+ "WellStatus": "Under development",
+ "OriginalOperator": "OFFICE2",
+ "WellName": "Data Platform Services",
+ "WellType": "Data Lake Cloud",
+ "EmptyAttribute": "",
+ "Rank": 1,
+ "Score" : 10,
+ "Established": "2000-03-27T23:38:48Z"
+ }
+ },
+ {
+ "id": "tenant1:ihs:testIngest9<timestamp>",
+ "data": {
+ "Field": "OSDU OFFICE - 2",
+ "Location": {
+ "latitude":32.406402588,
+ "longitude":-86.565592762
+ },
+ "Basin": "Houston",
+ "County": "Harris",
+ "State": "TX",
+ "Country": "USA",
+ "WellStatus": "Under development",
+ "OriginalOperator": "OFFICE2",
+ "WellName": "Data Platform Services",
+ "WellType": "Data Lake Cloud",
+ "EmptyAttribute": "",
+ "Rank": 1,
+ "Score" : 10,
+ "Established": "2000-03-27T23:38:48Z"
+ }
+ },
+ {
+ "id": "tenant1:ihs:testIngest10<timestamp>",
+ "data": {
+ "Field": "OSDU OFFICE - 2",
+ "Location": {
+ "latitude":32.406402588,
+ "longitude":-86.565592762
+ },
+ "Basin": "Houston",
+ "County": "Harris",
+ "State": "TX",
+ "Country": "USA",
+ "WellStatus": "Under development",
+ "OriginalOperator": "OFFICE2",
+ "WellName": "Data Platform Services",
+ "WellType": "Data Lake Cloud",
+ "EmptyAttribute": "",
+ "Rank": 1,
+ "Score" : 10,
+ "Established": "2000-03-27T23:38:48Z"
+ }
+ },
+ {
+ "id": "tenant1:ihs:testIngest11<timestamp>",
+ "data": {
+ "Field": 1234,
+ "Location": {
+ "latitude":"BA1",
+ "longitude":-86.565592762
+ },
+ "Basin": 789,
+ "County": 0.99,
+ "State": 0.56,
+ "Country": 1234,
+ "WellStatus": 528693,
+ "OriginalOperator": 564,
+ "WellName": 0.98,
+ "WellType": 454476578,
+ "EmptyAttribute": 1234,
+ "Rank": "Test",
+ "Score" : 10,
+ "Established": 123456,
+ "InvalidInteger": 2147483648
+ }
+ }
+]
\ No newline at end of file
diff --git a/testing/indexer-test-core/src/main/resources/testData/index_records_3.schema b/testing/indexer-test-core/src/main/resources/testData/index_records_3.schema
new file mode 100644
index 000000000..1a50a1747
--- /dev/null
+++ b/testing/indexer-test-core/src/main/resources/testData/index_records_3.schema
@@ -0,0 +1,69 @@
+{
+ "kind": "KIND_VAL",
+ "schema": [
+ {
+ "path": "Field",
+ "kind": "string"
+ },
+ {
+ "path": "Location",
+ "kind": "core:dl:geopoint:1.0.0"
+ },
+ {
+ "path": "GeoShape",
+ "kind": "core:dl:geoshape:1.0.0"
+ },
+ {
+ "path": "Basin",
+ "kind": "string"
+ },
+ {
+ "path": "County",
+ "kind": "string"
+ },
+ {
+ "path": "State",
+ "kind": "string"
+ },
+ {
+ "path": "Country",
+ "kind": "string"
+ },
+ {
+ "path": "WellStatus",
+ "kind": "string"
+ },
+ {
+ "path": "OriginalOperator",
+ "kind": "string"
+ },
+ {
+ "path": "WellName",
+ "kind": "string"
+ },
+ {
+ "path": "WellType",
+ "kind": "string"
+ },
+ {
+ "path": "EmptyAttribute",
+ "kind": "string"
+ },
+ {
+ "path": "Rank",
+ "kind": "int"
+ },
+ {
+ "path": "Score",
+ "kind": "int"
+ },
+ {
+ "path": "Established",
+ "kind": "datetime"
+ },
+ {
+ "path": "InvalidInteger",
+ "kind": "int"
+ }
+ ]
+}
\ No newline at end of file
--
GitLab