Merge branch 'gcp-nested-arrays-objects' into 'master'

Nested arrays of objects support

See merge request !114

indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java

@@ -208,6 +208,16 @@ public class AttributeParsingServiceImpl implements IAttributeParsingService {
         }
     }
 
+	@Override
+	public void tryParseNested(String recordId, String name, Object value, Map<String, Object> dataMap) {
+		dataMap.put(name,value);
+	}
+
+	@Override
+	public void tryParseObject(String recordId, String name, Object value, Map<String, Object> dataMap) {
+		dataMap.put(name,value);
+	}
+
 
     private List<String> isArrayType(Object attributeVal) {
         try {

indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java

 package org.opengroup.osdu.indexer.service;
 
-import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
-
 import java.util.Map;
 
 public interface IAttributeParsingService {
@@ -26,4 +24,8 @@ public interface IAttributeParsingService {
     void tryParseGeopoint(String recordId, String attributeName, Map<String, Object> storageRecordData, Map<String, Object> dataMap);
 
     void tryParseGeojson(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap);
+
+    void tryParseNested(String recordId, String name, Object value, Map<String, Object> dataMap);
+
+    void tryParseObject(String recordId, String name, Object value, Map<String, Object> dataMap);
 }

indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java

@@ -23,6 +23,7 @@ import org.elasticsearch.client.RestHighLevelClient;
 import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
 import org.opengroup.osdu.core.common.model.http.AppException;
 import org.opengroup.osdu.core.common.model.http.RequestStatus;
+import org.opengroup.osdu.core.common.model.indexer.ElasticType;
 import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
 import org.opengroup.osdu.core.common.model.indexer.OperationType;
 import org.opengroup.osdu.core.common.model.indexer.StorageType;
@@ -47,6 +48,10 @@ import java.util.Map;
 public class IndexSchemaServiceImpl implements IndexSchemaService {
 
     private static final String FLATTENED_SCHEMA = "_flattened";
+    private static final String WELLBORE_MARKER_SET = "WellboreMarkerSet";
+    private static final String MARKERS = "Markers";
+    private static final String WELL_LOG = "WellLog";
+    private static final String CURVES = "Curves";
 
     private final Gson gson = new Gson();
 
@@ -228,6 +233,14 @@ public class IndexSchemaServiceImpl implements IndexSchemaService {
             String kind = schemaObj.getKind();
             String type = kind.split(":")[2];
 
+            //TODO temporary fix for https://community.opengroup.org/osdu/platform/system/indexer-service/-/issues/1
+            if(data.get(MARKERS) != null){
+                data.put(MARKERS, ElasticType.NESTED.getValue());
+            }
+            if(data.get(CURVES) != null){
+                data.put(CURVES, ElasticType.NESTED.getValue());
+            }
+
             return IndexSchema.builder().dataSchema(data).metaSchema(meta).kind(kind).type(type).build();
 
         } catch (Exception e) {

indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java

@@ -91,7 +91,11 @@ public class StorageIndexerPayloadMapper {
                     this.attributeParsingService.tryParseGeojson(recordId, name, value, dataMap);
                     break;
                 case NESTED:
+                    this.attributeParsingService.tryParseNested(recordId, name, value, dataMap);
+                    break;
                 case OBJECT:
+                    this.attributeParsingService.tryParseObject(recordId, name, value, dataMap);
+                    break;
                 case UNDEFINED:
                     // don't do anything for now
                     break;

indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java

@@ -32,6 +32,8 @@ public class TypeMapper {
 
     private static final Map<String, Object> metaAttributeIndexerType = new HashMap<>();
 
+    private static final String STORAGE_TYPE_OBJECTS = "[]object";
+
     static {
 
         metaAttributeIndexerType.put(RecordMetaAttribute.KIND.getValue(), ElasticType.KEYWORD.getValue());
@@ -63,6 +65,9 @@ public class TypeMapper {
         storageToIndexerType.put(StorageType.DATETIME_ARRAY.getValue(), ElasticType.DATE_ARRAY.getValue());
         storageToIndexerType.put(StorageType.GEO_POINT.getValue(), ElasticType.GEO_POINT.getValue());
         storageToIndexerType.put(StorageType.GEO_SHAPE.getValue(), ElasticType.GEO_SHAPE.getValue());
+
+        //TODO temporary fix for https://community.opengroup.org/osdu/platform/system/indexer-service/-/issues/1
+        storageToIndexerType.put(STORAGE_TYPE_OBJECTS, ElasticType.OBJECT.getValue());
     }
 
     public static String getIndexerType(String storageType) {

provider/indexer-gcp/src/main/resources/application.properties

@@ -43,3 +43,4 @@ indexer.que.service.mail=default@iam.gserviceaccount.com
 SCHEMA_HOST=${HOST}/api/schema-service/v1/schema
 storage-query-kinds-host=https://${STORAGE_HOSTNAME}/api/storage/v2/query/kinds
 
+schema.converter.supported-array-types=boolean,integer,number,string,object