Merged PR 906: Pullback index tag changes from Gitlab

Gitlab pull

Related work items: #353

indexer-core/pom.xml

@@ -16,6 +16,7 @@
 
 	<properties>
 		<commons-beanutils.version>1.9.4</commons-beanutils.version>
+		<osdu.oscorecommon.version>0.6.5</osdu.oscorecommon.version>
 	</properties>
 
 	<dependencies>
@@ -33,6 +34,7 @@
 		<dependency>
 			<groupId>org.opengroup.osdu</groupId>
 			<artifactId>os-core-common</artifactId>
+			<version>${osdu.oscorecommon.version}</version>
 		</dependency>
 
 		<!-- spring boot dependencies -->

indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java

@@ -56,8 +56,10 @@ public class PropertiesProcessor {
     public Stream<Map<String, Object>> processItem(AllOfItem allOfItem) {
         Preconditions.checkNotNull(allOfItem, "allOfItem cannot be null");
 
-        if (Objects.nonNull(allOfItem.getAllOf())) {
-            return allOfItem.getAllOf().stream().flatMap(this::processItem);
+        Stream<Map<String, Object>> ofItems = processOfItems(allOfItem.getAllOf(), allOfItem.getAnyOf(), allOfItem.getOneOf());
+
+        if (Objects.nonNull(ofItems)) {
+            return ofItems;
         }
 
         String ref = allOfItem.getRef();
@@ -89,13 +91,34 @@ public class PropertiesProcessor {
                 new AppException(HttpStatus.SC_NOT_FOUND, "Failed to find definition:" + definitionSubRef,
                         "Unknown definition:" + definitionSubRef));
 
-        if (Objects.nonNull(definition.getAllOf())) {
-            return definition.getAllOf().stream().flatMap(this::processItem);
+        Stream<Map<String, Object>> ofItems =
+                processOfItems(definition.getAllOf(), definition.getAnyOf(), definition.getOneOf());
+
+        if (Objects.nonNull(ofItems)) {
+            return ofItems;
         }
 
         return processProperties(definition.getProperties());
     }
 
+    private Stream<Map<String, Object>> processOfItems(List<AllOfItem> allOf, List<AllOfItem> anyOf, List<AllOfItem> oneOf) {
+        Stream<Map<String, Object>> ofItems = null;
+
+        if (Objects.nonNull(allOf)) {
+            ofItems = allOf.stream().flatMap(this::processItem);
+        }
+
+        if (Objects.nonNull(anyOf)) {
+            ofItems = Stream.concat(Optional.ofNullable(ofItems).orElseGet(Stream::empty), anyOf.stream().flatMap(this::processItem));
+        }
+
+        if (Objects.nonNull(oneOf)) {
+            ofItems = Stream.concat(Optional.ofNullable(ofItems).orElseGet(Stream::empty), oneOf.stream().flatMap(this::processItem));
+        }
+
+        return ofItems;
+    }
+
     public Stream<Map<String, Object>> processProperties(Map<String, TypeProperty> properties){
         return properties.entrySet().stream().flatMap(this::processPropertyEntry);
     }
@@ -118,11 +141,10 @@ public class PropertiesProcessor {
             return Stream.empty();
         }
 
-        if (Objects.nonNull(entry.getValue().getAllOf())) {
-            PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey()
-                    , log, new SchemaConverterPropertiesConfig());
+        Stream<Map<String, Object>> ofItems = processOfItems(entry);
 
-            return entry.getValue().getAllOf().stream().flatMap(propertiesProcessor::processItem);
+        if (Objects.nonNull(ofItems)) {
+            return ofItems;
         }
 
         if (Objects.nonNull(entry.getValue().getProperties())) {
@@ -139,6 +161,35 @@ public class PropertiesProcessor {
         return storageSchemaEntry(getTypeByDefinitionProperty(entry.getValue()), pathPrefixWithDot + entry.getKey());
     }
 
+    private Stream<Map<String, Object>> processOfItems(Map.Entry<String, TypeProperty> entry) {
+        Stream<Map<String, Object>> ofItems = null;
+
+        if (Objects.nonNull(entry.getValue().getAllOf())) {
+            PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey()
+                    , log, new SchemaConverterPropertiesConfig());
+
+            ofItems = entry.getValue().getAllOf().stream().flatMap(propertiesProcessor::processItem);
+        }
+
+        if (Objects.nonNull(entry.getValue().getAnyOf())) {
+            PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey()
+                    , log, new SchemaConverterPropertiesConfig());
+
+            ofItems =  Stream.concat(Optional.ofNullable(ofItems).orElseGet(Stream::empty),
+                    entry.getValue().getAnyOf().stream().flatMap(propertiesProcessor::processItem));
+        }
+
+        if (Objects.nonNull(entry.getValue().getOneOf())) {
+            PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey()
+                    , log, new SchemaConverterPropertiesConfig());
+
+            ofItems = Stream.concat(Optional.ofNullable(ofItems).orElseGet(Stream::empty),
+                    entry.getValue().getOneOf().stream().flatMap(propertiesProcessor::processItem));
+        }
+
+        return ofItems;
+    }
+
     private Stream<Map<String, Object>> storageSchemaEntry(String kind, String path) {
         Preconditions.checkNotNullOrEmpty(kind, "kind cannot be null or empty");
         Preconditions.checkNotNullOrEmpty(path, "path cannot be null or empty");

indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java

@@ -97,6 +97,18 @@ public class SchemaToStorageFormatImpl implements SchemaToStorageFormat {
                             .collect(Collectors.toList()));
                 }
 
+                if (schemaData.getAnyOf() != null) {
+                    storageSchemaItems.addAll(schemaServiceSchema.getProperties().getData().getAnyOf().stream()
+                            .flatMap(propertiesProcessor::processItem)
+                            .collect(Collectors.toList()));
+                }
+
+                if (schemaData.getOneOf() != null) {
+                    storageSchemaItems.addAll(schemaServiceSchema.getProperties().getData().getOneOf().stream()
+                            .flatMap(propertiesProcessor::processItem)
+                            .collect(Collectors.toList()));
+                }
+
                 if (schemaData.getRef() != null) {
                     storageSchemaItems.addAll(propertiesProcessor.processRef(schemaData.getRef())
                             .collect(Collectors.toList()));

indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/readme.md

@@ -299,4 +299,67 @@ Ignored for now (array of references)
 
 ```
 
-\"kind\": \"long\"
\ No newline at end of file
+\"kind\": \"long\"
+
+Processing specifics
+----------------------------------------------------------------------------
+
+allOf, anyOf and oneOf tags are processed at the same way. All internal data(properties) are included into converted schema.
+
+For instance
+```json
+{
+  "definitions": {
+    "wellboreData1": {
+      "properties": {
+        "prop1": {
+          "type": "string"
+        }
+      }
+    },
+    "wellboreData2": {
+      "properties": {
+        "prop2": {
+          "type": "string"
+        }
+      }
+    }
+  },
+  "properties": {
+    "data": {
+      "allOf": [
+        {
+          "anyOf": [
+            {
+              "$ref": "#/definitions/wellboreData1"
+            } ],
+          "oneOf": [
+            {
+              "$ref": "#/definitions/wellboreData2"
+            }
+          ]
+        }
+      ]
+    }
+  }
+}
+
+```
+
+is converted to
+
+```json
+{
+  "kind": "KIND_VAL",
+  "schema": [
+    {
+      "kind": "string",
+      "path": "prop1"
+    },
+    {
+      "kind": "string",
+      "path": "prop2"
+    }
+    ]
+}
+```

indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/AllOfItem.java

@@ -27,4 +27,6 @@ public class AllOfItem {
     private String type;
     private Map<String, TypeProperty> properties;
     private List<AllOfItem> allOf;
+    private List<AllOfItem> oneOf;
+    private List<AllOfItem> anyOf;
 }
\ No newline at end of file

indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/Definition.java

@@ -23,4 +23,6 @@ import java.util.Map;
 public class Definition {
     private Map<String, TypeProperty> properties;
     private List<AllOfItem> allOf;
+    private List<AllOfItem> oneOf;
+    private List<AllOfItem> anyOf;
 }

indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/PropertiesData.java

@@ -23,6 +23,8 @@ import java.util.Map;
 @Data
 public class PropertiesData {
     private List<AllOfItem> allOf;
+    private List<AllOfItem> oneOf;
+    private List<AllOfItem> anyOf;
     @JsonProperty("$ref")
     private String ref;
     private Map<String, TypeProperty> properties;

indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java

@@ -30,4 +30,6 @@ public class TypeProperty {
     private Items items;
     private Map<String, TypeProperty> properties;
     private List<AllOfItem> allOf;
+    private List<AllOfItem> oneOf;
+    private List<AllOfItem> anyOf;
 }

indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java

@@ -208,6 +208,16 @@ public class AttributeParsingServiceImpl implements IAttributeParsingService {
         }
     }
 
+	@Override
+	public void tryParseNested(String recordId, String name, Object value, Map<String, Object> dataMap) {
+		dataMap.put(name,value);
+	}
+
+	@Override
+	public void tryParseObject(String recordId, String name, Object value, Map<String, Object> dataMap) {
+		dataMap.put(name,value);
+	}
+
 
     private List<String> isArrayType(Object attributeVal) {
         try {

indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java

 package org.opengroup.osdu.indexer.service;
 
-import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
-
 import java.util.Map;
 
 public interface IAttributeParsingService {
@@ -26,4 +24,8 @@ public interface IAttributeParsingService {
     void tryParseGeopoint(String recordId, String attributeName, Map<String, Object> storageRecordData, Map<String, Object> dataMap);
 
     void tryParseGeojson(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap);
+
+    void tryParseNested(String recordId, String name, Object value, Map<String, Object> dataMap);
+
+    void tryParseObject(String recordId, String name, Object value, Map<String, Object> dataMap);
 }

indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java

@@ -23,6 +23,7 @@ import org.elasticsearch.client.RestHighLevelClient;
 import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
 import org.opengroup.osdu.core.common.model.http.AppException;
 import org.opengroup.osdu.core.common.model.http.RequestStatus;
+import org.opengroup.osdu.core.common.model.indexer.ElasticType;
 import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
 import org.opengroup.osdu.core.common.model.indexer.OperationType;
 import org.opengroup.osdu.core.common.model.indexer.StorageType;
@@ -47,6 +48,10 @@ import java.util.Map;
 public class IndexSchemaServiceImpl implements IndexSchemaService {
 
     private static final String FLATTENED_SCHEMA = "_flattened";
+    private static final String WELLBORE_MARKER_SET = "WellboreMarkerSet";
+    private static final String MARKERS = "Markers";
+    private static final String WELL_LOG = "WellLog";
+    private static final String CURVES = "Curves";
 
     private final Gson gson = new Gson();
 
@@ -221,6 +226,7 @@ public class IndexSchemaServiceImpl implements IndexSchemaService {
             meta.put(RecordMetaAttribute.TYPE.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.TYPE));
             meta.put(RecordMetaAttribute.ACL.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.ACL));
             meta.put(RecordMetaAttribute.X_ACL.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.X_ACL));
+            meta.put(RecordMetaAttribute.TAGS.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.TAGS));
             meta.put(RecordMetaAttribute.LEGAL.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.LEGAL));
             meta.put(RecordMetaAttribute.ANCESTRY.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.ANCESTRY));
             meta.put(RecordMetaAttribute.INDEX_STATUS.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.INDEX_STATUS));
@@ -228,6 +234,14 @@ public class IndexSchemaServiceImpl implements IndexSchemaService {
             String kind = schemaObj.getKind();
             String type = kind.split(":")[2];
 
+            //TODO temporary fix for https://community.opengroup.org/osdu/platform/system/indexer-service/-/issues/1
+            if(data.get(MARKERS) != null){
+                data.put(MARKERS, ElasticType.NESTED.getValue());
+            }
+            if(data.get(CURVES) != null){
+                data.put(CURVES, ElasticType.NESTED.getValue());
+            }
+
             return IndexSchema.builder().dataSchema(data).metaSchema(meta).kind(kind).type(type).build();
 
         } catch (Exception e) {

indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java

@@ -48,6 +48,7 @@ import org.opengroup.osdu.indexer.util.ElasticClientHandler;
 import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
 import org.apache.commons.beanutils.PropertyUtils;
 import org.apache.commons.beanutils.NestedNullException;
+import org.springframework.context.annotation.Primary;
 import org.springframework.stereotype.Service;
 
 import javax.inject.Inject;
@@ -59,6 +60,7 @@ import java.util.logging.Level;
 import java.util.stream.Collectors;
 
 @Service
+@Primary
 public class IndexerServiceImpl implements IndexerService {
 
     private static final TimeValue BULK_REQUEST_TIMEOUT = TimeValue.timeValueMinutes(1);
@@ -309,6 +311,9 @@ public class IndexerServiceImpl implements IndexerService {
             document.setVersion(storageRecord.getVersion());
             document.setAcl(storageRecord.getAcl());
             document.setLegal(storageRecord.getLegal());
+            if (storageRecord.getTags() != null) {
+                document.setTags(storageRecord.getTags());
+            }
             RecordStatus recordStatus = this.jobStatus.getJobStatusByRecordId(storageRecord.getId());
             if (recordStatus.getIndexProgress().getStatusCode() == 0) {
                 recordStatus.getIndexProgress().setStatusCode(HttpStatus.SC_OK);
@@ -463,6 +468,7 @@ public class IndexerServiceImpl implements IndexerService {
         indexerPayload.put(RecordMetaAttribute.TYPE.getValue(), record.getType());
         indexerPayload.put(RecordMetaAttribute.VERSION.getValue(), record.getVersion());
         indexerPayload.put(RecordMetaAttribute.ACL.getValue(), record.getAcl());
+        indexerPayload.put(RecordMetaAttribute.TAGS.getValue(), record.getTags());
         indexerPayload.put(RecordMetaAttribute.X_ACL.getValue(), Acl.flattenAcl(record.getAcl()));
         indexerPayload.put(RecordMetaAttribute.LEGAL.getValue(), record.getLegal());
         indexerPayload.put(RecordMetaAttribute.INDEX_STATUS.getValue(), record.getIndexProgress());

indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java

@@ -91,7 +91,11 @@ public class StorageIndexerPayloadMapper {
                     this.attributeParsingService.tryParseGeojson(recordId, name, value, dataMap);
                     break;
                 case NESTED:
+                    this.attributeParsingService.tryParseNested(recordId, name, value, dataMap);
+                    break;
                 case OBJECT:
+                    this.attributeParsingService.tryParseObject(recordId, name, value, dataMap);
+                    break;
                 case UNDEFINED:
                     // don't do anything for now
                     break;

indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java

@@ -32,6 +32,8 @@ public class TypeMapper {
 
     private static final Map<String, Object> metaAttributeIndexerType = new HashMap<>();
 
+    private static final String STORAGE_TYPE_OBJECTS = "[]object";
+
     static {
 
         metaAttributeIndexerType.put(RecordMetaAttribute.KIND.getValue(), ElasticType.KEYWORD.getValue());
@@ -41,6 +43,7 @@ public class TypeMapper {
         metaAttributeIndexerType.put(RecordMetaAttribute.VERSION.getValue(), ElasticType.LONG.getValue());
         metaAttributeIndexerType.put(RecordMetaAttribute.X_ACL.getValue(), ElasticType.KEYWORD.getValue());
         metaAttributeIndexerType.put(RecordMetaAttribute.ACL.getValue(), getAclIndexerMapping());
+        metaAttributeIndexerType.put(RecordMetaAttribute.TAGS.getValue(), ElasticType.FLATTENED.getValue());
         metaAttributeIndexerType.put(RecordMetaAttribute.LEGAL.getValue(), getLegalIndexerMapping());
         metaAttributeIndexerType.put(RecordMetaAttribute.ANCESTRY.getValue(), getAncestryIndexerMapping());
         metaAttributeIndexerType.put(RecordMetaAttribute.INDEX_STATUS.getValue(), getIndexStatusMapping());
@@ -63,6 +66,9 @@ public class TypeMapper {
         storageToIndexerType.put(StorageType.DATETIME_ARRAY.getValue(), ElasticType.DATE_ARRAY.getValue());
         storageToIndexerType.put(StorageType.GEO_POINT.getValue(), ElasticType.GEO_POINT.getValue());
         storageToIndexerType.put(StorageType.GEO_SHAPE.getValue(), ElasticType.GEO_SHAPE.getValue());
+
+        //TODO temporary fix for https://community.opengroup.org/osdu/platform/system/indexer-service/-/issues/1
+        storageToIndexerType.put(STORAGE_TYPE_OBJECTS, ElasticType.OBJECT.getValue());
     }
 
     public static String getIndexerType(String storageType) {

indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImplTest.java

@@ -94,6 +94,21 @@ public class SchemaToStorageFormatImplTest {
         testSingleFile("/converter/tags/allOf/indefinitions.json", KIND);
     }
 
+    @Test
+    public void oneOfInDefinitions() {
+        testSingleFile("/converter/tags/oneOf/indefinitions.json", KIND);
+    }
+
+    @Test
+    public void anyOfInDefinitions() {
+        testSingleFile("/converter/tags/anyOf/indefinitions.json", KIND);
+    }
+
+    @Test
+    public void mixAllAnyOneOf() {
+        testSingleFile("/converter/tags/mixAllAnyOneOf/mix.json", KIND);
+    }
+
     @Test
     public void folderPassed() throws URISyntaxException, IOException {
 

indexer-core/src/test/java/org/opengroup/osdu/indexer/service/impl/SchemaProviderImplTest.java

@@ -77,7 +77,7 @@ public class SchemaProviderImplTest {
         Assert.assertEquals("{\n" +
                 "  \"kind\" : \"fake\",\n" +
                 "  \"schema\" : [ ]\n" +
-                "}", schema);
+                "}", schema.replaceAll("\r", ""));
 
     }
 

indexer-core/src/test/resources/converter/integration-tests/index_records_1.schema

@@ -20,7 +20,11 @@
             },
             "County": {
               "type": "string"
-            },
+            }}}],
+      "anyOf": [
+        {
+          "type": "object",
+          "properties": {
             "State": {
               "type": "string"
             },
@@ -41,7 +45,11 @@
             },
             "EmptyAttribute": {
               "type": "string"
-            },
+            }}}],
+      "oneOf": [
+        {
+          "type": "object",
+          "properties": {
             "Rank": {
               "type": "integer"
             },

indexer-core/src/test/resources/converter/tags/anyOf/indefinitions.json

+{
+  "definitions": {
+    "wellboreData1": {
+      "properties": {
+        "prop1": {
+          "type": "string"
+        }
+      }
+    },
+    "wellboreData2": {
+      "anyOf": [
+        {
+          "$ref": "#/definitions/wellboreData1"
+        }
+      ]
+    }
+  },
+  "properties": {
+    "data": {
+      "type": "object",
+      "properties": {
+        "Field": {
+          "$ref": "#/definitions/wellboreData2"
+        }
+      }
+    }
+  }
+}
+

indexer-core/src/test/resources/converter/tags/anyOf/indefinitions.json.res

+{
+  "kind": "KIND_VAL",
+  "schema": [
+    {
+      "kind": "string",
+      "path": "Field.prop1"
+    }
+  ]
+}
\ No newline at end of file

indexer-core/src/test/resources/converter/tags/mixAllAnyOneOf/mix.json

+{
+  "definitions": {
+    "wellboreData1": {
+      "properties": {
+        "prop1": {
+          "type": "string"
+        }
+      }
+    },
+    "wellboreData2": {
+      "properties": {
+        "prop2": {
+          "type": "string"
+        }
+      }
+    },
+    "wellboreData3": {
+      "properties": {
+        "prop3": {
+          "type": "string"
+        }
+      }
+    },
+    "wellboreData4": {
+      "properties": {
+        "prop4": {
+          "type": "string"
+        }
+      }
+    }
+  },
+  "properties": {
+    "data": {
+      "allOf": [
+        {
+          "anyOf": [
+            {
+              "$ref": "#/definitions/wellboreData1"
+            } ],
+          "oneOf": [
+            {
+              "$ref": "#/definitions/wellboreData2"
+            }
+          ]
+        },
+        {
+          "$ref": "#/definitions/wellboreData3"
+        },
+        {
+          "$ref": "#/definitions/wellboreData4"
+        }
+      ]
+    }
+  }
+}
+
+