diff --git a/indexer-core/pom.xml b/indexer-core/pom.xml
index 08b4a00a5edbe2f94fac86ab2c2b253dae7ad9f6..ba48b3cc7976a73c3b69b4433ca9110e91f03562 100644
--- a/indexer-core/pom.xml
+++ b/indexer-core/pom.xml
@@ -16,7 +16,7 @@
<properties>
<commons-beanutils.version>1.9.4</commons-beanutils.version>
- <osdu.oscorecommon.version>0.6.9</osdu.oscorecommon.version>
+ <osdu.oscorecommon.version>0.9.0-SNAPSHOT</osdu.oscorecommon.version>
</properties>
<dependencies>
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java
index f58605545479e2b686959a43033fcd146e20b2f0..04a3211fda1a6a91a71fc7cea639951b41b90642 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessor.java
@@ -14,26 +14,34 @@
package org.opengroup.osdu.indexer.schema.converter;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
import org.apache.http.HttpStatus;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.Constants;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.search.Preconditions;
import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterConfig;
import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterPropertiesConfig;
+import org.opengroup.osdu.indexer.schema.converter.exeption.SchemaProcessingException;
import org.opengroup.osdu.indexer.schema.converter.tags.AllOfItem;
import org.opengroup.osdu.indexer.schema.converter.tags.Definition;
import org.opengroup.osdu.indexer.schema.converter.tags.Definitions;
+import org.opengroup.osdu.indexer.schema.converter.tags.Items;
import org.opengroup.osdu.indexer.schema.converter.tags.TypeProperty;
-import java.util.*;
-import java.util.function.Supplier;
-import java.util.stream.Stream;
-
public class PropertiesProcessor {
- private JaxRsDpsLog log;
private SchemaConverterConfig schemaConverterConfig;
+ private static final String TYPE_KEY = "type";
private static final String DEF_PREFIX = "#/definitions/";
private static final String LINK_PREFIX = "^srn";
private static final String LINK_TYPE = "link";
@@ -42,12 +50,13 @@ public class PropertiesProcessor {
private final String pathPrefix;
private final String pathPrefixWithDot;
- public PropertiesProcessor(Definitions definitions, JaxRsDpsLog log, SchemaConverterConfig schemaConverterConfig) {
- this(definitions, null, log, schemaConverterConfig);
+ private final List<String> errors = new LinkedList<>();
+
+ public PropertiesProcessor(Definitions definitions, SchemaConverterConfig schemaConverterConfig) {
+ this(definitions, null, schemaConverterConfig);
}
- public PropertiesProcessor(Definitions definitions, String pathPrefix, JaxRsDpsLog log, SchemaConverterConfig schemaConverterConfig) {
- this.log = log;
+ public PropertiesProcessor(Definitions definitions, String pathPrefix, SchemaConverterConfig schemaConverterConfig) {
this.definitions = definitions;
this.pathPrefix = pathPrefix;
this.pathPrefixWithDot = Objects.isNull(pathPrefix) || pathPrefix.isEmpty() ? "" : pathPrefix + ".";
@@ -55,55 +64,71 @@ public class PropertiesProcessor {
}
public Stream<Map<String, Object>> processItem(AllOfItem allOfItem) {
- Preconditions.checkNotNull(allOfItem, "allOfItem cannot be null");
+ try {
+ Preconditions.checkNotNull(allOfItem, "allOfItem cannot be null");
- Stream<Map<String, Object>> ofItems = processOfItems(allOfItem.getAllOf(), allOfItem.getAnyOf(), allOfItem.getOneOf());
+ Stream<Map<String, Object>> ofItems = processOfItems(allOfItem.getAllOf(), allOfItem.getAnyOf(), allOfItem.getOneOf());
- if (Objects.nonNull(ofItems)) {
- return ofItems;
- }
+ if (Objects.nonNull(ofItems)) {
+ return ofItems;
+ }
- String ref = allOfItem.getRef();
+ String ref = allOfItem.getRef();
- return Objects.isNull(ref) ?
- allOfItem.getProperties().entrySet().stream().flatMap(this::processPropertyEntry) : processRef(ref);
+ return Objects.isNull(ref) ?
+ allOfItem.getProperties().entrySet().stream().flatMap(this::processPropertyEntry) : processRef(ref);
+ } catch (RuntimeException exception) {
+ errors.add(exception.getMessage());
+ return Stream.empty();
+ }
}
public Stream<Map<String, Object>> processRef(String ref) {
Preconditions.checkNotNull(ref, "reference cannot be null");
- if (!ref.contains(DEF_PREFIX)) {
- log.warning("Unknown definition:" + ref);
- return Stream.empty();
- }
+ try {
+ if (!ref.contains(DEF_PREFIX)) {
+ throw new SchemaProcessingException("Unknown definition format, no prefix:" + ref);
+ }
- String definitionSubRef = ref.substring(DEF_PREFIX.length());
+ String definitionSubRef = ref.substring(DEF_PREFIX.length());
- String definitionIdentity = getDefinitionIdentity(definitionSubRef);
+ String definitionIdentity = getDefinitionIdentity(definitionSubRef);
- if (schemaConverterConfig.getSkippedDefinitions().contains(definitionIdentity)) {
- return Stream.empty();
- }
+ if (schemaConverterConfig.getSkippedDefinitions().contains(definitionIdentity)) {
+ return Stream.empty();
+ }
- if (Objects.nonNull(schemaConverterConfig.getSpecialDefinitionsMap().get(definitionIdentity))) {
- return storageSchemaEntry(
- schemaConverterConfig.getSpecialDefinitionsMap().get(definitionIdentity) + getDefinitionColonVersion(definitionSubRef),
- pathPrefix);
- }
+ if (Objects.nonNull(schemaConverterConfig.getSpecialDefinitionsMap().get(definitionIdentity))) {
+ return storageSchemaEntry(
+ schemaConverterConfig.getSpecialDefinitionsMap().get(definitionIdentity) + getDefinitionColonVersion(definitionSubRef),
+ pathPrefix);
+ }
- Definition definition = definitions.getDefinition(definitionSubRef);
- Optional.ofNullable(definition).orElseThrow(() ->
- new AppException(HttpStatus.SC_NOT_FOUND, "Failed to find definition:" + definitionSubRef,
- "Unknown definition:" + definitionSubRef));
+ Definition definition = definitions.getDefinition(definitionSubRef);
+ Optional.ofNullable(definition).orElseThrow(() ->
+ new SchemaProcessingException("Failed to find definition:" + definitionSubRef));
- Stream<Map<String, Object>> ofItems =
- processOfItems(definition.getAllOf(), definition.getAnyOf(), definition.getOneOf());
+ Stream<Map<String, Object>> ofItems =
+ processOfItems(definition.getAllOf(), definition.getAnyOf(), definition.getOneOf());
+
+ if (Objects.nonNull(ofItems)) {
+ return ofItems;
+ }
- if (Objects.nonNull(ofItems)) {
- return ofItems;
+ return processProperties(definition.getProperties());
+ } catch (RuntimeException exception) {
+ errors.add(exception.getMessage());
+ return Stream.empty();
}
+ }
- return processProperties(definition.getProperties());
+ public Stream<Map<String, Object>> processProperties(Map<String, TypeProperty> properties) {
+ return properties.entrySet().stream().flatMap(this::processPropertyEntry);
+ }
+
+ public List<String> getErrors() {
+ return errors;
}
private String getDefinitionIdentity(String definitionSubRef) {
@@ -148,46 +173,89 @@ public class PropertiesProcessor {
return ofItems;
}
- public Stream<Map<String, Object>> processProperties(Map<String, TypeProperty> properties) {
- return properties.entrySet().stream().flatMap(this::processPropertyEntry);
- }
-
private Stream<Map<String, Object>> processPropertyEntry(Map.Entry<String, TypeProperty> entry) {
Preconditions.checkNotNull(entry, "entry cannot be null");
- if ("object".equals(entry.getValue().getType())
- && Objects.isNull(entry.getValue().getItems())
- && Objects.isNull(entry.getValue().getRef())
- && Objects.isNull(entry.getValue().getProperties())) {
- return Stream.empty();
- }
+ try {
+ if ("object".equals(entry.getValue().getType())
+ && Objects.isNull(entry.getValue().getItems())
+ && Objects.isNull(entry.getValue().getRef())
+ && Objects.isNull(entry.getValue().getProperties())) {
+ return Stream.empty();
+ }
+
+ if ("array".equals(entry.getValue().getType())) {
+
+ Items items = entry.getValue().getItems();
+ if(Objects.nonNull(items) && items.isComplexTypeItems()){
+ return processComplexTypeItems(entry, items);
+ }
+
+ if (schemaConverterConfig.getSupportedArrayTypes().contains(entry.getValue().getItems().getType()) && !items.isComplexTypeItems()) {
+ return storageSchemaEntry("[]" + getTypeByDefinitionProperty(entry.getValue()), pathPrefixWithDot + entry.getKey());
+ }
+
+ return Stream.empty();
+ }
+
+ Stream<Map<String, Object>> ofItems = processOfItems(entry);
- if ("array".equals(entry.getValue().getType())) {
- if (schemaConverterConfig.getSupportedArrayTypes().contains(entry.getValue().getItems().getType())) {
- return storageSchemaEntry("[]" + getTypeByDefinitionProperty(entry.getValue()), pathPrefixWithDot + entry.getKey());
+ if (Objects.nonNull(ofItems)) {
+ return ofItems;
}
+ if (Objects.nonNull(entry.getValue().getProperties())) {
+ PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey()
+ , new SchemaConverterPropertiesConfig());
+ Stream<Map<String, Object>> result = entry.getValue().getProperties().entrySet().stream().flatMap(propertiesProcessor::processPropertyEntry);
+ errors.addAll(propertiesProcessor.getErrors());
+ return result;
+ }
+
+ if (Objects.nonNull(entry.getValue().getRef())) {
+ PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions
+ , pathPrefixWithDot + entry.getKey(), new SchemaConverterPropertiesConfig());
+ Stream<Map<String, Object>> refResult = propertiesProcessor.processRef(entry.getValue().getRef());
+ errors.addAll(propertiesProcessor.getErrors());
+ return refResult;
+ }
+
+ return storageSchemaEntry(getTypeByDefinitionProperty(entry.getValue()), pathPrefixWithDot + entry.getKey());
+ } catch (RuntimeException ex) {
+ errors.add(ex.getMessage());
return Stream.empty();
}
- Stream<Map<String, Object>> ofItems = processOfItems(entry);
+ }
- if (Objects.nonNull(ofItems)) {
- return ofItems;
- }
+ private Stream<Map<String, Object>> processComplexTypeItems(Entry<String, TypeProperty> entry, Items items) {
+ Map<String, String> indexHint = entry.getValue().getIndexHint();
- if (Objects.nonNull(entry.getValue().getProperties())) {
- PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey()
- , log, new SchemaConverterPropertiesConfig());
- return entry.getValue().getProperties().entrySet().stream().flatMap(propertiesProcessor::processPropertyEntry);
- }
+ String indexingType = Objects.isNull(indexHint) ?
+ schemaConverterConfig.getDefaultObjectArraysType() :
+ indexHint.getOrDefault(TYPE_KEY,schemaConverterConfig.getDefaultObjectArraysType());
- if (Objects.nonNull(entry.getValue().getRef())) {
- return new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey(), log, new SchemaConverterPropertiesConfig())
- .processRef(entry.getValue().getRef());
- }
+ if(schemaConverterConfig.getProcessedArraysTypes().contains(indexingType)){
+ PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, new SchemaConverterPropertiesConfig());
- return storageSchemaEntry(getTypeByDefinitionProperty(entry.getValue()), pathPrefixWithDot + entry.getKey());
+ Stream<Map<String, Object>> propertiesStream = Stream.empty();
+
+ if(Objects.nonNull(items.getProperties())){
+ propertiesStream = items.getProperties().entrySet().stream().flatMap(propertiesProcessor::processPropertyEntry);
+ }
+ if (Objects.nonNull(items.getRef())){
+ propertiesStream = Stream.concat(propertiesStream, propertiesProcessor.processRef(items.getRef()));
+ }
+ if(Objects.nonNull(items.getAllOf())){
+ propertiesStream = Stream.concat(propertiesStream, items.getAllOf().stream().flatMap(propertiesProcessor::processItem));
+ }
+ return storageSchemaObjectArrayEntry(
+ indexingType,
+ entry.getKey(),
+ propertiesStream);
+ }else {
+ return storageSchemaEntry(indexingType, pathPrefixWithDot + entry.getKey());
+ }
}
private Stream<Map<String, Object>> processOfItems(Map.Entry<String, TypeProperty> entry) {
@@ -195,37 +263,51 @@ public class PropertiesProcessor {
if (Objects.nonNull(entry.getValue().getAllOf())) {
PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey()
- , log, new SchemaConverterPropertiesConfig());
+ , new SchemaConverterPropertiesConfig());
ofItems = entry.getValue().getAllOf().stream().flatMap(propertiesProcessor::processItem);
+ errors.addAll(propertiesProcessor.getErrors());
}
if (Objects.nonNull(entry.getValue().getAnyOf())) {
PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey()
- , log, new SchemaConverterPropertiesConfig());
+ , new SchemaConverterPropertiesConfig());
ofItems = Stream.concat(Optional.ofNullable(ofItems).orElseGet(Stream::empty),
entry.getValue().getAnyOf().stream().flatMap(propertiesProcessor::processItem));
+ errors.addAll(propertiesProcessor.getErrors());
}
if (Objects.nonNull(entry.getValue().getOneOf())) {
PropertiesProcessor propertiesProcessor = new PropertiesProcessor(definitions, pathPrefixWithDot + entry.getKey()
- , log, new SchemaConverterPropertiesConfig());
+ , new SchemaConverterPropertiesConfig());
ofItems = Stream.concat(Optional.ofNullable(ofItems).orElseGet(Stream::empty),
entry.getValue().getOneOf().stream().flatMap(propertiesProcessor::processItem));
+ errors.addAll(propertiesProcessor.getErrors());
}
return ofItems;
}
private Stream<Map<String, Object>> storageSchemaEntry(String kind, String path) {
+ Preconditions.checkNotNullOrEmpty(path, "path cannot be null or empty");
+ Preconditions.checkNotNullOrEmpty(kind, String.format("kind cannot be null or empty for path '%s'", path));
+
+ Map<String, Object> map = new HashMap<>();
+ map.put("kind", kind);
+ map.put("path", path);
+ return Stream.of(map);
+ }
+
+ private Stream<Map<String, Object>> storageSchemaObjectArrayEntry(String kind, String path,Stream<Map<String, Object>> mapStream) {
Preconditions.checkNotNullOrEmpty(kind, "kind cannot be null or empty");
Preconditions.checkNotNullOrEmpty(path, "path cannot be null or empty");
Map<String, Object> map = new HashMap<>();
map.put("kind", kind);
map.put("path", path);
+ map.put(Constants.PROPERTIES,mapStream.collect(Collectors.toList()));
return Stream.of(map);
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java
index f613d9f9c77e433f62a333b4688ad47b8e67a245..2ed1506a4f646548c69f9009aa4f2bb931a9af2c 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java
@@ -16,11 +16,10 @@ package org.opengroup.osdu.indexer.schema.converter;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.search.Preconditions;
import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterConfig;
+import org.opengroup.osdu.indexer.schema.converter.exeption.SchemaProcessingException;
import org.opengroup.osdu.indexer.schema.converter.interfaces.SchemaToStorageFormat;
import org.opengroup.osdu.indexer.schema.converter.tags.PropertiesData;
import org.opengroup.osdu.indexer.schema.converter.tags.SchemaRoot;
@@ -37,7 +36,6 @@ import java.util.stream.Collectors;
public class SchemaToStorageFormatImpl implements SchemaToStorageFormat {
private ObjectMapper objectMapper;
- private JaxRsDpsLog log;
private SchemaConverterConfig schemaConverterConfig;
@Inject
@@ -45,7 +43,6 @@ public class SchemaToStorageFormatImpl implements SchemaToStorageFormat {
Preconditions.checkNotNull(objectMapper, "objectMapper cannot be null");
this.objectMapper = objectMapper;
- this.log = log;
this.schemaConverterConfig = schemaConverterConfig;
}
@@ -68,7 +65,7 @@ public class SchemaToStorageFormatImpl implements SchemaToStorageFormat {
try {
return objectMapper.readValue(schemaServiceFormat, SchemaRoot.class);
} catch (JsonProcessingException e) {
- throw new AppException(HttpStatus.SC_BAD_REQUEST, "Loading shchem error", "Failed to load schema", e);
+ throw new SchemaProcessingException("Failed to parse the schema");
}
}
@@ -76,7 +73,7 @@ public class SchemaToStorageFormatImpl implements SchemaToStorageFormat {
try {
return objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(schemaServiceFormat);
} catch (JsonProcessingException e) {
- throw new AppException(HttpStatus.SC_UNPROCESSABLE_ENTITY, "Saving JSON error", "Failed to save a JSON file", e);
+ throw new SchemaProcessingException("Failed to save the JSON file");
}
}
@@ -84,43 +81,49 @@ public class SchemaToStorageFormatImpl implements SchemaToStorageFormat {
Preconditions.checkNotNull(objectMapper, "schemaServiceSchema cannot be null");
Preconditions.checkNotNullOrEmpty(kind, "kind cannot be null or empty");
- PropertiesProcessor propertiesProcessor = new PropertiesProcessor(schemaServiceSchema.getDefinitions(), log, schemaConverterConfig);
+ PropertiesProcessor propertiesProcessor = new PropertiesProcessor(schemaServiceSchema.getDefinitions(), schemaConverterConfig);
final List<Map<String, Object>> storageSchemaItems = new ArrayList<>();
- if (schemaServiceSchema.getProperties() != null) {
- PropertiesData schemaData = schemaServiceSchema.getProperties().getData();
- if (!Objects.isNull(schemaData)) {
-
- if (schemaData.getAllOf() != null) {
- storageSchemaItems.addAll(schemaServiceSchema.getProperties().getData().getAllOf().stream()
- .flatMap(propertiesProcessor::processItem)
- .collect(Collectors.toList()));
- }
-
- if (schemaData.getAnyOf() != null) {
- storageSchemaItems.addAll(schemaServiceSchema.getProperties().getData().getAnyOf().stream()
- .flatMap(propertiesProcessor::processItem)
- .collect(Collectors.toList()));
- }
-
- if (schemaData.getOneOf() != null) {
- storageSchemaItems.addAll(schemaServiceSchema.getProperties().getData().getOneOf().stream()
- .flatMap(propertiesProcessor::processItem)
- .collect(Collectors.toList()));
- }
-
- if (schemaData.getRef() != null) {
- storageSchemaItems.addAll(propertiesProcessor.processRef(schemaData.getRef())
- .collect(Collectors.toList()));
- }
-
- if (schemaData.getProperties() != null) {
- storageSchemaItems.addAll(propertiesProcessor.processProperties(schemaData.getProperties())
- .collect(Collectors.toList()));
- }
- }
- } else {
- log.warning("Schema doesn't have properties, kind:" + kind);
+ if (schemaServiceSchema.getProperties() == null) {
+ throw new SchemaProcessingException(String.format("Schema doesn't have data section, kind: %s", kind));
+ }
+
+ PropertiesData schemaData = schemaServiceSchema.getProperties().getData();
+ if (Objects.isNull(schemaData)) {
+ throw new SchemaProcessingException(String.format("Schema doesn't have properties section, kind: %s", kind));
+ }
+
+ if (schemaData.getAllOf() != null) {
+ storageSchemaItems.addAll(schemaServiceSchema.getProperties().getData().getAllOf().stream()
+ .flatMap(propertiesProcessor::processItem)
+ .collect(Collectors.toList()));
+ }
+
+ if (schemaData.getAnyOf() != null) {
+ storageSchemaItems.addAll(schemaServiceSchema.getProperties().getData().getAnyOf().stream()
+ .flatMap(propertiesProcessor::processItem)
+ .collect(Collectors.toList()));
+ }
+
+ if (schemaData.getOneOf() != null) {
+ storageSchemaItems.addAll(schemaServiceSchema.getProperties().getData().getOneOf().stream()
+ .flatMap(propertiesProcessor::processItem)
+ .collect(Collectors.toList()));
+ }
+
+ if (schemaData.getRef() != null) {
+ storageSchemaItems.addAll(propertiesProcessor.processRef(schemaData.getRef())
+ .collect(Collectors.toList()));
+ }
+
+ if (schemaData.getProperties() != null) {
+ storageSchemaItems.addAll(propertiesProcessor.processProperties(schemaData.getProperties())
+ .collect(Collectors.toList()));
+ }
+
+ if (!propertiesProcessor.getErrors().isEmpty()) {
+ throw new SchemaProcessingException(String.format("Errors occurred during parsing the schema, kind: %s | errors: %s" ,
+ kind, String.join(",", propertiesProcessor.getErrors())));
}
final Map<String, Object> result = new LinkedHashMap<>();
@@ -129,5 +132,4 @@ public class SchemaToStorageFormatImpl implements SchemaToStorageFormat {
return result;
}
-
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterConfig.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterConfig.java
index 539b2bdfd16b67422e680d7b3b58ac64948937e4..f31c28b663b314295389df432156b5dc1af29548 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterConfig.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterConfig.java
@@ -11,4 +11,6 @@ public interface SchemaConverterConfig {
Set<String> getSupportedArrayTypes();
Map<String, String> getSpecialDefinitionsMap();
Map<String, String> getPrimitiveTypesMap();
+ Set<String> getProcessedArraysTypes();
+ String getDefaultObjectArraysType();
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterPropertiesConfig.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterPropertiesConfig.java
index 9e5eaff440b905d9bce2db41eff1f69f555e4469..e969f8e31970a105e8ca7becf06b8831baaa5b9d 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterPropertiesConfig.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/config/SchemaConverterPropertiesConfig.java
@@ -1,12 +1,15 @@
package org.opengroup.osdu.indexer.schema.converter.config;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
-import java.util.*;
-
@Configuration
@ConfigurationProperties(prefix = "schema.converter")
@Getter
@@ -17,10 +20,13 @@ public class SchemaConverterPropertiesConfig implements SchemaConverterConfig {
private Set<String> supportedArrayTypes = getDefaultSupportedArrayTypes();
private Map<String, String> specialDefinitionsMap = getDefaultSpecialDefinitionsMap();
private Map<String, String> primitiveTypesMap = getDefaultPrimitiveTypesMap();
+ private Set<String> processedArraysTypes = getDefaultArraysTypesForProcessing();
+ private String defaultObjectArraysType = getObjectArraysDefaultType();
+
private Set<String> getDefaultSkippedDefinitions() {
return new HashSet<>(Arrays.asList("AbstractAnyCrsFeatureCollection",
- "anyCrsGeoJsonFeatureCollection"));
+ "anyCrsGeoJsonFeatureCollection"));
}
private Set<String> getDefaultSupportedArrayTypes() {
@@ -51,4 +57,12 @@ public class SchemaConverterPropertiesConfig implements SchemaConverterConfig {
return defaultPrimitiveTypesMap;
}
+
+ private Set<String> getDefaultArraysTypesForProcessing() {
+ return new HashSet<>(Arrays.asList("nested"));
+ }
+
+ private String getObjectArraysDefaultType() {
+ return "[]object";
+ }
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/exeption/SchemaProcessingException.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/exeption/SchemaProcessingException.java
new file mode 100644
index 0000000000000000000000000000000000000000..df4dcf456795f394af448a84da121113eb9a6f9a
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/exeption/SchemaProcessingException.java
@@ -0,0 +1,7 @@
+package org.opengroup.osdu.indexer.schema.converter.exeption;
+
+public class SchemaProcessingException extends RuntimeException {
+ public SchemaProcessingException(final String message) {
+ super(message);
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/readme.md b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/readme.md
index 4c39a276469a7a3b43d7c4caf2d78d940fb70a42..17a4ca99a040170dd656acdd655a55d3b6cd3a06 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/readme.md
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/readme.md
@@ -203,8 +203,68 @@ Is converted to
2. Arrays
-Arrays of complex types are ignored, only following arrays of primitive
-types are supported
+Arrays of complex types by default will be consumed as object type
+```json
+"Markers": {
+"type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "NegativeVerticalDelta"{
+ "description": "The distance vertically below the Marker position that marks the limit of the high confidence range for the Marker pick.",
+ "x-osdu-frame-of-reference": "UOM:length",
+ "type": "number"
+ },
+ .....
+ }
+ }
+}
+
+```
+Without inner objects processing
+```json
+{
+ path = Markers,
+ kind = []object
+}
+```
+
+Processing can be specified with optional "x-osdu-indexing" property
+```json
+"properties": {
+ "Markers": {
+ "x-osdu-indexing": {
+ "type": "nested"
+ },
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "NegativeVerticalDelta"{
+ "description": "The distance vertically below the Marker position that marks the limit of the high confidence range for the Marker pick.",
+ "x-osdu-frame-of-reference": "UOM:length",
+ "type": "number"
+ },
+ .....
+```
+"x-osdu-indexing" property values
+```json
+"nested" , "flattened"
+```
+By default, only "nested" type will lead to inner objects processing
+```json
+{
+path = Markers,
+ kind = nested,
+ properties = [{
+ path = NegativeVerticalDelta,
+ kind = double
+ },
+ .....
+}
+```
+
+Arrays of primitive types are supported
```json
"number", "string", "integer", "boolean"
@@ -370,4 +430,4 @@ is converted to
}
]
}
-```
+```
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/Items.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/Items.java
index dc3ef69e9ef19c7cdfbfebb48ddedc393067f98f..ed1678cf0238ccb89557bcb8fefc01429b6698ab 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/Items.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/Items.java
@@ -14,10 +14,23 @@
package org.opengroup.osdu.indexer.schema.converter.tags;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
import lombok.Data;
@Data
public class Items {
+ @JsonProperty("$ref")
+ private String ref;
+ private List<AllOfItem> allOf;
private String type;
private String pattern;
+ private Map<String, TypeProperty> properties;
+
+ public boolean isComplexTypeItems(){
+ return Objects.nonNull(ref) || Objects.nonNull(allOf) || Objects.nonNull(properties);
+ }
+
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java
index b929bad43f401e97fbc4f5b3ce8fd22344e14a76..d06a47c265e268b2a95322574dd6fcdfecc019ee 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/tags/TypeProperty.java
@@ -15,13 +15,14 @@
package org.opengroup.osdu.indexer.schema.converter.tags;
import com.fasterxml.jackson.annotation.JsonProperty;
-import lombok.Data;
-
import java.util.List;
import java.util.Map;
+import lombok.Data;
@Data
public class TypeProperty {
+ @JsonProperty("x-osdu-indexing")
+ private Map<String, String> indexHint;
private String type;
private String pattern;
private String format;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java
index 733408ab3257219f75899e1e57b6a491abad08d2..66d47186931a086c5cf952da892612da185a6fd5 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AttributeParsingServiceImpl.java
@@ -19,13 +19,18 @@ import com.google.gson.Gson;
import com.google.gson.JsonSyntaxException;
import com.google.gson.internal.LinkedTreeMap;
import com.google.gson.reflect.TypeToken;
+import java.lang.reflect.Array;
+import java.lang.reflect.Type;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.function.BiFunction;
+import javax.inject.Inject;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.model.indexer.ElasticType;
-import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
import org.opengroup.osdu.core.common.model.indexer.IndexingStatus;
import org.opengroup.osdu.core.common.model.indexer.JobStatus;
-import org.opengroup.osdu.core.common.Constants;
-import org.opengroup.osdu.indexer.model.geojson.FeatureCollection;
import org.opengroup.osdu.indexer.util.parser.BooleanParser;
import org.opengroup.osdu.indexer.util.parser.DateTimeParser;
import org.opengroup.osdu.indexer.util.parser.GeoShapeParser;
@@ -33,12 +38,6 @@ import org.opengroup.osdu.indexer.util.parser.NumberParser;
import org.springframework.stereotype.Service;
import org.springframework.web.context.annotation.RequestScope;
-import javax.inject.Inject;
-import java.lang.reflect.Array;
-import java.lang.reflect.Type;
-import java.util.*;
-import java.util.function.BiFunction;
-
@Service
@RequestScope
public class AttributeParsingServiceImpl implements IAttributeParsingService {
@@ -219,6 +218,11 @@ public class AttributeParsingServiceImpl implements IAttributeParsingService {
dataMap.put(name,value);
}
+ @Override
+ public void tryParseFlattened(String recordId, String name, Object value, Map<String, Object> dataMap) {
+ dataMap.put(name,value);
+ }
+
private List<String> isArrayType(Object attributeVal) {
try {
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java
index c4fb8ecf2966c3b354eb15e9fb32be8608554feb..a658cc6a1f289d5140cabb39d94864d4f173ef00 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IAttributeParsingService.java
@@ -28,4 +28,6 @@ public interface IAttributeParsingService {
void tryParseNested(String recordId, String name, Object value, Map<String, Object> dataMap);
void tryParseObject(String recordId, String name, Object value, Map<String, Object> dataMap);
+
+ void tryParseFlattened(String recordId, String name, Object value, Map<String, Object> dataMap);
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
index a63848140947c2a425c69458671733c3beaae684..095b66a84fd8b3174140d0294012e0aa0c025b5e 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
@@ -16,6 +16,12 @@ package org.opengroup.osdu.indexer.service;
import com.google.common.base.Strings;
import com.google.gson.Gson;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URISyntaxException;
+import java.util.HashMap;
+import java.util.Map;
+import javax.inject.Inject;
import org.apache.http.HttpStatus;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchStatusException;
@@ -26,7 +32,6 @@ import org.opengroup.osdu.core.common.model.http.RequestStatus;
import org.opengroup.osdu.core.common.model.indexer.ElasticType;
import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
import org.opengroup.osdu.core.common.model.indexer.OperationType;
-import org.opengroup.osdu.core.common.model.indexer.StorageType;
import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
import org.opengroup.osdu.core.common.model.storage.Schema;
import org.opengroup.osdu.core.common.model.storage.SchemaItem;
@@ -37,13 +42,6 @@ import org.opengroup.osdu.indexer.util.ElasticClientHandler;
import org.opengroup.osdu.indexer.util.TypeMapper;
import org.springframework.stereotype.Service;
-import javax.inject.Inject;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.net.URISyntaxException;
-import java.util.HashMap;
-import java.util.Map;
-
@Service
public class IndexSchemaServiceImpl implements IndexSchemaService {
@@ -204,15 +202,16 @@ public class IndexSchemaServiceImpl implements IndexSchemaService {
if (schemaObj == null) return null;
- Map<String, String> data = new HashMap<>();
+ Map<String, Object> data = new HashMap<>();
Map<String, Object> meta = new HashMap<>();
if (schemaObj.getSchema() != null && schemaObj.getSchema().length > 0) {
for (SchemaItem schemaItem : schemaObj.getSchema()) {
String dataType = schemaItem.getKind();
- String elasticDataType = TypeMapper.getIndexerType(dataType);
- if (elasticDataType == null) {
- elasticDataType = TypeMapper.getIndexerType(StorageType.STRING.getValue());
+ Object elasticDataType = TypeMapper.getIndexerType(dataType, ElasticType.TEXT.getValue());
+ if(schemaItem.getProperties() != null){
+ HashMap<String, Object> propertiesMap = normalizeInnerProperties(schemaItem);
+ elasticDataType = TypeMapper.getObjectsArrayMapping(dataType, propertiesMap);
}
data.put(schemaItem.getPath(), elasticDataType);
}
@@ -234,14 +233,6 @@ public class IndexSchemaServiceImpl implements IndexSchemaService {
String kind = schemaObj.getKind();
String type = kind.split(":")[2];
- //TODO temporary fix for https://community.opengroup.org/osdu/platform/system/indexer-service/-/issues/1
- if(data.get(MARKERS) != null){
- data.put(MARKERS, ElasticType.NESTED.getValue());
- }
- if(data.get(CURVES) != null){
- data.put(CURVES, ElasticType.NESTED.getValue());
- }
-
return IndexSchema.builder().dataSchema(data).metaSchema(meta).kind(kind).type(type).build();
} catch (Exception e) {
@@ -249,4 +240,14 @@ public class IndexSchemaServiceImpl implements IndexSchemaService {
}
}
+ private HashMap<String, Object> normalizeInnerProperties(SchemaItem schemaItem) {
+ HashMap<String, Object> propertiesMap = new HashMap<>();
+ for (SchemaItem propertiesItem : schemaItem.getProperties()) {
+ String propertiesItemKind = propertiesItem.getKind();
+ String propertiesElasticType = TypeMapper.getIndexerType(propertiesItemKind,ElasticType.TEXT.getValue());
+ propertiesMap.put(propertiesItem.getPath(),propertiesElasticType);
+ }
+ return propertiesMap;
+ }
+
}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
index acd87b30e4a909d57832758b5260aec100421d41..b70bb211f3f722d048d930e8c5dae437d1de3072 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
@@ -15,6 +15,12 @@
package org.opengroup.osdu.indexer.service;
import com.google.gson.Gson;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import javax.inject.Inject;
import org.apache.http.HttpStatus;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
@@ -38,13 +44,6 @@ import org.opengroup.osdu.indexer.util.ElasticClientHandler;
import org.opengroup.osdu.indexer.util.TypeMapper;
import org.springframework.stereotype.Service;
-import javax.inject.Inject;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-
@Service
public class IndexerMappingServiceImpl extends MappingServiceImpl implements IndexerMappingService {
@@ -112,7 +111,7 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
// data-source attributes
Map<String, Object> dataMapping = new HashMap<>();
if (schema.getDataSchema() != null) {
- for (Map.Entry<String, String> entry : schema.getDataSchema().entrySet()) {
+ for (Map.Entry<String, Object> entry : schema.getDataSchema().entrySet()) {
dataMapping.put(entry.getKey(), TypeMapper.getDataAttributeIndexerMapping(entry.getValue()));
}
@@ -131,7 +130,6 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements Ind
// don't add dynamic mapping
documentMapping.put("dynamic", false);
-
return documentMapping;
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/impl/SchemaProviderImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SchemaProviderImpl.java
similarity index 86%
rename from indexer-core/src/main/java/org/opengroup/osdu/indexer/service/impl/SchemaProviderImpl.java
rename to indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SchemaProviderImpl.java
index 2323aeecdf9218aa1fa6eef5610d96803100d17c..6f2bf52cca0a5addf8c3c59a297cb548cd9171c6 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/impl/SchemaProviderImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SchemaProviderImpl.java
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package org.opengroup.osdu.indexer.service.impl;
+package org.opengroup.osdu.indexer.service;
import com.google.api.client.http.HttpMethods;
import org.apache.http.HttpStatus;
@@ -22,9 +22,8 @@ import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.HttpResponse;
import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.opengroup.osdu.indexer.schema.converter.exeption.SchemaProcessingException;
import org.opengroup.osdu.indexer.schema.converter.interfaces.SchemaToStorageFormat;
-import org.opengroup.osdu.indexer.service.SchemaService;
-import org.opengroup.osdu.indexer.service.StorageService;
import org.springframework.stereotype.Component;
import javax.inject.Inject;
@@ -59,7 +58,17 @@ public class SchemaProviderImpl implements SchemaService {
@Override
public String getSchema(String kind) throws URISyntaxException, UnsupportedEncodingException {
- String schemaServiceSchema = getFromSchemaService(kind);
+ String schemaServiceSchema;
+
+ try {
+ schemaServiceSchema = getFromSchemaService(kind);
+ } catch (SchemaProcessingException ex) {
+ log.error(ex.getMessage(), ex);
+ return null;
+ } catch (RuntimeException ex) {
+ log.error(String.format("Failed to get the schema from the Schema service, kind: %s | message: %s", kind, ex.getMessage()), ex);
+ return null;
+ }
return Objects.nonNull(schemaServiceSchema) ? schemaServiceSchema : getFromStorageService(kind);
}
@@ -98,5 +107,4 @@ public class SchemaProviderImpl implements SchemaService {
return this.urlFetchService.sendRequest(request);
}
-
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java
index 6d9d8b8cde2bf7664b1a84e2bc7a14c7551e0db3..e550d4e0949cf2afb6458a341713d3b713106558 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java
@@ -1,115 +1,171 @@
package org.opengroup.osdu.indexer.service;
+import static org.opengroup.osdu.indexer.service.IAttributeParsingService.DATA_GEOJSON_TAG;
+import static org.opengroup.osdu.indexer.service.IAttributeParsingService.RECORD_GEOJSON_TAG;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import javax.inject.Inject;
import org.apache.commons.beanutils.NestedNullException;
import org.apache.commons.beanutils.PropertyUtils;
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.core.common.Constants;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.indexer.ElasticType;
import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
+import org.opengroup.osdu.core.common.model.indexer.IndexingStatus;
+import org.opengroup.osdu.core.common.model.indexer.JobStatus;
+import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterConfig;
import org.springframework.stereotype.Component;
-import javax.inject.Inject;
-import java.lang.reflect.InvocationTargetException;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.opengroup.osdu.indexer.service.IAttributeParsingService.DATA_GEOJSON_TAG;
-import static org.opengroup.osdu.indexer.service.IAttributeParsingService.RECORD_GEOJSON_TAG;
-
@Component
public class StorageIndexerPayloadMapper {
- @Inject
- private JaxRsDpsLog log;
- @Inject
- private IAttributeParsingService attributeParsingService;
-
- public Map<String, Object> mapDataPayload(IndexSchema storageSchema, Map<String, Object> storageRecordData, String recordId) {
-
- Map<String, Object> dataMap = new HashMap<>();
-
- if (storageSchema.isDataSchemaMissing()) return dataMap;
-
- // get the key and get the corresponding object from the storageRecord object
- for (Map.Entry<String, String> entry : storageSchema.getDataSchema().entrySet()) {
-
- String name = entry.getKey();
-
- Object value = getPropertyValue(recordId, storageRecordData, name);
-
- ElasticType elasticType = ElasticType.forValue(entry.getValue());
-
- if (value == null && !nullIndexedValueSupported(elasticType)) continue;
-
- switch (elasticType) {
- case KEYWORD:
- case KEYWORD_ARRAY:
- case TEXT:
- case TEXT_ARRAY:
- dataMap.put(name, value);
- break;
- case INTEGER_ARRAY:
- this.attributeParsingService.tryParseValueArray(Integer.class, recordId, name, value, dataMap);
- break;
- case INTEGER:
- this.attributeParsingService.tryParseInteger(recordId, name, value, dataMap);
- break;
- case LONG_ARRAY:
- this.attributeParsingService.tryParseValueArray(Long.class, recordId, name, value, dataMap);
- break;
- case LONG:
- this.attributeParsingService.tryParseLong(recordId, name, value, dataMap);
- break;
- case FLOAT_ARRAY:
- this.attributeParsingService.tryParseValueArray(Float.class, recordId, name, value, dataMap);
- break;
- case FLOAT:
- this.attributeParsingService.tryParseFloat(recordId, name, value, dataMap);
- break;
- case DOUBLE_ARRAY:
- this.attributeParsingService.tryParseValueArray(Double.class, recordId, name, value, dataMap);
- break;
- case DOUBLE:
- this.attributeParsingService.tryParseDouble(recordId, name, value, dataMap);
- break;
- case BOOLEAN_ARRAY:
- this.attributeParsingService.tryParseValueArray(Boolean.class, recordId, name, value, dataMap);
- break;
- case BOOLEAN:
- this.attributeParsingService.tryParseBoolean(recordId, name, value, dataMap);
- break;
- case DATE_ARRAY:
- this.attributeParsingService.tryParseValueArray(Date.class, recordId, name, value, dataMap);
- break;
- case DATE:
- this.attributeParsingService.tryParseDate(recordId, name, value, dataMap);
- break;
- case GEO_POINT:
- this.attributeParsingService.tryParseGeopoint(recordId, name, storageRecordData, dataMap);
- break;
- case GEO_SHAPE:
- this.attributeParsingService.tryParseGeojson(recordId, name, value, dataMap);
- break;
- case NESTED:
- this.attributeParsingService.tryParseNested(recordId, name, value, dataMap);
- break;
- case OBJECT:
- this.attributeParsingService.tryParseObject(recordId, name, value, dataMap);
- break;
- case UNDEFINED:
- // don't do anything for now
- break;
- }
- }
-
- // add these once iterated over the list
- storageSchema.getDataSchema().put(DATA_GEOJSON_TAG, ElasticType.GEO_SHAPE.getValue());
- storageSchema.getDataSchema().remove(RECORD_GEOJSON_TAG);
-
- return dataMap;
- }
-
- private Object getPropertyValue(String recordId, Map<String, Object> storageRecordData, String propertyKey) {
+ @Inject
+ private JaxRsDpsLog log;
+ @Inject
+ private IAttributeParsingService attributeParsingService;
+ @Inject
+ private JobStatus jobStatus;
+ @Inject
+ private SchemaConverterConfig schemaConfig;
+
+ public Map<String, Object> mapDataPayload(IndexSchema storageSchema, Map<String, Object> storageRecordData,
+ String recordId) {
+
+ Map<String, Object> dataCollectorMap = new HashMap<>();
+
+ if (storageSchema.isDataSchemaMissing()) {
+ this.log.warning(String.format("record-id: %s | schema mismatching: %s ", recordId, storageSchema.getKind()));
+ return dataCollectorMap;
+ }
+
+ mapDataPayload(storageSchema.getDataSchema(), storageRecordData, recordId, dataCollectorMap);
+
+ // add these once iterated over the list
+ storageSchema.getDataSchema().put(DATA_GEOJSON_TAG, ElasticType.GEO_SHAPE.getValue());
+ storageSchema.getDataSchema().remove(RECORD_GEOJSON_TAG);
+
+ return dataCollectorMap;
+ }
+
+ private Map<String, Object> mapDataPayload(Map<String, Object> dataSchema, Map<String, Object> storageRecordData,
+ String recordId, Map<String, Object> dataCollectorMap) {
+
+ // get the key and get the corresponding object from the storageRecord object
+ for (Map.Entry<String, Object> entry : dataSchema.entrySet()) {
+ String schemaPropertyName = entry.getKey();
+ Object storageRecordValue = getPropertyValue(recordId, storageRecordData, schemaPropertyName);
+ ElasticType elasticType = defineElasticType(entry.getValue());
+
+ if (Objects.isNull(elasticType)) {
+ this.jobStatus
+ .addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST,
+ String.format("record-id: %s | %s for entry %s", recordId, "Not resolvable elastic type", schemaPropertyName));
+ continue;
+ }
+
+ if (schemaConfig.getProcessedArraysTypes().contains(elasticType.getValue().toLowerCase()) && Objects.nonNull(storageRecordValue)) {
+ processInnerProperties(recordId, dataCollectorMap, entry.getValue(), schemaPropertyName, (List<Map>) storageRecordValue);
+ }
+
+ if (storageRecordValue == null && !nullIndexedValueSupported(elasticType)) {
+ continue;
+ }
+
+ switch (elasticType) {
+ case KEYWORD:
+ case KEYWORD_ARRAY:
+ case TEXT:
+ case TEXT_ARRAY:
+ dataCollectorMap.put(schemaPropertyName, storageRecordValue);
+ break;
+ case INTEGER_ARRAY:
+ this.attributeParsingService.tryParseValueArray(Integer.class, recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
+ break;
+ case INTEGER:
+ this.attributeParsingService.tryParseInteger(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
+ break;
+ case LONG_ARRAY:
+ this.attributeParsingService.tryParseValueArray(Long.class, recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
+ break;
+ case LONG:
+ this.attributeParsingService.tryParseLong(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
+ break;
+ case FLOAT_ARRAY:
+ this.attributeParsingService.tryParseValueArray(Float.class, recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
+ break;
+ case FLOAT:
+ this.attributeParsingService.tryParseFloat(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
+ break;
+ case DOUBLE_ARRAY:
+ this.attributeParsingService.tryParseValueArray(Double.class, recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
+ break;
+ case DOUBLE:
+ this.attributeParsingService.tryParseDouble(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
+ break;
+ case BOOLEAN_ARRAY:
+ this.attributeParsingService.tryParseValueArray(Boolean.class, recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
+ break;
+ case BOOLEAN:
+ this.attributeParsingService.tryParseBoolean(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
+ break;
+ case DATE_ARRAY:
+ this.attributeParsingService.tryParseValueArray(Date.class, recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
+ break;
+ case DATE:
+ this.attributeParsingService.tryParseDate(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
+ break;
+ case GEO_POINT:
+ this.attributeParsingService.tryParseGeopoint(recordId, schemaPropertyName, storageRecordData, dataCollectorMap);
+ break;
+ case GEO_SHAPE:
+ this.attributeParsingService.tryParseGeojson(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
+ break;
+ case FLATTENED:
+ // flattened type inner properties will be added "as is" without parsing as they types not present in schema
+ this.attributeParsingService.tryParseFlattened(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
+ break;
+ case OBJECT:
+ // object type inner properties will be added "as is" without parsing as they types not present in schema
+ this.attributeParsingService.tryParseObject(recordId, schemaPropertyName, storageRecordValue, dataCollectorMap);
+ break;
+ case UNDEFINED:
+ // don't do anything for now
+ break;
+ }
+ }
+
+ return dataCollectorMap;
+ }
+
+ private void processInnerProperties(String recordId, Map<String, Object> dataCollectorMap, Object schemaPropertyWithInnerProperties,
+ String name, List<Map> storageRecordValue) {
+ Map schemaPropertyMap = (Map) schemaPropertyWithInnerProperties;
+ Map innerProperties = (Map) schemaPropertyMap.get(Constants.PROPERTIES);
+ ArrayList<Map> innerPropertiesMappingCollector = new ArrayList<>();
+ storageRecordValue.forEach(recordData -> innerPropertiesMappingCollector.add(mapDataPayload(innerProperties, recordData, recordId, new HashMap<>())));
+ dataCollectorMap.put(name, innerPropertiesMappingCollector);
+ }
+
+ private ElasticType defineElasticType(Object entryValue) {
+ ElasticType elasticType = null;
+ if (entryValue instanceof String) {
+ elasticType = ElasticType.forValue(entryValue.toString());
+ } else if (entryValue instanceof Map) {
+ Map map = (Map) entryValue;
+ elasticType = ElasticType.forValue(map.get(Constants.TYPE).toString());
+ }
+ return elasticType;
+ }
+
+ private Object getPropertyValue(String recordId, Map<String, Object> storageRecordData, String propertyKey) {
try {
// try getting first level property using optimized collection
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/IndexerFilter.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/IndexerFilter.java
new file mode 100644
index 0000000000000000000000000000000000000000..633179c426acd8ed830568b1926821b9c0e77b7d
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/IndexerFilter.java
@@ -0,0 +1,49 @@
+
+package org.opengroup.osdu.indexer.util;
+
+import java.io.IOException;
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletResponse;
+import lombok.extern.java.Log;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Log
+@Component
+public class IndexerFilter implements Filter {
+
+ private final DpsHeaders dpsHeaders;
+
+ @Autowired
+ public IndexerFilter(DpsHeaders dpsHeaders) {
+ this.dpsHeaders = dpsHeaders;
+ }
+
+ @Override
+ public void init(FilterConfig filterConfig) throws ServletException {
+ }
+
+ @Override
+ public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse,
+ FilterChain filterChain)
+ throws IOException, ServletException {
+
+ HttpServletResponse httpResponse = (HttpServletResponse) servletResponse;
+
+ dpsHeaders.addCorrelationIdIfMissing();
+ httpResponse.addHeader(DpsHeaders.CORRELATION_ID, dpsHeaders.getCorrelationId());
+
+ filterChain.doFilter(servletRequest, servletResponse);
+ }
+
+ @Override
+ public void destroy() {
+ }
+
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java
index 416728dc3cb6689e4486e41798f37d74b7e7e627..7141317fa91b646dd22ed35a3746fa5ffefa275b 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java
@@ -14,6 +14,9 @@
package org.opengroup.osdu.indexer.util;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.commons.lang3.StringUtils;
import org.opengroup.osdu.core.common.Constants;
import org.opengroup.osdu.core.common.model.entitlements.AclRole;
import org.opengroup.osdu.core.common.model.indexer.ElasticType;
@@ -21,11 +24,6 @@ import org.opengroup.osdu.core.common.model.indexer.Records;
import org.opengroup.osdu.core.common.model.indexer.StorageType;
import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
-import org.apache.commons.lang3.StringUtils;
-
-import java.util.HashMap;
-import java.util.Map;
-
public class TypeMapper {
private static final Map<String, String> storageToIndexerType = new HashMap<>();
@@ -34,6 +32,10 @@ public class TypeMapper {
private static final String STORAGE_TYPE_OBJECTS = "[]object";
+ private static final String STORAGE_TYPE_NESTED = "nested";
+
+ private static final String STORAGE_TYPE_FLATTENED = "flattened";
+
static {
metaAttributeIndexerType.put(RecordMetaAttribute.KIND.getValue(), ElasticType.KEYWORD.getValue());
@@ -69,10 +71,12 @@ public class TypeMapper {
//TODO temporary fix for https://community.opengroup.org/osdu/platform/system/indexer-service/-/issues/1
storageToIndexerType.put(STORAGE_TYPE_OBJECTS, ElasticType.OBJECT.getValue());
+ storageToIndexerType.put(STORAGE_TYPE_NESTED, ElasticType.NESTED.getValue());
+ storageToIndexerType.put(STORAGE_TYPE_FLATTENED,ElasticType.FLATTENED.getValue());
}
- public static String getIndexerType(String storageType) {
- return storageToIndexerType.getOrDefault(storageType, null);
+ public static String getIndexerType(String storageType, String defaultType) {
+ return storageToIndexerType.getOrDefault(storageType, defaultType);
}
public static Object getIndexerType(RecordMetaAttribute attribute) {
@@ -87,16 +91,29 @@ public class TypeMapper {
return Records.Type.builder().type(metaAttributeIndexerType.get(key).toString()).build();
}
- public static Object getDataAttributeIndexerMapping(String indexerType) {
- if (ElasticType.TEXT.getValue().equalsIgnoreCase(indexerType)) {
+ public static Object getDataAttributeIndexerMapping(Object indexerType) {
+ if (ElasticType.TEXT.getValue().equalsIgnoreCase(indexerType.toString())) {
return getTextIndexerMapping();
}
- if (isArray(indexerType)) {
- return Records.Type.builder().type(getArrayMemberType(indexerType)).build();
+ if (isArray(indexerType.toString())) {
+ return Records.Type.builder().type(getArrayMemberType(indexerType.toString())).build();
+ }
+
+ if(isMap(indexerType)){
+ Map<String,Object> type = (Map<String, Object>) indexerType;
+ Map<String, Object> propertiesMap = (Map<String, Object>) type.get(Constants.PROPERTIES);
+ for (Map.Entry<String,Object> entry : propertiesMap.entrySet()){
+ entry.setValue(Records.Type.builder().type(entry.getValue().toString()).build());
+ }
+ return indexerType;
}
- return Records.Type.builder().type(indexerType).build();
+ return Records.Type.builder().type(indexerType.toString()).build();
+ }
+
+ private static boolean isMap(Object indexerType) {
+ return indexerType instanceof Map;
}
private static boolean isArray(String indexerType) {
@@ -140,6 +157,13 @@ public class TypeMapper {
return ancestryProperties;
}
+ public static Object getObjectsArrayMapping(String dataType, Object properties) {
+ Map<String, Object> nestedMapping = new HashMap<>();
+ nestedMapping.put(Constants.TYPE,storageToIndexerType.getOrDefault(dataType, dataType));
+ nestedMapping.put(Constants.PROPERTIES,properties);
+ return nestedMapping;
+ }
+
private static Object getIndexStatusMapping() {
Map<String, Object> indexStatusMapping = new HashMap<>();
indexStatusMapping.put("statusCode", Records.Type.builder().type(ElasticType.INTEGER.getValue()).build());
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessorTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessorTest.java
index 182bc58d33e4a1f1c491b805c6d1c1fe1dca9eed..fdcfeba5de48702c853597f8dcd5dbc5d642c821 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessorTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/PropertiesProcessorTest.java
@@ -14,14 +14,15 @@
package org.opengroup.osdu.indexer.schema.converter;
+import com.google.common.collect.ImmutableMap;
import org.junit.Test;
import org.mockito.Mockito;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterPropertiesConfig;
import org.opengroup.osdu.indexer.schema.converter.tags.AllOfItem;
import org.opengroup.osdu.indexer.schema.converter.tags.Definition;
import org.opengroup.osdu.indexer.schema.converter.tags.Definitions;
+import org.opengroup.osdu.indexer.schema.converter.tags.Items;
import org.opengroup.osdu.indexer.schema.converter.tags.TypeProperty;
import java.util.LinkedHashMap;
@@ -35,27 +36,29 @@ public class PropertiesProcessorTest {
private static final String PATH = "given_path";
private static final String DEFINITIONS_PREFIX = "#/definitions/";
- @Test(expected = AppException.class)
- public void should_fail_on_unknown_reference_definition() {
- JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class);
+ @Test
+ public void should_fail_on_bad_reference_definition() {
+ PropertiesProcessor propertiesProcessor = new PropertiesProcessor(Mockito.mock(Definitions.class), new SchemaConverterPropertiesConfig());
+ propertiesProcessor.processRef(DEFINITIONS_PREFIX + "unknownDefinition");
+ assertEquals(1, propertiesProcessor.getErrors().size());
+ }
- new PropertiesProcessor(Mockito.mock(Definitions.class), log, new SchemaConverterPropertiesConfig())
- .processRef(DEFINITIONS_PREFIX + "unknownDefinition");
+ @Test
+ public void should_fail_on_wrong_definition_format() {
+ PropertiesProcessor propertiesProcessor = new PropertiesProcessor(Mockito.mock(Definitions.class), new SchemaConverterPropertiesConfig());
+ propertiesProcessor.processRef("unknownDefinition");
+ assertEquals(1, propertiesProcessor.getErrors().size());
}
@Test
public void should_not_process_special_reference() {
- JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class);
-
- assertFalse(new PropertiesProcessor(null, log, new SchemaConverterPropertiesConfig())
+ assertFalse(new PropertiesProcessor(null, new SchemaConverterPropertiesConfig())
.processRef(DEFINITIONS_PREFIX + "a:b:anyCrsGeoJsonFeatureCollection:1.0.0").findAny().isPresent());
}
@Test
public void should_return_special_type() {
- JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class);
-
- String res = new PropertiesProcessor(null, PATH, log, new SchemaConverterPropertiesConfig())
+ String res = new PropertiesProcessor(null, PATH, new SchemaConverterPropertiesConfig())
.processRef(DEFINITIONS_PREFIX + "a:b:core_dl_geopoint:1.0.0").map(Object::toString).reduce("", String::concat);
assertEquals("{path=" + PATH + ", kind=core:dl:geopoint:1.0.0}", res);
}
@@ -79,7 +82,7 @@ public class PropertiesProcessorTest {
String defName = "a:b:defName:1.0.0";
definitions.add(defName, definition);
- String res = new PropertiesProcessor(definitions, PATH, log, new SchemaConverterPropertiesConfig())
+ String res = new PropertiesProcessor(definitions, PATH, new SchemaConverterPropertiesConfig())
.processRef(DEFINITIONS_PREFIX + defName).map(Object::toString).reduce("", String::concat);
assertEquals(res, "{path="+ PATH + "." + propertyName + ", kind=string}");
}
@@ -97,8 +100,113 @@ public class PropertiesProcessorTest {
properties.put(PATH, property);
allOfItem.setProperties(properties);
- String res = new PropertiesProcessor(Mockito.mock(Definitions.class), log, new SchemaConverterPropertiesConfig())
+ String res = new PropertiesProcessor(Mockito.mock(Definitions.class), new SchemaConverterPropertiesConfig())
.processItem(allOfItem).map(Object::toString).reduce("", String::concat);
assertEquals("{path=" + PATH + ", kind=int}", res);
}
+
+ @Test
+ public void should_return_processed_nested_array_items(){
+ JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class);
+
+ Map<String, TypeProperty> itemsProperties = new LinkedHashMap<>();
+ TypeProperty intProperty = new TypeProperty();
+ intProperty.setType("integer");
+ itemsProperties.put(PATH, intProperty);
+
+ Items items = new Items();
+ items.setProperties(itemsProperties);
+
+ TypeProperty arrayProperty = new TypeProperty();
+ arrayProperty.setIndexHint(ImmutableMap.of("type","nested"));
+ arrayProperty.setType("array");
+ arrayProperty.setItems(items);
+
+ Map<String, TypeProperty> allOfItemProperties = new LinkedHashMap<>();
+ allOfItemProperties.put(PATH,arrayProperty);
+
+ AllOfItem allOfItem = new AllOfItem();
+ allOfItem.setProperties(allOfItemProperties);
+
+ String res = new PropertiesProcessor(Mockito.mock(Definitions.class), new SchemaConverterPropertiesConfig())
+ .processItem(allOfItem).map(Object::toString).reduce("", String::concat);
+ assertEquals("{path="+ PATH + ", kind=nested, properties=[{path="+ PATH + ", kind=int}]}",res);
+ }
+
+ @Test
+ public void should_return_flattened_array_without_processing(){
+ JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class);
+
+ Map<String, TypeProperty> itemsProperties = new LinkedHashMap<>();
+ TypeProperty intProperty = new TypeProperty();
+ intProperty.setType("integer");
+ itemsProperties.put(PATH, intProperty);
+
+ Items items = new Items();
+ items.setProperties(itemsProperties);
+
+ TypeProperty arrayProperty = new TypeProperty();
+ arrayProperty.setIndexHint(ImmutableMap.of("type","flattened"));
+ arrayProperty.setType("array");
+ arrayProperty.setItems(items);
+
+ Map<String, TypeProperty> allOfItemProperties = new LinkedHashMap<>();
+ allOfItemProperties.put(PATH,arrayProperty);
+
+ AllOfItem allOfItem = new AllOfItem();
+ allOfItem.setProperties(allOfItemProperties);
+
+ String res = new PropertiesProcessor(Mockito.mock(Definitions.class), new SchemaConverterPropertiesConfig())
+ .processItem(allOfItem).map(Object::toString).reduce("", String::concat);
+ assertEquals("{path="+ PATH + ", kind=flattened}",res);
+ }
+
+ @Test
+ public void should_return_object_array_without_hints_in_schema_without_processing(){
+ JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class);
+
+ Map<String, TypeProperty> itemsProperties = new LinkedHashMap<>();
+ TypeProperty intProperty = new TypeProperty();
+ intProperty.setType("integer");
+ itemsProperties.put(PATH, intProperty);
+
+ Items items = new Items();
+ items.setProperties(itemsProperties);
+
+ TypeProperty arrayProperty = new TypeProperty();
+ arrayProperty.setType("array");
+ arrayProperty.setItems(items);
+
+ Map<String, TypeProperty> allOfItemProperties = new LinkedHashMap<>();
+ allOfItemProperties.put(PATH,arrayProperty);
+
+ AllOfItem allOfItem = new AllOfItem();
+ allOfItem.setProperties(allOfItemProperties);
+
+ String res = new PropertiesProcessor(Mockito.mock(Definitions.class), new SchemaConverterPropertiesConfig())
+ .processItem(allOfItem).map(Object::toString).reduce("", String::concat);
+ assertEquals("{path="+ PATH + ", kind=[]object}",res);
+ }
+
+ @Test
+ public void should_process_not_object_array_type(){
+ JaxRsDpsLog log = Mockito.mock(JaxRsDpsLog.class);
+
+ Items items = new Items();
+ items.setType("integer");
+
+ TypeProperty arrayProperty = new TypeProperty();
+ arrayProperty.setType("array");
+ arrayProperty.setItems(items);
+
+ Map<String, TypeProperty> allOfItemProperties = new LinkedHashMap<>();
+ allOfItemProperties.put(PATH,arrayProperty);
+
+ AllOfItem allOfItem = new AllOfItem();
+ allOfItem.setProperties(allOfItemProperties);
+
+ String res = new PropertiesProcessor(Mockito.mock(Definitions.class), new SchemaConverterPropertiesConfig())
+ .processItem(allOfItem).map(Object::toString).reduce("", String::concat);
+ assertEquals("{path="+ PATH + ", kind=[]int}",res);
+ }
}
\ No newline at end of file
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImplTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImplTest.java
index e9ad441db9e9dd44c18ec2b5b99b5c3791db5461..7947615ebcaac6ce5850a29d87fd4299fe6d2dfb 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImplTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImplTest.java
@@ -20,6 +20,7 @@ import org.junit.Test;
import org.mockito.Mockito;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterPropertiesConfig;
+import org.opengroup.osdu.indexer.schema.converter.exeption.SchemaProcessingException;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder;
@@ -54,6 +55,11 @@ public class SchemaToStorageFormatImplTest {
testSingleFile("/converter/new-definitions-format/colons-sample.json", "osdu:osdu:Wellbore:1.0.0");
}
+ @Test(expected = SchemaProcessingException.class)
+ public void wrongDefinitions() {
+ testSingleFile("/converter/bad-schema/wrong-definitions-and-missed-type.json", KIND);
+ }
+
@Test
public void firstSchemaPassed() {
testSingleFile("/converter/basic/schema.json", "osdu:osdu:Wellbore:1.0.0");
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/impl/SchemaProviderImplTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/SchemaProviderImplTest.java
similarity index 97%
rename from indexer-core/src/test/java/org/opengroup/osdu/indexer/service/impl/SchemaProviderImplTest.java
rename to indexer-core/src/test/java/org/opengroup/osdu/indexer/service/SchemaProviderImplTest.java
index 870f2b76c7178ff6430521f3b90ebf2d39c16d55..9e6da1a355f2057eec76feaaa87ad47451a7ae52 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/impl/SchemaProviderImplTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/SchemaProviderImplTest.java
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.package org.opengroup.osdu.indexer.service.impl;
-package org.opengroup.osdu.indexer.service.impl;
+package org.opengroup.osdu.indexer.service;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -70,7 +70,7 @@ public class SchemaProviderImplTest {
org.opengroup.osdu.core.common.model.http.HttpResponse httpResponse =
mock(org.opengroup.osdu.core.common.model.http.HttpResponse.class);
when(httpResponse.getResponseCode()).thenReturn(HttpStatus.SC_OK);
- when(httpResponse.getBody()).thenReturn("{}");
+ when(httpResponse.getBody()).thenReturn("{ \"properties\" : { \"data\": {} } }");
when(urlFetchService.sendRequest(any())).thenReturn(httpResponse);
String schema = sut.getSchema("fake");
@@ -140,7 +140,7 @@ public class SchemaProviderImplTest {
String kind = "tenant:test:test:1.0.0";
SchemaProviderImpl schemaService = Mockito.mock(SchemaProviderImpl.class);
- PowerMockito.when(schemaService.getSchema(any())).thenCallRealMethod();
+ when(schemaService.getSchema(any())).thenCallRealMethod();
InOrder inOrder = inOrder(schemaService);
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapperTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapperTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..f3a1ac0bf85b49eeee27380bb90010005a0ebf9a
--- /dev/null
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapperTest.java
@@ -0,0 +1,140 @@
+package org.opengroup.osdu.indexer.service;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import com.google.common.collect.ImmutableMap;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.opengroup.osdu.core.common.Constants;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
+import org.opengroup.osdu.core.common.model.indexer.JobStatus;
+import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterPropertiesConfig;
+import org.opengroup.osdu.indexer.util.parser.BooleanParser;
+import org.opengroup.osdu.indexer.util.parser.DateTimeParser;
+import org.opengroup.osdu.indexer.util.parser.GeoShapeParser;
+import org.opengroup.osdu.indexer.util.parser.NumberParser;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.junit4.SpringRunner;
+
+@RunWith(SpringRunner.class)
+@SpringBootTest(classes = {StorageIndexerPayloadMapper.class, AttributeParsingServiceImpl.class, NumberParser.class,
+ BooleanParser.class, DateTimeParser.class,
+ GeoShapeParser.class, GeometryConversionService.class, JobStatus.class, SchemaConverterPropertiesConfig.class,JaxRsDpsLog.class})
+public class StorageIndexerPayloadMapperTest {
+
+ public static final String FIRST_OBJECT_INNER_PROPERTY = "FirstObjectInnerProperty";
+ public static final String SECOND_OBJECT_INNER_PROPERTY = "SecondObjectInnerProperty";
+ public static final String FIRST_OBJECT_TEST_VALUE = "first-object-test-value";
+ public static final String SECOND_OBJECT_TEST_VALUE = "second-object-test-value";
+ public static final String OBJECT_PROPERTY = "ObjectProperty";
+ public static final String NESTED_PROPERTY = "NestedProperty";
+ public static final String FIRST_NESTED_INNER_PROPERTY = "FirstNestedInnerProperty";
+ public static final String SECOND_NESTED_INNER_PROPERTY = "SecondNestedInnerProperty";
+ public static final String FIRST_NESTED_VALUE = "first-nested-value";
+ public static final String SECOND_NESTED_VALUE = "second-nested-value";
+ public static final String FLATTENED_PROPERTY = "FlattenedProperty";
+ public static final String FIRST_FLATTENED_INNER_PROPERTY = "FirstFlattenedInnerProperty";
+ public static final String SECOND_FLATTENED_INNER_PROPERTY = "SecondFlattenedInnerProperty";
+ public static final String FIRST_FLATTENED_TEST_VALUE = "first-flattened-test-value";
+ public static final String SECOND_FLATTENED_TEST_VALUE = "second-flattened-test-value";
+ public static final String RECORD_TEST_ID = "test-id";
+
+ private static IndexSchema indexSchema;
+ private static Map<String, Object> storageRecordData;
+
+ @Autowired
+ private StorageIndexerPayloadMapper payloadMapper;
+
+ @BeforeClass
+ public static void setUp() {
+ HashMap<String, Object> dataMap = new HashMap<>();
+ dataMap.put("TextProperty", "text");
+ dataMap.put("TextArrayProperty", "text_array");
+ dataMap.put("DoubleProperty", "double");
+ dataMap.put(OBJECT_PROPERTY, "object");
+ dataMap.put(FLATTENED_PROPERTY, "flattened");
+ dataMap.put(NESTED_PROPERTY, ImmutableMap.of(
+ Constants.TYPE, "nested",
+ Constants.PROPERTIES, ImmutableMap.of(
+ FIRST_NESTED_INNER_PROPERTY, "text",
+ SECOND_NESTED_INNER_PROPERTY, "double")
+ ));
+ dataMap.put("DateProperty", "date");
+ indexSchema = IndexSchema.builder().kind("kind").type(Constants.TYPE).dataSchema(dataMap).build();
+
+ storageRecordData = new HashMap<>();
+ storageRecordData.put("TextProperty", "Testing");
+ storageRecordData.put("TextArrayProperty", Arrays.asList("test", "test-value"));
+ storageRecordData.put("DoubleProperty", "0.1");
+
+ storageRecordData.put(OBJECT_PROPERTY, Arrays.asList(
+ ImmutableMap.of(FIRST_OBJECT_INNER_PROPERTY, FIRST_OBJECT_TEST_VALUE),
+ ImmutableMap.of(SECOND_OBJECT_INNER_PROPERTY, SECOND_OBJECT_TEST_VALUE)
+ ));
+
+ storageRecordData.put(FLATTENED_PROPERTY, Arrays.asList(
+ ImmutableMap.of(FIRST_FLATTENED_INNER_PROPERTY, FIRST_FLATTENED_TEST_VALUE),
+ ImmutableMap.of(SECOND_FLATTENED_INNER_PROPERTY, SECOND_FLATTENED_TEST_VALUE)
+ ));
+
+ storageRecordData.put(NESTED_PROPERTY, Arrays.asList(
+ ImmutableMap.of(FIRST_NESTED_INNER_PROPERTY, FIRST_NESTED_VALUE, SECOND_NESTED_INNER_PROPERTY,"0.1"),
+ ImmutableMap.of(FIRST_NESTED_INNER_PROPERTY, SECOND_NESTED_VALUE, SECOND_NESTED_INNER_PROPERTY,"0.2")
+ ));
+ storageRecordData.put("DateProperty", "2021-03-02T00:17:20.640Z");
+ }
+
+ @Test
+ public void mapDataPayloadTestNested() {
+ Map<String, Object> stringObjectMap = payloadMapper.mapDataPayload(indexSchema, storageRecordData,
+ RECORD_TEST_ID);
+ Object nestedProperty = stringObjectMap.get(NESTED_PROPERTY);
+
+ assertTrue(nestedProperty instanceof List);
+ List<Map<String, Object>> nestedProperty1 = (List<Map<String, Object>>) nestedProperty;
+ Object firstNestedInnerProperty = nestedProperty1.get(0).get(FIRST_NESTED_INNER_PROPERTY);
+ assertEquals(FIRST_NESTED_VALUE,firstNestedInnerProperty);
+ Object secondNestedInnerProperty = nestedProperty1.get(0).get(SECOND_NESTED_INNER_PROPERTY);
+ assertEquals(0.1,secondNestedInnerProperty);
+ Object firstNestedInnerProperty1 = nestedProperty1.get(1).get(FIRST_NESTED_INNER_PROPERTY);
+ assertEquals(SECOND_NESTED_VALUE,firstNestedInnerProperty1);
+ Object secondNestedInnerProperty1 = nestedProperty1.get(1).get(SECOND_NESTED_INNER_PROPERTY);
+ assertEquals(0.2,secondNestedInnerProperty1);
+ }
+
+ @Test
+ public void mapDataPayloadTestFlattened() {
+ Map<String, Object> stringObjectMap = payloadMapper.mapDataPayload(indexSchema, storageRecordData,
+ RECORD_TEST_ID);
+ Object objectProperty = stringObjectMap.get(FLATTENED_PROPERTY);
+
+ assertTrue(objectProperty instanceof List);
+ List<Map<String, Object>> objectProperties = (List<Map<String, Object>>) objectProperty;
+ Object firstInnerProperty = objectProperties.get(0).get(FIRST_FLATTENED_INNER_PROPERTY);
+ assertEquals(FIRST_FLATTENED_TEST_VALUE,firstInnerProperty);
+ Object secondInnerProperty = objectProperties.get(1).get(SECOND_FLATTENED_INNER_PROPERTY);
+ assertEquals(SECOND_FLATTENED_TEST_VALUE,secondInnerProperty);
+ }
+
+ @Test
+ public void mapDataPayloadTestObject() {
+ Map<String, Object> stringObjectMap = payloadMapper.mapDataPayload(indexSchema, storageRecordData,
+ RECORD_TEST_ID);
+ Object objectProperty = stringObjectMap.get(OBJECT_PROPERTY);
+
+ assertTrue(objectProperty instanceof List);
+ List<Map<String, Object>> objectProperties = (List<Map<String, Object>>) objectProperty;
+ Object firstInnerProperty = objectProperties.get(0).get(FIRST_OBJECT_INNER_PROPERTY);
+ assertEquals(FIRST_OBJECT_TEST_VALUE,firstInnerProperty);
+ Object secondInnerProperty = objectProperties.get(1).get(SECOND_OBJECT_INNER_PROPERTY);
+ assertEquals(SECOND_OBJECT_TEST_VALUE,secondInnerProperty);
+ }
+}
\ No newline at end of file
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/IndexerFilterTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/IndexerFilterTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..9d73609676d11b1c91c8c2c21b9874f2015efe9e
--- /dev/null
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/IndexerFilterTest.java
@@ -0,0 +1,43 @@
+package org.opengroup.osdu.indexer.util;
+
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.MockitoJUnitRunner;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+
+import javax.servlet.FilterChain;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import java.io.IOException;
+
+@RunWith(MockitoJUnitRunner.class)
+public class IndexerFilterTest {
+
+
+ @InjectMocks
+ private IndexerFilter indexerFilter;
+
+ @Mock
+ private DpsHeaders dpsHeaders;
+
+ @Test
+ public void shouldSetCorrectResponseHeaders() throws IOException, ServletException {
+ HttpServletRequest httpServletRequest = Mockito.mock(HttpServletRequest.class);
+ HttpServletResponse httpServletResponse = Mockito.mock(HttpServletResponse.class);
+ FilterChain filterChain = Mockito.mock(FilterChain.class);
+
+ Mockito.when(dpsHeaders.getCorrelationId()).thenReturn("correlation-id-value");
+
+ indexerFilter.doFilter(httpServletRequest, httpServletResponse, filterChain);
+
+ Mockito.verify(httpServletResponse).addHeader("correlation-id", "correlation-id-value");
+ Mockito.verify(filterChain).doFilter(httpServletRequest, httpServletResponse);
+ }
+}
+
diff --git a/indexer-core/src/test/resources/converter/R3-json-schema/Generated/work-product-component/WellLog.1.0.0.json b/indexer-core/src/test/resources/converter/R3-json-schema/Generated/work-product-component/WellLog.1.0.0.json
index a7486367e98cae1ca66612cf7083a4ed872faa90..b2d6ce28ef2525fe813e86d2a2272495c2a380cb 100644
--- a/indexer-core/src/test/resources/converter/R3-json-schema/Generated/work-product-component/WellLog.1.0.0.json
+++ b/indexer-core/src/test/resources/converter/R3-json-schema/Generated/work-product-component/WellLog.1.0.0.json
@@ -113,12 +113,6 @@
},
"data": {
"allOf": [
- {
- "$ref": "../abstract/AbstractWPCGroupType.1.0.0.json"
- },
- {
- "$ref": "../abstract/AbstractWorkProductComponent.1.0.0.json"
- },
{
"type": "object",
"properties": {
diff --git a/indexer-core/src/test/resources/converter/R3-json-schema/Generated/work-product/WorkProduct.1.0.0.json b/indexer-core/src/test/resources/converter/R3-json-schema/Generated/work-product/WorkProduct.1.0.0.json
index bc1795ac985b92a955fe9bd75c8e6c09823d3759..2a098b7fca37874dd9df69837573774d90da1936 100644
--- a/indexer-core/src/test/resources/converter/R3-json-schema/Generated/work-product/WorkProduct.1.0.0.json
+++ b/indexer-core/src/test/resources/converter/R3-json-schema/Generated/work-product/WorkProduct.1.0.0.json
@@ -152,14 +152,6 @@
"type": "string"
}
},
- "SpatialPoint": {
- "description": "A centroid point that reflects the locale of the content of the work product (location of the subject matter).",
- "$ref": "../abstract/AbstractSpatialLocation.1.0.0.json"
- },
- "SpatialArea": {
- "description": "A polygon boundary that reflects the locale of the content of the work product (location of the subject matter).",
- "$ref": "../abstract/AbstractSpatialLocation.1.0.0.json"
- },
"SubmitterName": {
"type": "string",
"description": "Name of the person that first submitted the work product package to OSDU."
diff --git a/indexer-core/src/test/resources/converter/bad-schema/wrong-definitions-and-missed-type.json b/indexer-core/src/test/resources/converter/bad-schema/wrong-definitions-and-missed-type.json
new file mode 100644
index 0000000000000000000000000000000000000000..e321224b08c4322e3d7f14929a29fbd2d7501535
--- /dev/null
+++ b/indexer-core/src/test/resources/converter/bad-schema/wrong-definitions-and-missed-type.json
@@ -0,0 +1,1912 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "title": "Wellbore",
+ "description": "A hole in the ground extending from a point at the earth's surface to the maximum point of penetration.",
+ "type": "object",
+ "properties": {
+ "id": {
+ "description": "Previously called ResourceID or SRN which identifies this OSDU resource object without version.",
+ "title": "Entity ID",
+ "type": "string",
+ "pattern": "^srn:<namespace>:master-data\\/Wellbore:[^:]+$",
+ "example": "srn:<namespace>:master-data/Wellbore:2adac27b-5d84-5bcd-89f2-93ee709c06d9"
+ },
+ "kind": {
+ "description": "The schema identification for the OSDU resource object following the pattern <Namespace>:<Source>:<Type>:<VersionMajor>.<VersionMinor>.<VersionPatch>. The versioning scheme follows the semantic versioning, https://semver.org/.",
+ "title": "Entity Kind",
+ "type": "string",
+ "pattern": "^[A-Za-z0-9-_]+:[A-Za-z0-9-_]+:[A-Za-z0-9-_]+:[0-9]+.[0-9]+.[0-9]+$",
+ "example": "namespace:osdu:Wellbore:2.7.112"
+ },
+ "groupType": {
+ "description": "The OSDU group-type assigned to this resource object.",
+ "title": "Group Type",
+ "const": "master-data"
+ },
+ "version": {
+ "description": "The version number of this OSDU resource; set by the framework.",
+ "title": "Version Number",
+ "type": "integer",
+ "format": "int64",
+ "example": 1562066009929332
+ },
+ "resourceSecurityClassification": {
+ "description": "Classifies the security level of the resource.",
+ "title": "Resource Security Classification",
+ "type": "string",
+ "pattern": "^srn:<namespace>:reference-data\\/ResourceSecurityClassification:[^:]+:[0-9]*$"
+ },
+ "ancestry": {
+ "description": "The links to data, which constitute the inputs.",
+ "title": "Ancestry",
+ "$ref": "#/definitions/opendes:wks:AbstractLegalParentList:1.0.0"
+ },
+ "meta": {
+ "description": "The Frame of Reference meta data section linking the named properties to self-contained definitions.",
+ "title": "Frame of Reference Meta Data",
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/opendes:wks:AbstractMetaItem:1.0.0"
+ }
+ },
+ "source": {
+ "description": "The entity that produced the record, or from which it is received; could be an organization, agency, system, internal team, or individual. For informational purposes only, the list of sources is not governed.",
+ "title": "Data Source",
+ "type": "string"
+ },
+ "existenceKind": {
+ "description": "Where does this data resource sit in the cradle-to-grave span of its existence?",
+ "title": "Existence Kind",
+ "type": "string",
+ "pattern": "^srn:<namespace>:reference-data\\/ExistenceKind:[^:]+:[0-9]*$"
+ },
+ "data": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/AbstractFacility:1.0.0"
+ },
+ {
+ "$ref": "#/definitions/wrong:AbstractFacility:1.0.0"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "WellID": {
+
+ }
+
+ }
+ },
+ {
+ "type": "object",
+ "properties": {
+ "ExtensionProperties": {
+ "type": "object",
+ "properties": {}
+ }
+ }
+ }
+ ]
+ }
+ },
+ "required": [
+ "kind",
+ "acl",
+ "groupType",
+ "legal"
+ ],
+ "additionalProperties": false,
+ "definitions": {
+ "opendes:wks:AbstractAccessControlList:1.0.0": {
+ "title": "Access Control List",
+ "description": "The access control tags associated with this entity. This structure is included by the SystemProperties \"acl\", which is part of all OSDU records. Not extensible.",
+ "type": "object",
+ "properties": {
+ "owners": {
+ "title": "List of Owners",
+ "description": "The list of owners of this data record formatted as an email (core.common.model.storage.validation.ValidationDoc.EMAIL_REGEX).",
+ "type": "array",
+ "items": {
+ "type": "string",
+ "pattern": "^[a-zA-Z0-9_+&*-]+(?:\\.[a-zA-Z0-9_+&*-]+)*@(?:[a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,7}$"
+ }
+ },
+ "viewers": {
+ "title": "List of Viewers",
+ "description": "The list of viewers to which this data record is accessible/visible/discoverable formatted as an email (core.common.model.storage.validation.ValidationDoc.EMAIL_REGEX).",
+ "type": "array",
+ "items": {
+ "type": "string",
+ "pattern": "^[a-zA-Z0-9_+&*-]+(?:\\.[a-zA-Z0-9_+&*-]+)*@(?:[a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,7}$"
+ }
+ }
+ },
+ "required": [
+ "owners",
+ "viewers"
+ ],
+ "additionalProperties": false
+ },
+ "opendes:wks:AbstractLegalTags:1.0.0": {
+ "title": "Legal Meta Data",
+ "description": "Legal meta data like legal tags, relevant other countries, legal status. This structure is included by the SystemProperties \"legal\", which is part of all OSDU records. Not extensible.",
+ "type": "object",
+ "properties": {
+ "legaltags": {
+ "title": "Legal Tags",
+ "description": "The list of legal tags, which resolve to legal properties (like country of origin, export classification code, etc.) and rules with the help of the Compliance Service.",
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "otherRelevantDataCountries": {
+ "title": "Other Relevant Data Countries",
+ "description": "The list of other relevant data countries as an array of two-letter country codes, see https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2.",
+ "type": "array",
+ "items": {
+ "type": "string",
+ "pattern": "^[A-Z]{2}$"
+ }
+ },
+ "status": {
+ "title": "Legal Status",
+ "description": "The legal status. Set by the system after evaluation against the compliance rules associated with the \"legaltags\" using the Compliance Service.",
+ "type": "string",
+ "pattern": "^srn:<namespace>:reference-data\\/LegalStatus:[^:]+:[0-9]*$"
+ }
+ },
+ "required": [
+ "legaltags",
+ "otherRelevantDataCountries"
+ ],
+ "additionalProperties": false
+ },
+ "opendes:wks:AbstractLegalParentList:1.0.0": {
+ "title": "Parent List",
+ "description": "A list of entity IDs in the data ecosystem, which act as legal parents to the current entity. This structure is included by the SystemProperties \"ancestry\", which is part of all OSDU records. Not extensible.",
+ "type": "object",
+ "properties": {
+ "parents": {
+ "description": "An array of none, one or many entity references in the data ecosystem, which identify the source of data in the legal sense. Example: the 'parents' will be queried when e.g. the subscription of source data services is terminated; access to the derivatives is also terminated.",
+ "items": {
+ "type": "string"
+ },
+ "example": [],
+ "title": "Parents",
+ "type": "array"
+ }
+ },
+ "additionalProperties": false
+ },
+ "opendes:wks:AbstractMetaItem:1.0.0": {
+ "title": "Frame of Reference Meta Data Item",
+ "description": "A meta data item, which allows the association of named properties or property values to a Unit/Measurement/CRS/Azimuth/Time context.",
+ "oneOf": [
+ {
+ "title": "FrameOfReferenceUOM",
+ "type": "object",
+ "properties": {
+ "kind": {
+ "title": "UOM Reference Kind",
+ "description": "The kind of reference, 'Unit' for FrameOfReferenceUOM.",
+ "const": "Unit"
+ },
+ "name": {
+ "title": "UOM Unit Symbol",
+ "description": "The unit symbol or name of the unit.",
+ "type": "string",
+ "example": "ft[US]"
+ },
+ "persistableReference": {
+ "title": "UOM Persistable Reference",
+ "description": "The self-contained, persistable reference string uniquely identifying the Unit.",
+ "type": "string",
+ "example": "{\"abcd\":{\"a\":0.0,\"b\":1200.0,\"c\":3937.0,\"d\":0.0},\"symbol\":\"ft[US]\",\"baseMeasurement\":{\"ancestry\":\"L\",\"type\":\"UM\"},\"type\":\"UAD\"}"
+ },
+ "unitOfMeasureID": {
+ "description": "SRN to unit of measure reference.",
+ "type": "string",
+ "pattern": "^srn:<namespace>:reference-data\\/UnitOfMeasure:[^:]+:[0-9]*$",
+ "example": "srn:<namespace>:reference-data/UnitOfMeasure:Energistics_UoM_ftUS:"
+ },
+ "propertyNames": {
+ "title": "UOM Property Names",
+ "description": "The list of property names, to which this meta data item provides Unit context to. Data structures, which come in a single frame of reference, can register the property name, others require a full path like \"Data.StructureA.PropertyB\" to define a unique context.",
+ "type": "array",
+ "example": [
+ "HorizontalDeflection.EastWest",
+ "HorizontalDeflection.NorthSouth"
+ ],
+ "items": {
+ "type": "string"
+ }
+ }
+ },
+ "required": [
+ "kind",
+ "persistableReference"
+ ]
+ },
+ {
+ "title": "FrameOfReferenceCRS",
+ "type": "object",
+ "properties": {
+ "kind": {
+ "title": "CRS Reference Kind",
+ "description": "The kind of reference, constant 'CRS' for FrameOfReferenceCRS.",
+ "const": "CRS"
+ },
+ "name": {
+ "title": "CRS Name",
+ "description": "The name of the CRS.",
+ "type": "string",
+ "example": "NAD27 * OGP-Usa Conus / North Dakota South [32021,15851]"
+ },
+ "persistableReference": {
+ "title": "CRS Persistable Reference",
+ "description": "The self-contained, persistable reference string uniquely identifying the CRS.",
+ "type": "string",
+ "example": "{\"authCode\":{\"auth\":\"EPSG\",\"code\":\"32615\"},\"type\":\"LBC\",\"ver\":\"PE_10_3_1\",\"name\":\"WGS_1984_UTM_Zone_15N\",\"wkt\":\"PROJCS[\\\"WGS_1984_UTM_Zone_15N\\\",GEOGCS[\\\"GCS_WGS_1984\\\",DATUM[\\\"D_WGS_1984\\\",SPHEROID[\\\"WGS_1984\\\",6378137.0,298.257223563]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],PROJECTION[\\\"Transverse_Mercator\\\"],PARAMETER[\\\"False_Easting\\\",500000.0],PARAMETER[\\\"False_Northing\\\",0.0],PARAMETER[\\\"Central_Meridian\\\",-93.0],PARAMETER[\\\"Scale_Factor\\\",0.9996],PARAMETER[\\\"Latitude_Of_Origin\\\",0.0],UNIT[\\\"Meter\\\",1.0],AUTHORITY[\\\"EPSG\\\",32615]]\"}"
+ },
+ "coordinateReferenceSystemID": {
+ "description": "SRN to CRS reference.",
+ "type": "string",
+ "pattern": "^srn:<namespace>:reference-data\\/CoordinateReferenceSystem:[^:]+:[0-9]*$",
+ "example": "srn:<namespace>:reference-data/CoordinateReferenceSystem:EPSG.32615:"
+ },
+ "propertyNames": {
+ "title": "CRS Property Names",
+ "description": "The list of property names, to which this meta data item provides CRS context to. Data structures, which come in a single frame of reference, can register the property name, others require a full path like \"Data.StructureA.PropertyB\" to define a unique context.",
+ "type": "array",
+ "example": [
+ "KickOffPosition.X",
+ "KickOffPosition.Y"
+ ],
+ "items": {
+ "type": "string"
+ }
+ }
+ },
+ "required": [
+ "kind",
+ "persistableReference"
+ ]
+ },
+ {
+ "title": "FrameOfReferenceDateTime",
+ "type": "object",
+ "properties": {
+ "kind": {
+ "title": "DateTime Reference Kind",
+ "description": "The kind of reference, constant 'DateTime', for FrameOfReferenceDateTime.",
+ "const": "DateTime"
+ },
+ "name": {
+ "title": "DateTime Name",
+ "description": "The name of the DateTime format and reference.",
+ "type": "string",
+ "example": "UTC"
+ },
+ "persistableReference": {
+ "title": "DateTime Persistable Reference",
+ "description": "The self-contained, persistable reference string uniquely identifying DateTime reference.",
+ "type": "string",
+ "example": "{\"format\":\"yyyy-MM-ddTHH:mm:ssZ\",\"timeZone\":\"UTC\",\"type\":\"DTM\"}"
+ },
+ "propertyNames": {
+ "title": "DateTime Property Names",
+ "description": "The list of property names, to which this meta data item provides DateTime context to. Data structures, which come in a single frame of reference, can register the property name, others require a full path like \"Data.StructureA.PropertyB\" to define a unique context.",
+ "type": "array",
+ "example": [
+ "Acquisition.StartTime",
+ "Acquisition.EndTime"
+ ],
+ "items": {
+ "type": "string"
+ }
+ }
+ },
+ "required": [
+ "kind",
+ "persistableReference"
+ ]
+ },
+ {
+ "title": "FrameOfReferenceAzimuthReference",
+ "type": "object",
+ "properties": {
+ "kind": {
+ "title": "AzimuthReference Reference Kind",
+ "description": "The kind of reference, constant 'AzimuthReference', for FrameOfReferenceAzimuthReference.",
+ "const": "AzimuthReference"
+ },
+ "name": {
+ "title": "AzimuthReference Name",
+ "description": "The name of the CRS or the symbol/name of the unit.",
+ "type": "string",
+ "example": "TrueNorth"
+ },
+ "persistableReference": {
+ "title": "AzimuthReference Persistable Reference",
+ "description": "The self-contained, persistable reference string uniquely identifying AzimuthReference.",
+ "type": "string",
+ "example": "{\"code\":\"TrueNorth\",\"type\":\"AZR\"}"
+ },
+ "propertyNames": {
+ "title": "AzimuthReference Property Names",
+ "description": "The list of property names, to which this meta data item provides AzimuthReference context to. Data structures, which come in a single frame of reference, can register the property name, others require a full path like \"Data.StructureA.PropertyB\" to define a unique context.",
+ "type": "array",
+ "example": [
+ "Bearing"
+ ],
+ "items": {
+ "type": "string"
+ }
+ }
+ },
+ "required": [
+ "kind",
+ "persistableReference"
+ ]
+ }
+ ]
+ },
+ "opendes:wks:AbstractFacilityOperator:1.0.0": {
+ "title": "AbstractFacilityOperator",
+ "description": "The organisation that was responsible for a facility at some point in time.",
+ "type": "object",
+ "properties": {
+ "FacilityOperatorOrganisationID": {
+ "description": "The company that currently operates, or previously operated the facility",
+ "type": "string",
+ "pattern": "^srn:<namespace>:master-data\\/Organisation:[^:]+:[0-9]*$"
+ },
+ "EffectiveDateTime": {
+ "description": "The date and time at which the facility operator becomes effective.",
+ "type": "string",
+ "format": "date-time"
+ },
+ "TerminationDateTime": {
+ "description": "The date and time at which the facility operator is no longer in effect.",
+ "type": "string",
+ "format": "date-time"
+ }
+ }
+ },
+ "opendes:wks:AbstractAnyCrsFeatureCollection:1.0.0": {
+ "title": "AbstractAnyCrsFeatureCollection",
+ "description": "A schema like GeoJSON FeatureCollection with a non-WGS 84 CRS context; based on https://geojson.org/schema/FeatureCollection.json. Attention: the coordinate order is fixed: Longitude/Easting/Westing/X first, followed by Latitude/Northing/Southing/Y, optionally height as third coordinate.",
+ "type": "object",
+ "required": [
+ "type",
+ "persistableReferenceCRS",
+ "features"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsFeatureCollection"
+ ]
+ },
+ "CoordinateReferenceSystemID": {
+ "title": "Coordinate Reference System ID",
+ "description": "The CRS reference into the CoordinateReferenceSystem catalog.",
+ "type": "string",
+ "pattern": "^srn:<namespace>:reference-data\\/CoordinateReferenceSystem:[^:]+:[0-9]*$",
+ "example": "srn:<namespace>:reference-data/CoordinateReferenceSystem:BoundCRS.SLB.32021.15851:"
+ },
+ "VerticalCoordinateReferenceSystemID": {
+ "title": "Vertical Coordinate Reference System ID",
+ "description": "The explicit VerticalCRS reference into the CoordinateReferenceSystem catalog. This property stays empty for 2D geometries. Absent or empty values for 3D geometries mean the context may be provided by a CompoundCRS in 'CoordinateReferenceSystemID' or implicitly EPSG:5714 MSL height",
+ "type": "string",
+ "pattern": "^srn:<namespace>:reference-data\\/CoordinateReferenceSystem:[^:]+:[0-9]*$",
+ "example": "srn:<namespace>:reference-data/CoordinateReferenceSystem:VerticalCRS.EPSG.5773:"
+ },
+ "persistableReferenceCRS": {
+ "type": "string",
+ "title": "CRS Reference",
+ "description": "The CRS reference as persistableReference string. If populated, the CoordinateReferenceSystemID takes precedence.",
+ "example": "{\"lateBoundCRS\":{\"wkt\":\"PROJCS[\\\"NAD_1927_StatePlane_North_Dakota_South_FIPS_3302\\\",GEOGCS[\\\"GCS_North_American_1927\\\",DATUM[\\\"D_North_American_1927\\\",SPHEROID[\\\"Clarke_1866\\\",6378206.4,294.9786982]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],PROJECTION[\\\"Lambert_Conformal_Conic\\\"],PARAMETER[\\\"False_Easting\\\",2000000.0],PARAMETER[\\\"False_Northing\\\",0.0],PARAMETER[\\\"Central_Meridian\\\",-100.5],PARAMETER[\\\"Standard_Parallel_1\\\",46.1833333333333],PARAMETER[\\\"Standard_Parallel_2\\\",47.4833333333333],PARAMETER[\\\"Latitude_Of_Origin\\\",45.6666666666667],UNIT[\\\"Foot_US\\\",0.304800609601219],AUTHORITY[\\\"EPSG\\\",32021]]\",\"ver\":\"PE_10_3_1\",\"name\":\"NAD_1927_StatePlane_North_Dakota_South_FIPS_3302\",\"authCode\":{\"auth\":\"EPSG\",\"code\":\"32021\"},\"type\":\"LBC\"},\"singleCT\":{\"wkt\":\"GEOGTRAN[\\\"NAD_1927_To_WGS_1984_79_CONUS\\\",GEOGCS[\\\"GCS_North_American_1927\\\",DATUM[\\\"D_North_American_1927\\\",SPHEROID[\\\"Clarke_1866\\\",6378206.4,294.9786982]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],GEOGCS[\\\"GCS_WGS_1984\\\",DATUM[\\\"D_WGS_1984\\\",SPHEROID[\\\"WGS_1984\\\",6378137.0,298.257223563]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],METHOD[\\\"NADCON\\\"],PARAMETER[\\\"Dataset_conus\\\",0.0],AUTHORITY[\\\"EPSG\\\",15851]]\",\"ver\":\"PE_10_3_1\",\"name\":\"NAD_1927_To_WGS_1984_79_CONUS\",\"authCode\":{\"auth\":\"EPSG\",\"code\":\"15851\"},\"type\":\"ST\"},\"ver\":\"PE_10_3_1\",\"name\":\"NAD27 * OGP-Usa Conus / North Dakota South [32021,15851]\",\"authCode\":{\"auth\":\"SLB\",\"code\":\"32021079\"},\"type\":\"EBC\"}"
+ },
+ "persistableReferenceVerticalCRS": {
+ "type": "string",
+ "title": "Vertical CRS Reference",
+ "description": "The VerticalCRS reference as persistableReference string. If populated, the VerticalCoordinateReferenceSystemID takes precedence. The property is null or empty for 2D geometries. For 3D geometries and absent or null persistableReferenceVerticalCRS the vertical CRS is either provided via persistableReferenceCRS's CompoundCRS or it is implicitly defined as EPSG:5714 MSL height.",
+ "example": "{\"authCode\":{\"auth\":\"EPSG\",\"code\":\"5773\"},\"type\":\"LBC\",\"ver\":\"PE_10_3_1\",\"name\":\"EGM96_Geoid\",\"wkt\":\"VERTCS[\\\"EGM96_Geoid\\\",VDATUM[\\\"EGM96_Geoid\\\"],PARAMETER[\\\"Vertical_Shift\\\",0.0],PARAMETER[\\\"Direction\\\",1.0],UNIT[\\\"Meter\\\",1.0],AUTHORITY[\\\"EPSG\\\",5773]]\"}"
+ },
+ "persistableReferenceUnitZ": {
+ "type": "string",
+ "title": "Z-Unit Reference",
+ "description": "The unit of measure for the Z-axis (only for 3-dimensional coordinates, where the CRS does not describe the vertical unit). Note that the direction is upwards positive, i.e. Z means height.",
+ "example": "{\"scaleOffset\":{\"scale\":1.0,\"offset\":0.0},\"symbol\":\"m\",\"baseMeasurement\":{\"ancestry\":\"Length\",\"type\":\"UM\"},\"type\":\"USO\"}"
+ },
+ "features": {
+ "type": "array",
+ "items": {
+ "title": "AnyCrsGeoJSON Feature",
+ "type": "object",
+ "required": [
+ "type",
+ "properties",
+ "geometry"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsFeature"
+ ]
+ },
+ "properties": {
+ "oneOf": [
+ {
+ "type": "null"
+ },
+ {
+ "type": "object"
+ }
+ ]
+ },
+ "geometry": {
+ "oneOf": [
+ {
+ "type": "null"
+ },
+ {
+ "title": "AnyCrsGeoJSON Point",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsPoint"
+ ]
+ },
+ "coordinates": {
+ "type": "array",
+ "minItems": 2,
+ "items": {
+ "type": "number"
+ }
+ },
+ "bbox": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "AnyCrsGeoJSON LineString",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsLineString"
+ ]
+ },
+ "coordinates": {
+ "type": "array",
+ "minItems": 2,
+ "items": {
+ "type": "array",
+ "minItems": 2,
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "bbox": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "AnyCrsGeoJSON Polygon",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsPolygon"
+ ]
+ },
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "array",
+ "minItems": 2,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "bbox": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "AnyCrsGeoJSON MultiPoint",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsMultiPoint"
+ ]
+ },
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "minItems": 2,
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "bbox": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "AnyCrsGeoJSON MultiLineString",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsMultiLineString"
+ ]
+ },
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "minItems": 2,
+ "items": {
+ "type": "array",
+ "minItems": 2,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "bbox": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "AnyCrsGeoJSON MultiPolygon",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsMultiPolygon"
+ ]
+ },
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "array",
+ "minItems": 2,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ },
+ "bbox": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "AnyCrsGeoJSON GeometryCollection",
+ "type": "object",
+ "required": [
+ "type",
+ "geometries"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsGeometryCollection"
+ ]
+ },
+ "geometries": {
+ "type": "array",
+ "items": {
+ "oneOf": [
+ {
+ "title": "AnyCrsGeoJSON Point",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsPoint"
+ ]
+ },
+ "coordinates": {
+ "type": "array",
+ "minItems": 2,
+ "items": {
+ "type": "number"
+ }
+ },
+ "bbox": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "AnyCrsGeoJSON LineString",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsLineString"
+ ]
+ },
+ "coordinates": {
+ "type": "array",
+ "minItems": 2,
+ "items": {
+ "type": "array",
+ "minItems": 2,
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "bbox": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "AnyCrsGeoJSON Polygon",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsPolygon"
+ ]
+ },
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "array",
+ "minItems": 2,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "bbox": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "AnyCrsGeoJSON MultiPoint",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsMultiPoint"
+ ]
+ },
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "minItems": 2,
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "bbox": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "AnyCrsGeoJSON MultiLineString",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsMultiLineString"
+ ]
+ },
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "minItems": 2,
+ "items": {
+ "type": "array",
+ "minItems": 2,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "bbox": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "AnyCrsGeoJSON MultiPolygon",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsMultiPolygon"
+ ]
+ },
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "array",
+ "minItems": 2,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ },
+ "bbox": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ ]
+ }
+ },
+ "bbox": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ ]
+ },
+ "bbox": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ },
+ "bbox": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "opendes:wks:AbstractFeatureCollection:1.0.0": {
+ "description": "GeoJSON feature collection as originally published in https://geojson.org/schema/FeatureCollection.json. Attention: the coordinate order is fixed: Longitude first, followed by Latitude, optionally height above MSL (EPSG:5714) as third coordinate.",
+ "title": "GeoJSON FeatureCollection",
+ "type": "object",
+ "required": [
+ "type",
+ "features"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "FeatureCollection"
+ ]
+ },
+ "features": {
+ "type": "array",
+ "items": {
+ "title": "GeoJSON Feature",
+ "type": "object",
+ "required": [
+ "type",
+ "properties",
+ "geometry"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "Feature"
+ ]
+ },
+ "properties": {
+ "oneOf": [
+ {
+ "type": "null"
+ },
+ {
+ "type": "object"
+ }
+ ]
+ },
+ "geometry": {
+ "oneOf": [
+ {
+ "type": "null"
+ },
+ {
+ "title": "GeoJSON Point",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "Point"
+ ]
+ },
+ "coordinates": {
+ "type": "array",
+ "minItems": 2,
+ "items": {
+ "type": "number"
+ }
+ },
+ "bbox": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "GeoJSON LineString",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "LineString"
+ ]
+ },
+ "coordinates": {
+ "type": "array",
+ "minItems": 2,
+ "items": {
+ "type": "array",
+ "minItems": 2,
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "bbox": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "GeoJSON Polygon",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "Polygon"
+ ]
+ },
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "array",
+ "minItems": 2,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "bbox": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "GeoJSON MultiPoint",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "MultiPoint"
+ ]
+ },
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "minItems": 2,
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "bbox": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "GeoJSON MultiLineString",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "MultiLineString"
+ ]
+ },
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "minItems": 2,
+ "items": {
+ "type": "array",
+ "minItems": 2,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "bbox": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "GeoJSON MultiPolygon",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "MultiPolygon"
+ ]
+ },
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "array",
+ "minItems": 2,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ },
+ "bbox": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "GeoJSON GeometryCollection",
+ "type": "object",
+ "required": [
+ "type",
+ "geometries"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "GeometryCollection"
+ ]
+ },
+ "geometries": {
+ "type": "array",
+ "items": {
+ "oneOf": [
+ {
+ "title": "GeoJSON Point",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "Point"
+ ]
+ },
+ "coordinates": {
+ "type": "array",
+ "minItems": 2,
+ "items": {
+ "type": "number"
+ }
+ },
+ "bbox": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "GeoJSON LineString",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "LineString"
+ ]
+ },
+ "coordinates": {
+ "type": "array",
+ "minItems": 2,
+ "items": {
+ "type": "array",
+ "minItems": 2,
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "bbox": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "GeoJSON Polygon",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "Polygon"
+ ]
+ },
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "array",
+ "minItems": 2,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "bbox": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "GeoJSON MultiPoint",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "MultiPoint"
+ ]
+ },
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "minItems": 2,
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "bbox": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "GeoJSON MultiLineString",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "MultiLineString"
+ ]
+ },
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "minItems": 2,
+ "items": {
+ "type": "array",
+ "minItems": 2,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "bbox": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "GeoJSON MultiPolygon",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "MultiPolygon"
+ ]
+ },
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "array",
+ "minItems": 2,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ },
+ "bbox": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ ]
+ }
+ },
+ "bbox": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ ]
+ },
+ "bbox": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ },
+ "bbox": {
+ "type": "array",
+ "minItems": 4,
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "opendes:wks:AbstractSpatialLocation:1.0.0": {
+ "title": "AbstractSpatialLocation",
+ "description": "A geographic object which can be described by a set of points.",
+ "type": "object",
+ "properties": {
+ "SpatialLocationCoordinatesDate": {
+ "description": "Date when coordinates were measured or retrieved.",
+ "type": "string",
+ "format": "date-time"
+ },
+ "QuantitativeAccuracyBandID": {
+ "description": "An approximate quantitative assessment of the quality of a location (accurate to > 500 m (i.e. not very accurate)), to < 1 m, etc.",
+ "type": "string",
+ "pattern": "^srn:<namespace>:reference-data\\/QuantitativeAccuracyBand:[^:]+:[0-9]*$"
+ },
+ "QualitativeSpatialAccuracyTypeID": {
+ "description": "A qualitative description of the quality of a spatial location, e.g. unverifiable, not verified, basic validation.",
+ "type": "string",
+ "pattern": "^srn:<namespace>:reference-data\\/QualitativeSpatialAccuracyType:[^:]+:[0-9]*$"
+ },
+ "CoordinateQualityCheckPerformedBy": {
+ "description": "The user who performed the Quality Check.",
+ "type": "string"
+ },
+ "CoordinateQualityCheckDateTime": {
+ "description": "The date of the Quality Check.",
+ "type": "string",
+ "format": "date-time"
+ },
+ "CoordinateQualityCheckRemarks": {
+ "description": "Freetext remarks on Quality Check.",
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "AsIngestedCoordinates": {
+ "title": "As Ingested Coordinates",
+ "description": "The original or 'as ingested' coordinates (Point, MultiPoint, LineString, MultiLineString, Polygon or MultiPolygon). The name 'AsIngestedCoordinates' was chosen to contrast it to 'OriginalCoordinates', which carries the uncertainty whether any coordinate operations took place before ingestion. In cases where the original CRS is different from the as-ingested CRS, the OperationsApplied can also contain the list of operations applied to the coordinate prior to ingestion. The data structure is similar to GeoJSON FeatureCollection, however in a CRS context explicitly defined within the AbstractAnyCrsFeatureCollection. The coordinate sequence follows GeoJSON standard, i.e. 'eastward/longitude', 'northward/latitude' {, 'upward/height' unless overridden by an explicit direction in the AsIngestedCoordinates.VerticalCoordinateReferenceSystemID}.",
+ "$ref": "#/definitions/opendes:wks:AbstractAnyCrsFeatureCollection:1.0.0",
+ "x-osdu-frame-of-reference": "CRS:"
+ },
+ "Wgs84Coordinates": {
+ "title": "WGS 84 Coordinates",
+ "description": "The normalized coordinates (Point, MultiPoint, LineString, MultiLineString, Polygon or MultiPolygon) based on WGS 84 (EPSG:4326 for 2-dimensional coordinates, EPSG:4326 + EPSG:5714 (MSL) for 3-dimensional coordinates). This derived coordinate representation is intended for global discoverability only. The schema of this substructure is identical to the GeoJSON FeatureCollection https://geojson.org/schema/FeatureCollection.json. The coordinate sequence follows GeoJSON standard, i.e. longitude, latitude {, height}",
+ "$ref": "#/definitions/opendes:wks:AbstractFeatureCollection:1.0.0"
+ },
+ "OperationsApplied": {
+ "title": "Operations Applied",
+ "description": "The audit trail of operations applied to the coordinates from the original state to the current state. The list may contain operations applied prior to ingestion as well as the operations applied to produce the Wgs84Coordinates. The text elements refer to ESRI style CRS and Transformation names, which may have to be translated to EPSG standard names.",
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "example": [
+ "conversion from ED_1950_UTM_Zone_31N to GCS_European_1950; 1 points converted",
+ "transformation GCS_European_1950 to GCS_WGS_1984 using ED_1950_To_WGS_1984_24; 1 points successfully transformed"
+ ]
+ },
+ "SpatialParameterTypeID": {
+ "description": "A type of spatial representation of an object, often general (e.g. an Outline, which could be applied to Field, Reservoir, Facility, etc.) or sometimes specific (e.g. Onshore Outline, State Offshore Outline, Federal Offshore Outline, 3 spatial representations that may be used by Countries).",
+ "type": "string",
+ "pattern": "^srn:<namespace>:reference-data\\/SpatialParameterType:[^:]+:[0-9]*$"
+ },
+ "SpatialGeometryTypeID": {
+ "description": "Indicates the expected look of the SpatialParameterType, e.g. Point, MultiPoint, LineString, MultiLineString, Polygon, MultiPolygon. The value constrains the type of geometries in the GeoJSON Wgs84Coordinates and AsIngestedCoordinates.",
+ "type": "string",
+ "pattern": "^srn:<namespace>:reference-data\\/SpatialGeometryType:[^:]+:[0-9]*$"
+ }
+ }
+ },
+ "opendes:wks:AbstractGeoPoliticalContext:1.0.0": {
+ "title": "AbstractGeoPoliticalContext",
+ "description": "A single, typed geo-political entity reference, which is 'abstracted' to AbstractGeoContext and then aggregated by GeoContexts properties.",
+ "type": "object",
+ "properties": {
+ "GeoPoliticalEntityID": {
+ "type": "string",
+ "description": "Reference to GeoPoliticalEntity.",
+ "pattern": "^srn:<namespace>:master-data\\/GeoPoliticalEntity:[^:]+:[0-9]*$"
+ },
+ "GeoTypeID": {
+ "type": "string",
+ "description": "The GeoPoliticalEntityType reference of the GeoPoliticalEntity (via GeoPoliticalEntityID) for application convenience.",
+ "pattern": "^srn:<namespace>:reference-data\\/GeoPoliticalEntityType:[^:]+:[0-9]*$"
+ }
+ }
+ },
+ "opendes:wks:AbstractGeoBasinContext:1.0.0": {
+ "title": "AbstractGeoBasinContext",
+ "description": "A single, typed basin entity reference, which is 'abstracted' to AbstractGeoContext and then aggregated by GeoContexts properties.",
+ "type": "object",
+ "properties": {
+ "BasinID": {
+ "type": "string",
+ "description": "Reference to Basin.",
+ "pattern": "^srn:<namespace>:master-data\\/Basin:[^:]+:[0-9]*$"
+ },
+ "GeoTypeID": {
+ "description": "The BasinType reference of the Basin (via BasinID) for application convenience.",
+ "type": "string",
+ "pattern": "^srn:<namespace>:reference-data\\/BasinType:[^:]+:[0-9]*$"
+ }
+ }
+ },
+ "opendes:wks:AbstractGeoFieldContext:1.0.0": {
+ "title": "AbstractGeoFieldContext",
+ "description": "A single, typed field entity reference, which is 'abstracted' to AbstractGeoContext and then aggregated by GeoContexts properties.",
+ "type": "object",
+ "properties": {
+ "FieldID": {
+ "type": "string",
+ "description": "Reference to Field.",
+ "pattern": "^srn:<namespace>:master-data\\/Field:[^:]+:[0-9]*$"
+ },
+ "GeoTypeID": {
+ "const": "Field",
+ "description": "The fixed type 'Field' for this AbstractGeoFieldContext."
+ }
+ }
+ },
+ "opendes:wks:AbstractGeoPlayContext:1.0.0": {
+ "title": "AbstractGeoPlayContext",
+ "description": "A single, typed Play entity reference, which is 'abstracted' to AbstractGeoContext and then aggregated by GeoContexts properties.",
+ "type": "object",
+ "properties": {
+ "PlayID": {
+ "type": "string",
+ "description": "Reference to the play.",
+ "pattern": "^srn:<namespace>:master-data\\/Play:[^:]+:[0-9]*$"
+ },
+ "GeoTypeID": {
+ "type": "string",
+ "description": "The PlayType reference of the Play (via PlayID) for application convenience.",
+ "pattern": "^srn:<namespace>:reference-data\\/PlayType:[^:]+:[0-9]*$"
+ }
+ }
+ },
+ "opendes:wks:AbstractGeoProspectContext:1.0.0": {
+ "title": "AbstractGeoProspectContext",
+ "description": "A single, typed Prospect entity reference, which is 'abstracted' to AbstractGeoContext and then aggregated by GeoContexts properties.",
+ "type": "object",
+ "properties": {
+ "ProspectID": {
+ "type": "string",
+ "description": "Reference to the prospect.",
+ "pattern": "^srn:<namespace>:master-data\\/Prospect:[^:]+:[0-9]*$"
+ },
+ "GeoTypeID": {
+ "type": "string",
+ "description": "The ProspectType reference of the Prospect (via ProspectID) for application convenience.",
+ "pattern": "^srn:<namespace>:reference-data\\/ProspectType:[^:]+:[0-9]*$"
+ }
+ }
+ },
+ "opendes:wks:AbstractGeoContext:1.0.0": {
+ "title": "AbstractGeoContext",
+ "description": "A geographic context to an entity. It can be either a reference to a GeoPoliticalEntity, Basin, Field, Play or Prospect.",
+ "oneOf": [
+ {
+ "$ref": "#/definitions/opendes:wks:AbstractGeoPoliticalContext:1.0.0"
+ },
+ {
+ "$ref": "#/definitions/opendes:wks:AbstractGeoBasinContext:1.0.0"
+ },
+ {
+ "$ref": "#/definitions/opendes:wks:AbstractGeoFieldContext:1.0.0"
+ },
+ {
+ "$ref": "#/definitions/opendes:wks:AbstractGeoPlayContext:1.0.0"
+ },
+ {
+ "$ref": "#/definitions/opendes:wks:AbstractGeoProspectContext:1.0.0"
+ }
+ ]
+ },
+ "opendes:wks:AbstractAliasNames:1.0.0": {
+ "title": "AbstractAliasNames",
+ "description": "A list of alternative names for an object. The preferred name is in a separate, scalar property. It may or may not be repeated in the alias list, though a best practice is to include it if the list is present, but to omit the list if there are no other names. Note that the abstract entity is an array so the $ref to it is a simple property reference.",
+ "type": "object",
+ "properties": {
+ "AliasName": {
+ "description": "Alternative Name value of defined name type for an object.",
+ "type": "string"
+ },
+ "AliasNameTypeID": {
+ "description": "A classification of alias names such as by role played or type of source, such as regulatory name, regulatory code, company code, international standard name, etc.",
+ "type": "string",
+ "pattern": "^srn:<namespace>:reference-data\\/AliasNameType:[^:]+:[0-9]*$"
+ },
+ "DefinitionOrganisationID": {
+ "description": "Organisation that provided the name (the source).",
+ "type": "string",
+ "pattern": "^srn:<namespace>:master-data\\/Organisation:[^:]+:[0-9]*$"
+ },
+ "EffectiveDateTime": {
+ "description": "The date and time when an alias name becomes effective.",
+ "type": "string",
+ "format": "date-time"
+ },
+ "TerminationDateTime": {
+ "description": "The data and time when an alias name is no longer in effect.",
+ "type": "string",
+ "format": "date-time"
+ }
+ }
+ },
+ "opendes:wks:AbstractFacilityState:1.0.0": {
+ "title": "AbstractFacilityState",
+ "description": "The life cycle status of a facility at some point in time.",
+ "type": "object",
+ "properties": {
+ "EffectiveDateTime": {
+ "description": "The date and time at which the facility state becomes effective.",
+ "type": "string",
+ "format": "date-time"
+ },
+ "TerminationDateTime": {
+ "description": "The date and time at which the facility state is no longer in effect.",
+ "type": "string",
+ "format": "date-time"
+ },
+ "FacilityStateTypeID": {
+ "description": "The facility life cycle state from planning to abandonment.",
+ "type": "string",
+ "pattern": "^srn:<namespace>:reference-data\\/FacilityStateType:[^:]+:[0-9]*$"
+ }
+ }
+ },
+ "opendes:wks:AbstractFacilityEvent:1.0.0": {
+ "title": "AbstractFacilityEvent",
+ "description": "A significant occurrence in the life of a facility, which often changes its state, or the state of one of its components.",
+ "type": "object",
+ "properties": {
+ "FacilityEventTypeID": {
+ "description": "The facility event type is a picklist. Examples: Propose, Completion, Entry Date etc.",
+ "type": "string",
+ "pattern": "^srn:<namespace>:reference-data\\/FacilityEventType:[^:]+:[0-9]*$"
+ },
+ "EffectiveDateTime": {
+ "description": "The date and time at which the event becomes effective.",
+ "type": "string",
+ "format": "date-time"
+ },
+ "TerminationDateTime": {
+ "description": "The date and time at which the event is no longer in effect.",
+ "type": "string",
+ "format": "date-time"
+ }
+ }
+ },
+ "opendes:wks:AbstractFacilitySpecification:1.0.0": {
+ "title": "AbstractFacilitySpecification",
+ "description": "A property, characteristic, or attribute about a facility that is not described explicitly elsewhere.",
+ "type": "object",
+ "properties": {
+ "EffectiveDateTime": {
+ "description": "The date and time at which the facility specification instance becomes effective.",
+ "type": "string",
+ "format": "date-time"
+ },
+ "TerminationDateTime": {
+ "description": "The date and time at which the facility specification instance is no longer in effect.",
+ "format": "date-time",
+ "type": "string"
+ },
+ "FacilitySpecificationQuantity": {
+ "description": "The value for the specified parameter type.",
+ "type": "number",
+ "x-osdu-frame-of-reference": "UOM_via_property:UnitOfMeasureID"
+ },
+ "FacilitySpecificationDateTime": {
+ "description": "The actual date and time value of the parameter.",
+ "type": "string",
+ "format": "date-time"
+ },
+ "FacilitySpecificationIndicator": {
+ "description": "The actual indicator value of the parameter.",
+ "type": "bool"
+ },
+ "FacilitySpecificationText": {
+ "description": "The actual text value of the parameter.",
+ "type": "string"
+ },
+ "UnitOfMeasureID": {
+ "description": "The unit for the quantity parameter, like metre (m in SI units system) for quantity Length.",
+ "type": "string",
+ "pattern": "^srn:<namespace>:reference-data\\/UnitOfMeasure:[^:]+:[0-9]*$"
+ },
+ "ParameterTypeID": {
+ "description": "Parameter type of property or characteristic.",
+ "type": "string",
+ "pattern": "^srn:<namespace>:reference-data\\/ParameterType:[^:]+:[0-9]*$"
+ }
+ }
+ },
+ "opendes:wks:AbstractFacility:1.0.0": {
+ "title": "AbstractFacility",
+ "description": "",
+ "type": "object",
+ "properties": {
+ "FacilityID": {
+ "description": "A system-specified unique identifier of a Facility.",
+ "type": "string"
+ },
+ "FacilityTypeID": {
+ "description": "The definition of a kind of capability to perform a business function or a service.",
+ "type": "string",
+ "pattern": "^srn:<namespace>:reference-data\\/FacilityType:[^:]+:[0-9]*$"
+ },
+ "FacilityOperator": {
+ "description": "The history of operator organizations of the facility.",
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/opendes:wks:AbstractFacilityOperator:1.0.0"
+ }
+ },
+ "DataSourceOrganisationID": {
+ "description": "The main source of the header information.",
+ "type": "string",
+ "pattern": "^srn:<namespace>:master-data\\/Organisation:[^:]+:[0-9]*$"
+ },
+ "SpatialLocation": {
+ "description": "The spatial location information such as coordinates,CRS information.",
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/opendes:wks:AbstractSpatialLocation:1.0.0"
+ }
+ },
+ "GeoContexts": {
+ "description": "List of geographic entities which provide context to the facility. This may include multiple types or multiple values of the same type.",
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/opendes:wks:AbstractGeoContext:1.0.0"
+ }
+ },
+ "OperatingEnvironmentID": {
+ "description": "Identifies the Facility's general location as being onshore vs. offshore.",
+ "type": "string",
+ "pattern": "^srn:<namespace>:reference-data\\/OperatingEnvironment:[^:]+:[0-9]*$"
+ },
+ "FacilityName": {
+ "description": "Name of the Facility.",
+ "type": "string"
+ },
+ "FacilityNameAlias": {
+ "description": "Alternative names, including historical, by which this facility is/has been known.",
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/opendes:wks:AbstractAliasNames:1.0.0"
+ }
+ },
+ "FacilityState": {
+ "description": "The history of life cycle states the facility has been through.",
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/opendes:wks:AbstractFacilityState:1.0.0"
+ }
+ },
+ "FacilityEvent": {
+ "description": "A list of key facility events.",
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/opendes:wks:AbstractFacilityEvent:1.0.0"
+ }
+ },
+ "FacilitySpecification": {
+ "description": "facilitySpecification maintains the specification like slot name, wellbore drilling permit number, rig name etc.",
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/opendes:wks:AbstractFacilitySpecification:1.0.0"
+ }
+ }
+ }
+ },
+ "opendes:wks:AbstractFacilityVerticalMeasurement:1.0.0": {
+ "title": "AbstractFacilityVerticalMeasurement",
+ "description": "A location along a wellbore, _usually_ associated with some aspect of the drilling of the wellbore, but not with any intersecting _subsurface_ natural surfaces.",
+ "type": "object",
+ "properties": {
+ "VerticalMeasurementID": {
+ "description": "The ID for a distinct vertical measurement within the Facility array so that it may be referenced by other vertical measurements if necessary.",
+ "type": "string"
+ },
+ "EffectiveDateTime": {
+ "description": "The date and time at which a vertical measurement instance becomes effective.",
+ "type": "string",
+ "format": "date-time"
+ },
+ "VerticalMeasurement": {
+ "description": "The value of the elevation or depth. Depth is positive downwards from a vertical reference or geodetic datum along a path, which can be vertical; elevation is positive upwards from a geodetic datum along a vertical path. Either can be negative.",
+ "type": "number",
+ "x-osdu-frame-of-reference": "UOM_via_property:VerticalMeasurementUnitOfMeasureID"
+ },
+ "TerminationDateTime": {
+ "description": "The date and time at which a vertical measurement instance is no longer in effect.",
+ "type": "string",
+ "format": "date-time"
+ },
+ "VerticalMeasurementTypeID": {
+ "description": "Specifies the type of vertical measurement (TD, Plugback, Kickoff, Drill Floor, Rotary Table...).",
+ "type": "string",
+ "pattern": "^srn:<namespace>:reference-data\\/VerticalMeasurementType:[^:]+:[0-9]*$"
+ },
+ "VerticalMeasurementPathID": {
+ "description": "Specifies Measured Depth, True Vertical Depth, or Elevation.",
+ "type": "string",
+ "pattern": "^srn:<namespace>:reference-data\\/VerticalMeasurementPath:[^:]+:[0-9]*$"
+ },
+ "VerticalMeasurementSourceID": {
+ "description": "Specifies Driller vs Logger.",
+ "type": "string",
+ "pattern": "^srn:<namespace>:reference-data\\/VerticalMeasurementSource:[^:]+:[0-9]*$"
+ },
+ "WellboreTVDTrajectoryID": {
+ "description": "Specifies what directional survey or wellpath was used to calculate the TVD.",
+ "type": "string",
+ "pattern": "^srn:<namespace>:work-product-component\\/WellboreTrajectory:[^:]+:[0-9]*$"
+ },
+ "VerticalMeasurementUnitOfMeasureID": {
+ "description": "The unit of measure for the vertical measurement. If a unit of measure and a vertical CRS are provided, the unit of measure provided is taken over the unit of measure from the CRS.",
+ "type": "string",
+ "pattern": "^srn:<namespace>:reference-data\\/UnitOfMeasure:[^:]+:[0-9]*$"
+ },
+ "VerticalCRSID": {
+ "description": "Vertical CRS. It is expected that a Vertical CRS or a Vertical Reference is provided, but not both.",
+ "type": "string",
+ "pattern": "^srn:<namespace>:reference-data\\/CoordinateReferenceSystem:[^:]+:[0-9]*$"
+ },
+ "VerticalReferenceID": {
+ "description": "The reference point from which the vertical measurement is made. Must resolve ultimately to a vertical CRS. It is expected that a Vertical CRS or a Vertical Reference is provided, but not both.",
+ "type": "string"
+ },
+ "VerticalMeasurementDescription": {
+ "description": "Text which describes a vertical measurement in detail.",
+ "type": "string"
+ }
+ }
+ },
+ "opendes:wks:AbstractWellboreDrillingReason:1.0.0": {
+ "title": "AbstractWellboreDrillingReason",
+ "description": "Purpose for drilling a wellbore, which often is an indication of the level of risk.",
+ "type": "object",
+ "properties": {
+ "DrillingReasonTypeID": {
+ "description": "Identifier of the drilling reason type for the corresponding time period.",
+ "type": "string",
+ "pattern": "^srn:<namespace>:reference-data\\/DrillingReasonType:[^:]+:[0-9]*$"
+ },
+ "EffectiveDateTime": {
+ "description": "The date and time at which the event becomes effective.",
+ "type": "string",
+ "format": "date-time"
+ },
+ "TerminationDateTime": {
+ "description": "The date and time at which the event is no longer in effect.",
+ "type": "string",
+ "format": "date-time"
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/indexer-core/src/test/resources/converter/basic/schema.json.res b/indexer-core/src/test/resources/converter/basic/schema.json.res
index 416ac54e5d60a96858525c138d92186109a1f97e..e3b4fd5f35f322cba5814ffdfbee18eaac72fb0b 100644
--- a/indexer-core/src/test/resources/converter/basic/schema.json.res
+++ b/indexer-core/src/test/resources/converter/basic/schema.json.res
@@ -132,6 +132,42 @@
{
"kind": "link",
"path": "GeographicBottomHoleLocation.SpatialGeometryTypeID"
+ },
+ {
+ "path": "VerticalMeasurements",
+ "kind": "[]object"
+ },
+ {
+ "path": "DrillingReason",
+ "kind": "[]object"
+ },
+ {
+ "path": "FacilityNameAlias",
+ "kind": "[]object"
+ },
+ {
+ "path": "FacilityState",
+ "kind": "[]object"
+ },
+ {
+ "path": "FacilityEvent",
+ "kind": "[]object"
+ },
+ {
+ "path": "FacilitySpecification",
+ "kind": "[]object"
+ },
+ {
+ "path": "FacilityOperator",
+ "kind": "[]object"
+ },
+ {
+ "path": "SpatialLocation",
+ "kind": "[]object"
+ },
+ {
+ "path": "GeoContexts",
+ "kind": "[]object"
}
]
}
\ No newline at end of file
diff --git a/indexer-core/src/test/resources/converter/new-definitions-format/colons-sample.json.res b/indexer-core/src/test/resources/converter/new-definitions-format/colons-sample.json.res
index 8f3ca4d7266d4f92177459314efe2445f6a25949..4c099bb3224947924cc16d44f3a11037e3ea2724 100644
--- a/indexer-core/src/test/resources/converter/new-definitions-format/colons-sample.json.res
+++ b/indexer-core/src/test/resources/converter/new-definitions-format/colons-sample.json.res
@@ -72,6 +72,14 @@
{
"path": "VersionCreationReason",
"kind": "string"
+ },
+ {
+ "path": "GeoContexts",
+ "kind": "[]object"
+ },
+ {
+ "path": "NameAliases",
+ "kind": "[]object"
}
]
}
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index 61d185ec97ef7ce70c7c55a573f5fc4342d082a8..7546228a31d394b89b303783770b268ef74ded5a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -19,7 +19,7 @@
<java.version>1.8</java.version>
<springfox-version>2.7.0</springfox-version>
<spring-cloud.version>Greenwich.SR2</spring-cloud.version>
- <os-core-common.version>0.6.9</os-core-common.version>
+ <os-core-common.version>0.9.0-SNAPSHOT</os-core-common.version>
<snakeyaml.version>1.26</snakeyaml.version>
<hibernate-validator.version>6.1.5.Final</hibernate-validator.version>
<jackson.version>2.11.2</jackson.version>
diff --git a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexerMappingServiceTest.java b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexerMappingServiceTest.java
index 3fc9f04f711391c860528a1af7c9b34a01eb1cf9..01b50be0be12488791a03f54b8c46dce65c4bbd6 100644
--- a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexerMappingServiceTest.java
+++ b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/IndexerMappingServiceTest.java
@@ -88,7 +88,7 @@ public class IndexerMappingServiceTest {
@Before
public void setup() throws IOException {
- Map<String, String> dataMapping = new HashMap<>();
+ Map<String, Object> dataMapping = new HashMap<>();
dataMapping.put("Location", "geo_point");
Map<String, Object> metaMapping = new HashMap<>();
metaMapping.put(RecordMetaAttribute.ID.getValue(), "keyword");
diff --git a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/middleware/IndexFilter.java b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/middleware/IndexFilter.java
index cd7d4622c847e13f109c6ee89adc8c679b645144..525ccfa928b56f1bd5d08402621a8ffb331cedd5 100644
--- a/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/middleware/IndexFilter.java
+++ b/provider/indexer-gcp/src/main/java/org/opengroup/osdu/indexer/middleware/IndexFilter.java
@@ -75,7 +75,7 @@ public class IndexFilter implements Filter {
}
filterChain.doFilter(servletRequest, servletResponse);
-
+
HttpServletResponse httpResponse = (HttpServletResponse) servletResponse;
Map<String, List<Object>> standardHeaders = ResponseHeaders.STANDARD_RESPONSE_HEADERS;
for (Map.Entry<String, List<Object>> header : standardHeaders.entrySet()) {
diff --git a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java
index 019bae350d179f8d038f31883284af4e2d6e439b..f8debf207f2366c68fe36941e8a0c58eefdf54c6 100644
--- a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java
+++ b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java
@@ -74,7 +74,7 @@ public class IndexerMappingServiceTest {
@Before
public void setup() throws IOException {
- Map<String, String> dataMapping = new HashMap<>();
+ Map<String, Object> dataMapping = new HashMap<>();
dataMapping.put("Location", "geo_point");
Map<String, Object> metaMapping = new HashMap<>();
metaMapping.put(RecordMetaAttribute.ID.getValue(), "keyword");
diff --git a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java
index 6bae37e26cfb840db56275fe1a42f0afa8557677..631c840c22670a08200664c66aa8a0f82fc70ae6 100644
--- a/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java
+++ b/provider/indexer-gcp/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java
@@ -29,7 +29,7 @@ import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache;
import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
import org.opengroup.osdu.core.common.model.http.RequestStatus;
import org.opengroup.osdu.core.common.search.IndicesService;
-import org.opengroup.osdu.indexer.service.impl.SchemaProviderImpl;
+import org.opengroup.osdu.indexer.service.SchemaProviderImpl;
import org.opengroup.osdu.indexer.util.ElasticClientHandler;
import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
import org.powermock.core.classloader.annotations.PrepareForTest;
diff --git a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerMappingServiceTest.java b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerMappingServiceTest.java
index d6982c113fc059e87626d2f931e4f2288f7fd3fd..9e22a80fce0c02ff55d9bd940c07279dfcd39472 100644
--- a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerMappingServiceTest.java
+++ b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/IndexerMappingServiceTest.java
@@ -76,7 +76,7 @@ public class IndexerMappingServiceTest {
@Before
public void setup() throws IOException {
- Map<String, String> dataMapping = new HashMap<>();
+ Map<String, Object> dataMapping = new HashMap<>();
dataMapping.put("Location", "geo_point");
Map<String, Object> metaMapping = new HashMap<>();
metaMapping.put(RecordMetaAttribute.ID.getValue(), "keyword");
diff --git a/provider/indexer-reference/pom.xml b/provider/indexer-reference/pom.xml
index 3e5a5328dd666658d15098f5cb003613da6b0ee9..745a8c4fd3531ea7eb2bc5b64e71582ac57667f9 100644
--- a/provider/indexer-reference/pom.xml
+++ b/provider/indexer-reference/pom.xml
@@ -48,7 +48,6 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
- <version>0.6.5</version>
</dependency>
<dependency>
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/middleware/IndexFilter.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/middleware/IndexFilter.java
index 12cffcf01340bf7c7ce763e2ba7a19202ed1a8e6..97244e67ce9a4cd9130807cd0a437c77d4064c82 100644
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/middleware/IndexFilter.java
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/middleware/IndexFilter.java
@@ -90,7 +90,7 @@ public class IndexFilter implements Filter {
}
filterChain.doFilter(servletRequest, servletResponse);
-
+
HttpServletResponse httpResponse = (HttpServletResponse) servletResponse;
Map<String, List<Object>> standardHeaders = ResponseHeaders.STANDARD_RESPONSE_HEADERS;
for (Map.Entry<String, List<Object>> header : standardHeaders.entrySet()) {
@@ -99,6 +99,7 @@ public class IndexFilter implements Filter {
if (httpResponse.getHeader(DpsHeaders.CORRELATION_ID) == null) {
httpResponse.addHeader(DpsHeaders.CORRELATION_ID, dpsHeaders.getCorrelationId());
}
+
}
@Override
diff --git a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java
index 4ce6eb4e8af4a45eddbc163baad9bb9cd6c2ef12..3b59eb203d3c9bd4391b0a25325ec30e70d06257 100644
--- a/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java
+++ b/provider/indexer-reference/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java
@@ -54,7 +54,7 @@ import org.opengroup.osdu.core.common.model.indexer.OperationType;
import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
import org.opengroup.osdu.core.common.search.IndicesService;
import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache;
-import org.opengroup.osdu.indexer.service.impl.SchemaProviderImpl;
+import org.opengroup.osdu.indexer.service.SchemaProviderImpl;
import org.opengroup.osdu.indexer.util.ElasticClientHandler;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.springframework.test.context.junit4.SpringRunner;
diff --git a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
index 03912e841850bb4b0eb985ff9ea5f52bc1bed098..c565d38835769e8e5d5100583fd61202fbc26cd9 100644
--- a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
+++ b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -105,4 +105,25 @@ public class Steps extends SchemaServiceRecordSteps {
int expectedCount, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field) throws Throwable {
super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery(expectedCount, index, topLatitude, topLongitude, bottomLatitude, bottomLongitude, field);
}
+
+ @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by nested \"([^\"]*)\" and properties \\(\"([^\"]*)\", (\\d+)\\) and \\(\"([^\"]*)\", \"([^\"]*)\"\\)$")
+ public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery(
+ int expectedCount, String index, String path, String firstNestedProperty, String firstNestedValue, String secondNestedProperty,
+ String secondNestedValue) throws Throwable {
+ super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery(expectedCount, index, path, firstNestedProperty, firstNestedValue,
+ secondNestedProperty, secondNestedValue);
+ }
+
+ @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by flattened inner properties \\(\"([^\"]*)\", \"([^\"]*)\"\\)$")
+ public void i_should_be_able_search_documents_for_the_by_flattened_inner_properties(int expectedCount, String index, String flattenedField,
+ String flattenedFieldValue) throws Throwable {
+ super.i_should_be_able_search_documents_for_the_by_flattened_inner_properties(expectedCount, index, flattenedField, flattenedFieldValue);
+
+ }
+
+ @Then("^I should get \"([^\"]*)\" in response, without hints in schema for the \"([^\"]*)\" that present in the \"([^\"]*)\" with \"([^\"]*)\" for a given \"([^\"]*)\"$")
+ public void i_should_get_object_in_search_response_without_hints_in_schema(String objectInnerField, String index, String recordFile, String acl, String kind)
+ throws Throwable {
+ super.i_should_get_object_in_search_response_without_hints_in_schema(objectInnerField ,index, recordFile, acl, kind);
+ }
}
\ No newline at end of file
diff --git a/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
index 97779ee8b1f9f0cea9d6c23e0a88229225deb9fb..b1dc2343728b339278710726c0d37d4a1735dd0f 100644
--- a/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
+++ b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -73,4 +73,25 @@ public class Steps extends SchemaServiceRecordSteps {
int expectedCount, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field) throws Throwable {
super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery(expectedCount, index, topLatitude, topLongitude, bottomLatitude, bottomLongitude, field);
}
+
+ @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by nested \"([^\"]*)\" and properties \\(\"([^\"]*)\", (\\d+)\\) and \\(\"([^\"]*)\", \"([^\"]*)\"\\)$")
+ public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery(
+ int expectedCount, String index, String path, String firstNestedProperty, String firstNestedValue, String secondNestedProperty,
+ String secondNestedValue) throws Throwable {
+ super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery(expectedCount, index, path, firstNestedProperty, firstNestedValue,
+ secondNestedProperty, secondNestedValue);
+ }
+
+ @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by flattened inner properties \\(\"([^\"]*)\", \"([^\"]*)\"\\)$")
+ public void i_should_be_able_search_documents_for_the_by_flattened_inner_properties(int expectedCount, String index, String flattenedField,
+ String flattenedFieldValue) throws Throwable {
+ super.i_should_be_able_search_documents_for_the_by_flattened_inner_properties(expectedCount, index, flattenedField, flattenedFieldValue);
+
+ }
+
+ @Then("^I should get \"([^\"]*)\" in response, without hints in schema for the \"([^\"]*)\" that present in the \"([^\"]*)\" with \"([^\"]*)\" for a given \"([^\"]*)\"$")
+ public void i_should_get_object_in_search_response_without_hints_in_schema(String objectInnerField, String index, String recordFile, String acl, String kind)
+ throws Throwable {
+ super.i_should_get_object_in_search_response_without_hints_in_schema(objectInnerField ,index, recordFile, acl, kind);
+ }
}
\ No newline at end of file
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
index 4f3b563a3502abe88be51e3742c40c2bfe6a9f99..cd7f2eb02ee34ec824ca51b88a47c57460ef211e 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
@@ -1,5 +1,6 @@
package org.opengroup.osdu.common;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.MapDifference;
import com.google.common.collect.Maps;
import com.google.gson.Gson;
@@ -11,6 +12,7 @@ import org.elasticsearch.cluster.metadata.MappingMetadata;
import org.opengroup.osdu.core.common.model.entitlements.Acl;
import org.opengroup.osdu.models.Setup;
import org.opengroup.osdu.models.TestIndex;
+import org.opengroup.osdu.models.record.RecordData;
import org.opengroup.osdu.util.ElasticUtils;
import org.opengroup.osdu.util.FileHandler;
import org.opengroup.osdu.util.HTTPClient;
@@ -32,6 +34,7 @@ import static org.opengroup.osdu.util.Config.getStorageBaseURL;
@Log
public class RecordSteps extends TestsBase {
private Map<String, TestIndex> inputIndexMap = new HashMap<>();
+ private ObjectMapper mapper = new ObjectMapper();
private boolean shutDownHookAdded = false;
private String timeStamp = String.valueOf(System.currentTimeMillis());
@@ -156,6 +159,38 @@ public class RecordSteps extends TestsBase {
assertEquals(expectedNumber, actualNumberOfRecords);
}
+ public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery(
+ int expectedNumber, String index, String path, String firstNestedField, String firstNestedValue, String secondNestedField, String secondNestedValue)
+ throws Throwable {
+ index = generateActualName(index, timeStamp);
+ long numOfIndexedDocuments = createIndex(index);
+ long actualNumberOfRecords = elasticUtils.fetchRecordsByNestedQuery(index, path, firstNestedField, firstNestedValue, secondNestedField, secondNestedValue);
+ assertEquals(expectedNumber, actualNumberOfRecords);
+ }
+
+ public void i_should_be_able_search_documents_for_the_by_flattened_inner_properties(int expectedCount, String index, String flattenedField,
+ String flattenedFieldValue) throws Throwable {
+ index = generateActualName(index, timeStamp);
+ long numOfIndexedDocuments = createIndex(index);
+ long actualNumberOfRecords = elasticUtils.fetchRecordsWithFlattenedFieldsQuery(index, flattenedField, flattenedFieldValue);
+ assertEquals(expectedCount, actualNumberOfRecords);
+ }
+
+ public void i_should_get_object_in_search_response_without_hints_in_schema(String objectField, String index, String recordFile, String acl, String kind)
+ throws Throwable {
+ index = generateActualName(index, timeStamp);
+ long numOfIndexedDocuments = createIndex(index);
+ String expectedRecord = FileHandler.readFile(String.format("%s.%s", recordFile, "json"));
+
+ RecordData[] fileRecordData = mapper.readValue(expectedRecord, RecordData[].class);
+ RecordData expectedRecordData = fileRecordData[0];
+
+ String elasticRecordData = elasticUtils.fetchDataFromObjectsArrayRecords(index);
+ RecordData actualRecordData = mapper.readValue(elasticRecordData, RecordData.class);
+
+ assertEquals(expectedRecordData.getData().get(objectField),actualRecordData.getData().get(objectField));
+ }
+
private long createIndex(String index) throws InterruptedException, IOException {
long numOfIndexedDocuments = 0;
int iterator;
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/models/record/RecordData.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/models/record/RecordData.java
new file mode 100644
index 0000000000000000000000000000000000000000..ec0a6a221efe767579871b4877c7f6d61d0c7127
--- /dev/null
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/models/record/RecordData.java
@@ -0,0 +1,18 @@
+package org.opengroup.osdu.models.record;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.List;
+import java.util.Map;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class RecordData {
+
+ @JsonProperty("data")
+ private Map<String, List<Map<String,Object>>> data;
+
+ public Map<String, List<Map<String, Object>>> getData() {
+ return data;
+ }
+
+}
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
index 30a6104ec6de56ad349000e509e8a307372c4231..3d029aa6ecbd34fb9ec08c2e9909cf933e33843a 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
@@ -32,6 +32,7 @@ import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
import org.apache.http.message.BasicHeader;
import org.apache.http.ssl.SSLContextBuilder;
+import org.apache.lucene.search.join.ScoreMode;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchStatusException;
import org.elasticsearch.client.indices.CloseIndexRequest;
@@ -51,6 +52,7 @@ import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.client.indices.GetMappingsRequest;
import org.elasticsearch.client.indices.GetMappingsResponse;
import org.elasticsearch.cluster.metadata.MappingMetadata;
+import org.elasticsearch.search.SearchHits;
import org.locationtech.jts.geom.Coordinate;
import org.elasticsearch.common.geo.builders.EnvelopeBuilder;
import org.elasticsearch.common.settings.Settings;
@@ -274,6 +276,58 @@ public class ElasticUtils {
}
}
+ public long fetchRecordsByNestedQuery(String index, String path, String firstNestedField, String firstNestedValue, String secondNestedField, String secondNestedValue) throws Exception{
+ try (RestHighLevelClient client = this.createClient(username, password, host)) {
+ SearchRequest searchRequest = new SearchRequest(index);
+
+ SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+ searchSourceBuilder.query(nestedQuery(path,boolQuery().must(matchQuery(firstNestedField,firstNestedValue)).must(matchQuery(secondNestedField,secondNestedValue)), ScoreMode.Avg));
+
+ searchRequest.source(searchSourceBuilder);
+
+ SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
+ return searchResponse.getHits().getTotalHits().value;
+ } catch (ElasticsearchStatusException e) {
+ log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage()));
+ return -1;
+ }
+ }
+
+
+ public long fetchRecordsWithFlattenedFieldsQuery(String index, String flattenedField, String flattenedFieldValue) throws IOException {
+ try (RestHighLevelClient client = this.createClient(username, password, host)) {
+ SearchRequest searchRequest = new SearchRequest(index);
+
+ SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+ searchSourceBuilder.query(boolQuery().must(matchQuery(flattenedField,flattenedFieldValue)));
+ searchRequest.source(searchSourceBuilder);
+
+ SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
+ return searchResponse.getHits().getTotalHits().value;
+ } catch (ElasticsearchStatusException e) {
+ log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage()));
+ return -1;
+ }
+ }
+
+ public String fetchDataFromObjectsArrayRecords(String index) throws IOException {
+ try {
+ try (RestHighLevelClient client = this.createClient(username, password, host)) {
+ SearchRequest request = new SearchRequest(index);
+ SearchResponse searchResponse = client.search(request, RequestOptions.DEFAULT);
+
+ SearchHits searchHits = searchResponse.getHits();
+ if (searchHits.getHits().length != 0) {
+ return searchHits.getHits()[0].getSourceAsString();
+ }
+ return null;
+ }
+ } catch (ElasticsearchStatusException e) {
+ log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage()));
+ return null;
+ }
+ }
+
public Map<String, MappingMetadata> getMapping(String index) throws IOException {
try (RestHighLevelClient client = this.createClient(username, password, host)) {
GetMappingsRequest request = new GetMappingsRequest();
@@ -415,4 +469,5 @@ public class ElasticUtils {
}
return false;
}
+
}
\ No newline at end of file
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/IndexerClientUtil.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/IndexerClientUtil.java
new file mode 100644
index 0000000000000000000000000000000000000000..33771a92bcf0bf93852027e1a63e722d9cfd5752
--- /dev/null
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/IndexerClientUtil.java
@@ -0,0 +1,38 @@
+package org.opengroup.osdu.util;
+
+import static org.opengroup.osdu.util.Config.getDataPartitionIdTenant1;
+import static org.opengroup.osdu.util.Config.getIndexerBaseURL;
+
+import com.google.gson.Gson;
+import com.sun.jersey.api.client.ClientResponse;
+import java.util.Map;
+import javax.ws.rs.HttpMethod;
+import lombok.extern.java.Log;
+import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
+
+@Log
+public class IndexerClientUtil {
+
+ private final String purgeMessage = "{\"data\":\"[{\\\"kind\\\":\\\"%s\\\",\\\"op\\\":\\\"purge_schema\\\"}]\",\"attributes\":{\"account-id\":\"%s\"}}";
+
+ private final HTTPClient httpClient;
+ private Map<String, String> headers;
+
+ public IndexerClientUtil(HTTPClient httpClient) {
+ this.httpClient = httpClient;
+ headers = httpClient.getCommonHeader();
+ }
+
+ public void deleteIndex(String kind) {
+ String url = getIndexerBaseURL() + "index-cleanup";
+ log.info("URL: " + url);
+ ClientResponse response = httpClient.send(HttpMethod.POST, url, convertMessageIntoJson(kind), headers, httpClient.getAccessToken());
+ log.info(response.toString());
+ }
+
+ private String convertMessageIntoJson(String kind) {
+ RecordChangedMessages
+ recordChangedMessages = (new Gson()).fromJson(String.format(purgeMessage, kind, getDataPartitionIdTenant1()), RecordChangedMessages.class);
+ return new Gson().toJson(recordChangedMessages);
+ }
+}
diff --git a/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature b/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
index 711ce2de1bcad49d6f46e7578c26526b28590cf9..50051edcf49854757c25ce510b0125945f38aa7b 100644
--- a/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
+++ b/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
@@ -3,11 +3,12 @@ Feature: Indexing of the documents
Background:
Given the schema is created with the following kind
- | kind | index | schemaFile |
- | tenant1:indexer:test-data--Integration:1.0.0 | tenant1-indexer-test-data--integration-1.0.0 | index_records_1 |
- | tenant1:indexer:test-data--Integration:2.0.0 | tenant1-indexer-test-data--integration-2.0.0 | index_records_2 |
- | tenant1:indexer:test-data--Integration:3.0.0 | tenant1-indexer-test-data--integration-3.0.0 | index_records_3 |
- | tenant1:wks:master-data--Wellbore:2.0.3 | tenant1-wks-master-data--wellbore-2.0.3 | r3-index_record_wks_master |
+ | kind | index | schemaFile |
+ | tenant1:indexer:test-data--Integration:1.0.0 | tenant1-indexer-test-data--integration-1.0.0 | index_records_1 |
+ | tenant1:indexer:test-data--Integration:2.0.0 | tenant1-indexer-test-data--integration-2.0.0 | index_records_2 |
+ | tenant1:indexer:test-data--Integration:3.0.0 | tenant1-indexer-test-data--integration-3.0.0 | index_records_3 |
+ | tenant1:wks:master-data--Wellbore:2.0.3 | tenant1-wks-master-data--wellbore-2.0.3 | r3-index_record_wks_master |
+ | tenant1:wks:ArraysOfObjectsTestCollection:4.0.0 | tenant1-wks-arraysofobjectstestcollection-4.0.0 | r3-index_record_arrayofobjects |
Scenario Outline: Ingest the record and Index in the Elastic Search
When I ingest records with the <recordFile> with <acl> for a given <kind>
@@ -42,3 +43,13 @@ Feature: Indexing of the documents
Examples:
| kind | recordFile | number | index | acl | field | top_left_latitude | top_left_longitude | bottom_right_latitude | bottom_right_longitude |
| "tenant1:wks:master-data--Wellbore:2.0.3" | "r3-index_record_wks_master" | 1 | "tenant1-wks-master-data--wellbore-2.0.3" | "data.default.viewers@tenant1" | "data.SpatialLocation.Wgs84Coordinates" | 52 | -100 | 0 | 100 |
+
+ Scenario Outline: Ingest the r3-record with arrays of objects and hints in schema and Index in the Elastic Search
+ When I ingest records with the <recordFile> with <acl> for a given <kind>
+ Then I should be able search <number> documents for the <index> by nested <path> and properties (<first_nested_field>, <first_nested_value>) and (<second_nested_field>, <second_nested_value>)
+ Then I should be able search <number> documents for the <index> by flattened inner properties (<flattened_inner_field>, <flattened_inner_value>)
+ Then I should get <object_inner_field> in response, without hints in schema for the <index> that present in the <recordFile> with <acl> for a given <kind>
+
+ Examples:
+ | kind | recordFile | number | index | acl | path | first_nested_field | first_nested_value | second_nested_field | second_nested_value | flattened_inner_field | flattened_inner_value | object_inner_field |
+ | "tenant1:wks:ArraysOfObjectsTestCollection:4.0.0" | "r3-index_record_arrayofobjects" | 1 | "tenant1-wks-arraysofobjectstestcollection-4.0.0" | "data.default.viewers@tenant1" | "data.NestedTest" | "data.NestedTest.NumberTest" | 12345 | "data.NestedTest.StringTest" | "test string" | "data.FlattenedTest.StringTest" | "test string" | "ObjectTest" |
\ No newline at end of file
diff --git a/testing/indexer-test-core/src/main/resources/testData/r3-index_record_arrayofobjects.json b/testing/indexer-test-core/src/main/resources/testData/r3-index_record_arrayofobjects.json
new file mode 100644
index 0000000000000000000000000000000000000000..f37563c946a8b75e530dd1cd96017233ab40db8f
--- /dev/null
+++ b/testing/indexer-test-core/src/main/resources/testData/r3-index_record_arrayofobjects.json
@@ -0,0 +1,43 @@
+[
+ {
+ "id": "tenant1:<kindSubType>:testIngest70",
+ "data": {
+ "NestedTest": [
+ {
+ "DateTimeTest": "2020-02-13T09:13:15.55Z",
+ "NumberTest": "12345",
+ "StringTest": "test string"
+ },
+ {
+ "DateTimeTest": "2020-02-13T09:13:15.55Z",
+ "NumberTest": "567890",
+ "StringTest": "test string"
+ }
+ ],
+ "FlattenedTest": [
+ {
+ "DateTimeTest": "2020-02-13T09:13:15.55Z",
+ "NumberTest": "12345",
+ "StringTest": "test string"
+ },
+ {
+ "DateTimeTest": "2020-02-13T09:13:15.55Z",
+ "NumberTest": "567890",
+ "StringTest": "test string"
+ }
+ ],
+ "ObjectTest": [
+ {
+ "DateTimeTest": "2020-02-13T09:13:15.55Z",
+ "NumberTest": "12345",
+ "StringTest": "test string"
+ },
+ {
+ "DateTimeTest": "2020-02-13T09:13:15.55Z",
+ "NumberTest": "567890",
+ "StringTest": "test string"
+ }
+ ]
+ }
+ }
+]
\ No newline at end of file
diff --git a/testing/indexer-test-core/src/main/resources/testData/r3-index_record_arrayofobjects.schema.json b/testing/indexer-test-core/src/main/resources/testData/r3-index_record_arrayofobjects.schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..c68efaceb42e6b11cb8ea857a85459e1dedc89a1
--- /dev/null
+++ b/testing/indexer-test-core/src/main/resources/testData/r3-index_record_arrayofobjects.schema.json
@@ -0,0 +1,112 @@
+{
+ "schemaInfo": {
+ "schemaIdentity": {
+ "authority": "tenant1",
+ "source": "wks",
+ "entityType": "ArraysOfObjectsTestCollection",
+ "schemaVersionMajor": 4,
+ "schemaVersionMinor": 0,
+ "schemaVersionPatch": 0
+ },
+ "status": "DEVELOPMENT"
+ },
+ "schema": {
+ "x-osdu-license": "Copyright 2021, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "description": "Array of objects testing schema",
+ "title": "Test",
+ "type": "object",
+ "required": [
+ "kind",
+ "acl",
+ "legal"
+ ],
+ "properties": {
+ "data": {
+ "allOf": [
+ {
+ "type": "object",
+ "properties": {
+ "NestedTest": {
+ "description": "nested type test",
+ "type": "array",
+ "x-osdu-indexing": {
+ "type": "nested"
+ },
+ "items": {
+ "type": "object",
+ "properties": {
+ "DateTimeTest": {
+ "description": "date and time test",
+ "type": "string",
+ "format": "date-time",
+ "x-osdu-frame-of-reference": "DateTime"
+ },
+ "NumberTest": {
+ "description": "number test",
+ "type": "number"
+ },
+ "StringTest": {
+ "description": "string test",
+ "type": "string"
+ }
+ }
+ }
+ },
+ "FlattenedTest": {
+ "description": "flattened type test",
+ "type": "array",
+ "x-osdu-indexing": {
+ "type": "flattened"
+ },
+ "items": {
+ "type": "object",
+ "properties": {
+ "DateTimeTest": {
+ "description": "date and time test",
+ "type": "string",
+ "format": "date-time",
+ "x-osdu-frame-of-reference": "DateTime"
+ },
+ "NumberTest": {
+ "description": "number test",
+ "type": "number"
+ },
+ "StringTest": {
+ "description": "string test",
+ "type": "string"
+ }
+ }
+ }
+ },
+ "ObjectTest": {
+ "description": "default object type test",
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "DateTimeTest": {
+ "description": "date and time test",
+ "type": "string",
+ "format": "date-time",
+ "x-osdu-frame-of-reference": "DateTime"
+ },
+ "NumberTest": {
+ "description": "number test",
+ "type": "number"
+ },
+ "StringTest": {
+ "description": "string test",
+ "type": "string"
+ }
+ }
+ }
+ }
+ }
+ }
+ ]
+ }
+ },
+ "x-osdu-inheriting-from-kind": []
+ }
+}
\ No newline at end of file
diff --git a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java
index 4978ddfccad197628432d4ccb8b5985b4af5017e..4d9708e9794d942fd4a287a4bebfde5fce204695 100644
--- a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java
+++ b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java
@@ -6,7 +6,7 @@ import org.junit.runner.RunWith;
@RunWith(Cucumber.class)
@CucumberOptions(
- features = "classpath:features/indexrecord/IndexRecord.feature",
+ features = "classpath:features/indexrecord/indexRecord-schema-service.feature",
glue = {"classpath:org.opengroup.osdu.step_definitions/index/record"},
plugin = {"pretty", "junit:target/cucumber-reports/TEST-indexrecord.xml"})
public class RunTest {
diff --git a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
index 348474e7769bb54d2362d079ad9aa093760d6fa2..26060454cb859e06fb80bbfa8a8c0b2d4d8c45a2 100644
--- a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
+++ b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -1,27 +1,29 @@
package org.opengroup.osdu.step_definitions.index.record;
-import lombok.extern.java.Log;
-import org.opengroup.osdu.common.RecordSteps;
-import org.opengroup.osdu.util.GCPHTTPClient;
-
+import cucumber.api.DataTable;
import cucumber.api.Scenario;
import cucumber.api.java.Before;
-import cucumber.api.DataTable;
import cucumber.api.java.en.Given;
import cucumber.api.java.en.Then;
import cucumber.api.java.en.When;
+import lombok.extern.java.Log;
+import org.opengroup.osdu.common.SchemaServiceRecordSteps;
+import org.opengroup.osdu.util.ElasticUtils;
+import org.opengroup.osdu.util.GCPHTTPClient;
+import org.opengroup.osdu.util.GcpElasticUtils;
@Log
-public class Steps extends RecordSteps {
+public class Steps extends SchemaServiceRecordSteps {
public Steps() {
- super(new GCPHTTPClient());
+ super(new GCPHTTPClient(), new ElasticUtils());
}
@Before
public void before(Scenario scenario) {
this.scenario = scenario;
this.httpClient = new GCPHTTPClient();
+ this.elasticUtils = new GcpElasticUtils(this.httpClient);
}
@Given("^the schema is created with the following kind$")
@@ -40,13 +42,47 @@ public class Steps extends RecordSteps {
}
@Then("^I should get the elastic \"(.*?)\" for the \"([^\"]*)\" and \"([^\"]*)\" in the Elastic Search$")
- public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String type, String index) throws Throwable {
+ public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String type, String index)
+ throws Throwable {
super.i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(expectedMapping, type, index);
}
@Then("^I should get the (\\d+) documents for the \"([^\"]*)\" in the Elastic Search with out \"(.*?)\"$")
- public void iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(int expectedCount, String index, String skippedAttributes) throws Throwable {
+ public void iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(int expectedCount, String index, String skippedAttributes)
+ throws Throwable {
super.iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(expectedCount, index, skippedAttributes);
}
+ @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by bounding box query with points \\((-?\\d+), (-?\\d+)\\) and \\((-?\\d+), (-?\\d+)\\) on field \"([^\"]*)\"$")
+ public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery(
+ int expectedCount, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field)
+ throws Throwable {
+ String actualName = generateActualName(index, null);
+ super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery(expectedCount, actualName, topLatitude, topLongitude, bottomLatitude,
+ bottomLongitude, field);
+ }
+
+ @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by nested \"([^\"]*)\" and properties \\(\"([^\"]*)\", (\\d+)\\) and \\(\"([^\"]*)\", \"([^\"]*)\"\\)$")
+ public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery(
+ int expectedCount, String index, String path, String firstNestedProperty, String firstNestedValue, String secondNestedProperty,
+ String secondNestedValue) throws Throwable {
+ String actualName = generateActualName(index, null);
+ super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery(expectedCount, actualName, path, firstNestedProperty, firstNestedValue,
+ secondNestedProperty, secondNestedValue);
+ }
+
+ @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by flattened inner properties \\(\"([^\"]*)\", \"([^\"]*)\"\\)$")
+ public void i_should_be_able_search_documents_for_the_by_flattened_inner_properties(int expectedCount, String index, String flattenedField,
+ String flattenedFieldValue) throws Throwable {
+ String actualName = generateActualName(index, null);
+ super.i_should_be_able_search_documents_for_the_by_flattened_inner_properties(expectedCount, actualName, flattenedField, flattenedFieldValue);
+
+ }
+
+ @Then("^I should get \"([^\"]*)\" in response, without hints in schema for the \"([^\"]*)\" that present in the \"([^\"]*)\" with \"([^\"]*)\" for a given \"([^\"]*)\"$")
+ public void i_should_get_object_in_search_response_without_hints_in_schema(String objectInnerField, String index, String recordFile, String acl, String kind)
+ throws Throwable {
+ String actualName = generateActualName(index, null);
+ super.i_should_get_object_in_search_response_without_hints_in_schema(objectInnerField ,actualName, recordFile, acl, kind);
+ }
}
\ No newline at end of file
diff --git a/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/util/GcpElasticUtils.java b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/util/GcpElasticUtils.java
new file mode 100644
index 0000000000000000000000000000000000000000..ce781c6e8cafb3cd208c52e6ca86e5c670ce2a2e
--- /dev/null
+++ b/testing/indexer-test-gcp/src/test/java/org/opengroup/osdu/util/GcpElasticUtils.java
@@ -0,0 +1,20 @@
+package org.opengroup.osdu.util;
+
+public class GcpElasticUtils extends ElasticUtils {
+
+ private IndexerClientUtil indexerClientUtil;
+
+ public GcpElasticUtils(HTTPClient httpClient) {
+ super();
+ this.indexerClientUtil = new IndexerClientUtil(httpClient);
+ }
+
+ @Override
+ public void deleteIndex(String index) {
+ indexerClientUtil.deleteIndex(convertIndexToKindName(index));
+ }
+
+ private String convertIndexToKindName(String index) {
+ return index.replaceAll("-",":");
+ }
+}
diff --git a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java
index 897f02c15372583ef985dfc77e2884c4035640fd..cecc3b7ddf23d559c03a65f755d01e2c9184da1a 100644
--- a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java
+++ b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/RunTest.java
@@ -6,7 +6,7 @@ import org.junit.runner.RunWith;
@RunWith(Cucumber.class)
@CucumberOptions(
- features = {"classpath:features/indexrecord/IndexRecord.feature","classpath:apikey.feature"},
+ features = {"classpath:features/indexrecord/indexRecord-schema-service.feature","classpath:apikey.feature"},
glue = {"classpath:org.opengroup.osdu.step_definitions/index/record"},
plugin = {"pretty", "junit:target/cucumber-reports/TEST-indexrecord.xml"})
public class RunTest {
diff --git a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
index 3c885b11ca68f15dc3cad6ba15fae1ae7395b514..8caf85cacd30b7f146c13d0cb58cd156cbf9fc9a 100644
--- a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
+++ b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -8,6 +8,7 @@ import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.opengroup.osdu.common.RecordSteps;
+import org.opengroup.osdu.common.SchemaServiceRecordSteps;
import org.opengroup.osdu.core.common.Constants;
import org.opengroup.osdu.core.common.http.HttpClient;
import org.opengroup.osdu.core.common.http.HttpRequest;
@@ -16,6 +17,7 @@ import org.opengroup.osdu.core.common.model.http.AppError;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
import org.opengroup.osdu.core.ibm.util.Config;
+import org.opengroup.osdu.util.ElasticUtils;
import org.opengroup.osdu.util.IBMHTTPClient;
import com.google.gson.Gson;
@@ -29,10 +31,10 @@ import cucumber.api.java.en.When;
import lombok.extern.java.Log;
@Log
-public class Steps extends RecordSteps {
+public class Steps extends SchemaServiceRecordSteps {
public Steps() {
- super(new IBMHTTPClient());
+ super(new IBMHTTPClient(), new ElasticUtils());
}
@Before
@@ -66,6 +68,38 @@ public class Steps extends RecordSteps {
super.iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(expectedCount, index, skippedAttributes);
}
+ @Then("^I should be able to search (\\d+) record with index \"([^\"]*)\" by tag \"([^\"]*)\" and value \"([^\"]*)\"$")
+ public void iShouldBeAbleToSearchRecordByTagKeyAndTagValue(int expectedNumber, String index, String tagKey, String tagValue) throws Throwable {
+ super.iShouldBeAbleToSearchRecordByTagKeyAndTagValue(index, tagKey, tagValue, expectedNumber);
+ }
+
+ @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by bounding box query with points \\((-?\\d+), (-?\\d+)\\) and \\((-?\\d+), (-?\\d+)\\) on field \"(.*?)\"$")
+ public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery (
+ int expectedCount, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field) throws Throwable {
+ super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery(expectedCount, index, topLatitude, topLongitude, bottomLatitude, bottomLongitude, field);
+ }
+
+ @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by nested \"([^\"]*)\" and properties \\(\"([^\"]*)\", (\\d+)\\) and \\(\"([^\"]*)\", \"([^\"]*)\"\\)$")
+ public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery(
+ int expectedCount, String index, String path, String firstNestedProperty, String firstNestedValue, String secondNestedProperty,
+ String secondNestedValue) throws Throwable {
+ super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery(expectedCount, index, path, firstNestedProperty, firstNestedValue,
+ secondNestedProperty, secondNestedValue);
+ }
+
+ @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by flattened inner properties \\(\"([^\"]*)\", \"([^\"]*)\"\\)$")
+ public void i_should_be_able_search_documents_for_the_by_flattened_inner_properties(int expectedCount, String index, String flattenedField,
+ String flattenedFieldValue) throws Throwable {
+ super.i_should_be_able_search_documents_for_the_by_flattened_inner_properties(expectedCount, index, flattenedField, flattenedFieldValue);
+
+ }
+
+ @Then("^I should get \"([^\"]*)\" in response, without hints in schema for the \"([^\"]*)\" that present in the \"([^\"]*)\" with \"([^\"]*)\" for a given \"([^\"]*)\"$")
+ public void i_should_get_object_in_search_response_without_hints_in_schema(String objectInnerField, String index, String recordFile, String acl, String kind)
+ throws Throwable {
+ super.i_should_get_object_in_search_response_without_hints_in_schema(objectInnerField ,index, recordFile, acl, kind);
+ }
+
@When("^I pass api key$")
public void i_pass_the_api_key() {
}