diff --git a/devops/azure/chart/templates/deployment.yaml b/devops/azure/chart/templates/deployment.yaml
index 400579dc3d4032f1683cf0565cea9f315e1c8294..3c20b14a4997367ea6f5f3112813f136949edb15 100644
--- a/devops/azure/chart/templates/deployment.yaml
+++ b/devops/azure/chart/templates/deployment.yaml
@@ -125,4 +125,6 @@ spec:
value: "api://$(aad_client_id)"
- name: SPRING_CONFIG_NAME
value: "common,application"
- terminationGracePeriodSeconds: 101
\ No newline at end of file
+ - name: search_service_url
+ value: http://search/api/search/v2
+ terminationGracePeriodSeconds: 101
diff --git a/devops/gc/deploy/templates/configmap.yaml b/devops/gc/deploy/templates/configmap.yaml
index 73c17cfc8ff461e868f3f30a6164414d65a20dd0..9faeb58eec5831b880a8d355be89a512638354e4 100644
--- a/devops/gc/deploy/templates/configmap.yaml
+++ b/devops/gc/deploy/templates/configmap.yaml
@@ -22,3 +22,4 @@ data:
SECURITY_HTTPS_CERTIFICATE_TRUST: {{ .Values.data.securityHttpsCertificateTrust | quote }}
SPRING_PROFILES_ACTIVE: {{ .Values.data.springProfilesActive | quote }}
STORAGE_HOST: {{ .Values.data.storageHost | quote }}
+ SEARCH_BASE_HOST: {{ .Values.data.searchHost | quote }}
diff --git a/devops/gc/deploy/values.yaml b/devops/gc/deploy/values.yaml
index b657f2f62e2402cf28e70ccbec7d519bf682f6b4..ba923d9733dd6f257aef2bfe4f5522b6fd27caf0 100644
--- a/devops/gc/deploy/values.yaml
+++ b/devops/gc/deploy/values.yaml
@@ -15,6 +15,7 @@ data:
securityHttpsCertificateTrust: "true"
springProfilesActive: "gcp"
storageHost: "http://storage"
+ searchHost: "http://search"
# Deployment
requestsCpu: "35m"
requestsMemory: "640Mi"
diff --git a/devops/gc/pipeline/override-stages.yml b/devops/gc/pipeline/override-stages.yml
index 4da64870338f033683f3f1aae58a594fd6c7c1a6..09b708fa22b593178e6b7c6c5a28ab0667b64df1 100644
--- a/devops/gc/pipeline/override-stages.yml
+++ b/devops/gc/pipeline/override-stages.yml
@@ -2,6 +2,11 @@ variables:
GC_SERVICE: indexer
GC_VENDOR: gc
+gc-test:
+ variables:
+ CUCUMBER_OPTIONS: "--tags '~@* and @indexer-extended'"
+
gc-baremetal-test:
variables:
GC_VENDOR: baremetal
+ CUCUMBER_OPTIONS: "--tags '~@* and @indexer-extended'"
diff --git a/devops/ibm/ibm-indexer-config/templates/configmap.yaml b/devops/ibm/ibm-indexer-config/templates/configmap.yaml
index fbb4ea24338a5cf021c1c5307ca12692ca1c94a6..32262305e44f44d4f8614f7850430eb9a108403a 100644
--- a/devops/ibm/ibm-indexer-config/templates/configmap.yaml
+++ b/devops/ibm/ibm-indexer-config/templates/configmap.yaml
@@ -53,6 +53,7 @@ data:
STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST: "http://{{ .Release.Name }}-ibm-storage-deploy:8080/api/storage/v2/query/records:batch"
STORAGE_RECORDS_BATCH_SIZE: "{{ .Values.data.storageRecordsBatchSize }}"
STORAGE_RECORDS_BY_KIND_BATCH_SIZE: "{{ .Values.data.storageRecordsByKindBatchSize }}"
+ SEARCH_HOST: "http://{{ .Release.Name }}-ibm-search-deploy:8080/api/service/v2"
#db
diff --git a/docs/tutorial/PreviewFeatures.md b/docs/tutorial/PreviewFeatures.md
index 307360f59ddab73756195822f622d7ae9948142f..16981af3b7af845d163f9288ff738503423b070d 100644
--- a/docs/tutorial/PreviewFeatures.md
+++ b/docs/tutorial/PreviewFeatures.md
@@ -23,3 +23,28 @@ Here is an example to disable this feature by setting the property "indexer-deci
If the property "indexer-decimation-enabled" is not created or the property value is set to "true" (String type) in the
given data partition, the geo-shape decimation will be enabled.
+
+## Index extension
+
+OSDU Standard index extensions are defined by OSDU Data Definition work-streams with the intent to provide
+user/application friendly, derived properties. The standard set, together with the OSDU schemas, form the
+interoperability foundation. They can contribute to deliver domain specific APIs according to the Domain Driven Design
+principles.
+
+The configurations are encoded in OSDU reference-data records, one per each major schema version. The type name
+is IndexPropertyPathConfiguration. With this, the extension properties can be defined as if they were provided by a schema.
+
+In order to reduce the risk when extended evaluation of the solution is still on going, a feature flag that is managed by
+the Partition Service is applied to the solution. Here is an example to enable this feature by setting the property
+"index-augmenter-enabled" in a given data partition:
+```
+{
+ "index-augmenter-enabled": {
+ "sensitive": false,
+ "value": "true"
+ }
+}
+```
+
+If the property "index-augmenter-enabled" is not created or the property value is set to "false" (String type) in the
+given data partition, the configurations defined as type IndexPropertyPathConfiguration will be ignored and index extension will be disabled.
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/AbstractPartitionSafeCache.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/AbstractPartitionSafeCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..52d2cdf9e7e5220bd4c34293683fe6322b278d87
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/AbstractPartitionSafeCache.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.ICache;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.springframework.web.context.annotation.RequestScope;
+
+import javax.inject.Inject;
+
+@RequestScope
+public abstract class AbstractPartitionSafeCache<K, V> implements ICache<K, V> {
+ @Inject
+ private IRequestInfo requestInfo;
+
+ protected String cacheKey(String s) {
+ return this.requestInfo.getPartitionId() + "-" + s;
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IKindCache.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IKindCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..c474065deded2a236e41e7d43436b874d9c295c1
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IKindCache.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.ICache;
+
+public interface IKindCache extends ICache<String, String> {
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IParentChildRelationshipSpecsCache.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IParentChildRelationshipSpecsCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..74aa48e4fadeb7d477c1fed19aef9dc9662619d7
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IParentChildRelationshipSpecsCache.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.ICache;
+import org.opengroup.osdu.indexer.model.indexproperty.ParentChildRelationshipSpecs;
+
+public interface IParentChildRelationshipSpecsCache extends ICache<String, ParentChildRelationshipSpecs> {
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IPropertyConfigurationsCache.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IPropertyConfigurationsCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..24a51559f9955861969a2536de31c397b2e6e85f
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IPropertyConfigurationsCache.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.ICache;
+import org.opengroup.osdu.indexer.model.indexproperty.PropertyConfigurations;
+
+public interface IPropertyConfigurationsCache extends ICache<String, PropertyConfigurations> {
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IPropertyConfigurationsEnabledCache.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IPropertyConfigurationsEnabledCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..4b6c372d333c2c5539ab5d1fab3f41dfdabff7e7
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IPropertyConfigurationsEnabledCache.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.ICache;
+
+public interface IPropertyConfigurationsEnabledCache extends ICache<String, Boolean> {
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IRecordChangeInfoCache.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IRecordChangeInfoCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..25591747a59e9f2d1a4de0f7217331979365d499
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IRecordChangeInfoCache.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.ICache;
+import org.opengroup.osdu.indexer.model.RecordChangeInfo;
+
+public interface IRecordChangeInfoCache extends ICache<String, RecordChangeInfo> {
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IRelatedObjectCache.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IRelatedObjectCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..ca87122e9eb09f6fc495e392ef14591a5076afa7
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IRelatedObjectCache.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.ICache;
+
+import java.util.Map;
+
+public interface IRelatedObjectCache extends ICache<String, Map<String, Object>> {
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/KindCacheVmImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/KindCacheVmImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..2eec5cead55d7a78c62ff3c3a946070edbdca02f
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/KindCacheVmImpl.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.VmCache;
+import org.opengroup.osdu.indexer.model.Constants;
+import org.springframework.stereotype.Component;
+
+@Component
+public class KindCacheVmImpl implements IKindCache {
+
+ private VmCache<String, String> cache;
+
+ public KindCacheVmImpl() {
+ cache = new VmCache<>(Constants.SPEC_CACHE_EXPIRATION, Constants.SPEC_MAX_CACHE_SIZE);
+ }
+
+ @Override
+ public void put(String s, String o) {
+ this.cache.put(s, o);
+ }
+
+ @Override
+ public String get(String s) {
+ return this.cache.get(s);
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(s);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/ParentChildRelationshipSpecsCacheVmImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/ParentChildRelationshipSpecsCacheVmImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..a270ab77d5141ffca5130f607eec92e40264ba82
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/ParentChildRelationshipSpecsCacheVmImpl.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.VmCache;
+import org.opengroup.osdu.indexer.model.Constants;
+import org.opengroup.osdu.indexer.model.indexproperty.ParentChildRelationshipSpecs;
+import org.springframework.stereotype.Component;
+
+@Component
+public class ParentChildRelationshipSpecsCacheVmImpl implements IParentChildRelationshipSpecsCache {
+ private VmCache<String, ParentChildRelationshipSpecs> cache;
+
+ public ParentChildRelationshipSpecsCacheVmImpl() {
+ cache = new VmCache<>(Constants.SPEC_CACHE_EXPIRATION, Constants.SPEC_MAX_CACHE_SIZE);
+ }
+
+ @Override
+ public void put(String s, ParentChildRelationshipSpecs o) {
+ this.cache.put(s, o);
+ }
+
+ @Override
+ public ParentChildRelationshipSpecs get(String s) {
+ return this.cache.get(s);
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(s);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafeFlattenedSchemaCache.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafeFlattenedSchemaCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..862acb6e9399e3a7a32acc1a0d9303c6730de40d
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafeFlattenedSchemaCache.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+import javax.inject.Inject;
+
+@Component
+@RequestScope
+public class PartitionSafeFlattenedSchemaCache extends AbstractPartitionSafeCache<String, String> {
+ private static final String FLATTENED_SCHEMA = "_flattened";
+ @Inject
+ private ISchemaCache schemaCache;
+
+ @Override
+ public void put(String s, String o) {
+ this.schemaCache.put(getKey(s), o);
+ }
+
+ @Override
+ public String get(String s) {
+ return (String)this.schemaCache.get(getKey(s));
+ }
+
+ @Override
+ public void delete(String s) {
+ this.schemaCache.delete(getKey(s));
+ }
+
+ @Override
+ public void clearAll() {
+ this.schemaCache.clearAll();
+ }
+
+ private String getKey(String s) {
+ return cacheKey(s) + FLATTENED_SCHEMA;
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafeKindCache.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafeKindCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..e1bc5adce2686070b5cfed0587823b7b2e4700b4
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafeKindCache.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+import javax.inject.Inject;
+
+@Component
+@RequestScope
+public class PartitionSafeKindCache extends AbstractPartitionSafeCache<String, String> {
+ @Inject
+ private IKindCache cache;
+
+ @Override
+ public void put(String s, String o) {
+ this.cache.put(cacheKey(s), o);
+ }
+
+ @Override
+ public String get(String s) {
+ return this.cache.get(cacheKey(s));
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(cacheKey(s));
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafeParentChildRelationshipSpecsCache.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafeParentChildRelationshipSpecsCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..1ca0c87aa4215064fd15d4f9674db0830184994e
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafeParentChildRelationshipSpecsCache.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.indexer.model.indexproperty.ParentChildRelationshipSpecs;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+import javax.inject.Inject;
+
+@Component
+@RequestScope
+public class PartitionSafeParentChildRelationshipSpecsCache extends AbstractPartitionSafeCache<String, ParentChildRelationshipSpecs> {
+ @Inject
+ private IParentChildRelationshipSpecsCache cache;
+
+ @Override
+ public void put(String s, ParentChildRelationshipSpecs o) {
+ this.cache.put(cacheKey(s), o);
+ }
+
+ @Override
+ public ParentChildRelationshipSpecs get(String s) {
+ return this.cache.get(cacheKey(s));
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(cacheKey(s));
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafePropertyConfigurationsCache.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafePropertyConfigurationsCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..04a4d4082d91071722a2bc3500f86fd676ecae11
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafePropertyConfigurationsCache.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.indexer.model.indexproperty.PropertyConfigurations;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+import javax.inject.Inject;
+
+@Component
+@RequestScope
+public class PartitionSafePropertyConfigurationsCache extends AbstractPartitionSafeCache<String,PropertyConfigurations> {
+ @Inject
+ private IPropertyConfigurationsCache cache;
+
+ @Override
+ public void put(String s, PropertyConfigurations o) {
+ this.cache.put(cacheKey(s), o);
+ }
+
+ @Override
+ public PropertyConfigurations get(String s) {
+ return this.cache.get(cacheKey(s));
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(cacheKey(s));
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafePropertyConfigurationsEnabledCache.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafePropertyConfigurationsEnabledCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..e8bea9da70c3903302e5172dd20bcd9a622687b3
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafePropertyConfigurationsEnabledCache.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+import javax.inject.Inject;
+
+@Component
+@RequestScope
+public class PartitionSafePropertyConfigurationsEnabledCache extends AbstractPartitionSafeCache<String,Boolean> {
+ @Inject
+ private IPropertyConfigurationsEnabledCache cache;
+
+ @Override
+ public void put(String s, Boolean o) {
+ this.cache.put(cacheKey(s), o);
+ }
+
+ @Override
+ public Boolean get(String s) {
+ return this.cache.get(cacheKey(s));
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(cacheKey(s));
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafeSchemaCache.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafeSchemaCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..4cbb38658b60e8eedc209e8f75023d8f983ccf02
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafeSchemaCache.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+import javax.inject.Inject;
+
+@Component
+@RequestScope
+public class PartitionSafeSchemaCache extends AbstractPartitionSafeCache<String, String> {
+ @Inject
+ private ISchemaCache schemaCache;
+
+ @Override
+ public void put(String s, String o) {
+ this.schemaCache.put(cacheKey(s), o);
+ }
+
+ @Override
+ public String get(String s) {
+ return (String)this.schemaCache.get(cacheKey(s));
+ }
+
+ @Override
+ public void delete(String s) {
+ this.schemaCache.delete(cacheKey(s));
+ }
+
+ @Override
+ public void clearAll() {
+ this.schemaCache.clearAll();
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PropertyConfigurationsCacheVmImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PropertyConfigurationsCacheVmImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..4e8e0605237f42572ae0f4eb85e59c26a3d8248e
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PropertyConfigurationsCacheVmImpl.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.VmCache;
+import org.opengroup.osdu.indexer.model.Constants;
+import org.opengroup.osdu.indexer.model.indexproperty.PropertyConfigurations;
+import org.springframework.stereotype.Component;
+
+@Component
+public class PropertyConfigurationsCacheVmImpl implements IPropertyConfigurationsCache {
+ private VmCache<String, PropertyConfigurations> cache;
+
+ public PropertyConfigurationsCacheVmImpl() {
+ cache = new VmCache<>(Constants.SPEC_CACHE_EXPIRATION, Constants.SPEC_MAX_CACHE_SIZE);
+ }
+
+ @Override
+ public void put(String s, PropertyConfigurations o) {
+ this.cache.put(s, o);
+ }
+
+ @Override
+ public PropertyConfigurations get(String s) {
+ return this.cache.get(s);
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(s);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PropertyConfigurationsEnabledCacheVmImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PropertyConfigurationsEnabledCacheVmImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..99d2b6d3a6b8f09001164804d6135d06f659e4b5
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PropertyConfigurationsEnabledCacheVmImpl.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.VmCache;
+import org.opengroup.osdu.indexer.model.Constants;
+import org.springframework.stereotype.Component;
+
+@Component
+public class PropertyConfigurationsEnabledCacheVmImpl implements IPropertyConfigurationsEnabledCache {
+ private VmCache<String, Boolean> cache;
+
+ public PropertyConfigurationsEnabledCacheVmImpl() {
+ cache = new VmCache<>(Constants.SPEC_CACHE_EXPIRATION, Constants.SPEC_MAX_CACHE_SIZE);
+ }
+
+ @Override
+ public void put(String s, Boolean o) {
+ this.cache.put(s,o);
+ }
+
+ @Override
+ public Boolean get(String s) {
+ return this.cache.get(s);
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(s);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/RecordChangeInfoCacheVmImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/RecordChangeInfoCacheVmImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..24a3467f8037d4d78b97f034684f31742ca149db
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/RecordChangeInfoCacheVmImpl.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.VmCache;
+import org.opengroup.osdu.indexer.model.Constants;
+import org.opengroup.osdu.indexer.model.RecordChangeInfo;
+import org.springframework.stereotype.Component;
+
+@Component
+public class RecordChangeInfoCacheVmImpl implements IRecordChangeInfoCache {
+ private VmCache<String, RecordChangeInfo> cache;
+
+ public RecordChangeInfoCacheVmImpl() {
+ cache = new VmCache<>(Constants.DATA_CACHE_EXPIRATION, Constants.DATA_MAX_CACHE_SIZE);
+ }
+
+ @Override
+ public void put(String s, RecordChangeInfo o) {
+ this.cache.put(s, o);
+ }
+
+ @Override
+ public RecordChangeInfo get(String s) {
+ return this.cache.get(s);
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(s);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/RelatedObjectCacheVmImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/RelatedObjectCacheVmImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..c1a6f6988f7d4ebfafb9896c1583f4f30faa5b01
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/RelatedObjectCacheVmImpl.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.VmCache;
+import org.opengroup.osdu.indexer.model.Constants;
+import org.springframework.stereotype.Component;
+
+import java.util.Map;
+
+@Component
+public class RelatedObjectCacheVmImpl implements IRelatedObjectCache {
+ private VmCache<String, Map<String, Object>> cache;
+
+ public RelatedObjectCacheVmImpl() {
+ cache = new VmCache<>(Constants.DATA_CACHE_EXPIRATION, Constants.DATA_MAX_CACHE_SIZE);
+ }
+
+ @Override
+ public void put(String s, Map<String, Object> o) {
+ this.cache.put(s, o);
+ }
+
+ @Override
+ public Map<String, Object> get(String s) {
+ return this.cache.get(s);
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(s);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/Constants.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/Constants.java
new file mode 100644
index 0000000000000000000000000000000000000000..91017c2fe1fbfa1773fcd611773849e47a889d5c
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/Constants.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model;
+
+public class Constants {
+ // It should be moved to core common later
+ public static final String ANCESTRY_KINDS = "ancestry_kinds";
+
+ // Specifications using kind as key is not partition safe if the specifications are per data partition
+ public static final int SPEC_CACHE_EXPIRATION = 600;
+ public static final int SPEC_MAX_CACHE_SIZE = 2000;
+
+ // Data id itself is partition safe
+ public static final int DATA_CACHE_EXPIRATION = 120;
+ public static final int DATA_MAX_CACHE_SIZE = 2000;
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/RecordChangeInfo.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/RecordChangeInfo.java
new file mode 100644
index 0000000000000000000000000000000000000000..94dc1c62ed435796310e3545dabb91282d20cfba
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/RecordChangeInfo.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model;
+
+import lombok.Data;
+import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
+
+import java.util.List;
+
+@Data
+public class RecordChangeInfo {
+ private List<String> updatedProperties;
+ private RecordInfo recordInfo;
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SchemaIdentity.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SchemaIdentity.java
new file mode 100644
index 0000000000000000000000000000000000000000..0ad77f4229ce06827993d2b4090bd7da3c54ecd6
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SchemaIdentity.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import lombok.Data;
+
+@Data
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class SchemaIdentity {
+ String authority;
+ String source;
+ private String entityType;
+ private int schemaVersionMajor;
+ private int schemaVersionMinor;
+ private int schemaVersionPatch;
+ private String id;
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SchemaInfo.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SchemaInfo.java
new file mode 100644
index 0000000000000000000000000000000000000000..3cfa9ceb8cdb3e5378922b7303347134a662b6bf
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SchemaInfo.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model;
+
+import lombok.Data;
+import lombok.ToString;
+
+@Data
+@ToString
+public class SchemaInfo {
+ private SchemaIdentity schemaIdentity;
+ private String status;
+ private String scope;
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SchemaInfoResponse.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SchemaInfoResponse.java
new file mode 100644
index 0000000000000000000000000000000000000000..088d1a07b05d5e4a1c8d6883a87f93366dec8775
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SchemaInfoResponse.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model;
+
+import lombok.Data;
+import lombok.ToString;
+
+import java.util.List;
+
+@Data
+@ToString
+public class SchemaInfoResponse {
+ private List<SchemaInfo> schemaInfos;
+ private int offset;
+ private int count;
+ private int totalCount;
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SearchRecord.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SearchRecord.java
new file mode 100644
index 0000000000000000000000000000000000000000..5b15f66bc6b4952dd7e595b480bcc2ea9c02ce0f
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SearchRecord.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import lombok.Data;
+import lombok.ToString;
+
+import java.util.Map;
+
+@Data
+@ToString
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class SearchRecord {
+ private String id;
+ private String kind;
+ private Map<String, Object> data;
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SearchRequest.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SearchRequest.java
new file mode 100644
index 0000000000000000000000000000000000000000..c36bde212ed91a0f2e6a723b97488d7aaab47366
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SearchRequest.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model;
+
+import lombok.Data;
+import lombok.ToString;
+
+import javax.validation.constraints.NotNull;
+import java.util.List;
+
+@Data
+@ToString
+public class SearchRequest {
+ @NotNull(message = "Kind is missing")
+ private Object kind;
+ private String query;
+ private int limit;
+ private int offset;
+ private String cursor;
+ private List<String> returnedFields;
+ private boolean trackTotalCount = true;
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SearchResponse.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SearchResponse.java
new file mode 100644
index 0000000000000000000000000000000000000000..61136de58ce26dc958780dc1574dd6718cde4df0
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SearchResponse.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model;
+
+import lombok.Data;
+import lombok.ToString;
+
+import java.util.List;
+
+@Data
+@ToString
+public class SearchResponse {
+ private String cursor;
+ private List<SearchRecord> results;
+ private int totalCount;
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/ParentChildRelationshipSpec.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/ParentChildRelationshipSpec.java
new file mode 100644
index 0000000000000000000000000000000000000000..052479a9172ba06951dde9392a598d673294c564
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/ParentChildRelationshipSpec.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model.indexproperty;
+
+import lombok.Data;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@Data
+public class ParentChildRelationshipSpec {
+ private String parentKind;
+ private String parentObjectIdPath;
+ private String childKind;
+ private List<String> childValuePaths;
+
+ public ParentChildRelationshipSpec() {
+ childValuePaths = new ArrayList<>();
+ }
+
+ @Override
+ public boolean equals(Object another) {
+ if(another == null || !(another instanceof ParentChildRelationshipSpec))
+ return false;
+
+ ParentChildRelationshipSpec anotherSpec = (ParentChildRelationshipSpec)another;
+ return this.toString().equals(anotherSpec.toString());
+ }
+
+ @Override
+ public int hashCode() {
+ return this.toString().hashCode();
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder stringBuilder = new StringBuilder();
+ stringBuilder.append((parentKind != null)? parentKind : "__");
+ stringBuilder.append("<>");
+ stringBuilder.append((childKind != null)? childKind : "__");
+ stringBuilder.append("<>");
+ stringBuilder.append((parentObjectIdPath != null)? parentObjectIdPath : "__");
+ return stringBuilder.toString();
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/ParentChildRelationshipSpecs.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/ParentChildRelationshipSpecs.java
new file mode 100644
index 0000000000000000000000000000000000000000..35bd4e2edb8bcbc95f63eead1209a73d6f48489e
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/ParentChildRelationshipSpecs.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model.indexproperty;
+
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+public class ParentChildRelationshipSpecs {
+ private List<ParentChildRelationshipSpec> specList;
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/PropertyConfiguration.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/PropertyConfiguration.java
new file mode 100644
index 0000000000000000000000000000000000000000..de4e53f7a8dd1056d3f36cee3aecf659dd24010c
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/PropertyConfiguration.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model.indexproperty;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Data;
+import lombok.ToString;
+
+import java.util.List;
+
+@Data
+@ToString
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class PropertyConfiguration {
+ private final String EXTRACT_FIRST_MATCH_POLICY = "ExtractFirstMatch";
+ private final String EXTRACT_ALL_MATCHES_POLICY = "ExtractAllMatches";
+
+ @JsonProperty("Name")
+ private String name;
+
+ @JsonProperty("Policy")
+ private String policy;
+
+ @JsonProperty("UseCase")
+ private String useCase;
+
+ @JsonProperty("Paths")
+ private List<PropertyPath> paths;
+
+ public boolean isExtractFirstMatch() {
+ return EXTRACT_FIRST_MATCH_POLICY.equalsIgnoreCase(policy);
+ }
+
+ public boolean isExtractAllMatches() {
+ return EXTRACT_ALL_MATCHES_POLICY.equalsIgnoreCase(policy);
+ }
+
+ public boolean isValid() {
+ boolean hasValidPath = (paths != null && paths.stream().filter(p -> p.isValid()).findFirst().orElse(null) != null);
+ return hasValidPath && (isExtractFirstMatch() || isExtractAllMatches());
+ }
+
+ public String getRelatedObjectKind() {
+ if(paths != null) {
+ for (PropertyPath path : paths) {
+ if (path.isValid() && path.hasValidRelatedObjectsSpec()) {
+ return path.getRelatedObjectsSpec().getRelatedObjectKind();
+ }
+ }
+ }
+ return null;
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/PropertyConfigurations.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/PropertyConfigurations.java
new file mode 100644
index 0000000000000000000000000000000000000000..ab20d8755d9cfd8989ce9ec7fa64a1a7a97fd958
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/PropertyConfigurations.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model.indexproperty;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Strings;
+import lombok.Data;
+import lombok.ToString;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+@Data
+@ToString
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class PropertyConfigurations {
+ @JsonProperty("Name")
+ private String name;
+
+ @JsonProperty("Description")
+ private String description;
+
+ @JsonProperty("Code")
+ private String code;
+
+ @JsonProperty("AttributionAuthority")
+ private String attributionAuthority;
+
+ @JsonProperty("Configurations")
+ private List<PropertyConfiguration> configurations;
+
+ public List<String> getUniqueRelatedObjectKinds() {
+ if(configurations == null || configurations.isEmpty())
+ return new ArrayList<>();
+
+ Set<String> relatedObjectKinds = new HashSet<>();
+ for(PropertyConfiguration configuration : configurations) {
+ String relatedObjectKind = configuration.getRelatedObjectKind();
+ if(!Strings.isNullOrEmpty(relatedObjectKind)) {
+ relatedObjectKinds.add(relatedObjectKind);
+ }
+ }
+ return new ArrayList<>(relatedObjectKinds);
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/PropertyPath.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/PropertyPath.java
new file mode 100644
index 0000000000000000000000000000000000000000..e79a171ac2e2e66c5c3e3a450c6903c43274b818
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/PropertyPath.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model.indexproperty;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import lombok.Data;
+import lombok.ToString;
+import org.opengroup.osdu.indexer.model.indexproperty.jackson.PropertyPathDeserializer;
+
+@Data
+@ToString
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonDeserialize(using = PropertyPathDeserializer.class)
+public class PropertyPath {
+ private RelatedObjectsSpec relatedObjectsSpec;
+
+ private ValueExtraction valueExtraction;
+
+ public boolean hasValidRelatedObjectsSpec() {
+ return relatedObjectsSpec != null && relatedObjectsSpec.isValid();
+ }
+
+ public boolean hasValidValueExtraction() {
+ return valueExtraction != null && valueExtraction.isValid();
+ }
+
+ public boolean isValid() {
+ if(relatedObjectsSpec != null) {
+ return hasValidRelatedObjectsSpec() && hasValidValueExtraction();
+ }
+ else {
+ return hasValidValueExtraction();
+ }
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/RelatedCondition.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/RelatedCondition.java
new file mode 100644
index 0000000000000000000000000000000000000000..ba9683e26dfb1c73e52cf03d77200155c7ed2727
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/RelatedCondition.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model.indexproperty;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.google.api.client.util.Strings;
+import lombok.Data;
+import lombok.ToString;
+
+import java.util.List;
+
+@Data
+@ToString
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class RelatedCondition {
+ protected static final String ARRAY_SYMBOL = "[]";
+
+ protected String relatedConditionProperty;
+
+ protected List<String> relatedConditionMatches;
+
+ protected boolean hasValidCondition(String property) {
+ if(Strings.isNullOrEmpty(property) ||
+ Strings.isNullOrEmpty(relatedConditionProperty) ||
+ relatedConditionMatches == null ||
+ relatedConditionMatches.isEmpty())
+ return false;
+
+ if(property.indexOf(ARRAY_SYMBOL + "." ) <= 0 || property.endsWith(ARRAY_SYMBOL) ||
+ relatedConditionProperty.indexOf(ARRAY_SYMBOL + "." ) <= 0 || relatedConditionProperty.endsWith(ARRAY_SYMBOL))
+ return false;
+
+ String delimiter = "\\[\\]\\.";
+ String[] propertyParts = property.split(delimiter);
+ String[] relatedConditionPropertyParts = relatedConditionProperty.split(delimiter);
+ if(propertyParts.length != relatedConditionPropertyParts.length || propertyParts.length < 2)
+ return false;
+
+ for(int i = 0; i < propertyParts.length -1; i++) {
+ if(!propertyParts[i].equals(relatedConditionPropertyParts[i]))
+ return false;
+ }
+ return true;
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/RelatedObjectsSpec.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/RelatedObjectsSpec.java
new file mode 100644
index 0000000000000000000000000000000000000000..8ff9b57f55e75af1ebaeace577c8475426a0a2c5
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/RelatedObjectsSpec.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model.indexproperty;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.google.api.client.util.Strings;
+import lombok.Data;
+import lombok.ToString;
+
+@Data
+@ToString
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class RelatedObjectsSpec extends RelatedCondition {
+ public final static String CHILD_TO_PARENT = "ChildToParent";
+ public final static String PARENT_TO_CHILDREN = "ParentToChildren";
+
+ private String relatedObjectID;
+
+ private String relatedObjectKind;
+
+ private String relationshipDirection;
+
+ public boolean isChildToParent() { return CHILD_TO_PARENT.equalsIgnoreCase(relationshipDirection); }
+
+ public boolean isParentToChildren() { return PARENT_TO_CHILDREN.equalsIgnoreCase(relationshipDirection); }
+
+ public boolean isValid() {
+ return !Strings.isNullOrEmpty(relatedObjectID) && !Strings.isNullOrEmpty(relatedObjectKind) &&
+ (isChildToParent() || isParentToChildren());
+ }
+
+ /**
+ * To have a valid hasValidCondition, both relatedConditionProperty and relatedObjectID must refer to the same nested object but different property
+ * Only one level of nested is supported for now
+ * @return
+ */
+ public boolean hasValidCondition() {
+ return isValid() && super.hasValidCondition(relatedObjectID);
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/ValueExtraction.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/ValueExtraction.java
new file mode 100644
index 0000000000000000000000000000000000000000..1ae141d2b31595747afb9dfc1e8a806824a00c33
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/ValueExtraction.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model.indexproperty;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.api.client.util.Strings;
+import lombok.Data;
+import lombok.ToString;
+
+@Data
+@ToString
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class ValueExtraction extends RelatedCondition {
+ @JsonProperty("ValuePath")
+ private String valuePath;
+
+ public boolean isValid() {
+ return !Strings.isNullOrEmpty(valuePath);
+ }
+
+ /**
+ * To have a valid hasValidCondition, both relatedConditionProperty and relatedObjectID must refer to the same nested object but different property
+ * Only one level of nested is supported for now
+ * @return
+ */
+ public boolean hasValidCondition() {
+ return isValid() && super.hasValidCondition(valuePath);
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/jackson/PropertyPathDeserializer.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/jackson/PropertyPathDeserializer.java
new file mode 100644
index 0000000000000000000000000000000000000000..687776ed62597eab4365c3338607c03e5ff8a2ba
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/jackson/PropertyPathDeserializer.java
@@ -0,0 +1,114 @@
+package org.opengroup.osdu.indexer.model.indexproperty.jackson;
+
+import com.fasterxml.jackson.core.JacksonException;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.ObjectCodec;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.JsonNode;
+import org.opengroup.osdu.indexer.model.indexproperty.PropertyPath;
+import org.opengroup.osdu.indexer.model.indexproperty.RelatedObjectsSpec;
+import org.opengroup.osdu.indexer.model.indexproperty.ValueExtraction;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class PropertyPathDeserializer extends JsonDeserializer<PropertyPath> {
+ private final String RELATED_OBJECTS_SPEC_RELATIONSHIP_DIRECTION = "RelatedObjectsSpec.RelationshipDirection";
+ private final String RELATED_OBJECTS_SPEC_RELATED_OBJECT_KIND = "RelatedObjectsSpec.RelatedObjectKind";
+ private final String RELATED_OBJECTS_SPEC_RELATED_OBJECT_ID = "RelatedObjectsSpec.RelatedObjectID";
+ private final String RELATED_OBJECTS_SPEC_RELATED_CONDITION_PROPERTY = "RelatedObjectsSpec.RelatedConditionProperty";
+ private final String RELATED_OBJECTS_SPEC_RELATED_CONDITION_MATCHES = "RelatedObjectsSpec.RelatedConditionMatches";
+
+ private final String VALUE_EXTRACTION_VALUE_PATH = "ValueExtraction.ValuePath";
+ private final String VALUE_EXTRACTION_RELATED_CONDITION_PROPERTY = "ValueExtraction.RelatedConditionProperty";
+ private final String VALUE_EXTRACTION_RELATED_CONDITION_MATCHES = "ValueExtraction.RelatedConditionMatches";
+
+ @Override
+ public PropertyPath deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException, JacksonException {
+ PropertyPath propertyPath = new PropertyPath();
+ ObjectCodec codec = jsonParser.getCodec();
+ JsonNode node = codec.readTree(jsonParser);
+
+ RelatedObjectsSpec relatedObjectsSpec = deserializeRelatedObjects(node);
+ if(relatedObjectsSpec != null) {
+ propertyPath.setRelatedObjectsSpec(relatedObjectsSpec);
+ }
+ ValueExtraction valueExtraction = deserializeValueExtraction(node);
+ if(valueExtraction != null) {
+ propertyPath.setValueExtraction(valueExtraction);
+ }
+ return propertyPath;
+ }
+
+ private RelatedObjectsSpec deserializeRelatedObjects(JsonNode node) {
+ JsonNode relationshipDirection = node.get(RELATED_OBJECTS_SPEC_RELATIONSHIP_DIRECTION);
+ JsonNode relatedObjectKind = node.get(RELATED_OBJECTS_SPEC_RELATED_OBJECT_KIND);
+ JsonNode relatedObjectID = node.get(RELATED_OBJECTS_SPEC_RELATED_OBJECT_ID);
+ JsonNode relatedConditionProperty = node.get(RELATED_OBJECTS_SPEC_RELATED_CONDITION_PROPERTY);
+ JsonNode relatedConditionMatches = node.get(RELATED_OBJECTS_SPEC_RELATED_CONDITION_MATCHES);
+
+ if(isNotNull(relationshipDirection) ||
+ isNotNull(relatedObjectKind) ||
+ isNotNull(relatedObjectID) ||
+ isNotNull(relatedConditionProperty) ||
+ isNotNull(relatedConditionMatches)) {
+ RelatedObjectsSpec relatedObjectsSpec = new RelatedObjectsSpec();
+ if(isNotNull(relationshipDirection)) {
+ relatedObjectsSpec.setRelationshipDirection(relationshipDirection.asText());
+ }
+ if(isNotNull(relatedObjectKind)) {
+ relatedObjectsSpec.setRelatedObjectKind(relatedObjectKind.asText());
+ }
+ if(isNotNull(relatedObjectID)) {
+ relatedObjectsSpec.setRelatedObjectID(relatedObjectID.asText());
+ }
+ if(isNotNull(relatedConditionProperty)) {
+ relatedObjectsSpec.setRelatedConditionProperty(relatedConditionProperty.asText());
+ }
+ if(isNotNull(relatedConditionMatches) && relatedConditionMatches.isArray()) {
+ List<String> conditionMatches = new ArrayList<>();
+ for (JsonNode subNode : relatedConditionMatches) {
+ conditionMatches.add(subNode.asText());
+ }
+ relatedObjectsSpec.setRelatedConditionMatches(conditionMatches);
+ }
+ return relatedObjectsSpec;
+ }
+
+ return null;
+ }
+
+ private ValueExtraction deserializeValueExtraction(JsonNode node) {
+ JsonNode valuePath = node.get(VALUE_EXTRACTION_VALUE_PATH);
+ JsonNode relatedConditionProperty = node.get(VALUE_EXTRACTION_RELATED_CONDITION_PROPERTY);
+ JsonNode relatedConditionMatches = node.get(VALUE_EXTRACTION_RELATED_CONDITION_MATCHES);
+
+ if(isNotNull(valuePath) ||
+ isNotNull(relatedConditionProperty) ||
+ isNotNull(relatedConditionMatches)) {
+ ValueExtraction valueExtraction = new ValueExtraction();
+ if(isNotNull(valuePath)) {
+ valueExtraction.setValuePath(valuePath.asText());
+ }
+ if(isNotNull(relatedConditionProperty)) {
+ valueExtraction.setRelatedConditionProperty(relatedConditionProperty.asText());
+ }
+ if(isNotNull(relatedConditionMatches) && relatedConditionMatches.isArray()) {
+ List<String> conditionMatches = new ArrayList<>();
+ for (JsonNode subNode : relatedConditionMatches) {
+ conditionMatches.add(subNode.asText());
+ }
+ valueExtraction.setRelatedConditionMatches(conditionMatches);
+ }
+ return valueExtraction;
+ }
+
+ return null;
+ }
+
+ private boolean isNotNull(JsonNode node) {
+ return node != null && !node.isNull();
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java
index 4c985a0ff6abdd442743929318302986efe9fb1a..7d713d649b8fa6a0782943fc42c2c42488235c4d 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java
@@ -16,7 +16,6 @@ package org.opengroup.osdu.indexer.schema.converter;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.api.client.util.Strings;
import com.google.gson.Gson;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.search.Preconditions;
@@ -25,7 +24,7 @@ import org.opengroup.osdu.indexer.schema.converter.exeption.SchemaProcessingExce
import org.opengroup.osdu.indexer.schema.converter.interfaces.IVirtualPropertiesSchemaCache;
import org.opengroup.osdu.indexer.schema.converter.interfaces.SchemaToStorageFormat;
import org.opengroup.osdu.indexer.schema.converter.tags.*;
-import org.opengroup.osdu.indexer.util.VirtualPropertyUtil;
+import org.opengroup.osdu.indexer.util.PropertyUtil;
import org.springframework.stereotype.Component;
import javax.inject.Inject;
@@ -159,12 +158,12 @@ public class SchemaToStorageFormatImpl implements SchemaToStorageFormat {
// The schema for different properties in the list of Priority should be the same
Priority priority = entry.getValue().getPriorities().get(0);
- String virtualPropertyPath = VirtualPropertyUtil.removeDataPrefix(entry.getKey());
- hasVirtualDefaultLocation |= VirtualPropertyUtil.isPropertyPathMatched(virtualPropertyPath, VirtualPropertyUtil.VIRTUAL_DEFAULT_LOCATION);
+ String virtualPropertyPath = PropertyUtil.removeDataPrefix(entry.getKey());
+ hasVirtualDefaultLocation |= PropertyUtil.isPropertyPathMatched(virtualPropertyPath, PropertyUtil.VIRTUAL_DEFAULT_LOCATION);
- String originalPropertyPath = VirtualPropertyUtil.removeDataPrefix(priority.getPath());
+ String originalPropertyPath = PropertyUtil.removeDataPrefix(priority.getPath());
List<Map<String, Object>> matchedItems = storageSchemaItems.stream().filter(item ->
- VirtualPropertyUtil.isPropertyPathMatched((String) item.get("path"), originalPropertyPath))
+ PropertyUtil.isPropertyPathMatched((String) item.get("path"), originalPropertyPath))
.collect(Collectors.toList());
storageSchemaItems.addAll(matchedItems.stream().map(item ->
cloneVirtualProperty(item, virtualPropertyPath, originalPropertyPath))
@@ -173,7 +172,7 @@ public class SchemaToStorageFormatImpl implements SchemaToStorageFormat {
if(hasVirtualDefaultLocation) {
Map<String, Object> isDecimatedProperty = new HashMap<>();
- isDecimatedProperty.put("path", VirtualPropertyUtil.VIRTUAL_DEFAULT_LOCATION_IS_DECIMATED_PATH);
+ isDecimatedProperty.put("path", PropertyUtil.VIRTUAL_DEFAULT_LOCATION_IS_DECIMATED_PATH);
isDecimatedProperty.put("kind", "boolean");
storageSchemaItems.add(isDecimatedProperty);
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaService.java
index d19833f2578ff426e639856ff9ff1f05f46034ea..9b19787c35ec5247e1a88ef4c2f63e68d1a3d421 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaService.java
@@ -40,4 +40,6 @@ public interface IndexSchemaService {
void syncIndexMappingWithStorageSchema(String kind) throws ElasticsearchException, IOException, AppException, URISyntaxException;
boolean isStorageSchemaSyncRequired(String kind, boolean forceClean) throws IOException;
+
+ void invalidateSchemaCache(String kind);
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
index c469a2284aac9861af7e9db941034c8d3cd4ca2b..ab3cebb80195698fe9d3b280c801f95b31281c16 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
@@ -30,10 +30,13 @@ import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
import org.opengroup.osdu.core.common.model.storage.Schema;
import org.opengroup.osdu.core.common.model.storage.SchemaItem;
import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
+import org.opengroup.osdu.indexer.cache.PartitionSafeFlattenedSchemaCache;
+import org.opengroup.osdu.indexer.cache.PartitionSafeSchemaCache;
import org.opengroup.osdu.indexer.model.Kind;
-import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache;
+import org.opengroup.osdu.indexer.model.indexproperty.PropertyConfigurations;
import org.opengroup.osdu.indexer.schema.converter.exeption.SchemaProcessingException;
import org.opengroup.osdu.indexer.schema.converter.interfaces.IVirtualPropertiesSchemaCache;
+import org.opengroup.osdu.indexer.util.AugmenterSetting;
import org.opengroup.osdu.indexer.util.ElasticClientHandler;
import org.opengroup.osdu.indexer.util.TypeMapper;
import org.springframework.stereotype.Service;
@@ -42,15 +45,11 @@ import javax.inject.Inject;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URISyntaxException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
@Service
public class IndexSchemaServiceImpl implements IndexSchemaService {
- private static final String FLATTENED_SCHEMA = "_flattened";
-
private final Gson gson = new Gson();
@Inject
@@ -66,9 +65,15 @@ public class IndexSchemaServiceImpl implements IndexSchemaService {
@Inject
private IndicesService indicesService;
@Inject
- private ISchemaCache schemaCache;
+ private PartitionSafeSchemaCache schemaCache;
+ @Inject
+ private PartitionSafeFlattenedSchemaCache flattenedSchemaCache;
@Inject
private IVirtualPropertiesSchemaCache virtualPropertiesSchemaCache;
+ @Inject
+ private PropertyConfigurationsService propertyConfigurationsService;
+ @Inject
+ private AugmenterSetting augmenterSetting;
public void processSchemaMessages(Map<String, OperationType> schemaMsgs) throws IOException {
try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
@@ -154,25 +159,27 @@ public class IndexSchemaServiceImpl implements IndexSchemaService {
this.invalidateCache(kind);
}
- String schema = (String) this.schemaCache.get(kind);
+ String schema = this.schemaCache.get(kind);
if (Strings.isNullOrEmpty(schema)) {
// get from storage
schema = this.schemaProvider.getSchema(kind);
if (Strings.isNullOrEmpty(schema)) {
return this.getEmptySchema(kind);
} else {
- // cache the schema
- this.schemaCache.put(kind, schema);
- // get flatten schema and cache it
- IndexSchema flatSchemaObj = normalizeSchema(schema);
- if (flatSchemaObj != null) {
- this.schemaCache.put(kind + FLATTENED_SCHEMA, gson.toJson(flatSchemaObj));
+ if(augmenterSetting.isEnabled()) {
+ // Merge schema of the extended properties if needed
+ PropertyConfigurations propertyConfigurations = propertyConfigurationsService.getPropertyConfigurations(kind);
+ if (propertyConfigurations != null) {
+ schema = mergeSchemaFromPropertyConfiguration(schema, propertyConfigurations);
+ }
}
+
+ IndexSchema flatSchemaObj = cacheAndNormalizeSchema(kind, schema);
return flatSchemaObj;
}
} else {
// search flattened schema in memcache
- String flattenedSchema = (String) this.schemaCache.get(kind + FLATTENED_SCHEMA);
+ String flattenedSchema = this.flattenedSchemaCache.get(kind);
if (Strings.isNullOrEmpty(flattenedSchema)) {
return this.getEmptySchema(kind);
}
@@ -180,6 +187,57 @@ public class IndexSchemaServiceImpl implements IndexSchemaService {
}
}
+ private IndexSchema cacheAndNormalizeSchema(String kind, String schema) {
+ // cache the schema
+ this.schemaCache.put(kind, schema);
+ // get flatten schema and cache it
+ IndexSchema flatSchemaObj = normalizeSchema(schema);
+ if (flatSchemaObj != null) {
+ this.flattenedSchemaCache.put(kind, gson.toJson(flatSchemaObj));
+ }
+ return flatSchemaObj;
+ }
+
+ private String mergeSchemaFromPropertyConfiguration(String originalSchemaStr, PropertyConfigurations propertyConfigurations) throws UnsupportedEncodingException, URISyntaxException {
+ Map<String, Schema> relatedObjectKindSchemas = getSchemaOfRelatedObjectKinds(propertyConfigurations);
+ Schema originalSchema = gson.fromJson(originalSchemaStr, Schema.class);
+ List<SchemaItem> extendedSchemaItems = propertyConfigurationsService.getExtendedSchemaItems(originalSchema, relatedObjectKindSchemas, propertyConfigurations);
+ if (!extendedSchemaItems.isEmpty()) {
+ List<SchemaItem> originalSchemaItems = new ArrayList<>(Arrays.asList(originalSchema.getSchema()));
+ originalSchemaItems.addAll(extendedSchemaItems);
+ originalSchema.setSchema(originalSchemaItems.toArray(new SchemaItem[0]));
+ return gson.toJson(originalSchema);
+ } else {
+ return originalSchemaStr;
+ }
+ }
+
+ private Map<String, Schema> getSchemaOfRelatedObjectKinds(PropertyConfigurations propertyConfigurations) throws UnsupportedEncodingException, URISyntaxException {
+ List<String> relatedObjectKinds = propertyConfigurations.getUniqueRelatedObjectKinds();
+ Map<String, Schema> relatedObjectKindSchemas = new HashMap<>();
+ for (String relatedObjectKind : relatedObjectKinds) {
+ // The relatedObjectKind defined in property configuration can be kind having major version only
+ // e.g. "RelatedObjectKind": "osdu:wks:master-data--Wellbore:1."
+ String concreteRelatedObjectKind = propertyConfigurationsService.resolveConcreteKind(relatedObjectKind);
+ if (Strings.isNullOrEmpty(concreteRelatedObjectKind))
+ continue;
+
+ String relatedObjectKindSchema = this.schemaCache.get(concreteRelatedObjectKind);
+ if (Strings.isNullOrEmpty(relatedObjectKindSchema)) {
+ relatedObjectKindSchema = this.schemaProvider.getSchema(concreteRelatedObjectKind);
+ if (!Strings.isNullOrEmpty(relatedObjectKindSchema)) {
+ cacheAndNormalizeSchema(concreteRelatedObjectKind, relatedObjectKindSchema);
+ }
+ }
+
+ if (!Strings.isNullOrEmpty(relatedObjectKindSchema)) {
+ Schema schema = gson.fromJson(relatedObjectKindSchema, Schema.class);
+ relatedObjectKindSchemas.put(relatedObjectKind, schema);
+ }
+ }
+ return relatedObjectKindSchemas;
+ }
+
private IndexSchema getEmptySchema(String kind) {
Schema basicSchema = Schema.builder().kind(kind).build();
return normalizeSchema(gson.toJson(basicSchema));
@@ -205,14 +263,15 @@ public class IndexSchemaServiceImpl implements IndexSchemaService {
}
}
- private void invalidateCache(String kind) {
- String schema = (String) this.schemaCache.get(kind);
- if (!Strings.isNullOrEmpty(schema)) this.schemaCache.delete(kind);
-
- String flattenSchema = (String) this.schemaCache.get(kind + FLATTENED_SCHEMA);
- if (!Strings.isNullOrEmpty(flattenSchema)) this.schemaCache.delete(kind + FLATTENED_SCHEMA);
+ @Override
+ public void invalidateSchemaCache(String kind) {
+ this.invalidateCache(kind);
+ }
- virtualPropertiesSchemaCache.delete(kind);
+ private void invalidateCache(String kind) {
+ this.schemaCache.delete(kind);
+ this.flattenedSchemaCache.delete(kind);
+ this.virtualPropertiesSchemaCache.delete(kind);
}
private IndexSchema normalizeSchema(String schemaStr) throws AppException {
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
index f0e6ad373d6b811a604726819c74a14a0da7b66c..cdae2cb43b3c99b3cbc3376867f7c1e7cd142218 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
@@ -17,9 +17,6 @@ package org.opengroup.osdu.indexer.service;
import com.google.common.base.Strings;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
-import java.util.Collections;
-import lombok.AllArgsConstructor;
-import lombok.Data;
import org.apache.http.HttpStatus;
import org.elasticsearch.ElasticsearchStatusException;
import org.elasticsearch.action.DocWriteRequest;
@@ -28,7 +25,6 @@ import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.unit.TimeValue;
@@ -48,7 +44,9 @@ import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
import org.opengroup.osdu.indexer.logging.AuditLogger;
import org.opengroup.osdu.indexer.model.BulkRequestResult;
+import org.opengroup.osdu.indexer.model.indexproperty.PropertyConfigurations;
import org.opengroup.osdu.indexer.provider.interfaces.IPublisher;
+import org.opengroup.osdu.indexer.util.AugmenterSetting;
import org.opengroup.osdu.indexer.util.ElasticClientHandler;
import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
import org.springframework.context.annotation.Primary;
@@ -58,6 +56,7 @@ import javax.inject.Inject;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@@ -106,6 +105,10 @@ public class IndexerServiceImpl implements IndexerService {
private IRequestInfo requestInfo;
@Inject
private JobStatus jobStatus;
+ @Inject
+ private PropertyConfigurationsService propertyConfigurationsService;
+ @Inject
+ private AugmenterSetting augmenterSetting;
private DpsHeaders headers;
@@ -153,6 +156,14 @@ public class IndexerServiceImpl implements IndexerService {
if (retryRecordIds.size() > 0) {
retryAndEnqueueFailedRecords(recordInfos, retryRecordIds, message);
}
+
+ if(this.augmenterSetting.isEnabled()) {
+ Map<String, List<String>> upsertKindIds = getUpsertRecordIdsForConfigurationsEnabledKinds(upsertRecordMap, retryRecordIds);
+ Map<String, List<String>> deleteKindIds = getDeleteRecordIdsForConfigurationsEnabledKinds(deleteRecordMap, retryRecordIds);
+ if (!upsertKindIds.isEmpty() || !deleteKindIds.isEmpty()) {
+ propertyConfigurationsService.updateAssociatedRecords(message, upsertKindIds, deleteKindIds);
+ }
+ }
} catch (IOException e) {
errorMessage = e.getMessage();
throw new AppException(HttpStatus.SC_GATEWAY_TIMEOUT, "Internal communication failure", errorMessage, e);
@@ -187,6 +198,34 @@ public class IndexerServiceImpl implements IndexerService {
}
}
+ private Map<String, List<String>> getUpsertRecordIdsForConfigurationsEnabledKinds(Map<String, Map<String, OperationType>> upsertRecordMap, List<String> retryRecordIds) {
+ Map<String, List<String>> upsertKindIds = new HashMap<>();
+ for (Map.Entry<String, Map<String, OperationType>> entry : upsertRecordMap.entrySet()) {
+ String kind = entry.getKey();
+ if(propertyConfigurationsService.isPropertyConfigurationsEnabled(kind)) {
+ List<String> processedIds = entry.getValue().keySet().stream().filter(id -> !retryRecordIds.contains(id)).collect(Collectors.toList());
+ if (!processedIds.isEmpty()) {
+ upsertKindIds.put(kind, processedIds);
+ }
+ }
+ }
+ return upsertKindIds;
+ }
+
+ private Map<String, List<String>> getDeleteRecordIdsForConfigurationsEnabledKinds(Map<String, List<String>> deleteRecordMap, List<String> retryRecordIds) {
+ Map<String, List<String>> deletedRecordKindIdsMap = new HashMap<>();
+ for (Map.Entry<String, List<String>> entry : deleteRecordMap.entrySet()) {
+ String kind = entry.getKey();
+ if(propertyConfigurationsService.isPropertyConfigurationsEnabled(kind)) {
+ List<String> processedIds = entry.getValue().stream().filter(id -> !retryRecordIds.contains(id)).collect(Collectors.toList());
+ if (!processedIds.isEmpty()) {
+ deletedRecordKindIdsMap.put(kind, processedIds);
+ }
+ }
+ }
+ return deletedRecordKindIdsMap;
+ }
+
private void processSchemaEvents(RestHighLevelClient restClient,
Map.Entry<String, OperationType> msg) throws IOException, ElasticsearchStatusException {
String kind = msg.getKey();
@@ -195,6 +234,7 @@ public class IndexerServiceImpl implements IndexerService {
boolean indexExist = indicesService.isIndexExist(restClient, index);
if (indexExist && msg.getValue() == OperationType.purge_schema) {
indicesService.deleteIndex(restClient, index);
+ schemaService.invalidateSchemaCache(kind);
}
}
@@ -304,6 +344,19 @@ public class IndexerServiceImpl implements IndexerService {
String message = String.format("complete schema mismatch: none of the data attribute can be mapped | data: %s", storageRecordData);
this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.WARN, HttpStatus.SC_NOT_FOUND, message, String.format("record-id: %s | %s", storageRecord.getId(), message));
}
+
+ if(this.augmenterSetting.isEnabled()) {
+ if(propertyConfigurationsService.isPropertyConfigurationsEnabled(storageRecord.getKind())) {
+ PropertyConfigurations propertyConfigurations = propertyConfigurationsService.getPropertyConfigurations(storageRecord.getKind());
+ if (propertyConfigurations != null) {
+ // Merge extended properties
+ dataMap = mergeDataFromPropertyConfiguration(storageRecord.getId(), dataMap, propertyConfigurations);
+ }
+ // We cache the dataMap in case the update of this object will trigger update of the related objects.
+ propertyConfigurationsService.cacheDataRecord(storageRecord.getId(), storageRecord.getKind(), dataMap);
+ }
+ }
+
document.setData(dataMap);
}
} catch (AppException e) {
@@ -358,6 +411,15 @@ public class IndexerServiceImpl implements IndexerService {
return document;
}
+ private Map<String, Object> mergeDataFromPropertyConfiguration(String objectId, Map<String, Object> originalDataMap, PropertyConfigurations propertyConfigurations) {
+ Map<String, Object> extendedDataMap = propertyConfigurationsService.getExtendedProperties(objectId, originalDataMap, propertyConfigurations);
+ if (!extendedDataMap.isEmpty()) {
+ originalDataMap.putAll(extendedDataMap);
+ }
+
+ return originalDataMap;
+ }
+
private List<String> processElasticMappingAndUpsertRecords(RecordIndexerPayload recordIndexerPayload) throws Exception {
try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/PropertyConfigurationsService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/PropertyConfigurationsService.java
new file mode 100644
index 0000000000000000000000000000000000000000..7aa33277a5b925f0a9c9191b16300059b87f7bbd
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/PropertyConfigurationsService.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.service;
+
+import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
+import org.opengroup.osdu.core.common.model.storage.Schema;
+import org.opengroup.osdu.core.common.model.storage.SchemaItem;
+import org.opengroup.osdu.indexer.model.indexproperty.PropertyConfigurations;
+
+import java.util.List;
+import java.util.Map;
+
+public interface PropertyConfigurationsService {
+ boolean isPropertyConfigurationsEnabled(String kind);
+
+ PropertyConfigurations getPropertyConfigurations(String kind);
+
+ Map<String, Object> getExtendedProperties(String objectId, Map<String, Object> originalDataMap, PropertyConfigurations propertyConfigurations);
+
+ List<SchemaItem> getExtendedSchemaItems(Schema originalSchema, Map<String, Schema> relatedObjectKindSchemas, PropertyConfigurations propertyConfigurations);
+
+ String resolveConcreteKind(String kind);
+
+ void cacheDataRecord(String recordId, String kind, Map<String, Object> dataMap);
+
+ void updateAssociatedRecords(RecordChangedMessages message, Map<String, List<String>> upsertKindIds, Map<String, List<String>> deleteKindIds);
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/PropertyConfigurationsServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/PropertyConfigurationsServiceImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..5bb37704e6d48caaa5ee75f21fc6beeec91cf5db
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/PropertyConfigurationsServiceImpl.java
@@ -0,0 +1,951 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.service;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Strings;
+import com.google.gson.Gson;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.indexer.OperationType;
+import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
+import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
+import org.opengroup.osdu.core.common.model.storage.Schema;
+import org.opengroup.osdu.core.common.model.storage.SchemaItem;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.opengroup.osdu.indexer.cache.*;
+import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.opengroup.osdu.indexer.model.*;
+import org.opengroup.osdu.indexer.model.indexproperty.*;
+import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
+import org.opengroup.osdu.indexer.util.PropertyUtil;
+import org.springframework.stereotype.Component;
+
+import javax.inject.Inject;
+import java.io.UnsupportedEncodingException;
+import java.net.URISyntaxException;
+import java.util.*;
+import java.util.stream.Collectors;
+
+
+@Component
+public class PropertyConfigurationsServiceImpl implements PropertyConfigurationsService {
+ private static final String ASSOCIATED_IDENTITIES_PROPERTY = "AssociatedIdentities";
+ private static final String ASSOCIATED_IDENTITIES_PROPERTY_STORAGE_FORMAT_TYPE = "[]string";
+ private static final String WILD_CARD_KIND = "*:*:*:*";
+ private static final String INDEX_PROPERTY_PATH_CONFIGURATION_KIND = "osdu:wks:reference-data--IndexPropertyPathConfiguration:*";
+ private static final String ANCESTRY_KINDS_DELIMITER = ",";
+ private static final String PARENT_CHILDREN_CONFIGURATION_QUERY_FORMAT =
+ "nested(data.Configurations, nested(data.Configurations.Paths, (RelatedObjectsSpec.RelationshipDirection: ParentToChildren AND RelatedObjectsSpec.RelatedObjectKind:\"%s\")))";
+ private static final String HAS_CONFIGURATIONS_QUERY_FORMAT = "data.Code: \"%s\" OR nested(data.Configurations, nested(data.Configurations.Paths, (RelatedObjectsSpec.RelatedObjectKind:\"%s\")))";
+ private static final int MAX_SEARCH_LIMIT = 1000;
+
+ private static final String PROPERTY_DELIMITER = ".";
+ private static final String NESTED_OBJECT_DELIMITER = "[].";
+ private static final String ARRAY_SYMBOL = "[]";
+ private static final String SCHEMA_NESTED_KIND = "nested";
+
+ private final Gson gson = new Gson();
+ private final ObjectMapper objectMapper = new ObjectMapper();
+ private final PropertyConfigurations EMPTY_CONFIGURATIONS = new PropertyConfigurations();
+
+ @Inject
+ private IndexerConfigurationProperties configurationProperties;
+ @Inject
+ private PartitionSafePropertyConfigurationsCache propertyConfigurationCache;
+ @Inject
+ private PartitionSafePropertyConfigurationsEnabledCache propertyConfigurationsEnabledCache;
+ @Inject
+ private PartitionSafeParentChildRelationshipSpecsCache parentChildRelationshipSpecsCache;
+ @Inject
+ private PartitionSafeKindCache kindCache;
+ @Inject
+ private IRelatedObjectCache relatedObjectCache;
+ @Inject
+ private IRecordChangeInfoCache recordChangeInfoCache;
+ @Inject
+ private SearchService searchService;
+ @Inject
+ private SchemaService schemaService;
+ @Inject
+ private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
+ @Inject
+ private IRequestInfo requestInfo;
+ @Inject
+ private JaxRsDpsLog jaxRsDpsLog;
+
+ @Override
+ public boolean isPropertyConfigurationsEnabled(String kind) {
+ kind = PropertyUtil.getKindWithMajor(kind);
+ if (Strings.isNullOrEmpty(kind))
+ return false;
+
+ Boolean enabled = propertyConfigurationsEnabledCache.get(kind);
+ if(enabled == null) {
+ SearchRequest searchRequest = new SearchRequest();
+ searchRequest.setKind(INDEX_PROPERTY_PATH_CONFIGURATION_KIND);
+ String query = String.format(HAS_CONFIGURATIONS_QUERY_FORMAT, kind, kind);
+ searchRequest.setQuery(query);
+ if(searchFirstRecord(searchRequest) != null) {
+ enabled = true;
+ }
+ else {
+ enabled = false;
+ }
+ propertyConfigurationsEnabledCache.put(kind, enabled);
+ }
+
+ return enabled;
+ }
+
+ @Override
+ public PropertyConfigurations getPropertyConfigurations(String kind) {
+ kind = PropertyUtil.getKindWithMajor(kind);
+ if (Strings.isNullOrEmpty(kind))
+ return null;
+
+ PropertyConfigurations configuration = propertyConfigurationCache.get(kind);
+ if (configuration == null) {
+ configuration = searchConfigurations(kind);
+ if (configuration != null) {
+ propertyConfigurationCache.put(kind, configuration);
+ } else {
+ // It is common that a kind does not have extended property. So we need to cache an empty configuration
+ // to avoid unnecessary search
+ propertyConfigurationCache.put(kind, EMPTY_CONFIGURATIONS);
+ }
+ }
+
+ if (!isNullOrEmptyConfigurations(configuration)) {
+ return configuration;
+ }
+
+ return null;
+ }
+
+ @Override
+ public Map<String, Object> getExtendedProperties(String objectId, Map<String, Object> originalDataMap, PropertyConfigurations propertyConfigurations) {
+ Set<String> associatedIdentities = new HashSet<>();
+ Map<String, Object> extendedDataMap = new HashMap<>();
+ for (PropertyConfiguration configuration : propertyConfigurations.getConfigurations().stream().filter(c -> c.isValid()).collect(Collectors.toList())) {
+ String extendedPropertyName = configuration.getName();
+ if (originalDataMap.containsKey(extendedPropertyName) && originalDataMap.get(extendedPropertyName) != null) {
+ // If the original record already has the property, then we should not override.
+ // For example, if the trajectory record already SpatialLocation value, then it should not be overridden by the SpatialLocation of the well bore.
+ continue;
+ }
+
+ Map<String, Object> allPropertyValues = new HashMap<>();
+ for (PropertyPath path : configuration.getPaths().stream().filter(p -> p.hasValidValueExtraction()).collect(Collectors.toList())) {
+ if (path.hasValidRelatedObjectsSpec()) {
+ RelatedObjectsSpec relatedObjectsSpec = path.getRelatedObjectsSpec();
+ if (relatedObjectsSpec.isChildToParent()) {
+ List<String> relatedObjectIds = getRelatedObjectIds(originalDataMap, relatedObjectsSpec);
+ for (String relatedObjectId : relatedObjectIds) {
+ // Store all ids
+ associatedIdentities.add(PropertyUtil.removeIdPostfix(relatedObjectId));
+ }
+
+ for (String relatedObjectId : relatedObjectIds) {
+ Map<String, Object> relatedObject = getRelatedObjectData(relatedObjectsSpec.getRelatedObjectKind(), relatedObjectId);
+ Map<String, Object> propertyValues = getExtendedPropertyValues(extendedPropertyName, relatedObject, path.getValueExtraction(), configuration.isExtractFirstMatch());
+ if (allPropertyValues.isEmpty() && configuration.isExtractFirstMatch()) {
+ allPropertyValues = propertyValues;
+ break;
+ } else {
+ allPropertyValues = PropertyUtil.combineObjectMap(allPropertyValues, propertyValues);
+ }
+ }
+ } else {
+ List<SearchRecord> childrenRecords = searchChildrenRecords(relatedObjectsSpec.getRelatedObjectKind(), relatedObjectsSpec.getRelatedObjectID(), objectId);
+ for (SearchRecord record : childrenRecords) {
+ // If the child record is in the cache, that means the record was updated very recently.
+ // In this case, use the cache's record instead of the record from search result
+ Map<String, Object> childDataMap = this.relatedObjectCache.get(record.getId());
+ if(childDataMap == null) {
+ childDataMap = record.getData();
+ }
+
+ Map<String, Object> propertyValues = getExtendedPropertyValues(extendedPropertyName, childDataMap, path.getValueExtraction(), configuration.isExtractFirstMatch());
+ if (allPropertyValues.isEmpty() && configuration.isExtractFirstMatch()) {
+ allPropertyValues = propertyValues;
+ break;
+ } else {
+ allPropertyValues = PropertyUtil.combineObjectMap(allPropertyValues, propertyValues);
+ }
+ }
+ }
+ } else {
+ Map<String, Object> propertyValues = getExtendedPropertyValues(extendedPropertyName, originalDataMap, path.getValueExtraction(), configuration.isExtractFirstMatch());
+ if (allPropertyValues.isEmpty() && configuration.isExtractFirstMatch()) {
+ allPropertyValues = propertyValues;
+ } else {
+ allPropertyValues = PropertyUtil.combineObjectMap(allPropertyValues, propertyValues);
+ }
+ }
+
+ if (!allPropertyValues.isEmpty() && configuration.isExtractFirstMatch())
+ break;
+ }
+
+ extendedDataMap.putAll(allPropertyValues);
+ }
+ if (!associatedIdentities.isEmpty()) {
+ extendedDataMap.put(ASSOCIATED_IDENTITIES_PROPERTY, Arrays.asList(associatedIdentities.toArray()));
+ }
+
+ return extendedDataMap;
+ }
+
+ @Override
+ public List<SchemaItem> getExtendedSchemaItems(Schema originalSchema, Map<String, Schema> relatedObjectKindSchemas, PropertyConfigurations propertyConfigurations) {
+ List<SchemaItem> extendedSchemaItems = new ArrayList<>();
+ boolean hasChildToParentRelationship = false;
+ for (PropertyConfiguration configuration : propertyConfigurations.getConfigurations().stream().filter(c -> c.isValid()).collect(Collectors.toList())) {
+ Schema schema = null;
+ PropertyPath propertyPath = null;
+ for (PropertyPath path : configuration.getPaths().stream().filter(p -> p.hasValidRelatedObjectsSpec()).collect(Collectors.toList())) {
+ RelatedObjectsSpec relatedObjectsSpec = path.getRelatedObjectsSpec();
+ if (relatedObjectsSpec.isChildToParent()) {
+ hasChildToParentRelationship = true;
+ }
+ if (relatedObjectKindSchemas.containsKey(relatedObjectsSpec.getRelatedObjectKind())) {
+ // Refer to the schema of the related object
+ schema = relatedObjectKindSchemas.get(relatedObjectsSpec.getRelatedObjectKind());
+ propertyPath = path;
+ break;
+ }
+ }
+ if (schema == null) {
+ // Refer to the schema of the object itself
+ schema = originalSchema;
+ propertyPath = configuration.getPaths().stream().filter(p -> p.getRelatedObjectsSpec() == null && p.hasValidValueExtraction()).findFirst().orElse(null);
+ }
+
+ if (schema != null && propertyPath != null) {
+ List<SchemaItem> schemaItems = getExtendedSchemaItems(schema, configuration, propertyPath);
+ extendedSchemaItems.addAll(schemaItems);
+ }
+ }
+
+ if (hasChildToParentRelationship) {
+ extendedSchemaItems.add(createAssociatedIdentitiesSchemaItem());
+ }
+
+ return extendedSchemaItems;
+ }
+
+ @Override
+ public String resolveConcreteKind(String kind) {
+ if (Strings.isNullOrEmpty(kind) || PropertyUtil.isConcreteKind(kind)) {
+ return kind;
+ }
+
+ String concreteKind = kindCache.get(kind);
+ if (concreteKind == null) {
+ concreteKind = getLatestVersionOfKind(kind);
+ if (!Strings.isNullOrEmpty(concreteKind)) {
+ kindCache.put(kind, concreteKind);
+ }
+ }
+ return concreteKind;
+ }
+
+ @Override
+ public void cacheDataRecord(String recordId, String kind, Map<String, Object> dataMap) {
+ Map<String, Object> previousDataMap = this.getRelatedObjectData(kind, recordId);
+ RecordInfo recordInfo = new RecordInfo();
+ recordInfo.setId(recordId);
+ recordInfo.setKind(kind);
+ RecordChangeInfo changedInfo = new RecordChangeInfo();
+ changedInfo.setRecordInfo(recordInfo);
+ // Using recordChangeInfoCache is the best effort to avoid updating the associated records when unnecessary
+ // It should only store the updated records (ids) with updated properties. However, in order to
+ // handle the case that a new record is updated in a short period of time, the ids of the new records with OPT
+ // OperationType.create should be cached too.
+ if (previousDataMap == null || previousDataMap.isEmpty()) {
+ recordInfo.setOp(OperationType.create.getValue());
+ } else {
+ recordInfo.setOp(OperationType.update.getValue());
+ List<String> updatedProperties = PropertyUtil.getChangedProperties(previousDataMap, dataMap);
+
+ RecordChangeInfo previousChangedInfo = recordChangeInfoCache.get(recordId);
+ if(previousChangedInfo != null) {
+ if(previousChangedInfo.getRecordInfo().getOp().equals(OperationType.create.getValue())) {
+ recordInfo.setOp(OperationType.create.getValue());
+ }
+ else if(previousChangedInfo.getUpdatedProperties() != null) {
+ previousChangedInfo.getUpdatedProperties().forEach(p -> {
+ if(!updatedProperties.contains(p))
+ updatedProperties.add(p);
+ });
+ }
+ }
+
+ if(recordInfo.getOp().equals(OperationType.update.getValue()))
+ changedInfo.setUpdatedProperties(updatedProperties);
+ }
+ recordChangeInfoCache.put(recordId, changedInfo);
+ relatedObjectCache.put(recordId, dataMap);
+ }
+
+ @Override
+ public void updateAssociatedRecords(RecordChangedMessages message, Map<String, List<String>> upsertKindIds, Map<String, List<String>> deleteKindIds) {
+ if (upsertKindIds == null) {
+ upsertKindIds = new HashMap<>();
+ }
+ if (deleteKindIds == null) {
+ deleteKindIds = new HashMap<>();
+ }
+
+ Map<String, String> attributes = message.getAttributes();
+ String ancestors = attributes.containsKey(Constants.ANCESTRY_KINDS) ? attributes.get(Constants.ANCESTRY_KINDS) : "";
+ Map<String, List<RecordChangeInfo>> recordChangeInfoMap = createRecordChangeInfoMap(upsertKindIds, deleteKindIds);
+ for (Map.Entry<String, List<RecordChangeInfo>> entry : recordChangeInfoMap.entrySet()) {
+ String kind = entry.getKey();
+ List<RecordChangeInfo> recordChangeInfoList = entry.getValue();
+ String updatedAncestors = Strings.isNullOrEmpty(ancestors) ? kind : ancestors + ANCESTRY_KINDS_DELIMITER + kind;
+
+ updateAssociatedParentRecords(updatedAncestors, kind, recordChangeInfoList);
+ updateAssociatedChildrenRecords(updatedAncestors, recordChangeInfoList);
+ }
+ }
+
+ /******************************************************** Private methods **************************************************************/
+ private boolean isNullOrEmptyConfigurations(PropertyConfigurations configuration) {
+ return configuration == null || Strings.isNullOrEmpty(configuration.getCode());
+ }
+
+ private SchemaItem createAssociatedIdentitiesSchemaItem() {
+ SchemaItem extendedSchemaItem = new SchemaItem();
+ extendedSchemaItem.setPath(ASSOCIATED_IDENTITIES_PROPERTY);
+ extendedSchemaItem.setKind(ASSOCIATED_IDENTITIES_PROPERTY_STORAGE_FORMAT_TYPE);
+ return extendedSchemaItem;
+ }
+
+ private String createIdsFilter(List<String> ids) {
+ StringBuilder idsBuilder = new StringBuilder();
+ for (String id : ids) {
+ if (idsBuilder.length() > 0) {
+ idsBuilder.append(" OR ");
+ }
+ idsBuilder.append("\"");
+ idsBuilder.append(PropertyUtil.removeIdPostfix(id));
+ idsBuilder.append("\"");
+ }
+ return idsBuilder.toString();
+ }
+
+ private void createWorkerTask(String ancestors, List<RecordInfo> recordInfos) {
+ Map<String, String> attributes = new HashMap<>();
+ DpsHeaders headers = this.requestInfo.getHeadersWithDwdAuthZ();
+ attributes.put(DpsHeaders.ACCOUNT_ID, headers.getAccountId());
+ attributes.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
+ attributes.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
+ attributes.put(Constants.ANCESTRY_KINDS, ancestors);
+
+ RecordChangedMessages recordChangedMessages = RecordChangedMessages.builder().data(gson.toJson(recordInfos)).attributes(attributes).build();
+ String recordChangedMessagePayload = gson.toJson(recordChangedMessages);
+ this.indexerQueueTaskBuilder.createWorkerTask(recordChangedMessagePayload, 0L, this.requestInfo.getHeadersWithDwdAuthZ());
+ }
+
+ private Map<String, Object> getRelatedObjectData(String relatedObjectKind, String relatedObjectId) {
+ String key = PropertyUtil.removeIdPostfix(relatedObjectId);
+ Map<String, Object> relatedObject = relatedObjectCache.get(key);
+ if (relatedObject == null) {
+ SearchRecord searchRecord = searchRelatedRecord(relatedObjectKind, relatedObjectId);
+ if (searchRecord != null) {
+ relatedObject = searchRecord.getData();
+ relatedObjectCache.put(key, relatedObject);
+ }
+ }
+
+ return relatedObject;
+ }
+
+ private Map<String, List<RecordChangeInfo>> createRecordChangeInfoMap(Map<String, List<String>> upsertKindIds, Map<String, List<String>> deleteKindIds) {
+ Map<String, List<RecordChangeInfo>> recordChangeInfoMap = new HashMap<>();
+ for (Map.Entry<String, List<String>> entry : upsertKindIds.entrySet()) {
+ List<RecordChangeInfo> recordChangeInfoList = getOrCreateRecordChangeInfoList(entry.getKey(), recordChangeInfoMap);
+ for (String id : entry.getValue()) {
+ RecordChangeInfo changeInfo = recordChangeInfoCache.get(id);
+ if (changeInfo == null) {
+ changeInfo = new RecordChangeInfo();
+ changeInfo.setRecordInfo(this.createRecordInfo(entry.getKey(), id, OperationType.create));
+ }
+ recordChangeInfoList.add(changeInfo);
+ }
+ }
+ for (Map.Entry<String, List<String>> entry : deleteKindIds.entrySet()) {
+ List<RecordChangeInfo> recordChangeInfoList = getOrCreateRecordChangeInfoList(entry.getKey(), recordChangeInfoMap);
+ for (String id : entry.getValue()) {
+ RecordChangeInfo changeInfo = new RecordChangeInfo();
+ changeInfo.setRecordInfo(this.createRecordInfo(entry.getKey(), id, OperationType.delete));
+ recordChangeInfoList.add(changeInfo);
+ }
+ }
+
+ return recordChangeInfoMap;
+ }
+
+ private List<RecordChangeInfo> getOrCreateRecordChangeInfoList(String kind, Map<String, List<RecordChangeInfo>> recordChangeInfoMap) {
+ List<RecordChangeInfo> recordChangeInfoList;
+ if (recordChangeInfoMap.containsKey(kind)) {
+ recordChangeInfoList = recordChangeInfoMap.get(kind);
+ } else {
+ recordChangeInfoList = new ArrayList<>();
+ recordChangeInfoMap.put(kind, recordChangeInfoList);
+ }
+ return recordChangeInfoList;
+ }
+
+ private RecordInfo createRecordInfo(String kind, String id, OperationType operationType) {
+ RecordInfo recordInfo = new RecordInfo();
+ recordInfo.setKind(kind);
+ recordInfo.setId(id);
+ recordInfo.setOp(operationType.getValue());
+ return recordInfo;
+ }
+
+ private List<SchemaItem> getExtendedSchemaItems(Schema schema, PropertyConfiguration configuration, PropertyPath propertyPath) {
+ String relatedPropertyPath = PropertyUtil.removeDataPrefix(propertyPath.getValueExtraction().getValuePath());
+ if (relatedPropertyPath.contains(ARRAY_SYMBOL)) { // Nested
+ List<SchemaItem> extendedSchemaItems = new ArrayList<>();
+ extendedSchemaItems = cloneExtendedSchemaItemsFromNestedSchema(Arrays.asList(schema.getSchema()), configuration, relatedPropertyPath);
+ if (extendedSchemaItems.isEmpty()) {
+ // It is possible that the format of the source property is not defined
+ // In this case, we assume that the format of property is string in order to make its value(s) searchable
+ SchemaItem extendedSchemaItem = new SchemaItem();
+ extendedSchemaItem.setPath(configuration.getName());
+ if (configuration.isExtractFirstMatch()) {
+ extendedSchemaItem.setKind("string");
+ } else {
+ extendedSchemaItem.setKind("[]string");
+ }
+ extendedSchemaItems.add(extendedSchemaItem);
+ }
+ return extendedSchemaItems;
+ } else {// Flatten
+ List<SchemaItem> schemaItems = Arrays.asList(schema.getSchema());
+ return cloneExtendedSchemaItems(schemaItems, configuration, relatedPropertyPath);
+ }
+ }
+
+ private List<SchemaItem> cloneExtendedSchemaItems(List<SchemaItem> schemaItems, PropertyConfiguration configuration, String relatedPropertyPath) {
+ List<SchemaItem> extendedSchemaItems = new ArrayList<>();
+ for (SchemaItem schemaItem : schemaItems) {
+ if (PropertyUtil.isPropertyPathMatched(schemaItem.getPath(), relatedPropertyPath)) {
+ String path = schemaItem.getPath();
+ path = path.replace(relatedPropertyPath, configuration.getName());
+ SchemaItem extendedSchemaItem = new SchemaItem();
+ extendedSchemaItem.setPath(path);
+ if (configuration.isExtractFirstMatch()) {
+ extendedSchemaItem.setKind(schemaItem.getKind());
+ } else {
+ extendedSchemaItem.setKind("[]" + schemaItem.getKind());
+ }
+ extendedSchemaItems.add(extendedSchemaItem);
+ }
+ }
+ return extendedSchemaItems;
+ }
+
+ private List<SchemaItem> cloneExtendedSchemaItemsFromNestedSchema(List<SchemaItem> schemaItems, PropertyConfiguration configuration, String relatedPropertyPath) {
+ if (relatedPropertyPath.contains(ARRAY_SYMBOL)) {
+ List<SchemaItem> extendedSchemaItems = new ArrayList<>();
+ int idx = relatedPropertyPath.indexOf(NESTED_OBJECT_DELIMITER);
+ String prePath = relatedPropertyPath.substring(0, idx);
+ String postPath = relatedPropertyPath.substring(idx + NESTED_OBJECT_DELIMITER.length());
+ for (SchemaItem schemaItem : schemaItems) {
+ if (schemaItem.getPath().equals(prePath)) {
+ if (schemaItem.getKind().equals(SCHEMA_NESTED_KIND) && schemaItem.getProperties() != null) {
+ schemaItems = Arrays.asList(schemaItem.getProperties());
+ extendedSchemaItems = cloneExtendedSchemaItemsFromNestedSchema(schemaItems, configuration, postPath);
+ }
+ break;
+ }
+ }
+ return extendedSchemaItems;
+ } else {
+ return cloneExtendedSchemaItems(schemaItems, configuration, relatedPropertyPath);
+ }
+ }
+
+ private List<String> getRelatedObjectIds(Map<String, Object> dataMap, RelatedObjectsSpec relatedObjectsSpec) {
+ if (dataMap == null || dataMap.isEmpty() || relatedObjectsSpec == null || !relatedObjectsSpec.isValid())
+ return new ArrayList<>();
+
+ Map<String, Object> propertyValues = getPropertyValues(dataMap, relatedObjectsSpec.getRelatedObjectID(), relatedObjectsSpec, relatedObjectsSpec.hasValidCondition(), false);
+ List<String> relatedObjectIds = new ArrayList<>();
+ for (Object value : propertyValues.values()) {
+ if (value instanceof List) {
+ for (Object obj : (List) value) {
+ relatedObjectIds.add(obj.toString());
+ }
+ } else {
+ relatedObjectIds.add(value.toString());
+ }
+ }
+ return relatedObjectIds;
+ }
+
+ private Map<String, Object> getExtendedPropertyValues(String extendedPropertyName, Map<String, Object> dataMap, ValueExtraction valueExtraction, boolean isExtractFirstMatch) {
+ if (dataMap == null || dataMap.isEmpty() || valueExtraction == null || !valueExtraction.isValid())
+ return new HashMap<>();
+ Map<String, Object> propertyValues = getPropertyValues(dataMap, valueExtraction.getValuePath(), valueExtraction, valueExtraction.hasValidCondition(), isExtractFirstMatch);
+ return PropertyUtil.replacePropertyPaths(extendedPropertyName, valueExtraction.getValuePath(), propertyValues);
+ }
+
+ private Map<String, Object> getPropertyValues(Map<String, Object> dataMap, String valuePath, RelatedCondition relatedCondition, boolean hasValidCondition, boolean isExtractFirstMatch) {
+ valuePath = PropertyUtil.removeDataPrefix(valuePath);
+ Map<String, Object> propertyValues = new HashMap<>();
+ if (valuePath.contains(ARRAY_SYMBOL)) { // Nested
+ String conditionProperty = null;
+ List<String> conditionMatches = null;
+ if (hasValidCondition) {
+ int idx = relatedCondition.getRelatedConditionProperty().lastIndexOf(NESTED_OBJECT_DELIMITER);
+ conditionProperty = relatedCondition.getRelatedConditionProperty().substring(idx + NESTED_OBJECT_DELIMITER.length());
+ conditionMatches = relatedCondition.getRelatedConditionMatches();
+ }
+
+ List<Object> valueList = getPropertyValuesFromNestedObjects(dataMap, valuePath, conditionProperty, conditionMatches, hasValidCondition, isExtractFirstMatch);
+ if (!valueList.isEmpty()) {
+ if (isExtractFirstMatch) {
+ propertyValues.put(valuePath, valueList.get(0));
+ } else {
+ propertyValues.put(valuePath, valueList);
+ }
+ }
+ } else { // Flatten
+ for (Map.Entry<String, Object> entry : dataMap.entrySet()) {
+ String key = entry.getKey();
+ if (key.equals(valuePath) || key.startsWith(valuePath + PROPERTY_DELIMITER)) {
+ if (isExtractFirstMatch) {
+ propertyValues.put(key, entry.getValue());
+ } else {
+ List<Object> values = new ArrayList<>();
+ values.add(entry.getValue());
+ propertyValues.put(key, values);
+ }
+ }
+ }
+ }
+
+ return propertyValues;
+ }
+
+ private List<Object> getPropertyValuesFromNestedObjects(Map<String, Object> dataMap, String valuePath, String conditionProperty, List<String> conditionMatches, boolean hasCondition, boolean isExtractFirstMatch) {
+ Set<Object> propertyValues = new HashSet<>();
+
+ if (valuePath.contains(ARRAY_SYMBOL)) {
+ int idx = valuePath.indexOf(NESTED_OBJECT_DELIMITER);
+ String prePath = valuePath.substring(0, idx);
+ String postPath = valuePath.substring(idx + NESTED_OBJECT_DELIMITER.length());
+ try {
+ if (dataMap.containsKey(prePath) && dataMap.get(prePath) != null) {
+ List<Map<String, Object>> nestedObjects = (List<Map<String, Object>>) dataMap.get(prePath);
+ for (Map<String, Object> nestedObject : nestedObjects) {
+ List<Object> valueList = getPropertyValuesFromNestedObjects(nestedObject, postPath, conditionProperty, conditionMatches, hasCondition, isExtractFirstMatch);
+ if (valueList != null && !valueList.isEmpty()) {
+ propertyValues.addAll(valueList);
+ if (isExtractFirstMatch)
+ break;
+ }
+ }
+ }
+ } catch (Exception ex) {
+ //Ignore cast exception
+ }
+ } else if (dataMap.containsKey(valuePath) && dataMap.get(valuePath) != null) {
+ Object extractPropertyValue = dataMap.get(valuePath);
+ if (hasCondition) {
+ if (dataMap.containsKey(conditionProperty) && dataMap.get(conditionProperty) != null) {
+ String conditionPropertyValue = dataMap.get(conditionProperty).toString();
+ if (conditionMatches.contains(conditionPropertyValue) && extractPropertyValue != null) {
+ propertyValues.add(extractPropertyValue);
+ }
+ }
+ } else {
+ propertyValues.add(extractPropertyValue);
+ }
+ }
+
+ List<Object> propertyValueList = new ArrayList<>(propertyValues);
+ Collections.sort(propertyValueList, Comparator.comparing(Object::toString));
+ return propertyValueList;
+ }
+
+ private ParentChildRelationshipSpecs getParentChildRelatedObjectsSpecs(String childKind) {
+ final String childKindWithMajor = PropertyUtil.getKindWithMajor(childKind);
+
+ ParentChildRelationshipSpecs specs = parentChildRelationshipSpecsCache.get(childKindWithMajor);
+ if (specs == null) {
+ List<ParentChildRelationshipSpec> specsList = new ArrayList<>();
+ specs = new ParentChildRelationshipSpecs();
+ specs.setSpecList(specsList);
+
+ List<PropertyConfigurations> configurationsList = searchParentKindConfigurations((childKindWithMajor));
+ for (PropertyConfigurations configurations : configurationsList) {
+ for (PropertyConfiguration configuration : configurations.getConfigurations()) {
+ List<PropertyPath> matchedPropertyPaths = configuration.getPaths().stream().filter(p ->
+ p.hasValidRelatedObjectsSpec() &&
+ p.getRelatedObjectsSpec().isParentToChildren() &&
+ p.getRelatedObjectsSpec().getRelatedObjectKind().contains(childKindWithMajor))
+ .collect(Collectors.toList());
+ for(PropertyPath propertyPath: matchedPropertyPaths) {
+ ParentChildRelationshipSpec spec = toParentChildRelationshipSpec(propertyPath, configurations.getCode(), childKindWithMajor);
+ boolean merged = false;
+ for(ParentChildRelationshipSpec sp: specsList) {
+ if(sp.equals(spec)) {
+ List<String> childValuePaths = sp.getChildValuePaths();
+ if(!childValuePaths.contains(spec.getChildValuePaths().get(0))) {
+ childValuePaths.add(spec.getChildValuePaths().get(0));
+ }
+ merged = true;
+ break;
+ }
+ }
+ if(!merged) {
+ specsList.add(spec);
+ }
+ }
+ }
+ }
+
+ parentChildRelationshipSpecsCache.put(childKindWithMajor, specs);
+ }
+
+ return specs;
+ }
+
+ private ParentChildRelationshipSpec toParentChildRelationshipSpec(PropertyPath propertyPath, String parentKind, String childKind) {
+ ParentChildRelationshipSpec spec = new ParentChildRelationshipSpec();
+ spec.setParentKind(parentKind);
+ spec.setParentObjectIdPath(propertyPath.getRelatedObjectsSpec().getRelatedObjectID());
+ spec.setChildKind(childKind);
+ String valuePath = PropertyUtil.removeDataPrefix(propertyPath.getValueExtraction().getValuePath());
+ spec.getChildValuePaths().add(valuePath);
+ return spec;
+ }
+
+ private void updateAssociatedParentRecords(String ancestors, String childKind, List<RecordChangeInfo> childRecordChangeInfos) {
+ ParentChildRelationshipSpecs specs = getParentChildRelatedObjectsSpecs(childKind);
+ Set ancestorSet = new HashSet<>(Arrays.asList(ancestors.split(ANCESTRY_KINDS_DELIMITER)));
+ for (ParentChildRelationshipSpec spec : specs.getSpecList()) {
+ List childRecordIds = getChildRecordIdsWithExtendedPropertiesChanged(spec, childRecordChangeInfos);
+ if (childRecordIds.isEmpty())
+ continue;
+
+ List<String> parentIds = searchUniqueParentIds(childKind, childRecordIds, spec.getParentObjectIdPath());
+ if (parentIds.isEmpty())
+ continue;
+
+ final int limit = configurationProperties.getStorageRecordsByKindBatchSize();
+ Map<String, List<String>> parentKindIds = searchKindIds(spec.getParentKind(), parentIds);
+ List<RecordInfo> recordInfos = new ArrayList<>();
+ for (Map.Entry<String, List<String>> entry : parentKindIds.entrySet()) {
+ if (ancestorSet.contains(entry.getKey()))
+ continue; // circular indexing found.
+
+ for (String id : entry.getValue()) {
+ RecordInfo recordInfo = new RecordInfo();
+ recordInfo.setKind(entry.getKey());
+ recordInfo.setId(id);
+ recordInfo.setOp(OperationType.update.getValue());
+ recordInfos.add(recordInfo);
+
+ if (recordInfos.size() >= limit) {
+ createWorkerTask(ancestors, recordInfos);
+ recordInfos = new ArrayList<>();
+ }
+ }
+ }
+ if (!recordInfos.isEmpty()) {
+ createWorkerTask(ancestors, recordInfos);
+ }
+ }
+ }
+
+ private List<String> getChildRecordIdsWithExtendedPropertiesChanged(ParentChildRelationshipSpec spec, List<RecordChangeInfo> childRecordChangeInfos) {
+ List<String> childRecordIds = new ArrayList<>();
+ for (RecordChangeInfo recordChangeInfo : childRecordChangeInfos) {
+ if (recordChangeInfo.getRecordInfo().getOp().equals(OperationType.update.getValue())) {
+ String updatedExtendedProperty = recordChangeInfo.getUpdatedProperties().stream().filter(p -> {
+ for (String valuePath : spec.getChildValuePaths()) {
+ if (PropertyUtil.isPropertyPathMatched(valuePath, p) ||
+ PropertyUtil.isPropertyPathMatched(p, valuePath)) {
+ return true;
+ }
+ }
+ return false;
+ }).findFirst().orElse(null);
+
+ if (updatedExtendedProperty != null) {
+ // The parent property that is extended by the children was updated
+ childRecordIds.add(recordChangeInfo.getRecordInfo().getId());
+ }
+ }
+ else {
+ childRecordIds.add(recordChangeInfo.getRecordInfo().getId());
+ }
+ }
+ return childRecordIds;
+ }
+
+ private boolean areExtendedPropertiesChanged(String childKind, List<RecordChangeInfo> parentRecordChangeInfos) {
+ if (parentRecordChangeInfos.stream().filter(info -> !info.getRecordInfo().getOp().equals(OperationType.update.getValue())).findFirst().orElse(null) != null) {
+ // If there is any OP of the parent record(s) that is not OperationType.update. It must be OperationType.delete in this case. Then the child record should be updated
+ return true;
+ }
+
+ PropertyConfigurations propertyConfigurations = this.getPropertyConfigurations(childKind);
+ if(propertyConfigurations != null) {
+ for (PropertyConfiguration propertyConfiguration : propertyConfigurations.getConfigurations()) {
+ for (PropertyPath propertyPath : propertyConfiguration.getPaths().stream().filter(
+ p -> p.hasValidValueExtraction() && p.hasValidRelatedObjectsSpec()).collect(Collectors.toList())) {
+ String relatedObjectKind = propertyPath.getRelatedObjectsSpec().getRelatedObjectKind();
+ String valuePath = PropertyUtil.removeDataPrefix(propertyPath.getValueExtraction().getValuePath());
+
+ // Find any parent record which has changed property that is extended by the child (kind)
+ RecordChangeInfo parentRecordChangeInfo = parentRecordChangeInfos.stream().filter(info -> {
+ if (PropertyUtil.hasSameMajorKind(info.getRecordInfo().getKind(), relatedObjectKind)) {
+ List<String> matchedProperties = info.getUpdatedProperties().stream().filter(
+ p -> PropertyUtil.isPropertyPathMatched(p, valuePath) || PropertyUtil.isPropertyPathMatched(valuePath, p)).collect(Collectors.toList());
+ return !matchedProperties.isEmpty();
+ }
+ return false;
+ }).findFirst().orElse(null);
+ if (parentRecordChangeInfo != null) {
+ return true;
+ }
+ }
+ }
+ }
+
+ return false;
+ }
+
+ private void updateAssociatedChildrenRecords(String ancestors, List<RecordChangeInfo> recordChangeInfos) {
+ List<String> processedIds = recordChangeInfos.stream().map(recordChangeInfo -> recordChangeInfo.getRecordInfo().getId()).collect(Collectors.toList());
+ String query = String.format("data.%s:(%s)", ASSOCIATED_IDENTITIES_PROPERTY, createIdsFilter(processedIds));
+ String kind = WILD_CARD_KIND;
+ for (String ancestryKind : ancestors.split(ANCESTRY_KINDS_DELIMITER)) {
+ if (!ancestryKind.trim().isEmpty()) {
+ // Exclude the kinds in the ancestryKinds to prevent circular chasing
+ kind += ",-" + ancestryKind.trim();
+ }
+ }
+
+ final int limit = configurationProperties.getStorageRecordsByKindBatchSize();
+ SearchRequest searchRequest = new SearchRequest();
+ searchRequest.setKind(kind);
+ searchRequest.setQuery(query);
+ searchRequest.setReturnedFields(Arrays.asList("kind", "id", "data." + ASSOCIATED_IDENTITIES_PROPERTY));
+ List<RecordInfo> recordInfos = new ArrayList<>();
+ for (SearchRecord record : searchAllRecords(searchRequest)) {
+ Map<String, Object> data = record.getData();
+ if (!data.containsKey(ASSOCIATED_IDENTITIES_PROPERTY) || data.get(ASSOCIATED_IDENTITIES_PROPERTY) == null)
+ continue;
+
+ List<String> associatedParentIds = (List<String>) data.get(ASSOCIATED_IDENTITIES_PROPERTY);
+ List<RecordChangeInfo> associatedParentRecordChangeInfos = recordChangeInfos.stream().filter(
+ info -> associatedParentIds.contains(info.getRecordInfo().getId())).collect(Collectors.toList());
+ if (areExtendedPropertiesChanged(record.getKind(), associatedParentRecordChangeInfos)) {
+ RecordInfo recordInfo = new RecordInfo();
+ recordInfo.setKind(record.getKind());
+ recordInfo.setId(record.getId());
+ recordInfo.setOp(OperationType.update.getValue());
+ recordInfos.add(recordInfo);
+
+ if (recordInfos.size() >= limit) {
+ createWorkerTask(ancestors, recordInfos);
+ recordInfos = new ArrayList<>();
+ }
+ }
+ }
+ if (!recordInfos.isEmpty()) {
+ createWorkerTask(ancestors, recordInfos);
+ }
+ }
+
+ private String getLatestVersionOfKind(String kindWithMajor) {
+ Kind kind = new Kind(kindWithMajor);
+ String version = kind.getVersion();
+ String[] subVersions = version.split("\\.");
+ String majorVersion = subVersions[0];
+ String latestKind = null;
+ try {
+ SchemaInfoResponse response = schemaService.getSchemaInfos(kind.getAuthority(), kind.getSource(), kind.getType(), majorVersion, null, null, true);
+ if (response != null && response.getSchemaInfos() != null && response.getSchemaInfos().size() > 0) {
+ SchemaInfo schemaInfo = response.getSchemaInfos().get(0);
+ SchemaIdentity schemaIdentity = schemaInfo.getSchemaIdentity();
+ latestKind = schemaIdentity.getAuthority() + ":" +
+ schemaIdentity.getSource() + ":" +
+ schemaIdentity.getEntityType() + ":" +
+ schemaIdentity.getSchemaVersionMajor() + "." +
+ schemaIdentity.getSchemaVersionMinor() + "." +
+ schemaIdentity.getSchemaVersionPatch();
+ }
+ } catch (URISyntaxException e) {
+ jaxRsDpsLog.error("failed to get schema info", e);
+ } catch (UnsupportedEncodingException e) {
+ jaxRsDpsLog.error("failed to get schema info", e);
+ }
+
+ return latestKind;
+ }
+
+ /****************************** search methods that use search service to get the data **************************************/
+
+ private PropertyConfigurations searchConfigurations(String kind) {
+ SearchRequest searchRequest = new SearchRequest();
+ searchRequest.setKind(INDEX_PROPERTY_PATH_CONFIGURATION_KIND);
+ String query = String.format("data.Code: \"%s\"", kind);
+ searchRequest.setQuery(query);
+ for (SearchRecord searchRecord : searchAllRecords(searchRequest)) {
+ try {
+ String data = objectMapper.writeValueAsString(searchRecord.getData());
+ PropertyConfigurations configurations = objectMapper.readValue(data, PropertyConfigurations.class);
+ if (kind.equals(configurations.getCode())) {
+ return configurations;
+ }
+ } catch (JsonProcessingException e) {
+ jaxRsDpsLog.error("failed to deserialize PropertyConfigurations object", e);
+ }
+ }
+ return null;
+ }
+
+ private List<PropertyConfigurations> searchParentKindConfigurations(String childKind) {
+ SearchRequest searchRequest = new SearchRequest();
+ searchRequest.setKind(INDEX_PROPERTY_PATH_CONFIGURATION_KIND);
+ String query = String.format(PARENT_CHILDREN_CONFIGURATION_QUERY_FORMAT, childKind);
+ searchRequest.setQuery(query);
+ List<PropertyConfigurations> configurationsList = new ArrayList<>();
+ for (SearchRecord searchRecord : searchAllRecords(searchRequest)) {
+ try {
+ String data = objectMapper.writeValueAsString(searchRecord.getData());
+ PropertyConfigurations configurations = objectMapper.readValue(data, PropertyConfigurations.class);
+ configurationsList.add(configurations);
+ } catch (JsonProcessingException e) {
+ jaxRsDpsLog.error("failed to deserialize PropertyConfigurations object", e);
+ }
+ }
+ return configurationsList;
+ }
+
+ private SearchRecord searchRelatedRecord(String relatedObjectKind, String relatedObjectId) {
+ String kind = PropertyUtil.isConcreteKind(relatedObjectKind) ? relatedObjectKind : relatedObjectKind + "*";
+ String id = PropertyUtil.removeIdPostfix(relatedObjectId);
+ SearchRequest searchRequest = new SearchRequest();
+ searchRequest.setKind(kind);
+ String query = String.format("id: \"%s\"", id);
+ searchRequest.setQuery(query);
+ return searchFirstRecord(searchRequest);
+ }
+
+ private Map<String, List<String>> searchKindIds(String majorKind, List<String> ids) {
+ Map<String, List<String>> kindIds = new HashMap<>();
+ SearchRequest searchRequest = new SearchRequest();
+ String kind = PropertyUtil.isConcreteKind(majorKind) ? majorKind : majorKind + "*";
+ searchRequest.setKind(kind);
+ String query = String.format("id: (%s)", createIdsFilter(ids));
+ searchRequest.setReturnedFields(Arrays.asList("kind", "id"));
+ searchRequest.setQuery(query);
+ for (SearchRecord record : searchAllRecords(searchRequest)) {
+ if (kindIds.containsKey(record.getKind())) {
+ kindIds.get(record.getKind()).add(record.getId());
+ } else {
+ List<String> idList = new ArrayList<>();
+ idList.add(record.getId());
+ kindIds.put(record.getKind(), idList);
+ }
+ }
+ return kindIds;
+ }
+
+ private List<String> searchUniqueParentIds(String childKind, List<String> childRecordIds, String parentObjectIdPath) {
+ Set<String> parentIds = new HashSet<>();
+ SearchRequest searchRequest = new SearchRequest();
+ searchRequest.setKind(childKind);
+ String query = String.format("id: (%s)", createIdsFilter(childRecordIds));
+ searchRequest.setReturnedFields(Arrays.asList(parentObjectIdPath));
+ searchRequest.setQuery(query);
+ parentObjectIdPath = PropertyUtil.removeDataPrefix(parentObjectIdPath);
+ for (SearchRecord record : searchAllRecords(searchRequest)) {
+ if (record.getData().containsKey(parentObjectIdPath)) {
+ Object id = record.getData().get(parentObjectIdPath);
+ if (id != null && !parentIds.contains(id)) {
+ parentIds.add(id.toString());
+ }
+ }
+ }
+ return new ArrayList<>(parentIds);
+ }
+
+ private List<SearchRecord> searchChildrenRecords(String childrenObjectKind, String childrenObjectField, String parentId) {
+ String kind = PropertyUtil.isConcreteKind(childrenObjectKind) ? childrenObjectKind : childrenObjectKind + "*";
+ SearchRequest searchRequest = new SearchRequest();
+ searchRequest.setKind(kind);
+ String query = String.format("%s: \"%s\"", childrenObjectField, parentId);
+ searchRequest.setQuery(query);
+ return searchAllRecords(searchRequest);
+ }
+
+ /*
+ It is assumed that the search request in this method won't return millions of records
+ */
+ private List<SearchRecord> searchAllRecords(SearchRequest searchRequest) {
+ searchRequest.setLimit(MAX_SEARCH_LIMIT);
+ List<SearchRecord> allRecords = new ArrayList<>();
+ boolean done = false;
+ int offset = 0;
+ try {
+ while (!done) {
+ searchRequest.setOffset(offset);
+ SearchResponse searchResponse = searchService.query(searchRequest);
+ List<SearchRecord> results = searchResponse.getResults();
+ if (results != null && results.size() > 0) {
+ allRecords.addAll(results);
+ }
+
+ if (results != null && results.size() == MAX_SEARCH_LIMIT) {
+ offset += MAX_SEARCH_LIMIT;
+ } else {
+ done = true;
+ }
+ }
+ } catch (URISyntaxException e) {
+ jaxRsDpsLog.error("Failed to call search service.", e);
+ }
+ return allRecords;
+ }
+
+ private SearchRecord searchFirstRecord(SearchRequest searchRequest) {
+ searchRequest.setLimit(1);
+ try {
+ SearchResponse searchResponse = searchService.query(searchRequest);
+ List<SearchRecord> results = searchResponse.getResults();
+ if (results != null && !results.isEmpty()) {
+ return results.get(0);
+ }
+ } catch (URISyntaxException e) {
+ jaxRsDpsLog.error("Failed to call search service.", e);
+ }
+ return null;
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SchemaProviderImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SchemaProviderImpl.java
index 6a688858371e14480c9a090b9a66fa75108553ec..8a9fd0cac07e83c46e5e9e3a48076076bfc2a455 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SchemaProviderImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SchemaProviderImpl.java
@@ -15,12 +15,15 @@
package org.opengroup.osdu.indexer.service;
import com.google.api.client.http.HttpMethods;
+
import java.io.UnsupportedEncodingException;
import java.net.URISyntaxException;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
-import java.util.Objects;
import javax.inject.Inject;
+
+import com.google.api.client.util.Strings;
+import com.google.gson.Gson;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.http.FetchServiceHttpRequest;
import org.opengroup.osdu.core.common.http.IUrlFetchService;
@@ -29,6 +32,7 @@ import org.opengroup.osdu.core.common.model.http.HttpResponse;
import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
import org.opengroup.osdu.indexer.logging.AuditLogger;
+import org.opengroup.osdu.indexer.model.SchemaInfoResponse;
import org.opengroup.osdu.indexer.schema.converter.interfaces.SchemaToStorageFormat;
import org.springframework.stereotype.Component;
@@ -37,6 +41,8 @@ import org.springframework.stereotype.Component;
*/
@Component
public class SchemaProviderImpl implements SchemaService {
+ private final Gson gson = new Gson();
+ private final int MAX_NUMBER_OF_SCHEMA_INFOS = 1000;
@Inject
private JaxRsDpsLog log;
@@ -65,6 +71,20 @@ public class SchemaProviderImpl implements SchemaService {
return schemaServiceSchema;
}
+ @Override
+ public SchemaInfoResponse getSchemaInfos(String authority, String source, String entityType, String majorVersion, String minorVersion, String patchVersion, boolean latestVersion) throws URISyntaxException, UnsupportedEncodingException {
+ String queryParams = buildQueryString(authority, source, entityType, majorVersion, minorVersion, patchVersion, latestVersion);
+ String url = String.format("%s?%s", configurationProperties.getSchemaHost(), queryParams);
+ FetchServiceHttpRequest request = FetchServiceHttpRequest.builder()
+ .httpMethod(HttpMethods.GET)
+ .headers(this.requestInfo.getHeadersMapWithDwdAuthZ())
+ .url(url)
+ .build();
+
+ HttpResponse response = this.urlFetchService.sendRequest(request);
+ return gson.fromJson(response.getBody(), SchemaInfoResponse.class);
+ }
+
protected String getFromSchemaService(String kind) throws UnsupportedEncodingException, URISyntaxException {
HttpResponse response = getSchemaServiceResponse(kind);
@@ -99,4 +119,28 @@ public class SchemaProviderImpl implements SchemaService {
return this.urlFetchService.sendRequest(request);
}
+
+ private String buildQueryString(String authority, String source, String entityType, String majorVersion, String minorVersion, String patchVersion, boolean latestVersion) throws UnsupportedEncodingException {
+ StringBuilder stringBuilder = new StringBuilder();
+ addQueryParam(stringBuilder, "authority", authority);
+ addQueryParam(stringBuilder, "source", source);
+ addQueryParam(stringBuilder, "entityType", entityType);
+ addQueryParam(stringBuilder, "schemaVersionMajor", majorVersion);
+ addQueryParam(stringBuilder, "schemaVersionMinor", minorVersion);
+ addQueryParam(stringBuilder, "schemaVersionPatch", patchVersion);
+ addQueryParam(stringBuilder, "latestVersion", String.valueOf(latestVersion));
+ addQueryParam(stringBuilder, "limit", String.valueOf(MAX_NUMBER_OF_SCHEMA_INFOS));
+ return stringBuilder.toString();
+ }
+
+ private StringBuilder addQueryParam(StringBuilder stringBuilder, String paramName, String paramValue) {
+ if(!Strings.isNullOrEmpty(paramName) && !Strings.isNullOrEmpty(paramValue)) {
+ if (stringBuilder.length() > 0)
+ stringBuilder.append("&");
+ stringBuilder.append(paramName);
+ stringBuilder.append("=");
+ stringBuilder.append(paramValue);
+ }
+ return stringBuilder;
+ }
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SchemaService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SchemaService.java
index cb161958c76d27981321707df057b17c03cf3317..ccea60df40cb1593e65d56286c302626cf15b260 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SchemaService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SchemaService.java
@@ -14,15 +14,18 @@
package org.opengroup.osdu.indexer.service;
+import org.opengroup.osdu.indexer.model.SchemaInfo;
+import org.opengroup.osdu.indexer.model.SchemaInfoResponse;
+
import java.io.UnsupportedEncodingException;
import java.net.URISyntaxException;
+import java.util.List;
/**
* Interface to consume schemas from the Schema Service
*/
public interface SchemaService {
/**
- *
* @param kind key to retrieve schema
* @return obtained schema
* @throws URISyntaxException
@@ -30,4 +33,12 @@ public interface SchemaService {
*/
String getSchema(String kind) throws URISyntaxException, UnsupportedEncodingException;
+ /**
+ * @param authority
+ * @param source
+ * @param entityType
+ * @param majorVersion
+ * @return The latest version of the kind
+ */
+ SchemaInfoResponse getSchemaInfos(String authority, String source, String entityType, String majorVersion, String minorVersion, String patchVersion, boolean latestVersion) throws URISyntaxException, UnsupportedEncodingException;
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SearchService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SearchService.java
new file mode 100644
index 0000000000000000000000000000000000000000..6617e4b90bb8fccce1f0d0e2cc8ae6de941a5dcc
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SearchService.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.service;
+
+import org.opengroup.osdu.indexer.model.SearchRequest;
+import org.opengroup.osdu.indexer.model.SearchResponse;
+
+import java.net.URISyntaxException;
+
+/**
+ * Interface to consume schemas from the Schema Service
+ */
+public interface SearchService {
+ /**
+ *
+ * @param searchRequest
+ * @return
+ * @throws URISyntaxException
+ */
+ SearchResponse query(SearchRequest searchRequest) throws URISyntaxException;
+
+ /**
+ *
+ * @param searchRequest
+ * @return
+ * @throws URISyntaxException
+ */
+ SearchResponse queryWithCursor(SearchRequest searchRequest) throws URISyntaxException;
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SearchServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SearchServiceImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..5cb7c8d87544d6b17ba29183a079ea8f1ff3c633
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SearchServiceImpl.java
@@ -0,0 +1,87 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.service;
+
+import com.google.api.client.http.HttpMethods;
+import com.google.common.base.Strings;
+import com.google.gson.Gson;
+import org.opengroup.osdu.core.common.http.FetchServiceHttpRequest;
+import org.opengroup.osdu.core.common.http.IUrlFetchService;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.opengroup.osdu.core.common.model.http.HttpResponse;
+import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.opengroup.osdu.indexer.model.SearchRequest;
+import org.opengroup.osdu.indexer.model.SearchResponse;
+import org.springframework.stereotype.Component;
+
+import javax.inject.Inject;
+import java.net.URISyntaxException;
+
+@Component
+public class SearchServiceImpl implements SearchService {
+ private static final String QUERY_PATH = "query";
+ private static final String QUERY_WITH_CURSOR_PATH = "query_with_cursor";
+ private final Gson gson = new Gson();
+ private static final int OK_CODE = 200;
+
+ @Inject
+ private IUrlFetchService urlFetchService;
+ @Inject
+ private IRequestInfo requestInfo;
+ @Inject
+ private IndexerConfigurationProperties configurationProperties;
+ @Inject
+ private JaxRsDpsLog jaxRsDpsLog;
+
+ @Override
+ public SearchResponse query(SearchRequest searchRequest) throws URISyntaxException {
+ return searchRecords(searchRequest, QUERY_PATH);
+ }
+
+ @Override
+ public SearchResponse queryWithCursor(SearchRequest searchRequest) throws URISyntaxException {
+ return searchRecords(searchRequest, QUERY_WITH_CURSOR_PATH);
+ }
+
+ private SearchResponse searchRecords(SearchRequest searchRequest, String path) throws URISyntaxException {
+ if(Strings.isNullOrEmpty(configurationProperties.getSearchHost())) {
+ jaxRsDpsLog.error("SEARCH_HOST", "The environment variable SEARCH_HOST is not setup");
+ return new SearchResponse();
+ }
+
+ String body = this.gson.toJson(searchRequest);
+ String url = String.format("%s/%s", configurationProperties.getSearchHost(), path);
+ FetchServiceHttpRequest request = FetchServiceHttpRequest.builder()
+ .httpMethod(HttpMethods.POST)
+ .url(url)
+ .headers(this.requestInfo.getHeaders())
+ .body(body)
+ .build();
+ HttpResponse response = this.urlFetchService.sendRequest(request);
+
+ if(response != null && response.getResponseCode() == OK_CODE) {
+ return gson.fromJson(response.getBody(), SearchResponse.class);
+ }
+ else {
+ if(response != null)
+ jaxRsDpsLog.error(String.format("Search service: failed to call the search service: %d", response.getResponseCode()));
+ else
+ jaxRsDpsLog.error(String.format("Search service: failed to call the search service. The response is null."));
+ return new SearchResponse();
+ }
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java
index 194649737bb93817d69a0fb23dcb3f244015735f..d949e581fa000842f1d48ce91cec1b08164cecbb 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java
@@ -30,7 +30,7 @@ import org.opengroup.osdu.indexer.schema.converter.interfaces.IVirtualProperties
import org.opengroup.osdu.indexer.schema.converter.tags.Priority;
import org.opengroup.osdu.indexer.schema.converter.tags.VirtualProperties;
import org.opengroup.osdu.indexer.schema.converter.tags.VirtualProperty;
-import org.opengroup.osdu.indexer.util.VirtualPropertyUtil;
+import org.opengroup.osdu.indexer.util.PropertyUtil;
import org.opengroup.osdu.indexer.util.geo.decimator.DecimatedResult;
import org.opengroup.osdu.indexer.util.geo.decimator.GeoShapeDecimator;
import org.opengroup.osdu.indexer.util.geo.decimator.GeoShapeDecimationSetting;
@@ -223,21 +223,21 @@ public class StorageIndexerPayloadMapper {
continue;
}
Priority priority = chooseOriginalProperty(entry.getKey(), entry.getValue().getPriorities(), dataCollectorMap);
- String virtualPropertyPath = VirtualPropertyUtil.removeDataPrefix(entry.getKey());
- String originalPropertyPath = VirtualPropertyUtil.removeDataPrefix(priority.getPath());
+ String virtualPropertyPath = PropertyUtil.removeDataPrefix(entry.getKey());
+ String originalPropertyPath = PropertyUtil.removeDataPrefix(priority.getPath());
// Populate the virtual property values from the chosen original property
List<String> originalPropertyNames = dataCollectorMap.keySet().stream()
- .filter(originalPropertyName -> VirtualPropertyUtil.isPropertyPathMatched(originalPropertyName, originalPropertyPath))
+ .filter(originalPropertyName -> PropertyUtil.isPropertyPathMatched(originalPropertyName, originalPropertyPath))
.collect(Collectors.toList());
originalPropertyNames.forEach(originalPropertyName -> {
String virtualPropertyName = virtualPropertyPath + originalPropertyName.substring(originalPropertyPath.length());
dataCollectorMap.put(virtualPropertyName, dataCollectorMap.get(originalPropertyName));
});
- if(virtualPropertyPath.equals(VirtualPropertyUtil.VIRTUAL_DEFAULT_LOCATION) &&
- dataCollectorMap.containsKey(VirtualPropertyUtil.VIRTUAL_DEFAULT_LOCATION_WGS84_PATH)) {
- originalGeoShapeProperty = originalPropertyPath + VirtualPropertyUtil.FIELD_WGS84_COORDINATES;
+ if(virtualPropertyPath.equals(PropertyUtil.VIRTUAL_DEFAULT_LOCATION) &&
+ dataCollectorMap.containsKey(PropertyUtil.VIRTUAL_DEFAULT_LOCATION_WGS84_PATH)) {
+ originalGeoShapeProperty = originalPropertyPath + PropertyUtil.FIELD_WGS84_COORDINATES;
}
}
}
@@ -273,30 +273,30 @@ public class StorageIndexerPayloadMapper {
DecimatedResult result = decimator.decimateShapeObj(shapeObj);
if(result.isDecimated()) {
dataCollectorMap.put(originalGeoShapeProperty, result.getDecimatedShapeObj());
- if(dataCollectorMap.containsKey(VirtualPropertyUtil.VIRTUAL_DEFAULT_LOCATION_WGS84_PATH)) {
- dataCollectorMap.put(VirtualPropertyUtil.VIRTUAL_DEFAULT_LOCATION_WGS84_PATH, result.getDecimatedShapeObj());
+ if(dataCollectorMap.containsKey(PropertyUtil.VIRTUAL_DEFAULT_LOCATION_WGS84_PATH)) {
+ dataCollectorMap.put(PropertyUtil.VIRTUAL_DEFAULT_LOCATION_WGS84_PATH, result.getDecimatedShapeObj());
}
}
- if(dataCollectorMap.containsKey(VirtualPropertyUtil.VIRTUAL_DEFAULT_LOCATION_WGS84_PATH)) {
- dataCollectorMap.put(VirtualPropertyUtil.VIRTUAL_DEFAULT_LOCATION_IS_DECIMATED_PATH, result.isDecimated());
+ if(dataCollectorMap.containsKey(PropertyUtil.VIRTUAL_DEFAULT_LOCATION_WGS84_PATH)) {
+ dataCollectorMap.put(PropertyUtil.VIRTUAL_DEFAULT_LOCATION_IS_DECIMATED_PATH, result.isDecimated());
}
}
private Priority chooseOriginalProperty(String virtualPropertyPath, List<Priority> priorities, Map<String, Object> dataCollectorMap) {
- if (VirtualPropertyUtil.VIRTUAL_DEFAULT_LOCATION.equals(virtualPropertyPath) || VirtualPropertyUtil.DATA_VIRTUAL_DEFAULT_LOCATION.equals(virtualPropertyPath)) {
+ if (PropertyUtil.VIRTUAL_DEFAULT_LOCATION.equals(virtualPropertyPath) || PropertyUtil.DATA_VIRTUAL_DEFAULT_LOCATION.equals(virtualPropertyPath)) {
// Specially handle "data.VirtualProperties.DefaultLocation" -- check the value of the field "wgs84Coordinates"
for (Priority priority : priorities) {
- String originalPropertyPath = VirtualPropertyUtil.removeDataPrefix(priority.getPath());
- String wgs84PropertyField = originalPropertyPath + VirtualPropertyUtil.FIELD_WGS84_COORDINATES;
+ String originalPropertyPath = PropertyUtil.removeDataPrefix(priority.getPath());
+ String wgs84PropertyField = originalPropertyPath + PropertyUtil.FIELD_WGS84_COORDINATES;
if (dataCollectorMap.containsKey(wgs84PropertyField) && dataCollectorMap.get(wgs84PropertyField) != null)
return priority;
}
}
for (Priority priority : priorities) {
- String originalPropertyPath = VirtualPropertyUtil.removeDataPrefix(priority.getPath());
+ String originalPropertyPath = PropertyUtil.removeDataPrefix(priority.getPath());
List<String> originalPropertyNames = dataCollectorMap.keySet().stream()
- .filter(name -> VirtualPropertyUtil.isPropertyPathMatched(name, originalPropertyPath))
+ .filter(name -> PropertyUtil.isPropertyPathMatched(name, originalPropertyPath))
.collect(Collectors.toList());
for (String originalPropertyName : originalPropertyNames) {
if (dataCollectorMap.containsKey(originalPropertyName) && dataCollectorMap.get(originalPropertyName) != null)
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/AugmenterSetting.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/AugmenterSetting.java
new file mode 100644
index 0000000000000000000000000000000000000000..d11d41cc38f5b3c7aaf1cd63cfbafd38ca4250eb
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/AugmenterSetting.java
@@ -0,0 +1,16 @@
+package org.opengroup.osdu.indexer.util;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class AugmenterSetting {
+ private static final String PROPERTY_NAME = "index-augmenter-enabled";
+
+ @Autowired
+ private BooleanFeatureFlagClient booleanFeatureFlagClient;
+
+ public boolean isEnabled() {
+ return booleanFeatureFlagClient.isEnabled(PROPERTY_NAME, false);
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/BooleanFeatureFlagClient.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/BooleanFeatureFlagClient.java
new file mode 100644
index 0000000000000000000000000000000000000000..a27ec91cd02d9bb54060e8509ec470512b7bd36d
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/BooleanFeatureFlagClient.java
@@ -0,0 +1,81 @@
+package org.opengroup.osdu.indexer.util;
+
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.partition.*;
+import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient;
+import org.opengroup.osdu.indexer.util.geo.decimator.FeatureFlagCache;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.stereotype.Component;
+
+@Component
+public class BooleanFeatureFlagClient {
+ private static final String TOKEN_PREFIX = "Bearer ";
+
+ @Lazy
+ @Autowired
+ private FeatureFlagCache cache;
+
+ @Autowired
+ private JaxRsDpsLog logger;
+
+ @Autowired
+ private DpsHeaders headers;
+
+ @Autowired
+ private IPartitionFactory factory;
+
+ @Autowired
+ private IServiceAccountJwtClient tokenService;
+
+ public boolean isEnabled(String featureName, boolean defaultValue) {
+ String dataPartitionId = headers.getPartitionId();
+ String cacheKey = String.format("%s-%s", dataPartitionId, featureName);
+ if (cache != null && cache.containsKey(cacheKey))
+ return cache.get(cacheKey);
+
+ boolean isEnabled = defaultValue;
+ try {
+ PartitionInfo partitionInfo = getPartitionInfo(dataPartitionId);
+ isEnabled = getFeatureValue(partitionInfo, featureName, defaultValue);
+ } catch (Exception e) {
+ this.logger.error(String.format("PartitionService: Error getting %s for dataPartition with Id: %s. Turn on the feature flag by default.", featureName, dataPartitionId), e);
+ }
+ this.cache.put(cacheKey, isEnabled);
+ return isEnabled;
+ }
+
+ private PartitionInfo getPartitionInfo(String dataPartitionId) throws PartitionException {
+ try {
+ DpsHeaders partitionHeaders = DpsHeaders.createFromMap(headers.getHeaders());
+ partitionHeaders.put(DpsHeaders.AUTHORIZATION, this.getAuthorization(dataPartitionId));
+
+ IPartitionProvider partitionProvider = this.factory.create(partitionHeaders);
+ PartitionInfo partitionInfo = partitionProvider.get(dataPartitionId);
+ return partitionInfo;
+ } catch (PartitionException e) {
+ logger.error(String.format("Error getting partition info for data-partition: %s", dataPartitionId), e);
+ throw e;
+ }
+ }
+
+ private String getAuthorization(String dataPartitionId) {
+ String authorization = this.tokenService.getIdToken(dataPartitionId);
+ if(!authorization.startsWith(TOKEN_PREFIX)) {
+ authorization = TOKEN_PREFIX + authorization;
+ }
+ return authorization;
+ }
+
+ private boolean getFeatureValue(PartitionInfo partitionInfo, String featureName, boolean defaultValue) {
+ if(partitionInfo == null || partitionInfo.getProperties() == null)
+ return defaultValue;
+
+ if(partitionInfo.getProperties().containsKey(featureName)) {
+ Property property = partitionInfo.getProperties().get(featureName);
+ return Boolean.parseBoolean((String)property.getValue());
+ }
+ return defaultValue;
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/PropertyUtil.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/PropertyUtil.java
new file mode 100644
index 0000000000000000000000000000000000000000..49a7175ee43287395f3bfc28fe1a3f2d2acba903
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/PropertyUtil.java
@@ -0,0 +1,261 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.util;
+
+import com.google.api.client.util.Strings;
+import com.google.common.collect.MapDifference;
+import com.google.common.collect.Maps;
+import org.opengroup.osdu.indexer.model.Kind;
+
+import java.util.*;
+
+public class PropertyUtil {
+ public static final String DATA_VIRTUAL_DEFAULT_LOCATION = "data.VirtualProperties.DefaultLocation";
+ public static final String VIRTUAL_DEFAULT_LOCATION = "VirtualProperties.DefaultLocation";
+ public static final String FIELD_WGS84_COORDINATES = ".Wgs84Coordinates";
+ public static final String VIRTUAL_DEFAULT_LOCATION_WGS84_PATH = VIRTUAL_DEFAULT_LOCATION + FIELD_WGS84_COORDINATES;
+ public static final String VIRTUAL_DEFAULT_LOCATION_IS_DECIMATED_PATH = VIRTUAL_DEFAULT_LOCATION + ".IsDecimated";
+
+
+ private static final String PROPERTY_DELIMITER = ".";
+ private static final String NESTED_OBJECT_DELIMITER = "[].";
+ private static final String ARRAY_SYMBOL = "[]";
+ private static final String DATA_PREFIX = "data" + PROPERTY_DELIMITER;
+
+ public static boolean isPropertyPathMatched(String propertyPath, String parentPropertyPath) {
+ // We should not just use propertyPath.startsWith(parentPropertyPath)
+ // For example, if parentPropertyPath is "data.FacilityName" and propertyPath.startsWith(parentPropertyPath) is used,
+ // then the property "data.FacilityNameAlias" will be matched and unexpected result will be returned.
+ return !Strings.isNullOrEmpty(propertyPath) && (propertyPath.startsWith(parentPropertyPath + PROPERTY_DELIMITER) || propertyPath.equals(parentPropertyPath));
+ }
+
+ public static boolean hasSameMajorKind(String left, String right) {
+ try {
+ Kind leftKind = new Kind(left);
+ Kind rightKind = new Kind(right);
+
+ String[] leftVersions = leftKind.getVersion().split("\\.");
+ String[] rightVersions = rightKind.getVersion().split("\\.");
+ return leftKind.getAuthority().equals(rightKind.getAuthority()) &&
+ leftKind.getSource().equals(rightKind.getSource()) &&
+ leftKind.getType().equals(rightKind.getType()) &&
+ leftVersions[0].equals(rightVersions[0]);
+ }
+ catch(Exception ex) {
+ // catch exception in case either left kind or right kind is an invalid kind
+ return false;
+ }
+ }
+
+ public static String removeDataPrefix(String path) {
+ if (!Strings.isNullOrEmpty(path) && path.startsWith(DATA_PREFIX))
+ return path.substring(DATA_PREFIX.length());
+ return path;
+ }
+
+ public static String removeIdPostfix(String objectId) {
+ if (objectId != null && objectId.endsWith(":")) {
+ objectId = objectId.substring(0, objectId.length() - 1);
+ }
+ return objectId;
+ }
+
+ public static Map<String, Object> combineObjectMap(Map<String, Object> to, Map<String, Object> from) {
+ if((to == null || to.isEmpty()) && (from == null || from.isEmpty())) {
+ return new HashMap<>();
+ }
+ else if(to == null || to.isEmpty()) {
+ return from;
+ }
+ else if(from == null || from.isEmpty()) {
+ return to;
+ }
+
+ for (Map.Entry<String, Object> entry : from.entrySet()) {
+ if (to.containsKey(entry.getKey())) {
+ Object toObject = to.get(entry.getKey());
+ Object fromObject = entry.getValue();
+ if (toObject instanceof List && fromObject instanceof List) {
+ Set<Object> objectSet = new HashSet<>();
+ objectSet.addAll((List) toObject);
+ objectSet.addAll((List) fromObject);
+ List<Object> propertyValueList = new ArrayList<>(objectSet);
+ Collections.sort(propertyValueList, Comparator.comparing(Object::toString));
+ to.put(entry.getKey(), propertyValueList);
+ }
+ else if(toObject instanceof Map && fromObject instanceof Map) {
+ Object objectMap = combineObjectMap((Map<String, Object>) toObject, (Map<String, Object>) fromObject);
+ to.put(entry.getKey(), objectMap);
+ }
+ else if(!toObject.equals(fromObject)) {
+ if(toObject.getClass().equals(fromObject.getClass())) {
+ List<Object> propertyValueList = new ArrayList<>();
+ propertyValueList.add(toObject);
+ propertyValueList.add(fromObject);
+ Collections.sort(propertyValueList, Comparator.comparing(Object::toString));
+ to.put(entry.getKey(), propertyValueList);
+ }
+ else if(toObject instanceof List || fromObject instanceof List) {
+ List<Object> propertyValueList = toObject instanceof List? (List)toObject : (List)fromObject;
+ Object object = toObject instanceof List? fromObject : toObject;
+ if(!propertyValueList.isEmpty() && propertyValueList.get(0).getClass().equals(object.getClass())) {
+ propertyValueList.add(object);
+ Collections.sort(propertyValueList, Comparator.comparing(Object::toString));
+ to.put(entry.getKey(), propertyValueList);
+ }
+ else if(propertyValueList.isEmpty()) {
+ to.put(entry.getKey(), object);
+ }
+ }
+ }
+ } else {
+ to.put(entry.getKey(), entry.getValue());
+ }
+ }
+
+ return to;
+ }
+
+ public static Map<String, Object> replacePropertyPaths(String newPathPrefix, String valuePath, Map<String, Object> objectMap) {
+ if(Strings.isNullOrEmpty(newPathPrefix) || Strings.isNullOrEmpty(valuePath) || objectMap == null || objectMap.isEmpty()) {
+ return new HashMap<>();
+ }
+
+ newPathPrefix = removeDataPrefix(newPathPrefix);
+ valuePath = removeDataPrefix(valuePath);
+
+ Map<String, Object> values = new HashMap<>();
+ for (Map.Entry<String, Object> entry : objectMap.entrySet()) {
+ String key = entry.getKey();
+ if (key.equals(valuePath) || key.startsWith(valuePath + PROPERTY_DELIMITER)) {
+ key = key.replace(valuePath, newPathPrefix);
+ values.put(key, entry.getValue());
+ }
+ }
+ return values;
+ }
+
+ public static boolean isConcreteKind(String kind) {
+ if(Strings.isNullOrEmpty(kind)) {
+ return false;
+ }
+
+ String[] parts = kind.split(":");
+ if(parts.length != 4)
+ return false;
+ String[] subVersions = parts[3].split("\\.");
+ return (subVersions.length == 3);
+ }
+
+ public static String getKindWithMajor(String kind) {
+ if(Strings.isNullOrEmpty(kind)) {
+ return kind;
+ }
+ String[] parts = kind.split(":");
+ if(parts.length != 4)
+ return "";
+
+ int index = kind.lastIndexOf(":");
+ String[] subVersions = parts[3].split("\\.");
+ String kindWithMajor = kind.substring(0, index) + ":" + subVersions[0] + ".";
+ return kindWithMajor;
+ }
+
+ public static List<String> getChangedProperties(Map<String, Object> leftMap, Map<String, Object> rightMap) {
+ if(leftMap == null && rightMap == null) {
+ return new ArrayList<>();
+ }
+
+ if(leftMap == null) {
+ leftMap = new HashMap<>();
+ }
+ if(rightMap == null) {
+ rightMap = new HashMap<>();
+ }
+ MapDifference<String, Object> difference = Maps.difference(leftMap, rightMap);
+ if(difference.areEqual()) {
+ return new ArrayList<>();
+ }
+
+ Set<String> changedProperties = new HashSet<>();
+ if (difference.entriesOnlyOnLeft().size() > 0) {
+ difference.entriesOnlyOnLeft().forEach((key, value) -> changedProperties.add(key));
+ }
+ if (difference.entriesOnlyOnRight().size() > 0) {
+ difference.entriesOnlyOnRight().forEach((key, value) -> changedProperties.add(key));
+ }
+ if (difference.entriesDiffering().size() > 0) {
+ for(Map.Entry<String, MapDifference.ValueDifference<Object>> entry : difference.entriesDiffering().entrySet()) {
+ try {
+ MapDifference.ValueDifference<Object> valueDifference = entry.getValue();
+ Object left = valueDifference.leftValue();
+ Object right = valueDifference.rightValue();
+ if(left == null && right == null) {
+ continue;
+ }
+ else if(left == null || right == null) {
+ changedProperties.add(entry.getKey());
+ }
+ else if(left instanceof Map) {
+ Map<String, Object> innerLeftMap = (Map<String, Object>)left;
+ Map<String, Object> innerRightMap = (Map<String, Object>)right;
+ List<String> nestedChangedProperties = getChangedProperties(innerLeftMap, innerRightMap);
+ for (String nestedProperty: nestedChangedProperties) {
+ String p = entry.getKey() + PROPERTY_DELIMITER + nestedProperty;
+ changedProperties.add(p);
+ }
+ }
+ else if(left instanceof List) {
+ List<Object> innerLeftList = (List<Object>)left;
+ List<Object> innerRightList = (List<Object>)right;
+ if(innerLeftList.size() != innerRightList.size()) {
+ String p = entry.getKey() + ARRAY_SYMBOL;
+ changedProperties.add(p);
+ }
+ else {
+ for(int i = 0; i < innerLeftList.size(); i++) {
+ Map<String, Object> innerLeftMap = (Map<String, Object>)innerLeftList.get(i);
+ Map<String, Object> innerRightMap = (Map<String, Object>)innerRightList.get(i);
+ List<String> nestedChangedProperties = getChangedProperties(innerLeftMap, innerRightMap);
+ for (String nestedProperty: nestedChangedProperties) {
+ String p = entry.getKey() + NESTED_OBJECT_DELIMITER + nestedProperty;
+ changedProperties.add(p);
+ }
+ }
+ }
+ }
+ else {
+ // Special handle of number. The integer value from the search result could be converted to double
+ if(left instanceof Double || right instanceof Double) {
+ double leftValue = Double.parseDouble(left.toString());
+ double rightValue = Double.parseDouble(right.toString());
+ if(Double.compare(leftValue, rightValue) == 0) {
+ continue; // The left/right values are the same in this case
+ }
+ }
+ changedProperties.add(entry.getKey());
+ }
+ }
+ catch (Exception ex) {
+ // assume there is difference in this case
+ changedProperties.add(entry.getKey());
+ }
+ }
+ }
+
+ return new ArrayList<>(changedProperties);
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/VirtualPropertyUtil.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/VirtualPropertyUtil.java
deleted file mode 100644
index ba9cea6085728ede4305fdacc63a568ab32d63c0..0000000000000000000000000000000000000000
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/VirtualPropertyUtil.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Copyright © Schlumberger
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0 *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.util;
-
-import com.google.api.client.util.Strings;
-
-public class VirtualPropertyUtil {
- public static final String DATA_VIRTUAL_DEFAULT_LOCATION = "data.VirtualProperties.DefaultLocation";
- public static final String VIRTUAL_DEFAULT_LOCATION = "VirtualProperties.DefaultLocation";
- public static final String FIELD_WGS84_COORDINATES = ".Wgs84Coordinates";
- public static final String VIRTUAL_DEFAULT_LOCATION_WGS84_PATH = VIRTUAL_DEFAULT_LOCATION + FIELD_WGS84_COORDINATES;
- public static final String VIRTUAL_DEFAULT_LOCATION_IS_DECIMATED_PATH = VIRTUAL_DEFAULT_LOCATION + ".IsDecimated";
-
-
- private static final String PROPERTY_DELIMITER = ".";
- private static final String DATA_PREFIX = "data" + PROPERTY_DELIMITER;
-
- public static boolean isPropertyPathMatched(String propertyPath, String parentPropertyPath) {
- // We should not just use propertyPath.startsWith(parentPropertyPath)
- // For example, if parentPropertyPath is "data.FacilityName" and propertyPath.startsWith(parentPropertyPath) is used,
- // then the property "data.FacilityNameAlias" will be matched and unexpected result will be returned.
- return !Strings.isNullOrEmpty(propertyPath) && (propertyPath.startsWith(parentPropertyPath + PROPERTY_DELIMITER) || propertyPath.equals(parentPropertyPath));
- }
-
- public static String removeDataPrefix(String path) {
- if (!Strings.isNullOrEmpty(path) && path.startsWith(DATA_PREFIX))
- return path.substring(DATA_PREFIX.length());
- return path;
- }
-}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/geo/decimator/DecimationSettingCache.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/geo/decimator/FeatureFlagCache.java
similarity index 89%
rename from indexer-core/src/main/java/org/opengroup/osdu/indexer/util/geo/decimator/DecimationSettingCache.java
rename to indexer-core/src/main/java/org/opengroup/osdu/indexer/util/geo/decimator/FeatureFlagCache.java
index 127d9e7079bad1ac0d6c6c12d2aa50d32c09aca9..c24d200b7ce3452c89304213d69f0cde47112a9a 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/geo/decimator/DecimationSettingCache.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/geo/decimator/FeatureFlagCache.java
@@ -19,8 +19,8 @@ import org.opengroup.osdu.core.common.cache.VmCache;
import org.springframework.stereotype.Component;
@Component
-public class DecimationSettingCache extends VmCache<String, Boolean> {
- public DecimationSettingCache() {
+public class FeatureFlagCache extends VmCache<String, Boolean> {
+ public FeatureFlagCache() {
super(300, 1000);
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/geo/decimator/GeoShapeDecimationSetting.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/geo/decimator/GeoShapeDecimationSetting.java
index a062c1ac6e645e3e5bdcec4d9c066e0d0230d199..ddab10f0180684167776593c5b064816dadd84e5 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/geo/decimator/GeoShapeDecimationSetting.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/geo/decimator/GeoShapeDecimationSetting.java
@@ -15,74 +15,18 @@
package org.opengroup.osdu.indexer.util.geo.decimator;
-import org.apache.http.HttpStatus;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.partition.*;
-import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient;
+import org.opengroup.osdu.indexer.util.BooleanFeatureFlagClient;
import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
@Component
public class GeoShapeDecimationSetting {
private static final String PROPERTY_NAME = "indexer-decimation-enabled";
- @Lazy
@Autowired
- private DecimationSettingCache cache;
-
- @Autowired
- private JaxRsDpsLog logger;
-
- @Autowired
- private DpsHeaders headers;
-
- @Autowired
- private IPartitionFactory factory;
-
- @Autowired
- private IServiceAccountJwtClient tokenService;
+ private BooleanFeatureFlagClient booleanFeatureFlagClient;
public boolean isDecimationEnabled() {
- String dataPartitionId = headers.getPartitionId();
- String cacheKey = String.format("%s-%s", dataPartitionId, PROPERTY_NAME);
- if (cache != null && cache.containsKey(cacheKey))
- return cache.get(cacheKey);
-
- boolean decimationEnabled = true;
- try {
- PartitionInfo partitionInfo = getPartitionInfo(dataPartitionId);
- decimationEnabled = getDecimationSetting(partitionInfo);
- } catch (Exception e) {
- this.logger.error(String.format("PartitionService: Error getting %s for dataPartition with Id: %s. Turn on the feature flag by default.", PROPERTY_NAME, dataPartitionId), e);
- }
- this.cache.put(cacheKey, decimationEnabled);
- return decimationEnabled;
- }
-
- private PartitionInfo getPartitionInfo(String dataPartitionId) {
- try {
- DpsHeaders partitionHeaders = DpsHeaders.createFromMap(headers.getHeaders());
- partitionHeaders.put(DpsHeaders.AUTHORIZATION, this.tokenService.getIdToken(dataPartitionId));
-
- IPartitionProvider partitionProvider = this.factory.create(partitionHeaders);
- PartitionInfo partitionInfo = partitionProvider.get(dataPartitionId);
- return partitionInfo;
- } catch (PartitionException e) {
- throw new AppException(HttpStatus.SC_FORBIDDEN, "Service unavailable", String.format("Error getting partition info for data-partition: %s", dataPartitionId), e);
- }
- }
-
- private boolean getDecimationSetting(PartitionInfo partitionInfo) {
- if(partitionInfo == null || partitionInfo.getProperties() == null)
- return true;
-
- if(partitionInfo.getProperties().containsKey(PROPERTY_NAME)) {
- Property property = partitionInfo.getProperties().get(PROPERTY_NAME);
- return Boolean.parseBoolean((String)property.getValue());
- }
- return true;
+ return booleanFeatureFlagClient.isEnabled(PROPERTY_NAME, true);
}
}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/indexproperty/RelatedObjectsSpecTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/indexproperty/RelatedObjectsSpecTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..63a53c37d41db25c2ae8284fe6947016ad40906c
--- /dev/null
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/indexproperty/RelatedObjectsSpecTest.java
@@ -0,0 +1,121 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model.indexproperty;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+@RunWith(SpringRunner.class)
+public class RelatedObjectsSpecTest {
+
+ @Test
+ public void hasValidCondition_return_true() {
+ RelatedObjectsSpec spec = new RelatedObjectsSpec();
+ spec.setRelatedObjectKind("osdu:wks:master-data--GeoPoliticalEntity:1.");
+ spec.setRelationshipDirection(RelatedObjectsSpec.CHILD_TO_PARENT);
+ spec.setRelatedObjectID("data.GeoContexts[].GeoPoliticalEntityID");
+ spec.setRelatedConditionProperty("data.GeoContexts[].GeoTypeID");
+ List<String> matches = Arrays.asList("opendes:reference-data--GeoPoliticalEntityType:Country:");
+ spec.setRelatedConditionMatches(matches);
+
+ Assert.assertTrue(spec.hasValidCondition());
+ }
+
+ @Test
+ public void hasValidCondition_return_true_with_multi_nested() {
+ RelatedObjectsSpec spec = new RelatedObjectsSpec();
+ spec.setRelatedObjectKind("osdu:wks:master-data--Organisation:1.");
+ spec.setRelationshipDirection(RelatedObjectsSpec.CHILD_TO_PARENT);
+ spec.setRelatedObjectID("data.TechnicalAssurances[].Reviewers[].OrganisationID");
+ spec.setRelatedConditionProperty("data.TechnicalAssurances[].Reviewers[].RoleTypeID");
+ List<String> matches = Arrays.asList("opendes:reference-data--ContactRoleType:AccountOwner:");
+ spec.setRelatedConditionMatches(matches);
+
+ Assert.assertTrue(spec.hasValidCondition());
+ }
+
+ @Test
+ public void hasValidCondition_return_false_with_unmatched_multi_nested() {
+ RelatedObjectsSpec spec = new RelatedObjectsSpec();
+ spec.setRelatedObjectKind("osdu:wks:master-data--Organisation:1.");
+ spec.setRelatedObjectID("data.TechnicalAssurances[].Reviewers[].OrganisationID");
+ spec.setRelatedConditionProperty("data.TechnicalAssurances[].Auditors[].RoleTypeID");
+ List<String> matches = Arrays.asList("opendes:reference-data--ContactRoleType:AccountOwner:");
+ spec.setRelatedConditionMatches(matches);
+
+ Assert.assertFalse(spec.hasValidCondition());
+ }
+
+ @Test
+ public void hasValidCondition_return_false_with_null_propertyPath() {
+ RelatedObjectsSpec spec = new RelatedObjectsSpec();
+ spec.setRelatedObjectKind("osdu:wks:master-data--GeoPoliticalEntity:1.");
+ spec.setRelatedObjectID("data.GeoContexts[].GeoPoliticalEntityID");
+ Assert.assertFalse(spec.hasValidCondition());
+ }
+
+ @Test
+ public void hasValidCondition_return_false_with_empty_matches() {
+ RelatedObjectsSpec spec = new RelatedObjectsSpec();
+ spec.setRelatedObjectKind("osdu:wks:master-data--GeoPoliticalEntity:1.");
+ spec.setRelatedObjectID("data.GeoContexts[].GeoPoliticalEntityID");
+ spec.setRelatedConditionProperty("data.GeoContexts[].GeoTypeID");
+ spec.setRelatedConditionMatches(new ArrayList<>());
+
+ Assert.assertFalse(spec.hasValidCondition());
+ }
+
+ @Test
+ public void hasValidCondition_return_false_with_unmatched_propertyPath() {
+ RelatedObjectsSpec spec = new RelatedObjectsSpec();
+ spec.setRelatedObjectKind("osdu:wks:master-data--GeoPoliticalEntity:1.");
+ spec.setRelatedObjectID("data.GeoContexts[].GeoPoliticalEntityID");
+ spec.setRelatedConditionProperty("data.VerticalMeasurementID[].VerticalMeasurementID");
+ List<String> matches = Arrays.asList("KB");
+ spec.setRelatedConditionMatches(matches);
+
+ Assert.assertFalse(spec.hasValidCondition());
+ }
+
+ @Test
+ public void hasValidCondition_return_false_without_nested_property() {
+ RelatedObjectsSpec spec = new RelatedObjectsSpec();
+ spec.setRelatedObjectKind("osdu:wks:master-data--GeoPoliticalEntity:1.");
+ spec.setRelatedObjectID("data.GeoContexts[]");
+ spec.setRelatedConditionProperty("data.GeoContexts[]");
+ List<String> matches = Arrays.asList("opendes:reference-data--GeoPoliticalEntityType:Country:");
+ spec.setRelatedConditionMatches(matches);
+ Assert.assertFalse(spec.hasValidCondition());
+ }
+
+ @Test
+ public void hasValidCondition_return_false_for_none_nested_property() {
+ RelatedObjectsSpec spec = new RelatedObjectsSpec();
+ spec.setRelatedObjectKind("osdu:wks:master-data--GeoPoliticalEntity:1.");
+ spec.setRelatedObjectID("data.GeoContexts.GeoPoliticalEntityID");
+ spec.setRelatedConditionProperty("data.GeoContexts.GeoTypeID");
+ List<String> matches = Arrays.asList("opendes:reference-data--GeoPoliticalEntityType:Country:");
+ spec.setRelatedConditionMatches(matches);
+ Assert.assertFalse(spec.hasValidCondition());
+ }
+
+}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/indexproperty/ValueExtractionTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/indexproperty/ValueExtractionTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..4c578c9a400250d5c69cb5259d8cd0ee53031281
--- /dev/null
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/indexproperty/ValueExtractionTest.java
@@ -0,0 +1,109 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model.indexproperty;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+@RunWith(SpringRunner.class)
+public class ValueExtractionTest {
+ @Test
+ public void hasValidCondition_return_true() {
+ ValueExtraction valueExtraction = new ValueExtraction();
+ valueExtraction.setValuePath("data.NameAliases[].AliasName");
+ valueExtraction.setRelatedConditionProperty("data.NameAliases[].AliasNameTypeID");
+ List<String> matches = Arrays.asList("opendes:reference-data--AliasNameType:UniqueIdentifier:","reference-data--AliasNameType:RegulatoryName:");
+ valueExtraction.setRelatedConditionMatches(matches);
+
+ Assert.assertTrue(valueExtraction.hasValidCondition());
+ }
+
+ @Test
+ public void hasValidCondition_return_true_with_multi_nested() {
+ ValueExtraction valueExtraction = new ValueExtraction();
+ valueExtraction.setValuePath("data.TechnicalAssurances[].Reviewers[].Name");
+ valueExtraction.setRelatedConditionProperty("data.TechnicalAssurances[].Reviewers[].RoleTypeID");
+ List<String> matches = Arrays.asList("opendes:reference-data--ContactRoleType:AccountOwner:");
+ valueExtraction.setRelatedConditionMatches(matches);
+
+ Assert.assertTrue(valueExtraction.hasValidCondition());
+ }
+
+ @Test
+ public void hasValidCondition_return_false_with_unmatched_multi_nested() {
+ ValueExtraction valueExtraction = new ValueExtraction();
+ valueExtraction.setValuePath("data.TechnicalAssurances[].Reviewers[].Name");
+ valueExtraction.setRelatedConditionProperty("data.TechnicalAssurances[].Auditors[].RoleTypeID");
+ List<String> matches = Arrays.asList("opendes:reference-data--ContactRoleType:AccountOwner:");
+ valueExtraction.setRelatedConditionMatches(matches);
+
+ Assert.assertFalse(valueExtraction.hasValidCondition());
+ }
+
+ @Test
+ public void hasValidCondition_return_false_with_null_propertyPath() {
+ ValueExtraction valueExtraction = new ValueExtraction();
+ valueExtraction.setValuePath("data.NameAliases[].AliasName");
+ Assert.assertFalse(valueExtraction.hasValidCondition());
+ }
+
+ @Test
+ public void hasValidCondition_return_false_with_empty_matches() {
+ ValueExtraction valueExtraction = new ValueExtraction();
+ valueExtraction.setValuePath("data.NameAliases[].AliasName");
+ valueExtraction.setRelatedConditionProperty("data.NameAliases[].AliasNameTypeID");
+ valueExtraction.setRelatedConditionMatches(new ArrayList<>());
+
+ Assert.assertFalse(valueExtraction.hasValidCondition());
+ }
+
+ @Test
+ public void hasValidCondition_return_false_with_unmatched_propertyPath() {
+ ValueExtraction valueExtraction = new ValueExtraction();
+ valueExtraction.setValuePath("data.NameAliases[].AliasName");
+ valueExtraction.setRelatedConditionProperty("data.VerticalMeasurementID[].VerticalMeasurementID");
+ List<String> matches = Arrays.asList("KB");
+ valueExtraction.setRelatedConditionMatches(matches);
+
+ Assert.assertFalse(valueExtraction.hasValidCondition());
+ }
+
+ @Test
+ public void hasValidCondition_return_false_without_nested_property() {
+ ValueExtraction valueExtraction = new ValueExtraction();
+ valueExtraction.setValuePath("data.NameAliases[]");
+ valueExtraction.setRelatedConditionProperty("data.NameAliases[]");
+ List<String> matches = Arrays.asList("opendes:reference-data--AliasNameType:UniqueIdentifier:","reference-data--AliasNameType:RegulatoryName:");
+ valueExtraction.setRelatedConditionMatches(matches);
+ Assert.assertFalse(valueExtraction.hasValidCondition());
+ }
+
+ @Test
+ public void hasValidCondition_return_false_for_none_nested_property() {
+ ValueExtraction valueExtraction = new ValueExtraction();
+ valueExtraction.setValuePath("data.NameAliases.AliasName");
+ valueExtraction.setRelatedConditionProperty("data.NameAliases.AliasNameTypeID");
+ List<String> matches = Arrays.asList("opendes:reference-data--AliasNameType:UniqueIdentifier:","reference-data--AliasNameType:RegulatoryName:");
+ valueExtraction.setRelatedConditionMatches(matches);
+ Assert.assertFalse(valueExtraction.hasValidCondition());
+ }
+}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/indexproperty/jackson/PropertyPathDeserializerTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/indexproperty/jackson/PropertyPathDeserializerTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..d149703ab706598c8667ac354d9e6598fa7148a9
--- /dev/null
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/indexproperty/jackson/PropertyPathDeserializerTest.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model.indexproperty.jackson;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.SneakyThrows;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.opengroup.osdu.indexer.model.indexproperty.PropertyConfiguration;
+import org.opengroup.osdu.indexer.model.indexproperty.PropertyConfigurations;
+import org.opengroup.osdu.indexer.model.indexproperty.PropertyPath;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+
+@RunWith(SpringRunner.class)
+public class PropertyPathDeserializerTest {
+ @Test
+ public void deserialize_configurations_test() throws JsonProcessingException {
+ String jsonText = getJsonFromFile("well_configuration_record.json");
+ ObjectMapper objectMapper = new ObjectMapper();
+ PropertyConfigurations configurations = objectMapper.readValue(jsonText, PropertyConfigurations.class);
+ Assert.assertNotNull(configurations);
+ Assert.assertEquals(2, configurations.getConfigurations().size());
+ PropertyConfiguration countryNameConfiguration = configurations.getConfigurations().get(0);
+ PropertyConfiguration wellUWIConfiguration = configurations.getConfigurations().get(1);
+
+ Assert.assertEquals("CountryNames", countryNameConfiguration.getName());
+ Assert.assertEquals(1, countryNameConfiguration.getPaths().size());
+ PropertyPath path1 = countryNameConfiguration.getPaths().get(0);
+ Assert.assertTrue(path1.hasValidRelatedObjectsSpec());
+ Assert.assertTrue(path1.getRelatedObjectsSpec().hasValidCondition());
+ Assert.assertEquals(1, path1.getRelatedObjectsSpec().getRelatedConditionMatches().size());
+ Assert.assertTrue(path1.hasValidValueExtraction());
+ Assert.assertFalse(path1.getValueExtraction().hasValidCondition());
+
+ Assert.assertEquals("WellUWI", wellUWIConfiguration.getName());
+ Assert.assertEquals(1, wellUWIConfiguration.getPaths().size());
+ PropertyPath path2 = wellUWIConfiguration.getPaths().get(0);
+ Assert.assertFalse(path2.hasValidRelatedObjectsSpec());
+ Assert.assertTrue(path2.hasValidValueExtraction());
+ Assert.assertTrue(path2.getValueExtraction().hasValidCondition());
+ Assert.assertEquals(5, path2.getValueExtraction().getRelatedConditionMatches().size());
+
+ }
+
+ @SneakyThrows
+ private String getJsonFromFile(String file) {
+ InputStream inStream = this.getClass().getResourceAsStream("/indexproperty/" + file);
+ BufferedReader br = new BufferedReader(new InputStreamReader(inStream));
+ StringBuilder stringBuilder = new StringBuilder();
+ String sCurrentLine;
+ while ((sCurrentLine = br.readLine()) != null)
+ {
+ stringBuilder.append(sCurrentLine).append("\n");
+ }
+ return stringBuilder.toString();
+ }
+
+}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java
index d1ea8650ed05dac34ead0ad8f88dbc0f57b6de47..4c488fb13c7b1d66e4a7f7113edf003e40b5bc1e 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java
@@ -14,6 +14,9 @@
package org.opengroup.osdu.indexer.service;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
import org.apache.http.HttpStatus;
import org.elasticsearch.client.RestHighLevelClient;
import org.junit.Assert;
@@ -27,35 +30,30 @@ import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.RequestStatus;
import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
import org.opengroup.osdu.core.common.model.indexer.OperationType;
+import org.opengroup.osdu.core.common.model.storage.SchemaItem;
import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
-import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache;
+import org.opengroup.osdu.indexer.cache.PartitionSafeFlattenedSchemaCache;
+import org.opengroup.osdu.indexer.cache.PartitionSafeSchemaCache;
+import org.opengroup.osdu.indexer.model.indexproperty.PropertyConfigurations;
import org.opengroup.osdu.indexer.schema.converter.exeption.SchemaProcessingException;
import org.opengroup.osdu.indexer.schema.converter.interfaces.IVirtualPropertiesSchemaCache;
+import org.opengroup.osdu.indexer.util.AugmenterSetting;
import org.opengroup.osdu.indexer.util.ElasticClientHandler;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.springframework.test.context.junit4.SpringRunner;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
+import java.lang.reflect.Type;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyBoolean;
-import static org.mockito.ArgumentMatchers.anyString;
-import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.junit.Assert.*;
+import static org.mockito.ArgumentMatchers.*;
+import static org.mockito.Mockito.*;
import static org.mockito.MockitoAnnotations.initMocks;
import static org.powermock.api.mockito.PowerMockito.mock;
import static org.powermock.api.mockito.PowerMockito.when;
@@ -81,9 +79,15 @@ public class IndexerSchemaServiceTest {
@Mock
private SchemaService schemaService;
@Mock
- private ISchemaCache schemaCache;
+ private PartitionSafeSchemaCache schemaCache;
+ @Mock
+ private PartitionSafeFlattenedSchemaCache flattenedSchemaCache;
@Mock
private IVirtualPropertiesSchemaCache virtualPropertiesSchemaCache;
+ @Mock
+ private PropertyConfigurationsService propertyConfigurationsService;
+ @Mock
+ private AugmenterSetting augmenterSetting;
@InjectMocks
private IndexSchemaServiceImpl sut;
@@ -92,6 +96,7 @@ public class IndexerSchemaServiceTest {
initMocks(this);
RestHighLevelClient restHighLevelClient = mock(RestHighLevelClient.class);
when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ when(augmenterSetting.isEnabled()).thenReturn(true);
}
@Test
@@ -221,6 +226,151 @@ public class IndexerSchemaServiceTest {
verifyNoMoreInteractions(this.mappingService);
}
+ @Test
+ public void should_merge_schema_without_invalidateCache_when_kind_has_property_configuration() throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"startDate\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ String extendedProperties = "[{\n" +
+ " \"path\": \"ext_p1\",\n" +
+ " \"kind\": \"string\"\n" +
+ " }, {\n" +
+ " \"path\": \"ext_p2\",\n" +
+ " \"kind\": \"string\"\n" +
+ " }\n" +
+ "]";
+ Gson gson = new Gson();
+ Type listOfSchemaItems = new TypeToken<ArrayList<SchemaItem>>() {}.getType();
+ List<SchemaItem> extendedSchemaItems = gson.fromJson(extendedProperties, listOfSchemaItems);
+
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.schemaService.getSchema(kind)).thenReturn(storageSchema);
+ when(this.propertyConfigurationsService.getPropertyConfigurations(kind)).thenReturn(new PropertyConfigurations());
+ when(this.propertyConfigurationsService.getExtendedSchemaItems(any(), any(), any())).thenReturn(extendedSchemaItems);
+
+ IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false);
+ assertNotNull(indexSchema);
+ assertEquals(4, indexSchema.getDataSchema().size());
+ verify(this.schemaCache, times(0)).delete(any());
+ verify(this.flattenedSchemaCache, times(0)).delete(any());
+ verify(this.virtualPropertiesSchemaCache, times(0)).delete(any());
+ }
+
+ @Test
+ public void should_merge_schema_with_invalidateCache_when_kind_has_property_configuration() throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"startDate\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ String extendedProperties = "[{\n" +
+ " \"path\": \"ext_p1\",\n" +
+ " \"kind\": \"string\"\n" +
+ " }, {\n" +
+ " \"path\": \"ext_p2\",\n" +
+ " \"kind\": \"string\"\n" +
+ " }\n" +
+ "]";
+ Gson gson = new Gson();
+ Type listOfSchemaItems = new TypeToken<ArrayList<SchemaItem>>() {}.getType();
+ List<SchemaItem> extendedSchemaItems = gson.fromJson(extendedProperties, listOfSchemaItems);
+
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.schemaService.getSchema(kind)).thenReturn(storageSchema);
+ when(this.propertyConfigurationsService.getPropertyConfigurations(kind)).thenReturn(new PropertyConfigurations());
+ when(this.propertyConfigurationsService.getExtendedSchemaItems(any(), any(), any())).thenReturn(extendedSchemaItems);
+
+ IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, true);
+ assertNotNull(indexSchema);
+ assertEquals(4, indexSchema.getDataSchema().size());
+ verify(this.schemaCache, times(1)).delete(any());
+ verify(this.flattenedSchemaCache, times(1)).delete(any());
+ verify(this.virtualPropertiesSchemaCache, times(1)).delete(any());
+ }
+
+ @Test
+ public void should_get_schema_of_related_object_kinds_when__kind_has_property_configuration() throws IOException, URISyntaxException {
+ String kind = "osdu:wks:work-product-component--WellLog:1.0.0";
+ String storageSchema = "{" +
+ " \"kind\": \"osdu:wks:work-product-component--WellLog:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"startDate\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ String propertyConfigurations = "{\n" +
+ " \"Name\": \"WellLogIndex-PropertyPathConfiguration\",\n" +
+ " \"Code\": \"osdu:wks:work-product-component--WellLog:1.\",\n" +
+ " \"AttributionAuthority\": \"OSDU\",\n" +
+ " \"Configurations\": [{\n" +
+ " \"Name\": \"WellboreName\",\n" +
+ " \"Policy\": \"ExtractFirstMatch\",\n" +
+ " \"Paths\": [{\n" +
+ " \"RelatedObjectsSpec.RelationshipDirection\": \"ChildToParent\",\n" +
+ " \"RelatedObjectsSpec.RelatedObjectKind\": \"osdu:wks:master-data--Wellbore:1.\",\n" +
+ " \"RelatedObjectsSpec.RelatedObjectID\": \"data.WellboreID\",\n" +
+ " \"ValueExtraction.ValuePath\": \"data.FacilityName\"\n" +
+ " }\n" +
+ " ]\n" +
+ " }, {\n" +
+ " \"Name\": \"TechnicalAssuranceReviewerOrganisationNames\",\n" +
+ " \"Policy\": \"ExtractAllMatches\",\n" +
+ " \"Paths\": [{\n" +
+ " \"RelatedObjectsSpec.RelationshipDirection\": \"ChildToParent\",\n" +
+ " \"RelatedObjectsSpec.RelatedObjectKind\": \"osdu:wks:master-data--Organisation:1.\",\n" +
+ " \"RelatedObjectsSpec.RelatedObjectID\": \"data.TechnicalAssurances[].Reviewers[].OrganisationID\",\n" +
+ " \"ValueExtraction.ValuePath\": \"data.OrganisationName\"\n" +
+ " }\n" +
+ " ]\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+ ObjectMapper objectMapper = new ObjectMapper();
+ PropertyConfigurations configurations = objectMapper.readValue(propertyConfigurations, PropertyConfigurations.class);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.schemaService.getSchema(kind)).thenReturn(storageSchema);
+ when(this.propertyConfigurationsService.getPropertyConfigurations(kind)).thenReturn(configurations);
+ when(this.propertyConfigurationsService.resolveConcreteKind(anyString())).thenAnswer(invocation -> {
+ String relatedObjectKind = invocation.getArgument(0);
+ return relatedObjectKind + "0.0";
+ });
+
+ IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false);
+ assertEquals(2, indexSchema.getDataSchema().size());
+ verify(this.propertyConfigurationsService, times(2)).resolveConcreteKind(any());
+ verify(this.schemaCache, times(3)).get(any());
+ verify(this.schemaService, times(3)).getSchema(any());
+ }
+
@Test
public void should_throw_mapping_conflict_when_elastic_backend_cannot_process_schema_changes() throws IOException, URISyntaxException {
String kind = "tenant1:avocet:completion:1.0.0";
@@ -315,13 +465,12 @@ public class IndexerSchemaServiceTest {
when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
- when(this.schemaCache.get(kind)).thenReturn("schema");
- when(this.schemaCache.get(kind + "_flattened")).thenReturn("flattened schema");
this.sut.processSchemaMessages(schemaMessages);
- verify(this.schemaCache, times(2)).get(anyString());
- verify(this.schemaCache, times(2)).delete(anyString());
+ verify(this.schemaCache, times(1)).delete(anyString());
+ verify(this.flattenedSchemaCache, times(1)).delete(anyString());
+ verify(this.virtualPropertiesSchemaCache, times(1)).delete(anyString());
}
@Test
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceImplTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceImplTest.java
index b7e24d13458e006224755595796098c8204be545..5ab9d69ec9d284b08c8dfde688021832d45d8c9d 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceImplTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceImplTest.java
@@ -1,3 +1,18 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.opengroup.osdu.indexer.service;
import com.google.common.reflect.TypeToken;
@@ -10,6 +25,7 @@ import org.elasticsearch.rest.RestStatus;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Spy;
@@ -23,8 +39,11 @@ import org.opengroup.osdu.core.common.model.storage.ConversionStatus;
import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
import org.opengroup.osdu.indexer.logging.AuditLogger;
+import org.opengroup.osdu.indexer.model.indexproperty.PropertyConfigurations;
import org.opengroup.osdu.indexer.provider.interfaces.IPublisher;
+import org.opengroup.osdu.indexer.util.AugmenterSetting;
import org.opengroup.osdu.indexer.util.ElasticClientHandler;
+import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -35,8 +54,7 @@ import java.net.URISyntaxException;
import java.util.*;
import static java.util.Collections.singletonList;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
+import static org.junit.Assert.*;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@@ -78,11 +96,20 @@ public class IndexerServiceImplTest {
private IMappingService mappingService;
@Mock
private IPublisher progressPublisher;
+ @Mock
+ private PropertyConfigurationsService propertyConfigurationsService;
+ @Mock
+ private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
+ @Mock
+ private AugmenterSetting augmenterSetting;
private List<RecordInfo> recordInfos = new ArrayList<>();
private final String pubsubMsg = "[{\"id\":\"opendes:doc:test1\",\"kind\":\"opendes:testindexer1:well:1.0.0\",\"op\":\"update\"}," +
"{\"id\":\"opendes:doc:test2\",\"kind\":\"opendes:testindexer2:well:1.0.0\",\"op\":\"create\"}, {\"id\":\"opendes:doc:test3\",\"kind\":\"opendes:testindexer2:well:1.0.0\",\"op\":\"create\"}]";
+ private final String pubsubMsgForDeletion = "[{\"id\":\"opendes:doc:test1\",\"kind\":\"opendes:testindexer1:well:1.0.0\",\"op\":\"delete\"}," +
+ "{\"id\":\"opendes:doc:test2\",\"kind\":\"opendes:testindexer2:well:1.0.0\",\"op\":\"delete\"}, {\"id\":\"opendes:doc:test3\",\"kind\":\"opendes:testindexer2:well:1.0.0\",\"op\":\"delete\"}]";
+
private final String kind1 = "opendes:testindexer1:well:1.0.0";
private final String kind2 = "opendes:testindexer2:well:1.0.0";
private final String recordId1 = "opendes:doc:test1";
@@ -96,6 +123,7 @@ public class IndexerServiceImplTest {
@Before
public void setup() throws IOException {
+ when(augmenterSetting.isEnabled()).thenReturn(true);
}
@Test
@@ -118,49 +146,7 @@ public class IndexerServiceImplTest {
@Test
public void should_properlyUpdateAuditLogs_givenValidCreateAndUpdateRecords() {
try {
- mockStatic(Acl.class);
-
- // setup headers
- this.dpsHeaders = new DpsHeaders();
- this.dpsHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
- when(this.requestInfo.getHeaders()).thenReturn(dpsHeaders);
- when(this.requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(dpsHeaders.getHeaders());
-
- // setup message
- Type listType = new TypeToken<List<RecordInfo>>() {}.getType();
- this.recordInfos = (new Gson()).fromJson(this.pubsubMsg, listType);
- Map<String, String> messageAttributes = new HashMap<>();
- messageAttributes.put(DpsHeaders.DATA_PARTITION_ID, "opendes");
- this.recordChangedMessages = RecordChangedMessages.builder().attributes(messageAttributes).messageId("xxxx").publishTime("2000-01-02T10:10:44+0000").data("{}").build();
-
- // setup schema
- Map<String, Object> schema = createSchema();
- indexSchemaServiceMock(kind2, schema);
- indexSchemaServiceMock(kind1, null);
-
- // setup storage records
- Map<String, Object> storageData = new HashMap<>();
- storageData.put("schema1", "test-value");
- List<Records.Entity> validRecords = new ArrayList<>();
- validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build());
- validRecords.add(Records.Entity.builder().id(recordId3).kind(kind2).data(storageData).build());
- List<ConversionStatus> conversionStatus = new LinkedList<>();
- Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build();
- when(this.storageService.getStorageRecords(any(), any())).thenReturn(storageRecords);
-
- // setup elastic, index and mapped document
- when(this.indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true);
- when(this.mappingService.getIndexMappingFromRecordSchema(any())).thenReturn(new HashMap<>());
-
- when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient);
- when(this.restHighLevelClient.bulk(any(), any(RequestOptions.class))).thenReturn(this.bulkResponse);
-
- Map<String, Object> indexerMappedPayload = new HashMap<>();
- indexerMappedPayload.put("id", "keyword");
- when(this.storageIndexerPayloadMapper.mapDataPayload(any(), any(), any())).thenReturn(indexerMappedPayload);
-
- BulkItemResponse[] responses = new BulkItemResponse[]{prepareFailedResponse(), prepareSuccessfulResponse(), prepare400Response()};
- when(this.bulkResponse.getItems()).thenReturn(responses);
+ prepareTestDataAndEnv(this.pubsubMsg);
// test
JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
@@ -178,6 +164,125 @@ public class IndexerServiceImplTest {
}
}
+ @Test
+ public void should_updateAssociatedRecords_givenValidCreateAndUpdateRecords() {
+ try {
+ prepareTestDataAndEnv(this.pubsubMsg);
+
+ // setup property configuration
+ when(this.propertyConfigurationsService.isPropertyConfigurationsEnabled(any())).thenReturn(true);
+ ArgumentCaptor<Map<String, List<String>>> upsertArgumentCaptor = ArgumentCaptor.forClass(Map.class);
+ ArgumentCaptor<Map<String, List<String>>> deleteArgumentCaptor = ArgumentCaptor.forClass(Map.class);
+
+ // test
+ this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
+
+ // validate
+ verify(this.propertyConfigurationsService, times(1)).updateAssociatedRecords(any(), upsertArgumentCaptor.capture(), deleteArgumentCaptor.capture());
+ Map<String, List<String>> upsertKindIds = upsertArgumentCaptor.getValue();
+ Map<String, List<String>> deleteKindIds = deleteArgumentCaptor.getValue();
+ assertEquals(2, upsertKindIds.size());
+ assertEquals(1, upsertKindIds.get(kind1).size());
+ assertEquals(2, upsertKindIds.get(kind2).size());
+ assertEquals(0, deleteKindIds.size());
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_updateAssociatedRecords_givenValidDeleteRecords() {
+ try {
+ prepareTestDataAndEnv(this.pubsubMsgForDeletion);
+
+ // setup property configuration
+ when(this.propertyConfigurationsService.isPropertyConfigurationsEnabled(any())).thenReturn(true);
+ ArgumentCaptor<Map<String, List<String>>> upsertArgumentCaptor = ArgumentCaptor.forClass(Map.class);
+ ArgumentCaptor<Map<String, List<String>>> deleteArgumentCaptor = ArgumentCaptor.forClass(Map.class);
+
+ // test
+ this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
+
+ // validate
+ verify(this.propertyConfigurationsService, times(1)).updateAssociatedRecords(any(), upsertArgumentCaptor.capture(), deleteArgumentCaptor.capture());
+ Map<String, List<String>> upsertKindIds = upsertArgumentCaptor.getValue();
+ Map<String, List<String>> deleteKindIds = deleteArgumentCaptor.getValue();
+ assertEquals(0, upsertKindIds.size());
+ assertEquals(2, deleteKindIds.size());
+ assertEquals(1, deleteKindIds.get(kind1).size());
+ assertEquals(2, deleteKindIds.get(kind2).size());
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_mergeExtendedProperties_givenValidCreateAndUpdateRecords_and_kindsHavingPropertyConfigurations() {
+ try {
+ prepareTestDataAndEnv(this.pubsubMsg);
+
+ // setup property configuration
+ when(this.propertyConfigurationsService.isPropertyConfigurationsEnabled(any())).thenReturn(true);
+ when(this.propertyConfigurationsService.getPropertyConfigurations(any())).thenReturn(new PropertyConfigurations());
+
+ // test
+ this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
+
+ // validate
+ verify(this.propertyConfigurationsService, times(2)).getPropertyConfigurations(any());
+ verify(this.propertyConfigurationsService, times(2)).getExtendedProperties(any(), any(), any());
+ verify(this.propertyConfigurationsService, times(2)).cacheDataRecord(any(), any(), any());
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ private void prepareTestDataAndEnv(String pubsubMsg) throws IOException, URISyntaxException {
+ mockStatic(Acl.class);
+
+ // setup headers
+ this.dpsHeaders = new DpsHeaders();
+ this.dpsHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
+ when(this.requestInfo.getHeaders()).thenReturn(dpsHeaders);
+ when(this.requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(dpsHeaders.getHeaders());
+
+ // setup message
+ Type listType = new TypeToken<List<RecordInfo>>() {}.getType();
+ this.recordInfos = (new Gson()).fromJson(pubsubMsg, listType);
+ Map<String, String> messageAttributes = new HashMap<>();
+ messageAttributes.put(DpsHeaders.DATA_PARTITION_ID, "opendes");
+ this.recordChangedMessages = RecordChangedMessages.builder().attributes(messageAttributes).messageId("xxxx").publishTime("2000-01-02T10:10:44+0000").data("{}").build();
+
+ // setup schema
+ Map<String, Object> schema = createSchema();
+ indexSchemaServiceMock(kind2, schema);
+ indexSchemaServiceMock(kind1, null);
+
+ // setup storage records
+ Map<String, Object> storageData = new HashMap<>();
+ storageData.put("schema1", "test-value");
+ List<Records.Entity> validRecords = new ArrayList<>();
+ validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build());
+ validRecords.add(Records.Entity.builder().id(recordId3).kind(kind2).data(storageData).build());
+ List<ConversionStatus> conversionStatus = new LinkedList<>();
+ Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build();
+ when(this.storageService.getStorageRecords(any(), any())).thenReturn(storageRecords);
+
+ // setup elastic, index and mapped document
+ when(this.indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true);
+ when(this.mappingService.getIndexMappingFromRecordSchema(any())).thenReturn(new HashMap<>());
+
+ when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient);
+ when(this.restHighLevelClient.bulk(any(), any(RequestOptions.class))).thenReturn(this.bulkResponse);
+
+ Map<String, Object> indexerMappedPayload = new HashMap<>();
+ indexerMappedPayload.put("id", "keyword");
+ when(this.storageIndexerPayloadMapper.mapDataPayload(any(), any(), any())).thenReturn(indexerMappedPayload);
+
+ BulkItemResponse[] responses = new BulkItemResponse[]{prepareFailedResponse(), prepareSuccessfulResponse(), prepare400Response()};
+ when(this.bulkResponse.getItems()).thenReturn(responses);
+ }
+
private BulkItemResponse prepareFailedResponse() {
BulkItemResponse responseFail = mock(BulkItemResponse.class);
when(responseFail.isFailed()).thenReturn(true);
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/PropertyConfigurationsServiceImplTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/PropertyConfigurationsServiceImplTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..caeacd2cff10bb51a537c834bc8f416b29a971df
--- /dev/null
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/PropertyConfigurationsServiceImplTest.java
@@ -0,0 +1,901 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.service;
+
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Strings;
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import lombok.SneakyThrows;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.indexer.OperationType;
+import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
+import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
+import org.opengroup.osdu.core.common.model.storage.Schema;
+import org.opengroup.osdu.core.common.model.storage.SchemaItem;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.opengroup.osdu.indexer.cache.*;
+import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.opengroup.osdu.indexer.model.*;
+import org.opengroup.osdu.indexer.model.indexproperty.PropertyConfigurations;
+import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.UnsupportedEncodingException;
+import java.lang.reflect.Type;
+import java.net.URISyntaxException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@RunWith(SpringRunner.class)
+public class PropertyConfigurationsServiceImplTest {
+ private final Gson gson = new Gson();
+
+ @InjectMocks
+ private PropertyConfigurationsServiceImpl sut;
+
+ @Mock
+ private IndexerConfigurationProperties configurationProperties;
+ @Mock
+ private PartitionSafePropertyConfigurationsCache propertyConfigurationCache;
+ @Mock
+ private PartitionSafePropertyConfigurationsEnabledCache propertyConfigurationsEnabledCache;
+ @Mock
+ private PartitionSafeParentChildRelationshipSpecsCache parentChildRelationshipSpecsCache;
+ @Mock
+ private PartitionSafeKindCache kindCache;
+ @Mock
+ private IRelatedObjectCache relatedObjectCache;
+ @Mock
+ private IRecordChangeInfoCache recordChangeInfoCache;
+ @Mock
+ private SearchService searchService;
+ @Mock
+ private SchemaService schemaService;
+ @Mock
+ private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
+ @Mock
+ private IRequestInfo requestInfo;
+ @Mock
+ private JaxRsDpsLog jaxRsDpsLog;
+
+ private final String propertyConfigurationKind = "osdu:wks:reference-data--IndexPropertyPathConfiguration:*";
+ private String childKind;
+ private String childId;
+ private String parentKind;
+ private String parentId;
+
+ @Test
+ public void isPropertyConfigurationsEnabled_invalid_kind() {
+ Assert.assertFalse(sut.isPropertyConfigurationsEnabled(null));
+ Assert.assertFalse(sut.isPropertyConfigurationsEnabled(""));
+ Assert.assertFalse(sut.isPropertyConfigurationsEnabled("anyAuth:anySource:anyEntity"));
+ }
+
+ @Test
+ public void isPropertyConfigurationsEnabled_with_value_true_in_cache() {
+ String kind = "anyAuth:anySource:anyEntity:1.";
+ when(this.propertyConfigurationsEnabledCache.get(any())).thenReturn(true);
+ Assert.assertTrue(sut.isPropertyConfigurationsEnabled(kind));
+ verify(this.propertyConfigurationsEnabledCache, times(0)).put(any(), any());
+ }
+
+ @Test
+ public void isPropertyConfigurationsEnabled_with_value_false_in_cache() {
+ String kind = "anyAuth:anySource:anyEntity:1.";
+ when(this.propertyConfigurationsEnabledCache.get(any())).thenReturn(false);
+ Assert.assertFalse(sut.isPropertyConfigurationsEnabled(kind));
+ verify(this.propertyConfigurationsEnabledCache, times(0)).put(any(), any());
+ }
+
+ @Test
+ public void isPropertyConfigurationsEnabled_with_result_from_search() throws URISyntaxException {
+ String kind = "anyAuth:anySource:anyEntity:1.";
+ SearchResponse response = new SearchResponse();
+ response.setResults(Arrays.asList(new SearchRecord()));
+ when(this.propertyConfigurationsEnabledCache.get(any())).thenReturn(null);
+ when(this.searchService.query(any())).thenReturn(response);
+ Assert.assertTrue(sut.isPropertyConfigurationsEnabled(kind));
+ verify(this.propertyConfigurationsEnabledCache, times(1)).put(any(), any());
+ }
+
+ @Test
+ public void isPropertyConfigurationsEnabled_without_result_from_search() throws URISyntaxException {
+ String kind = "anyAuth:anySource:anyEntity:1.";
+ SearchResponse response = new SearchResponse();
+ when(this.propertyConfigurationsEnabledCache.get(any())).thenReturn(null);
+ when(this.searchService.query(any())).thenReturn(response);
+ Assert.assertFalse(sut.isPropertyConfigurationsEnabled(kind));
+ verify(this.propertyConfigurationsEnabledCache, times(1)).put(any(), any());
+ }
+
+ @Test
+ public void getPropertyConfigurations_invalid_kind() {
+ Assert.assertNull(sut.getPropertyConfigurations(null));
+ Assert.assertNull(sut.getPropertyConfigurations(""));
+ Assert.assertNull(sut.getPropertyConfigurations("anyAuth:anySource:anyEntity"));
+ }
+
+ @Test
+ public void getPropertyConfigurations_with_configuration_in_cache() {
+ String code = "anyAuth:anySource:anyEntity:1.";
+ String kind = "anyAuth:anySource:anyEntity:1.0.0";
+ PropertyConfigurations configuration = new PropertyConfigurations();
+ configuration.setCode(code);
+ when(this.propertyConfigurationCache.get(eq(code))).thenReturn(configuration);
+ PropertyConfigurations configuration2 = sut.getPropertyConfigurations(kind);
+
+ Assert.assertNotNull(configuration2);
+ Assert.assertEquals(code, configuration2.getCode());
+ }
+
+ @Test
+ public void getPropertyConfigurations_with_empty_configuration_in_cache() {
+ String code = "anyAuth:anySource:anyEntity:1.";
+ String kind = "anyAuth:anySource:anyEntity:1.0.0";
+ PropertyConfigurations configuration = new PropertyConfigurations();
+ when(this.propertyConfigurationCache.get(eq(code))).thenReturn(configuration);
+ PropertyConfigurations configuration2 = sut.getPropertyConfigurations(kind);
+
+ Assert.assertNull(configuration2);
+ }
+
+ @Test
+ public void getPropertyConfigurations_with_result_from_search() throws URISyntaxException {
+ Map<String, Object> data = this.getDataMap("well_configuration_record.json");
+ SearchRecord searchRecord = new SearchRecord();
+ searchRecord.setData(data);
+ List<SearchRecord> results = Arrays.asList(searchRecord);
+ SearchResponse searchResponse = new SearchResponse();
+ searchResponse.setResults(results);
+ searchResponse.setTotalCount(results.size());
+ when(this.searchService.query(any())).thenReturn(searchResponse);
+
+ String kind = "osdu:wks:master-data--Well:1.0.0";
+ String code = "osdu:wks:master-data--Well:1.";
+ PropertyConfigurations configuration = sut.getPropertyConfigurations(kind);
+
+ ArgumentCaptor<PropertyConfigurations> argumentCaptor = ArgumentCaptor.forClass(PropertyConfigurations.class);
+ verify(this.propertyConfigurationCache, times(1)).put(any(), argumentCaptor.capture());
+ Assert.assertNotNull(configuration);
+ Assert.assertEquals(code, configuration.getCode());
+ Assert.assertEquals(code, argumentCaptor.getValue().getCode());
+ }
+
+ @Test
+ public void getPropertyConfigurations_without_result_from_search() throws URISyntaxException {
+ when(this.searchService.query(any())).thenReturn(new SearchResponse());
+
+ String kind = "osdu:wks:master-data--Well:1.0.0";
+ PropertyConfigurations configuration = sut.getPropertyConfigurations(kind);
+
+ ArgumentCaptor<PropertyConfigurations> argumentCaptor = ArgumentCaptor.forClass(PropertyConfigurations.class);
+ verify(this.propertyConfigurationCache, times(1)).put(any(), argumentCaptor.capture());
+ Assert.assertNull(configuration);
+ Assert.assertNull(argumentCaptor.getValue().getCode());
+ }
+
+ @Test
+ public void getExtendedProperties_from_children_objects() throws JsonProcessingException, URISyntaxException {
+ PropertyConfigurations propertyConfigurations = getConfigurations("wellbore_configuration_record.json");
+ Map<String, Object> originalDataMap = getDataMap("wellbore_data.json");
+ String jsonText = getJsonFromFile("welllog_search_records.json");
+ Type type = new TypeToken<List<SearchRecord>>() {}.getType();
+ List<SearchRecord> childrenRecords = gson.fromJson(jsonText, type);
+ SearchResponse response = new SearchResponse();
+ response.setResults(childrenRecords);
+ when(this.searchService.query(any())).thenReturn(response);
+
+ Map<String, Object> extendedProperties = this.sut.getExtendedProperties("anyId", originalDataMap, propertyConfigurations);
+ Map<String, Object> expectedExtendedProperties = getDataMap("wellbore_extended_data.json");
+ verifyMap(expectedExtendedProperties, extendedProperties);
+ }
+
+ @Test
+ public void getExtendedProperties_from_self_and_parent_objects() throws JsonProcessingException, URISyntaxException {
+ PropertyConfigurations propertyConfigurations = getConfigurations("welllog_configuration_record.json");
+ Map<String, Object> originalDataMap = getDataMap("welllog_original_data.json");
+ Map<String, Object> relatedObjectData;
+ Map<String, Map<String, Object>> relatedObjects = new HashMap<>();
+ relatedObjectData = getDataMap("wellbore_data.json");
+ relatedObjects.put("opendes:master-data--Wellbore:nz-100000113552", relatedObjectData);
+ relatedObjectData = getDataMap("organisation_data1.json");
+ relatedObjects.put("opendes:master-data--Organisation:BigOil-Department-SeismicInterpretation", relatedObjectData);
+ relatedObjectData = getDataMap("organisation_data2.json");
+ relatedObjects.put("opendes:master-data--Organisation:BigOil-Department-SeismicProcessing", relatedObjectData);
+
+ // Setup search response for searchService.query(...)
+ when(this.searchService.query(any())).thenAnswer(invocation -> {
+ SearchRequest searchRequest = invocation.getArgument(0);
+ String query = searchRequest.getQuery();
+ Map<String, Object> data = null;
+ for(Map.Entry<String, Map<String, Object>> entry: relatedObjects.entrySet()) {
+ if(query.contains(entry.getKey())) {
+ data = entry.getValue();
+ break;
+ }
+ }
+ if(data == null)
+ throw new Exception("Unexpected search");
+ SearchResponse searchResponse = new SearchResponse();
+ SearchRecord record = new SearchRecord();
+ record.setData(data);
+ searchResponse.setResults(Arrays.asList(record));
+ return searchResponse;
+ });
+
+ Map<String, Object> extendedProperties = this.sut.getExtendedProperties("anyId", originalDataMap, propertyConfigurations);
+ Map<String, Object> expectedExtendedProperties = getDataMap("welllog_extended_data.json");
+ verifyMap(expectedExtendedProperties, extendedProperties);
+ }
+
+ private void verifyMap(Map<String, Object> expectedExtendedProperties, Map<String, Object> extendedProperties) {
+ Assert.assertEquals(expectedExtendedProperties.size(), extendedProperties.size());
+
+ for(Map.Entry<String, Object> entry: expectedExtendedProperties.entrySet()) {
+ String name = entry.getKey();
+ Object value = entry.getValue();
+ Assert.assertTrue(extendedProperties.containsKey(name));
+ if(value instanceof String) {
+ Assert.assertEquals(value, extendedProperties.get(name));
+ }
+ else if(value instanceof List) {
+ List<String> expectedValues = (List<String>)value;
+ List<String> values = (List<String>)extendedProperties.get(name);
+ Assert.assertEquals(expectedValues.size(), values.size());
+ for(int i = 0; i < expectedValues.size(); i++) {
+ Assert.assertEquals(expectedValues.get(i), values.get(i));
+ }
+ }
+ else {
+ Assert.assertEquals(value, extendedProperties.get(name));
+ }
+ }
+ }
+
+ @Test
+ public void getExtendedSchemaItems_from_self_and_parent_object_kind() throws JsonProcessingException {
+ PropertyConfigurations propertyConfigurations = getConfigurations("well_configuration_record.json");
+ Schema originalSchema = getSchema("well_storage_schema.json");
+ Schema geoPoliticalEntitySchema = getSchema("geo_political_entity_storage_schema.json");
+ String relatedObjectKind = "osdu:wks:master-data--GeoPoliticalEntity:1.";
+ Map<String, Schema> relatedObjectKindSchemas = new HashMap<>();
+ relatedObjectKindSchemas.put(relatedObjectKind, geoPoliticalEntitySchema);
+
+ List<SchemaItem> extendedSchemaItems = this.sut.getExtendedSchemaItems(originalSchema, relatedObjectKindSchemas, propertyConfigurations);
+ Assert.assertEquals(3, extendedSchemaItems.size());
+ SchemaItem countryNameItem = extendedSchemaItems.stream().filter(item -> item.getPath().equals("CountryNames")).findFirst().orElse(null);
+ Assert.assertNotNull(countryNameItem);
+ Assert.assertEquals("[]string", countryNameItem.getKind());
+
+ SchemaItem wellUWIItem = extendedSchemaItems.stream().filter(item -> item.getPath().equals("WellUWI")).findFirst().orElse(null);
+ Assert.assertNotNull(wellUWIItem);
+ Assert.assertEquals("string", wellUWIItem.getKind());
+
+ SchemaItem associatedIdentitiesItem = extendedSchemaItems.stream().filter(item -> item.getPath().equals("AssociatedIdentities")).findFirst().orElse(null);
+ Assert.assertNotNull(associatedIdentitiesItem);
+ Assert.assertEquals("[]string", associatedIdentitiesItem.getKind());
+ }
+
+ @Test
+ public void getExtendedSchemaItems_from_multiple_object_kinds() throws JsonProcessingException {
+ PropertyConfigurations propertyConfigurations = getConfigurations("welllog_configuration_record.json");
+ Schema originalSchema = getSchema("welllog_storage_schema.json");
+ Map<String, Schema> relatedObjectKindSchemas = new HashMap<>();
+ Schema wellboreSchema = getSchema("wellbore_storage_schema.json");
+ relatedObjectKindSchemas.put("osdu:wks:master-data--Wellbore:1.", wellboreSchema);
+ Schema organisationSchema = getSchema("organisation_storage_schema.json");
+ relatedObjectKindSchemas.put("osdu:wks:master-data--Organisation:1.", organisationSchema);
+
+ String jsonText = getJsonFromFile("welllog_extended_schema_items.json");
+ Type type = new TypeToken<List<SchemaItem>>() {}.getType();
+ List<SchemaItem> expectedExtendedSchemaItems = gson.fromJson(jsonText, type);
+
+ List<SchemaItem> extendedSchemaItems = this.sut.getExtendedSchemaItems(originalSchema, relatedObjectKindSchemas, propertyConfigurations);
+ Assert.assertEquals(expectedExtendedSchemaItems.size(), extendedSchemaItems.size());
+ for(int i = 0; i < expectedExtendedSchemaItems.size(); i++) {
+ SchemaItem expectedExtendedSchemaItem = expectedExtendedSchemaItems.get(i);
+ SchemaItem extendedSchemaItem = extendedSchemaItems.get(i);
+ Assert.assertEquals(expectedExtendedSchemaItem.getKind(), extendedSchemaItem.getKind());
+ Assert.assertEquals(expectedExtendedSchemaItem.getPath(), extendedSchemaItem.getPath());
+ }
+ }
+
+ @Test
+ public void resolveConcreteKind_with_concreteKind() {
+ String kind = "osdu:wks:master-data--Well:1.0.0";
+ Assert.assertEquals(kind, sut.resolveConcreteKind(kind));
+ }
+
+ @Test
+ public void resolveConcreteKind_with_null_empty_kind() {
+ Assert.assertTrue(Strings.isNullOrEmpty(sut.resolveConcreteKind(null)));
+ Assert.assertTrue(Strings.isNullOrEmpty(sut.resolveConcreteKind("")));
+ }
+
+ @Test
+ public void resolveConcreteKind_with_value_in_cache() {
+ String kind = "osdu:wks:master-data--Well:1.";
+ String expectedKind = kind + "2.3";
+
+ when(this.kindCache.get(any())).thenReturn(expectedKind);
+ Assert.assertEquals(expectedKind, sut.resolveConcreteKind(kind));
+ }
+
+ @Test
+ public void resolveConcreteKind_with_result_from_schemaService() throws UnsupportedEncodingException, URISyntaxException {
+ String kind = "osdu:wks:master-data--Well:1.";
+ String expectedKind = kind + "2.3";
+
+ SchemaIdentity schemaIdentity = new SchemaIdentity();
+ schemaIdentity.setAuthority("osdu");
+ schemaIdentity.setSource("wks");
+ schemaIdentity.setEntityType("master-data--Well");
+ schemaIdentity.setSchemaVersionMajor(1);
+ schemaIdentity.setSchemaVersionMinor(2);
+ schemaIdentity.setSchemaVersionPatch(3);
+ SchemaInfo schemaInfo = new SchemaInfo();
+ schemaInfo.setSchemaIdentity(schemaIdentity);
+ List<SchemaInfo> schemaInfos = Arrays.asList(schemaInfo);
+ SchemaInfoResponse response = new SchemaInfoResponse();
+ response.setSchemaInfos(schemaInfos);
+ response.setTotalCount(schemaInfos.size());
+ when(this.schemaService.getSchemaInfos(any(), any(), any(), any(), eq(null), eq(null), eq(true))).thenReturn(response);
+ String latestKind = sut.resolveConcreteKind(kind);
+
+ ArgumentCaptor<String> argumentCaptor = ArgumentCaptor.forClass(String.class);
+ verify(this.kindCache, times(1)).put(any(), argumentCaptor.capture());
+ Assert.assertEquals(expectedKind, latestKind);
+ Assert.assertEquals(expectedKind, argumentCaptor.getValue());
+ }
+
+ @Test
+ public void resolveConcreteKind_without_result_from_schemaService() throws UnsupportedEncodingException, URISyntaxException {
+ String kind = "osdu:wks:master-data--Well:1.";
+
+ SchemaInfoResponse response = new SchemaInfoResponse();
+ when(this.schemaService.getSchemaInfos(any(), any(), any(), any(), eq(null), eq(null), eq(true))).thenReturn(response);
+ String latestKind = sut.resolveConcreteKind(kind);
+
+ verify(this.kindCache, times(0)).put(any(), any());
+ Assert.assertNull(latestKind);
+ }
+
+ @Test
+ public void cacheDataRecord_create_record() throws URISyntaxException {
+ ArgumentCaptor<RecordChangeInfo> recordInfoArgumentCaptor = ArgumentCaptor.forClass(RecordChangeInfo.class);
+ ArgumentCaptor<Map<String, Object>> dataMapArgumentCaptor = ArgumentCaptor.forClass(Map.class);
+ String recordId = "anyId";
+ String kind = "anyKind";
+ Map<String, Object> dataMap = new HashMap<>();
+ dataMap.put("p1", "v1");
+
+ when(this.searchService.query(any())).thenReturn(new SearchResponse());
+
+ this.sut.cacheDataRecord(recordId, kind, dataMap);
+
+ verify(this.recordChangeInfoCache, times(1)).put(any(), recordInfoArgumentCaptor.capture());
+ verify(this.relatedObjectCache, times(1)).put(any(), dataMapArgumentCaptor.capture());
+
+ Assert.assertEquals(OperationType.create.getValue(), recordInfoArgumentCaptor.getValue().getRecordInfo().getOp());
+ Assert.assertEquals(1, dataMapArgumentCaptor.getValue().size());
+ }
+
+ @Test
+ public void cacheDataRecord_update_record() throws URISyntaxException {
+ ArgumentCaptor<RecordChangeInfo> recordInfoArgumentCaptor = ArgumentCaptor.forClass(RecordChangeInfo.class);
+ ArgumentCaptor<Map<String, Object>> dataMapArgumentCaptor = ArgumentCaptor.forClass(Map.class);
+ String recordId = "anyId";
+ String kind = "anyKind";
+ Map<String, Object> dataMap = new HashMap<>();
+ dataMap.put("p1", "v1");
+ Map<String, Object> previousDataMap = new HashMap<>();
+ previousDataMap.put("p1", "v10");
+ previousDataMap.put("p2", "v2");
+
+ SearchRecord searchRecord = new SearchRecord();
+ searchRecord.setKind(kind);
+ searchRecord.setId(recordId);
+ searchRecord.setData(previousDataMap);
+ SearchResponse searchResponse = new SearchResponse();
+ searchResponse.setResults(Arrays.asList(searchRecord));
+ searchResponse.setTotalCount(1);
+ when(this.searchService.query(any())).thenReturn(searchResponse);
+
+ this.sut.cacheDataRecord(recordId, kind, dataMap);
+
+ verify(this.recordChangeInfoCache, times(1)).put(any(), recordInfoArgumentCaptor.capture());
+ verify(this.relatedObjectCache, times(2)).put(any(), dataMapArgumentCaptor.capture());
+
+ RecordChangeInfo changedInfo = recordInfoArgumentCaptor.getValue();
+ Assert.assertEquals(OperationType.update.getValue(), changedInfo.getRecordInfo().getOp());
+ Assert.assertEquals(2, changedInfo.getUpdatedProperties().size());
+ Assert.assertTrue(changedInfo.getUpdatedProperties().contains("p1"));
+ Assert.assertTrue(changedInfo.getUpdatedProperties().contains("p2"));
+ Assert.assertEquals(1, dataMapArgumentCaptor.getValue().size());
+ Assert.assertEquals("v1", dataMapArgumentCaptor.getValue().get("p1"));
+ }
+
+ @Test
+ public void cacheDataRecord_update_record_merge_previous_UpdateChangedInfo() throws URISyntaxException {
+ ArgumentCaptor<RecordChangeInfo> recordInfoArgumentCaptor = ArgumentCaptor.forClass(RecordChangeInfo.class);
+ ArgumentCaptor<Map<String, Object>> dataMapArgumentCaptor = ArgumentCaptor.forClass(Map.class);
+ String recordId = "anyId";
+ String kind = "anyKind";
+ Map<String, Object> dataMap = new HashMap<>();
+ dataMap.put("p1", "v1");
+ Map<String, Object> previousDataMap = new HashMap<>();
+ previousDataMap.put("p1", "v1");
+ previousDataMap.put("p2", "v2");
+
+ RecordChangeInfo previousChangedInfo = new RecordChangeInfo();
+ RecordInfo recordInfo = new RecordInfo();
+ recordInfo.setId(recordId);
+ recordInfo.setKind(kind);
+ recordInfo.setOp(OperationType.update.getValue());
+ previousChangedInfo.setRecordInfo(recordInfo);
+ previousChangedInfo.setUpdatedProperties(Arrays.asList("p1"));
+
+ SearchRecord searchRecord = new SearchRecord();
+ searchRecord.setKind(kind);
+ searchRecord.setId(recordId);
+ searchRecord.setData(previousDataMap);
+ SearchResponse searchResponse = new SearchResponse();
+ searchResponse.setResults(Arrays.asList(searchRecord));
+ searchResponse.setTotalCount(1);
+
+ when(this.searchService.query(any())).thenReturn(searchResponse);
+ when(this.recordChangeInfoCache.get(any())).thenReturn(previousChangedInfo);
+
+ this.sut.cacheDataRecord(recordId, kind, dataMap);
+
+ verify(this.recordChangeInfoCache, times(1)).put(any(), recordInfoArgumentCaptor.capture());
+ verify(this.relatedObjectCache, times(2)).put(any(), dataMapArgumentCaptor.capture());
+
+ RecordChangeInfo changedInfo = recordInfoArgumentCaptor.getValue();
+ Assert.assertEquals(OperationType.update.getValue(), changedInfo.getRecordInfo().getOp());
+ Assert.assertEquals(2, changedInfo.getUpdatedProperties().size());
+ Assert.assertTrue(changedInfo.getUpdatedProperties().contains("p1"));
+ Assert.assertTrue(changedInfo.getUpdatedProperties().contains("p2"));
+ Assert.assertEquals(1, dataMapArgumentCaptor.getValue().size());
+ Assert.assertEquals("v1", dataMapArgumentCaptor.getValue().get("p1"));
+ }
+
+ @Test
+ public void cacheDataRecord_update_record_merge_previous_CreateChangedInfo() throws URISyntaxException {
+ ArgumentCaptor<RecordChangeInfo> recordInfoArgumentCaptor = ArgumentCaptor.forClass(RecordChangeInfo.class);
+ ArgumentCaptor<Map<String, Object>> dataMapArgumentCaptor = ArgumentCaptor.forClass(Map.class);
+ String recordId = "anyId";
+ String kind = "anyKind";
+ Map<String, Object> dataMap = new HashMap<>();
+ dataMap.put("p1", "v1");
+ Map<String, Object> previousDataMap = new HashMap<>();
+ previousDataMap.put("p1", "v1");
+ previousDataMap.put("p2", "v2");
+
+ RecordChangeInfo previousChangedInfo = new RecordChangeInfo();
+ RecordInfo recordInfo = new RecordInfo();
+ recordInfo.setId(recordId);
+ recordInfo.setKind(kind);
+ recordInfo.setOp(OperationType.create.getValue());
+ previousChangedInfo.setRecordInfo(recordInfo);
+
+ SearchRecord searchRecord = new SearchRecord();
+ searchRecord.setKind(kind);
+ searchRecord.setId(recordId);
+ searchRecord.setData(previousDataMap);
+ SearchResponse searchResponse = new SearchResponse();
+ searchResponse.setResults(Arrays.asList(searchRecord));
+ searchResponse.setTotalCount(1);
+
+ when(this.searchService.query(any())).thenReturn(searchResponse);
+ when(this.recordChangeInfoCache.get(any())).thenReturn(previousChangedInfo);
+
+ this.sut.cacheDataRecord(recordId, kind, dataMap);
+
+ verify(this.recordChangeInfoCache, times(1)).put(any(), recordInfoArgumentCaptor.capture());
+ verify(this.relatedObjectCache, times(2)).put(any(), dataMapArgumentCaptor.capture());
+
+ RecordChangeInfo changedInfo = recordInfoArgumentCaptor.getValue();
+ Assert.assertEquals(OperationType.create.getValue(), changedInfo.getRecordInfo().getOp());
+ Assert.assertNull(changedInfo.getUpdatedProperties());
+ Assert.assertEquals(1, dataMapArgumentCaptor.getValue().size());
+ Assert.assertEquals("v1", dataMapArgumentCaptor.getValue().get("p1"));
+ }
+
+ @Test
+ public void updateAssociatedRecords_updateAssociatedParentRecords_for_created_childRecord() throws URISyntaxException {
+ updateAssociatedRecords_updateAssociatedParentRecords_for_created_delete(OperationType.create);
+ }
+
+ @Test
+ public void updateAssociatedRecords_updateAssociatedParentRecords_for_deleted_childRecord() throws URISyntaxException {
+ updateAssociatedRecords_updateAssociatedParentRecords_for_created_delete(OperationType.delete);
+ }
+
+ private void updateAssociatedRecords_updateAssociatedParentRecords_for_created_delete(OperationType operationType) throws URISyntaxException {
+ updateAssociatedRecords_updateAssociatedParentRecords_baseSetup();
+
+ // Test
+ RecordChangedMessages recordChangedMessages = new RecordChangedMessages();
+ recordChangedMessages.setAttributes(new HashMap<>());
+ Map<String, List<String>> upsertKindIds = new HashMap<>();
+ Map<String, List<String>> deleteKindIds = new HashMap<>();
+ if(operationType == OperationType.create)
+ upsertKindIds.put(childKind, Arrays.asList(childId));
+ else
+ deleteKindIds.put(childKind, Arrays.asList(childId));
+ this.sut.updateAssociatedRecords(recordChangedMessages, upsertKindIds, deleteKindIds);
+
+ // Verify
+ ArgumentCaptor<String> payloadArgumentCaptor = ArgumentCaptor.forClass(String.class);
+ verify(this.indexerQueueTaskBuilder,times(1)).createWorkerTask(payloadArgumentCaptor.capture(), any(), any());
+
+ RecordChangedMessages newMessages = gson.fromJson(payloadArgumentCaptor.getValue(), RecordChangedMessages.class);
+ Type type = new TypeToken<List<RecordInfo>>() {}.getType();
+ List<RecordInfo> infoList = gson.fromJson(newMessages.getData(), type);
+ Assert.assertEquals(childKind, newMessages.getAttributes().get(Constants.ANCESTRY_KINDS));
+ Assert.assertEquals(1, infoList.size());
+ Assert.assertEquals(parentKind, infoList.get(0).getKind());
+ Assert.assertEquals(parentId, infoList.get(0).getId());
+ }
+
+ @Test
+ public void updateAssociatedRecords_updateAssociatedParentRecords_for_updated_childRecord_with_extendedPropertyChanged() throws URISyntaxException {
+ updateAssociatedRecords_updateAssociatedParentRecords_baseSetup();
+
+ RecordChangeInfo recordChangeInfo = new RecordChangeInfo();
+ RecordInfo recordInfo = new RecordInfo();
+ recordInfo.setKind(childKind);
+ recordInfo.setId(childId);
+ recordInfo.setOp(OperationType.update.getValue());
+ recordChangeInfo.setRecordInfo(recordInfo);
+ recordChangeInfo.setUpdatedProperties(Arrays.asList("Curves[].Mnemonic", "Name"));
+ when(this.recordChangeInfoCache.get(any())).thenReturn(recordChangeInfo);
+
+ // Test
+ RecordChangedMessages recordChangedMessages = new RecordChangedMessages();
+ recordChangedMessages.setAttributes(new HashMap<>());
+ Map<String, List<String>> upsertKindIds = new HashMap<>();
+ Map<String, List<String>> deleteKindIds = new HashMap<>();
+ upsertKindIds.put(childKind, Arrays.asList(childId));
+ this.sut.updateAssociatedRecords(recordChangedMessages, upsertKindIds, deleteKindIds);
+
+ // Verify
+ ArgumentCaptor<String> payloadArgumentCaptor = ArgumentCaptor.forClass(String.class);
+ verify(this.indexerQueueTaskBuilder,times(1)).createWorkerTask(payloadArgumentCaptor.capture(), any(), any());
+
+ RecordChangedMessages newMessages = gson.fromJson(payloadArgumentCaptor.getValue(), RecordChangedMessages.class);
+ Type type = new TypeToken<List<RecordInfo>>() {}.getType();
+ List<RecordInfo> infoList = gson.fromJson(newMessages.getData(), type);
+ Assert.assertEquals(childKind, newMessages.getAttributes().get(Constants.ANCESTRY_KINDS));
+ Assert.assertEquals(1, infoList.size());
+ Assert.assertEquals(parentKind, infoList.get(0).getKind());
+ Assert.assertEquals(parentId, infoList.get(0).getId());
+ }
+
+ @Test
+ public void updateAssociatedRecords_updateAssociatedParentRecords_for_updated_childRecord_without_extendedPropertyChanged() throws URISyntaxException {
+ updateAssociatedRecords_updateAssociatedParentRecords_baseSetup();
+
+ RecordChangeInfo recordChangeInfo = new RecordChangeInfo();
+ RecordInfo recordInfo = new RecordInfo();
+ recordInfo.setKind(childKind);
+ recordInfo.setId(childId);
+ recordInfo.setOp(OperationType.update.getValue());
+ recordChangeInfo.setRecordInfo(recordInfo);
+ recordChangeInfo.setUpdatedProperties(Arrays.asList("Name"));
+ when(this.recordChangeInfoCache.get(any())).thenReturn(recordChangeInfo);
+
+ // Test
+ RecordChangedMessages recordChangedMessages = new RecordChangedMessages();
+ recordChangedMessages.setAttributes(new HashMap<>());
+ Map<String, List<String>> upsertKindIds = new HashMap<>();
+ Map<String, List<String>> deleteKindIds = new HashMap<>();
+ upsertKindIds.put(childKind, Arrays.asList(childId));
+ this.sut.updateAssociatedRecords(recordChangedMessages, upsertKindIds, deleteKindIds);
+
+ // Verify
+ verify(this.indexerQueueTaskBuilder,times(0)).createWorkerTask(any(), any(), any());
+ }
+
+ @Test
+ public void updateAssociatedRecords_updateAssociatedParentRecords_circularIndexing() throws URISyntaxException {
+ updateAssociatedRecords_updateAssociatedParentRecords_baseSetup();
+
+ // Test
+ RecordChangedMessages recordChangedMessages = new RecordChangedMessages();
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(Constants.ANCESTRY_KINDS, parentKind);
+ recordChangedMessages.setAttributes(attributes);
+ Map<String, List<String>> upsertKindIds = new HashMap<>();
+ Map<String, List<String>> deleteKindIds = new HashMap<>();
+ upsertKindIds.put(childKind, Arrays.asList(childId));
+ this.sut.updateAssociatedRecords(recordChangedMessages, upsertKindIds, deleteKindIds);
+
+ // Verify
+ verify(this.indexerQueueTaskBuilder,times(0)).createWorkerTask(any(), any(), any());
+ }
+
+ private void updateAssociatedRecords_updateAssociatedParentRecords_baseSetup() throws URISyntaxException {
+ childKind = "osdu:wks:work-product-component--WellLog:1.0.0";
+ childId = "anyChildId";
+ parentKind = "osdu:wks:master-data--Wellbore:1.0.0";
+ parentId = "anyParentId";
+
+ // Setup search response for searchService.query(...)
+ when(this.searchService.query(any())).thenAnswer(invocation -> {
+ SearchRequest searchRequest = invocation.getArgument(0);
+ SearchResponse searchResponse = new SearchResponse();
+ if (searchRequest.getKind().toString().equals(propertyConfigurationKind)) {
+ if (searchRequest.getQuery().contains("nested")) {
+ // Return of getParentChildRelatedObjectsSpecs(...)
+ Map<String, Object> dataMap = getDataMap("wellbore_configuration_record.json");
+ SearchRecord searchRecord = new SearchRecord();
+ searchRecord.setData(dataMap);
+ searchResponse.setResults(Arrays.asList(searchRecord));
+ } else {
+ // search ChildToParent.
+ // NO result
+ }
+ } else {
+ if(searchRequest.getKind().toString().equals(childKind)) {
+ // Return of searchUniqueParentIds(...)
+ SearchRecord searchRecord = new SearchRecord();
+ Map<String, Object> childDataMap = new HashMap<>();
+ childDataMap.put("WellboreID", parentId);
+ searchRecord.setKind(childKind);
+ searchRecord.setData(childDataMap);
+ searchResponse.setResults(Arrays.asList(searchRecord));
+ }
+ else if(searchRequest.getKind().toString().equals("osdu:wks:master-data--Wellbore:1.*")) {
+ // Return of searchKindIds(...)
+ SearchRecord searchRecord = new SearchRecord();
+ searchRecord.setKind(parentKind);
+ searchRecord.setId(parentId);
+ searchResponse.setResults(Arrays.asList(searchRecord));
+ }
+ }
+ return searchResponse;
+ });
+
+ // setup headers
+ DpsHeaders dpsHeaders = new DpsHeaders();
+ dpsHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
+ dpsHeaders.put(DpsHeaders.DATA_PARTITION_ID, "opendes");
+ dpsHeaders.put(DpsHeaders.CORRELATION_ID, "123");
+ when(this.requestInfo.getHeadersWithDwdAuthZ()).thenReturn(dpsHeaders);
+ }
+
+ @Test
+ public void updateAssociatedRecords_updateAssociatedChildrenRecords_for_created_parentRecord() throws URISyntaxException {
+ updateAssociatedRecords_updateAssociatedChildrenRecords_for_created_delete(OperationType.create);
+ }
+
+ @Test
+ public void updateAssociatedRecords_updateAssociatedChildrenRecords_for_deleted_parentRecord() throws URISyntaxException {
+ updateAssociatedRecords_updateAssociatedChildrenRecords_for_created_delete(OperationType.delete);
+ }
+
+ private void updateAssociatedRecords_updateAssociatedChildrenRecords_for_created_delete(OperationType op) throws URISyntaxException {
+ updateAssociatedRecords_updateAssociatedChildrenRecords_baseSetup();
+
+ // Test
+ RecordChangedMessages recordChangedMessages = new RecordChangedMessages();
+ recordChangedMessages.setAttributes(new HashMap<>());
+ Map<String, List<String>> upsertKindIds = new HashMap<>();
+ Map<String, List<String>> deleteKindIds = new HashMap<>();
+ if(op == OperationType.create)
+ upsertKindIds.put(parentKind, Arrays.asList(parentId));
+ else
+ deleteKindIds.put(parentKind, Arrays.asList(parentId));
+ this.sut.updateAssociatedRecords(recordChangedMessages, upsertKindIds, deleteKindIds);
+
+ // Verify
+ ArgumentCaptor<String> payloadArgumentCaptor = ArgumentCaptor.forClass(String.class);
+ verify(this.indexerQueueTaskBuilder,times(1)).createWorkerTask(payloadArgumentCaptor.capture(), any(), any());
+
+ RecordChangedMessages newMessages = gson.fromJson(payloadArgumentCaptor.getValue(), RecordChangedMessages.class);
+ Type type = new TypeToken<List<RecordInfo>>() {}.getType();
+ List<RecordInfo> infoList = gson.fromJson(newMessages.getData(), type);
+ Assert.assertEquals(parentKind, newMessages.getAttributes().get(Constants.ANCESTRY_KINDS));
+ Assert.assertEquals(1, infoList.size());
+ Assert.assertEquals(childKind, infoList.get(0).getKind());
+ Assert.assertEquals(childId, infoList.get(0).getId());
+ }
+
+ @Test
+ public void updateAssociatedRecords_updateAssociatedChildrenRecords_for_updated_parentRecord_with_extendedPropertyChanged() throws URISyntaxException {
+ updateAssociatedRecords_updateAssociatedChildrenRecords_baseSetup();
+
+ RecordChangeInfo recordChangeInfo = new RecordChangeInfo();
+ RecordInfo recordInfo = new RecordInfo();
+ recordInfo.setKind(parentKind);
+ recordInfo.setId(parentId);
+ recordInfo.setOp(OperationType.update.getValue());
+ recordChangeInfo.setRecordInfo(recordInfo);
+ recordChangeInfo.setUpdatedProperties(Arrays.asList("GeoPoliticalEntityName"));
+ when(this.recordChangeInfoCache.get(any())).thenReturn(recordChangeInfo);
+
+ // Test
+ RecordChangedMessages recordChangedMessages = new RecordChangedMessages();
+ recordChangedMessages.setAttributes(new HashMap<>());
+ Map<String, List<String>> upsertKindIds = new HashMap<>();
+ Map<String, List<String>> deleteKindIds = new HashMap<>();
+ upsertKindIds.put(parentKind, Arrays.asList(parentId));
+ this.sut.updateAssociatedRecords(recordChangedMessages, upsertKindIds, deleteKindIds);
+
+ // Verify
+ ArgumentCaptor<String> payloadArgumentCaptor = ArgumentCaptor.forClass(String.class);
+ verify(this.indexerQueueTaskBuilder,times(1)).createWorkerTask(payloadArgumentCaptor.capture(), any(), any());
+
+ RecordChangedMessages newMessages = gson.fromJson(payloadArgumentCaptor.getValue(), RecordChangedMessages.class);
+ Type type = new TypeToken<List<RecordInfo>>() {}.getType();
+ List<RecordInfo> infoList = gson.fromJson(newMessages.getData(), type);
+ Assert.assertEquals(parentKind, newMessages.getAttributes().get(Constants.ANCESTRY_KINDS));
+ Assert.assertEquals(1, infoList.size());
+ Assert.assertEquals(childKind, infoList.get(0).getKind());
+ Assert.assertEquals(childId, infoList.get(0).getId());
+ }
+
+ @Test
+ public void updateAssociatedRecords_updateAssociatedChildrenRecords_for_updated_parentRecord_without_extendedPropertyChanged() throws URISyntaxException {
+ updateAssociatedRecords_updateAssociatedChildrenRecords_baseSetup();
+
+ RecordChangeInfo recordChangeInfo = new RecordChangeInfo();
+ RecordInfo recordInfo = new RecordInfo();
+ recordInfo.setKind(parentKind);
+ recordInfo.setId(parentId);
+ recordInfo.setOp(OperationType.update.getValue());
+ recordChangeInfo.setRecordInfo(recordInfo);
+ recordChangeInfo.setUpdatedProperties(Arrays.asList("abc"));
+ when(this.recordChangeInfoCache.get(any())).thenReturn(recordChangeInfo);
+
+ // Test
+ RecordChangedMessages recordChangedMessages = new RecordChangedMessages();
+ recordChangedMessages.setAttributes(new HashMap<>());
+ Map<String, List<String>> upsertKindIds = new HashMap<>();
+ Map<String, List<String>> deleteKindIds = new HashMap<>();
+ upsertKindIds.put(parentKind, Arrays.asList(parentId));
+ this.sut.updateAssociatedRecords(recordChangedMessages, upsertKindIds, deleteKindIds);
+
+ // Verify
+ verify(this.indexerQueueTaskBuilder,times(0)).createWorkerTask(any(), any(), any());
+ }
+
+ @Test
+ public void updateAssociatedRecords_updateAssociatedChildrenRecords_circularIndexing() throws URISyntaxException {
+ updateAssociatedRecords_updateAssociatedChildrenRecords_baseSetup();
+
+ // Test
+ RecordChangedMessages recordChangedMessages = new RecordChangedMessages();
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(Constants.ANCESTRY_KINDS, childKind);
+ recordChangedMessages.setAttributes(attributes);
+ Map<String, List<String>> upsertKindIds = new HashMap<>();
+ Map<String, List<String>> deleteKindIds = new HashMap<>();
+ upsertKindIds.put(parentKind, Arrays.asList(parentId));
+ this.sut.updateAssociatedRecords(recordChangedMessages, upsertKindIds, deleteKindIds);
+
+ // Verify
+ verify(this.indexerQueueTaskBuilder,times(0)).createWorkerTask(any(), any(), any());
+ }
+
+ private void updateAssociatedRecords_updateAssociatedChildrenRecords_baseSetup() throws URISyntaxException {
+ childKind = "osdu:wks:master-data--Well:1.0.0";
+ childId = "anyChildId";
+ parentKind = "osdu:wks:master-data--GeoPoliticalEntity:1.0.0";
+ parentId = "anyParentId";
+
+ // Setup search response for searchService.query(...)
+ when(this.searchService.query(any())).thenAnswer(invocation -> {
+ SearchRequest searchRequest = invocation.getArgument(0);
+ SearchResponse searchResponse = new SearchResponse();
+ if (searchRequest.getKind().toString().equals(propertyConfigurationKind)) {
+ if (!searchRequest.getQuery().contains("nested")) {
+ // Return of getParentChildRelatedObjectsSpecs(...)
+ Map<String, Object> dataMap = getDataMap("well_configuration_record.json");
+ SearchRecord searchRecord = new SearchRecord();
+ searchRecord.setData(dataMap);
+ searchResponse.setResults(Arrays.asList(searchRecord));
+ } else {
+ // Search ParentToChildren
+ // No result
+ }
+ } else {
+ String kind = "*:*:*:*,-"+parentKind;
+ if(searchRequest.getKind().toString().equals(kind)) {
+ // Return of searchUniqueParentIds(...)
+ SearchRecord searchRecord = new SearchRecord();
+ Map<String, Object> childDataMap = new HashMap<>();
+ childDataMap.put("AssociatedIdentities", Arrays.asList(parentId));
+ searchRecord.setKind(childKind);
+ searchRecord.setId(childId);
+ searchRecord.setData(childDataMap);
+ searchResponse.setResults(Arrays.asList(searchRecord));
+ }
+ else {
+ // This branch is a setup for test case:
+ // updateAssociatedRecords_updateAssociatedChildrenRecords_circularIndexing
+ kind = "*:*:*:*,-"+ childKind + ",-" + parentKind;
+ if(!searchRequest.getKind().toString().equals(kind)) {
+ throw new Exception("Unexpected search");
+ }
+ }
+ }
+ return searchResponse;
+ });
+
+ // setup headers
+ DpsHeaders dpsHeaders = new DpsHeaders();
+ dpsHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
+ dpsHeaders.put(DpsHeaders.DATA_PARTITION_ID, "opendes");
+ dpsHeaders.put(DpsHeaders.CORRELATION_ID, "123");
+ when(this.requestInfo.getHeadersWithDwdAuthZ()).thenReturn(dpsHeaders);
+ }
+
+ private Schema getSchema(String file) {
+ String jsonText = getJsonFromFile(file);
+ return gson.fromJson(jsonText, Schema.class);
+ }
+
+ private PropertyConfigurations getConfigurations(String file) throws JsonProcessingException {
+ Map<String, Object> dataMap = getDataMap(file);
+ ObjectMapper objectMapper = new ObjectMapper();
+ String data = objectMapper.writeValueAsString(dataMap);
+ PropertyConfigurations configurations = objectMapper.readValue(data, PropertyConfigurations.class);
+ return configurations;
+ }
+
+ private Map<String, Object> getDataMap(String file) {
+ String jsonText = getJsonFromFile(file);
+ Type type = new TypeToken<Map<String, Object>>() {}.getType();
+ return gson.fromJson(jsonText, type);
+ }
+
+ @SneakyThrows
+ private String getJsonFromFile(String file) {
+ InputStream inStream = this.getClass().getResourceAsStream("/indexproperty/" + file);
+ BufferedReader br = new BufferedReader(new InputStreamReader(inStream));
+ StringBuilder stringBuilder = new StringBuilder();
+ String sCurrentLine;
+ while ((sCurrentLine = br.readLine()) != null)
+ {
+ stringBuilder.append(sCurrentLine).append("\n");
+ }
+ return stringBuilder.toString();
+ }
+}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/SchemaProviderImplTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/SchemaProviderImplTest.java
index fe91e5ffbca0e444ee037753a66341fe31d56cca..68a35ad4494d33acadfc7c11bbf634ce78aab17d 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/SchemaProviderImplTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/SchemaProviderImplTest.java
@@ -14,41 +14,36 @@
package org.opengroup.osdu.indexer.service;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.inOrder;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.powermock.api.mockito.PowerMockito.when;
-
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
-import java.io.UnsupportedEncodingException;
-import java.net.URISyntaxException;
-import java.util.Map;
import org.apache.http.HttpStatus;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
-import org.mockito.InOrder;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.mockito.Mockito;
-import org.mockito.Spy;
+import org.mockito.*;
+import org.opengroup.osdu.core.common.http.FetchServiceHttpRequest;
import org.opengroup.osdu.core.common.http.IUrlFetchService;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.HttpResponse;
import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
import org.opengroup.osdu.indexer.logging.AuditLogger;
+import org.opengroup.osdu.indexer.model.SchemaInfoResponse;
import org.opengroup.osdu.indexer.schema.converter.SchemaToStorageFormatImpl;
import org.powermock.api.mockito.PowerMockito;
import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder;
import org.springframework.test.context.junit4.SpringRunner;
+import java.io.UnsupportedEncodingException;
+import java.net.URISyntaxException;
+import java.util.Map;
+
+import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.*;
+import static org.powermock.api.mockito.PowerMockito.when;
+
@RunWith(SpringRunner.class)
public class SchemaProviderImplTest {
@@ -167,4 +162,74 @@ public class SchemaProviderImplTest {
verify(schemaService, times(0)).getFromStorageService(any());
}
+ @Test
+ public void getSchemaInfos() throws URISyntaxException, UnsupportedEncodingException {
+ HttpResponse httpResponse = createSchemaInfoResponse();
+ PowerMockito.when(this.urlFetchService.sendRequest(any())).thenReturn(httpResponse);
+
+ SchemaInfoResponse schemaInfoResponse = sut.getSchemaInfos("osdu", "wks", "master-data--Wellbore", "1", null, null, true);
+ assertEquals(1, schemaInfoResponse.getCount());
+ assertEquals(1, schemaInfoResponse.getSchemaInfos().size());
+ assertEquals("osdu:wks:master-data--Wellbore:1.3.0", schemaInfoResponse.getSchemaInfos().get(0).getSchemaIdentity().getId());
+ }
+
+ @Test
+ public void getSchemaInfos_latestSchemaInfo_url() throws URISyntaxException, UnsupportedEncodingException {
+ String schemaHost = "http://localhost/api/schema-service/v1/schema";
+ ArgumentCaptor<FetchServiceHttpRequest> argumentCaptor = ArgumentCaptor.forClass(FetchServiceHttpRequest.class);
+ HttpResponse httpResponse = createSchemaInfoResponse();
+ PowerMockito.when(this.configurationProperties.getSchemaHost()).thenReturn(schemaHost);
+ PowerMockito.when(this.urlFetchService.sendRequest(any())).thenReturn(httpResponse);
+
+ sut.getSchemaInfos("osdu", "wks", "master-data--Wellbore", "1", null, null, true);
+ verify(this.urlFetchService).sendRequest(argumentCaptor.capture());
+ FetchServiceHttpRequest request = argumentCaptor.getValue();
+ String url = request.getUrl();
+ String expectedUrl = "http://localhost/api/schema-service/v1/schema?authority=osdu&source=wks&entityType=master-data--Wellbore&schemaVersionMajor=1&latestVersion=true&limit=1000";
+ assertEquals(expectedUrl, url);
+ }
+
+ @Test
+ public void getSchemaInfos_allSchemaInfo_url() throws URISyntaxException, UnsupportedEncodingException {
+ String schemaHost = "http://localhost/api/schema-service/v1/schema";
+ ArgumentCaptor<FetchServiceHttpRequest> argumentCaptor = ArgumentCaptor.forClass(FetchServiceHttpRequest.class);
+ HttpResponse httpResponse = createSchemaInfoResponse();
+ PowerMockito.when(this.configurationProperties.getSchemaHost()).thenReturn(schemaHost);
+ PowerMockito.when(this.urlFetchService.sendRequest(any())).thenReturn(httpResponse);
+
+ sut.getSchemaInfos("osdu", "wks", "master-data--Wellbore", "1", "2", null, false);
+ verify(this.urlFetchService).sendRequest(argumentCaptor.capture());
+ FetchServiceHttpRequest request = argumentCaptor.getValue();
+ String url = request.getUrl();
+ String expectedUrl = "http://localhost/api/schema-service/v1/schema?authority=osdu&source=wks&entityType=master-data--Wellbore&schemaVersionMajor=1&schemaVersionMinor=2&latestVersion=false&limit=1000";
+ assertEquals(expectedUrl, url);
+ }
+
+ private HttpResponse createSchemaInfoResponse() {
+ String schemaInfos = "{\n" +
+ " \"schemaInfos\": [{\n" +
+ " \"schemaIdentity\": {\n" +
+ " \"authority\": \"osdu\",\n" +
+ " \"source\": \"wks\",\n" +
+ " \"entityType\": \"master-data--Wellbore\",\n" +
+ " \"schemaVersionMajor\": 1,\n" +
+ " \"schemaVersionMinor\": 3,\n" +
+ " \"schemaVersionPatch\": 0,\n" +
+ " \"id\": \"osdu:wks:master-data--Wellbore:1.3.0\"\n" +
+ " },\n" +
+ " \"createdBy\": \"ServiceAdminUser\",\n" +
+ " \"dateCreated\": \"2023-03-27T12:49:13.822+00:00\",\n" +
+ " \"status\": \"PUBLISHED\",\n" +
+ " \"scope\": \"SHARED\"\n" +
+ " }\n" +
+ " ],\n" +
+ " \"offset\": 0,\n" +
+ " \"count\": 1,\n" +
+ " \"totalCount\": 1\n" +
+ "}";
+ HttpResponse httpResponse = new HttpResponse();
+ httpResponse.setBody(schemaInfos);
+ return httpResponse;
+ }
+
}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/SearchServiceImplTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/SearchServiceImplTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..2018809bfec26c654bc5040669c85c041528d248
--- /dev/null
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/SearchServiceImplTest.java
@@ -0,0 +1,146 @@
+package org.opengroup.osdu.indexer.service;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.http.IUrlFetchService;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.HttpResponse;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.opengroup.osdu.indexer.model.SearchRequest;
+import org.opengroup.osdu.indexer.model.SearchResponse;
+import org.powermock.modules.junit4.PowerMockRunner;
+import static org.mockito.ArgumentMatchers.any;
+
+import java.net.URISyntaxException;
+
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@RunWith(PowerMockRunner.class)
+public class SearchServiceImplTest {
+ @InjectMocks
+ private SearchServiceImpl sut;
+
+ @Mock
+ private IUrlFetchService urlFetchService;
+ @Mock
+ private IRequestInfo requestInfo;
+ @Mock
+ private IndexerConfigurationProperties configurationProperties;
+ @Mock
+ private JaxRsDpsLog jaxRsDpsLog;
+
+ private String searchHost = "http://localhost";
+
+ @Test
+ public void query_without_searchHostSetting() throws URISyntaxException {
+ when(this.configurationProperties.getSearchHost()).thenReturn(null);
+ SearchResponse response = sut.query(new SearchRequest());
+ Assert.assertNotNull(response);
+ Assert.assertNull(response.getResults());
+ }
+
+ @Test
+ public void query_with_responseCode_OK() throws URISyntaxException {
+ String bodyJson = "{\n" +
+ " \"results\": [\n" +
+ " {\n" +
+ " \"data\": {\n" +
+ " \"FacilityName\": \"A123\"\n" +
+ " },\n" +
+ " \"kind\": \"osdu:wks:master-data--Wellbore:1.0.0\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"data\": {\n" +
+ " \"FacilityName\": \"B123\"\n" +
+ " },\n" +
+ " \"kind\": \"osdu:wks:master-data--Wellbore:1.0.0\"\n" +
+ " }\n" +
+ " ],\n" +
+ " \"aggregations\": null,\n" +
+ " \"totalCount\": 10000\n" +
+ "}";
+ HttpResponse response = new HttpResponse();
+ response.setResponseCode(200);
+ response.setBody(bodyJson);
+ when(this.configurationProperties.getSearchHost()).thenReturn(searchHost);
+ when(this.urlFetchService.sendRequest(any())).thenReturn(response);
+ SearchResponse searchResponse = sut.query(new SearchRequest());
+ Assert.assertNotNull(searchResponse);
+ Assert.assertEquals(10000,searchResponse.getTotalCount());
+ Assert.assertEquals(2,searchResponse.getResults().size());
+ }
+
+ @Test
+ public void query_with_cursor_with_responseCode_OK() throws URISyntaxException {
+ String bodyJson = "{\n" +
+ " \"cursor\": \"509E144E7F9B81F8148327D6CB73BB6F\",\n" +
+ " \"results\": [\n" +
+ " {\n" +
+ " \"kind\": \"osdu:wks:master-data--Wellbore:1.0.0\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"kind\": \"osdu:wks:master-data--Wellbore:1.0.1\"\n" +
+ " }\n" +
+ " ],\n" +
+ " \"totalCount\": 1000\n" +
+ "}";
+ HttpResponse response = new HttpResponse();
+ response.setResponseCode(200);
+ response.setBody(bodyJson);
+ when(this.configurationProperties.getSearchHost()).thenReturn(searchHost);
+ when(this.urlFetchService.sendRequest(any())).thenReturn(response);
+ SearchResponse searchResponse = sut.query(new SearchRequest());
+ Assert.assertNotNull(searchResponse);
+ Assert.assertNotNull(searchResponse.getCursor());
+ Assert.assertEquals(1000,searchResponse.getTotalCount());
+ Assert.assertEquals(2,searchResponse.getResults().size());
+ }
+
+ @Test
+ public void query_with_responseCode_OK_EmptyResult() throws URISyntaxException {
+ String bodyJson = "{\n" +
+ " \"results\": [],\n" +
+ " \"aggregations\": [],\n" +
+ " \"totalCount\": 0\n" +
+ "}";
+ HttpResponse response = new HttpResponse();
+ response.setResponseCode(200);
+ response.setBody(bodyJson);
+ when(this.configurationProperties.getSearchHost()).thenReturn(searchHost);
+ when(this.urlFetchService.sendRequest(any())).thenReturn(response);
+ SearchResponse searchResponse = sut.query(new SearchRequest());
+ Assert.assertNotNull(searchResponse);
+ Assert.assertEquals(0,searchResponse.getTotalCount());
+ Assert.assertEquals(0,searchResponse.getResults().size());
+ }
+
+ @Test
+ public void query_with_responseCode_BadRequest() throws URISyntaxException {
+ String bodyJson = "{\n" +
+ " \"code\": 400,\n" +
+ " \"reason\": \"Bad Request\",\n" +
+ " \"message\": \"Invalid parameters were given on search request\"\n" +
+ "}";
+ HttpResponse response = new HttpResponse();
+ response.setResponseCode(200);
+ response.setBody(bodyJson);
+ when(this.configurationProperties.getSearchHost()).thenReturn(searchHost);
+ when(this.urlFetchService.sendRequest(any())).thenReturn(response);
+ SearchResponse searchResponse = sut.query(new SearchRequest());
+ Assert.assertNotNull(searchResponse);
+ Assert.assertNull(searchResponse.getResults());
+ }
+
+ @Test
+ public void query_with_null_response() throws URISyntaxException {
+ when(this.configurationProperties.getSearchHost()).thenReturn(searchHost);
+ when(this.urlFetchService.sendRequest(any())).thenReturn(null);
+ SearchResponse searchResponse = sut.query(new SearchRequest());
+ Assert.assertNotNull(searchResponse);
+ Assert.assertNull(searchResponse.getResults());
+ }
+}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapperTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapperTest.java
index a8ede574da14ec07ad905907c98d5854e132a6e0..f11d6cb80a2b258d742bf13e26d2e2ee62dabfa8 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapperTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapperTest.java
@@ -21,6 +21,7 @@ import org.opengroup.osdu.indexer.service.mock.PartitionFactoryMock;
import org.opengroup.osdu.indexer.service.mock.PartitionProviderMock;
import org.opengroup.osdu.indexer.service.mock.ServiceAccountJwtClientMock;
import org.opengroup.osdu.indexer.service.mock.VirtualPropertiesSchemaCacheMock;
+import org.opengroup.osdu.indexer.util.BooleanFeatureFlagClient;
import org.opengroup.osdu.indexer.util.geo.decimator.*;
import org.opengroup.osdu.indexer.util.parser.BooleanParser;
import org.opengroup.osdu.indexer.util.parser.DateTimeParser;
@@ -45,8 +46,8 @@ import static org.junit.Assert.*;
@RunWith(SpringRunner.class)
@SpringBootTest(classes = {StorageIndexerPayloadMapper.class, AttributeParsingServiceImpl.class, NumberParser.class,
BooleanParser.class, DateTimeParser.class, GeoShapeParser.class, DouglasPeuckerReducer.class, GeoShapeDecimator.class,
- GeometryDecimator.class, GeometryConversionService.class, DecimationSettingCache.class,
- GeoShapeDecimationSetting.class, DpsHeaders.class, JobStatus.class, SchemaConverterPropertiesConfig.class, JaxRsDpsLog.class,
+ GeometryDecimator.class, GeometryConversionService.class, FeatureFlagCache.class,
+ GeoShapeDecimationSetting.class, BooleanFeatureFlagClient.class,DpsHeaders.class, JobStatus.class, SchemaConverterPropertiesConfig.class, JaxRsDpsLog.class,
PartitionFactoryMock.class, PartitionProviderMock.class, ServiceAccountJwtClientMock.class, VirtualPropertiesSchemaCacheMock.class, })
public class StorageIndexerPayloadMapperTest {
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/geo/decimator/GeoShapeDecimationSettingTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/BooleanFeatureFlagClientTest.java
similarity index 73%
rename from indexer-core/src/test/java/org/opengroup/osdu/indexer/util/geo/decimator/GeoShapeDecimationSettingTest.java
rename to indexer-core/src/test/java/org/opengroup/osdu/indexer/util/BooleanFeatureFlagClientTest.java
index 82fc83db7ccd0a520eabb90ea3a0f414ca0338c2..10e73df399a9ba64b858c121df5e7ad365e2fbcd 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/geo/decimator/GeoShapeDecimationSettingTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/BooleanFeatureFlagClientTest.java
@@ -13,7 +13,7 @@
* limitations under the License.
*/
-package org.opengroup.osdu.indexer.util.geo.decimator;
+package org.opengroup.osdu.indexer.util;
import org.junit.Assert;
import org.junit.Before;
@@ -25,6 +25,7 @@ import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.partition.*;
import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient;
+import org.opengroup.osdu.indexer.util.geo.decimator.FeatureFlagCache;
import org.springframework.test.context.junit4.SpringRunner;
import java.util.HashMap;
@@ -34,14 +35,14 @@ import static org.mockito.ArgumentMatchers.anyString;
import static org.powermock.api.mockito.PowerMockito.when;
@RunWith(SpringRunner.class)
-public class GeoShapeDecimationSettingTest {
+public class BooleanFeatureFlagClientTest {
private static final String PROPERTY_NAME = "indexer-decimation-enabled";
@InjectMocks
- private GeoShapeDecimationSetting sut;
+ private BooleanFeatureFlagClient sut;
@Mock
- private DecimationSettingCache cache;
+ private FeatureFlagCache cache;
@Mock
private JaxRsDpsLog logger;
@@ -74,7 +75,12 @@ public class GeoShapeDecimationSettingTest {
property.setValue("true");
partitionInfo.getProperties().put(PROPERTY_NAME, property);
when(this.partitionProvider.get(anyString())).thenReturn(partitionInfo);
- boolean enabled = sut.isDecimationEnabled();
+
+ // Default value won't take any effect
+ boolean enabled = sut.isEnabled(PROPERTY_NAME, true);
+ Assert.assertTrue(enabled);
+
+ enabled = sut.isEnabled(PROPERTY_NAME, false);
Assert.assertTrue(enabled);
}
@@ -86,25 +92,36 @@ public class GeoShapeDecimationSettingTest {
property.setValue("false");
partitionInfo.getProperties().put(PROPERTY_NAME, property);
when(this.partitionProvider.get(anyString())).thenReturn(partitionInfo);
- boolean enabled = sut.isDecimationEnabled();
+
+ // Default value won't take any effect
+ boolean enabled = sut.isEnabled(PROPERTY_NAME, true);
+ Assert.assertFalse(enabled);
+
+ enabled = sut.isEnabled(PROPERTY_NAME, false);
Assert.assertFalse(enabled);
}
@Test
- public void isDecimationEnabled_return_true_when_property_does_not_exist() throws PartitionException {
+ public void isDecimationEnabled_return_default_value_when_property_does_not_exist() throws PartitionException {
// The feature flag is enabled by default
PartitionInfo partitionInfo = new PartitionInfo();
when(this.partitionProvider.get(anyString())).thenReturn(partitionInfo);
- boolean enabled = sut.isDecimationEnabled();
+ boolean enabled = sut.isEnabled(PROPERTY_NAME, true);;
Assert.assertTrue(enabled);
+
+ enabled = sut.isEnabled(PROPERTY_NAME, false);;
+ Assert.assertFalse(enabled);
}
@Test
- public void isDecimationEnabled_return_true_when_partitionProvider_throws_exception() throws PartitionException {
+ public void isDecimationEnabled_return_default_value_when_partitionProvider_throws_exception() throws PartitionException {
// The feature flag is enabled by default
when(this.partitionProvider.get(anyString())).thenThrow(PartitionException.class);
- boolean enabled = sut.isDecimationEnabled();
+ boolean enabled = sut.isEnabled(PROPERTY_NAME, true);;
Assert.assertTrue(enabled);
+
+ enabled = sut.isEnabled(PROPERTY_NAME, false);;
+ Assert.assertFalse(enabled);
}
}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/PropertyUtilTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/PropertyUtilTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..683953925ea222178228f6ed4b4b348148b9b476
--- /dev/null
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/PropertyUtilTest.java
@@ -0,0 +1,294 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.util;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import lombok.SneakyThrows;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.lang.reflect.Type;
+import java.util.*;
+
+@RunWith(SpringRunner.class)
+public class PropertyUtilTest {
+ private static final Gson gson = new Gson();
+
+ @Test
+ public void isPropertyPathMatched() {
+ Assert.assertTrue(PropertyUtil.isPropertyPathMatched("data.FacilityName", "data.FacilityName"));
+ Assert.assertTrue(PropertyUtil.isPropertyPathMatched("data.ProjectedBottomHoleLocation.Wgs84Coordinates", "data.ProjectedBottomHoleLocation"));
+
+ Assert.assertFalse(PropertyUtil.isPropertyPathMatched("data.FacilityName", "data.FacilityNameAliase"));
+ Assert.assertFalse(PropertyUtil.isPropertyPathMatched("data.ProjectedBottomHoleLocation.Wgs84Coordinates", "data.ProjectedBottomHole"));
+ Assert.assertFalse(PropertyUtil.isPropertyPathMatched("", "data.ProjectedBottomHole"));
+ Assert.assertFalse(PropertyUtil.isPropertyPathMatched(null, "data.ProjectedBottomHole"));
+ }
+
+ @Test
+ public void hasSameMajorVersion() {
+ Assert.assertTrue(PropertyUtil.hasSameMajorKind("osdu:wks:master-data--Well:1.0.0", "osdu:wks:master-data--Well:1.0.0"));
+ Assert.assertTrue(PropertyUtil.hasSameMajorKind("osdu:wks:master-data--Well:1.1.0", "osdu:wks:master-data--Well:1.0.0"));
+ Assert.assertTrue(PropertyUtil.hasSameMajorKind("osdu:wks:master-data--Well:1.0.2", "osdu:wks:master-data--Well:1.0.0"));
+ Assert.assertFalse(PropertyUtil.hasSameMajorKind("osdu:wks:master-data--Well:2.0.0", "osdu:wks:master-data--Well:1.0.0"));
+ Assert.assertFalse(PropertyUtil.hasSameMajorKind("osdu:wks:master-data--Well:2.0.0", null));
+ }
+
+ @Test
+ public void removeDataPrefix() {
+ Assert.assertEquals("FacilityName", PropertyUtil.removeDataPrefix("data.FacilityName"));
+ Assert.assertEquals("FacilityName", PropertyUtil.removeDataPrefix("FacilityName"));
+ Assert.assertEquals("ProjectedBottomHoleLocation", PropertyUtil.removeDataPrefix("data.ProjectedBottomHoleLocation"));
+ Assert.assertEquals("ProjectedBottomHoleLocation", PropertyUtil.removeDataPrefix("ProjectedBottomHoleLocation"));
+ Assert.assertEquals("", PropertyUtil.removeDataPrefix(""));
+ Assert.assertNull(PropertyUtil.removeDataPrefix(null));
+ }
+
+ @Test
+ public void removeIdPostfix() {
+ Assert.assertEquals("data-partition:entity:12345", PropertyUtil.removeIdPostfix("data-partition:entity:12345"));
+ Assert.assertEquals("data-partition:entity:12345", PropertyUtil.removeIdPostfix("data-partition:entity:12345:"));
+ }
+
+ @Test
+ public void combineObjectMap_listObject() {
+ Map<String, Object> leftObjectMap = new HashMap<>();
+ List<Object> leftList = new ArrayList<>(Arrays.asList("v1", "v3"));
+ leftObjectMap.put("key", leftList);
+ Map<String, Object> rightObjectMap = new HashMap<>();
+ List<Object> rightList = new ArrayList<>(Arrays.asList("v1", "v2", "v4"));
+ rightObjectMap.put("key", rightList);
+
+ Map<String, Object> combinedObjectMap = PropertyUtil.combineObjectMap(leftObjectMap, new HashMap<>());
+ Assert.assertEquals(1, combinedObjectMap.size());
+ Assert.assertEquals(2, ((List)combinedObjectMap.get("key")).size());
+
+ combinedObjectMap = PropertyUtil.combineObjectMap(new HashMap<>(), rightObjectMap);
+ Assert.assertEquals(1, combinedObjectMap.size());
+ Assert.assertEquals(3, ((List)combinedObjectMap.get("key")).size());
+
+ combinedObjectMap = PropertyUtil.combineObjectMap(leftObjectMap, rightObjectMap);
+ Assert.assertEquals(1, combinedObjectMap.size());
+ Assert.assertEquals(4, ((List)combinedObjectMap.get("key")).size());
+ }
+
+ @Test
+ public void combineObjectMap_mapObject() {
+ Map<String, Object> leftObjectMap = new HashMap<>();
+ Map<String, Object> leftInnerMap = new HashMap<>();
+ leftInnerMap.put("key2_1", "value1");
+ leftObjectMap.put("key", leftInnerMap);
+
+ Map<String, Object> rightObjectMap = new HashMap<>();
+ Map<String, Object> rightInnerMap = new HashMap<>();
+ rightInnerMap.put("key2_2", "value2");
+ rightObjectMap.put("key", rightInnerMap);
+
+ Map<String, Object> combinedObjectMap = PropertyUtil.combineObjectMap(leftObjectMap, new HashMap<>());
+ Assert.assertEquals(1, combinedObjectMap.size());
+ Assert.assertEquals(1, ((Map)combinedObjectMap.get("key")).size());
+
+ combinedObjectMap = PropertyUtil.combineObjectMap(new HashMap<>(), rightObjectMap);
+ Assert.assertEquals(1, combinedObjectMap.size());
+ Assert.assertEquals(1, ((Map)combinedObjectMap.get("key")).size());
+
+ combinedObjectMap = PropertyUtil.combineObjectMap(leftObjectMap, rightObjectMap);
+ Assert.assertEquals(1, combinedObjectMap.size());
+ Assert.assertEquals(2, ((Map)combinedObjectMap.get("key")).size());
+ }
+
+ @Test
+ public void combineObjectMap_sameStringValue() {
+ Map<String, Object> leftObjectMap = new HashMap<>();
+ leftObjectMap.put("key", "value");
+
+ Map<String, Object> rightObjectMap = new HashMap<>();
+ rightObjectMap.put("key", "value");
+
+ Map<String, Object> combinedObjectMap = PropertyUtil.combineObjectMap(leftObjectMap, new HashMap<>());
+ Assert.assertEquals(1, combinedObjectMap.size());
+ Assert.assertEquals("value", combinedObjectMap.get("key"));
+
+ combinedObjectMap = PropertyUtil.combineObjectMap(new HashMap<>(), rightObjectMap);
+ Assert.assertEquals(1, combinedObjectMap.size());
+ Assert.assertEquals("value", combinedObjectMap.get("key"));
+
+ combinedObjectMap = PropertyUtil.combineObjectMap(leftObjectMap, rightObjectMap);
+ Assert.assertEquals(1, combinedObjectMap.size());
+ Assert.assertEquals("value", combinedObjectMap.get("key"));
+
+ }
+
+ @Test
+ public void combineObjectMap_differentStringValue() {
+ Map<String, Object> leftObjectMap = new HashMap<>();
+ leftObjectMap.put("key", "value1");
+
+ Map<String, Object> rightObjectMap = new HashMap<>();
+ rightObjectMap.put("key", "value2");
+
+ Map<String, Object> combinedObjectMap = PropertyUtil.combineObjectMap(leftObjectMap, new HashMap<>());
+ Assert.assertEquals(1, combinedObjectMap.size());
+ Assert.assertEquals("value1", combinedObjectMap.get("key"));
+
+ combinedObjectMap = PropertyUtil.combineObjectMap(new HashMap<>(), rightObjectMap);
+ Assert.assertEquals(1, combinedObjectMap.size());
+ Assert.assertEquals("value2", combinedObjectMap.get("key"));
+
+ combinedObjectMap = PropertyUtil.combineObjectMap(leftObjectMap, rightObjectMap);
+ Assert.assertEquals(1, combinedObjectMap.size());
+ Assert.assertEquals(2, ((List)combinedObjectMap.get("key")).size());
+ }
+
+ @Test
+ public void combineObjectMap_ObjectList() {
+ Map<String, Object> leftObjectMap = new HashMap<>();
+ leftObjectMap.put("key", "value1");
+
+ Map<String, Object> rightObjectMap = new HashMap<>();
+ List<Object> rightList = new ArrayList<>(Arrays.asList("v1", "v2", "v4"));
+ rightObjectMap.put("key", rightList);
+
+ Map<String, Object> combinedObjectMap = PropertyUtil.combineObjectMap(leftObjectMap, rightObjectMap);
+ Assert.assertEquals(1, combinedObjectMap.size());
+ Assert.assertEquals(4, ((List)combinedObjectMap.get("key")).size());
+
+ leftObjectMap = new HashMap<>();
+ leftObjectMap.put("key", "value1");
+ rightObjectMap = new HashMap<>();
+ rightObjectMap.put("key", new ArrayList<>());
+ combinedObjectMap = PropertyUtil.combineObjectMap(leftObjectMap, rightObjectMap);
+ Assert.assertEquals(1, combinedObjectMap.size());
+ Assert.assertEquals("value1", combinedObjectMap.get("key"));
+ }
+
+ @Test
+ public void combineObjectMap_misMatchTypeValue() {
+ Map<String, Object> leftObjectMap = new HashMap<>();
+ leftObjectMap.put("key", "value1");
+
+ Map<String, Object> rightObjectMap = new HashMap<>();
+ rightObjectMap.put("key", new HashMap<>());
+
+ // The object values in rightObjectMap will be ignored in this case
+ Map<String, Object> combinedObjectMap = PropertyUtil.combineObjectMap(leftObjectMap, rightObjectMap);
+ Assert.assertEquals(1, combinedObjectMap.size());
+ Assert.assertTrue(combinedObjectMap.get("key") instanceof String);
+ Assert.assertEquals("value1", combinedObjectMap.get("key"));
+
+ combinedObjectMap = PropertyUtil.combineObjectMap(rightObjectMap, leftObjectMap);
+ Assert.assertEquals(1, combinedObjectMap.size());
+ Assert.assertTrue(combinedObjectMap.get("key") instanceof Map);
+ }
+
+ @Test
+ public void replacePropertyPaths() {
+ Map<String, Object> map = new HashMap<>();
+ map.put("aaa", "value1");
+ map.put("bbb.ccc", "value2");
+ map.put("bbb.ddd", "value3");
+
+ Map<String, Object> newMap;
+ newMap = PropertyUtil.replacePropertyPaths("111", "aaa", map);
+ Assert.assertEquals("value1", newMap.get("111"));
+ newMap = PropertyUtil.replacePropertyPaths("111", "data.aaa", map);
+ Assert.assertEquals("value1", newMap.get("111"));
+ newMap = PropertyUtil.replacePropertyPaths("data.111", "aaa", map);
+ Assert.assertEquals("value1", newMap.get("111"));
+
+ newMap = PropertyUtil.replacePropertyPaths("222", "bbb", map);
+ Assert.assertEquals("value2", newMap.get("222.ccc"));
+ Assert.assertEquals("value3", newMap.get("222.ddd"));
+ newMap = PropertyUtil.replacePropertyPaths("222", "data.bbb", map);
+ Assert.assertEquals("value2", newMap.get("222.ccc"));
+ Assert.assertEquals("value3", newMap.get("222.ddd"));
+ newMap = PropertyUtil.replacePropertyPaths("data.222", "bbb", map);
+ Assert.assertEquals("value2", newMap.get("222.ccc"));
+ Assert.assertEquals("value3", newMap.get("222.ddd"));
+ }
+
+ @Test
+ public void replacePropertyPaths_emptyValues() {
+ Map<String, Object> map = new HashMap<>();
+ map.put("abc", "value1");
+ Assert.assertTrue(PropertyUtil.replacePropertyPaths(null, "abc", map).isEmpty());
+ Assert.assertTrue(PropertyUtil.replacePropertyPaths("abc", null, map).isEmpty());
+ Assert.assertTrue(PropertyUtil.replacePropertyPaths("a12", "abc", new HashMap<>()).isEmpty());
+ Assert.assertTrue(PropertyUtil.replacePropertyPaths("a12", "abc", null).isEmpty());
+ }
+
+ @Test
+ public void isConcreteKind() {
+ Assert.assertTrue(PropertyUtil.isConcreteKind("osdu:wks:master-data--Well:1.0.0"));
+ Assert.assertFalse(PropertyUtil.isConcreteKind("osdu:wks:master-data--Well:1.0."));
+ Assert.assertFalse(PropertyUtil.isConcreteKind("osdu:master-data--Well:1.0.0"));
+ Assert.assertFalse(PropertyUtil.isConcreteKind(null));
+ Assert.assertFalse(PropertyUtil.isConcreteKind(""));
+ }
+
+ @Test
+ public void getKindWithMajor() {
+ Assert.assertEquals("osdu:wks:master-data--Well:1.", PropertyUtil.getKindWithMajor("osdu:wks:master-data--Well:1.0.0"));
+ Assert.assertEquals("osdu:wks:master-data--Well:1.", PropertyUtil.getKindWithMajor("osdu:wks:master-data--Well:1."));
+ Assert.assertEquals("osdu:wks:master-data--Well:1.", PropertyUtil.getKindWithMajor("osdu:wks:master-data--Well:1"));
+
+ Assert.assertEquals("", PropertyUtil.getKindWithMajor("osdu:wks:master-data--Well"));
+ Assert.assertEquals("", PropertyUtil.getKindWithMajor(""));
+ Assert.assertNull(null, PropertyUtil.getKindWithMajor(null));
+ }
+
+ @Test
+ public void getChangedProperties() {
+ Map<String, Object> dataMapLeft = getDataMap("well.json");
+ Map<String, Object> dataMapRight = getDataMap("well2.json");
+ List<String> changedProperties = PropertyUtil.getChangedProperties(dataMapLeft, dataMapRight);
+ Assert.assertEquals(3, changedProperties.size());
+ List<String> expectedChangedWellProperties = Arrays.asList("VirtualProperties.DefaultName", "VerticalMeasurements[].VerticalMeasurementID", "FacilityName");
+ changedProperties.forEach(p -> Assert.assertTrue(expectedChangedWellProperties.contains(p)));
+
+ dataMapLeft = getDataMap("wellLog.json");
+ dataMapRight = getDataMap("wellLog2.json");
+ changedProperties = PropertyUtil.getChangedProperties(dataMapLeft, dataMapRight);
+ List<String> expectedChangedWellLogProperties = Arrays.asList("Curves[].CurveID");
+ changedProperties.forEach(p -> Assert.assertTrue(expectedChangedWellLogProperties.contains(p)));
+ }
+
+ private Map<String, Object> getDataMap(String file) {
+ String jsonText = getJsonFromFile(file);
+ Type type = new TypeToken<Map<String, Object>>() {}.getType();
+ return gson.fromJson(jsonText, type);
+ }
+
+ @SneakyThrows
+ private String getJsonFromFile(String file) {
+ InputStream inStream = this.getClass().getResourceAsStream("/indexproperty/" + file);
+ BufferedReader br = new BufferedReader(new InputStreamReader(inStream));
+ StringBuilder stringBuilder = new StringBuilder();
+ String sCurrentLine;
+ while ((sCurrentLine = br.readLine()) != null)
+ {
+ stringBuilder.append(sCurrentLine).append("\n");
+ }
+ return stringBuilder.toString();
+ }
+
+}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/VirtualPropertyUtilTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/VirtualPropertyUtilTest.java
deleted file mode 100644
index 63c33460fbfed619231f2b529e2596482966e23b..0000000000000000000000000000000000000000
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/VirtualPropertyUtilTest.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Copyright © Schlumberger
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0 *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.util;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.springframework.test.context.junit4.SpringRunner;
-
-@RunWith(SpringRunner.class)
-public class VirtualPropertyUtilTest {
-
- @Test
- public void isPropertyPathMatched() {
- Assert.assertTrue(VirtualPropertyUtil.isPropertyPathMatched("data.FacilityName", "data.FacilityName"));
- Assert.assertTrue(VirtualPropertyUtil.isPropertyPathMatched("data.ProjectedBottomHoleLocation.Wgs84Coordinates", "data.ProjectedBottomHoleLocation"));
-
- Assert.assertFalse(VirtualPropertyUtil.isPropertyPathMatched("data.FacilityName", "data.FacilityNameAliase"));
- Assert.assertFalse(VirtualPropertyUtil.isPropertyPathMatched("data.ProjectedBottomHoleLocation.Wgs84Coordinates", "data.ProjectedBottomHole"));
- Assert.assertFalse(VirtualPropertyUtil.isPropertyPathMatched("", "data.ProjectedBottomHole"));
- Assert.assertFalse(VirtualPropertyUtil.isPropertyPathMatched(null, "data.ProjectedBottomHole"));
- }
-
- @Test
- public void removeDataPrefix() {
- Assert.assertEquals("FacilityName", VirtualPropertyUtil.removeDataPrefix("data.FacilityName"));
- Assert.assertEquals("FacilityName", VirtualPropertyUtil.removeDataPrefix("FacilityName"));
- Assert.assertEquals("ProjectedBottomHoleLocation", VirtualPropertyUtil.removeDataPrefix("data.ProjectedBottomHoleLocation"));
- Assert.assertEquals("ProjectedBottomHoleLocation", VirtualPropertyUtil.removeDataPrefix("ProjectedBottomHoleLocation"));
- Assert.assertEquals("", VirtualPropertyUtil.removeDataPrefix(""));
- Assert.assertNull(VirtualPropertyUtil.removeDataPrefix(null));
- }
-}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/geo/decimator/DecimationSettingCacheTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/geo/decimator/DecimationSettingCacheTest.java
index e850126fe2880d457bd3fcd15ede9e0953b1d779..c32159b5e1186cefa6f72e4bc478d4095aa97bc9 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/geo/decimator/DecimationSettingCacheTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/geo/decimator/DecimationSettingCacheTest.java
@@ -25,11 +25,11 @@ import org.springframework.test.context.junit4.SpringRunner;
public class DecimationSettingCacheTest {
private static final String VALID_KEY = "Tenant1-indexer-decimation-enabled";
private static final String INVALID_KEY = "Tenant2-indexer-decimation-enabled";
- DecimationSettingCache cache;
+ FeatureFlagCache cache;
@Before
public void setup() {
- cache = new DecimationSettingCache();
+ cache = new FeatureFlagCache();
cache.put(VALID_KEY, true);
}
diff --git a/indexer-core/src/test/resources/indexproperty/geo_political_entity_storage_schema.json b/indexer-core/src/test/resources/indexproperty/geo_political_entity_storage_schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..9021c6eabe177343a0a220bc8e4d67466adaedd6
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/geo_political_entity_storage_schema.json
@@ -0,0 +1,186 @@
+{
+ "kind": "osdu:wks:master-data--GeoPoliticalEntity:1.0.0",
+ "schema": [{
+ "path": "ResourceHomeRegionID",
+ "kind": "string"
+ }, {
+ "path": "ResourceHostRegionIDs",
+ "kind": "[]string"
+ }, {
+ "path": "ResourceLifecycleStatus",
+ "kind": "string"
+ }, {
+ "path": "ResourceSecurityClassification",
+ "kind": "string"
+ }, {
+ "path": "ResourceCurationStatus",
+ "kind": "string"
+ }, {
+ "path": "ExistenceKind",
+ "kind": "string"
+ }, {
+ "path": "TechnicalAssuranceID",
+ "kind": "string"
+ }, {
+ "path": "Source",
+ "kind": "string"
+ }, {
+ "path": "NameAliases",
+ "kind": "nested",
+ "properties": [{
+ "path": "AliasName",
+ "kind": "string"
+ }, {
+ "path": "TerminationDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "AliasNameTypeID",
+ "kind": "string"
+ }, {
+ "path": "EffectiveDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "DefinitionOrganisationID",
+ "kind": "string"
+ }
+ ]
+ }, {
+ "path": "SpatialLocation.SpatialParameterTypeID",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.QuantitativeAccuracyBandID",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.CoordinateQualityCheckRemarks",
+ "kind": "[]string"
+ }, {
+ "path": "SpatialLocation.AppliedOperations",
+ "kind": "[]string"
+ }, {
+ "path": "SpatialLocation.QualitativeSpatialAccuracyTypeID",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.CoordinateQualityCheckPerformedBy",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.SpatialLocationCoordinatesDate",
+ "kind": "datetime"
+ }, {
+ "path": "SpatialLocation.CoordinateQualityCheckDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "SpatialLocation.Wgs84Coordinates",
+ "kind": "core:dl:geoshape:1.0.0"
+ }, {
+ "path": "SpatialLocation.SpatialGeometryTypeID",
+ "kind": "string"
+ }, {
+ "path": "VersionCreationReason",
+ "kind": "string"
+ }, {
+ "path": "TechnicalAssuranceTypeID",
+ "kind": "string"
+ }, {
+ "path": "GeoContexts",
+ "kind": "nested",
+ "properties": [{
+ "path": "GeoPoliticalEntityID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "BasinID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "FieldID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "PlayID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "ProspectID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }
+ ]
+ }, {
+ "path": "ParentGeoPoliticalEntityID",
+ "kind": "string"
+ }, {
+ "path": "GeoPoliticalEntityTypeID",
+ "kind": "string"
+ }, {
+ "path": "TerminationDate",
+ "kind": "datetime"
+ }, {
+ "path": "DisputedIndicator",
+ "kind": "bool"
+ }, {
+ "path": "DaylightSavingTimeStartDate",
+ "kind": "datetime"
+ }, {
+ "path": "GeoPoliticalEntityName",
+ "kind": "string"
+ }, {
+ "path": "GeoPoliticalEntityID",
+ "kind": "string"
+ }, {
+ "path": "DaylightSavingTimeEndDate",
+ "kind": "datetime"
+ }, {
+ "path": "GeoPoliticalEntityNameAliases",
+ "kind": "[]object"
+ }, {
+ "path": "EffectiveDate",
+ "kind": "datetime"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.SpatialParameterTypeID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.QuantitativeAccuracyBandID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.CoordinateQualityCheckRemarks",
+ "kind": "[]string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.AppliedOperations",
+ "kind": "[]string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.QualitativeSpatialAccuracyTypeID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.CoordinateQualityCheckPerformedBy",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.SpatialLocationCoordinatesDate",
+ "kind": "datetime"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.CoordinateQualityCheckDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.Wgs84Coordinates",
+ "kind": "core:dl:geoshape:1.0.0"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.SpatialGeometryTypeID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultName",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.IsDecimated",
+ "kind": "boolean"
+ }
+ ]
+}
diff --git a/indexer-core/src/test/resources/indexproperty/organisation_data1.json b/indexer-core/src/test/resources/indexproperty/organisation_data1.json
new file mode 100644
index 0000000000000000000000000000000000000000..dad4b5c3da7d61ededf41066b2c0bc5dfe3c523c
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/organisation_data1.json
@@ -0,0 +1,8 @@
+{
+ "InternalOrganisationIndicator": true,
+ "VirtualProperties.DefaultName": "BigOil SeismicInterpretation Department (original)",
+ "VersionCreationReason": "Index testing",
+ "OrganisationName": "BigOil SeismicInterpretation Department (original)",
+ "TechnicalAssuranceTypeID": "opendes:reference-data--TechnicalAssuranceType:Certified:",
+ "OrganisationTypeID": "opendes:reference-data--OrganisationType:OrganizationUnit:"
+}
diff --git a/indexer-core/src/test/resources/indexproperty/organisation_data2.json b/indexer-core/src/test/resources/indexproperty/organisation_data2.json
new file mode 100644
index 0000000000000000000000000000000000000000..e486b811c3f5c12390b127321bd42e395919751c
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/organisation_data2.json
@@ -0,0 +1,8 @@
+{
+ "InternalOrganisationIndicator": true,
+ "VirtualProperties.DefaultName": "BigOil SeismicProcessing Department (original)",
+ "VersionCreationReason": "Index testing",
+ "OrganisationName": "BigOil SeismicProcessing Department (original)",
+ "TechnicalAssuranceTypeID": "opendes:reference-data--TechnicalAssuranceType:Certified:",
+ "OrganisationTypeID": "opendes:reference-data--OrganisationType:OrganizationUnit:"
+}
diff --git a/indexer-core/src/test/resources/indexproperty/organisation_storage_schema.json b/indexer-core/src/test/resources/indexproperty/organisation_storage_schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..ba6563b82bdfc59e8511c0298e430f1214ebd626
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/organisation_storage_schema.json
@@ -0,0 +1,186 @@
+{
+ "kind": "osdu:wks:master-data--Organisation:1.1.0",
+ "schema": [{
+ "path": "ResourceHomeRegionID",
+ "kind": "string"
+ }, {
+ "path": "ResourceHostRegionIDs",
+ "kind": "[]string"
+ }, {
+ "path": "ResourceLifecycleStatus",
+ "kind": "string"
+ }, {
+ "path": "ResourceSecurityClassification",
+ "kind": "string"
+ }, {
+ "path": "ResourceCurationStatus",
+ "kind": "string"
+ }, {
+ "path": "ExistenceKind",
+ "kind": "string"
+ }, {
+ "path": "TechnicalAssuranceID",
+ "kind": "string"
+ }, {
+ "path": "Source",
+ "kind": "string"
+ }, {
+ "path": "NameAliases",
+ "kind": "nested",
+ "properties": [{
+ "path": "AliasName",
+ "kind": "string"
+ }, {
+ "path": "TerminationDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "AliasNameTypeID",
+ "kind": "string"
+ }, {
+ "path": "EffectiveDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "DefinitionOrganisationID",
+ "kind": "string"
+ }
+ ]
+ }, {
+ "path": "SpatialLocation.SpatialParameterTypeID",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.QuantitativeAccuracyBandID",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.CoordinateQualityCheckRemarks",
+ "kind": "[]string"
+ }, {
+ "path": "SpatialLocation.AppliedOperations",
+ "kind": "[]string"
+ }, {
+ "path": "SpatialLocation.QualitativeSpatialAccuracyTypeID",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.CoordinateQualityCheckPerformedBy",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.SpatialLocationCoordinatesDate",
+ "kind": "datetime"
+ }, {
+ "path": "SpatialLocation.CoordinateQualityCheckDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "SpatialLocation.Wgs84Coordinates",
+ "kind": "core:dl:geoshape:1.0.0"
+ }, {
+ "path": "SpatialLocation.SpatialGeometryTypeID",
+ "kind": "string"
+ }, {
+ "path": "VersionCreationReason",
+ "kind": "string"
+ }, {
+ "path": "TechnicalAssuranceTypeID",
+ "kind": "string"
+ }, {
+ "path": "GeoContexts",
+ "kind": "nested",
+ "properties": [{
+ "path": "GeoPoliticalEntityID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "BasinID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "FieldID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "PlayID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "ProspectID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }
+ ]
+ }, {
+ "path": "OrganisationPurposeDescription",
+ "kind": "string"
+ }, {
+ "path": "TerminationDate",
+ "kind": "datetime"
+ }, {
+ "path": "OrganisationTypeID",
+ "kind": "string"
+ }, {
+ "path": "OrganisationID",
+ "kind": "string"
+ }, {
+ "path": "InternalOrganisationIndicator",
+ "kind": "bool"
+ }, {
+ "path": "OrganisationName",
+ "kind": "string"
+ }, {
+ "path": "OrganisationNameAliases",
+ "kind": "[]object"
+ }, {
+ "path": "ParentOrganisationID",
+ "kind": "string"
+ }, {
+ "path": "OrganisationDescription",
+ "kind": "string"
+ }, {
+ "path": "EffectiveDate",
+ "kind": "datetime"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.SpatialParameterTypeID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.QuantitativeAccuracyBandID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.CoordinateQualityCheckRemarks",
+ "kind": "[]string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.AppliedOperations",
+ "kind": "[]string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.QualitativeSpatialAccuracyTypeID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.CoordinateQualityCheckPerformedBy",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.SpatialLocationCoordinatesDate",
+ "kind": "datetime"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.CoordinateQualityCheckDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.Wgs84Coordinates",
+ "kind": "core:dl:geoshape:1.0.0"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.SpatialGeometryTypeID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultName",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.IsDecimated",
+ "kind": "boolean"
+ }
+ ]
+}
diff --git a/indexer-core/src/test/resources/indexproperty/well.json b/indexer-core/src/test/resources/indexproperty/well.json
new file mode 100644
index 0000000000000000000000000000000000000000..918b86432c450ac5d85409a4f7c99aea9815dd9e
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/well.json
@@ -0,0 +1,136 @@
+{
+ "VirtualProperties.DefaultLocation.QuantitativeAccuracyBandID": null,
+ "ResourceLifecycleStatus": null,
+ "DefaultVerticalMeasurementID": "KB",
+ "WasBusinessInterestObligatory": null,
+ "SpatialLocation.Wgs84Coordinates": {
+ "type": "geometrycollection",
+ "geometries": [{
+ "type": "point",
+ "coordinates": [171.8745361, -41.2456122]
+ }
+ ]
+ },
+ "ResourceCurationStatus": null,
+ "TechnicalAssuranceID": "opendes:reference-data--TechnicalAssuranceType:Unsuitable:",
+ "VirtualProperties.DefaultLocation.SpatialGeometryTypeID": "opendes:reference-data--SpatialGeometryType:Point:",
+ "RoleID": null,
+ "FacilityName": "Kongahu-1",
+ "Source": null,
+ "FacilityID": null,
+ "OutcomeID": null,
+ "VirtualProperties.DefaultName": "Kongahu-1",
+ "VerticalMeasurements": [{
+ "WellboreTVDTrajectoryID": null,
+ "VerticalCRSID": null,
+ "VerticalReferenceID": null,
+ "VerticalMeasurementSourceID": null,
+ "VerticalMeasurementID": "KB",
+ "VerticalMeasurementPathID": "opendes:reference-data--VerticalMeasurementPath:ELEV:",
+ "VerticalMeasurement": 26.0,
+ "VerticalMeasurementTypeID": "opendes:reference-data--VerticalMeasurementType:KB:",
+ "VerticalMeasurementDescription": null,
+ "VerticalMeasurementUnitOfMeasureID": "opendes:reference-data--UnitOfMeasure:m:",
+ "VerticalReferenceEntityID": null,
+ "RigID": null
+ }, {
+ "WellboreTVDTrajectoryID": null,
+ "VerticalCRSID": null,
+ "VerticalReferenceID": null,
+ "VerticalMeasurementSourceID": null,
+ "VerticalMeasurementID": "Well Head Elevation",
+ "VerticalMeasurementPathID": "opendes:reference-data--VerticalMeasurementPath:ELEV:",
+ "VerticalMeasurement": 26.0,
+ "VerticalMeasurementTypeID": null,
+ "VerticalMeasurementDescription": null,
+ "VerticalMeasurementUnitOfMeasureID": "opendes:reference-data--UnitOfMeasure:m:",
+ "VerticalReferenceEntityID": null,
+ "RigID": null
+ }, {
+ "WellboreTVDTrajectoryID": null,
+ "VerticalCRSID": null,
+ "VerticalReferenceID": null,
+ "VerticalMeasurementSourceID": null,
+ "VerticalMeasurementID": "Water Depth",
+ "VerticalMeasurementPathID": "opendes:reference-data--VerticalMeasurementPath:ELEV:",
+ "VerticalMeasurement": 94.0,
+ "VerticalMeasurementTypeID": null,
+ "VerticalMeasurementDescription": null,
+ "VerticalMeasurementUnitOfMeasureID": "opendes:reference-data--UnitOfMeasure:m:",
+ "VerticalReferenceEntityID": null,
+ "RigID": null
+ }
+ ],
+ "VirtualProperties.DefaultLocation.CoordinateQualityCheckPerformedBy": null,
+ "VersionCreationReason": null,
+ "ResourceSecurityClassification": null,
+ "InterestTypeID": null,
+ "DataSourceOrganisationID": null,
+ "SpatialLocation.SpatialParameterTypeID": null,
+ "ExistenceKind": null,
+ "SpatialLocation.CoordinateQualityCheckPerformedBy": null,
+ "FacilityOperators": [{
+ "FacilityOperatorID": "Operator",
+ "FacilityOperatorOrganisationID": "opendes:master-data--Organisation:HOME%20ENERGY%20NZ%20LTD:"
+ }
+ ],
+ "FacilityTypeID": "opendes:reference-data--FacilityType:Well:",
+ "BusinessIntentionID": null,
+ "NameAliases": [{
+ "AliasName": "100000113552",
+ "AliasNameTypeID": "opendes:reference-data--AliasNameType:UniqueIdentifier:",
+ "DefinitionOrganisationID": null
+ }, {
+ "AliasName": "Well1",
+ "AliasNameTypeID": "opendes:reference-data--AliasNameType:CommonName:",
+ "DefinitionOrganisationID": null
+ }
+ ],
+ "DefaultVerticalCRSID": null,
+ "FacilityEvents": [{
+ "EffectiveDateTime": "1984-06-21T00:00:00+0000",
+ "FacilityEventTypeID": "opendes:reference-data--FacilityEventType:Spud:"
+ }
+ ],
+ "VirtualProperties.DefaultLocation.IsDecimated": false,
+ "TechnicalAssuranceTypeID": null,
+ "GeoContexts": [{
+ "BasinID": null,
+ "FieldID": null,
+ "PlayID": null,
+ "GeoPoliticalEntityID": "opendes:master-data--GeoPoliticalEntity:New%20Zealand:",
+ "GeoTypeID": "opendes:reference-data--GeoPoliticalEntityType:Country:",
+ "ProspectID": null
+ }, {
+ "BasinID": null,
+ "FieldID": null,
+ "PlayID": null,
+ "GeoPoliticalEntityID": "opendes:master-data--GeoPoliticalEntity:38058:",
+ "GeoTypeID": "opendes:reference-data--GeoPoliticalEntityType:LicenseBlock:",
+ "ProspectID": null
+ }
+ ],
+ "FakeGeoContexts": ["opendes:reference-data--GeoPoliticalEntityType:Country:","opendes:reference-data--GeoPoliticalEntityType:LicenseBlock:","opendes:reference-data--GeoPoliticalEntityType:Country:"],
+ "WasBusinessInterestFinancialNonOperated": null,
+ "CurrentOperatorID": null,
+ "SpatialLocation.QualitativeSpatialAccuracyTypeID": null,
+ "WasBusinessInterestTechnical": null,
+ "SpatialLocation.SpatialGeometryTypeID": "opendes:reference-data--SpatialGeometryType:Point:",
+ "OperatingEnvironmentID": "opendes:reference-data--OperatingEnvironment:Offshore:",
+ "VirtualProperties.DefaultLocation.SpatialParameterTypeID": null,
+ "ResourceHomeRegionID": null,
+ "StatusSummaryID": null,
+ "VirtualProperties.DefaultLocation.QualitativeSpatialAccuracyTypeID": null,
+ "ConditionID": null,
+ "InitialOperatorID": null,
+ "WasBusinessInterestFinancialOperated": null,
+ "VirtualProperties.DefaultLocation.Wgs84Coordinates": {
+ "type": "geometrycollection",
+ "geometries": [{
+ "type": "point",
+ "coordinates": [171.8745361, -41.2456122]
+ }
+ ]
+ },
+ "SpatialLocation.QuantitativeAccuracyBandID": null
+}
diff --git a/indexer-core/src/test/resources/indexproperty/well2.json b/indexer-core/src/test/resources/indexproperty/well2.json
new file mode 100644
index 0000000000000000000000000000000000000000..76fce04a10dfdc6a1b09661d0879fa4465f7f620
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/well2.json
@@ -0,0 +1,136 @@
+{
+ "VirtualProperties.DefaultLocation.QuantitativeAccuracyBandID": null,
+ "ResourceLifecycleStatus": null,
+ "DefaultVerticalMeasurementID": "KB",
+ "WasBusinessInterestObligatory": null,
+ "SpatialLocation.Wgs84Coordinates": {
+ "type": "geometrycollection",
+ "geometries": [{
+ "type": "point",
+ "coordinates": [171.8745361, -41.2456122]
+ }
+ ]
+ },
+ "ResourceCurationStatus": null,
+ "TechnicalAssuranceID": "opendes:reference-data--TechnicalAssuranceType:Unsuitable:",
+ "VirtualProperties.DefaultLocation.SpatialGeometryTypeID": "opendes:reference-data--SpatialGeometryType:Point:",
+ "RoleID": null,
+ "FacilityName": "Kongahu-2",
+ "Source": null,
+ "FacilityID": null,
+ "OutcomeID": null,
+ "VirtualProperties.DefaultName": "Kongahu-2",
+ "VerticalMeasurements": [{
+ "WellboreTVDTrajectoryID": null,
+ "VerticalCRSID": null,
+ "VerticalReferenceID": null,
+ "VerticalMeasurementSourceID": null,
+ "VerticalMeasurementID": "MB",
+ "VerticalMeasurementPathID": "opendes:reference-data--VerticalMeasurementPath:ELEV:",
+ "VerticalMeasurement": 26.0,
+ "VerticalMeasurementTypeID": "opendes:reference-data--VerticalMeasurementType:KB:",
+ "VerticalMeasurementDescription": null,
+ "VerticalMeasurementUnitOfMeasureID": "opendes:reference-data--UnitOfMeasure:m:",
+ "VerticalReferenceEntityID": null,
+ "RigID": null
+ }, {
+ "WellboreTVDTrajectoryID": null,
+ "VerticalCRSID": null,
+ "VerticalReferenceID": null,
+ "VerticalMeasurementSourceID": null,
+ "VerticalMeasurementID": "Well Head Elevation",
+ "VerticalMeasurementPathID": "opendes:reference-data--VerticalMeasurementPath:ELEV:",
+ "VerticalMeasurement": 26.0,
+ "VerticalMeasurementTypeID": null,
+ "VerticalMeasurementDescription": null,
+ "VerticalMeasurementUnitOfMeasureID": "opendes:reference-data--UnitOfMeasure:m:",
+ "VerticalReferenceEntityID": null,
+ "RigID": null
+ }, {
+ "WellboreTVDTrajectoryID": null,
+ "VerticalCRSID": null,
+ "VerticalReferenceID": null,
+ "VerticalMeasurementSourceID": null,
+ "VerticalMeasurementID": "Water Depth",
+ "VerticalMeasurementPathID": "opendes:reference-data--VerticalMeasurementPath:ELEV:",
+ "VerticalMeasurement": 94.0,
+ "VerticalMeasurementTypeID": null,
+ "VerticalMeasurementDescription": null,
+ "VerticalMeasurementUnitOfMeasureID": "opendes:reference-data--UnitOfMeasure:m:",
+ "VerticalReferenceEntityID": null,
+ "RigID": null
+ }
+ ],
+ "VirtualProperties.DefaultLocation.CoordinateQualityCheckPerformedBy": null,
+ "VersionCreationReason": null,
+ "ResourceSecurityClassification": null,
+ "InterestTypeID": null,
+ "DataSourceOrganisationID": null,
+ "SpatialLocation.SpatialParameterTypeID": null,
+ "ExistenceKind": null,
+ "SpatialLocation.CoordinateQualityCheckPerformedBy": null,
+ "FacilityOperators": [{
+ "FacilityOperatorID": "Operator",
+ "FacilityOperatorOrganisationID": "opendes:master-data--Organisation:HOME%20ENERGY%20NZ%20LTD:"
+ }
+ ],
+ "FacilityTypeID": "opendes:reference-data--FacilityType:Well:",
+ "BusinessIntentionID": null,
+ "NameAliases": [{
+ "AliasName": "100000113552",
+ "AliasNameTypeID": "opendes:reference-data--AliasNameType:UniqueIdentifier:",
+ "DefinitionOrganisationID": null
+ }, {
+ "AliasName": "Well1",
+ "AliasNameTypeID": "opendes:reference-data--AliasNameType:CommonName:",
+ "DefinitionOrganisationID": null
+ }
+ ],
+ "DefaultVerticalCRSID": null,
+ "FacilityEvents": [{
+ "EffectiveDateTime": "1984-06-21T00:00:00+0000",
+ "FacilityEventTypeID": "opendes:reference-data--FacilityEventType:Spud:"
+ }
+ ],
+ "VirtualProperties.DefaultLocation.IsDecimated": false,
+ "TechnicalAssuranceTypeID": null,
+ "GeoContexts": [{
+ "BasinID": null,
+ "FieldID": null,
+ "PlayID": null,
+ "GeoPoliticalEntityID": "opendes:master-data--GeoPoliticalEntity:New%20Zealand:",
+ "GeoTypeID": "opendes:reference-data--GeoPoliticalEntityType:Country:",
+ "ProspectID": null
+ }, {
+ "BasinID": null,
+ "FieldID": null,
+ "PlayID": null,
+ "GeoPoliticalEntityID": "opendes:master-data--GeoPoliticalEntity:38058:",
+ "GeoTypeID": "opendes:reference-data--GeoPoliticalEntityType:LicenseBlock:",
+ "ProspectID": null
+ }
+ ],
+ "FakeGeoContexts": ["opendes:reference-data--GeoPoliticalEntityType:Country:","opendes:reference-data--GeoPoliticalEntityType:LicenseBlock:","opendes:reference-data--GeoPoliticalEntityType:Country:"],
+ "WasBusinessInterestFinancialNonOperated": null,
+ "CurrentOperatorID": null,
+ "SpatialLocation.QualitativeSpatialAccuracyTypeID": null,
+ "WasBusinessInterestTechnical": null,
+ "SpatialLocation.SpatialGeometryTypeID": "opendes:reference-data--SpatialGeometryType:Point:",
+ "OperatingEnvironmentID": "opendes:reference-data--OperatingEnvironment:Offshore:",
+ "VirtualProperties.DefaultLocation.SpatialParameterTypeID": null,
+ "ResourceHomeRegionID": null,
+ "StatusSummaryID": null,
+ "VirtualProperties.DefaultLocation.QualitativeSpatialAccuracyTypeID": null,
+ "ConditionID": null,
+ "InitialOperatorID": null,
+ "WasBusinessInterestFinancialOperated": null,
+ "VirtualProperties.DefaultLocation.Wgs84Coordinates": {
+ "type": "geometrycollection",
+ "geometries": [{
+ "type": "point",
+ "coordinates": [171.8745361, -41.2456122]
+ }
+ ]
+ },
+ "SpatialLocation.QuantitativeAccuracyBandID": null
+}
diff --git a/indexer-core/src/test/resources/indexproperty/wellLog.json b/indexer-core/src/test/resources/indexproperty/wellLog.json
new file mode 100644
index 0000000000000000000000000000000000000000..0caafe08589d987a025b07393236a236f77d683d
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/wellLog.json
@@ -0,0 +1,352 @@
+{
+ "LogRemark": null,
+ "SpatialArea.QuantitativeAccuracyBandID": null,
+ "CompanyID": null,
+ "VirtualProperties.DefaultLocation.QuantitativeAccuracyBandID": null,
+ "SpatialArea.SpatialParameterTypeID": null,
+ "ResourceCurationStatus": null,
+ "SpatialArea.SpatialGeometryTypeID": null,
+ "IsExtendedLoad": null,
+ "Name": "EC_CURVES",
+ "VerticalMeasurement.VerticalCRSID": null,
+ "VirtualProperties.DefaultName": "EC_CURVES",
+ "SeismicReferenceElevation.VerticalMeasurementUnitOfMeasureID": null,
+ "VirtualProperties.DefaultLocation.CoordinateQualityCheckPerformedBy": null,
+ "ResourceSecurityClassification": null,
+ "SeismicReferenceElevation.VerticalMeasurementDescription": null,
+ "VerticalMeasurement.VerticalMeasurementPathID": null,
+ "SpatialLocation.SpatialParameterTypeID": null,
+ "ExistenceKind": null,
+ "HoleTypeLogging": null,
+ "LoggingDirection": null,
+ "VerticalMeasurementID": null,
+ "SeismicReferenceElevation.VerticalMeasurementPathID": null,
+ "SpatialArea.QualitativeSpatialAccuracyTypeID": null,
+ "VerticalMeasurement.VerticalReferenceEntityID": null,
+ "SpatialPoint.SpatialGeometryTypeID": null,
+ "SpatialLocation.SpatialGeometryTypeID": "opendes:reference-data--SpatialGeometryType:Point:",
+ "VerticalMeasurement.VerticalMeasurementSourceID": null,
+ "WellLogTypeID": null,
+ "IsDiscoverable": null,
+ "SeismicReferenceElevation.VerticalMeasurementTypeID": null,
+ "LoggingService": null,
+ "VirtualProperties.DefaultLocation.QualitativeSpatialAccuracyTypeID": null,
+ "VerticalMeasurement.VerticalReferenceID": null,
+ "SubmitterName": null,
+ "LogRun": null,
+ "SpatialPoint.QualitativeSpatialAccuracyTypeID": null,
+ "LogActivity": null,
+ "Description": null,
+ "ResourceLifecycleStatus": null,
+ "ServiceCompanyID": null,
+ "SpatialLocation.Wgs84Coordinates": {
+ "geometries": [{
+ "coordinates": [171.8745361, -41.2456122],
+ "type": "point"
+ }
+ ],
+ "type": "geometrycollection"
+ },
+ "TechnicalAssuranceID": null,
+ "SeismicReferenceElevation.WellboreTVDTrajectoryID": null,
+ "VirtualProperties.DefaultLocation.SpatialGeometryTypeID": null,
+ "Source": null,
+ "AssociatedIdentities": ["opendes:master-data--Wellbore:nz-100000113552"],
+ "LogVersion": null,
+ "SeismicReferenceElevation.VerticalCRSID": null,
+ "VerticalMeasurement.VerticalMeasurementTypeID": null,
+ "ToolStringDescription": null,
+ "LogSource": null,
+ "SpatialPoint.CoordinateQualityCheckPerformedBy": null,
+ "IsRegular": null,
+ "DrillingFluidProperty": null,
+ "SeismicReferenceElevation.VerticalReferenceEntityID": null,
+ "VerticalMeasurement.WellboreTVDTrajectoryID": null,
+ "SpatialLocation.CoordinateQualityCheckPerformedBy": null,
+ "VerticalMeasurement.VerticalMeasurementDescription": null,
+ "SpatialPoint.SpatialParameterTypeID": null,
+ "WellboreID": "opendes:master-data--Wellbore:nz-100000113552:",
+ "ActivityType": null,
+ "VerticalMeasurement.VerticalMeasurementUnitOfMeasureID": null,
+ "SpatialPoint.QuantitativeAccuracyBandID": null,
+ "SamplingDomainTypeID": null,
+ "SpatialArea.CoordinateQualityCheckPerformedBy": null,
+ "ReferenceCurveID": "DEPTH",
+ "SpatialLocation.IsDecimated": false,
+ "WellboreName": "Kongahu-5",
+ "SpatialLocation.QualitativeSpatialAccuracyTypeID": null,
+ "VirtualProperties.DefaultLocation.SpatialParameterTypeID": null,
+ "ResourceHomeRegionID": null,
+ "FrameIdentifier": null,
+ "SeismicReferenceElevation.VerticalReferenceID": null,
+ "SeismicReferenceElevation.VerticalMeasurementSourceID": null,
+ "Curves": [{
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Measured%20Depth:",
+ "CurveID": "DEPTH",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "DEPTH",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:M:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": null,
+ "CurveID": "CT",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "CT",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:MH%2FM:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Conductivity%20-%20Flushed%20Zone:",
+ "CurveID": "CXO",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "CXO",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:MH%2FM:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Invasion%20Diameter:",
+ "CurveID": "DI",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "DI",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Fracture%20Pressure:",
+ "CurveID": "FPRESS",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "FPRESS",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:PSI:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Formation%20Temperature%20%28Estimate%29:",
+ "CurveID": "FTEMP",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "FTEMP",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:DEGC:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": null,
+ "CurveID": "HMC_POR",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "HMC_POR",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": null,
+ "CurveID": "HMC_RES",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "HMC_RES",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Mud%20Resistivity:",
+ "CurveID": "RM",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "RM",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Mudcake%20Resistivity:",
+ "CurveID": "RMC",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "RMC",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Mud%20Filtrate%20Resistivity:",
+ "CurveID": "RMF",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "RMF",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20True%20Formation:",
+ "CurveID": "RT",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "RT",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Flushed%20Zone:",
+ "CurveID": "RXO",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "RXO",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Volumetric%20Photoelectric%20Factor:",
+ "CurveID": "U",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "U",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:B%2FC3:",
+ "CurveDescription": null
+ }
+ ],
+ "SpatialLocation.QuantitativeAccuracyBandID": null
+}
diff --git a/indexer-core/src/test/resources/indexproperty/wellLog2.json b/indexer-core/src/test/resources/indexproperty/wellLog2.json
new file mode 100644
index 0000000000000000000000000000000000000000..7a071091f7a76dc83ccc2f5fb80a8438cd0a25a9
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/wellLog2.json
@@ -0,0 +1,352 @@
+{
+ "LogRemark": null,
+ "SpatialArea.QuantitativeAccuracyBandID": null,
+ "CompanyID": null,
+ "VirtualProperties.DefaultLocation.QuantitativeAccuracyBandID": null,
+ "SpatialArea.SpatialParameterTypeID": null,
+ "ResourceCurationStatus": null,
+ "SpatialArea.SpatialGeometryTypeID": null,
+ "IsExtendedLoad": null,
+ "Name": "EC_CURVES",
+ "VerticalMeasurement.VerticalCRSID": null,
+ "VirtualProperties.DefaultName": "EC_CURVES",
+ "SeismicReferenceElevation.VerticalMeasurementUnitOfMeasureID": null,
+ "VirtualProperties.DefaultLocation.CoordinateQualityCheckPerformedBy": null,
+ "ResourceSecurityClassification": null,
+ "SeismicReferenceElevation.VerticalMeasurementDescription": null,
+ "VerticalMeasurement.VerticalMeasurementPathID": null,
+ "SpatialLocation.SpatialParameterTypeID": null,
+ "ExistenceKind": null,
+ "HoleTypeLogging": null,
+ "LoggingDirection": null,
+ "VerticalMeasurementID": null,
+ "SeismicReferenceElevation.VerticalMeasurementPathID": null,
+ "SpatialArea.QualitativeSpatialAccuracyTypeID": null,
+ "VerticalMeasurement.VerticalReferenceEntityID": null,
+ "SpatialPoint.SpatialGeometryTypeID": null,
+ "SpatialLocation.SpatialGeometryTypeID": "opendes:reference-data--SpatialGeometryType:Point:",
+ "VerticalMeasurement.VerticalMeasurementSourceID": null,
+ "WellLogTypeID": null,
+ "IsDiscoverable": null,
+ "SeismicReferenceElevation.VerticalMeasurementTypeID": null,
+ "LoggingService": null,
+ "VirtualProperties.DefaultLocation.QualitativeSpatialAccuracyTypeID": null,
+ "VerticalMeasurement.VerticalReferenceID": null,
+ "SubmitterName": null,
+ "LogRun": null,
+ "SpatialPoint.QualitativeSpatialAccuracyTypeID": null,
+ "LogActivity": null,
+ "Description": null,
+ "ResourceLifecycleStatus": null,
+ "ServiceCompanyID": null,
+ "SpatialLocation.Wgs84Coordinates": {
+ "geometries": [{
+ "coordinates": [171.8745361, -41.2456122],
+ "type": "point"
+ }
+ ],
+ "type": "geometrycollection"
+ },
+ "TechnicalAssuranceID": null,
+ "SeismicReferenceElevation.WellboreTVDTrajectoryID": null,
+ "VirtualProperties.DefaultLocation.SpatialGeometryTypeID": null,
+ "Source": null,
+ "AssociatedIdentities": ["opendes:master-data--Wellbore:nz-100000113552"],
+ "LogVersion": null,
+ "SeismicReferenceElevation.VerticalCRSID": null,
+ "VerticalMeasurement.VerticalMeasurementTypeID": null,
+ "ToolStringDescription": null,
+ "LogSource": null,
+ "SpatialPoint.CoordinateQualityCheckPerformedBy": null,
+ "IsRegular": null,
+ "DrillingFluidProperty": null,
+ "SeismicReferenceElevation.VerticalReferenceEntityID": null,
+ "VerticalMeasurement.WellboreTVDTrajectoryID": null,
+ "SpatialLocation.CoordinateQualityCheckPerformedBy": null,
+ "VerticalMeasurement.VerticalMeasurementDescription": null,
+ "SpatialPoint.SpatialParameterTypeID": null,
+ "WellboreID": "opendes:master-data--Wellbore:nz-100000113552:",
+ "ActivityType": null,
+ "VerticalMeasurement.VerticalMeasurementUnitOfMeasureID": null,
+ "SpatialPoint.QuantitativeAccuracyBandID": null,
+ "SamplingDomainTypeID": null,
+ "SpatialArea.CoordinateQualityCheckPerformedBy": null,
+ "ReferenceCurveID": "DEPTH",
+ "SpatialLocation.IsDecimated": false,
+ "WellboreName": "Kongahu-5",
+ "SpatialLocation.QualitativeSpatialAccuracyTypeID": null,
+ "VirtualProperties.DefaultLocation.SpatialParameterTypeID": null,
+ "ResourceHomeRegionID": null,
+ "FrameIdentifier": null,
+ "SeismicReferenceElevation.VerticalReferenceID": null,
+ "SeismicReferenceElevation.VerticalMeasurementSourceID": null,
+ "Curves": [{
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Measured%20Depth:",
+ "CurveID": "DEPTH",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "DEPTH",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:M:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": null,
+ "CurveID": "ABC",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "CT",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:MH%2FM:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Conductivity%20-%20Flushed%20Zone:",
+ "CurveID": "CXO",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "CXO",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:MH%2FM:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Invasion%20Diameter:",
+ "CurveID": "DI",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "DI",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Fracture%20Pressure:",
+ "CurveID": "FPRESS",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "FPRESS",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:PSI:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Formation%20Temperature%20%28Estimate%29:",
+ "CurveID": "FTEMP",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "FTEMP",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:DEGC:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": null,
+ "CurveID": "HMC_POR",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "HMC_POR",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": null,
+ "CurveID": "HMC_RES",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "HMC_RES",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Mud%20Resistivity:",
+ "CurveID": "RM",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "RM",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Mudcake%20Resistivity:",
+ "CurveID": "RMC",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "RMC",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Mud%20Filtrate%20Resistivity:",
+ "CurveID": "RMF",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "RMF",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20True%20Formation:",
+ "CurveID": "RT",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "RT",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Flushed%20Zone:",
+ "CurveID": "RXO",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "RXO",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Volumetric%20Photoelectric%20Factor:",
+ "CurveID": "U",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "U",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:B%2FC3:",
+ "CurveDescription": null
+ }
+ ],
+ "SpatialLocation.QuantitativeAccuracyBandID": null
+}
diff --git a/indexer-core/src/test/resources/indexproperty/well_configuration_record.json b/indexer-core/src/test/resources/indexproperty/well_configuration_record.json
new file mode 100644
index 0000000000000000000000000000000000000000..0cf4995d3cdd40b537d685b2fd7af394605df845
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/well_configuration_record.json
@@ -0,0 +1,56 @@
+{
+ "AttributionPublication": null,
+ "InactiveIndicator": null,
+ "Description": "The index property list for master-data--Well:1., valid for all master-data--Well kinds for major version 1.",
+ "ResourceLifecycleStatus": null,
+ "ResourceCurationStatus": null,
+ "TechnicalAssuranceID": null,
+ "Code": "osdu:wks:master-data--Well:1.",
+ "Source": null,
+ "Name": "Well-IndexPropertyPathConfiguration",
+ "AttributionAuthority": "OSDU",
+ "ResourceHomeRegionID": null,
+ "VirtualProperties.DefaultName": "Well-IndexPropertyPathConfiguration",
+ "AttributionRevision": null,
+ "ResourceSecurityClassification": null,
+ "ID": null,
+ "ExistenceKind": null,
+ "Configurations": [{
+ "Policy": "ExtractAllMatches",
+ "UseCase": "As a user I want to find objects by a country name, with the understanding that an object may extend over country boundaries.",
+ "Paths": [{
+ "RelatedObjectsSpec.RelatedObjectID": "data.GeoContexts[].GeoPoliticalEntityID",
+ "RelatedObjectsSpec.RelatedConditionMatches": [
+ "opendes:reference-data--GeoPoliticalEntityType:Country:"
+ ],
+ "ValueExtraction.ValuePath": "data.GeoPoliticalEntityName",
+ "RelatedObjectsSpec.RelatedObjectKind": "osdu:wks:master-data--GeoPoliticalEntity:1.",
+ "RelatedObjectsSpec.RelatedConditionProperty": "data.GeoContexts[].GeoTypeID",
+ "RelatedObjectsSpec.RelationshipDirection": "ChildToParent",
+ "ValueExtraction.RelatedConditionProperty": null
+ }
+ ],
+ "Name": "CountryNames"
+ }, {
+ "Policy": "ExtractFirstMatch",
+ "UseCase": "As a user I want to discover and match Wells by their UWI. I am aware that this is not globally reliable, however, I am able to specify a prioritized AliasNameType list to look up value in the NameAliases array.",
+ "Paths": [{
+ "RelatedObjectsSpec.RelatedObjectID": null,
+ "ValueExtraction.ValuePath": "data.NameAliases[].AliasName",
+ "RelatedObjectsSpec.RelatedObjectKind": null,
+ "RelatedObjectsSpec.RelatedConditionProperty": null,
+ "RelatedObjectsSpec.RelationshipDirection": null,
+ "ValueExtraction.RelatedConditionMatches": [
+ "opendes:reference-data--AliasNameType:UniqueIdentifier:",
+ "opendes:reference-data--AliasNameType:RegulatoryName:",
+ "opendes:reference-data--AliasNameType:PreferredName:",
+ "opendes:reference-data--AliasNameType:CommonName:",
+ "opendes:reference-data--AliasNameType:ShortName:"
+ ],
+ "ValueExtraction.RelatedConditionProperty": "data.NameAliases[].AliasNameTypeID"
+ }
+ ],
+ "Name": "WellUWI"
+ }
+ ]
+}
diff --git a/indexer-core/src/test/resources/indexproperty/well_storage_schema.json b/indexer-core/src/test/resources/indexproperty/well_storage_schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..ee6178a3868902da8681181b0bf2016ef8e93394
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/well_storage_schema.json
@@ -0,0 +1,314 @@
+{
+ "kind": "osdu:wks:master-data--Well:1.1.0",
+ "schema": [{
+ "path": "ResourceHomeRegionID",
+ "kind": "string"
+ }, {
+ "path": "ResourceHostRegionIDs",
+ "kind": "[]string"
+ }, {
+ "path": "ResourceLifecycleStatus",
+ "kind": "string"
+ }, {
+ "path": "ResourceSecurityClassification",
+ "kind": "string"
+ }, {
+ "path": "ResourceCurationStatus",
+ "kind": "string"
+ }, {
+ "path": "ExistenceKind",
+ "kind": "string"
+ }, {
+ "path": "TechnicalAssuranceID",
+ "kind": "string"
+ }, {
+ "path": "Source",
+ "kind": "string"
+ }, {
+ "path": "NameAliases",
+ "kind": "nested",
+ "properties": [{
+ "path": "AliasNameTypeID",
+ "kind": "string"
+ }, {
+ "path": "EffectiveDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "AliasName",
+ "kind": "string"
+ }, {
+ "path": "TerminationDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "DefinitionOrganisationID",
+ "kind": "string"
+ }
+ ]
+ }, {
+ "path": "SpatialLocation.SpatialParameterTypeID",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.QuantitativeAccuracyBandID",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.CoordinateQualityCheckRemarks",
+ "kind": "[]string"
+ }, {
+ "path": "SpatialLocation.AppliedOperations",
+ "kind": "[]string"
+ }, {
+ "path": "SpatialLocation.QualitativeSpatialAccuracyTypeID",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.CoordinateQualityCheckPerformedBy",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.SpatialLocationCoordinatesDate",
+ "kind": "datetime"
+ }, {
+ "path": "SpatialLocation.CoordinateQualityCheckDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "SpatialLocation.Wgs84Coordinates",
+ "kind": "core:dl:geoshape:1.0.0"
+ }, {
+ "path": "SpatialLocation.SpatialGeometryTypeID",
+ "kind": "string"
+ }, {
+ "path": "VersionCreationReason",
+ "kind": "string"
+ }, {
+ "path": "TechnicalAssuranceTypeID",
+ "kind": "string"
+ }, {
+ "path": "GeoContexts",
+ "kind": "nested",
+ "properties": [{
+ "path": "GeoPoliticalEntityID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "BasinID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "FieldID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "PlayID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "ProspectID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }
+ ]
+ }, {
+ "path": "FacilityStates",
+ "kind": "nested",
+ "properties": [{
+ "path": "EffectiveDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "FacilityStateTypeID",
+ "kind": "string"
+ }, {
+ "path": "TerminationDateTime",
+ "kind": "datetime"
+ }
+ ]
+ }, {
+ "path": "FacilityID",
+ "kind": "string"
+ }, {
+ "path": "OperatingEnvironmentID",
+ "kind": "string"
+ }, {
+ "path": "FacilityNameAliases",
+ "kind": "[]object"
+ }, {
+ "path": "FacilityEvents",
+ "kind": "nested",
+ "properties": [{
+ "path": "EffectiveDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "TerminationDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "FacilityEventTypeID",
+ "kind": "string"
+ }
+ ]
+ }, {
+ "path": "FacilitySpecifications",
+ "kind": "flattened"
+ }, {
+ "path": "DataSourceOrganisationID",
+ "kind": "string"
+ }, {
+ "path": "InitialOperatorID",
+ "kind": "string"
+ }, {
+ "path": "CurrentOperatorID",
+ "kind": "string"
+ }, {
+ "path": "FacilityOperators",
+ "kind": "nested",
+ "properties": [{
+ "path": "FacilityOperatorID",
+ "kind": "string"
+ }, {
+ "path": "EffectiveDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "FacilityOperatorOrganisationID",
+ "kind": "string"
+ }, {
+ "path": "TerminationDateTime",
+ "kind": "datetime"
+ }
+ ]
+ }, {
+ "path": "FacilityName",
+ "kind": "string"
+ }, {
+ "path": "FacilityTypeID",
+ "kind": "string"
+ }, {
+ "path": "BusinessIntentionID",
+ "kind": "string"
+ }, {
+ "path": "DefaultVerticalCRSID",
+ "kind": "string"
+ }, {
+ "path": "WasBusinessInterestFinancialNonOperated",
+ "kind": "bool"
+ }, {
+ "path": "DefaultVerticalMeasurementID",
+ "kind": "string"
+ }, {
+ "path": "WasBusinessInterestObligatory",
+ "kind": "bool"
+ }, {
+ "path": "RoleID",
+ "kind": "string"
+ }, {
+ "path": "WasBusinessInterestTechnical",
+ "kind": "bool"
+ }, {
+ "path": "OutcomeID",
+ "kind": "string"
+ }, {
+ "path": "VerticalMeasurements",
+ "kind": "nested",
+ "properties": [{
+ "path": "VerticalMeasurementID",
+ "kind": "string"
+ }, {
+ "path": "RigID",
+ "kind": "string"
+ }, {
+ "path": "WellboreTVDTrajectoryID",
+ "kind": "string"
+ }, {
+ "path": "VerticalCRSID",
+ "kind": "string"
+ }, {
+ "path": "VerticalMeasurementSourceID",
+ "kind": "string"
+ }, {
+ "path": "VerticalReferenceID",
+ "kind": "string"
+ }, {
+ "path": "TerminationDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "VerticalMeasurementPathID",
+ "kind": "string"
+ }, {
+ "path": "EffectiveDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "VerticalMeasurement",
+ "kind": "double"
+ }, {
+ "path": "VerticalMeasurementTypeID",
+ "kind": "string"
+ }, {
+ "path": "VerticalMeasurementDescription",
+ "kind": "string"
+ }, {
+ "path": "VerticalMeasurementUnitOfMeasureID",
+ "kind": "string"
+ }, {
+ "path": "VerticalReferenceEntityID",
+ "kind": "string"
+ }
+ ]
+ }, {
+ "path": "HistoricalInterests",
+ "kind": "[]object"
+ }, {
+ "path": "StatusSummaryID",
+ "kind": "string"
+ }, {
+ "path": "InterestTypeID",
+ "kind": "string"
+ }, {
+ "path": "ConditionID",
+ "kind": "string"
+ }, {
+ "path": "WasBusinessInterestFinancialOperated",
+ "kind": "bool"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.SpatialParameterTypeID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.QuantitativeAccuracyBandID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.CoordinateQualityCheckRemarks",
+ "kind": "[]string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.AppliedOperations",
+ "kind": "[]string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.QualitativeSpatialAccuracyTypeID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.CoordinateQualityCheckPerformedBy",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.SpatialLocationCoordinatesDate",
+ "kind": "datetime"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.CoordinateQualityCheckDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.Wgs84Coordinates",
+ "kind": "core:dl:geoshape:1.0.0"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.SpatialGeometryTypeID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultName",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.IsDecimated",
+ "kind": "boolean"
+ }
+ ]
+}
diff --git a/indexer-core/src/test/resources/indexproperty/wellbore_configuration_record.json b/indexer-core/src/test/resources/indexproperty/wellbore_configuration_record.json
new file mode 100644
index 0000000000000000000000000000000000000000..0eb75a54ad5af31e9d7e3143e8689754af495ce1
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/wellbore_configuration_record.json
@@ -0,0 +1,33 @@
+{
+ "AttributionPublication": null,
+ "InactiveIndicator": null,
+ "Description": "The index property list for master-data--Wellbore:1., valid for all master-data--Wellbore kinds for major version 1.",
+ "ResourceLifecycleStatus": null,
+ "ResourceCurationStatus": null,
+ "TechnicalAssuranceID": null,
+ "Code": "osdu:wks:master-data--Wellbore:1.",
+ "Source": null,
+ "Name": "Well-IndexPropertyPathConfiguration",
+ "AttributionAuthority": "OSDU",
+ "ResourceHomeRegionID": null,
+ "VirtualProperties.DefaultName": "Well-IndexPropertyPathConfiguration",
+ "AttributionRevision": null,
+ "ResourceSecurityClassification": null,
+ "ID": null,
+ "ExistenceKind": null,
+ "Configurations": [{
+ "Policy": "ExtractAllMatches",
+ "UseCase": "As a user I want to find wellbores by well log type(s).",
+ "Paths": [{
+ "RelatedObjectsSpec.RelatedObjectID": "data.WellboreID",
+ "ValueExtraction.ValuePath": "data.Curves[].Mnemonic",
+ "RelatedObjectsSpec.RelatedObjectKind": "osdu:wks:work-product-component--WellLog:1.",
+ "RelatedObjectsSpec.RelatedConditionProperty": null,
+ "RelatedObjectsSpec.RelationshipDirection": "ParentToChildren",
+ "ValueExtraction.RelatedConditionProperty": null
+ }
+ ],
+ "Name": "WellLogs"
+ }
+ ]
+}
diff --git a/indexer-core/src/test/resources/indexproperty/wellbore_data.json b/indexer-core/src/test/resources/indexproperty/wellbore_data.json
new file mode 100644
index 0000000000000000000000000000000000000000..1150d64100d254247eea99305faeb131b79a5437
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/wellbore_data.json
@@ -0,0 +1,65 @@
+{
+ "DefaultVerticalMeasurementID": "KB",
+ "FacilityName": "Kongahu-1",
+ "VirtualProperties.DefaultName": "Kongahu-1",
+ "FacilityTypeID": "opendes:reference-data--FacilityType:Wellbore:",
+ "FacilityEvents": [{
+ "EffectiveDateTime": "1984-06-21T00:00:00+0000",
+ "FacilityEventTypeID": "opendes:reference-data--FacilityEventType:Spud:"
+ }
+ ],
+ "VirtualProperties.DefaultLocation.IsDecimated": false,
+ "CurrentOperatorID": "opendes:master-data--Organisation:HOME%20ENERGY%20NZ%20LTD:",
+ "SpatialLocation.SpatialGeometryTypeID": "opendes:reference-data--SpatialGeometryType:Point:",
+ "WellID": "opendes:master-data--Well:1816bd81",
+ "SpatialLocation.Wgs84Coordinates": {
+ "type": "geometrycollection",
+ "geometries": [{
+ "type": "point",
+ "coordinates": [171.8745361, -41.2456122]
+ }
+ ]
+ },
+ "TechnicalAssuranceID": "opendes:reference-data--TechnicalAssuranceType:Unsuitable:",
+ "VirtualProperties.DefaultLocation.SpatialGeometryTypeID": "opendes:reference-data--SpatialGeometryType:Point:",
+ "DefinitiveTrajectoryID": "opendes:work-product-component--WellboreTrajectory:nz-100000113552:",
+ "VerticalMeasurements": [{
+ "VerticalMeasurementID": "KB",
+ "VerticalMeasurementPathID": "opendes:reference-data--VerticalMeasurementPath:ELEV:",
+ "VerticalMeasurement": 26.0,
+ "VerticalMeasurementTypeID": "opendes:reference-data--VerticalMeasurementType:KB:",
+ "VerticalMeasurementUnitOfMeasureID": "opendes:reference-data--UnitOfMeasure:m:"
+ }, {
+ "VerticalMeasurementID": "Total Depth MD",
+ "VerticalMeasurementPathID": "opendes:reference-data--VerticalMeasurementPath:MD:",
+ "VerticalMeasurement": 2015.5,
+ "VerticalMeasurementUnitOfMeasureID": "opendes:reference-data--UnitOfMeasure:m:"
+ }, {
+ "VerticalMeasurementID": "Well Head Elevation",
+ "VerticalMeasurementPathID": "opendes:reference-data--VerticalMeasurementPath:ELEV:",
+ "VerticalMeasurement": 26.0,
+ "VerticalMeasurementUnitOfMeasureID": "opendes:reference-data--UnitOfMeasure:m:"
+ }
+ ],
+ "NameAliases": [{
+ "AliasName": "100000113552",
+ "AliasNameTypeID": "opendes:reference-data--AliasNameType:UniqueIdentifier:"
+ }
+ ],
+ "GeoContexts": [{
+ "GeoPoliticalEntityID": "opendes:master-data--GeoPoliticalEntity:New%20Zealand:",
+ "GeoTypeID": "opendes:reference-data--GeoPoliticalEntityType:Country:"
+ }, {
+ "GeoPoliticalEntityID": "opendes:master-data--GeoPoliticalEntity:38058:",
+ "GeoTypeID": "opendes:reference-data--GeoPoliticalEntityType:LicenseBlock:"
+ }
+ ],
+ "VirtualProperties.DefaultLocation.Wgs84Coordinates": {
+ "type": "geometrycollection",
+ "geometries": [{
+ "type": "point",
+ "coordinates": [171.8745361, -41.2456122]
+ }
+ ]
+ }
+}
diff --git a/indexer-core/src/test/resources/indexproperty/wellbore_extended_data.json b/indexer-core/src/test/resources/indexproperty/wellbore_extended_data.json
new file mode 100644
index 0000000000000000000000000000000000000000..0ed1e2b9b88266ccd9fe24960b7281454efa094d
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/wellbore_extended_data.json
@@ -0,0 +1,3 @@
+{
+ "WellLogs": ["BS", "BS_2", "BS_3", "BS_4", "BS_5", "BS_6", "CALI", "CALI_2", "CALI_3", "CALI_4", "CALI_5", "CALI_6", "CT", "CXO", "DENS", "DENS_CORR", "DEPTH", "DI", "DRHO", "DT", "DTC", "DT_2", "DT_3", "DT_4", "FPRESS", "FTEMP", "GR", "GR_10", "GR_2", "GR_3", "GR_4", "GR_5", "GR_6", "GR_7", "GR_8", "GR_9", "GR_CORR", "HMC_POR", "HMC_RES", "LLD", "LLD_2", "LLD_3", "LLD_4", "LLS", "LLS_2", "LLS_3", "LLS_4", "MINV", "MINV_2", "MINV_3", "MINV_4", "MNOR", "MNOR_2", "MNOR_3", "MNOR_4", "MSFL", "MSFL_2", "MSFL_3", "MSFL_4", "NEUT", "NEUT_CORR", "NPHI", "PEF", "RESD", "RESD_CORR", "RESS", "RESS_CORR", "RHOB", "RM", "RMC", "RMF", "RT", "RXO", "SP", "SP_2", "SP_3", "SP_4", "TENS", "TENS_10", "TENS_2", "TENS_3", "TENS_4", "TENS_5", "TENS_6", "TENS_7", "TENS_8", "TENS_9", "U"]
+}
diff --git a/indexer-core/src/test/resources/indexproperty/wellbore_storage_schema.json b/indexer-core/src/test/resources/indexproperty/wellbore_storage_schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..3d52ae29a04d50b3bc48a27b778ed944f3e4666f
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/wellbore_storage_schema.json
@@ -0,0 +1,442 @@
+{
+ "kind": "osdu:wks:master-data--Wellbore:1.3.0",
+ "schema": [{
+ "path": "ResourceHomeRegionID",
+ "kind": "string"
+ }, {
+ "path": "ResourceHostRegionIDs",
+ "kind": "[]string"
+ }, {
+ "path": "ResourceLifecycleStatus",
+ "kind": "string"
+ }, {
+ "path": "ResourceSecurityClassification",
+ "kind": "string"
+ }, {
+ "path": "ResourceCurationStatus",
+ "kind": "string"
+ }, {
+ "path": "ExistenceKind",
+ "kind": "string"
+ }, {
+ "path": "TechnicalAssuranceID",
+ "kind": "string"
+ }, {
+ "path": "Source",
+ "kind": "string"
+ }, {
+ "path": "NameAliases",
+ "kind": "nested",
+ "properties": [{
+ "path": "AliasNameTypeID",
+ "kind": "string"
+ }, {
+ "path": "EffectiveDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "AliasName",
+ "kind": "string"
+ }, {
+ "path": "TerminationDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "DefinitionOrganisationID",
+ "kind": "string"
+ }
+ ]
+ }, {
+ "path": "SpatialLocation.SpatialParameterTypeID",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.QuantitativeAccuracyBandID",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.CoordinateQualityCheckRemarks",
+ "kind": "[]string"
+ }, {
+ "path": "SpatialLocation.AppliedOperations",
+ "kind": "[]string"
+ }, {
+ "path": "SpatialLocation.QualitativeSpatialAccuracyTypeID",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.CoordinateQualityCheckPerformedBy",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.SpatialLocationCoordinatesDate",
+ "kind": "datetime"
+ }, {
+ "path": "SpatialLocation.CoordinateQualityCheckDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "SpatialLocation.Wgs84Coordinates",
+ "kind": "core:dl:geoshape:1.0.0"
+ }, {
+ "path": "SpatialLocation.SpatialGeometryTypeID",
+ "kind": "string"
+ }, {
+ "path": "VersionCreationReason",
+ "kind": "string"
+ }, {
+ "path": "TechnicalAssuranceTypeID",
+ "kind": "string"
+ }, {
+ "path": "GeoContexts",
+ "kind": "nested",
+ "properties": [{
+ "path": "GeoPoliticalEntityID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "BasinID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "FieldID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "PlayID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "ProspectID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }
+ ]
+ }, {
+ "path": "FacilityStates",
+ "kind": "nested",
+ "properties": [{
+ "path": "EffectiveDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "FacilityStateTypeID",
+ "kind": "string"
+ }, {
+ "path": "TerminationDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "Remark",
+ "kind": "string"
+ }
+ ]
+ }, {
+ "path": "FacilityNameAliases",
+ "kind": "[]object"
+ }, {
+ "path": "FacilityEvents",
+ "kind": "nested",
+ "properties": [{
+ "path": "EffectiveDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "FacilityEventTypeID",
+ "kind": "string"
+ }, {
+ "path": "TerminationDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "Remark",
+ "kind": "string"
+ }
+ ]
+ }, {
+ "path": "CurrentOperatorID",
+ "kind": "string"
+ }, {
+ "path": "FacilityName",
+ "kind": "string"
+ }, {
+ "path": "FacilityID",
+ "kind": "string"
+ }, {
+ "path": "OperatingEnvironmentID",
+ "kind": "string"
+ }, {
+ "path": "FacilitySpecifications",
+ "kind": "flattened"
+ }, {
+ "path": "DataSourceOrganisationID",
+ "kind": "string"
+ }, {
+ "path": "FacilityDescription",
+ "kind": "string"
+ }, {
+ "path": "InitialOperatorID",
+ "kind": "string"
+ }, {
+ "path": "FacilityOperators",
+ "kind": "nested",
+ "properties": [{
+ "path": "FacilityOperatorID",
+ "kind": "string"
+ }, {
+ "path": "EffectiveDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "FacilityOperatorOrganisationID",
+ "kind": "string"
+ }, {
+ "path": "TerminationDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "Remark",
+ "kind": "string"
+ }
+ ]
+ }, {
+ "path": "FacilityTypeID",
+ "kind": "string"
+ }, {
+ "path": "FluidDirectionID",
+ "kind": "string"
+ }, {
+ "path": "TargetFormation",
+ "kind": "string"
+ }, {
+ "path": "FormationNameAtTotalDepth",
+ "kind": "string"
+ }, {
+ "path": "DefaultVerticalMeasurementID",
+ "kind": "string"
+ }, {
+ "path": "WasBusinessInterestObligatory",
+ "kind": "bool"
+ }, {
+ "path": "RoleID",
+ "kind": "string"
+ }, {
+ "path": "DefinitiveTrajectoryID",
+ "kind": "string"
+ }, {
+ "path": "GeographicBottomHoleLocation.SpatialParameterTypeID",
+ "kind": "string"
+ }, {
+ "path": "GeographicBottomHoleLocation.QuantitativeAccuracyBandID",
+ "kind": "string"
+ }, {
+ "path": "GeographicBottomHoleLocation.CoordinateQualityCheckRemarks",
+ "kind": "[]string"
+ }, {
+ "path": "GeographicBottomHoleLocation.AppliedOperations",
+ "kind": "[]string"
+ }, {
+ "path": "GeographicBottomHoleLocation.QualitativeSpatialAccuracyTypeID",
+ "kind": "string"
+ }, {
+ "path": "GeographicBottomHoleLocation.CoordinateQualityCheckPerformedBy",
+ "kind": "string"
+ }, {
+ "path": "GeographicBottomHoleLocation.SpatialLocationCoordinatesDate",
+ "kind": "datetime"
+ }, {
+ "path": "GeographicBottomHoleLocation.CoordinateQualityCheckDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "GeographicBottomHoleLocation.Wgs84Coordinates",
+ "kind": "core:dl:geoshape:1.0.0"
+ }, {
+ "path": "GeographicBottomHoleLocation.SpatialGeometryTypeID",
+ "kind": "string"
+ }, {
+ "path": "DrillingReasons",
+ "kind": "[]object"
+ }, {
+ "path": "OutcomeID",
+ "kind": "string"
+ }, {
+ "path": "VerticalMeasurements",
+ "kind": "nested",
+ "properties": [{
+ "path": "VerticalMeasurementID",
+ "kind": "string"
+ }, {
+ "path": "RigID",
+ "kind": "string"
+ }, {
+ "path": "WellboreTVDTrajectoryID",
+ "kind": "string"
+ }, {
+ "path": "VerticalCRSID",
+ "kind": "string"
+ }, {
+ "path": "VerticalMeasurementSourceID",
+ "kind": "string"
+ }, {
+ "path": "VerticalReferenceID",
+ "kind": "string"
+ }, {
+ "path": "TerminationDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "VerticalMeasurementPathID",
+ "kind": "string"
+ }, {
+ "path": "EffectiveDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "VerticalMeasurement",
+ "kind": "double"
+ }, {
+ "path": "VerticalMeasurementTypeID",
+ "kind": "string"
+ }, {
+ "path": "VerticalMeasurementDescription",
+ "kind": "string"
+ }, {
+ "path": "VerticalMeasurementUnitOfMeasureID",
+ "kind": "string"
+ }, {
+ "path": "VerticalReferenceEntityID",
+ "kind": "string"
+ }
+ ]
+ }, {
+ "path": "WellboreCosts",
+ "kind": "nested",
+ "properties": [{
+ "path": "ActivityTypeID",
+ "kind": "string"
+ }, {
+ "path": "Cost",
+ "kind": "double"
+ }
+ ]
+ }, {
+ "path": "PrimaryProductTypeID",
+ "kind": "string"
+ }, {
+ "path": "SequenceNumber",
+ "kind": "int"
+ }, {
+ "path": "InterestTypeID",
+ "kind": "string"
+ }, {
+ "path": "KickOffWellbore",
+ "kind": "string"
+ }, {
+ "path": "TertiaryProductTypeID",
+ "kind": "string"
+ }, {
+ "path": "ShowProductTypeID",
+ "kind": "string"
+ }, {
+ "path": "BusinessIntentionID",
+ "kind": "string"
+ }, {
+ "path": "WasBusinessInterestFinancialNonOperated",
+ "kind": "bool"
+ }, {
+ "path": "ProjectedBottomHoleLocation.SpatialParameterTypeID",
+ "kind": "string"
+ }, {
+ "path": "ProjectedBottomHoleLocation.QuantitativeAccuracyBandID",
+ "kind": "string"
+ }, {
+ "path": "ProjectedBottomHoleLocation.CoordinateQualityCheckRemarks",
+ "kind": "[]string"
+ }, {
+ "path": "ProjectedBottomHoleLocation.AppliedOperations",
+ "kind": "[]string"
+ }, {
+ "path": "ProjectedBottomHoleLocation.QualitativeSpatialAccuracyTypeID",
+ "kind": "string"
+ }, {
+ "path": "ProjectedBottomHoleLocation.CoordinateQualityCheckPerformedBy",
+ "kind": "string"
+ }, {
+ "path": "ProjectedBottomHoleLocation.SpatialLocationCoordinatesDate",
+ "kind": "datetime"
+ }, {
+ "path": "ProjectedBottomHoleLocation.CoordinateQualityCheckDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "ProjectedBottomHoleLocation.Wgs84Coordinates",
+ "kind": "core:dl:geoshape:1.0.0"
+ }, {
+ "path": "ProjectedBottomHoleLocation.SpatialGeometryTypeID",
+ "kind": "string"
+ }, {
+ "path": "WasBusinessInterestTechnical",
+ "kind": "bool"
+ }, {
+ "path": "WellboreReasonID",
+ "kind": "string"
+ }, {
+ "path": "WellboreTrajectoryTypeID",
+ "kind": "string"
+ }, {
+ "path": "PrimaryMaterialID",
+ "kind": "string"
+ }, {
+ "path": "HistoricalInterests",
+ "kind": "[]object"
+ }, {
+ "path": "StatusSummaryID",
+ "kind": "string"
+ }, {
+ "path": "ConditionID",
+ "kind": "string"
+ }, {
+ "path": "WasBusinessInterestFinancialOperated",
+ "kind": "bool"
+ }, {
+ "path": "SecondaryProductTypeID",
+ "kind": "string"
+ }, {
+ "path": "WellID",
+ "kind": "string"
+ }, {
+ "path": "TrajectoryTypeID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.SpatialParameterTypeID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.QuantitativeAccuracyBandID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.CoordinateQualityCheckRemarks",
+ "kind": "[]string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.AppliedOperations",
+ "kind": "[]string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.QualitativeSpatialAccuracyTypeID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.CoordinateQualityCheckPerformedBy",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.SpatialLocationCoordinatesDate",
+ "kind": "datetime"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.CoordinateQualityCheckDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.Wgs84Coordinates",
+ "kind": "core:dl:geoshape:1.0.0"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.SpatialGeometryTypeID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultName",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.IsDecimated",
+ "kind": "boolean"
+ }
+ ]
+}
diff --git a/indexer-core/src/test/resources/indexproperty/welllog_configuration_record.json b/indexer-core/src/test/resources/indexproperty/welllog_configuration_record.json
new file mode 100644
index 0000000000000000000000000000000000000000..2807dd86384ffad35ce5d75e09e94e414b20bd07
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/welllog_configuration_record.json
@@ -0,0 +1,100 @@
+{
+ "AttributionPublication": null,
+ "InactiveIndicator": null,
+ "Description": "The index property list for WellLog:1., valid for all WellLog kinds for major version 1.",
+ "ResourceLifecycleStatus": null,
+ "ResourceCurationStatus": null,
+ "TechnicalAssuranceID": null,
+ "Code": "osdu:wks:work-product-component--WellLog:1.",
+ "Source": null,
+ "Name": "WellLogIndex-PropertyPathConfiguration",
+ "AttributionAuthority": "OSDU",
+ "ResourceHomeRegionID": null,
+ "VirtualProperties.DefaultName": "WellLogIndex-PropertyPathConfiguration",
+ "AttributionRevision": null,
+ "ResourceSecurityClassification": null,
+ "ID": null,
+ "ExistenceKind": null,
+ "Configurations": [{
+ "Policy": "ExtractFirstMatch",
+ "UseCase": "As a user I want to discover WellLog instances by the wellbore's name value.",
+ "Paths": [{
+ "RelatedObjectsSpec.RelatedObjectID": "data.WellboreID",
+ "ValueExtraction.ValuePath": "data.VirtualProperties.DefaultName",
+ "RelatedObjectsSpec.RelatedObjectKind": "osdu:wks:master-data--Wellbore:1.",
+ "RelatedObjectsSpec.RelatedConditionProperty": null,
+ "RelatedObjectsSpec.RelationshipDirection": "ChildToParent",
+ "ValueExtraction.RelatedConditionProperty": null
+ }, {
+ "RelatedObjectsSpec.RelatedObjectID": "data.WellboreID",
+ "ValueExtraction.ValuePath": "data.FacilityName",
+ "RelatedObjectsSpec.RelatedObjectKind": "osdu:wks:master-data--Wellbore:1.",
+ "RelatedObjectsSpec.RelatedConditionProperty": null,
+ "RelatedObjectsSpec.RelationshipDirection": "ChildToParent",
+ "ValueExtraction.RelatedConditionProperty": null
+ }
+ ],
+ "Name": "WellboreName"
+ }, {
+ "Policy": "ExtractFirstMatch",
+ "UseCase": "As a user I want to discover WellLog instances by spatial location.",
+ "Paths": [{
+ "RelatedObjectsSpec.RelatedObjectID": "data.WellboreID",
+ "ValueExtraction.ValuePath": "data.VirtualProperties.DefaultLocation",
+ "RelatedObjectsSpec.RelatedObjectKind": "osdu:wks:master-data--Wellbore:1.",
+ "RelatedObjectsSpec.RelatedConditionProperty": null,
+ "RelatedObjectsSpec.RelationshipDirection": "ChildToParent",
+ "ValueExtraction.RelatedConditionProperty": null
+ }, {
+ "RelatedObjectsSpec.RelatedObjectID": "data.WellboreID",
+ "ValueExtraction.ValuePath": "data.ProjectedBottomHoleLocation",
+ "RelatedObjectsSpec.RelatedObjectKind": "osdu:wks:master-data--Wellbore:1.",
+ "RelatedObjectsSpec.RelatedConditionProperty": null,
+ "RelatedObjectsSpec.RelationshipDirection": "ChildToParent",
+ "ValueExtraction.RelatedConditionProperty": null
+ }, {
+ "RelatedObjectsSpec.RelatedObjectID": "data.WellboreID",
+ "ValueExtraction.ValuePath": "data.GeographicBottomHoleLocation",
+ "RelatedObjectsSpec.RelatedObjectKind": "osdu:wks:master-data--Wellbore:1.",
+ "RelatedObjectsSpec.RelatedConditionProperty": null,
+ "RelatedObjectsSpec.RelationshipDirection": "ChildToParent",
+ "ValueExtraction.RelatedConditionProperty": null
+ }, {
+ "RelatedObjectsSpec.RelatedObjectID": "data.WellboreID",
+ "ValueExtraction.ValuePath": "data.SpatialLocation",
+ "RelatedObjectsSpec.RelatedObjectKind": "osdu:wks:master-data--Wellbore:1.",
+ "RelatedObjectsSpec.RelatedConditionProperty": null,
+ "RelatedObjectsSpec.RelationshipDirection": "ChildToParent",
+ "ValueExtraction.RelatedConditionProperty": null
+ }
+ ],
+ "Name": "SpatialLocation"
+ }, {
+ "Policy": "ExtractAllMatches",
+ "UseCase": "As a user I want to discover WellLog instances by the Technical Assurance reviewer name.",
+ "Paths": [{
+ "RelatedObjectsSpec.RelatedObjectID": null,
+ "ValueExtraction.ValuePath": "data.TechnicalAssurances[].Reviewers[].Name",
+ "RelatedObjectsSpec.RelatedObjectKind": null,
+ "RelatedObjectsSpec.RelatedConditionProperty": null,
+ "RelatedObjectsSpec.RelationshipDirection": null,
+ "ValueExtraction.RelatedConditionProperty": null
+ }
+ ],
+ "Name": "TechnicalAssuranceReviewerNames"
+ }, {
+ "Policy": "ExtractAllMatches",
+ "UseCase": "As a user I want to discover WellLog instances by the Technical Assurance Organisation name.",
+ "Paths": [{
+ "RelatedObjectsSpec.RelatedObjectID": "data.TechnicalAssurances[].Reviewers[].OrganisationID",
+ "ValueExtraction.ValuePath": "data.OrganisationName",
+ "RelatedObjectsSpec.RelatedObjectKind": "osdu:wks:master-data--Organisation:1.",
+ "RelatedObjectsSpec.RelatedConditionProperty": null,
+ "RelatedObjectsSpec.RelationshipDirection": "ChildToParent",
+ "ValueExtraction.RelatedConditionProperty": null
+ }
+ ],
+ "Name": "TechnicalAssuranceReviewerOrganisationNames"
+ }
+ ]
+}
diff --git a/indexer-core/src/test/resources/indexproperty/welllog_extended_data.json b/indexer-core/src/test/resources/indexproperty/welllog_extended_data.json
new file mode 100644
index 0000000000000000000000000000000000000000..a4e6bece65b88730b2503aeadf8e457e404ae16a
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/welllog_extended_data.json
@@ -0,0 +1,16 @@
+{
+ "SpatialLocation.SpatialGeometryTypeID": "opendes:reference-data--SpatialGeometryType:Point:",
+ "TechnicalAssuranceReviewerOrganisationNames": ["BigOil SeismicInterpretation Department (original)", "BigOil SeismicProcessing Department (original)"],
+ "SpatialLocation.Wgs84Coordinates": {
+ "geometries": [{
+ "coordinates": [171.8745361, -41.2456122],
+ "type": "point"
+ }
+ ],
+ "type": "geometrycollection"
+ },
+ "SpatialLocation.IsDecimated": false,
+ "TechnicalAssuranceReviewerNames": ["BigOil Department SeismicInterpretation (denormalized)", "BigOil Department SeismicProcessing (denormalized)"],
+ "WellboreName": "Kongahu-1",
+ "AssociatedIdentities": ["opendes:master-data--Organisation:BigOil-Department-SeismicProcessing", "opendes:master-data--Wellbore:nz-100000113552", "opendes:master-data--Organisation:BigOil-Department-SeismicInterpretation"]
+}
diff --git a/indexer-core/src/test/resources/indexproperty/welllog_extended_schema_items.json b/indexer-core/src/test/resources/indexproperty/welllog_extended_schema_items.json
new file mode 100644
index 0000000000000000000000000000000000000000..db795359e48669299f07c0a7287ab7a440fdead1
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/welllog_extended_schema_items.json
@@ -0,0 +1,47 @@
+[{
+ "path": "WellboreName",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.SpatialParameterTypeID",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.QuantitativeAccuracyBandID",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.CoordinateQualityCheckRemarks",
+ "kind": "[]string"
+ }, {
+ "path": "SpatialLocation.AppliedOperations",
+ "kind": "[]string"
+ }, {
+ "path": "SpatialLocation.QualitativeSpatialAccuracyTypeID",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.CoordinateQualityCheckPerformedBy",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.SpatialLocationCoordinatesDate",
+ "kind": "datetime"
+ }, {
+ "path": "SpatialLocation.CoordinateQualityCheckDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "SpatialLocation.Wgs84Coordinates",
+ "kind": "core:dl:geoshape:1.0.0"
+ }, {
+ "path": "SpatialLocation.SpatialGeometryTypeID",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.IsDecimated",
+ "kind": "boolean"
+ }, {
+ "path": "TechnicalAssuranceReviewerNames",
+ "kind": "[]string"
+ }, {
+ "path": "TechnicalAssuranceReviewerOrganisationNames",
+ "kind": "[]string"
+ }, {
+ "path": "AssociatedIdentities",
+ "kind": "[]string"
+ }
+]
diff --git a/indexer-core/src/test/resources/indexproperty/welllog_original_data.json b/indexer-core/src/test/resources/indexproperty/welllog_original_data.json
new file mode 100644
index 0000000000000000000000000000000000000000..047bce6d5ca99a1bedfb180479afffcf69804c9a
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/welllog_original_data.json
@@ -0,0 +1,142 @@
+{
+ "Name": "A0_FINAL_INTER_REF_D",
+ "VirtualProperties.DefaultName": "A0_FINAL_INTER_REF_D",
+ "WellboreID": "opendes:master-data--Wellbore:nz-100000113552:",
+ "ReferenceCurveID": "DEPTH",
+ "TechnicalAssurances": [{
+ "Comment": "This is free form text from reviewer, e.g. restrictions on use",
+ "UnacceptableUsage": [{
+ "WorkflowUsage": "opendes:reference-data--WorkflowUsageType:SeismicInterpretation:",
+ "WorkflowPersona": "opendes:reference-data--WorkflowPersonaType:SeismicInterpreter:"
+ }
+ ],
+ "TechnicalAssuranceTypeID": "opendes:reference-data--TechnicalAssuranceType:Trusted:",
+ "Reviewers": [{
+ "RoleTypeID": "opendes:reference-data--ContactRoleType:ProjectManager:AccountOwner:",
+ "OrganisationID": "opendes:master-data--Organisation:BigOil-Department-SeismicProcessing:",
+ "Name": "BigOil Department SeismicProcessing (denormalized)"
+ }, {
+ "RoleTypeID": "opendes:reference-data--ContactRoleType:AccountOwner:",
+ "OrganisationID": "opendes:master-data--Organisation:BigOil-Department-SeismicInterpretation:",
+ "Name": "BigOil Department SeismicInterpretation (denormalized)"
+ }
+ ],
+ "AcceptableUsage": [{
+ "WorkflowUsage": "opendes:reference-data--WorkflowUsageType:SeismicProcessing:",
+ "WorkflowPersona": "opendes:reference-data--WorkflowPersonaType:SeismicProcessor:"
+ }
+ ],
+ "EffectiveDate": "2020-02-13T00:00:00+0000"
+ }
+ ],
+ "Curves": [{
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Measured%20Depth:",
+ "CurveID": "DEPTH",
+ "Mnemonic": "DEPTH",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:M:"
+ }, {
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Bit%20Size:",
+ "CurveID": "BS",
+ "Mnemonic": "BS",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Caliper:",
+ "CurveID": "CALI",
+ "Mnemonic": "CALI",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Bulk%20Density:",
+ "CurveID": "DENS",
+ "Mnemonic": "DENS",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:G%2FC3:"
+ }, {
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Bulk%20Density:",
+ "CurveID": "DENS_CORR",
+ "Mnemonic": "DENS_CORR",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:G%2FC3:"
+ }, {
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Bulk%20Density%20Correction:",
+ "CurveID": "DRHO",
+ "Mnemonic": "DRHO",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:G%2FC3:"
+ }, {
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Compressional%20Slowness:",
+ "CurveID": "DTC",
+ "Mnemonic": "DTC",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:US%2FF:"
+ }, {
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Gamma%20Ray:",
+ "CurveID": "GR",
+ "Mnemonic": "GR",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"
+ }, {
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Gamma%20Ray:",
+ "CurveID": "GR_CORR",
+ "Mnemonic": "GR_CORR",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"
+ }, {
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Neutron%20Porosity:",
+ "CurveID": "NEUT",
+ "Mnemonic": "NEUT",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:V%2FV:"
+ }, {
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Neutron%20Porosity:",
+ "CurveID": "NEUT_CORR",
+ "Mnemonic": "NEUT_CORR",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:V%2FV:"
+ }, {
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Photoelectric%20Factor:",
+ "CurveID": "PEF",
+ "Mnemonic": "PEF",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:B%2FE:"
+ }, {
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Deep:",
+ "CurveID": "RESD",
+ "Mnemonic": "RESD",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Deep:",
+ "CurveID": "RESD_CORR",
+ "Mnemonic": "RESD_CORR",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Tool%20Status:",
+ "CurveID": "RESS",
+ "Mnemonic": "RESS",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Tool%20Status:",
+ "CurveID": "RESS_CORR",
+ "Mnemonic": "RESS_CORR",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Spontaneous%20Potential:",
+ "CurveID": "SP",
+ "Mnemonic": "SP",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:MV:"
+ }, {
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Tension:",
+ "CurveID": "TENS",
+ "Mnemonic": "TENS",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:KGF:"
+ }
+ ]
+}
diff --git a/indexer-core/src/test/resources/indexproperty/welllog_search_records.json b/indexer-core/src/test/resources/indexproperty/welllog_search_records.json
new file mode 100644
index 0000000000000000000000000000000000000000..f2abecb03c274744dfcb3e64bddad49794b3ab6d
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/welllog_search_records.json
@@ -0,0 +1,678 @@
+[{
+ "id": "opendes:work-product-component--WellLog:9b579416f23c4f36af4a00c10657babe",
+ "kind": "osdu:wks:work-product-component--WellLog:1.2.0",
+ "data": {
+ "Name": "EC_CURVES",
+ "VirtualProperties.DefaultName": "EC_CURVES",
+ "SpatialLocation.SpatialGeometryTypeID": "opendes:reference-data--SpatialGeometryType:Point:",
+ "SpatialLocation.Wgs84Coordinates": {
+ "geometries": [{
+ "coordinates": [171.8745361, -41.2456122],
+ "type": "point"
+ }
+ ],
+ "type": "geometrycollection"
+ },
+ "AssociatedIdentities": ["opendes:master-data--Wellbore:nz-100000113552"],
+ "WellboreID": "opendes:master-data--Wellbore:nz-100000113552:",
+ "ReferenceCurveID": "DEPTH",
+ "SpatialLocation.IsDecimated": false,
+ "WellboreName": "SLB",
+ "Curves": [{
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Measured%20Depth:",
+ "CurveID": "DEPTH",
+ "Mnemonic": "DEPTH",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:M:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "CurveID": "CT",
+ "Mnemonic": "CT",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:MH%2FM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Conductivity%20-%20Flushed%20Zone:",
+ "CurveID": "CXO",
+ "Mnemonic": "CXO",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:MH%2FM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Invasion%20Diameter:",
+ "CurveID": "DI",
+ "Mnemonic": "DI",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Fracture%20Pressure:",
+ "CurveID": "FPRESS",
+ "Mnemonic": "FPRESS",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:PSI:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Formation%20Temperature%20%28Estimate%29:",
+ "CurveID": "FTEMP",
+ "Mnemonic": "FTEMP",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:DEGC:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "CurveID": "HMC_POR",
+ "Mnemonic": "HMC_POR",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "CurveID": "HMC_RES",
+ "Mnemonic": "HMC_RES",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Mud%20Resistivity:",
+ "CurveID": "RM",
+ "Mnemonic": "RM",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Mudcake%20Resistivity:",
+ "CurveID": "RMC",
+ "Mnemonic": "RMC",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Mud%20Filtrate%20Resistivity:",
+ "CurveID": "RMF",
+ "Mnemonic": "RMF",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20True%20Formation:",
+ "CurveID": "RT",
+ "Mnemonic": "RT",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Flushed%20Zone:",
+ "CurveID": "RXO",
+ "Mnemonic": "RXO",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Volumetric%20Photoelectric%20Factor:",
+ "CurveID": "U",
+ "Mnemonic": "U",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:B%2FC3:"
+ }
+ ]
+ }
+ }, {
+ "id": "opendes:work-product-component--WellLog:bee5994052564bb0ac9db531601a96aa",
+ "kind": "osdu:wks:work-product-component--WellLog:1.2.0",
+ "data": {
+ "Name": "RAW_INTER_REF_D",
+ "VirtualProperties.DefaultName": "RAW_INTER_REF_D",
+ "SpatialLocation.SpatialGeometryTypeID": "opendes:reference-data--SpatialGeometryType:Point:",
+ "SpatialLocation.Wgs84Coordinates": {
+ "geometries": [{
+ "coordinates": [171.8745361, -41.2456122],
+ "type": "point"
+ }
+ ],
+ "type": "geometrycollection"
+ },
+ "AssociatedIdentities": ["opendes:master-data--Wellbore:nz-100000113552"],
+ "WellboreID": "opendes:master-data--Wellbore:nz-100000113552:",
+ "ReferenceCurveID": "DEPTH",
+ "SpatialLocation.IsDecimated": false,
+ "WellboreName": "SLB",
+ "Curves": [{
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Measured%20Depth:",
+ "CurveID": "DEPTH",
+ "Mnemonic": "DEPTH",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:M:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Bit%20Size:",
+ "CurveID": "BS",
+ "Mnemonic": "BS",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Bit%20Size:",
+ "CurveID": "BS_2",
+ "Mnemonic": "BS_2",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Bit%20Size:",
+ "CurveID": "BS_3",
+ "Mnemonic": "BS_3",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Bit%20Size:",
+ "CurveID": "BS_4",
+ "Mnemonic": "BS_4",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Bit%20Size:",
+ "CurveID": "BS_5",
+ "Mnemonic": "BS_5",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Bit%20Size:",
+ "CurveID": "BS_6",
+ "Mnemonic": "BS_6",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Caliper:",
+ "CurveID": "CALI",
+ "Mnemonic": "CALI",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Caliper:",
+ "CurveID": "CALI_2",
+ "Mnemonic": "CALI_2",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Caliper:",
+ "CurveID": "CALI_3",
+ "Mnemonic": "CALI_3",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Caliper:",
+ "CurveID": "CALI_4",
+ "Mnemonic": "CALI_4",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Caliper:",
+ "CurveID": "CALI_5",
+ "Mnemonic": "CALI_5",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Caliper:",
+ "CurveID": "CALI_6",
+ "Mnemonic": "CALI_6",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Bulk%20Density%20Correction:",
+ "CurveID": "DRHO",
+ "Mnemonic": "DRHO",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:G%2FC3:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Compressional%20Slowness:",
+ "CurveID": "DT",
+ "Mnemonic": "DT",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:US%2FF:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Acoustic%20Slowness:",
+ "CurveID": "DT_2",
+ "Mnemonic": "DT_2",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:US%2FF:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Acoustic%20Slowness:",
+ "CurveID": "DT_3",
+ "Mnemonic": "DT_3",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:US%2FF:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Acoustic%20Slowness:",
+ "CurveID": "DT_4",
+ "Mnemonic": "DT_4",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:US%2FF:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Gamma%20Ray:",
+ "CurveID": "GR",
+ "Mnemonic": "GR",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Gamma%20Ray:",
+ "CurveID": "GR_2",
+ "Mnemonic": "GR_2",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Gamma%20Ray:",
+ "CurveID": "GR_3",
+ "Mnemonic": "GR_3",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Gamma%20Ray:",
+ "CurveID": "GR_4",
+ "Mnemonic": "GR_4",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Gamma%20Ray:",
+ "CurveID": "GR_5",
+ "Mnemonic": "GR_5",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Gamma%20Ray:",
+ "CurveID": "GR_6",
+ "Mnemonic": "GR_6",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Gamma%20Ray:",
+ "CurveID": "GR_7",
+ "Mnemonic": "GR_7",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Gamma%20Ray:",
+ "CurveID": "GR_8",
+ "Mnemonic": "GR_8",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Gamma%20Ray:",
+ "CurveID": "GR_9",
+ "Mnemonic": "GR_9",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Gamma%20Ray:",
+ "CurveID": "GR_10",
+ "Mnemonic": "GR_10",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Deep%20Laterolog:",
+ "CurveID": "LLD",
+ "Mnemonic": "LLD",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Deep%20Laterolog:",
+ "CurveID": "LLD_2",
+ "Mnemonic": "LLD_2",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Deep%20Laterolog:",
+ "CurveID": "LLD_3",
+ "Mnemonic": "LLD_3",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Deep%20Laterolog:",
+ "CurveID": "LLD_4",
+ "Mnemonic": "LLD_4",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Shallow%20Laterolog:",
+ "CurveID": "LLS",
+ "Mnemonic": "LLS",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Shallow%20Laterolog:",
+ "CurveID": "LLS_2",
+ "Mnemonic": "LLS_2",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Shallow%20Laterolog:",
+ "CurveID": "LLS_3",
+ "Mnemonic": "LLS_3",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Shallow%20Laterolog:",
+ "CurveID": "LLS_4",
+ "Mnemonic": "LLS_4",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Micro%20Inverse:",
+ "CurveID": "MINV",
+ "Mnemonic": "MINV",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Micro%20Inverse:",
+ "CurveID": "MINV_2",
+ "Mnemonic": "MINV_2",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Micro%20Inverse:",
+ "CurveID": "MINV_3",
+ "Mnemonic": "MINV_3",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Micro%20Inverse:",
+ "CurveID": "MINV_4",
+ "Mnemonic": "MINV_4",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Micro%20Normal:",
+ "CurveID": "MNOR",
+ "Mnemonic": "MNOR",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Micro%20Normal:",
+ "CurveID": "MNOR_2",
+ "Mnemonic": "MNOR_2",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Micro%20Normal:",
+ "CurveID": "MNOR_3",
+ "Mnemonic": "MNOR_3",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Micro%20Normal:",
+ "CurveID": "MNOR_4",
+ "Mnemonic": "MNOR_4",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Micro%20Laterolog:",
+ "CurveID": "MSFL",
+ "Mnemonic": "MSFL",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Micro%20Laterolog:",
+ "CurveID": "MSFL_2",
+ "Mnemonic": "MSFL_2",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Micro%20Laterolog:",
+ "CurveID": "MSFL_3",
+ "Mnemonic": "MSFL_3",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Micro%20Laterolog:",
+ "CurveID": "MSFL_4",
+ "Mnemonic": "MSFL_4",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Thermal%20Neutron%20Porosity:",
+ "CurveID": "NPHI",
+ "Mnemonic": "NPHI",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:V%2FV:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Photoelectric%20Factor:",
+ "CurveID": "PEF",
+ "Mnemonic": "PEF",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:B%2FE:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Bulk%20Density:",
+ "CurveID": "RHOB",
+ "Mnemonic": "RHOB",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:G%2FC3:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Spontaneous%20Potential:",
+ "CurveID": "SP",
+ "Mnemonic": "SP",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:MV:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Spontaneous%20Potential:",
+ "CurveID": "SP_2",
+ "Mnemonic": "SP_2",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:MV:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Spontaneous%20Potential:",
+ "CurveID": "SP_3",
+ "Mnemonic": "SP_3",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:MV:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Spontaneous%20Potential:",
+ "CurveID": "SP_4",
+ "Mnemonic": "SP_4",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:MV:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Tension:",
+ "CurveID": "TENS",
+ "Mnemonic": "TENS",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:KGF:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Tension:",
+ "CurveID": "TENS_2",
+ "Mnemonic": "TENS_2",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:KGF:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Tension:",
+ "CurveID": "TENS_3",
+ "Mnemonic": "TENS_3",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:KGF:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Tension:",
+ "CurveID": "TENS_4",
+ "Mnemonic": "TENS_4",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:KGF:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Tension:",
+ "CurveID": "TENS_5",
+ "Mnemonic": "TENS_5",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:KGF:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Tension:",
+ "CurveID": "TENS_6",
+ "Mnemonic": "TENS_6",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:KGF:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Tension:",
+ "CurveID": "TENS_7",
+ "Mnemonic": "TENS_7",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:KGF:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Tension:",
+ "CurveID": "TENS_8",
+ "Mnemonic": "TENS_8",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:KGF:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Tension:",
+ "CurveID": "TENS_9",
+ "Mnemonic": "TENS_9",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:KGF:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Tension:",
+ "CurveID": "TENS_10",
+ "Mnemonic": "TENS_10",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:KGF:"
+ }
+ ]
+ }
+ }, {
+ "id": "opendes:work-product-component--WellLog:23b7dfde2c1349d58a0f97ae78bff9df",
+ "kind": "osdu:wks:work-product-component--WellLog:1.2.0",
+ "data": {
+ "Name": "A0_FINAL_INTER_REF_D",
+ "VirtualProperties.DefaultName": "A0_FINAL_INTER_REF_D",
+ "TechnicalAssuranceReviewerNames": ["BigOil Department SeismicInterpretation (denormalized)", "BigOil Department SeismicProcessing (denormalized)"],
+ "SpatialLocation.SpatialGeometryTypeID": "opendes:reference-data--SpatialGeometryType:Point:",
+ "SpatialLocation.Wgs84Coordinates": {
+ "geometries": [{
+ "coordinates": [171.8745361, -41.2456122],
+ "type": "point"
+ }
+ ],
+ "type": "geometrycollection"
+ },
+ "AssociatedIdentities": ["opendes:master-data--Organisation:BigOil-Department-SeismicProcessing", "opendes:master-data--Wellbore:nz-100000113552", "opendes:master-data--Organisation:BigOil-Department-SeismicInterpretation"],
+ "WellboreID": "opendes:master-data--Wellbore:nz-100000113552:",
+ "ReferenceCurveID": "DEPTH",
+ "SpatialLocation.IsDecimated": false,
+ "WellboreName": "Kongahu-1",
+ "TechnicalAssurances": [{
+ "Comment": "This is free form text from reviewer, e.g. restrictions on use",
+ "UnacceptableUsage": [{
+ "WorkflowUsage": "opendes:reference-data--WorkflowUsageType:SeismicInterpretation:",
+ "WorkflowPersona": "opendes:reference-data--WorkflowPersonaType:SeismicInterpreter:"
+ }
+ ],
+ "TechnicalAssuranceTypeID": "opendes:reference-data--TechnicalAssuranceType:Trusted:",
+ "Reviewers": [{
+ "RoleTypeID": "opendes:reference-data--ContactRoleType:ProjectManager:AccountOwner:",
+ "OrganisationID": "opendes:master-data--Organisation:BigOil-Department-SeismicProcessing:",
+ "Name": "BigOil Department SeismicProcessing (denormalized)"
+ }, {
+ "RoleTypeID": "opendes:reference-data--ContactRoleType:AccountOwner:",
+ "OrganisationID": "opendes:master-data--Organisation:BigOil-Department-SeismicInterpretation:",
+ "Name": "BigOil Department SeismicInterpretation (denormalized)"
+ }
+ ],
+ "AcceptableUsage": [{
+ "WorkflowUsage": "opendes:reference-data--WorkflowUsageType:SeismicProcessing:",
+ "WorkflowPersona": "opendes:reference-data--WorkflowPersonaType:SeismicProcessor:"
+ }
+ ],
+ "EffectiveDate": "2020-02-13T00:00:00+0000"
+ }
+ ],
+ "TechnicalAssuranceReviewerOrganisationNames": ["BigOil SeismicInterpretation Department (original)", "BigOil SeismicProcessing Department (original)"],
+ "Curves": [{
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Measured%20Depth:",
+ "CurveID": "DEPTH",
+ "Mnemonic": "DEPTH",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:M:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Bit%20Size:",
+ "CurveID": "BS",
+ "Mnemonic": "BS",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Caliper:",
+ "CurveID": "CALI",
+ "Mnemonic": "CALI",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Bulk%20Density:",
+ "CurveID": "DENS",
+ "Mnemonic": "DENS",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:G%2FC3:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Bulk%20Density:",
+ "CurveID": "DENS_CORR",
+ "Mnemonic": "DENS_CORR",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:G%2FC3:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Bulk%20Density%20Correction:",
+ "CurveID": "DRHO",
+ "Mnemonic": "DRHO",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:G%2FC3:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Compressional%20Slowness:",
+ "CurveID": "DTC",
+ "Mnemonic": "DTC",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:US%2FF:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Gamma%20Ray:",
+ "CurveID": "GR",
+ "Mnemonic": "GR",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Gamma%20Ray:",
+ "CurveID": "GR_CORR",
+ "Mnemonic": "GR_CORR",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Neutron%20Porosity:",
+ "CurveID": "NEUT",
+ "Mnemonic": "NEUT",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:V%2FV:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Neutron%20Porosity:",
+ "CurveID": "NEUT_CORR",
+ "Mnemonic": "NEUT_CORR",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:V%2FV:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Photoelectric%20Factor:",
+ "CurveID": "PEF",
+ "Mnemonic": "PEF",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:B%2FE:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Deep:",
+ "CurveID": "RESD",
+ "Mnemonic": "RESD",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Deep:",
+ "CurveID": "RESD_CORR",
+ "Mnemonic": "RESD_CORR",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Tool%20Status:",
+ "CurveID": "RESS",
+ "Mnemonic": "RESS",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Tool%20Status:",
+ "CurveID": "RESS_CORR",
+ "Mnemonic": "RESS_CORR",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Spontaneous%20Potential:",
+ "CurveID": "SP",
+ "Mnemonic": "SP",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:MV:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Tension:",
+ "CurveID": "TENS",
+ "Mnemonic": "TENS",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:KGF:"
+ }
+ ]
+ }
+ }
+]
diff --git a/indexer-core/src/test/resources/indexproperty/welllog_storage_schema.json b/indexer-core/src/test/resources/indexproperty/welllog_storage_schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..734ecd5c6e3954ec57ee15db09478623519bab4e
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/welllog_storage_schema.json
@@ -0,0 +1,449 @@
+{
+ "kind": "osdu:wks:work-product-component--WellLog:1.2.0",
+ "schema": [{
+ "path": "ResourceHomeRegionID",
+ "kind": "string"
+ }, {
+ "path": "ResourceHostRegionIDs",
+ "kind": "[]string"
+ }, {
+ "path": "ResourceLifecycleStatus",
+ "kind": "string"
+ }, {
+ "path": "ResourceSecurityClassification",
+ "kind": "string"
+ }, {
+ "path": "ResourceCurationStatus",
+ "kind": "string"
+ }, {
+ "path": "ExistenceKind",
+ "kind": "string"
+ }, {
+ "path": "TechnicalAssuranceID",
+ "kind": "string"
+ }, {
+ "path": "Source",
+ "kind": "string"
+ }, {
+ "path": "Datasets",
+ "kind": "[]string"
+ }, {
+ "path": "IsDiscoverable",
+ "kind": "bool"
+ }, {
+ "path": "TechnicalAssurances",
+ "kind": "nested",
+ "properties": [{
+ "path": "Comment",
+ "kind": "string"
+ }, {
+ "path": "Reviewers",
+ "kind": "[]object"
+ }, {
+ "path": "UnacceptableUsage",
+ "kind": "flattened"
+ }, {
+ "path": "AcceptableUsage",
+ "kind": "flattened"
+ }, {
+ "path": "TechnicalAssuranceTypeID",
+ "kind": "string"
+ }, {
+ "path": "EffectiveDate",
+ "kind": "datetime"
+ }
+ ]
+ }, {
+ "path": "Artefacts",
+ "kind": "flattened"
+ }, {
+ "path": "IsExtendedLoad",
+ "kind": "bool"
+ }, {
+ "path": "SpatialArea.SpatialParameterTypeID",
+ "kind": "string"
+ }, {
+ "path": "SpatialArea.QuantitativeAccuracyBandID",
+ "kind": "string"
+ }, {
+ "path": "SpatialArea.CoordinateQualityCheckRemarks",
+ "kind": "[]string"
+ }, {
+ "path": "SpatialArea.AppliedOperations",
+ "kind": "[]string"
+ }, {
+ "path": "SpatialArea.QualitativeSpatialAccuracyTypeID",
+ "kind": "string"
+ }, {
+ "path": "SpatialArea.CoordinateQualityCheckPerformedBy",
+ "kind": "string"
+ }, {
+ "path": "SpatialArea.SpatialLocationCoordinatesDate",
+ "kind": "datetime"
+ }, {
+ "path": "SpatialArea.CoordinateQualityCheckDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "SpatialArea.Wgs84Coordinates",
+ "kind": "core:dl:geoshape:1.0.0"
+ }, {
+ "path": "SpatialArea.SpatialGeometryTypeID",
+ "kind": "string"
+ }, {
+ "path": "Description",
+ "kind": "string"
+ }, {
+ "path": "CreationDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "BusinessActivities",
+ "kind": "[]string"
+ }, {
+ "path": "SpatialPoint.SpatialParameterTypeID",
+ "kind": "string"
+ }, {
+ "path": "SpatialPoint.QuantitativeAccuracyBandID",
+ "kind": "string"
+ }, {
+ "path": "SpatialPoint.CoordinateQualityCheckRemarks",
+ "kind": "[]string"
+ }, {
+ "path": "SpatialPoint.AppliedOperations",
+ "kind": "[]string"
+ }, {
+ "path": "SpatialPoint.QualitativeSpatialAccuracyTypeID",
+ "kind": "string"
+ }, {
+ "path": "SpatialPoint.CoordinateQualityCheckPerformedBy",
+ "kind": "string"
+ }, {
+ "path": "SpatialPoint.SpatialLocationCoordinatesDate",
+ "kind": "datetime"
+ }, {
+ "path": "SpatialPoint.CoordinateQualityCheckDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "SpatialPoint.Wgs84Coordinates",
+ "kind": "core:dl:geoshape:1.0.0"
+ }, {
+ "path": "SpatialPoint.SpatialGeometryTypeID",
+ "kind": "string"
+ }, {
+ "path": "GeoContexts",
+ "kind": "nested",
+ "properties": [{
+ "path": "GeoPoliticalEntityID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "BasinID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "FieldID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "PlayID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "ProspectID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }
+ ]
+ }, {
+ "path": "AuthorIDs",
+ "kind": "[]string"
+ }, {
+ "path": "SubmitterName",
+ "kind": "string"
+ }, {
+ "path": "LineageAssertions",
+ "kind": "flattened"
+ }, {
+ "path": "Tags",
+ "kind": "[]string"
+ }, {
+ "path": "Name",
+ "kind": "string"
+ }, {
+ "path": "LogRemark",
+ "kind": "string"
+ }, {
+ "path": "SamplingStop",
+ "kind": "double"
+ }, {
+ "path": "CompanyID",
+ "kind": "string"
+ }, {
+ "path": "TopMeasuredDepth",
+ "kind": "double"
+ }, {
+ "path": "ServiceCompanyID",
+ "kind": "string"
+ }, {
+ "path": "SamplingInterval",
+ "kind": "double"
+ }, {
+ "path": "LogVersion",
+ "kind": "string"
+ }, {
+ "path": "LogSource",
+ "kind": "string"
+ }, {
+ "path": "ToolStringDescription",
+ "kind": "string"
+ }, {
+ "path": "DrillingFluidProperty",
+ "kind": "string"
+ }, {
+ "path": "IsRegular",
+ "kind": "bool"
+ }, {
+ "path": "LogServiceDateInterval.StartDate",
+ "kind": "datetime"
+ }, {
+ "path": "LogServiceDateInterval.EndDate",
+ "kind": "datetime"
+ }, {
+ "path": "HoleTypeLogging",
+ "kind": "string"
+ }, {
+ "path": "LoggingDirection",
+ "kind": "string"
+ }, {
+ "path": "SamplingStart",
+ "kind": "double"
+ }, {
+ "path": "WellboreID",
+ "kind": "string"
+ }, {
+ "path": "PassNumber",
+ "kind": "int"
+ }, {
+ "path": "ActivityType",
+ "kind": "string"
+ }, {
+ "path": "VerticalMeasurementID",
+ "kind": "string"
+ }, {
+ "path": "SamplingDomainTypeID",
+ "kind": "string"
+ }, {
+ "path": "ReferenceCurveID",
+ "kind": "string"
+ }, {
+ "path": "CandidateReferenceCurveIDs",
+ "kind": "[]string"
+ }, {
+ "path": "SeismicReferenceElevation.WellboreTVDTrajectoryID",
+ "kind": "string"
+ }, {
+ "path": "SeismicReferenceElevation.VerticalCRSID",
+ "kind": "string"
+ }, {
+ "path": "SeismicReferenceElevation.VerticalMeasurementSourceID",
+ "kind": "string"
+ }, {
+ "path": "SeismicReferenceElevation.VerticalReferenceID",
+ "kind": "string"
+ }, {
+ "path": "SeismicReferenceElevation.TerminationDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "SeismicReferenceElevation.VerticalMeasurementPathID",
+ "kind": "string"
+ }, {
+ "path": "SeismicReferenceElevation.EffectiveDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "SeismicReferenceElevation.VerticalMeasurement",
+ "kind": "double"
+ }, {
+ "path": "SeismicReferenceElevation.VerticalMeasurementTypeID",
+ "kind": "string"
+ }, {
+ "path": "SeismicReferenceElevation.VerticalMeasurementDescription",
+ "kind": "string"
+ }, {
+ "path": "SeismicReferenceElevation.VerticalMeasurementUnitOfMeasureID",
+ "kind": "string"
+ }, {
+ "path": "SeismicReferenceElevation.VerticalReferenceEntityID",
+ "kind": "string"
+ }, {
+ "path": "WellLogTypeID",
+ "kind": "string"
+ }, {
+ "path": "FrameIdentifier",
+ "kind": "string"
+ }, {
+ "path": "LoggingService",
+ "kind": "string"
+ }, {
+ "path": "BottomMeasuredDepth",
+ "kind": "double"
+ }, {
+ "path": "VerticalMeasurement.WellboreTVDTrajectoryID",
+ "kind": "string"
+ }, {
+ "path": "VerticalMeasurement.VerticalCRSID",
+ "kind": "string"
+ }, {
+ "path": "VerticalMeasurement.VerticalMeasurementSourceID",
+ "kind": "string"
+ }, {
+ "path": "VerticalMeasurement.VerticalReferenceID",
+ "kind": "string"
+ }, {
+ "path": "VerticalMeasurement.TerminationDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "VerticalMeasurement.VerticalMeasurementPathID",
+ "kind": "string"
+ }, {
+ "path": "VerticalMeasurement.EffectiveDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "VerticalMeasurement.VerticalMeasurement",
+ "kind": "double"
+ }, {
+ "path": "VerticalMeasurement.VerticalMeasurementTypeID",
+ "kind": "string"
+ }, {
+ "path": "VerticalMeasurement.VerticalMeasurementDescription",
+ "kind": "string"
+ }, {
+ "path": "VerticalMeasurement.VerticalMeasurementUnitOfMeasureID",
+ "kind": "string"
+ }, {
+ "path": "VerticalMeasurement.VerticalReferenceEntityID",
+ "kind": "string"
+ }, {
+ "path": "LogRun",
+ "kind": "string"
+ }, {
+ "path": "ZeroTime",
+ "kind": "datetime"
+ }, {
+ "path": "LogActivity",
+ "kind": "string"
+ }, {
+ "path": "Curves",
+ "kind": "nested",
+ "properties": [{
+ "path": "IsProcessed",
+ "kind": "bool"
+ }, {
+ "path": "LogCurveMainFamilyID",
+ "kind": "string"
+ }, {
+ "path": "DateStamp",
+ "kind": "datetime"
+ }, {
+ "path": "NumberOfColumns",
+ "kind": "int"
+ }, {
+ "path": "LogCurveFamilyID",
+ "kind": "string"
+ }, {
+ "path": "CurveID",
+ "kind": "string"
+ }, {
+ "path": "TopDepth",
+ "kind": "double"
+ }, {
+ "path": "CurveVersion",
+ "kind": "string"
+ }, {
+ "path": "CurveSampleTypeID",
+ "kind": "string"
+ }, {
+ "path": "InterpreterName",
+ "kind": "string"
+ }, {
+ "path": "CurveQuality",
+ "kind": "string"
+ }, {
+ "path": "NullValue",
+ "kind": "bool"
+ }, {
+ "path": "Interpolate",
+ "kind": "bool"
+ }, {
+ "path": "DepthUnit",
+ "kind": "string"
+ }, {
+ "path": "DepthCoding",
+ "kind": "string"
+ }, {
+ "path": "Mnemonic",
+ "kind": "string"
+ }, {
+ "path": "BaseDepth",
+ "kind": "double"
+ }, {
+ "path": "LogCurveTypeID",
+ "kind": "string"
+ }, {
+ "path": "LogCurveBusinessValueID",
+ "kind": "string"
+ }, {
+ "path": "CurveUnit",
+ "kind": "string"
+ }, {
+ "path": "CurveDescription",
+ "kind": "string"
+ }
+ ]
+ }, {
+ "path": "VirtualProperties.DefaultLocation.SpatialParameterTypeID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.QuantitativeAccuracyBandID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.CoordinateQualityCheckRemarks",
+ "kind": "[]string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.AppliedOperations",
+ "kind": "[]string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.QualitativeSpatialAccuracyTypeID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.CoordinateQualityCheckPerformedBy",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.SpatialLocationCoordinatesDate",
+ "kind": "datetime"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.CoordinateQualityCheckDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.Wgs84Coordinates",
+ "kind": "core:dl:geoshape:1.0.0"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.SpatialGeometryTypeID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultName",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.IsDecimated",
+ "kind": "boolean"
+ }
+ ]
+}
diff --git a/provider/indexer-aws/README.md b/provider/indexer-aws/README.md
index 3c2ad5d3ac00b155f30c9cf2fcd2bc7b77b2d9b9..01b9d492c12801e3b6adfb002eabff7b3ae2a3d2 100644
--- a/provider/indexer-aws/README.md
+++ b/provider/indexer-aws/README.md
@@ -123,27 +123,28 @@ You should see in the logs that pop up what url and port it runs on. By default
export ELASTIC_PASSWORD=$ELASTIC_PASSWORD
export ELASTIC_USER_NAME=$ELASTIC_USERNAME
- | name | example value | description | sensitive?
- | --- | --- | --- | --- |
+ | name | example value | description | sensitive?
+ | --- |----------------------------------------------------------------------------------------| --- | --- |
| `AWS_ACCESS_KEY_ID` | `ASIAXXXXXXXXXXXXXX` | The AWS Access Key for a user with access to Backend Resources required by the service | yes |
| `AWS_SECRET_ACCESS_KEY` | `super-secret-key==` | The AWS Secret Key for a user with access to Backend Resources required by the service | yes |
- | `AWS_SESSION_TOKEN` | `session-token-xxxxxxxxx` | AWS Session token needed if using an SSO user session to authenticate | yes |
- | `AWS_COGNITO_USER_POOL_ID` | `us-east-1_xxxxxxxx` | User Pool Id for the reference cognito | no |
- | `AWS_COGNITO_CLIENT_ID` | `xxxxxxxxxxxx` | Client ID for the Auth Flow integrated with the Cognito User Pool | no |
- | `AWS_COGNITO_AUTH_FLOW` | `USER_PASSWORD_AUTH` | Auth flow used by reference cognito deployment | no |
- | `DEFAULT_DATA_PARTITION_ID_TENANT1` | `opendes` | Partition used to create and index record | no |
- | `DEFAULT_DATA_PARTITION_ID_TENANT2` | `common` | Another needed partition| no |
- | `AWS_COGNITO_AUTH_PARAMS_USER` | `int-test-user@testing.com` | Int Test Username | no |
- | `AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS` | `noaccess@testing.com` | No Access Username | no |
- | `AWS_COGNITO_AUTH_PARAMS_PASSWORD` | `some-secure-password` | Int Test User/NoAccessUser Password | yes |
- | `ENTITLEMENTS_DOMAIN` | `example.com` | Domain for user's groups | no |
- | `OTHER_RELEVANT_DATA_COUNTRIES` | `US` | Used to create demo legal tag | no |
- | `STORAGE_HOST` | `http://localhost:8080/api/storage/v2/` | The url where the storage API is hosted | no |
- | `HOST` | `http://localhost:8080` | Base url for deployment | no |
- | `ELASTIC_HOST` | `localhost` | Url for elasticsearch | no |
- | `ELASTIC_PORT` | `9300` | Port for elasticsearch | no |
- | `ELASTIC_PASSWORD` | `xxxxxxxxxxxxxxx` | Password for user to access elasticsearch | yes |
- | `ELASTIC_USER_NAME` | `xxxxxxxxxxxxxxxx` | Username for user to access elasticsearch | yes |
+ | `AWS_SESSION_TOKEN` | `session-token-xxxxxxxxx` | AWS Session token needed if using an SSO user session to authenticate | yes |
+ | `AWS_COGNITO_USER_POOL_ID` | `us-east-1_xxxxxxxx` | User Pool Id for the reference cognito | no |
+ | `AWS_COGNITO_CLIENT_ID` | `xxxxxxxxxxxx` | Client ID for the Auth Flow integrated with the Cognito User Pool | no |
+ | `AWS_COGNITO_AUTH_FLOW` | `USER_PASSWORD_AUTH` | Auth flow used by reference cognito deployment | no |
+ | `DEFAULT_DATA_PARTITION_ID_TENANT1` | `opendes` | Partition used to create and index record | no |
+ | `DEFAULT_DATA_PARTITION_ID_TENANT2` | `common` | Another needed partition | no |
+ | `AWS_COGNITO_AUTH_PARAMS_USER` | `int-test-user@testing.com` | Int Test Username | no |
+ | `AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS` | `noaccess@testing.com` | No Access Username | no |
+ | `AWS_COGNITO_AUTH_PARAMS_PASSWORD` | `some-secure-password` | Int Test User/NoAccessUser Password | yes |
+ | `ENTITLEMENTS_DOMAIN` | `example.com` | Domain for user's groups | no |
+ | `OTHER_RELEVANT_DATA_COUNTRIES` | `US` | Used to create demo legal tag | no |
+ | `STORAGE_HOST` | `http://localhost:8080/api/storage/v2/` | The url where the storage API is hosted | no |
+ | `HOST` | `http://localhost:8080` | Base url for deployment | no |
+ | `ELASTIC_HOST` | `localhost` | Url for elasticsearch | no |
+ | `ELASTIC_PORT` | `9300` | Port for elasticsearch | no |
+ | `ELASTIC_PASSWORD` | `xxxxxxxxxxxxxxx` | Password for user to access elasticsearch | yes |
+ | `ELASTIC_USER_NAME` | `xxxxxxxxxxxxxxxx` | Username for user to access elasticsearch | yes |
+ | `CUCUMBER_OPTIONS` | `--tags '~@indexer-extended'` OR `--tags '~@* and @indexer-extended'` | By default `--tags '~@* and @indexer-extended'` to enable experimental feature testing | no |
**Creating a new user to use for integration tests**
@@ -199,4 +200,4 @@ Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
-limitations under the License.
\ No newline at end of file
+limitations under the License.
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java
index 0944a9091c2ac6533ac5b26c88d06fec12808c70..dbc4e36fb84a4f8b2d4a2a449e61174610f94aea 100644
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java
@@ -23,6 +23,7 @@ import org.opengroup.osdu.core.aws.ssm.K8sLocalParameterProvider;
import org.opengroup.osdu.core.aws.ssm.K8sParameterNotFoundException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
+import org.opengroup.osdu.indexer.model.Constants;
import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Primary;
@@ -134,6 +135,7 @@ public class IndexerQueueTaskBuilderAws extends IndexerQueueTaskBuilder {
retryCount = 1;
retryDelay = INITIAL_RETRY_DELAY_SECONDS;
}
+
System.out.println("Re-queuing for retry attempt #: " + retryCount);
System.out.println("Delay (in seconds) before next retry: " + retryDelay);
@@ -142,6 +144,12 @@ public class IndexerQueueTaskBuilderAws extends IndexerQueueTaskBuilder {
.withDataType("String")
.withStringValue(String.valueOf(retryCount))
);
+ // Append the ancestry kinds used to prevent circular chasing
+ if(message.getAttributes().containsKey(Constants.ANCESTRY_KINDS)) {
+ messageAttributes.put(Constants.ANCESTRY_KINDS, new MessageAttributeValue()
+ .withDataType("String")
+ .withStringValue(message.getAttributes().get(Constants.ANCESTRY_KINDS)));
+ }
// Send a message with an attribute and a delay
final SendMessageRequest sendMessageRequest ;
diff --git a/provider/indexer-aws/src/main/resources/application.properties b/provider/indexer-aws/src/main/resources/application.properties
index 6529bf15accf2f4b6045d51c5d76de14f7620f7a..a0d1a419f6a6aa71128963495390ae748e7e9649 100644
--- a/provider/indexer-aws/src/main/resources/application.properties
+++ b/provider/indexer-aws/src/main/resources/application.properties
@@ -24,6 +24,7 @@ GAE_SERVICE=indexer
#reusing STORAGE_BASE_URL variable here as the base url to point to schema service
SCHEMA_HOST=${SCHEMA_BASE_URL}/api/schema-service/v1/schema
+SEARCH_HOST=${SEARCH_BASE_URL}/api/search/v2
PARTITION_PATH=/api/partition/v1
PARTITION_API=${PARTITION_BASE_URL}${PARTITION_PATH}
diff --git a/provider/indexer-azure/README.md b/provider/indexer-azure/README.md
index cb13417cb40902bdfa6cea76f058604ad25bae21..03a4a114ea9dad4923f4289cba4b59f152c0acd8 100644
--- a/provider/indexer-azure/README.md
+++ b/provider/indexer-azure/README.md
@@ -43,6 +43,7 @@ az keyvault secret show --vault-name $KEY_VAULT_NAME --name $KEY_VAULT_SECRET_NA
| `server.servlet.contextPath` | `/api/indexer/v2/` | Servlet context path | no | - |
| `schema_service_url` | ex `https://schema.azurewebsites.net` | Endpoint of schema service | no | output of infrastructure deployments |
| `SCHEMA_HOST` | `${schema_service_url}/schema` | Endpoint of schema API | no | - |
+| `SEARCH_HOST` | `${search_service_endpoint}` | Endpoint of search API | no | - |
| `storage_service_url` | ex `https://storage.azurewebsites.net` | Endpoint of storage service | no | output of infrastructure deployments |
| `STORAGE_SCHEMA_HOST` | `${storage_service_url}/schemas` | Endpoint of schema API | no | - |
| `STORAGE_QUERY_RECORD_HOST` | `${storage_service_url}/query/records` | Endpoint of records API | no | - |
@@ -79,10 +80,12 @@ az keyvault secret show --vault-name $KEY_VAULT_NAME --name $KEY_VAULT_SECRET_NA
| `DEFAULT_DATA_PARTITION_ID_TENANT2` | ex `common` | Secondary data partition for queries | no | Data in search index |
| `STORAGE_HOST` | ex `https://storage.azurewebsites.net/` | Storage service endpoint | no | output of infrastructure deployment |
| `SCHEMA_HOST` | ex `https://schema.azurewebsites.net/` | Endpoint of schema API | no | - |
+| `SEARCH_HOST` | ex `https://search.azurewebsites.net/` | Endpoint of search API | no | - |
| `ENVIRONMENT` | `CLOUD` | Deployment environment | no | - |
| `ENTITLEMENTS_DOMAIN` | `contoso.com` | OSDU R2 service domain | no | - |
| `LEGAL_TAG` | `opendes-public-usa-dataset-7643990` | Legal tag used for test records | no | Needs to be in DB. The referenced tag should already exist. |
| `OTHER_RELEVANT_DATA_COUNTRIES` | `US` | ? | no | - |
+| `CUCUMBER_OPTIONS` | `--tags '~@indexer-extended'` OR `--tags '~@* and @indexer-extended'` | By default `--tags '~@* and @indexer-extended'` to enable experimental feature testing | no | - |
### Configure Maven
diff --git a/provider/indexer-azure/pom.xml b/provider/indexer-azure/pom.xml
index fa2638d805b7a5b3568288702c9f2a89641b2f9d..9be42bd821dc93e11175b100279ab6a482b765e0 100644
--- a/provider/indexer-azure/pom.xml
+++ b/provider/indexer-azure/pom.xml
@@ -42,7 +42,7 @@
<indexer-core.version>0.22.0-SNAPSHOT</indexer-core.version>
<spring-security-jwt.version>1.1.1.RELEASE</spring-security-jwt.version>
<osdu.corelibazure.version>0.20.0-rc5</osdu.corelibazure.version>
- <os-core-common.version>0.19.0-rc6</os-core-common.version>
+ <os-core-common.version>0.21.0-rc4</os-core-common.version>
<reactor-netty.version>0.9.12.RELEASE</reactor-netty.version>
<java-jwt.version>3.8.1</java-jwt.version>
<powermock.version>2.0.2</powermock.version>
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java
index 639f6e4a0e7fbf9b2bd56656d68e593df70b18c7..cf24317df9db9a6316ec7a399c6fd0b515728175 100644
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java
+++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java
@@ -33,6 +33,7 @@ import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
import org.opengroup.osdu.indexer.azure.di.PublisherConfig;
import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.opengroup.osdu.indexer.model.Constants;
import org.opengroup.osdu.indexer.service.StorageService;
import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
import org.springframework.beans.factory.annotation.Autowired;
@@ -161,6 +162,7 @@ public class IndexerQueueTaskBuilderAzure extends IndexerQueueTaskBuilder {
// data
List<RecordInfo> recordInfos = parseRecordsAsJSON(receivedPayload.getData());
+ Map<String, String> attributes = receivedPayload.getAttributes();
// add all to body {"message": {"data":[], "id":...}}
JsonObject jo = new JsonObject();
@@ -168,6 +170,10 @@ public class IndexerQueueTaskBuilderAzure extends IndexerQueueTaskBuilder {
jo.addProperty(DpsHeaders.ACCOUNT_ID, headers.getPartitionIdWithFallbackToAccountId());
jo.addProperty(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
jo.addProperty(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
+ // Append the ancestry kinds used to prevent circular chasing
+ if(attributes != null && attributes.containsKey(Constants.ANCESTRY_KINDS)) {
+ jo.addProperty(Constants.ANCESTRY_KINDS, attributes.get(Constants.ANCESTRY_KINDS));
+ }
JsonObject jomsg = new JsonObject();
jomsg.add("message", jo);
diff --git a/provider/indexer-azure/src/main/resources/application.properties b/provider/indexer-azure/src/main/resources/application.properties
index d2dd19ac5bc19b00222901d71b828f970198f90a..2e870669aa046452a3d94e847578089fd3a3afe6 100644
--- a/provider/indexer-azure/src/main/resources/application.properties
+++ b/provider/indexer-azure/src/main/resources/application.properties
@@ -48,6 +48,7 @@ STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=${storage_service_url}/query/records:ba
STORAGE_RECORDS_BATCH_SIZE=20
STORAGE_RECORDS_BY_KIND_BATCH_SIZE=1000
+SEARCH_HOST=${search_service_url}
INDEXER_QUEUE_HOST=http://127.0.0.1:9000
diff --git a/provider/indexer-gc/docs/anthos/README.md b/provider/indexer-gc/docs/anthos/README.md
index fd4b4c3bda463cd66d0abf591803f577fade610a..bcbe2b3562471fa3270cfb906a1a36e6a8fa9381 100644
--- a/provider/indexer-gc/docs/anthos/README.md
+++ b/provider/indexer-gc/docs/anthos/README.md
@@ -233,24 +233,25 @@ Give `client-id` and `client-secret` to services, which should be authorized wit
You will need to have the following environment variables defined.
-| name | value | description | sensitive? | source |
-|--------------------------------------|-----------------------------------------------------------------|---------------------------------------------------------------------------------------------------|-----------------------------------------|-------------------------------------|
-| `ELASTIC_PASSWORD` | `********` | Password for Elasticsearch | yes | output of infrastructure deployment |
-| `ELASTIC_USER_NAME` | `********` | User name for Elasticsearch | yes | output of infrastructure deployment |
-| `ELASTIC_HOST` | ex `elastic.domain.com` | Host Elasticsearch | yes | output of infrastructure deployment |
-| `ELASTIC_PORT` | ex `9243` | Port Elasticsearch | yes | output of infrastructure deployment |
-| `INDEXER_HOST` | ex `https://os-indexer-dot-opendes.appspot.com/api/indexer/v2/` | Indexer API endpoint | no | output of infrastructure deployment |
-| `GROUP_ID` | ex `opendes-gcp.projects.com` | OSDU R2 to run tests under | no | - |
-| `OTHER_RELEVANT_DATA_COUNTRIES` | ex `US` | valid legal tag with a other relevant data countries | no | - |
-| `LEGAL_TAG` | ex `opendes-demo-legaltag` | valid legal tag with a other relevant data countries from `DEFAULT_OTHER_RELEVANT_DATA_COUNTRIES` | no | - |
-| `DEFAULT_DATA_PARTITION_ID_TENANT1` | ex `opendes` | HTTP Header 'Data-Partition-ID' | no | - |
-| `DEFAULT_DATA_PARTITION_ID_TENANT2` | ex `opendes` | HTTP Header 'Data-Partition-ID' | no | - |
-| `SEARCH_HOST` | ex `http://localhost:8080/api/search/v2/` | Endpoint of search service | no | - |
-| `STORAGE_HOST` | ex `http://os-storage-dot-opendes.appspot.com/api/storage/v2/` | Storage API endpoint | no | output of infrastructure deployment |
-| `SECURITY_HTTPS_CERTIFICATE_TRUST` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment |
-| `TEST_OPENID_PROVIDER_CLIENT_ID` | `********` | Client Id for `$INTEGRATION_TESTER` | yes | -- |
-| `TEST_OPENID_PROVIDER_CLIENT_SECRET` | `********` | | Client secret for `$INTEGRATION_TESTER` | -- |
-| `TEST_OPENID_PROVIDER_URL` | `https://keycloak.com/auth/realms/osdu` | OpenID provider url | yes | -- |
+| name | value | description | sensitive? | source |
+|--------------------------------------|-----------------------------------------------------------------------|---------------------------------------------------------------------------------------------------|-----------------------------------------|-------------------------------------|
+| `ELASTIC_PASSWORD` | `********` | Password for Elasticsearch | yes | output of infrastructure deployment |
+| `ELASTIC_USER_NAME` | `********` | User name for Elasticsearch | yes | output of infrastructure deployment |
+| `ELASTIC_HOST` | ex `elastic.domain.com` | Host Elasticsearch | yes | output of infrastructure deployment |
+| `ELASTIC_PORT` | ex `9243` | Port Elasticsearch | yes | output of infrastructure deployment |
+| `INDEXER_HOST` | ex `https://os-indexer-dot-opendes.appspot.com/api/indexer/v2/` | Indexer API endpoint | no | output of infrastructure deployment |
+| `GROUP_ID` | ex `opendes-gcp.projects.com` | OSDU R2 to run tests under | no | - |
+| `OTHER_RELEVANT_DATA_COUNTRIES` | ex `US` | valid legal tag with a other relevant data countries | no | - |
+| `LEGAL_TAG` | ex `opendes-demo-legaltag` | valid legal tag with a other relevant data countries from `DEFAULT_OTHER_RELEVANT_DATA_COUNTRIES` | no | - |
+| `DEFAULT_DATA_PARTITION_ID_TENANT1` | ex `opendes` | HTTP Header 'Data-Partition-ID' | no | - |
+| `DEFAULT_DATA_PARTITION_ID_TENANT2` | ex `opendes` | HTTP Header 'Data-Partition-ID' | no | - |
+| `SEARCH_HOST` | ex `http://localhost:8080/api/search/v2/` | Endpoint of search service | no | - |
+| `STORAGE_HOST` | ex `http://os-storage-dot-opendes.appspot.com/api/storage/v2/` | Storage API endpoint | no | output of infrastructure deployment |
+| `SECURITY_HTTPS_CERTIFICATE_TRUST` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment |
+| `TEST_OPENID_PROVIDER_CLIENT_ID` | `********` | Client Id for `$INTEGRATION_TESTER` | yes | -- |
+| `TEST_OPENID_PROVIDER_CLIENT_SECRET` | `********` | | Client secret for `$INTEGRATION_TESTER` | -- |
+| `TEST_OPENID_PROVIDER_URL` | `https://keycloak.com/auth/realms/osdu` | OpenID provider url | yes | -- |
+| `CUCUMBER_OPTIONS` | `--tags '~@indexer-extended'` OR `--tags '~@* and @indexer-extended'` | By default `--tags '~@indexer-extended'` to disable experimental feature testing | no | -- |
**Entitlements configuration for integration accounts**
diff --git a/provider/indexer-gc/docs/gc/README.md b/provider/indexer-gc/docs/gc/README.md
index 6a8b9e5c107f7cc4e67f482427cdb6bcee5a08a1..c3944a2bdb546f766cd410caacd405d286b21a76 100644
--- a/provider/indexer-gc/docs/gc/README.md
+++ b/provider/indexer-gc/docs/gc/README.md
@@ -168,23 +168,24 @@ TBD
You will need to have the following environment variables defined.
-| name | value | description | sensitive? | source |
-|-------------------------------------|----------------------------------------------------------------|---------------------------------------------------------------------------------------------------|------------|------------------------------------------------------------|
-| `ELASTIC_PASSWORD` | `********` | Password for Elasticsearch | yes | output of infrastructure deployment |
-| `ELASTIC_USER_NAME` | `********` | User name for Elasticsearch | yes | output of infrastructure deployment |
-| `ELASTIC_HOST` | ex `elastic.domain.com` | Host Elasticsearch | yes | output of infrastructure deployment |
-| `ELASTIC_PORT` | ex `9243` | Port Elasticsearch | yes | output of infrastructure deployment |
-| `GCLOUD_PROJECT` | ex `opendes` | Google Cloud Project Id | no | output of infrastructure deployment |
-| `INDEXER_HOST` | ex `https://os-indexer-dot-opendes.appspot.com/api/indexer/v2/` | Indexer API endpoint | no | output of infrastructure deployment |
-| `GROUP_ID` | ex `opendes-gc.projects.com` | OSDU R2 to run tests under | no | - |
-| `OTHER_RELEVANT_DATA_COUNTRIES` | ex `US` | valid legal tag with a other relevant data countries | no | - |
-| `LEGAL_TAG` | ex `opendes-demo-legaltag` | valid legal tag with a other relevant data countries from `DEFAULT_OTHER_RELEVANT_DATA_COUNTRIES` | no | - |
-| `DEFAULT_DATA_PARTITION_ID_TENANT1` | ex `opendes` | HTTP Header 'Data-Partition-ID' | no | - |
-| `DEFAULT_DATA_PARTITION_ID_TENANT2` | ex `opendes` | HTTP Header 'Data-Partition-ID' | no | - |
-| `SEARCH_INTEGRATION_TESTER` | `********` | Service account for API calls. Note: this user must have entitlements configured already | yes | <https://console.cloud.google.com/iam-admin/serviceaccounts> |
-| `SEARCH_HOST` | ex `http://localhost:8080/api/search/v2/` | Endpoint of search service | no | - |
-| `STORAGE_HOST` | ex `http://os-storage-dot-opendes.appspot.com/api/storage/v2/` | Storage API endpoint | no | output of infrastructure deployment |
-| `SECURITY_HTTPS_CERTIFICATE_TRUST` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment |
+| name | value | description | sensitive? | source |
+|-------------------------------------|-----------------------------------------------------------------------|---------------------------------------------------------------------------------------------------|------------|--------------------------------------------------------------|
+| `ELASTIC_PASSWORD` | `********` | Password for Elasticsearch | yes | output of infrastructure deployment |
+| `ELASTIC_USER_NAME` | `********` | User name for Elasticsearch | yes | output of infrastructure deployment |
+| `ELASTIC_HOST` | ex `elastic.domain.com` | Host Elasticsearch | yes | output of infrastructure deployment |
+| `ELASTIC_PORT` | ex `9243` | Port Elasticsearch | yes | output of infrastructure deployment |
+| `GCLOUD_PROJECT` | ex `opendes` | Google Cloud Project Id | no | output of infrastructure deployment |
+| `INDEXER_HOST` | ex `https://os-indexer-dot-opendes.appspot.com/api/indexer/v2/` | Indexer API endpoint | no | output of infrastructure deployment |
+| `GROUP_ID` | ex `opendes-gc.projects.com` | OSDU R2 to run tests under | no | - |
+| `OTHER_RELEVANT_DATA_COUNTRIES` | ex `US` | valid legal tag with a other relevant data countries | no | - |
+| `LEGAL_TAG` | ex `opendes-demo-legaltag` | valid legal tag with a other relevant data countries from `DEFAULT_OTHER_RELEVANT_DATA_COUNTRIES` | no | - |
+| `DEFAULT_DATA_PARTITION_ID_TENANT1` | ex `opendes` | HTTP Header 'Data-Partition-ID' | no | - |
+| `DEFAULT_DATA_PARTITION_ID_TENANT2` | ex `opendes` | HTTP Header 'Data-Partition-ID' | no | - |
+| `SEARCH_INTEGRATION_TESTER` | `********` | Service account for API calls. Note: this user must have entitlements configured already | yes | <https://console.cloud.google.com/iam-admin/serviceaccounts> |
+| `SEARCH_HOST` | ex `http://localhost:8080/api/search/v2/` | Endpoint of search service | no | - |
+| `STORAGE_HOST` | ex `http://os-storage-dot-opendes.appspot.com/api/storage/v2/` | Storage API endpoint | no | output of infrastructure deployment |
+| `SECURITY_HTTPS_CERTIFICATE_TRUST` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment |
+| `CUCUMBER_OPTIONS` | `--tags '~@indexer-extended'` OR `--tags '~@* and @indexer-extended'` | By default `--tags '~@indexer-extended'` to disable experimental feature testing | no | -- |
**Entitlements configuration for integration accounts**
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/publish/ReprocessingTaskPublisher.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/publish/ReprocessingTaskPublisher.java
index 720628c8a042376ddf4d2ac549a428918674c95c..ce93ee0f68c149b5f8f051d0b17cf74c754e093a 100644
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/publish/ReprocessingTaskPublisher.java
+++ b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/publish/ReprocessingTaskPublisher.java
@@ -26,6 +26,7 @@ import org.opengroup.osdu.core.gcp.oqm.driver.OqmDriver;
import org.opengroup.osdu.core.gcp.oqm.model.OqmDestination;
import org.opengroup.osdu.core.gcp.oqm.model.OqmMessage;
import org.opengroup.osdu.core.gcp.oqm.model.OqmTopic;
+import org.opengroup.osdu.indexer.model.Constants;
import org.opengroup.osdu.indexer.provider.gcp.indexing.processing.IndexerMessagingConfigProperties;
import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
import org.springframework.context.annotation.Primary;
@@ -128,10 +129,16 @@ public class ReprocessingTaskPublisher extends IndexerQueueTaskBuilder {
RecordChangedMessages recordChangedMessages = gson.fromJson(payload,
RecordChangedMessages.class);
+ Map<String, String> attributes = getAttributesFromHeaders(headers);
+ // Append the ancestry kinds used to prevent circular chasing
+ if(recordChangedMessages.getAttributes().containsKey(Constants.ANCESTRY_KINDS)) {
+ attributes.put(Constants.ANCESTRY_KINDS, recordChangedMessages.getAttributes().get(Constants.ANCESTRY_KINDS));
+ }
+
OqmMessage oqmMessage = OqmMessage.builder()
.id(headers.getCorrelationId())
.data(recordChangedMessages.getData())
- .attributes(getAttributesFromHeaders(headers))
+ .attributes(attributes)
.build();
log.info("Reprocessing task: {} ,has been published.", oqmMessage);
diff --git a/provider/indexer-gc/src/main/resources/application.properties b/provider/indexer-gc/src/main/resources/application.properties
index 1a2613ffe5df6a87e20724b7fcfb6dd1f027c239..d26678140c5b47019f09362a24de19abdbb01884 100644
--- a/provider/indexer-gc/src/main/resources/application.properties
+++ b/provider/indexer-gc/src/main/resources/application.properties
@@ -49,6 +49,7 @@ STORAGE_SCHEMA_HOST=${STORAGE_API}/schemas
SCHEMA_BASE_HOST=http://schema
SCHEMA_PATH=/api/schema-service/v1/schema
SCHEMA_HOST=${SCHEMA_BASE_HOST}${SCHEMA_PATH}
+SEARCH_HOST=${SEARCH_BASE_HOST}/api/search/v2
records-changed-topic-name=records-changed
schema-changed-topic-name=schema-changed
diff --git a/provider/indexer-ibm/src/main/resources/application.properties b/provider/indexer-ibm/src/main/resources/application.properties
index aaf4ceb6cf42aaa7a1fcaa8d72e71830700d4b27..e22d699633b9017f30a20d39d6862d214f6ef89c 100644
--- a/provider/indexer-ibm/src/main/resources/application.properties
+++ b/provider/indexer-ibm/src/main/resources/application.properties
@@ -27,6 +27,7 @@ CRON_INDEX_CLEANUP_THRESHOLD_DAYS=3
CRON_EMPTY_INDEX_CLEANUP_THRESHOLD_DAYS=7
SCHEMA_HOST=${HOST}/api/schema-service/v1/schema
+SEARCH_HOST=${search_service_url}/api/search/v2
storage_service_url=http://localhost:8082
#storage_service_url=https://os-storage-ibm-osdu-r2.osduadev-a1c3eaf78a86806e299f5f3f207556f0-0000.us-south.containers.appdomain.cloud
diff --git a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
index 37dd8f26ea74440a4c9affde1844a74d4168ec0b..62d0edc5586490e71850ac197614560e367e81c1 100644
--- a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
+++ b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -75,6 +75,16 @@ public class Steps extends SchemaServiceRecordSteps {
super.the_schema_is_created_with_the_following_kind(dataTable);
}
+ @Then("^I set starting stateful scenarios$")
+ public void i_set_starting_stateful_scenarios() throws Throwable {
+ super.i_set_scenarios_as_stateful(true);
+ }
+
+ @Then("^I set ending stateful scenarios$")
+ public void i_set_ending_stateful_scenarios() throws Throwable {
+ super.i_set_scenarios_as_stateful(false);
+ }
+
@When("^I ingest records with the \"(.*?)\" with \"(.*?)\" for a given \"(.*?)\"$")
public void i_ingest_records_with_the_for_a_given(String record, String dataGroup, String kind) {
super.i_ingest_records_with_the_for_a_given(record, dataGroup, kind);
@@ -115,6 +125,16 @@ public class Steps extends SchemaServiceRecordSteps {
super.iShouldBeAbleToSearchRecordByTagKeyAndTagValue(index, tagKey, tagValue, expectedNumber);
}
+ @Then("^I clean up the index of the extended kinds \"([^\"]*)\" in the Elastic Search$")
+ public void iShouldCleanupIndicesOfExtendedKinds(String extendedKinds) throws Throwable {
+ super.iShouldCleanupIndicesOfExtendedKinds(extendedKinds);
+ }
+
+ @Then("^I should be able to search (\\d+) record with index \"([^\"]*)\" by extended data field \"([^\"]*)\" and value \"([^\"]*)\"$")
+ public void iShouldBeAbleToSearchRecordByFieldAndFieldValue(int expectedNumber, String index, String fieldKey, String fieldValue) throws Throwable {
+ super.iShouldBeAbleToSearchRecordByFieldAndFieldValue(index, fieldKey, fieldValue, expectedNumber);
+ }
+
@Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by bounding box query with points \\((-?\\d+), (-?\\d+)\\) and \\((-?\\d+), (-?\\d+)\\) on field \"(.*?)\"$")
public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery (
int expectedCount, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field) throws Throwable {
@@ -141,4 +161,4 @@ public class Steps extends SchemaServiceRecordSteps {
throws Throwable {
super.i_should_get_object_in_search_response_without_hints_in_schema(objectInnerField ,index, recordFile, acl, kind);
}
-}
\ No newline at end of file
+}
diff --git a/testing/indexer-test-aws/src/test/resources/cucumber.properties b/testing/indexer-test-aws/src/test/resources/cucumber.properties
new file mode 100644
index 0000000000000000000000000000000000000000..4b0c14c43ba6053662d611f07881939ef39e4110
--- /dev/null
+++ b/testing/indexer-test-aws/src/test/resources/cucumber.properties
@@ -0,0 +1,2 @@
+# tag indexer-extended disabled by default
+cucumber.options=--tags ~@indexer-extended
diff --git a/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
index ae5e82f842d377236fccbedf449f30d25167f811..3cad1b4b8634a1786e9c620a1e18d7d9a49d29ee 100644
--- a/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
+++ b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -43,6 +43,16 @@ public class Steps extends SchemaServiceRecordSteps {
super.the_schema_is_created_with_the_following_kind(dataTable);
}
+ @Then("^I set starting stateful scenarios$")
+ public void i_set_starting_stateful_scenarios() throws Throwable {
+ super.i_set_scenarios_as_stateful(true);
+ }
+
+ @Then("^I set ending stateful scenarios$")
+ public void i_set_ending_stateful_scenarios() throws Throwable {
+ super.i_set_scenarios_as_stateful(false);
+ }
+
@When("^I ingest records with the \"(.*?)\" with \"(.*?)\" for a given \"(.*?)\"$")
public void i_ingest_records_with_the_for_a_given(String record, String dataGroup, String kind) {
super.i_ingest_records_with_the_for_a_given(record, dataGroup, kind);
@@ -83,6 +93,16 @@ public class Steps extends SchemaServiceRecordSteps {
super.iShouldBeAbleToSearchRecordByTagKeyAndTagValue(index, tagKey, tagValue, expectedNumber);
}
+ @Then("^I clean up the index of the extended kinds \"([^\"]*)\" in the Elastic Search$")
+ public void iShouldCleanupIndicesOfExtendedKinds(String extendedKinds) throws Throwable {
+ super.iShouldCleanupIndicesOfExtendedKinds(extendedKinds);
+ }
+
+ @Then("^I should be able to search (\\d+) record with index \"([^\"]*)\" by extended data field \"([^\"]*)\" and value \"([^\"]*)\"$")
+ public void iShouldBeAbleToSearchRecordByFieldAndFieldValue(int expectedNumber, String index, String fieldKey, String fieldValue) throws Throwable {
+ super.iShouldBeAbleToSearchRecordByFieldAndFieldValue(index, fieldKey, fieldValue, expectedNumber);
+ }
+
@Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by bounding box query with points \\((-?\\d+), (-?\\d+)\\) and \\((-?\\d+), (-?\\d+)\\) on field \"([^\"]*)\"$")
public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery(
int expectedCount, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field) throws Throwable {
@@ -109,4 +129,4 @@ public class Steps extends SchemaServiceRecordSteps {
throws Throwable {
super.i_should_get_object_in_search_response_without_hints_in_schema(objectInnerField ,index, recordFile, acl, kind);
}
-}
\ No newline at end of file
+}
diff --git a/testing/indexer-test-azure/src/test/resources/cucumber.properties b/testing/indexer-test-azure/src/test/resources/cucumber.properties
new file mode 100644
index 0000000000000000000000000000000000000000..2ca785a2f55f5f35f2c0f4b755c2a6cd072669c0
--- /dev/null
+++ b/testing/indexer-test-azure/src/test/resources/cucumber.properties
@@ -0,0 +1,2 @@
+# tag indexer-extended enabled by default
+cucumber.options=--tags '~@* and @indexer-extended'
diff --git a/testing/indexer-test-baremetal/src/test/java/org/opengroup/osdu/step_definitions/record/Steps.java b/testing/indexer-test-baremetal/src/test/java/org/opengroup/osdu/step_definitions/record/Steps.java
index 20ecbca20e53cc9a99e16f08446a0ac1d5c5a96c..c88b45c901145e00567c752fea0770a31177a66e 100644
--- a/testing/indexer-test-baremetal/src/test/java/org/opengroup/osdu/step_definitions/record/Steps.java
+++ b/testing/indexer-test-baremetal/src/test/java/org/opengroup/osdu/step_definitions/record/Steps.java
@@ -48,6 +48,16 @@ public class Steps extends SchemaServiceRecordSteps {
super.the_schema_is_created_with_the_following_kind(dataTable);
}
+ @Then("^I set starting stateful scenarios$")
+ public void i_set_starting_stateful_scenarios() throws Throwable {
+ super.i_set_scenarios_as_stateful(true);
+ }
+
+ @Then("^I set ending stateful scenarios$")
+ public void i_set_ending_stateful_scenarios() throws Throwable {
+ super.i_set_scenarios_as_stateful(false);
+ }
+
@When("^I ingest records with the \"(.*?)\" with \"(.*?)\" for a given \"(.*?)\"$")
public void i_ingest_records_with_the_for_a_given(String record, String dataGroup, String kind) {
super.i_ingest_records_with_the_for_a_given(record, dataGroup, kind);
@@ -90,6 +100,16 @@ public class Steps extends SchemaServiceRecordSteps {
super.iShouldBeAbleToSearchRecordByTagKeyAndTagValue(index, tagKey, tagValue, expectedNumber);
}
+ @Then("^I clean up the index of the extended kinds \"([^\"]*)\" in the Elastic Search$")
+ public void iShouldCleanupIndicesOfExtendedKinds(String extendedKinds) throws Throwable {
+ super.iShouldCleanupIndicesOfExtendedKinds(extendedKinds);
+ }
+
+ @Then("^I should be able to search (\\d+) record with index \"([^\"]*)\" by extended data field \"([^\"]*)\" and value \"([^\"]*)\"$")
+ public void iShouldBeAbleToSearchRecordByFieldAndFieldValue(int expectedNumber, String index, String fieldKey, String fieldValue) throws Throwable {
+ super.iShouldBeAbleToSearchRecordByFieldAndFieldValue(index, fieldKey, fieldValue, expectedNumber);
+ }
+
@Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by bounding box query with points \\((-?\\d+), (-?\\d+)\\) and \\((-?\\d+), (-?\\d+)\\) on field \"([^\"]*)\"$")
public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery(
int expectedCount, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field)
diff --git a/testing/indexer-test-baremetal/src/test/resources/cucumber.properties b/testing/indexer-test-baremetal/src/test/resources/cucumber.properties
new file mode 100644
index 0000000000000000000000000000000000000000..6884be422aa64d5c1c7a2dfb832c803f77371e43
--- /dev/null
+++ b/testing/indexer-test-baremetal/src/test/resources/cucumber.properties
@@ -0,0 +1,18 @@
+#
+# Copyright 2020-2023 Google LLC
+# Copyright 2020-2023 EPAM Systems, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# tag indexer-extended disabled by default
+cucumber.options=--tags ~@indexer-extended
diff --git a/testing/indexer-test-baremetal/src/test/java/resources/logback-test.xml b/testing/indexer-test-baremetal/src/test/resources/logback-test.xml
similarity index 100%
rename from testing/indexer-test-baremetal/src/test/java/resources/logback-test.xml
rename to testing/indexer-test-baremetal/src/test/resources/logback-test.xml
diff --git a/testing/indexer-test-core/pom.xml b/testing/indexer-test-core/pom.xml
index f341d13b07788c46a4fd0671d36a21daf022abf3..da0787021778e02c2ce55dbf9b90acd9dd29b06f 100644
--- a/testing/indexer-test-core/pom.xml
+++ b/testing/indexer-test-core/pom.xml
@@ -85,7 +85,7 @@
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
- <version>1.18.2</version>
+ <version>1.18.26</version>
<scope>provided</scope>
</dependency>
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
index ebfaba6b5ef99eaf3a5429799057d459172c5063..da0676cf610850a43b80e0f31cc6e465525b2de1 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
@@ -21,6 +21,7 @@ import org.springframework.util.CollectionUtils;
import javax.ws.rs.HttpMethod;
import java.io.IOException;
import java.lang.reflect.Type;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -101,6 +102,14 @@ public class RecordSteps extends TestsBase {
String createTime = java.time.Instant.now().toString();
for (Map<String, Object> testRecord : records) {
+ if(testRecord.containsKey("data")) {
+ Map<String, Object> data = (Map<String, Object>)testRecord.get("data");
+ if(data != null && data.size() > 0) {
+ data = replaceValues(data, timeStamp);
+ testRecord.put("data", data);
+ }
+ }
+
testRecord.put("kind", actualKind);
testRecord.put("id", generateRecordId(testRecord));
testRecord.put("legal", generateLegalTag());
@@ -184,6 +193,22 @@ public class RecordSteps extends TestsBase {
assertEquals(expectedNumber, actualNumberOfRecords);
}
+ public void iShouldCleanupIndicesOfExtendedKinds(String extendedKinds) throws Throwable {
+ String[] kinds = extendedKinds.split(",");
+ for(String kind : kinds) {
+ String actualKind = this.generateActualName(kind.trim(), timeStamp);
+ TestIndex testIndex = this.getInputIndexMap().get(actualKind);
+ testIndex.cleanupIndex(actualKind);
+ }
+ }
+
+ public void iShouldBeAbleToSearchRecordByFieldAndFieldValue(String index, String fieldKey, String fieldValue, int expectedNumber) throws Throwable {
+ TimeUnit.SECONDS.sleep(60);
+ index = generateActualName(index, timeStamp);
+ long actualNumberOfRecords = elasticUtils.fetchRecordsByFieldAndFieldValue(index, fieldKey, fieldValue);
+ assertEquals(expectedNumber, actualNumberOfRecords);
+ }
+
public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery (
int expectedNumber, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field) throws Throwable {
index = generateActualName(index, timeStamp);
@@ -232,6 +257,46 @@ public class RecordSteps extends TestsBase {
testIndex.addIndex();
}
+ private Map<String, Object> replaceValues(Map<String, Object> data, String timeStamp) {
+ for(String key : data.keySet()) {
+ Object value = data.get(key);
+ Object replacedValue = replaceValue(value, timeStamp);
+ data.put(key, replacedValue);
+ }
+ return data;
+ }
+
+ private List<Object> replaceValues(List<Object> values, String timeStamp) {
+ List<Object> replacedValues = new ArrayList<>();
+ for(Object value : values) {
+ Object replacedValue = replaceValue(value, timeStamp);
+ replacedValues.add(replacedValue);
+ }
+
+ return replacedValues;
+ }
+
+ private Object replaceValue(Object value, String timeStamp) {
+ Object replacedValue = value;
+
+ if(value instanceof String) {
+ String rawValue = (String) value;
+ for (Map.Entry<String, String> tenant : tenantMap.entrySet()) {
+ rawValue = rawValue.replaceAll(tenant.getKey() + ":", tenant.getValue() + ":");
+ }
+ replacedValue = rawValue.replaceAll("<timestamp>", timeStamp);
+ }
+ else if(value instanceof List) {
+ replacedValue = replaceValues((List)value, timeStamp);
+ }
+ else if(value instanceof Map) {
+ replacedValue = replaceValues((Map<String, Object>) value, timeStamp);
+ }
+
+ return replacedValue;
+ }
+
+
private long createIndex(String index) throws InterruptedException, IOException {
long numOfIndexedDocuments = 0;
int iterator;
@@ -320,4 +385,4 @@ public class RecordSteps extends TestsBase {
shutDownHookAdded = true;
}
}
-}
\ No newline at end of file
+}
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/SchemaServiceRecordSteps.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/SchemaServiceRecordSteps.java
index af48584f1b8e315f0435c65d78442913b8cbfd6f..53e19fb874a5c27f07e30af25fe85bb9c1172aa5 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/SchemaServiceRecordSteps.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/SchemaServiceRecordSteps.java
@@ -10,15 +10,22 @@ import java.util.List;
import java.util.Map;
public class SchemaServiceRecordSteps extends RecordSteps {
+ private static boolean runStatefulScenario = false;
public SchemaServiceRecordSteps(HTTPClient httpClient, ElasticUtils elasticUtils) {
super(httpClient, elasticUtils);
}
public void the_schema_is_created_with_the_following_kind(DataTable dataTable) {
- List<Setup> inputList = dataTable.asList(Setup.class);
- inputList.forEach(this::setup);
- super.addShutDownHook();
+ if(!SchemaServiceRecordSteps.runStatefulScenario) {
+ List<Setup> inputList = dataTable.asList(Setup.class);
+ inputList.forEach(this::setup);
+ super.addShutDownHook();
+ }
+ }
+
+ public void i_set_scenarios_as_stateful(boolean stateful) throws Throwable {
+ SchemaServiceRecordSteps.runStatefulScenario = stateful;
}
private void setup(Setup input) {
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
index b37e7daa324d2f73d3c85598dcb64a5317abaf5b..de2b99f64cf6857596062d614281dfcd345321a0 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
@@ -244,6 +244,24 @@ public class ElasticUtils {
}
}
+ public long fetchRecordsByFieldAndFieldValue(String index, String fieldKey, String fieldValue) throws IOException {
+ try {
+ try (RestHighLevelClient client = this.createClient(username, password, host)) {
+ SearchRequest request = new SearchRequest(index);
+ if(!Strings.isNullOrEmpty(fieldKey)) {
+ SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
+ sourceBuilder.query(boolQuery().must(matchQuery(fieldKey, fieldValue)));
+ request.source(sourceBuilder);
+ }
+ SearchResponse searchResponse = client.search(request, RequestOptions.DEFAULT);
+ return searchResponse.getHits().getTotalHits().value;
+ }
+ } catch (ElasticsearchStatusException e) {
+ log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage()));
+ return -1;
+ }
+ }
+
public List<Map<String, Object>> fetchRecordsByAttribute(String index, String attributeKey, String attributeValue) throws IOException {
List<Map<String, Object>> out = new ArrayList<>();
try {
@@ -499,4 +517,4 @@ public class ElasticUtils {
return false;
}
-}
\ No newline at end of file
+}
diff --git a/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature b/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
index d11cd33d5bf4076f0881a0966f8e25d573394644..556a969a6d9aca64f28bf7d4aa544e3bf7c034bb 100644
--- a/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
+++ b/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
@@ -13,7 +13,45 @@ Feature: Indexing of the documents
| tenant1:indexer:test-update-data--Integration:1.0.1 | tenant1-indexer-test-update-data--integration-1.0.1 | index_update_records_kind_v1 |
| tenant1:indexer:test-update-data--Integration:2.0.1 | tenant1-indexer-test-update-data--integration-2.0.1 | index_update_records_kind_v2 |
| tenant1:indexer:virtual-properties-Integration:1.0.0 | tenant1-indexer-virtual-properties-integration-1.0.0 | index_record_virtual_properties |
- | tenant1:indexer:decimation-Integration:1.0.0 | tenant1-indexer-decimation-integration-1.0.0 | index_record_seismic_survey |
+ | tenant1:indexer:decimation-Integration:1.0.0 | tenant1-indexer-decimation-integration-1.0.0 | index_record_seismic_survey |
+ | osdu:wks:reference-data--IndexPropertyPathConfiguration:1.0.0 | osdu-wks-reference-data--indexpropertypathconfiguration-1.0.0 | osdu_wks_IndexPropertyPathConfiguration_v1 |
+ | test:indexer:index-property--Wellbore:1.0.0 | test-indexer-index-property--wellbore-1.0.0 | index-property-wellbore_v1 |
+ | test:indexer:index-property--WellLog:1.0.0 | test-indexer-index-property--welllog-1.0.0 | index-property-welllog_v1 |
+
+ @indexer-extended
+ Scenario Outline: Prepare the index property configuration records and clean up index of the extended kinds in the Elastic Search
+ When I ingest records with the <recordFile> with <acl> for a given <kind>
+ Then I should get the <number> documents for the <index> in the Elastic Search
+ Then I clean up the index of the extended kinds <extendedKinds> in the Elastic Search
+ Then I set starting stateful scenarios
+
+ Examples:
+ | kind | recordFile | number | index | acl | extendedKinds |
+ | "osdu:wks:reference-data--IndexPropertyPathConfiguration:1.0.0" | "osdu_wks_IndexPropertyPathConfiguration_v1" | 2 | "osdu-wks-reference-data--indexpropertypathconfiguration-1.0.0" | "data.default.viewers@tenant1" | "test:indexer:index-property--Wellbore:1.0.0,test:indexer:index-property--WellLog:1.0.0" |
+
+ @indexer-extended
+ Scenario Outline: Ingest the records of the extended kinds, Index in the Elastic Search and Search string field
+ When I ingest records with the <recordFile> with <acl> for a given <kind>
+ Then I should be able to search <number> record with index <index> by extended data field <field> and value <value>
+
+ Examples:
+ | kind | recordFile | number | index | acl | field | value |
+ | "test:indexer:index-property--Wellbore:1.0.0" | "index-property-wellbore_v1" | 1 | "test-indexer-index-property--wellbore-1.0.0" | "data.default.viewers@tenant1" | "data.WellUWI" | "123454321" |
+ | "test:indexer:index-property--WellLog:1.0.0" | "index-property-welllog_v1" | 1 | "test-indexer-index-property--welllog-1.0.0" | "data.default.viewers@tenant1" | "data.WellboreName" | "Facility_123" |
+
+ @indexer-extended
+ Scenario Outline: Ingest the records of the extended kinds, Index in the Elastic Search and Search spatial field
+ When I ingest records with the <recordFile> with <acl> for a given <kind>
+ Then I should be able search <number> documents for the <index> by bounding box query with points (<top_left_latitude>, <top_left_longitude>) and (<bottom_right_latitude>, <bottom_right_longitude>) on field <field>
+
+ Examples:
+ | kind | recordFile | number | index | acl | field | top_left_latitude | top_left_longitude | bottom_right_latitude | bottom_right_longitude |
+ | "test:indexer:index-property--Wellbore:1.0.0" | "index-property-wellbore_v1" | 1 | "test-indexer-index-property--wellbore-1.0.0" | "data.default.viewers@tenant1" | "data.Location" | 30 | -96 | 29 | -95 |
+ | "test:indexer:index-property--WellLog:1.0.0" | "index-property-welllog_v1" | 1 | "test-indexer-index-property--welllog-1.0.0" | "data.default.viewers@tenant1" | "data.SpatialLocation" | 30 | -96 | 29 | -95 |
+
+ @indexer-extended
+ Scenario: End Stateful Scenarios
+ Then I set ending stateful scenarios
Scenario Outline: Ingest the record and Index in the Elastic Search
When I ingest records with the <recordFile> with <acl> for a given <kind>
diff --git a/testing/indexer-test-core/src/main/resources/testData/index-property-wellbore_v1.json b/testing/indexer-test-core/src/main/resources/testData/index-property-wellbore_v1.json
new file mode 100644
index 0000000000000000000000000000000000000000..469a0d22c7637a8fd867bbf2ed9aad733b87b839
--- /dev/null
+++ b/testing/indexer-test-core/src/main/resources/testData/index-property-wellbore_v1.json
@@ -0,0 +1,16 @@
+[{
+ "id": "tenant1:index-property--Wellbore:testIngest1",
+ "data": {
+ "Location": {
+ "latitude": 29.170640,
+ "longitude": -95.002875
+ },
+ "FacilityName": "Facility_123",
+ "NameAliases": [{
+ "AliasName": "123454321",
+ "AliasNameTypeID": "tenant1:reference-data--AliasNameType:UniqueIdentifier:"
+ }
+ ]
+ }
+ }
+]
diff --git a/testing/indexer-test-core/src/main/resources/testData/index-property-wellbore_v1.schema.json b/testing/indexer-test-core/src/main/resources/testData/index-property-wellbore_v1.schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..93d616f44e393c27e176724c2a97c80028f613bb
--- /dev/null
+++ b/testing/indexer-test-core/src/main/resources/testData/index-property-wellbore_v1.schema.json
@@ -0,0 +1,118 @@
+{
+ "schemaInfo": {
+ "schemaIdentity": {
+ "authority": "test",
+ "source": "indexer",
+ "entityType": "index-property--Wellbore",
+ "schemaVersionMajor": 1,
+ "schemaVersionMinor": 0,
+ "schemaVersionPatch": 0,
+ "id": "test:indexer:index-property--Wellbore:1.0.0"
+ },
+ "status": "DEVELOPMENT"
+ },
+ "schema": {
+ "definitions": {
+ "opendes:wks:core_dl_geopoint:1.0.0": {
+ "description": "A 2D point location in latitude and longitude referenced to WGS 84 if not specified otherwise.",
+ "properties": {
+ "latitude": {
+ "description": "The latitude value in degrees of arc (dega). Value range [-90, 90].",
+ "maximum": 90,
+ "minimum": -90,
+ "title": "Latitude",
+ "type": "number"
+ },
+ "longitude": {
+ "description": "The longitude value in degrees of arc (dega). Value range [-180, 180]",
+ "maximum": 180,
+ "minimum": -180,
+ "title": "Longitude",
+ "type": "number"
+ }
+ },
+ "required": [
+ "latitude",
+ "longitude"
+ ],
+ "title": "2D Map Location",
+ "type": "object"
+ },
+ "opendes:wks:AbstractAliasNames:1.0.0": {
+ "title": "AbstractAliasNames",
+ "type": "object",
+ "properties": {
+ "AliasNameTypeID": {
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-AliasNameType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "A classification of alias names such as by role played or type of source, such as regulatory name, regulatory code, company code, international standard name, etc.",
+ "x-osdu-relationship": [{
+ "EntityType": "AliasNameType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "type": "string"
+ },
+ "EffectiveDateTime": {
+ "format": "date-time",
+ "type": "string",
+ "description": "The date and time when an alias name becomes effective."
+ },
+ "AliasName": {
+ "type": "string",
+ "description": "Alternative Name value of defined name type for an object."
+ },
+ "TerminationDateTime": {
+ "format": "date-time",
+ "type": "string",
+ "description": "The data and time when an alias name is no longer in effect."
+ },
+ "DefinitionOrganisationID": {
+ "pattern": "^[\\w\\-\\.]+:(reference-data\\-\\-StandardsOrganisation|master-data\\-\\-Organisation):[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The StandardsOrganisation (reference-data) or Organisation (master-data) that provided the name (the source).",
+ "x-osdu-relationship": [{
+ "EntityType": "StandardsOrganisation",
+ "GroupType": "reference-data"
+ }, {
+ "EntityType": "Organisation",
+ "GroupType": "master-data"
+ }
+ ],
+ "type": "string"
+ }
+ }
+ }
+ },
+ "properties": {
+ "data": {
+ "allOf": [
+ {
+ "type": "object",
+ "properties": {
+ "Location": {
+ "$ref": "#/definitions/opendes:wks:core_dl_geopoint:1.0.0",
+ "description": "The wellbore's position .",
+ "format": "core:dl:geopoint:1.0.0",
+ "title": "WGS 84 Position",
+ "type": "object"
+ },
+ "NameAliases": {
+ "x-osdu-indexing": {
+ "type": "nested"
+ },
+ "description": "Alternative names, including historical, by which this master data is/has been known (it should include all the identifiers).",
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/opendes:wks:AbstractAliasNames:1.0.0"
+ }
+ },
+ "FacilityName": {
+ "description": "Name of the Facility.",
+ "type": "string"
+ }
+ }
+ }
+ ]
+ }
+ }
+ }
+}
diff --git a/testing/indexer-test-core/src/main/resources/testData/index-property-welllog_v1.json b/testing/indexer-test-core/src/main/resources/testData/index-property-welllog_v1.json
new file mode 100644
index 0000000000000000000000000000000000000000..b22a36722b78de757f1631809f9f4ea51fa02f25
--- /dev/null
+++ b/testing/indexer-test-core/src/main/resources/testData/index-property-welllog_v1.json
@@ -0,0 +1,8 @@
+[{
+ "id": "tenant1:index-property--WellLog:testIngest2",
+ "data": {
+ "Name": "Log_ABC",
+ "WellboreID": "tenant1:index-property--Wellbore:testIngest1"
+ }
+ }
+]
diff --git a/testing/indexer-test-core/src/main/resources/testData/index-property-welllog_v1.schema.json b/testing/indexer-test-core/src/main/resources/testData/index-property-welllog_v1.schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..1b9549b616e5cc889ed200f08decc0b08d4a481b
--- /dev/null
+++ b/testing/indexer-test-core/src/main/resources/testData/index-property-welllog_v1.schema.json
@@ -0,0 +1,36 @@
+{
+ "schemaInfo": {
+ "schemaIdentity": {
+ "authority": "test",
+ "source": "indexer",
+ "entityType": "index-property--WellLog",
+ "schemaVersionMajor": 1,
+ "schemaVersionMinor": 0,
+ "schemaVersionPatch": 0,
+ "id": "test:indexer:index-property--WellLog:1.0.0"
+ },
+ "status": "DEVELOPMENT"
+ },
+ "schema": {
+ "properties": {
+ "data": {
+ "allOf": [
+ {
+ "type": "object",
+ "properties": {
+ "WellboreID": {
+ "description": "Id of parent wellbore.",
+ "type": "string"
+ },
+ "Name": {
+ "description": "Name of welllog.",
+ "type": "string"
+ }
+ }
+ }
+ ]
+ }
+ }
+ }
+
+}
diff --git a/testing/indexer-test-core/src/main/resources/testData/osdu_wks_IndexPropertyPathConfiguration_v1.json b/testing/indexer-test-core/src/main/resources/testData/osdu_wks_IndexPropertyPathConfiguration_v1.json
new file mode 100644
index 0000000000000000000000000000000000000000..f8ba28eb63ab5e68676610c029c12defbd962e2e
--- /dev/null
+++ b/testing/indexer-test-core/src/main/resources/testData/osdu_wks_IndexPropertyPathConfiguration_v1.json
@@ -0,0 +1,70 @@
+[{
+ "id": "tenant1:reference-data--IndexPropertyPathConfiguration:index-property--WellLog:1.",
+ "data": {
+ "Name": "WellLogIndex-PropertyPathConfiguration",
+ "Description": "The index property list for WellLog:1., valid for all WellLog kinds for major version 1.",
+ "Code": "test:indexer:index-property--WellLog:1.",
+ "AttributionAuthority": "OSDU",
+ "Configurations": [{
+ "Name": "WellboreName",
+ "Policy": "ExtractFirstMatch",
+ "Paths": [{
+ "RelatedObjectsSpec": {
+ "RelationshipDirection": "ChildToParent",
+ "RelatedObjectKind": "test:indexer:index-property--Wellbore:1.",
+ "RelatedObjectID": "data.WellboreID"
+ },
+ "ValueExtraction": {
+ "ValuePath": "data.FacilityName"
+ }
+ }
+ ],
+ "UseCase": "As a user I want to discover WellLog instances by the wellbore's name value."
+ }, {
+ "Name": "SpatialLocation",
+ "Policy": "ExtractFirstMatch",
+ "Paths": [{
+ "RelatedObjectsSpec": {
+ "RelationshipDirection": "ChildToParent",
+ "RelatedObjectKind": "test:indexer:index-property--Wellbore:1.",
+ "RelatedObjectID": "data.WellboreID"
+ },
+ "ValueExtraction": {
+ "ValuePath": "data.Location"
+ }
+ }
+ ],
+ "UseCase": "As a user I want to discover WellLog instances by spatial location."
+ }
+ ]
+ }
+ }, {
+ "id": "tenant1:reference-data--IndexPropertyPathConfiguration:index-property--Wellbore:1.",
+ "data": {
+ "Name": "Wellbore-IndexPropertyPathConfiguration",
+ "Description": "The index property list for index-property--Wellbore:1., valid for all index-property--Wellbore kinds for major version 1.",
+ "Code": "test:indexer:index-property--Wellbore:1.",
+ "AttributionAuthority": "OSDU",
+ "Configurations": [{
+ "Name": "WellUWI",
+ "Policy": "ExtractFirstMatch",
+ "Paths": [{
+ "ValueExtraction": {
+ "RelatedConditionMatches": [
+ "tenant1:reference-data--AliasNameType:UniqueIdentifier:",
+ "tenant1:reference-data--AliasNameType:RegulatoryName:",
+ "tenant1:reference-data--AliasNameType:PreferredName:",
+ "tenant1:reference-data--AliasNameType:CommonName:",
+ "tenant1:reference-data--AliasNameType:ShortName:"
+ ],
+ "RelatedConditionProperty": "data.NameAliases[].AliasNameTypeID",
+ "ValuePath": "data.NameAliases[].AliasName"
+ }
+ }
+ ],
+ "UseCase": "As a user I want to discover and match Wells by their UWI. I am aware that this is not globally reliable, however, I am able to specify a prioritized AliasNameType list to look up value in the NameAliases array."
+ }
+ ]
+ }
+ }
+]
diff --git a/testing/indexer-test-core/src/main/resources/testData/osdu_wks_IndexPropertyPathConfiguration_v1.schema.json b/testing/indexer-test-core/src/main/resources/testData/osdu_wks_IndexPropertyPathConfiguration_v1.schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..7cf15888121f227c11303d934c9af6340f384798
--- /dev/null
+++ b/testing/indexer-test-core/src/main/resources/testData/osdu_wks_IndexPropertyPathConfiguration_v1.schema.json
@@ -0,0 +1,284 @@
+{
+ "schemaInfo": {
+ "schemaIdentity": {
+ "authority": "osdu",
+ "source": "wks",
+ "entityType": "reference-data--IndexPropertyPathConfiguration",
+ "schemaVersionMajor": 1,
+ "schemaVersionMinor": 0,
+ "schemaVersionPatch": 0,
+ "id": "osdu:wks:reference-data--IndexPropertyPathConfiguration:1.0.0"
+ },
+ "status": "DEVELOPMENT"
+ },
+ "schema": {
+ "x-osdu-license": "Copyright 2023, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$id": "https://schema.osdu.opengroup.org/json/reference-data/IndexPropertyPathConfiguration.1.0.0.json",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:reference-data--IndexPropertyPathConfiguration:1.0.0",
+ "title": "IndexPropertyPathConfiguration",
+ "description": "IndexPropertyPathConfiguration contains the de-normalization configuration settings for the Search index. The record id contains the kind so that there is a unique relationship between the kind and the kind's index extensions.",
+ "type": "object",
+ "properties": {
+ "id": {
+ "description": "Previously called ResourceID or SRN which identifies this OSDU resource object without version.",
+ "title": "Entity ID",
+ "type": "string",
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-IndexPropertyPathConfiguration:[\\w\\-\\.\\:\\%]+$",
+ "example": "namespace:reference-data--IndexPropertyPathConfiguration:c9d84708-2b1b-5e0b-954e-9621132f7154"
+ },
+ "kind": {
+ "description": "The schema identification for the OSDU resource object following the pattern {Namespace}:{Source}:{Type}:{VersionMajor}.{VersionMinor}.{VersionPatch}. The versioning scheme follows the semantic versioning, https://semver.org/.",
+ "title": "Entity Kind",
+ "type": "string",
+ "pattern": "^[\\w\\-\\.]+:[\\w\\-\\.]+:[\\w\\-\\.]+:[0-9]+.[0-9]+.[0-9]+$",
+ "example": "osdu:wks:reference-data--IndexPropertyPathConfiguration:1.0.0"
+ },
+ "version": {
+ "description": "The version number of this OSDU resource; set by the framework.",
+ "title": "Version Number",
+ "type": "integer",
+ "format": "int64",
+ "example": 1562066009929332
+ },
+ "acl": {
+ "description": "The access control tags associated with this entity.",
+ "title": "Access Control List",
+ "$ref": "osdu:wks:AbstractAccessControlList:1.0.0"
+ },
+ "legal": {
+ "description": "The entity's legal tags and compliance status. The actual contents associated with the legal tags is managed by the Compliance Service.",
+ "title": "Legal Tags",
+ "$ref": "osdu:wks:AbstractLegalTags:1.0.0"
+ },
+ "tags": {
+ "title": "Tag Dictionary",
+ "description": "A generic dictionary of string keys mapping to string value. Only strings are permitted as keys and values.",
+ "type": "object",
+ "additionalProperties": {
+ "type": "string"
+ },
+ "example": {
+ "NameOfKey": "String value"
+ }
+ },
+ "createTime": {
+ "description": "Timestamp of the time at which initial version of this OSDU resource object was created. Set by the System. The value is a combined date-time string in ISO-8601 given in UTC.",
+ "title": "Resource Object Creation DateTime",
+ "type": "string",
+ "format": "date-time",
+ "example": "2020-12-16T11:46:20.163Z"
+ },
+ "createUser": {
+ "title": "Resource Object Creation User Reference",
+ "description": "The user reference, which created the first version of this resource object. Set by the System.",
+ "type": "string",
+ "example": "some-user@some-company-cloud.com"
+ },
+ "modifyTime": {
+ "description": "Timestamp of the time at which this version of the OSDU resource object was created. Set by the System. The value is a combined date-time string in ISO-8601 given in UTC.",
+ "title": "Resource Object Version Creation DateTime",
+ "type": "string",
+ "format": "date-time",
+ "example": "2020-12-16T11:52:24.477Z"
+ },
+ "modifyUser": {
+ "title": "Resource Object Version Creation User Reference",
+ "description": "The user reference, which created this version of this resource object. Set by the System.",
+ "type": "string",
+ "example": "some-user@some-company-cloud.com"
+ },
+ "ancestry": {
+ "description": "The links to data, which constitute the inputs, from which this record instance is derived.",
+ "title": "Ancestry",
+ "$ref": "osdu:wks:AbstractLegalParentList:1.0.0"
+ },
+ "meta": {
+ "description": "The Frame of Reference meta data section linking the named properties to self-contained definitions.",
+ "title": "Frame of Reference Meta Data",
+ "type": "array",
+ "items": {
+ "$ref": "osdu:wks:AbstractMetaItem:1.0.0"
+ }
+ },
+ "data": {
+ "allOf": [
+ {
+ "$ref": "osdu:wks:AbstractCommonResources:1.0.0"
+ },
+ {
+ "$ref": "osdu:wks:AbstractReferenceType:1.0.0"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "Configurations": {
+ "type": "array",
+ "title": "Configurations",
+ "description": "The list of index property configurations for the specific kind.",
+ "x-osdu-indexing": {
+ "type": "nested"
+ },
+ "items": {
+ "type": "object",
+ "title": "Configuration",
+ "description": "One single configuration to derive an Search index property value and assign it to the index 'column' with Name.",
+ "properties": {
+ "Name": {
+ "type": "string",
+ "title": "Name",
+ "description": "The name of the indexed property, i.e., this is the property name used in Search.",
+ "example": "CountryNames"
+ },
+ "UseCase": {
+ "type": "string",
+ "title": "Use Case",
+ "description": "The use case description this configuration satisfies.",
+ "example": "As a user I want to find objects by a country name, with the understanding that an object may extend over country boundaries."
+ },
+ "Policy": {
+ "type": "string",
+ "title": "Extraction Policy",
+ "description": "Current supported policies are 'ExtractAllMatches' resulting in an array of values or 'ExtractFirstMatch' single value. The policy applies only to the Paths[].ValueExtraction.",
+ "example": "ExtractAllMatches",
+ "pattern": "^(ExtractFirstMatch|ExtractAllMatches)$"
+ },
+ "Paths": {
+ "type": "array",
+ "title": "Paths",
+ "description": "The list of path definitions to derive the property value from.",
+ "x-osdu-indexing": {
+ "type": "nested"
+ },
+ "items": {
+ "type": "object",
+ "title": "Path",
+ "description": "A single path definition to derive a property value from.",
+ "properties": {
+ "RelatedObjectsSpec": {
+ "type": "object",
+ "title": "Related Objects Specification",
+ "description": "The specification to extract related objects, from which to derive the ValueExtraction. If this property is empty or absent, the ValueExtraction is done on the current object to be indexed.",
+ "properties": {
+ "RelatedObjectID": {
+ "type": "string",
+ "title": "Related Object ID",
+ "description": "The path to the property containing the ID of the target record to chase. This property is only populated if the property is extracted from a related object, which must be chased. If the property is derived from 'within' the same record, which triggered the indexing, the RelatedObjectID is left absent.",
+ "example": "GeoContexts[].GeoPoliticalEntityID"
+ },
+ "RelatedObjectKind": {
+ "type": "string",
+ "title": "Related Object Kind",
+ "description": "The kind or schema id expected as the target object type. This property is only populated if the property is extracted from a related object, which must be chased. If the property is derived from 'within' the same record, which triggered the indexing, the RelatedObjectKind is left absent.",
+ "example": "osdu:wks:master-data--GeoPoliticalEntity:1.",
+ "pattern": "^[\\w\\-\\.]+:[\\w\\-\\.]+:[\\w\\-\\.]+:[0-9]+.$"
+ },
+ "RelationshipDirection": {
+ "type": "string",
+ "title": "Relationship Direction",
+ "description": "The direction of the relationship definition seen from the object being indexed. 'ChildToParent' assumes an outgoing relationship with the target record defined in the object being indexed. 'ParentToChildren' assumes that the related objects have a relationship by RelatedObjectID to the id of the record being indexed.",
+ "example": "ChildToParent",
+ "pattern": "^(ChildToParent|ParentToChildren)$"
+ },
+ "RelatedConditionProperty": {
+ "type": "string",
+ "title": "Related Condition Property",
+ "description": "The property path of the target record data block, which needs subjected to the conditional matching. The data prefix is not required.",
+ "example": "GeoContexts[].GeoTypeID"
+ },
+ "RelatedConditionMatches": {
+ "type": "array",
+ "title": "Related Condition Matches",
+ "description": "The RelatedConditionProperty values, which need to match in order to be accepted as de-normalized value(s). If the Policy is ExtractFirstMatch, the list is prioritized and the first match is accepted as final value. Policy ExtractAllMatches collects all matching values as array.",
+ "example": [
+ "namespace:reference-data--GeoPoliticalEntityType:Country:"
+ ],
+ "items": {
+ "type": "string"
+ }
+ }
+ }
+ },
+ "ValueExtraction": {
+ "type": "object",
+ "title": "Value Extraction",
+ "description": "The instructions from where to derive the value.",
+ "properties": {
+ "RelatedConditionProperty": {
+ "type": "string",
+ "title": "Related Condition Property",
+ "description": "The property path of the target record data block, which needs to be subjected to the conditional matching. The data prefix is not required in the path."
+ },
+ "RelatedConditionMatches": {
+ "type": "array",
+ "title": "Related Condition Matches",
+ "description": "The RelatedConditionProperty values, which need to match in order to be accepted as de-normalized value(s). If the Policy is ExtractFirstMatch, the list is prioritized and the first match is accepted as final value. Policy ExtractAllMatches collects all matching values as array.",
+ "items": {
+ "type": "string"
+ }
+ },
+ "ValuePath": {
+ "type": "string",
+ "title": "Value Path",
+ "description": "The path to the property from where to extract the de-normalized value. The data prefix is not required in the path.",
+ "example": "GeoPoliticalEntityName"
+ }
+ },
+ "required": [
+ "ValuePath"
+ ]
+ }
+ },
+ "required": [
+ "ValueExtraction"
+ ]
+ }
+ }
+ }
+ }
+ }
+ },
+ "title": "IndividualProperties"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "ExtensionProperties": {
+ "type": "object"
+ }
+ },
+ "title": "ExtensionProperties"
+ }
+ ]
+ }
+ },
+ "required": [
+ "kind",
+ "acl",
+ "legal"
+ ],
+ "additionalProperties": false,
+ "x-osdu-review-status": "Accepted",
+ "x-osdu-governance-model": "OPEN",
+ "x-osdu-governance-authorities": [
+ "Energistics",
+ "OSDU"
+ ],
+ "x-osdu-virtual-properties": {
+ "data.VirtualProperties.DefaultName": {
+ "type": "string",
+ "priority": [
+ {
+ "path": "data.Name"
+ }
+ ]
+ }
+ },
+ "x-osdu-inheriting-from-kind": [
+ {
+ "name": "ReferenceType",
+ "kind": "osdu:wks:AbstractReferenceType:1.0.0"
+ }
+ ]
+ }
+}
diff --git a/testing/indexer-test-gc/pom.xml b/testing/indexer-test-gc/pom.xml
index ab4cdaa78fda0605159fab207e9f2810f4dd8727..5518883c91c18ac2825493006e0143f0c33a3700 100644
--- a/testing/indexer-test-gc/pom.xml
+++ b/testing/indexer-test-gc/pom.xml
@@ -87,7 +87,7 @@
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
- <version>1.18.2</version>
+ <version>1.18.26</version>
<scope>provided</scope>
</dependency>
diff --git a/testing/indexer-test-gc/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-gc/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
index 7164300a8d7eb06c477999c8ba8bdeb3a2bc522e..591f8a68f097bff0a0e4ed53d32b1246b95d858a 100644
--- a/testing/indexer-test-gc/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
+++ b/testing/indexer-test-gc/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -31,6 +31,16 @@ public class Steps extends SchemaServiceRecordSteps {
super.the_schema_is_created_with_the_following_kind(dataTable);
}
+ @Then("^I set starting stateful scenarios$")
+ public void i_set_starting_stateful_scenarios() throws Throwable {
+ super.i_set_scenarios_as_stateful(true);
+ }
+
+ @Then("^I set ending stateful scenarios$")
+ public void i_set_ending_stateful_scenarios() throws Throwable {
+ super.i_set_scenarios_as_stateful(false);
+ }
+
@When("^I ingest records with the \"(.*?)\" with \"(.*?)\" for a given \"(.*?)\"$")
public void i_ingest_records_with_the_for_a_given(String record, String dataGroup, String kind) {
super.i_ingest_records_with_the_for_a_given(record, dataGroup, kind);
@@ -72,6 +82,16 @@ public class Steps extends SchemaServiceRecordSteps {
super.iShouldBeAbleToSearchRecordByTagKeyAndTagValue(index, tagKey, tagValue, expectedNumber);
}
+ @Then("^I clean up the index of the extended kinds \"([^\"]*)\" in the Elastic Search$")
+ public void iShouldCleanupIndicesOfExtendedKinds(String extendedKinds) throws Throwable {
+ super.iShouldCleanupIndicesOfExtendedKinds(extendedKinds);
+ }
+
+ @Then("^I should be able to search (\\d+) record with index \"([^\"]*)\" by extended data field \"([^\"]*)\" and value \"([^\"]*)\"$")
+ public void iShouldBeAbleToSearchRecordByFieldAndFieldValue(int expectedNumber, String index, String fieldKey, String fieldValue) throws Throwable {
+ super.iShouldBeAbleToSearchRecordByFieldAndFieldValue(index, fieldKey, fieldValue, expectedNumber);
+ }
+
@Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by bounding box query with points \\((-?\\d+), (-?\\d+)\\) and \\((-?\\d+), (-?\\d+)\\) on field \"([^\"]*)\"$")
public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery(
int expectedCount, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field)
diff --git a/testing/indexer-test-gc/src/test/resources/cucumber.properties b/testing/indexer-test-gc/src/test/resources/cucumber.properties
new file mode 100644
index 0000000000000000000000000000000000000000..6884be422aa64d5c1c7a2dfb832c803f77371e43
--- /dev/null
+++ b/testing/indexer-test-gc/src/test/resources/cucumber.properties
@@ -0,0 +1,18 @@
+#
+# Copyright 2020-2023 Google LLC
+# Copyright 2020-2023 EPAM Systems, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# tag indexer-extended disabled by default
+cucumber.options=--tags ~@indexer-extended
diff --git a/testing/indexer-test-ibm/src/main/resources/cucumber.properties b/testing/indexer-test-ibm/src/main/resources/cucumber.properties
new file mode 100644
index 0000000000000000000000000000000000000000..2ca785a2f55f5f35f2c0f4b755c2a6cd072669c0
--- /dev/null
+++ b/testing/indexer-test-ibm/src/main/resources/cucumber.properties
@@ -0,0 +1,2 @@
+# tag indexer-extended enabled by default
+cucumber.options=--tags '~@* and @indexer-extended'
diff --git a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
index 624f994deeeded56f15e7acd660e8873e7ed0518..f7eb42a590c9f9faca81b2e36d3b9b7d4067855c 100644
--- a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
+++ b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -48,6 +48,16 @@ public class Steps extends SchemaServiceRecordSteps {
super.the_schema_is_created_with_the_following_kind(dataTable);
}
+ @Then("^I set starting stateful scenarios$")
+ public void i_set_starting_stateful_scenarios() throws Throwable {
+ super.i_set_scenarios_as_stateful(true);
+ }
+
+ @Then("^I set ending stateful scenarios$")
+ public void i_set_ending_stateful_scenarios() throws Throwable {
+ super.i_set_scenarios_as_stateful(false);
+ }
+
@When("^I ingest records with the \"(.*?)\" with \"(.*?)\" for a given \"(.*?)\"$")
public void i_ingest_records_with_the_for_a_given(String record, String dataGroup, String kind) {
super.i_ingest_records_with_the_for_a_given(record, dataGroup, kind);
@@ -88,6 +98,16 @@ public class Steps extends SchemaServiceRecordSteps {
super.iShouldBeAbleToSearchRecordByTagKeyAndTagValue(index, tagKey, tagValue, expectedNumber);
}
+ @Then("^I clean up the index of the extended kinds \"([^\"]*)\" in the Elastic Search$")
+ public void iShouldCleanupIndicesOfExtendedKinds(String extendedKinds) throws Throwable {
+ super.iShouldCleanupIndicesOfExtendedKinds(extendedKinds);
+ }
+
+ @Then("^I should be able to search (\\d+) record with index \"([^\"]*)\" by extended data field \"([^\"]*)\" and value \"([^\"]*)\"$")
+ public void iShouldBeAbleToSearchRecordByFieldAndFieldValue(int expectedNumber, String index, String fieldKey, String fieldValue) throws Throwable {
+ super.iShouldBeAbleToSearchRecordByFieldAndFieldValue(index, fieldKey, fieldValue, expectedNumber);
+ }
+
@Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by bounding box query with points \\((-?\\d+), (-?\\d+)\\) and \\((-?\\d+), (-?\\d+)\\) on field \"(.*?)\"$")
public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery (
int expectedCount, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field) throws Throwable {
@@ -156,4 +176,4 @@ public class Steps extends SchemaServiceRecordSteps {
}
}
-}
\ No newline at end of file
+}