diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a8ca2bffe630a107f1c5bdefe2a0c27d94752b35..d611679b713eeac77e034ee7e8c639c70a97fcc7 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -5,7 +5,7 @@ variables:
AWS_SERVICE: indexer
AWS_SERVICE_GATEWAY: osdu-gateway
AWS_ENVIRONMENT: dev
- AWS_DEPLOY_TARGET: HELM
+ AWS_DEPLOY_TARGET: TF
AWS_EKS_DEPLOYMENT_NAME: os-indexer
GCP_BUILD_SUBDIR: provider/indexer-gcp
@@ -60,6 +60,7 @@ include:
file: "cloud-providers/gc-global.yml"
- local: "devops/gc/pipeline/override-stages.yml"
+ - local: "devops/aws/pipeline/override-stages.yml"
aws-test-java:
tags: ["aws-internal-test"]
diff --git a/NOTICE b/NOTICE
index ba7891fad64d315704b373871343c6a9acbd3352..60d60ed3a6a3f2f3300f41c3fb13d1ef9f61fe63 100644
--- a/NOTICE
+++ b/NOTICE
@@ -17,7 +17,6 @@ The following software have components provided under the terms of this license:
- Apache Commons CLI (from https://commons.apache.org/proper/commons-cli/, https://repo1.maven.org/maven2/commons-cli/commons-cli)
- Apache Geronimo JMS Spec 2.0 (from http://geronimo.apache.org/maven/${siteId}/${version})
-- Apache Log4j Core (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core)
- Apache Log4j JUL Adapter (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-jul)
- Apache Maven Invoker (from https://repo1.maven.org/maven2/org/apache/maven/shared/maven-invoker)
- Apache Maven Reporting API (from https://repo1.maven.org/maven2/org/apache/maven/reporting/maven-reporting-api)
@@ -53,6 +52,7 @@ The following software have components provided under the terms of this license:
- AMQP 1.0 JMS Spring Boot AutoConfiguration (from https://repo1.maven.org/maven2/org/amqphub/spring/amqp-10-jms-spring-boot-autoconfigure)
- AMQP 1.0 JMS Spring Boot Starter (from https://repo1.maven.org/maven2/org/amqphub/spring/amqp-10-jms-spring-boot-starter)
+- API Common (from https://github.com/googleapis, https://github.com/googleapis/api-common-java, https://repo1.maven.org/maven2/com/google/api/api-common)
- ASM Analysis (from http://asm.ow2.io/)
- ASM Commons (from http://asm.ow2.io/, https://repo1.maven.org/maven2/org/ow2/asm/asm-commons)
- ASM Tree (from http://asm.ow2.io/)
@@ -372,7 +372,7 @@ The following software have components provided under the terms of this license:
- Byte Buddy (without dependencies) (from https://repo1.maven.org/maven2/net/bytebuddy/byte-buddy)
- Byte Buddy Java agent (from https://repo1.maven.org/maven2/net/bytebuddy/byte-buddy-agent)
- ClassMate (from http://github.com/cowtowncoder/java-classmate)
-- Cloud Key Management Service (KMS) API v1-rev20230407-2.0.0 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-cloudkms)
+- Cloud Key Management Service (KMS) API v1-rev20230421-2.0.0 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-cloudkms)
- CloudWatch Metrics for AWS Java SDK (from https://aws.amazon.com/sdkforjava)
- Cobertura (from http://cobertura.sourceforge.net)
- Cobertura Limited Runtime (from http://cobertura.sourceforge.net)
@@ -392,6 +392,8 @@ The following software have components provided under the terms of this license:
- Doxia Sitetools :: Site Renderer (from http://maven.apache.org/doxia/doxia-sitetools/doxia-site-renderer/, https://repo1.maven.org/maven2/org/apache/maven/doxia/doxia-site-renderer)
- Elastic JNA Distribution (from https://github.com/java-native-access/jna)
- FindBugs-jsr305 (from http://findbugs.sourceforge.net/)
+- GAX (Google Api eXtensions) for Java (Core) (from https://github.com/googleapis, https://github.com/googleapis/gax-java, https://repo1.maven.org/maven2/com/google/api/gax)
+- GAX (Google Api eXtensions) for Java (gRPC) (from <https://repo1.maven.org/maven2/com/google/api/gax-grpc>, https://repo1.maven.org/maven2/com/google/api/gax-grpc)
- GSON extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-gson)
- Google APIs Client Library for Java (from https://repo1.maven.org/maven2/com/google/api-client/google-api-client)
- Google Cloud Core (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-core, https://github.com/googleapis/java-core, https://repo1.maven.org/maven2/com/google/cloud/google-cloud-core)
@@ -427,8 +429,8 @@ The following software have components provided under the terms of this license:
- JSON.simple (from http://code.google.com/p/json-simple/)
- JSONassert (from http://github.com/skyscreamer/yoga, https://github.com/skyscreamer/JSONassert)
- JSR107 API and SPI (from https://github.com/jsr107/jsr107spec)
+- JSpecify annotations (from http://jspecify.org/)
- Jackson 2 extensions to the Google APIs Client Library for Java (from https://repo1.maven.org/maven2/com/google/api-client/google-api-client-jackson2)
-- Jackson 2 extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-jackson2)
- Jackson dataformat: CBOR (from http://github.com/FasterXML/jackson-dataformats-binary)
- Jackson dataformat: Smile (from http://github.com/FasterXML/jackson-dataformat-smile, http://github.com/FasterXML/jackson-dataformats-binary)
- Jackson datatype: JSR310 (from http://wiki.fasterxml.com/JacksonModuleJSR310, https://repo1.maven.org/maven2/com/fasterxml/jackson/datatype/jackson-datatype-jsr310)
@@ -531,7 +533,7 @@ The following software have components provided under the terms of this license:
- OkHttp Logging Interceptor (from https://github.com/square/okhttp, https://repo1.maven.org/maven2/com/squareup/okhttp3/logging-interceptor, https://square.github.io/okhttp/)
- OkHttp URLConnection (from https://repo1.maven.org/maven2/com/squareup/okhttp3/okhttp-urlconnection, https://square.github.io/okhttp/)
- Okio (from https://github.com/square/okio/, https://repo1.maven.org/maven2/com/squareup/okio/okio)
-- OpenCensus (from https://github.com/census-instrumentation/opencensus-java)
+- OpenCensus (from https://github.com/census-instrumentation/opencensus-java, https://github.com/census-instrumentation/opencensus-proto)
- PWDB :: Database (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/database)
- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils, https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-utils)
- Plexus I18N Component (from https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-i18n)
@@ -620,11 +622,14 @@ The following software have components provided under the terms of this license:
- io.grpc:grpc-auth (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-context (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-core (from https://github.com/grpc/grpc-java)
+- io.grpc:grpc-googleapis (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-grpclb (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-netty-shaded (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-protobuf (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-protobuf-lite (from https://github.com/grpc/grpc-java)
+- io.grpc:grpc-services (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-stub (from https://github.com/grpc/grpc-java)
+- io.grpc:grpc-xds (from https://github.com/grpc/grpc-java)
- ion-java (from https://github.com/amzn/ion-java/, https://github.com/amznlabs/ion-java/)
- jackson-databind (from http://github.com/FasterXML/jackson, http://wiki.fasterxml.com/JacksonHome, https://github.com/FasterXML/jackson)
- jakarta.inject (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/jakarta.inject)
@@ -652,8 +657,8 @@ The following software have components provided under the terms of this license:
- perfmark:perfmark-api (from https://github.com/perfmark/perfmark)
- proto-google-cloud-logging-v2 (from https://github.com/googleapis/java-logging/proto-google-cloud-logging-v2, https://repo1.maven.org/maven2/com/google/api/grpc/proto-google-cloud-logging-v2)
- proto-google-cloud-pubsub-v1 (from https://github.com/googleapis/googleapis, https://github.com/googleapis/java-pubsub/proto-google-cloud-pubsub-v1)
-- proto-google-common-protos (from https://github.com/googleapis/api-client-staging, https://github.com/googleapis/gapic-generator-java, https://github.com/googleapis/googleapis, https://github.com/googleapis/java-iam/proto-google-common-protos)
-- proto-google-iam-v1 (from https://github.com/googleapis/gapic-generator-java, https://github.com/googleapis/googleapis, https://github.com/googleapis/java-iam/proto-google-iam-v1)
+- proto-google-common-protos (from https://github.com/googleapis/api-client-staging, https://github.com/googleapis/googleapis, https://github.com/googleapis/java-iam/proto-google-common-protos, https://github.com/googleapis/sdk-platform-java)
+- proto-google-iam-v1 (from https://github.com/googleapis/googleapis, https://github.com/googleapis/java-iam/proto-google-iam-v1, https://github.com/googleapis/sdk-platform-java)
- rank-eval (from https://github.com/elastic/elasticsearch, https://github.com/elastic/elasticsearch.git)
- resilience4j (from https://github.com/resilience4j/resilience4j, https://resilience4j.readme.io, ttps://resilience4j.readme.io)
- rest (from https://github.com/elastic/elasticsearch, https://github.com/elastic/elasticsearch.git)
@@ -681,13 +686,11 @@ BSD-2-Clause
========================================================================
The following software have components provided under the terms of this license:
-- API Common (from https://github.com/googleapis, https://github.com/googleapis/api-common-java, https://repo1.maven.org/maven2/com/google/api/api-common)
+- Apache Log4j Core (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core)
- Apache Lucene (module: memory) (from https://lucene.apache.org/)
- Apache Lucene (module: misc) (from https://lucene.apache.org/, https://repo1.maven.org/maven2/org/apache/lucene/lucene-misc)
- Apache Lucene (module: spatial-extras) (from https://lucene.apache.org/, https://repo1.maven.org/maven2/org/apache/lucene/lucene-spatial-extras)
- Apache Lucene (module: suggest) (from https://lucene.apache.org/, https://repo1.maven.org/maven2/org/apache/lucene/lucene-suggest)
-- GAX (Google Api eXtensions) for Java (Core) (from https://github.com/googleapis, https://github.com/googleapis/gax-java, https://repo1.maven.org/maven2/com/google/api/gax)
-- GAX (Google Api eXtensions) for Java (gRPC) (from <https://repo1.maven.org/maven2/com/google/api/gax-grpc>, https://repo1.maven.org/maven2/com/google/api/gax-grpc)
- Hamcrest (from http://hamcrest.org/JavaHamcrest/)
- Hamcrest Core (from http://hamcrest.org/, http://hamcrest.org/JavaHamcrest/, https://repo1.maven.org/maven2/org/hamcrest/hamcrest-core)
- JSch (from http://www.jcraft.com/jsch/)
@@ -758,6 +761,7 @@ The following software have components provided under the terms of this license:
- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils, https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-utils)
- Protocol Buffer Java API (from http://code.google.com/p/protobuf, https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java)
- Protocol Buffers [Util] (from https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java-util)
+- RE2/J (from http://github.com/google/re2j)
- Redisson (from http://redisson.org)
- ServiceLocator Default Implementation (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-locator)
- Spring Core (from http://www.springframework.org, https://github.com/spring-projects/spring-framework, https://repo1.maven.org/maven2/org/springframework/spring-core)
@@ -963,8 +967,6 @@ The following software have components provided under the terms of this license:
- HK2 core module (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-core)
- Jakarta Annotations API (from https://projects.eclipse.org/projects/ee4j.ca)
- Jakarta RESTful WS API (from https://github.com/eclipse-ee4j/jaxrs-api)
-- Java Architecture for XML Binding (from http://jaxb.java.net/, https://repo1.maven.org/maven2/javax/xml/bind/jaxb-api)
-- JavaBeans Activation Framework (from <http://java.sun.com/javase/technologies/desktop/javabeans/jaf/index.jsp>, http://java.sun.com/javase/technologies/desktop/javabeans/jaf/index.jsp, https://repo1.maven.org/maven2/com/sun/activation/javax.activation)
- RabbitMQ Java Client (from http://www.rabbitmq.com, https://www.rabbitmq.com)
- ServiceLocator Default Implementation (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-locator)
- aopalliance-repackaged (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/aopalliance-repackaged)
@@ -1026,7 +1028,6 @@ GPL-3.0-only
The following software have components provided under the terms of this license:
- Jakarta Annotations API (from https://projects.eclipse.org/projects/ee4j.ca)
-- Java Architecture for XML Binding (from http://jaxb.java.net/, https://repo1.maven.org/maven2/javax/xml/bind/jaxb-api)
========================================================================
GPL-3.0-or-later
diff --git a/devops/aws/pipeline/override-stages.yml b/devops/aws/pipeline/override-stages.yml
new file mode 100644
index 0000000000000000000000000000000000000000..0c48375d5baf61c09aceda46477925e53750d0d7
--- /dev/null
+++ b/devops/aws/pipeline/override-stages.yml
@@ -0,0 +1,40 @@
+aws-test-java:
+ extends:
+ - .maven
+ - .aws
+ - .aws_common_variables
+ - .aws_variables
+ stage: integration
+ needs: [{ job: 'aws-update-tf', optional: true }, { job: 'aws-update-helm', optional: true }, { job: 'aws-update-eks', optional: true }]
+ retry: 1
+ before_script:
+ - !reference [.maven, before_script]
+ - !reference [.aws, before_script]
+ - !reference [.aws_variables, before_script]
+ script:
+ - echo os-indexer override
+ - export ELASTIC_HOST=localhost
+ - export KUBECONFIG=/tmp/kubeconfig-${RANDOM}.yaml
+ - aws eks update-kubeconfig --region $AWS_REGION --name $EKS_CLUSTER_NAME --role-arn $EKS_CLUSTER_MGMT_ROLE
+
+ - localPort=$ELASTIC_PORT
+ - echo $localPort
+ - kubectl port-forward -n $TENANT_GROUP_NAME-tenant-$EKS_TENANT_NAME-elasticsearch svc/elasticsearch-es-http $localPort:$ELASTIC_PORT > /dev/null 2>&1 &
+ - export ELASTIC_PORT=$localPort
+ - pid=$!
+ - |
+ trap '{
+ echo killing "Port forward process: "$pid
+ kill $pid
+ rm $KUBECONFIG
+ }' EXIT
+ - $MAVEN_BUILD $INTEGRATION_TEST_DIR maven-aws-integration-test-output.txt ${AWS_MAVEN_TEST_COMMAND_OVERRIDE:-test} --update-snapshots -DdisableXmlReport=true
+ only:
+ variables:
+ #Default if not defined
+ - $AWS_SKIP_DEPLOY != 'true' && $AWS_SKIP_TESTS != 'true' && $AWS == '1' && ($AWS_INT_TEST_TYPE == 'java' || $AWS_INT_TEST_TYPE == null)
+ artifacts:
+ when: always
+ paths:
+ - $INTEGRATION_TEST_DIR
+ expire_in: 2 days
diff --git a/devops/azure/chart/templates/deployment.yaml b/devops/azure/chart/templates/deployment.yaml
index 400579dc3d4032f1683cf0565cea9f315e1c8294..baa8ce9c00c6847da3ddd5b2cb76371a819efd57 100644
--- a/devops/azure/chart/templates/deployment.yaml
+++ b/devops/azure/chart/templates/deployment.yaml
@@ -102,7 +102,7 @@ spec:
- name: servicebus_topic_name
value: indexing-progress
- name: reindex_topic_name
- value: recordstopic
+ value: reindextopic
- name: entitlements_service_endpoint
value: http://entitlements/api/entitlements/v2
- name: entitlements_service_api_key
@@ -125,4 +125,6 @@ spec:
value: "api://$(aad_client_id)"
- name: SPRING_CONFIG_NAME
value: "common,application"
- terminationGracePeriodSeconds: 101
\ No newline at end of file
+ - name: search_service_url
+ value: http://search/api/search/v2
+ terminationGracePeriodSeconds: 101
diff --git a/devops/gc/deploy/README.md b/devops/gc/deploy/README.md
index e2d9e0b9c1f8642f077e1d5139375a44d28ee686..eca88b9b059252c218980123b147d5bd137ec915 100644
--- a/devops/gc/deploy/README.md
+++ b/devops/gc/deploy/README.md
@@ -31,6 +31,7 @@ First you need to set variables in **values.yaml** file using any code editor. S
|------|-------------|------|---------|---------|
**global.domain** | your domain for the external endpoint, ex `example.com` | string | - | yes
**global.onPremEnabled** | whether on-prem is enabled | boolean | false | yes
+**global.limitsEnabled** | whether CPU and memory limits are enabled | boolean | true | yes
### Configmap variables
@@ -52,8 +53,8 @@ First you need to set variables in **values.yaml** file using any code editor. S
|------|-------------|------|---------|---------|
**data.requestsCpu** | amount of requested CPU | string | 35m | yes
**data.requestsMemory** | amount of requested memory| string | 640Mi | yes
-**data.limitsCpu** | CPU limit | string | 1 | yes
-**data.limitsMemory** | memory limit | string | 1G | yes
+**data.limitsCpu** | CPU limit | string | 1 | only if `global.limitsEnabled` is true
+**data.limitsMemory** | memory limit | string | 1G | only if `global.limitsEnabled` is true
**data.image** | service image | string | - | yes
**data.imagePullPolicy** | when to pull image | string | IfNotPresent | yes
**data.serviceAccountName** | name of your service account | string | indexer | yes
diff --git a/devops/gc/deploy/templates/configmap.yaml b/devops/gc/deploy/templates/configmap.yaml
index 73c17cfc8ff461e868f3f30a6164414d65a20dd0..9faeb58eec5831b880a8d355be89a512638354e4 100644
--- a/devops/gc/deploy/templates/configmap.yaml
+++ b/devops/gc/deploy/templates/configmap.yaml
@@ -22,3 +22,4 @@ data:
SECURITY_HTTPS_CERTIFICATE_TRUST: {{ .Values.data.securityHttpsCertificateTrust | quote }}
SPRING_PROFILES_ACTIVE: {{ .Values.data.springProfilesActive | quote }}
STORAGE_HOST: {{ .Values.data.storageHost | quote }}
+ SEARCH_BASE_HOST: {{ .Values.data.searchHost | quote }}
diff --git a/devops/gc/deploy/templates/deployment.yaml b/devops/gc/deploy/templates/deployment.yaml
index e42da5931b5959ba4850e0ecdb5d0e82b54aa7c2..06855a6c7b586f3bdb1b9b75d68488d4dce2c456 100644
--- a/devops/gc/deploy/templates/deployment.yaml
+++ b/devops/gc/deploy/templates/deployment.yaml
@@ -49,14 +49,16 @@ spec:
{{- end }}
securityContext:
allowPrivilegeEscalation: false
- runAsUser: 0
+ runAsNonRoot: true
ports:
- containerPort: 8080
resources:
requests:
cpu: {{ .Values.data.requestsCpu | quote }}
memory: {{ .Values.data.requestsMemory | quote }}
+ {{- if .Values.global.limitsEnabled }}
limits:
cpu: {{ .Values.data.limitsCpu | quote }}
memory: {{ .Values.data.limitsMemory | quote }}
+ {{- end }}
serviceAccountName: {{ .Values.data.serviceAccountName | quote }}
diff --git a/devops/gc/deploy/values.yaml b/devops/gc/deploy/values.yaml
index b657f2f62e2402cf28e70ccbec7d519bf682f6b4..96f0648927b09fdf48f165b4ce272180f4882862 100644
--- a/devops/gc/deploy/values.yaml
+++ b/devops/gc/deploy/values.yaml
@@ -5,6 +5,7 @@
global:
domain: ""
onPremEnabled: false
+ limitsEnabled: true
data:
# Configmap
@@ -15,6 +16,7 @@ data:
securityHttpsCertificateTrust: "true"
springProfilesActive: "gcp"
storageHost: "http://storage"
+ searchHost: "http://search"
# Deployment
requestsCpu: "35m"
requestsMemory: "640Mi"
diff --git a/devops/gc/pipeline/override-stages.yml b/devops/gc/pipeline/override-stages.yml
index e14557ddd5d4093b0d0a8a92a2c9d1214ec4217c..09b708fa22b593178e6b7c6c5a28ab0667b64df1 100644
--- a/devops/gc/pipeline/override-stages.yml
+++ b/devops/gc/pipeline/override-stages.yml
@@ -2,6 +2,11 @@ variables:
GC_SERVICE: indexer
GC_VENDOR: gc
-gc-anthos-test:
+gc-test:
variables:
- GC_VENDOR: anthos
+ CUCUMBER_OPTIONS: "--tags '~@* and @indexer-extended'"
+
+gc-baremetal-test:
+ variables:
+ GC_VENDOR: baremetal
+ CUCUMBER_OPTIONS: "--tags '~@* and @indexer-extended'"
diff --git a/devops/ibm/ibm-indexer-config/templates/configmap.yaml b/devops/ibm/ibm-indexer-config/templates/configmap.yaml
index fbb4ea24338a5cf021c1c5307ca12692ca1c94a6..32262305e44f44d4f8614f7850430eb9a108403a 100644
--- a/devops/ibm/ibm-indexer-config/templates/configmap.yaml
+++ b/devops/ibm/ibm-indexer-config/templates/configmap.yaml
@@ -53,6 +53,7 @@ data:
STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST: "http://{{ .Release.Name }}-ibm-storage-deploy:8080/api/storage/v2/query/records:batch"
STORAGE_RECORDS_BATCH_SIZE: "{{ .Values.data.storageRecordsBatchSize }}"
STORAGE_RECORDS_BY_KIND_BATCH_SIZE: "{{ .Values.data.storageRecordsByKindBatchSize }}"
+ SEARCH_HOST: "http://{{ .Release.Name }}-ibm-search-deploy:8080/api/service/v2"
#db
diff --git a/docs/tutorial/IndexerService.md b/docs/tutorial/IndexerService.md
index 7d32460d94b8774dcaf5e888baae2e17fd35fad6..f63d8c72510c6103c97c53bc93566d19030b5449 100644
--- a/docs/tutorial/IndexerService.md
+++ b/docs/tutorial/IndexerService.md
@@ -4,14 +4,14 @@
- [Indexer service](#indexer-service)
- [Introduction](#introduction)
+- [Features](#features)
+ - [Geoshape Decimation](#geoshape-decimation)
- [Indexer API access](#indexer-api-access)
- [API Reference](#api-reference)
- [Version info endpoint](#version-info-endpoint)
- [Reindex](#reindex)
- [Data Partition provision](#data-partition-provision)
- [Schema change](#schema-change)
-- [Schema Service adoption](#schema-service-adoption)
- - [R3 Schema Support](#r3-schema-support)
- [Troubleshoot Indexing Issues](#troubleshoot-indexing-issues)
- [Get indexing status](#get-indexing-status)
@@ -25,6 +25,20 @@ The indexer is indexes attributes defined in the schema. Schema can be created a
via Schema Service. The Indexer service also adds number of OSDU Data Platform meta attributes such as id, kind,
parent, acl, namespace, type, version, legaltags, index to each record at the time of indexing.
+## Features <a name="features"></a>
+
+### Geoshape Decimation <a name="geoshape-decimation"></a>
+
+In order to improve indexing and search performance for documents with large geometry, the geo-shape of the following
+GeoJSON types in the original shape attribute and virtual shape attribute if exists are decimated
+by implementing Ramer–Douglas–Peucker algorithm:
+- LineString
+- MultiLineString
+- Polygon
+- MultiPolygon
+
+The feature is enabled for all data partitions since M19.
+
## Indexer API access <a name="indexer-api-access"></a>
- Required roles
@@ -101,7 +115,9 @@ This endpoint takes information from files, generated by `spring-boot-maven-plug
### Reindex <a name="reindex"></a>
-Reindex API allows users to re-index a `kind` without re-ingesting the records via storage API. Reindexing a kind is an asynchronous operation and when a user calls this API, it will respond with HTTP 200 if it can launch the re-indexing or
+#### Reindex a 'kind'
+
+Reindex kind API allows users to re-index a `kind` without re-ingesting the records via storage API. Reindexing a kind is an asynchronous operation and when a user calls this API, it will respond with HTTP 200 if it can launch the re-indexing or
appropriate error code if it cannot. The current status of the indexing can be tracked by calling search API and making query with this particular kind. Please be advised, it may take few seconds to few hours to finish the re-indexing as
multiple factors contribute to latency, such as number of records in the kind, current load at the indexer service etc.
@@ -128,7 +144,7 @@ curl --request POST \
}'
```
-</details>
+</details><br>
#### Prerequisite
@@ -145,6 +161,58 @@ will use the same schema and overwrite records with the same ids. Default value
`kind` <br />
  (required, String) Kind to be re-indexed.
+#### Reindex given records
+
+Reindex records API allows users to re-index the given records by providing the record ids without re-ingesting the records via storage API. Reindexing a kind is an asynchronous operation and when a user calls this API, it will respond with HTTP 202 if it can launch the re-indexing or
+appropriate error code if it cannot. The response body indicates which given records were re-indexed and which ones were not found in storage. It supports up to 1000 records per API call.
+
+#### Request
+
+```http
+POST /api/indexer/v2/reindex/records HTTP/1.1
+{
+ "recordIds": ["opendes:work-product-component--WellLog:17763fcc18864f4f8eab62e320f8913d", "opendes:work-product-component--WellLog:566edebc-1a9f-4f4d-9a30-ed458e959ac7"]
+}
+```
+
+<details><summary>**Curl**</summary>
+
+```bash
+curl --request POST \
+ --url '/api/indexer/v2/reindex/records' \
+ --header 'accept: application/json' \
+ --header 'authorization: Bearer <JWT>' \
+ --header 'content-type: application/json' \
+ --header 'data-partition-id: opendes' \
+ --data '{
+ "recordIds": ["opendes:work-product-component--WellLog:17763fcc18864f4f8eab62e320f8913d", "opendes:work-product-component--WellLog:566edebc-1a9f-4f4d-9a30-ed458e959ac7"]
+}'
+```
+
+</details><br>
+
+#### Prerequisite
+
+Users must be a member of `users.datalake.admins` or `users.datalake.ops` group.
+
+#### Request body
+
+`recordIds` <br />
+  (required, Array of String) Storage records to be re-indexed.
+
+#### Example response
+
+```json
+{
+ "reIndexedRecords": [
+ "opendes:work-product-component--WellLog:566edebc-1a9f-4f4d-9a30-ed458e959ac7"
+ ],
+ "notFoundRecords": [
+ "opendes:work-product-component--WellLog:17763fcc18864f4f8eab62e320f8913d"
+ ]
+}
+```
+
[Back to table of contents](#TOC)
## Delete API <a name="delete"></a>
@@ -165,6 +233,7 @@ curl --request DELETE \
--header 'content-type: application/json' \
--header 'data-partition-id: opendes'
```
+</details><br>
### Data Partition provision <a name="data-partition-provision"></a>
@@ -185,7 +254,7 @@ curl --request PUT \
--header 'data-partition-id: opendes''
```
-</details>
+</details><br>
#### Prerequisite
@@ -233,52 +302,6 @@ POST /api/indexer/v2/_dps/task-handlers/schema-worker HTTP/1.1
`data` <br />
  (required, String) Schema change event message json string. Only `create` and `update` events are supported.
-## Schema Service adoption <a name="schema-service-adoption"></a>
-
-Indexer service is in adaptation process to use schemas from the Schema service instead of Storage Service. The Indexer
-Service retrieves a schema from the Schema Service if the schema is not found on the Storage Service. Change affects
-only Azure implementation so far. Later call to the Storage Service will be deprecated and then removed (after the end
-of the deprecation period).
-
-[Back to table of contents](#TOC)
-
-### R3 Schema Support <a name="r3-schema-support"></a>
-
-Indexer service support r3 schema. These schemas are created via Schema service.
-
-Here is an example following end-to-end workflow can be exercised (please update the schema based on your environment):
-
-- Ingest r3 schema for `opendes:wks:master-data--Wellbore:1.0.0`. Schema service payload can be
- found [here](https://community.opengroup.org/osdu/platform/system/indexer-service/-/blob/master/testing/indexer-test-core/src/main/resources/testData/r3-index_record_wks_master.schema.json)
- .
-
-- Ingest r3 master-data Wellbore record. Storage service payload can be
- found [here](https://community.opengroup.org/osdu/platform/system/indexer-service/-/blob/master/testing/indexer-test-core/src/main/resources/testData/r3-index_record_wks_master.json)
-
-- Records can be searched via Search service. Here is sample payload:
-
-```
-POST /api/search/v2/query HTTP/1.1
-Content-Type: application/json
-data-partition-id: opendes
-{
- "kind": "opendes:wks:master-data--Wellbore:1.0.0",
- "spatialFilter": {
- "field": "data.SpatialLocation.Wgs84Coordinates",
- "byBoundingBox": {
- "topLeft": {
- "longitude": -100.0,
- "latitude": 52.0
- },
- "bottomRight": {
- "longitude": 100.0,
- "latitude": 0.0
- }
- }
- }
-}
-```
-
[Back to table of contents](#TOC)
# Troubleshoot Indexing Issues <a name="troubleshoot-indexing-issues"></a>
@@ -324,28 +347,43 @@ Example:
Details of the index block:
-1) trace: This field collects all the issues related to the indexing and concatenates using '|'. This is a String field.
-2) statusCode: This field determines the category of the error. This is integer field. It can have the following values:
- - 200 - All OK
- - 404 - Schema is missing in Storage
- - 400 - Some fields were not properly mapped with the schema defined
-3) lastUpdateTime: This field captures the last time the record was updated by the indexer service. This is datetime
- field so you can do range queries on this field.
+1. `trace`: This field collects all the issues related to the indexing and concatenates using '|'. This is a string field.
+
+2. `statusCode`: This field determines the category of the error. This is an integer field. It can have the following values:
+ - 200 - All OK
+ - 404 - Schema is missing in Schema service.
+ - 400 - Some fields were not properly mapped with the schema defined, such as the schema defined as `int` for field, but the input record had an attribute value of `text` etc.
+
+3. `lastUpdateTime`: This field captures the last time the record was updated by the Indexer service. This is datetime field, so you can do range queries on this field.
You can query the index status using the following example query:
+```http
+POST /search/v2/query HTTP/1.1
+{
+ "kind": "*:*:*:*",
+ "query": "index.statusCode:404",
+ "limit": 1000,
+ "returnedFields": [ "id", "index" ]
+}
+```
+
+<details><summary>**Curl**</summary>
+
```bash
curl --request POST \
- --url /api/search/v2/query \
+ --url /search/v2/query \
--header 'Authorization: Token' \
--header 'Content-Type: application/json' \
- --header 'data-partition-id: Data partition id' \
+ --header 'Data-Partition-Id: opendes' \
--data '{"kind": "*:*:*:*","query": "index.statusCode:404","returnedFields": ["index"]}'
-
-NOTE: By default, the API response excludes the 'index' attribute block. The user must specify 'index' as the 'returnedFields" in order to see it in the response.
```
-The above query will return all records which had problems due to fields mismatch.
+</details><br>
+
+__Note__: By default, the API response excludes the `index` attribute block. You must specify `index` field in `returnedFields` in order to see it in the response.
+
+The above query returns all records which had problems due to fields mismatch.
Please refer to the [Search service](https://community.opengroup.org/osdu/platform/system/search-service/-/blob/master/docs/api/search_openapi.yaml#L28) documentation for examples on different kinds of search
queries.
diff --git a/docs/tutorial/PreviewFeatures.md b/docs/tutorial/PreviewFeatures.md
index 307360f59ddab73756195822f622d7ae9948142f..bdc3dc8800c05ebd64c937911452c8dbd74a2a54 100644
--- a/docs/tutorial/PreviewFeatures.md
+++ b/docs/tutorial/PreviewFeatures.md
@@ -1,25 +1,36 @@
## Geoshape Decimation
-In order to improve indexing and search performance for documents with large geometry, the geo-shape of the following
-GeoJSON types in the original shape attribute and virtual shape attribute if exists are decimated
+In order to improve indexing and search performance for documents with large geometry, the geo-shape of the following
+GeoJSON types in the original shape attribute and virtual shape attribute if exists are decimated
by implementing Ramer–Douglas–Peucker algorithm:
- LineString
- MultiLineString
- Polygon
-- MultiPolygon
+- MultiPolygon
+The feature is enabled for all data partitions since M19.
-The feature is enabled by default for all data partitions. If client does not want the geo-shape to be decimated in their
-data partitions, they can disable geo-shape decimation through the Partition Service.
-Here is an example to disable this feature by setting the property "indexer-decimation-enabled" in a given data partition:
+## Index extension
+
+OSDU Standard index extensions are defined by OSDU Data Definition work-streams with the intent to provide
+user/application friendly, derived properties. The standard set, together with the OSDU schemas, form the
+interoperability foundation. They can contribute to deliver domain specific APIs according to the Domain Driven Design
+principles.
+
+The configurations are encoded in OSDU reference-data records, one per each major schema version. The type name
+is IndexPropertyPathConfiguration. With this, the extension properties can be defined as if they were provided by a schema.
+
+In order to reduce the risk when extended evaluation of the solution is still on going, a feature flag that is managed by
+the Partition Service is applied to the solution. Here is an example to enable this feature by setting the property
+"index-augmenter-enabled" in a given data partition:
```
{
- "indexer-decimation-enabled": {
+ "index-augmenter-enabled": {
"sensitive": false,
- "value": "false"
+ "value": "true"
}
}
```
-If the property "indexer-decimation-enabled" is not created or the property value is set to "true" (String type) in the
-given data partition, the geo-shape decimation will be enabled.
+If the property "index-augmenter-enabled" is not created or the property value is set to "false" (String type) in the
+given data partition, the configurations defined as type IndexPropertyPathConfiguration will be ignored and index extension will be disabled.
diff --git a/indexer-core/pom.xml b/indexer-core/pom.xml
index b888f32f7dc8a12d46e2ff07887954dbb1da7af5..e81cf110809ed34b185a482b4a6c2cc71ccee31a 100644
--- a/indexer-core/pom.xml
+++ b/indexer-core/pom.xml
@@ -4,12 +4,12 @@
<parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-service</artifactId>
- <version>0.21.0-SNAPSHOT</version>
+ <version>0.22.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>indexer-core</artifactId>
- <version>0.21.0-SNAPSHOT</version>
+ <version>0.22.0-SNAPSHOT</version>
<name>indexer-core</name>
<description>Indexer Service Core</description>
<packaging>jar</packaging>
@@ -19,7 +19,7 @@
<gson.version>2.9.1</gson.version>
<netty.version>4.1.70.Final</netty.version>
<spring-webmvc.version>5.3.22</spring-webmvc.version>
- <os-core-common.version>0.21.0-rc4</os-core-common.version>
+ <os-core-common.version>0.21.0</os-core-common.version>
</properties>
<dependencyManagement>
@@ -48,6 +48,7 @@
<artifactId>snakeyaml</artifactId>
</dependency>
+
<dependency>
<groupId>org.hibernate.validator</groupId>
<artifactId>hibernate-validator</artifactId>
@@ -63,6 +64,12 @@
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-jersey</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.tomcat.embed</groupId>
+ <artifactId>tomcat-embed-core</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
@@ -107,6 +114,7 @@
<dependency>
<groupId>org.apache.tomcat.embed</groupId>
<artifactId>tomcat-embed-core</artifactId>
+ <version>9.0.75</version>
</dependency>
<dependency>
@@ -237,16 +245,18 @@
<version>2.7</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>xerces</groupId>
+ <artifactId>xercesImpl</artifactId>
+ <version>2.12.2</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.codehaus.plexus</groupId>
<artifactId>plexus-utils</artifactId>
<version>3.0.16</version>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.apache.tomcat.embed</groupId>
- <artifactId>tomcat-embed-core</artifactId>
- </dependency>
</dependencies>
<build>
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/PartitionSetupApi.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/PartitionSetupApi.java
index 56566ed015536ad821fe7328215f8e3f7ed98b24..6a5aae39f978613e70ecf1bf95bc6f1950707e47 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/PartitionSetupApi.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/PartitionSetupApi.java
@@ -21,9 +21,11 @@ import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.responses.ApiResponses;
import io.swagger.v3.oas.annotations.security.SecurityRequirement;
import io.swagger.v3.oas.annotations.tags.Tag;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppError;
import org.opengroup.osdu.indexer.logging.AuditLogger;
import org.opengroup.osdu.indexer.service.IClusterConfigurationService;
+import org.opengroup.osdu.indexer.service.IndexAliasService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.ResponseEntity;
import org.springframework.security.access.prepost.PreAuthorize;
@@ -46,27 +48,34 @@ public class PartitionSetupApi {
private static final String OPS = "users.datalake.ops";
+ @Autowired
+ private IndexAliasService indexAliasService;
@Autowired
private IClusterConfigurationService clusterConfigurationService;
@Autowired
+ private JaxRsDpsLog jaxRsDpsLog;
+ @Autowired
private AuditLogger auditLogger;
@Operation(summary = "${partitionSetupApi.provisionPartition.summary}", description = "${partitionSetupApi.provisionPartition.description}",
- security = {@SecurityRequirement(name = "Authorization")}, tags = { "partition-setup-api" })
+ security = {@SecurityRequirement(name = "Authorization")}, tags = {"partition-setup-api"})
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "OK"),
- @ApiResponse(responseCode = "400", description = "Bad Request", content = {@Content(schema = @Schema(implementation = AppError.class))}),
- @ApiResponse(responseCode = "401", description = "Unauthorized", content = {@Content(schema = @Schema(implementation = AppError.class))}),
- @ApiResponse(responseCode = "403", description = "User not authorized to perform the action", content = {@Content(schema = @Schema(implementation = AppError.class))}),
- @ApiResponse(responseCode = "404", description = "Not Found", content = {@Content(schema = @Schema(implementation = AppError.class))}),
- @ApiResponse(responseCode = "500", description = "Internal Server Error", content = {@Content(schema = @Schema(implementation = AppError.class))}),
- @ApiResponse(responseCode = "502", description = "Bad Gateway", content = {@Content(schema = @Schema(implementation = AppError.class))}),
- @ApiResponse(responseCode = "503", description = "Service Unavailable", content = {@Content(schema = @Schema(implementation = AppError.class))})
+ @ApiResponse(responseCode = "400", description = "Bad Request", content = {@Content(schema = @Schema(implementation = AppError.class))}),
+ @ApiResponse(responseCode = "401", description = "Unauthorized", content = {@Content(schema = @Schema(implementation = AppError.class))}),
+ @ApiResponse(responseCode = "403", description = "User not authorized to perform the action", content = {@Content(schema = @Schema(implementation = AppError.class))}),
+ @ApiResponse(responseCode = "404", description = "Not Found", content = {@Content(schema = @Schema(implementation = AppError.class))}),
+ @ApiResponse(responseCode = "500", description = "Internal Server Error", content = {@Content(schema = @Schema(implementation = AppError.class))}),
+ @ApiResponse(responseCode = "502", description = "Bad Gateway", content = {@Content(schema = @Schema(implementation = AppError.class))}),
+ @ApiResponse(responseCode = "503", description = "Service Unavailable", content = {@Content(schema = @Schema(implementation = AppError.class))})
})
@PreAuthorize("@authorizationFilter.hasPermission('" + OPS + "')")
@PutMapping(path = "/provision", consumes = "application/json")
public ResponseEntity<?> provisionPartition(@RequestHeader(DATA_PARTITION_ID) String dataPartitionId) throws IOException {
+ this.jaxRsDpsLog.info("applying cluster configuration for partition: " + dataPartitionId);
this.clusterConfigurationService.updateClusterConfiguration();
+ this.jaxRsDpsLog.info("creating default alias for all pre-exiting indices for partition: " + dataPartitionId);
+ this.indexAliasService.createIndexAliasesForAll();
this.auditLogger.getConfigurePartition(singletonList(dataPartitionId));
return new ResponseEntity<>(org.springframework.http.HttpStatus.OK);
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/RecordIndexerApi.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/RecordIndexerApi.java
index 6f38cd0644e62c2ade527f2a69d79ede2c2181e6..93d09b0da1f98686c1bb88bbd0c2157bfa6cc481 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/RecordIndexerApi.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/RecordIndexerApi.java
@@ -103,7 +103,7 @@ public class RecordIndexerApi {
public ResponseEntity<?> reindex(
@RequestBody @NotNull(message = SwaggerDoc.REQUEST_VALIDATION_NOT_NULL_BODY)
@Valid RecordReindexRequest recordReindexRequest) {
- return new ResponseEntity<>(reIndexService.reindexRecords(recordReindexRequest, false), HttpStatus.OK);
+ return new ResponseEntity<>(reIndexService.reindexKind(recordReindexRequest, false), HttpStatus.OK);
}
// THIS IS AN INTERNAL USE API ONLY
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java
index 8d719c247bb1a6aa8fcc8f24877703a31aa743b8..c4ea75f991793fcea471a4cee6f51b369be316c1 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java
@@ -24,10 +24,14 @@ import io.swagger.v3.oas.annotations.security.SecurityRequirement;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.opengroup.osdu.core.common.model.http.AppError;
import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
+import org.opengroup.osdu.core.common.model.indexer.Records;
import org.opengroup.osdu.core.common.model.search.SearchServiceRole;
import org.opengroup.osdu.indexer.logging.AuditLogger;
+import org.opengroup.osdu.indexer.model.ReindexRecordsRequest;
+import org.opengroup.osdu.indexer.model.ReindexRecordsResponse;
import org.opengroup.osdu.indexer.service.IndexSchemaService;
import org.opengroup.osdu.indexer.service.ReindexService;
+import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.PatchMapping;
@@ -42,6 +46,8 @@ import javax.inject.Inject;
import javax.validation.Valid;
import javax.validation.constraints.NotNull;
import java.io.IOException;
+import java.util.List;
+import java.util.stream.Collectors;
import static java.util.Collections.singletonList;
@@ -58,6 +64,29 @@ public class ReindexApi {
@Inject
private AuditLogger auditLogger;
+ @Operation(summary = "${reindexApi.reindexRecords.summary}", description = "${reindexApi.reindexRecords.description}",
+ security = {@SecurityRequirement(name = "Authorization")}, tags = { "reindex-api" })
+ @ApiResponses(value = {
+ @ApiResponse(responseCode = "202", description = "Accepted"),
+ @ApiResponse(responseCode = "400", description = "Bad Request", content = {@Content(schema = @Schema(implementation = AppError.class))}),
+ @ApiResponse(responseCode = "401", description = "Unauthorized", content = {@Content(schema = @Schema(implementation = AppError.class))}),
+ @ApiResponse(responseCode = "403", description = "User not authorized to perform the action", content = {@Content(schema = @Schema(implementation = AppError.class))}),
+ @ApiResponse(responseCode = "404", description = "Not Found", content = {@Content(schema = @Schema(implementation = AppError.class))}),
+ @ApiResponse(responseCode = "500", description = "Internal Server Error", content = {@Content(schema = @Schema(implementation = AppError.class))}),
+ @ApiResponse(responseCode = "502", description = "Bad Gateway", content = {@Content(schema = @Schema(implementation = AppError.class))}),
+ @ApiResponse(responseCode = "503", description = "Service Unavailable", content = {@Content(schema = @Schema(implementation = AppError.class))})
+ })
+ @PreAuthorize("@authorizationFilter.hasPermission('" + SearchServiceRole.ADMIN + "')")
+ @PostMapping(path = "/records", consumes = "application/json")
+ public ResponseEntity<?> reindexRecords(@NotNull @Valid @RequestBody ReindexRecordsRequest reindexRecordsRequest) {
+ Records records = this.reIndexService.reindexRecords(reindexRecordsRequest.getRecordIds());
+ List<String> reindexedRecords = records.getRecords().stream().map(Records.Entity::getId).collect(Collectors.toList());
+ if (!reindexedRecords.isEmpty()) {
+ this.auditLogger.getReindexRecords(reindexedRecords);
+ }
+ return new ResponseEntity<>(ReindexRecordsResponse.builder().reIndexedRecords(records.getRecords().stream().map(Records.Entity::getId).collect(Collectors.toList())).notFoundRecords(records.getNotFound()).build(), HttpStatus.ACCEPTED);
+ }
+
@Operation(summary = "${reindexApi.reindex.summary}", description = "${reindexApi.reindex.description}",
security = {@SecurityRequirement(name = "Authorization")}, tags = { "reindex-api" })
@ApiResponses(value = {
@@ -75,7 +104,7 @@ public class ReindexApi {
public ResponseEntity<?> reindex(@NotNull @Valid @RequestBody RecordReindexRequest recordReindexRequest,
@Parameter(description = "Force Clean")
@RequestParam(value = "force_clean", defaultValue = "false") boolean forceClean) throws IOException {
- this.reIndexService.reindexRecords(recordReindexRequest, this.indexSchemaService.isStorageSchemaSyncRequired(recordReindexRequest.getKind(), forceClean));
+ this.reIndexService.reindexKind(recordReindexRequest, this.indexSchemaService.isStorageSchemaSyncRequired(recordReindexRequest.getKind(), forceClean));
this.auditLogger.getReindex(singletonList(recordReindexRequest.getKind()));
return new ResponseEntity<>(org.springframework.http.HttpStatus.OK);
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/AbstractPartitionSafeCache.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/AbstractPartitionSafeCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..52d2cdf9e7e5220bd4c34293683fe6322b278d87
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/AbstractPartitionSafeCache.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.ICache;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.springframework.web.context.annotation.RequestScope;
+
+import javax.inject.Inject;
+
+@RequestScope
+public abstract class AbstractPartitionSafeCache<K, V> implements ICache<K, V> {
+ @Inject
+ private IRequestInfo requestInfo;
+
+ protected String cacheKey(String s) {
+ return this.requestInfo.getPartitionId() + "-" + s;
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/ChildrenKindsCacheVmImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/ChildrenKindsCacheVmImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..a200d0b25a166a40316e320f4d70d4cae247fef7
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/ChildrenKindsCacheVmImpl.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.VmCache;
+import org.opengroup.osdu.indexer.model.Constants;
+import org.opengroup.osdu.indexer.model.indexproperty.ChildrenKinds;
+import org.springframework.stereotype.Component;
+
+@Component
+public class ChildrenKindsCacheVmImpl implements IChildrenKindsCache{
+ private VmCache<String, ChildrenKinds> cache;
+
+ public ChildrenKindsCacheVmImpl() {
+ cache = new VmCache<>(Constants.SPEC_CACHE_EXPIRATION, Constants.SPEC_MAX_CACHE_SIZE);
+ }
+
+ @Override
+ public void put(String s, ChildrenKinds o) {
+ this.cache.put(s, o);
+ }
+
+ @Override
+ public ChildrenKinds get(String s) {
+ return this.cache.get(s);
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(s);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/geo/decimator/DecimationSettingCache.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/FeatureFlagCache.java
similarity index 83%
rename from indexer-core/src/main/java/org/opengroup/osdu/indexer/util/geo/decimator/DecimationSettingCache.java
rename to indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/FeatureFlagCache.java
index 127d9e7079bad1ac0d6c6c12d2aa50d32c09aca9..d57a5dbffda83a72c785a692da47d0bb3b756b64 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/geo/decimator/DecimationSettingCache.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/FeatureFlagCache.java
@@ -13,14 +13,14 @@
* limitations under the License.
*/
-package org.opengroup.osdu.indexer.util.geo.decimator;
+package org.opengroup.osdu.indexer.cache;
import org.opengroup.osdu.core.common.cache.VmCache;
import org.springframework.stereotype.Component;
@Component
-public class DecimationSettingCache extends VmCache<String, Boolean> {
- public DecimationSettingCache() {
+public class FeatureFlagCache extends VmCache<String, Boolean> {
+ public FeatureFlagCache() {
super(300, 1000);
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IChildrenKindsCache.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IChildrenKindsCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..dda43845814be42a0cee4131922343acebe9120f
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IChildrenKindsCache.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.ICache;
+import org.opengroup.osdu.indexer.model.indexproperty.ChildrenKinds;
+
+public interface IChildrenKindsCache extends ICache<String, ChildrenKinds> {
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IKindCache.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IKindCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..c474065deded2a236e41e7d43436b874d9c295c1
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IKindCache.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.ICache;
+
+public interface IKindCache extends ICache<String, String> {
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IParentChildRelationshipSpecsCache.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IParentChildRelationshipSpecsCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..74aa48e4fadeb7d477c1fed19aef9dc9662619d7
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IParentChildRelationshipSpecsCache.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.ICache;
+import org.opengroup.osdu.indexer.model.indexproperty.ParentChildRelationshipSpecs;
+
+public interface IParentChildRelationshipSpecsCache extends ICache<String, ParentChildRelationshipSpecs> {
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IPropertyConfigurationsCache.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IPropertyConfigurationsCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..24a51559f9955861969a2536de31c397b2e6e85f
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IPropertyConfigurationsCache.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.ICache;
+import org.opengroup.osdu.indexer.model.indexproperty.PropertyConfigurations;
+
+public interface IPropertyConfigurationsCache extends ICache<String, PropertyConfigurations> {
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IPropertyConfigurationsEnabledCache.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IPropertyConfigurationsEnabledCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..4b6c372d333c2c5539ab5d1fab3f41dfdabff7e7
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IPropertyConfigurationsEnabledCache.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.ICache;
+
+public interface IPropertyConfigurationsEnabledCache extends ICache<String, Boolean> {
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IRecordChangeInfoCache.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IRecordChangeInfoCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..25591747a59e9f2d1a4de0f7217331979365d499
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IRecordChangeInfoCache.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.ICache;
+import org.opengroup.osdu.indexer.model.RecordChangeInfo;
+
+public interface IRecordChangeInfoCache extends ICache<String, RecordChangeInfo> {
+}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/mock/PartitionFactoryMock.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IRelatedObjectCache.java
similarity index 58%
rename from indexer-core/src/test/java/org/opengroup/osdu/indexer/service/mock/PartitionFactoryMock.java
rename to indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IRelatedObjectCache.java
index dbe1ee22b0020d99eff0cc1b585c48400ce958e8..e33200e568cc575fc8d7f3ad1aeb9f24415a03e5 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/mock/PartitionFactoryMock.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/IRelatedObjectCache.java
@@ -13,15 +13,12 @@
* limitations under the License.
*/
-package org.opengroup.osdu.indexer.service.mock;
+package org.opengroup.osdu.indexer.cache;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.partition.IPartitionFactory;
-import org.opengroup.osdu.core.common.partition.IPartitionProvider;
+import org.opengroup.osdu.core.common.cache.ICache;
+import org.opengroup.osdu.core.common.model.storage.RecordData;
-public class PartitionFactoryMock implements IPartitionFactory {
- @Override
- public IPartitionProvider create(DpsHeaders dpsHeaders) {
- return new PartitionProviderMock();
- }
+import java.util.Map;
+
+public interface IRelatedObjectCache extends ICache<String, RecordData> {
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/KindCacheVmImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/KindCacheVmImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..2eec5cead55d7a78c62ff3c3a946070edbdca02f
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/KindCacheVmImpl.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.VmCache;
+import org.opengroup.osdu.indexer.model.Constants;
+import org.springframework.stereotype.Component;
+
+@Component
+public class KindCacheVmImpl implements IKindCache {
+
+ private VmCache<String, String> cache;
+
+ public KindCacheVmImpl() {
+ cache = new VmCache<>(Constants.SPEC_CACHE_EXPIRATION, Constants.SPEC_MAX_CACHE_SIZE);
+ }
+
+ @Override
+ public void put(String s, String o) {
+ this.cache.put(s, o);
+ }
+
+ @Override
+ public String get(String s) {
+ return this.cache.get(s);
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(s);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/ParentChildRelationshipSpecsCacheVmImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/ParentChildRelationshipSpecsCacheVmImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..a270ab77d5141ffca5130f607eec92e40264ba82
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/ParentChildRelationshipSpecsCacheVmImpl.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.VmCache;
+import org.opengroup.osdu.indexer.model.Constants;
+import org.opengroup.osdu.indexer.model.indexproperty.ParentChildRelationshipSpecs;
+import org.springframework.stereotype.Component;
+
+@Component
+public class ParentChildRelationshipSpecsCacheVmImpl implements IParentChildRelationshipSpecsCache {
+ private VmCache<String, ParentChildRelationshipSpecs> cache;
+
+ public ParentChildRelationshipSpecsCacheVmImpl() {
+ cache = new VmCache<>(Constants.SPEC_CACHE_EXPIRATION, Constants.SPEC_MAX_CACHE_SIZE);
+ }
+
+ @Override
+ public void put(String s, ParentChildRelationshipSpecs o) {
+ this.cache.put(s, o);
+ }
+
+ @Override
+ public ParentChildRelationshipSpecs get(String s) {
+ return this.cache.get(s);
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(s);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafeChildrenKindsCache.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafeChildrenKindsCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..59dd354871e2d722a444dffb6204dcd9f6a8e018
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafeChildrenKindsCache.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.indexer.model.indexproperty.ChildrenKinds;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+import javax.inject.Inject;
+
+
+@Component
+@RequestScope
+public class PartitionSafeChildrenKindsCache extends AbstractPartitionSafeCache<String, ChildrenKinds>{
+ @Inject
+ private IChildrenKindsCache cache;
+
+ @Override
+ public void put(String s, ChildrenKinds o) {
+ this.cache.put(cacheKey(s), o);
+ }
+
+ @Override
+ public ChildrenKinds get(String s) {
+ return this.cache.get(cacheKey(s));
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(cacheKey(s));
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafeFlattenedSchemaCache.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafeFlattenedSchemaCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..862acb6e9399e3a7a32acc1a0d9303c6730de40d
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafeFlattenedSchemaCache.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+import javax.inject.Inject;
+
+@Component
+@RequestScope
+public class PartitionSafeFlattenedSchemaCache extends AbstractPartitionSafeCache<String, String> {
+ private static final String FLATTENED_SCHEMA = "_flattened";
+ @Inject
+ private ISchemaCache schemaCache;
+
+ @Override
+ public void put(String s, String o) {
+ this.schemaCache.put(getKey(s), o);
+ }
+
+ @Override
+ public String get(String s) {
+ return (String)this.schemaCache.get(getKey(s));
+ }
+
+ @Override
+ public void delete(String s) {
+ this.schemaCache.delete(getKey(s));
+ }
+
+ @Override
+ public void clearAll() {
+ this.schemaCache.clearAll();
+ }
+
+ private String getKey(String s) {
+ return cacheKey(s) + FLATTENED_SCHEMA;
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafeKindCache.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafeKindCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..e1bc5adce2686070b5cfed0587823b7b2e4700b4
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafeKindCache.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+import javax.inject.Inject;
+
+@Component
+@RequestScope
+public class PartitionSafeKindCache extends AbstractPartitionSafeCache<String, String> {
+ @Inject
+ private IKindCache cache;
+
+ @Override
+ public void put(String s, String o) {
+ this.cache.put(cacheKey(s), o);
+ }
+
+ @Override
+ public String get(String s) {
+ return this.cache.get(cacheKey(s));
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(cacheKey(s));
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafeParentChildRelationshipSpecsCache.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafeParentChildRelationshipSpecsCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..1ca0c87aa4215064fd15d4f9674db0830184994e
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafeParentChildRelationshipSpecsCache.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.indexer.model.indexproperty.ParentChildRelationshipSpecs;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+import javax.inject.Inject;
+
+@Component
+@RequestScope
+public class PartitionSafeParentChildRelationshipSpecsCache extends AbstractPartitionSafeCache<String, ParentChildRelationshipSpecs> {
+ @Inject
+ private IParentChildRelationshipSpecsCache cache;
+
+ @Override
+ public void put(String s, ParentChildRelationshipSpecs o) {
+ this.cache.put(cacheKey(s), o);
+ }
+
+ @Override
+ public ParentChildRelationshipSpecs get(String s) {
+ return this.cache.get(cacheKey(s));
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(cacheKey(s));
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafePropertyConfigurationsCache.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafePropertyConfigurationsCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..04a4d4082d91071722a2bc3500f86fd676ecae11
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafePropertyConfigurationsCache.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.indexer.model.indexproperty.PropertyConfigurations;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+import javax.inject.Inject;
+
+@Component
+@RequestScope
+public class PartitionSafePropertyConfigurationsCache extends AbstractPartitionSafeCache<String,PropertyConfigurations> {
+ @Inject
+ private IPropertyConfigurationsCache cache;
+
+ @Override
+ public void put(String s, PropertyConfigurations o) {
+ this.cache.put(cacheKey(s), o);
+ }
+
+ @Override
+ public PropertyConfigurations get(String s) {
+ return this.cache.get(cacheKey(s));
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(cacheKey(s));
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafePropertyConfigurationsEnabledCache.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafePropertyConfigurationsEnabledCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..e8bea9da70c3903302e5172dd20bcd9a622687b3
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafePropertyConfigurationsEnabledCache.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+import javax.inject.Inject;
+
+@Component
+@RequestScope
+public class PartitionSafePropertyConfigurationsEnabledCache extends AbstractPartitionSafeCache<String,Boolean> {
+ @Inject
+ private IPropertyConfigurationsEnabledCache cache;
+
+ @Override
+ public void put(String s, Boolean o) {
+ this.cache.put(cacheKey(s), o);
+ }
+
+ @Override
+ public Boolean get(String s) {
+ return this.cache.get(cacheKey(s));
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(cacheKey(s));
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafeSchemaCache.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafeSchemaCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..4cbb38658b60e8eedc209e8f75023d8f983ccf02
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PartitionSafeSchemaCache.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache;
+import org.springframework.stereotype.Component;
+import org.springframework.web.context.annotation.RequestScope;
+
+import javax.inject.Inject;
+
+@Component
+@RequestScope
+public class PartitionSafeSchemaCache extends AbstractPartitionSafeCache<String, String> {
+ @Inject
+ private ISchemaCache schemaCache;
+
+ @Override
+ public void put(String s, String o) {
+ this.schemaCache.put(cacheKey(s), o);
+ }
+
+ @Override
+ public String get(String s) {
+ return (String)this.schemaCache.get(cacheKey(s));
+ }
+
+ @Override
+ public void delete(String s) {
+ this.schemaCache.delete(cacheKey(s));
+ }
+
+ @Override
+ public void clearAll() {
+ this.schemaCache.clearAll();
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PropertyConfigurationsCacheVmImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PropertyConfigurationsCacheVmImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..4e8e0605237f42572ae0f4eb85e59c26a3d8248e
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PropertyConfigurationsCacheVmImpl.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.VmCache;
+import org.opengroup.osdu.indexer.model.Constants;
+import org.opengroup.osdu.indexer.model.indexproperty.PropertyConfigurations;
+import org.springframework.stereotype.Component;
+
+@Component
+public class PropertyConfigurationsCacheVmImpl implements IPropertyConfigurationsCache {
+ private VmCache<String, PropertyConfigurations> cache;
+
+ public PropertyConfigurationsCacheVmImpl() {
+ cache = new VmCache<>(Constants.SPEC_CACHE_EXPIRATION, Constants.SPEC_MAX_CACHE_SIZE);
+ }
+
+ @Override
+ public void put(String s, PropertyConfigurations o) {
+ this.cache.put(s, o);
+ }
+
+ @Override
+ public PropertyConfigurations get(String s) {
+ return this.cache.get(s);
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(s);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PropertyConfigurationsEnabledCacheVmImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PropertyConfigurationsEnabledCacheVmImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..99d2b6d3a6b8f09001164804d6135d06f659e4b5
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/PropertyConfigurationsEnabledCacheVmImpl.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.VmCache;
+import org.opengroup.osdu.indexer.model.Constants;
+import org.springframework.stereotype.Component;
+
+@Component
+public class PropertyConfigurationsEnabledCacheVmImpl implements IPropertyConfigurationsEnabledCache {
+ private VmCache<String, Boolean> cache;
+
+ public PropertyConfigurationsEnabledCacheVmImpl() {
+ cache = new VmCache<>(Constants.SPEC_CACHE_EXPIRATION, Constants.SPEC_MAX_CACHE_SIZE);
+ }
+
+ @Override
+ public void put(String s, Boolean o) {
+ this.cache.put(s,o);
+ }
+
+ @Override
+ public Boolean get(String s) {
+ return this.cache.get(s);
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(s);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/RecordChangeInfoCacheVmImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/RecordChangeInfoCacheVmImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..24a3467f8037d4d78b97f034684f31742ca149db
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/RecordChangeInfoCacheVmImpl.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.VmCache;
+import org.opengroup.osdu.indexer.model.Constants;
+import org.opengroup.osdu.indexer.model.RecordChangeInfo;
+import org.springframework.stereotype.Component;
+
+@Component
+public class RecordChangeInfoCacheVmImpl implements IRecordChangeInfoCache {
+ private VmCache<String, RecordChangeInfo> cache;
+
+ public RecordChangeInfoCacheVmImpl() {
+ cache = new VmCache<>(Constants.DATA_CACHE_EXPIRATION, Constants.DATA_MAX_CACHE_SIZE);
+ }
+
+ @Override
+ public void put(String s, RecordChangeInfo o) {
+ this.cache.put(s, o);
+ }
+
+ @Override
+ public RecordChangeInfo get(String s) {
+ return this.cache.get(s);
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(s);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/RelatedObjectCacheVmImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/RelatedObjectCacheVmImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..f3d2e166ab0eefe6151f18ac9ad52fd033450abd
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/RelatedObjectCacheVmImpl.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.cache;
+
+import org.opengroup.osdu.core.common.cache.VmCache;
+import org.opengroup.osdu.core.common.model.storage.RecordData;
+import org.opengroup.osdu.indexer.model.Constants;
+import org.springframework.stereotype.Component;
+
+import java.util.Map;
+
+@Component
+public class RelatedObjectCacheVmImpl implements IRelatedObjectCache {
+ private VmCache<String, RecordData> cache;
+
+ public RelatedObjectCacheVmImpl() {
+ cache = new VmCache<>(Constants.DATA_CACHE_EXPIRATION, Constants.DATA_MAX_CACHE_SIZE);
+ }
+
+ @Override
+ public void put(String s, RecordData o) {
+ this.cache.put(s, o);
+ }
+
+ @Override
+ public RecordData get(String s) {
+ return this.cache.get(s);
+ }
+
+ @Override
+ public void delete(String s) {
+ this.cache.delete(s);
+ }
+
+ @Override
+ public void clearAll() {
+ this.cache.clearAll();
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditEvents.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditEvents.java
index ad0a6cc98435dd6efbf1e6457617656ac1ded7f0..9a183e80df3e9216481c6d2f1673f3c62e50cf3f 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditEvents.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditEvents.java
@@ -63,6 +63,9 @@ public class AuditEvents {
private static final String INDEX_DELETE_SUCCESS = "Successfully deleted index";
private static final String INDEX_DELETE_FAILURE = "Failed deleting index";
+ private static final String REINDEX_RECORDS_ACTION_ID = "IN0014";
+ private static final String REINDEX_RECORDS_OPERATION = "Reindex records";
+
private final String user;
public AuditEvents(String user) {
@@ -204,6 +207,17 @@ public class AuditEvents {
.build();
}
+ public AuditPayload getReindexRecordsEvent(List<String> resources) {
+ return AuditPayload.builder()
+ .action(AuditAction.CREATE)
+ .status(AuditStatus.SUCCESS)
+ .actionId(REINDEX_RECORDS_ACTION_ID)
+ .message(REINDEX_RECORDS_OPERATION)
+ .resources(resources)
+ .user(this.user)
+ .build();
+ }
+
public AuditPayload getCopyIndexEvent(List<String> resources) {
return AuditPayload.builder()
.action(AuditAction.CREATE)
@@ -269,4 +283,4 @@ public class AuditEvents {
.build();
}
}
-}
\ No newline at end of file
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditLogger.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditLogger.java
index e02f7a2ea8ab30659fd943f81567472f13400151..792f46de627e96f2f5162cde7865693712ef5bc0 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditLogger.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditLogger.java
@@ -89,6 +89,10 @@ public class AuditLogger {
this.writeLog(this.getAuditEvents().getReindexEvent(resources));
}
+ public void getReindexRecords(List<String> resources) {
+ this.writeLog(this.getAuditEvents().getReindexRecordsEvent(resources));
+ }
+
public void copyIndex(List<String> resources) {
this.writeLog(this.getAuditEvents().getCopyIndexEvent(resources));
}
@@ -116,4 +120,4 @@ public class AuditLogger {
private void writeLog(AuditPayload log) {
this.logger.audit(log);
}
-}
\ No newline at end of file
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/BulkRequestResult.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/BulkRequestResult.java
new file mode 100644
index 0000000000000000000000000000000000000000..ca58f271454556057b3e41c078bb3c2b397bfd27
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/BulkRequestResult.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model;
+
+import java.util.List;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+@Data
+@AllArgsConstructor
+public class BulkRequestResult {
+ private List<String> failureRecordIds;
+ private List<String> retryUpsertRecordIds;
+}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/Constants.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/Constants.java
new file mode 100644
index 0000000000000000000000000000000000000000..91017c2fe1fbfa1773fcd611773849e47a889d5c
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/Constants.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model;
+
+public class Constants {
+ // It should be moved to core common later
+ public static final String ANCESTRY_KINDS = "ancestry_kinds";
+
+ // Specifications using kind as key is not partition safe if the specifications are per data partition
+ public static final int SPEC_CACHE_EXPIRATION = 600;
+ public static final int SPEC_MAX_CACHE_SIZE = 2000;
+
+ // Data id itself is partition safe
+ public static final int DATA_CACHE_EXPIRATION = 120;
+ public static final int DATA_MAX_CACHE_SIZE = 2000;
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/IndexAliasesResult.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/IndexAliasesResult.java
new file mode 100644
index 0000000000000000000000000000000000000000..c53f64d4ca68cb6f43709b2f6283f231f26f66b2
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/IndexAliasesResult.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import lombok.Data;
+import lombok.ToString;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@Data
+@ToString
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class IndexAliasesResult {
+ private List<String> indicesWithAliases;
+ private List<String> indicesWithoutAliases;
+
+ public IndexAliasesResult() {
+ indicesWithAliases = new ArrayList<>();
+ indicesWithoutAliases = new ArrayList<>();
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/RecordChangeInfo.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/RecordChangeInfo.java
new file mode 100644
index 0000000000000000000000000000000000000000..94dc1c62ed435796310e3545dabb91282d20cfba
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/RecordChangeInfo.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model;
+
+import lombok.Data;
+import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
+
+import java.util.List;
+
+@Data
+public class RecordChangeInfo {
+ private List<String> updatedProperties;
+ private RecordInfo recordInfo;
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/ReindexRecordsRequest.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/ReindexRecordsRequest.java
new file mode 100644
index 0000000000000000000000000000000000000000..d11570ec1f0e5402ad8fbb0b3bba2603ccfe391a
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/ReindexRecordsRequest.java
@@ -0,0 +1,33 @@
+// Copyright 2023, SLB
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import javax.validation.constraints.NotBlank;
+import javax.validation.constraints.NotNull;
+import javax.validation.constraints.Size;
+import java.util.List;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class ReindexRecordsRequest {
+ @NotNull
+ @Size(min = 1, max = 1000)
+ private List<@NotBlank String> recordIds;
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/ReindexRecordsResponse.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/ReindexRecordsResponse.java
new file mode 100644
index 0000000000000000000000000000000000000000..b0f4eee4ccc29fae4465f04117cf12fea5afb253
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/ReindexRecordsResponse.java
@@ -0,0 +1,27 @@
+// Copyright 2023, SLB
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.model;
+
+import lombok.Builder;
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+@Builder
+public class ReindexRecordsResponse {
+ private List<String> reIndexedRecords;
+ private List<String> notFoundRecords;
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SchemaIdentity.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SchemaIdentity.java
new file mode 100644
index 0000000000000000000000000000000000000000..0ad77f4229ce06827993d2b4090bd7da3c54ecd6
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SchemaIdentity.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import lombok.Data;
+
+@Data
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class SchemaIdentity {
+ String authority;
+ String source;
+ private String entityType;
+ private int schemaVersionMajor;
+ private int schemaVersionMinor;
+ private int schemaVersionPatch;
+ private String id;
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SchemaInfo.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SchemaInfo.java
new file mode 100644
index 0000000000000000000000000000000000000000..3cfa9ceb8cdb3e5378922b7303347134a662b6bf
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SchemaInfo.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model;
+
+import lombok.Data;
+import lombok.ToString;
+
+@Data
+@ToString
+public class SchemaInfo {
+ private SchemaIdentity schemaIdentity;
+ private String status;
+ private String scope;
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SchemaInfoResponse.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SchemaInfoResponse.java
new file mode 100644
index 0000000000000000000000000000000000000000..088d1a07b05d5e4a1c8d6883a87f93366dec8775
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SchemaInfoResponse.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model;
+
+import lombok.Data;
+import lombok.ToString;
+
+import java.util.List;
+
+@Data
+@ToString
+public class SchemaInfoResponse {
+ private List<SchemaInfo> schemaInfos;
+ private int offset;
+ private int count;
+ private int totalCount;
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SearchRecord.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SearchRecord.java
new file mode 100644
index 0000000000000000000000000000000000000000..5b15f66bc6b4952dd7e595b480bcc2ea9c02ce0f
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SearchRecord.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import lombok.Data;
+import lombok.ToString;
+
+import java.util.Map;
+
+@Data
+@ToString
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class SearchRecord {
+ private String id;
+ private String kind;
+ private Map<String, Object> data;
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SearchRequest.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SearchRequest.java
new file mode 100644
index 0000000000000000000000000000000000000000..c36bde212ed91a0f2e6a723b97488d7aaab47366
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SearchRequest.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model;
+
+import lombok.Data;
+import lombok.ToString;
+
+import javax.validation.constraints.NotNull;
+import java.util.List;
+
+@Data
+@ToString
+public class SearchRequest {
+ @NotNull(message = "Kind is missing")
+ private Object kind;
+ private String query;
+ private int limit;
+ private int offset;
+ private String cursor;
+ private List<String> returnedFields;
+ private boolean trackTotalCount = true;
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SearchResponse.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SearchResponse.java
new file mode 100644
index 0000000000000000000000000000000000000000..61136de58ce26dc958780dc1574dd6718cde4df0
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SearchResponse.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model;
+
+import lombok.Data;
+import lombok.ToString;
+
+import java.util.List;
+
+@Data
+@ToString
+public class SearchResponse {
+ private String cursor;
+ private List<SearchRecord> results;
+ private int totalCount;
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/ChildrenKinds.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/ChildrenKinds.java
new file mode 100644
index 0000000000000000000000000000000000000000..8ee6c98d0a49465919a7ec472f62f7f9094f57d7
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/ChildrenKinds.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model.indexproperty;
+
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+public class ChildrenKinds {
+ private List<String> kinds;
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/ParentChildRelationshipSpec.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/ParentChildRelationshipSpec.java
new file mode 100644
index 0000000000000000000000000000000000000000..052479a9172ba06951dde9392a598d673294c564
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/ParentChildRelationshipSpec.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model.indexproperty;
+
+import lombok.Data;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@Data
+public class ParentChildRelationshipSpec {
+ private String parentKind;
+ private String parentObjectIdPath;
+ private String childKind;
+ private List<String> childValuePaths;
+
+ public ParentChildRelationshipSpec() {
+ childValuePaths = new ArrayList<>();
+ }
+
+ @Override
+ public boolean equals(Object another) {
+ if(another == null || !(another instanceof ParentChildRelationshipSpec))
+ return false;
+
+ ParentChildRelationshipSpec anotherSpec = (ParentChildRelationshipSpec)another;
+ return this.toString().equals(anotherSpec.toString());
+ }
+
+ @Override
+ public int hashCode() {
+ return this.toString().hashCode();
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder stringBuilder = new StringBuilder();
+ stringBuilder.append((parentKind != null)? parentKind : "__");
+ stringBuilder.append("<>");
+ stringBuilder.append((childKind != null)? childKind : "__");
+ stringBuilder.append("<>");
+ stringBuilder.append((parentObjectIdPath != null)? parentObjectIdPath : "__");
+ return stringBuilder.toString();
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/ParentChildRelationshipSpecs.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/ParentChildRelationshipSpecs.java
new file mode 100644
index 0000000000000000000000000000000000000000..35bd4e2edb8bcbc95f63eead1209a73d6f48489e
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/ParentChildRelationshipSpecs.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model.indexproperty;
+
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+public class ParentChildRelationshipSpecs {
+ private List<ParentChildRelationshipSpec> specList;
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/PropertyConfiguration.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/PropertyConfiguration.java
new file mode 100644
index 0000000000000000000000000000000000000000..de4e53f7a8dd1056d3f36cee3aecf659dd24010c
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/PropertyConfiguration.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model.indexproperty;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Data;
+import lombok.ToString;
+
+import java.util.List;
+
+@Data
+@ToString
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class PropertyConfiguration {
+ private final String EXTRACT_FIRST_MATCH_POLICY = "ExtractFirstMatch";
+ private final String EXTRACT_ALL_MATCHES_POLICY = "ExtractAllMatches";
+
+ @JsonProperty("Name")
+ private String name;
+
+ @JsonProperty("Policy")
+ private String policy;
+
+ @JsonProperty("UseCase")
+ private String useCase;
+
+ @JsonProperty("Paths")
+ private List<PropertyPath> paths;
+
+ public boolean isExtractFirstMatch() {
+ return EXTRACT_FIRST_MATCH_POLICY.equalsIgnoreCase(policy);
+ }
+
+ public boolean isExtractAllMatches() {
+ return EXTRACT_ALL_MATCHES_POLICY.equalsIgnoreCase(policy);
+ }
+
+ public boolean isValid() {
+ boolean hasValidPath = (paths != null && paths.stream().filter(p -> p.isValid()).findFirst().orElse(null) != null);
+ return hasValidPath && (isExtractFirstMatch() || isExtractAllMatches());
+ }
+
+ public String getRelatedObjectKind() {
+ if(paths != null) {
+ for (PropertyPath path : paths) {
+ if (path.isValid() && path.hasValidRelatedObjectsSpec()) {
+ return path.getRelatedObjectsSpec().getRelatedObjectKind();
+ }
+ }
+ }
+ return null;
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/PropertyConfigurations.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/PropertyConfigurations.java
new file mode 100644
index 0000000000000000000000000000000000000000..ab20d8755d9cfd8989ce9ec7fa64a1a7a97fd958
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/PropertyConfigurations.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model.indexproperty;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Strings;
+import lombok.Data;
+import lombok.ToString;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+@Data
+@ToString
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class PropertyConfigurations {
+ @JsonProperty("Name")
+ private String name;
+
+ @JsonProperty("Description")
+ private String description;
+
+ @JsonProperty("Code")
+ private String code;
+
+ @JsonProperty("AttributionAuthority")
+ private String attributionAuthority;
+
+ @JsonProperty("Configurations")
+ private List<PropertyConfiguration> configurations;
+
+ public List<String> getUniqueRelatedObjectKinds() {
+ if(configurations == null || configurations.isEmpty())
+ return new ArrayList<>();
+
+ Set<String> relatedObjectKinds = new HashSet<>();
+ for(PropertyConfiguration configuration : configurations) {
+ String relatedObjectKind = configuration.getRelatedObjectKind();
+ if(!Strings.isNullOrEmpty(relatedObjectKind)) {
+ relatedObjectKinds.add(relatedObjectKind);
+ }
+ }
+ return new ArrayList<>(relatedObjectKinds);
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/PropertyPath.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/PropertyPath.java
new file mode 100644
index 0000000000000000000000000000000000000000..e79a171ac2e2e66c5c3e3a450c6903c43274b818
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/PropertyPath.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model.indexproperty;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import lombok.Data;
+import lombok.ToString;
+import org.opengroup.osdu.indexer.model.indexproperty.jackson.PropertyPathDeserializer;
+
+@Data
+@ToString
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonDeserialize(using = PropertyPathDeserializer.class)
+public class PropertyPath {
+ private RelatedObjectsSpec relatedObjectsSpec;
+
+ private ValueExtraction valueExtraction;
+
+ public boolean hasValidRelatedObjectsSpec() {
+ return relatedObjectsSpec != null && relatedObjectsSpec.isValid();
+ }
+
+ public boolean hasValidValueExtraction() {
+ return valueExtraction != null && valueExtraction.isValid();
+ }
+
+ public boolean isValid() {
+ if(relatedObjectsSpec != null) {
+ return hasValidRelatedObjectsSpec() && hasValidValueExtraction();
+ }
+ else {
+ return hasValidValueExtraction();
+ }
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/RelatedCondition.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/RelatedCondition.java
new file mode 100644
index 0000000000000000000000000000000000000000..ba9683e26dfb1c73e52cf03d77200155c7ed2727
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/RelatedCondition.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model.indexproperty;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.google.api.client.util.Strings;
+import lombok.Data;
+import lombok.ToString;
+
+import java.util.List;
+
+@Data
+@ToString
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class RelatedCondition {
+ protected static final String ARRAY_SYMBOL = "[]";
+
+ protected String relatedConditionProperty;
+
+ protected List<String> relatedConditionMatches;
+
+ protected boolean hasValidCondition(String property) {
+ if(Strings.isNullOrEmpty(property) ||
+ Strings.isNullOrEmpty(relatedConditionProperty) ||
+ relatedConditionMatches == null ||
+ relatedConditionMatches.isEmpty())
+ return false;
+
+ if(property.indexOf(ARRAY_SYMBOL + "." ) <= 0 || property.endsWith(ARRAY_SYMBOL) ||
+ relatedConditionProperty.indexOf(ARRAY_SYMBOL + "." ) <= 0 || relatedConditionProperty.endsWith(ARRAY_SYMBOL))
+ return false;
+
+ String delimiter = "\\[\\]\\.";
+ String[] propertyParts = property.split(delimiter);
+ String[] relatedConditionPropertyParts = relatedConditionProperty.split(delimiter);
+ if(propertyParts.length != relatedConditionPropertyParts.length || propertyParts.length < 2)
+ return false;
+
+ for(int i = 0; i < propertyParts.length -1; i++) {
+ if(!propertyParts[i].equals(relatedConditionPropertyParts[i]))
+ return false;
+ }
+ return true;
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/RelatedObjectsSpec.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/RelatedObjectsSpec.java
new file mode 100644
index 0000000000000000000000000000000000000000..8ff9b57f55e75af1ebaeace577c8475426a0a2c5
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/RelatedObjectsSpec.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model.indexproperty;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.google.api.client.util.Strings;
+import lombok.Data;
+import lombok.ToString;
+
+@Data
+@ToString
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class RelatedObjectsSpec extends RelatedCondition {
+ public final static String CHILD_TO_PARENT = "ChildToParent";
+ public final static String PARENT_TO_CHILDREN = "ParentToChildren";
+
+ private String relatedObjectID;
+
+ private String relatedObjectKind;
+
+ private String relationshipDirection;
+
+ public boolean isChildToParent() { return CHILD_TO_PARENT.equalsIgnoreCase(relationshipDirection); }
+
+ public boolean isParentToChildren() { return PARENT_TO_CHILDREN.equalsIgnoreCase(relationshipDirection); }
+
+ public boolean isValid() {
+ return !Strings.isNullOrEmpty(relatedObjectID) && !Strings.isNullOrEmpty(relatedObjectKind) &&
+ (isChildToParent() || isParentToChildren());
+ }
+
+ /**
+ * To have a valid hasValidCondition, both relatedConditionProperty and relatedObjectID must refer to the same nested object but different property
+ * Only one level of nested is supported for now
+ * @return
+ */
+ public boolean hasValidCondition() {
+ return isValid() && super.hasValidCondition(relatedObjectID);
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/ValueExtraction.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/ValueExtraction.java
new file mode 100644
index 0000000000000000000000000000000000000000..1ae141d2b31595747afb9dfc1e8a806824a00c33
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/ValueExtraction.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model.indexproperty;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.api.client.util.Strings;
+import lombok.Data;
+import lombok.ToString;
+
+@Data
+@ToString
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class ValueExtraction extends RelatedCondition {
+ @JsonProperty("ValuePath")
+ private String valuePath;
+
+ public boolean isValid() {
+ return !Strings.isNullOrEmpty(valuePath);
+ }
+
+ /**
+ * To have a valid hasValidCondition, both relatedConditionProperty and relatedObjectID must refer to the same nested object but different property
+ * Only one level of nested is supported for now
+ * @return
+ */
+ public boolean hasValidCondition() {
+ return isValid() && super.hasValidCondition(valuePath);
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/jackson/PropertyPathDeserializer.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/jackson/PropertyPathDeserializer.java
new file mode 100644
index 0000000000000000000000000000000000000000..687776ed62597eab4365c3338607c03e5ff8a2ba
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/indexproperty/jackson/PropertyPathDeserializer.java
@@ -0,0 +1,114 @@
+package org.opengroup.osdu.indexer.model.indexproperty.jackson;
+
+import com.fasterxml.jackson.core.JacksonException;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.ObjectCodec;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.JsonNode;
+import org.opengroup.osdu.indexer.model.indexproperty.PropertyPath;
+import org.opengroup.osdu.indexer.model.indexproperty.RelatedObjectsSpec;
+import org.opengroup.osdu.indexer.model.indexproperty.ValueExtraction;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class PropertyPathDeserializer extends JsonDeserializer<PropertyPath> {
+ private final String RELATED_OBJECTS_SPEC_RELATIONSHIP_DIRECTION = "RelatedObjectsSpec.RelationshipDirection";
+ private final String RELATED_OBJECTS_SPEC_RELATED_OBJECT_KIND = "RelatedObjectsSpec.RelatedObjectKind";
+ private final String RELATED_OBJECTS_SPEC_RELATED_OBJECT_ID = "RelatedObjectsSpec.RelatedObjectID";
+ private final String RELATED_OBJECTS_SPEC_RELATED_CONDITION_PROPERTY = "RelatedObjectsSpec.RelatedConditionProperty";
+ private final String RELATED_OBJECTS_SPEC_RELATED_CONDITION_MATCHES = "RelatedObjectsSpec.RelatedConditionMatches";
+
+ private final String VALUE_EXTRACTION_VALUE_PATH = "ValueExtraction.ValuePath";
+ private final String VALUE_EXTRACTION_RELATED_CONDITION_PROPERTY = "ValueExtraction.RelatedConditionProperty";
+ private final String VALUE_EXTRACTION_RELATED_CONDITION_MATCHES = "ValueExtraction.RelatedConditionMatches";
+
+ @Override
+ public PropertyPath deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException, JacksonException {
+ PropertyPath propertyPath = new PropertyPath();
+ ObjectCodec codec = jsonParser.getCodec();
+ JsonNode node = codec.readTree(jsonParser);
+
+ RelatedObjectsSpec relatedObjectsSpec = deserializeRelatedObjects(node);
+ if(relatedObjectsSpec != null) {
+ propertyPath.setRelatedObjectsSpec(relatedObjectsSpec);
+ }
+ ValueExtraction valueExtraction = deserializeValueExtraction(node);
+ if(valueExtraction != null) {
+ propertyPath.setValueExtraction(valueExtraction);
+ }
+ return propertyPath;
+ }
+
+ private RelatedObjectsSpec deserializeRelatedObjects(JsonNode node) {
+ JsonNode relationshipDirection = node.get(RELATED_OBJECTS_SPEC_RELATIONSHIP_DIRECTION);
+ JsonNode relatedObjectKind = node.get(RELATED_OBJECTS_SPEC_RELATED_OBJECT_KIND);
+ JsonNode relatedObjectID = node.get(RELATED_OBJECTS_SPEC_RELATED_OBJECT_ID);
+ JsonNode relatedConditionProperty = node.get(RELATED_OBJECTS_SPEC_RELATED_CONDITION_PROPERTY);
+ JsonNode relatedConditionMatches = node.get(RELATED_OBJECTS_SPEC_RELATED_CONDITION_MATCHES);
+
+ if(isNotNull(relationshipDirection) ||
+ isNotNull(relatedObjectKind) ||
+ isNotNull(relatedObjectID) ||
+ isNotNull(relatedConditionProperty) ||
+ isNotNull(relatedConditionMatches)) {
+ RelatedObjectsSpec relatedObjectsSpec = new RelatedObjectsSpec();
+ if(isNotNull(relationshipDirection)) {
+ relatedObjectsSpec.setRelationshipDirection(relationshipDirection.asText());
+ }
+ if(isNotNull(relatedObjectKind)) {
+ relatedObjectsSpec.setRelatedObjectKind(relatedObjectKind.asText());
+ }
+ if(isNotNull(relatedObjectID)) {
+ relatedObjectsSpec.setRelatedObjectID(relatedObjectID.asText());
+ }
+ if(isNotNull(relatedConditionProperty)) {
+ relatedObjectsSpec.setRelatedConditionProperty(relatedConditionProperty.asText());
+ }
+ if(isNotNull(relatedConditionMatches) && relatedConditionMatches.isArray()) {
+ List<String> conditionMatches = new ArrayList<>();
+ for (JsonNode subNode : relatedConditionMatches) {
+ conditionMatches.add(subNode.asText());
+ }
+ relatedObjectsSpec.setRelatedConditionMatches(conditionMatches);
+ }
+ return relatedObjectsSpec;
+ }
+
+ return null;
+ }
+
+ private ValueExtraction deserializeValueExtraction(JsonNode node) {
+ JsonNode valuePath = node.get(VALUE_EXTRACTION_VALUE_PATH);
+ JsonNode relatedConditionProperty = node.get(VALUE_EXTRACTION_RELATED_CONDITION_PROPERTY);
+ JsonNode relatedConditionMatches = node.get(VALUE_EXTRACTION_RELATED_CONDITION_MATCHES);
+
+ if(isNotNull(valuePath) ||
+ isNotNull(relatedConditionProperty) ||
+ isNotNull(relatedConditionMatches)) {
+ ValueExtraction valueExtraction = new ValueExtraction();
+ if(isNotNull(valuePath)) {
+ valueExtraction.setValuePath(valuePath.asText());
+ }
+ if(isNotNull(relatedConditionProperty)) {
+ valueExtraction.setRelatedConditionProperty(relatedConditionProperty.asText());
+ }
+ if(isNotNull(relatedConditionMatches) && relatedConditionMatches.isArray()) {
+ List<String> conditionMatches = new ArrayList<>();
+ for (JsonNode subNode : relatedConditionMatches) {
+ conditionMatches.add(subNode.asText());
+ }
+ valueExtraction.setRelatedConditionMatches(conditionMatches);
+ }
+ return valueExtraction;
+ }
+
+ return null;
+ }
+
+ private boolean isNotNull(JsonNode node) {
+ return node != null && !node.isNull();
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java
index 4c985a0ff6abdd442743929318302986efe9fb1a..7d713d649b8fa6a0782943fc42c2c42488235c4d 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/schema/converter/SchemaToStorageFormatImpl.java
@@ -16,7 +16,6 @@ package org.opengroup.osdu.indexer.schema.converter;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.api.client.util.Strings;
import com.google.gson.Gson;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.search.Preconditions;
@@ -25,7 +24,7 @@ import org.opengroup.osdu.indexer.schema.converter.exeption.SchemaProcessingExce
import org.opengroup.osdu.indexer.schema.converter.interfaces.IVirtualPropertiesSchemaCache;
import org.opengroup.osdu.indexer.schema.converter.interfaces.SchemaToStorageFormat;
import org.opengroup.osdu.indexer.schema.converter.tags.*;
-import org.opengroup.osdu.indexer.util.VirtualPropertyUtil;
+import org.opengroup.osdu.indexer.util.PropertyUtil;
import org.springframework.stereotype.Component;
import javax.inject.Inject;
@@ -159,12 +158,12 @@ public class SchemaToStorageFormatImpl implements SchemaToStorageFormat {
// The schema for different properties in the list of Priority should be the same
Priority priority = entry.getValue().getPriorities().get(0);
- String virtualPropertyPath = VirtualPropertyUtil.removeDataPrefix(entry.getKey());
- hasVirtualDefaultLocation |= VirtualPropertyUtil.isPropertyPathMatched(virtualPropertyPath, VirtualPropertyUtil.VIRTUAL_DEFAULT_LOCATION);
+ String virtualPropertyPath = PropertyUtil.removeDataPrefix(entry.getKey());
+ hasVirtualDefaultLocation |= PropertyUtil.isPropertyPathMatched(virtualPropertyPath, PropertyUtil.VIRTUAL_DEFAULT_LOCATION);
- String originalPropertyPath = VirtualPropertyUtil.removeDataPrefix(priority.getPath());
+ String originalPropertyPath = PropertyUtil.removeDataPrefix(priority.getPath());
List<Map<String, Object>> matchedItems = storageSchemaItems.stream().filter(item ->
- VirtualPropertyUtil.isPropertyPathMatched((String) item.get("path"), originalPropertyPath))
+ PropertyUtil.isPropertyPathMatched((String) item.get("path"), originalPropertyPath))
.collect(Collectors.toList());
storageSchemaItems.addAll(matchedItems.stream().map(item ->
cloneVirtualProperty(item, virtualPropertyPath, originalPropertyPath))
@@ -173,7 +172,7 @@ public class SchemaToStorageFormatImpl implements SchemaToStorageFormat {
if(hasVirtualDefaultLocation) {
Map<String, Object> isDecimatedProperty = new HashMap<>();
- isDecimatedProperty.put("path", VirtualPropertyUtil.VIRTUAL_DEFAULT_LOCATION_IS_DECIMATED_PATH);
+ isDecimatedProperty.put("path", PropertyUtil.VIRTUAL_DEFAULT_LOCATION_IS_DECIMATED_PATH);
isDecimatedProperty.put("kind", "boolean");
storageSchemaItems.add(isDecimatedProperty);
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexAliasService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexAliasService.java
new file mode 100644
index 0000000000000000000000000000000000000000..a817d979f245c639d3576ab48eeab873d06d9736
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexAliasService.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.service;
+
+import org.elasticsearch.client.RestHighLevelClient;
+import org.opengroup.osdu.indexer.model.IndexAliasesResult;
+
+public interface IndexAliasService {
+ IndexAliasesResult createIndexAliasesForAll();
+ boolean createIndexAlias(RestHighLevelClient restClient, String kind);
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexAliasServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexAliasServiceImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..f8b2c4f482a9f2a90ece49670c10b06a2707f439
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexAliasServiceImpl.java
@@ -0,0 +1,212 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.service;
+
+import com.google.api.client.util.Strings;
+import org.apache.http.HttpStatus;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.GetAliasesResponse;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.cluster.metadata.AliasMetadata;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.search.aggregations.bucket.terms.Terms;
+import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
+import org.opengroup.osdu.indexer.model.IndexAliasesResult;
+import org.opengroup.osdu.indexer.util.ElasticClientHandler;
+import org.springframework.stereotype.Component;
+
+import javax.inject.Inject;
+import java.io.IOException;
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Component
+public class IndexAliasServiceImpl implements IndexAliasService{
+ private static final String KIND_COMPLETE_VERSION_PATTERN = "[\\w-\\.\\*]+:[\\w-\\.\\*]+:[\\w-\\.\\*]+:(\\d+\\.\\d+\\.\\d+)$";
+
+ @Inject
+ private ElasticIndexNameResolver elasticIndexNameResolver;
+ @Inject
+ private ElasticClientHandler elasticClientHandler;
+ @Inject
+ private JaxRsDpsLog jaxRsDpsLog;
+
+ @Override
+ public IndexAliasesResult createIndexAliasesForAll() {
+ IndexAliasesResult result = new IndexAliasesResult();
+ try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
+ List<String> allKinds = getAllKinds(restClient);
+ Set<String> allExistingAliases = getAllExistingAliases(restClient);
+ for (String kind : allKinds) {
+ String alias = elasticIndexNameResolver.getIndexAliasFromKind(kind);
+ String indexName = elasticIndexNameResolver.getIndexNameFromKind(kind);
+ if(allExistingAliases.contains(alias)) {
+ result.getIndicesWithAliases().add(indexName);
+ }
+ else {
+ if(createIndexAlias(restClient, kind)) {
+ result.getIndicesWithAliases().add(indexName);
+ }
+ else {
+ result.getIndicesWithoutAliases().add(indexName);
+ }
+ }
+ }
+ }
+ catch (Exception e) {
+ jaxRsDpsLog.error("elastic search request failed", e);
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "elastic search cannot respond", "an unknown error has occurred.", e);
+ }
+
+ return result;
+ }
+
+ @Override
+ public boolean createIndexAlias(RestHighLevelClient restClient, String kind) {
+ if(!elasticIndexNameResolver.isIndexAliasSupported(kind)) {
+ return false;
+ }
+
+ try {
+ // To create an alias for an index, the index name must the concrete index name, not alias
+ String actualIndexName = resolveConcreteIndexName(restClient, kind);
+ if(Strings.isNullOrEmpty(actualIndexName))
+ return false;
+
+ Map<String, String> indexAliasMap = new HashMap<>();
+ indexAliasMap.put(actualIndexName, elasticIndexNameResolver.getIndexAliasFromKind(kind));
+ String kindWithMajorVersion = getKindWithMajorVersion(kind);
+ if(elasticIndexNameResolver.isIndexAliasSupported(kindWithMajorVersion)) {
+ String index = elasticIndexNameResolver.getIndexNameFromKind(kindWithMajorVersion);
+ String alias = elasticIndexNameResolver.getIndexAliasFromKind(kindWithMajorVersion);
+ indexAliasMap.put(index, alias);
+ }
+
+ boolean ok = true;
+ for (Map.Entry<String, String> entry: indexAliasMap.entrySet()) {
+ IndicesAliasesRequest addRequest = new IndicesAliasesRequest();
+ IndicesAliasesRequest.AliasActions aliasActions = new IndicesAliasesRequest.AliasActions(IndicesAliasesRequest.AliasActions.Type.ADD)
+ .index(entry.getKey())
+ .alias(entry.getValue());
+ addRequest.addAliasAction(aliasActions);
+ AcknowledgedResponse response = restClient.indices().updateAliases(addRequest, RequestOptions.DEFAULT);
+ ok &= response.isAcknowledged();
+ }
+ return ok;
+ }
+ catch(Exception e) {
+ jaxRsDpsLog.error(String.format("Fail to create index alias for kind '%s'", kind), e);
+ }
+
+ return false;
+ }
+
+ private Set<String> getAllExistingAliases(RestHighLevelClient restClient) throws IOException {
+ GetAliasesRequest request = new GetAliasesRequest();
+ GetAliasesResponse response = restClient.indices().getAlias(request, RequestOptions.DEFAULT);
+ if(response.status() != RestStatus.OK)
+ return new HashSet<>();
+
+ Set<String> allAliases = new HashSet<>();
+ for (Set<AliasMetadata> aliasSet: response.getAliases().values()) {
+ List<String> aliases = aliasSet.stream().map(a -> a.getAlias()).collect(Collectors.toList());
+ allAliases.addAll(aliases);
+ }
+ return allAliases;
+ }
+
+ private String getKindWithMajorVersion(String kind) {
+ // If kind is common:welldb:wellbore:1.2.0, then kind with major version is common:welldb:wellbore:1.*.*
+ int idx = kind.lastIndexOf(":");
+ String version = kind.substring(idx+1);
+ if(version.indexOf(".") > 0) {
+ String kindWithoutVersion = kind.substring(0, idx);
+ String majorVersion = version.substring(0, version.indexOf("."));
+ return String.format("%s:%s.*.*", kindWithoutVersion, majorVersion);
+ }
+ return null;
+ }
+
+ private String resolveConcreteIndexName(RestHighLevelClient restClient, String kind) throws IOException {
+ String index = elasticIndexNameResolver.getIndexNameFromKind(kind);
+ if(!isCompleteVersionKind(kind)) {
+ return index;
+ }
+
+ GetAliasesRequest request = new GetAliasesRequest(index);
+ GetAliasesResponse response = restClient.indices().getAlias(request, RequestOptions.DEFAULT);
+ if(response.status() == RestStatus.NOT_FOUND) {
+ /* index resolved from kind is actual concrete index
+ * Example:
+ * {
+ * "opendes-wke-well-1.0.7": {
+ * "aliases": {}
+ * }
+ * }
+ */
+ return index;
+ }
+ if(response.status() == RestStatus.OK) {
+ /* index resolved from kind is NOT actual create index. It is just an alias
+ * The concrete index name in this example is "opendes-osdudemo-wellbore-1.0.0_1649167113090"
+ * Example:
+ * {
+ * "opendes-osdudemo-wellbore-1.0.0_1649167113090": {
+ * "aliases": {
+ * "opendes-osdudemo-wellbore-1.0.0": {}
+ * }
+ * }
+ * }
+ */
+ Map<String, Set<AliasMetadata>> aliases = response.getAliases();
+ for (Map.Entry<String, Set<AliasMetadata>> entry: aliases.entrySet()) {
+ String actualIndex = entry.getKey();
+ List<String> aliaseNames = entry.getValue().stream().map(a -> a.getAlias()).collect(Collectors.toList());
+ if(aliaseNames.contains(index))
+ return actualIndex;
+ }
+ }
+ return index;
+ }
+
+ private boolean isCompleteVersionKind(String kind) {
+ return !Strings.isNullOrEmpty(kind) && kind.matches(KIND_COMPLETE_VERSION_PATTERN);
+ }
+
+ private List<String> getAllKinds(RestHighLevelClient client) throws IOException {
+ List<String> kinds;
+ SearchRequest elasticSearchRequest = new SearchRequest("_all");
+ TermsAggregationBuilder termsAggregationBuilder = new TermsAggregationBuilder("kinds");
+ termsAggregationBuilder.field("kind");
+ termsAggregationBuilder.size(10000);
+ SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
+ sourceBuilder.aggregation(termsAggregationBuilder);
+ elasticSearchRequest.source(sourceBuilder);
+ SearchResponse searchResponse = client.search(elasticSearchRequest, RequestOptions.DEFAULT);
+ Terms kindBuckets = searchResponse.getAggregations().get("kinds");
+ kinds = kindBuckets.getBuckets().stream().map(bucket -> bucket.getKey().toString()).collect(Collectors.toList());
+ return kinds;
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaService.java
index d19833f2578ff426e639856ff9ff1f05f46034ea..9b19787c35ec5247e1a88ef4c2f63e68d1a3d421 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaService.java
@@ -40,4 +40,6 @@ public interface IndexSchemaService {
void syncIndexMappingWithStorageSchema(String kind) throws ElasticsearchException, IOException, AppException, URISyntaxException;
boolean isStorageSchemaSyncRequired(String kind, boolean forceClean) throws IOException;
+
+ void invalidateSchemaCache(String kind);
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
index c469a2284aac9861af7e9db941034c8d3cd4ca2b..ab3cebb80195698fe9d3b280c801f95b31281c16 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
@@ -30,10 +30,13 @@ import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
import org.opengroup.osdu.core.common.model.storage.Schema;
import org.opengroup.osdu.core.common.model.storage.SchemaItem;
import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
+import org.opengroup.osdu.indexer.cache.PartitionSafeFlattenedSchemaCache;
+import org.opengroup.osdu.indexer.cache.PartitionSafeSchemaCache;
import org.opengroup.osdu.indexer.model.Kind;
-import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache;
+import org.opengroup.osdu.indexer.model.indexproperty.PropertyConfigurations;
import org.opengroup.osdu.indexer.schema.converter.exeption.SchemaProcessingException;
import org.opengroup.osdu.indexer.schema.converter.interfaces.IVirtualPropertiesSchemaCache;
+import org.opengroup.osdu.indexer.util.AugmenterSetting;
import org.opengroup.osdu.indexer.util.ElasticClientHandler;
import org.opengroup.osdu.indexer.util.TypeMapper;
import org.springframework.stereotype.Service;
@@ -42,15 +45,11 @@ import javax.inject.Inject;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URISyntaxException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
@Service
public class IndexSchemaServiceImpl implements IndexSchemaService {
- private static final String FLATTENED_SCHEMA = "_flattened";
-
private final Gson gson = new Gson();
@Inject
@@ -66,9 +65,15 @@ public class IndexSchemaServiceImpl implements IndexSchemaService {
@Inject
private IndicesService indicesService;
@Inject
- private ISchemaCache schemaCache;
+ private PartitionSafeSchemaCache schemaCache;
+ @Inject
+ private PartitionSafeFlattenedSchemaCache flattenedSchemaCache;
@Inject
private IVirtualPropertiesSchemaCache virtualPropertiesSchemaCache;
+ @Inject
+ private PropertyConfigurationsService propertyConfigurationsService;
+ @Inject
+ private AugmenterSetting augmenterSetting;
public void processSchemaMessages(Map<String, OperationType> schemaMsgs) throws IOException {
try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
@@ -154,25 +159,27 @@ public class IndexSchemaServiceImpl implements IndexSchemaService {
this.invalidateCache(kind);
}
- String schema = (String) this.schemaCache.get(kind);
+ String schema = this.schemaCache.get(kind);
if (Strings.isNullOrEmpty(schema)) {
// get from storage
schema = this.schemaProvider.getSchema(kind);
if (Strings.isNullOrEmpty(schema)) {
return this.getEmptySchema(kind);
} else {
- // cache the schema
- this.schemaCache.put(kind, schema);
- // get flatten schema and cache it
- IndexSchema flatSchemaObj = normalizeSchema(schema);
- if (flatSchemaObj != null) {
- this.schemaCache.put(kind + FLATTENED_SCHEMA, gson.toJson(flatSchemaObj));
+ if(augmenterSetting.isEnabled()) {
+ // Merge schema of the extended properties if needed
+ PropertyConfigurations propertyConfigurations = propertyConfigurationsService.getPropertyConfigurations(kind);
+ if (propertyConfigurations != null) {
+ schema = mergeSchemaFromPropertyConfiguration(schema, propertyConfigurations);
+ }
}
+
+ IndexSchema flatSchemaObj = cacheAndNormalizeSchema(kind, schema);
return flatSchemaObj;
}
} else {
// search flattened schema in memcache
- String flattenedSchema = (String) this.schemaCache.get(kind + FLATTENED_SCHEMA);
+ String flattenedSchema = this.flattenedSchemaCache.get(kind);
if (Strings.isNullOrEmpty(flattenedSchema)) {
return this.getEmptySchema(kind);
}
@@ -180,6 +187,57 @@ public class IndexSchemaServiceImpl implements IndexSchemaService {
}
}
+ private IndexSchema cacheAndNormalizeSchema(String kind, String schema) {
+ // cache the schema
+ this.schemaCache.put(kind, schema);
+ // get flatten schema and cache it
+ IndexSchema flatSchemaObj = normalizeSchema(schema);
+ if (flatSchemaObj != null) {
+ this.flattenedSchemaCache.put(kind, gson.toJson(flatSchemaObj));
+ }
+ return flatSchemaObj;
+ }
+
+ private String mergeSchemaFromPropertyConfiguration(String originalSchemaStr, PropertyConfigurations propertyConfigurations) throws UnsupportedEncodingException, URISyntaxException {
+ Map<String, Schema> relatedObjectKindSchemas = getSchemaOfRelatedObjectKinds(propertyConfigurations);
+ Schema originalSchema = gson.fromJson(originalSchemaStr, Schema.class);
+ List<SchemaItem> extendedSchemaItems = propertyConfigurationsService.getExtendedSchemaItems(originalSchema, relatedObjectKindSchemas, propertyConfigurations);
+ if (!extendedSchemaItems.isEmpty()) {
+ List<SchemaItem> originalSchemaItems = new ArrayList<>(Arrays.asList(originalSchema.getSchema()));
+ originalSchemaItems.addAll(extendedSchemaItems);
+ originalSchema.setSchema(originalSchemaItems.toArray(new SchemaItem[0]));
+ return gson.toJson(originalSchema);
+ } else {
+ return originalSchemaStr;
+ }
+ }
+
+ private Map<String, Schema> getSchemaOfRelatedObjectKinds(PropertyConfigurations propertyConfigurations) throws UnsupportedEncodingException, URISyntaxException {
+ List<String> relatedObjectKinds = propertyConfigurations.getUniqueRelatedObjectKinds();
+ Map<String, Schema> relatedObjectKindSchemas = new HashMap<>();
+ for (String relatedObjectKind : relatedObjectKinds) {
+ // The relatedObjectKind defined in property configuration can be kind having major version only
+ // e.g. "RelatedObjectKind": "osdu:wks:master-data--Wellbore:1."
+ String concreteRelatedObjectKind = propertyConfigurationsService.resolveConcreteKind(relatedObjectKind);
+ if (Strings.isNullOrEmpty(concreteRelatedObjectKind))
+ continue;
+
+ String relatedObjectKindSchema = this.schemaCache.get(concreteRelatedObjectKind);
+ if (Strings.isNullOrEmpty(relatedObjectKindSchema)) {
+ relatedObjectKindSchema = this.schemaProvider.getSchema(concreteRelatedObjectKind);
+ if (!Strings.isNullOrEmpty(relatedObjectKindSchema)) {
+ cacheAndNormalizeSchema(concreteRelatedObjectKind, relatedObjectKindSchema);
+ }
+ }
+
+ if (!Strings.isNullOrEmpty(relatedObjectKindSchema)) {
+ Schema schema = gson.fromJson(relatedObjectKindSchema, Schema.class);
+ relatedObjectKindSchemas.put(relatedObjectKind, schema);
+ }
+ }
+ return relatedObjectKindSchemas;
+ }
+
private IndexSchema getEmptySchema(String kind) {
Schema basicSchema = Schema.builder().kind(kind).build();
return normalizeSchema(gson.toJson(basicSchema));
@@ -205,14 +263,15 @@ public class IndexSchemaServiceImpl implements IndexSchemaService {
}
}
- private void invalidateCache(String kind) {
- String schema = (String) this.schemaCache.get(kind);
- if (!Strings.isNullOrEmpty(schema)) this.schemaCache.delete(kind);
-
- String flattenSchema = (String) this.schemaCache.get(kind + FLATTENED_SCHEMA);
- if (!Strings.isNullOrEmpty(flattenSchema)) this.schemaCache.delete(kind + FLATTENED_SCHEMA);
+ @Override
+ public void invalidateSchemaCache(String kind) {
+ this.invalidateCache(kind);
+ }
- virtualPropertiesSchemaCache.delete(kind);
+ private void invalidateCache(String kind) {
+ this.schemaCache.delete(kind);
+ this.flattenedSchemaCache.delete(kind);
+ this.virtualPropertiesSchemaCache.delete(kind);
}
private IndexSchema normalizeSchema(String schemaStr) throws AppException {
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
index 8e2895fc2b0807e9ad6fedcc2a09f7b060ab27af..cdae2cb43b3c99b3cbc3376867f7c1e7cd142218 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
@@ -25,7 +25,6 @@ import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.unit.TimeValue;
@@ -38,12 +37,16 @@ import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.http.RequestStatus;
import org.opengroup.osdu.core.common.model.indexer.*;
+import org.opengroup.osdu.core.common.model.indexer.RecordIndexerPayload.Record;
import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
import org.opengroup.osdu.indexer.logging.AuditLogger;
+import org.opengroup.osdu.indexer.model.BulkRequestResult;
+import org.opengroup.osdu.indexer.model.indexproperty.PropertyConfigurations;
import org.opengroup.osdu.indexer.provider.interfaces.IPublisher;
+import org.opengroup.osdu.indexer.util.AugmenterSetting;
import org.opengroup.osdu.indexer.util.ElasticClientHandler;
import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
import org.springframework.context.annotation.Primary;
@@ -53,6 +56,7 @@ import javax.inject.Inject;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@@ -68,6 +72,8 @@ public class IndexerServiceImpl implements IndexerService {
private static final List<RestStatus> RETRY_ELASTIC_EXCEPTION = new ArrayList<>(Arrays.asList(RestStatus.TOO_MANY_REQUESTS, RestStatus.BAD_GATEWAY, RestStatus.SERVICE_UNAVAILABLE, RestStatus.FORBIDDEN));
+ private static final String MAPPER_PARSING_EXCEPTION_TYPE = "type=mapper_parsing_exception";
+
private final Gson gson = new GsonBuilder().serializeNulls().create();
// we index a normalized kind (authority + source + entity type + major version) as a tags attribute for all records
@@ -99,6 +105,10 @@ public class IndexerServiceImpl implements IndexerService {
private IRequestInfo requestInfo;
@Inject
private JobStatus jobStatus;
+ @Inject
+ private PropertyConfigurationsService propertyConfigurationsService;
+ @Inject
+ private AugmenterSetting augmenterSetting;
private DpsHeaders headers;
@@ -146,6 +156,14 @@ public class IndexerServiceImpl implements IndexerService {
if (retryRecordIds.size() > 0) {
retryAndEnqueueFailedRecords(recordInfos, retryRecordIds, message);
}
+
+ if(this.augmenterSetting.isEnabled()) {
+ Map<String, List<String>> upsertKindIds = getUpsertRecordIdsForConfigurationsEnabledKinds(upsertRecordMap, retryRecordIds);
+ Map<String, List<String>> deleteKindIds = getDeleteRecordIdsForConfigurationsEnabledKinds(deleteRecordMap, retryRecordIds);
+ if (!upsertKindIds.isEmpty() || !deleteKindIds.isEmpty()) {
+ propertyConfigurationsService.updateAssociatedRecords(message, upsertKindIds, deleteKindIds);
+ }
+ }
} catch (IOException e) {
errorMessage = e.getMessage();
throw new AppException(HttpStatus.SC_GATEWAY_TIMEOUT, "Internal communication failure", errorMessage, e);
@@ -180,6 +198,34 @@ public class IndexerServiceImpl implements IndexerService {
}
}
+ private Map<String, List<String>> getUpsertRecordIdsForConfigurationsEnabledKinds(Map<String, Map<String, OperationType>> upsertRecordMap, List<String> retryRecordIds) {
+ Map<String, List<String>> upsertKindIds = new HashMap<>();
+ for (Map.Entry<String, Map<String, OperationType>> entry : upsertRecordMap.entrySet()) {
+ String kind = entry.getKey();
+ if(propertyConfigurationsService.isPropertyConfigurationsEnabled(kind)) {
+ List<String> processedIds = entry.getValue().keySet().stream().filter(id -> !retryRecordIds.contains(id)).collect(Collectors.toList());
+ if (!processedIds.isEmpty()) {
+ upsertKindIds.put(kind, processedIds);
+ }
+ }
+ }
+ return upsertKindIds;
+ }
+
+ private Map<String, List<String>> getDeleteRecordIdsForConfigurationsEnabledKinds(Map<String, List<String>> deleteRecordMap, List<String> retryRecordIds) {
+ Map<String, List<String>> deletedRecordKindIdsMap = new HashMap<>();
+ for (Map.Entry<String, List<String>> entry : deleteRecordMap.entrySet()) {
+ String kind = entry.getKey();
+ if(propertyConfigurationsService.isPropertyConfigurationsEnabled(kind)) {
+ List<String> processedIds = entry.getValue().stream().filter(id -> !retryRecordIds.contains(id)).collect(Collectors.toList());
+ if (!processedIds.isEmpty()) {
+ deletedRecordKindIdsMap.put(kind, processedIds);
+ }
+ }
+ }
+ return deletedRecordKindIdsMap;
+ }
+
private void processSchemaEvents(RestHighLevelClient restClient,
Map.Entry<String, OperationType> msg) throws IOException, ElasticsearchStatusException {
String kind = msg.getKey();
@@ -188,6 +234,7 @@ public class IndexerServiceImpl implements IndexerService {
boolean indexExist = indicesService.isIndexExist(restClient, index);
if (indexExist && msg.getValue() == OperationType.purge_schema) {
indicesService.deleteIndex(restClient, index);
+ schemaService.invalidateSchemaCache(kind);
}
}
@@ -297,6 +344,19 @@ public class IndexerServiceImpl implements IndexerService {
String message = String.format("complete schema mismatch: none of the data attribute can be mapped | data: %s", storageRecordData);
this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.WARN, HttpStatus.SC_NOT_FOUND, message, String.format("record-id: %s | %s", storageRecord.getId(), message));
}
+
+ if(this.augmenterSetting.isEnabled()) {
+ if(propertyConfigurationsService.isPropertyConfigurationsEnabled(storageRecord.getKind())) {
+ PropertyConfigurations propertyConfigurations = propertyConfigurationsService.getPropertyConfigurations(storageRecord.getKind());
+ if (propertyConfigurations != null) {
+ // Merge extended properties
+ dataMap = mergeDataFromPropertyConfiguration(storageRecord.getId(), dataMap, propertyConfigurations);
+ }
+ // We cache the dataMap in case the update of this object will trigger update of the related objects.
+ propertyConfigurationsService.cacheDataRecord(storageRecord.getId(), storageRecord.getKind(), dataMap);
+ }
+ }
+
document.setData(dataMap);
}
} catch (AppException e) {
@@ -351,6 +411,15 @@ public class IndexerServiceImpl implements IndexerService {
return document;
}
+ private Map<String, Object> mergeDataFromPropertyConfiguration(String objectId, Map<String, Object> originalDataMap, PropertyConfigurations propertyConfigurations) {
+ Map<String, Object> extendedDataMap = propertyConfigurationsService.getExtendedProperties(objectId, originalDataMap, propertyConfigurations);
+ if (!extendedDataMap.isEmpty()) {
+ originalDataMap.putAll(extendedDataMap);
+ }
+
+ return originalDataMap;
+ }
+
private List<String> processElasticMappingAndUpsertRecords(RecordIndexerPayload recordIndexerPayload) throws Exception {
try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
@@ -363,7 +432,29 @@ public class IndexerServiceImpl implements IndexerService {
this.cacheOrCreateElasticMapping(schemas, restClient);
// process the records
- return this.upsertRecords(recordIndexerPayload.getRecords(), restClient);
+ List<RecordIndexerPayload.Record> records = recordIndexerPayload.getRecords();
+ BulkRequestResult bulkRequestResult = this.upsertRecords(records, restClient);
+ List<String> failedRecordIds = bulkRequestResult.getFailureRecordIds();
+
+ processRetryUpsertRecords(restClient, records, bulkRequestResult, failedRecordIds);
+
+ return failedRecordIds;
+ }
+ }
+
+ private void processRetryUpsertRecords(RestHighLevelClient restClient, List<Record> records,
+ BulkRequestResult bulkRequestResult, List<String> failedRecordIds) {
+ List<String> retryUpsertRecordIds = bulkRequestResult.getRetryUpsertRecordIds();
+ if (!retryUpsertRecordIds.isEmpty()) {
+ List<Record> retryUpsertRecords = records.stream()
+ .filter(record -> retryUpsertRecordIds.contains(record.getId()))
+ .collect(Collectors.toList());
+ retryUpsertRecords.forEach(record -> {
+ record.setData(Collections.emptyMap());
+ record.setTags(Collections.emptyMap());
+ });
+ bulkRequestResult = upsertRecords(retryUpsertRecords, restClient);
+ failedRecordIds.addAll(bulkRequestResult.getFailureRecordIds());
}
}
@@ -386,8 +477,8 @@ public class IndexerServiceImpl implements IndexerService {
}
}
- private List<String> upsertRecords(List<RecordIndexerPayload.Record> records, RestHighLevelClient restClient) throws AppException {
- if (records == null || records.isEmpty()) return new LinkedList<>();
+ private BulkRequestResult upsertRecords(List<RecordIndexerPayload.Record> records, RestHighLevelClient restClient) throws AppException {
+ if (records == null || records.isEmpty()) return new BulkRequestResult(Collections.emptyList(), Collections.emptyList());
BulkRequest bulkRequest = new BulkRequest();
bulkRequest.timeout(BULK_REQUEST_TIMEOUT);
@@ -423,14 +514,15 @@ public class IndexerServiceImpl implements IndexerService {
}
try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
- return processBulkRequest(restClient, bulkRequest);
+ return processBulkRequest(restClient, bulkRequest).getFailureRecordIds();
}
}
- private List<String> processBulkRequest(RestHighLevelClient restClient, BulkRequest bulkRequest) throws AppException {
+ private BulkRequestResult processBulkRequest(RestHighLevelClient restClient, BulkRequest bulkRequest) throws AppException {
+ if (bulkRequest.numberOfActions() == 0) return new BulkRequestResult(Collections.emptyList(), Collections.emptyList());
List<String> failureRecordIds = new LinkedList<>();
- if (bulkRequest.numberOfActions() == 0) return failureRecordIds;
+ List<String> retryUpsertRecordIds = new LinkedList<>();
int failedRequestStatus = 500;
Exception failedRequestCause = null;
@@ -449,7 +541,10 @@ public class IndexerServiceImpl implements IndexerService {
BulkItemResponse.Failure failure = bulkItemResponse.getFailure();
bulkFailures.add(String.format("elasticsearch bulk service status: %s | id: %s | message: %s", failure.getStatus(), failure.getId(), failure.getMessage()));
this.jobStatus.addOrUpdateRecordStatus(bulkItemResponse.getId(), IndexingStatus.FAIL, failure.getStatus().getStatus(), bulkItemResponse.getFailureMessage());
- if (canIndexerRetry(bulkItemResponse)) {
+
+ if (RestStatus.BAD_REQUEST.equals(failure.getStatus()) && failure.getCause() != null && failure.getCause().getMessage().contains(MAPPER_PARSING_EXCEPTION_TYPE)) {
+ retryUpsertRecordIds.add(bulkItemResponse.getId());
+ } else if (canIndexerRetry(bulkItemResponse)) {
failureRecordIds.add(bulkItemResponse.getId());
if (failedRequestCause == null) {
@@ -481,7 +576,7 @@ public class IndexerServiceImpl implements IndexerService {
}
throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Elastic error", "Error indexing records.", e);
}
- return failureRecordIds;
+ return new BulkRequestResult(failureRecordIds, retryUpsertRecordIds);
}
private Map<String, Object> getSourceMap(RecordIndexerPayload.Record record) {
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java
index bc8d633ac9be3a39336ea79a634709e1f34c2940..e585fca490a13a11f53acac4c9a1c5f54ee1a2cf 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java
@@ -50,6 +50,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.web.context.annotation.RequestScope;
+import javax.inject.Inject;
import java.io.IOException;
import java.lang.reflect.Type;
import java.util.*;
@@ -64,6 +65,8 @@ public class IndicesServiceImpl implements IndicesService {
private ElasticIndexNameResolver elasticIndexNameResolver;
@Autowired
private PartitionSafeIndexCache indexCache;
+ @Inject
+ private IndexAliasService indexAliasService;
@Autowired
private JaxRsDpsLog log;
@@ -106,7 +109,8 @@ public class IndicesServiceImpl implements IndicesService {
this.indexCache.put(index, true);
this.log.info(String.format("Time taken to successfully create new index %s : %d milliseconds", request.index(), stopTime-startTime));
- createIndexAlias(client, index);
+ // Create alias for index
+ indexAliasService.createIndexAlias(client, elasticIndexNameResolver.getKindFromIndexName(index));
}
return indexStatus;
@@ -303,48 +307,4 @@ public class IndicesServiceImpl implements IndicesService {
throw exception;
}
}
-
- private void createIndexAlias(RestHighLevelClient client, String index) {
- String kind = this.elasticIndexNameResolver.getKindFromIndexName(index);
- if(!elasticIndexNameResolver.isIndexAliasSupported(kind))
- return;
-
- try {
- List<String> kinds = new ArrayList<>();
- kinds.add(kind);
- String kindWithMajorVersion = getKindWithMajorVersion(kind);
- if(elasticIndexNameResolver.isIndexAliasSupported(kindWithMajorVersion)) {
- kinds.add(kindWithMajorVersion);
- }
- for (String kd : kinds) {
- index = elasticIndexNameResolver.getIndexNameFromKind(kd);
- String alias = elasticIndexNameResolver.getIndexAliasFromKind(kd);
- IndicesAliasesRequest addRequest = new IndicesAliasesRequest();
- IndicesAliasesRequest.AliasActions aliasActions = new IndicesAliasesRequest.AliasActions(IndicesAliasesRequest.AliasActions.Type.ADD)
- .index(index)
- .alias(alias);
- addRequest.addAliasAction(aliasActions);
- AcknowledgedResponse response = client.indices().updateAliases(addRequest, RequestOptions.DEFAULT);
- if (response.isAcknowledged()) {
- this.log.info(String.format("Alias %s was created for index %s", alias, index));
- }
- }
- }
- catch(Exception ex) {
- // Failed to create alias is not the end. It should not affect the status of index creation
- this.log.error(String.format("Fail to create aliases for index %s", index), ex);
- }
- }
-
- private String getKindWithMajorVersion(String kind) {
- // If kind is common:welldb:wellbore:1.2.0, then kind with major version is common:welldb:wellbore:1.*.*
- int idx = kind.lastIndexOf(":");
- String version = kind.substring(idx+1);
- if(version.indexOf(".") > 0) {
- String kindWithoutVersion = kind.substring(0, idx);
- String majorVersion = version.substring(0, version.indexOf("."));
- return String.format("%s:%s.*.*", kindWithoutVersion, majorVersion);
- }
- return null;
- }
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/PropertyConfigurationsService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/PropertyConfigurationsService.java
new file mode 100644
index 0000000000000000000000000000000000000000..7aa33277a5b925f0a9c9191b16300059b87f7bbd
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/PropertyConfigurationsService.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.service;
+
+import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
+import org.opengroup.osdu.core.common.model.storage.Schema;
+import org.opengroup.osdu.core.common.model.storage.SchemaItem;
+import org.opengroup.osdu.indexer.model.indexproperty.PropertyConfigurations;
+
+import java.util.List;
+import java.util.Map;
+
+public interface PropertyConfigurationsService {
+ boolean isPropertyConfigurationsEnabled(String kind);
+
+ PropertyConfigurations getPropertyConfigurations(String kind);
+
+ Map<String, Object> getExtendedProperties(String objectId, Map<String, Object> originalDataMap, PropertyConfigurations propertyConfigurations);
+
+ List<SchemaItem> getExtendedSchemaItems(Schema originalSchema, Map<String, Schema> relatedObjectKindSchemas, PropertyConfigurations propertyConfigurations);
+
+ String resolveConcreteKind(String kind);
+
+ void cacheDataRecord(String recordId, String kind, Map<String, Object> dataMap);
+
+ void updateAssociatedRecords(RecordChangedMessages message, Map<String, List<String>> upsertKindIds, Map<String, List<String>> deleteKindIds);
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/PropertyConfigurationsServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/PropertyConfigurationsServiceImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..de45b4db3aaa1a6809bf200b422db530625617ff
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/PropertyConfigurationsServiceImpl.java
@@ -0,0 +1,983 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.service;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Strings;
+import com.google.gson.Gson;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.indexer.OperationType;
+import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
+import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
+import org.opengroup.osdu.core.common.model.storage.RecordData;
+import org.opengroup.osdu.core.common.model.storage.Schema;
+import org.opengroup.osdu.core.common.model.storage.SchemaItem;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.opengroup.osdu.indexer.cache.*;
+import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.opengroup.osdu.indexer.model.*;
+import org.opengroup.osdu.indexer.model.indexproperty.*;
+import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
+import org.opengroup.osdu.indexer.util.PropertyUtil;
+import org.springframework.stereotype.Component;
+
+import javax.inject.Inject;
+import java.io.UnsupportedEncodingException;
+import java.net.URISyntaxException;
+import java.util.*;
+import java.util.stream.Collectors;
+
+
+@Component
+public class PropertyConfigurationsServiceImpl implements PropertyConfigurationsService {
+ private static final String ASSOCIATED_IDENTITIES_PROPERTY = "AssociatedIdentities";
+ private static final String ASSOCIATED_IDENTITIES_PROPERTY_STORAGE_FORMAT_TYPE = "[]string";
+ private static final String INDEX_PROPERTY_PATH_CONFIGURATION_KIND = "osdu:wks:reference-data--IndexPropertyPathConfiguration:*";
+ private static final String ANCESTRY_KINDS_DELIMITER = ",";
+ private static final String PARENT_CHILDREN_CONFIGURATION_QUERY_FORMAT =
+ "nested(data.Configurations, nested(data.Configurations.Paths, (RelatedObjectsSpec.RelationshipDirection: ParentToChildren AND RelatedObjectsSpec.RelatedObjectKind:\"%s\")))";
+ private static final String CHILDREN_PARENT_CONFIGURATION_QUERY_FORMAT =
+ "nested(data.Configurations, nested(data.Configurations.Paths, (RelatedObjectsSpec.RelationshipDirection: ChildToParent AND RelatedObjectsSpec.RelatedObjectKind:\"%s\")))";
+ private static final String HAS_CONFIGURATIONS_QUERY_FORMAT = "data.Code: \"%s\" OR nested(data.Configurations, nested(data.Configurations.Paths, (RelatedObjectsSpec.RelatedObjectKind:\"%s\")))";
+ private static final int MAX_SEARCH_LIMIT = 1000;
+
+ private static final String PROPERTY_DELIMITER = ".";
+ private static final String NESTED_OBJECT_DELIMITER = "[].";
+ private static final String ARRAY_SYMBOL = "[]";
+ private static final String SCHEMA_NESTED_KIND = "nested";
+
+ private final Gson gson = new Gson();
+ private final ObjectMapper objectMapper = new ObjectMapper();
+ private final PropertyConfigurations EMPTY_CONFIGURATIONS = new PropertyConfigurations();
+
+ @Inject
+ private IndexerConfigurationProperties configurationProperties;
+ @Inject
+ private PartitionSafePropertyConfigurationsCache propertyConfigurationCache;
+ @Inject
+ private PartitionSafePropertyConfigurationsEnabledCache propertyConfigurationsEnabledCache;
+ @Inject
+ private PartitionSafeParentChildRelationshipSpecsCache parentChildRelationshipSpecsCache;
+ @Inject
+ private PartitionSafeChildrenKindsCache childrenKindsCache;
+ @Inject
+ private PartitionSafeKindCache kindCache;
+ @Inject
+ private IRelatedObjectCache relatedObjectCache;
+ @Inject
+ private IRecordChangeInfoCache recordChangeInfoCache;
+ @Inject
+ private SearchService searchService;
+ @Inject
+ private SchemaService schemaService;
+ @Inject
+ private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
+ @Inject
+ private IRequestInfo requestInfo;
+ @Inject
+ private JaxRsDpsLog jaxRsDpsLog;
+
+ @Override
+ public boolean isPropertyConfigurationsEnabled(String kind) {
+ kind = PropertyUtil.getKindWithMajor(kind);
+ if (Strings.isNullOrEmpty(kind))
+ return false;
+
+ Boolean enabled = propertyConfigurationsEnabledCache.get(kind);
+ if(enabled == null) {
+ SearchRequest searchRequest = new SearchRequest();
+ searchRequest.setKind(INDEX_PROPERTY_PATH_CONFIGURATION_KIND);
+ String query = String.format(HAS_CONFIGURATIONS_QUERY_FORMAT, kind, kind);
+ searchRequest.setQuery(query);
+ if(searchFirstRecord(searchRequest) != null) {
+ enabled = true;
+ }
+ else {
+ enabled = false;
+ }
+ propertyConfigurationsEnabledCache.put(kind, enabled);
+ }
+
+ return enabled;
+ }
+
+ @Override
+ public PropertyConfigurations getPropertyConfigurations(String kind) {
+ kind = PropertyUtil.getKindWithMajor(kind);
+ if (Strings.isNullOrEmpty(kind))
+ return null;
+
+ PropertyConfigurations configuration = propertyConfigurationCache.get(kind);
+ if (configuration == null) {
+ configuration = searchConfigurations(kind);
+ if (configuration != null) {
+ propertyConfigurationCache.put(kind, configuration);
+ } else {
+ // It is common that a kind does not have extended property. So we need to cache an empty configuration
+ // to avoid unnecessary search
+ propertyConfigurationCache.put(kind, EMPTY_CONFIGURATIONS);
+ }
+ }
+
+ if (!isNullOrEmptyConfigurations(configuration)) {
+ return configuration;
+ }
+
+ return null;
+ }
+
+ @Override
+ public Map<String, Object> getExtendedProperties(String objectId, Map<String, Object> originalDataMap, PropertyConfigurations propertyConfigurations) {
+ Set<String> associatedIdentities = new HashSet<>();
+ Map<String, Object> extendedDataMap = new HashMap<>();
+ for (PropertyConfiguration configuration : propertyConfigurations.getConfigurations().stream().filter(c -> c.isValid()).collect(Collectors.toList())) {
+ String extendedPropertyName = configuration.getName();
+ if (originalDataMap.containsKey(extendedPropertyName) && originalDataMap.get(extendedPropertyName) != null) {
+ // If the original record already has the property, then we should not override.
+ // For example, if the trajectory record already SpatialLocation value, then it should not be overridden by the SpatialLocation of the well bore.
+ continue;
+ }
+
+ Map<String, Object> allPropertyValues = new HashMap<>();
+ for (PropertyPath path : configuration.getPaths().stream().filter(p -> p.hasValidValueExtraction()).collect(Collectors.toList())) {
+ if (path.hasValidRelatedObjectsSpec()) {
+ RelatedObjectsSpec relatedObjectsSpec = path.getRelatedObjectsSpec();
+ if (relatedObjectsSpec.isChildToParent()) {
+ List<String> relatedObjectIds = getRelatedObjectIds(originalDataMap, relatedObjectsSpec);
+ for (String relatedObjectId : relatedObjectIds) {
+ // Store all ids
+ associatedIdentities.add(PropertyUtil.removeIdPostfix(relatedObjectId));
+ }
+
+ for (String relatedObjectId : relatedObjectIds) {
+ Map<String, Object> relatedObject = getRelatedObjectData(relatedObjectsSpec.getRelatedObjectKind(), relatedObjectId);
+ Map<String, Object> propertyValues = getExtendedPropertyValues(extendedPropertyName, relatedObject, path.getValueExtraction(), configuration.isExtractFirstMatch());
+ if (allPropertyValues.isEmpty() && configuration.isExtractFirstMatch()) {
+ allPropertyValues = propertyValues;
+ break;
+ } else {
+ allPropertyValues = PropertyUtil.combineObjectMap(allPropertyValues, propertyValues);
+ }
+ }
+ } else {
+ List<SearchRecord> childrenRecords = searchChildrenRecords(relatedObjectsSpec.getRelatedObjectKind(), relatedObjectsSpec.getRelatedObjectID(), objectId);
+ for (SearchRecord record : childrenRecords) {
+ // If the child record is in the cache, that means the record was updated very recently.
+ // In this case, use the cache's record instead of the record from search result
+ RecordData cachedRecordData = this.relatedObjectCache.get(record.getId());
+ Map<String, Object> childDataMap = (cachedRecordData != null)? cachedRecordData.getData() : record.getData();
+ Map<String, Object> propertyValues = getExtendedPropertyValues(extendedPropertyName, childDataMap, path.getValueExtraction(), configuration.isExtractFirstMatch());
+ if (allPropertyValues.isEmpty() && configuration.isExtractFirstMatch()) {
+ allPropertyValues = propertyValues;
+ break;
+ } else {
+ allPropertyValues = PropertyUtil.combineObjectMap(allPropertyValues, propertyValues);
+ }
+ }
+ }
+ } else {
+ Map<String, Object> propertyValues = getExtendedPropertyValues(extendedPropertyName, originalDataMap, path.getValueExtraction(), configuration.isExtractFirstMatch());
+ if (allPropertyValues.isEmpty() && configuration.isExtractFirstMatch()) {
+ allPropertyValues = propertyValues;
+ } else {
+ allPropertyValues = PropertyUtil.combineObjectMap(allPropertyValues, propertyValues);
+ }
+ }
+
+ if (!allPropertyValues.isEmpty() && configuration.isExtractFirstMatch())
+ break;
+ }
+
+ extendedDataMap.putAll(allPropertyValues);
+ }
+ if (!associatedIdentities.isEmpty()) {
+ extendedDataMap.put(ASSOCIATED_IDENTITIES_PROPERTY, Arrays.asList(associatedIdentities.toArray()));
+ }
+
+ return extendedDataMap;
+ }
+
+ @Override
+ public List<SchemaItem> getExtendedSchemaItems(Schema originalSchema, Map<String, Schema> relatedObjectKindSchemas, PropertyConfigurations propertyConfigurations) {
+ List<SchemaItem> extendedSchemaItems = new ArrayList<>();
+ boolean hasChildToParentRelationship = false;
+ for (PropertyConfiguration configuration : propertyConfigurations.getConfigurations().stream().filter(c -> c.isValid()).collect(Collectors.toList())) {
+ Schema schema = null;
+ PropertyPath propertyPath = null;
+ for (PropertyPath path : configuration.getPaths().stream().filter(p -> p.hasValidRelatedObjectsSpec()).collect(Collectors.toList())) {
+ RelatedObjectsSpec relatedObjectsSpec = path.getRelatedObjectsSpec();
+ if (relatedObjectsSpec.isChildToParent()) {
+ hasChildToParentRelationship = true;
+ }
+ if (relatedObjectKindSchemas.containsKey(relatedObjectsSpec.getRelatedObjectKind())) {
+ // Refer to the schema of the related object
+ schema = relatedObjectKindSchemas.get(relatedObjectsSpec.getRelatedObjectKind());
+ propertyPath = path;
+ break;
+ }
+ }
+ if (schema == null) {
+ // Refer to the schema of the object itself
+ schema = originalSchema;
+ propertyPath = configuration.getPaths().stream().filter(p -> p.getRelatedObjectsSpec() == null && p.hasValidValueExtraction()).findFirst().orElse(null);
+ }
+
+ if (schema != null && propertyPath != null) {
+ List<SchemaItem> schemaItems = getExtendedSchemaItems(schema, configuration, propertyPath);
+ extendedSchemaItems.addAll(schemaItems);
+ }
+ }
+
+ if (hasChildToParentRelationship) {
+ extendedSchemaItems.add(createAssociatedIdentitiesSchemaItem());
+ }
+
+ return extendedSchemaItems;
+ }
+
+ @Override
+ public String resolveConcreteKind(String kind) {
+ if (Strings.isNullOrEmpty(kind) || PropertyUtil.isConcreteKind(kind)) {
+ return kind;
+ }
+
+ String concreteKind = kindCache.get(kind);
+ if (concreteKind == null) {
+ concreteKind = getLatestVersionOfKind(kind);
+ if (!Strings.isNullOrEmpty(concreteKind)) {
+ kindCache.put(kind, concreteKind);
+ }
+ }
+ return concreteKind;
+ }
+
+ @Override
+ public void cacheDataRecord(String recordId, String kind, Map<String, Object> dataMap) {
+ Map<String, Object> previousDataMap = this.getRelatedObjectData(kind, recordId);
+ RecordInfo recordInfo = new RecordInfo();
+ recordInfo.setId(recordId);
+ recordInfo.setKind(kind);
+ RecordChangeInfo changedInfo = new RecordChangeInfo();
+ changedInfo.setRecordInfo(recordInfo);
+ // Using recordChangeInfoCache is the best effort to avoid updating the associated records when unnecessary
+ // It should only store the updated records (ids) with updated properties. However, in order to
+ // handle the case that a new record is updated in a short period of time, the ids of the new records with OPT
+ // OperationType.create should be cached too.
+ if (previousDataMap == null || previousDataMap.isEmpty()) {
+ recordInfo.setOp(OperationType.create.getValue());
+ } else {
+ recordInfo.setOp(OperationType.update.getValue());
+ List<String> updatedProperties = PropertyUtil.getChangedProperties(previousDataMap, dataMap);
+
+ RecordChangeInfo previousChangedInfo = recordChangeInfoCache.get(recordId);
+ if(previousChangedInfo != null) {
+ if(previousChangedInfo.getRecordInfo().getOp().equals(OperationType.create.getValue())) {
+ recordInfo.setOp(OperationType.create.getValue());
+ }
+ else if(previousChangedInfo.getUpdatedProperties() != null) {
+ previousChangedInfo.getUpdatedProperties().forEach(p -> {
+ if(!updatedProperties.contains(p))
+ updatedProperties.add(p);
+ });
+ }
+ }
+
+ if(recordInfo.getOp().equals(OperationType.update.getValue()))
+ changedInfo.setUpdatedProperties(updatedProperties);
+ }
+ recordChangeInfoCache.put(recordId, changedInfo);
+ RecordData recordData = new RecordData();
+ recordData.setData(dataMap);
+ relatedObjectCache.put(recordId, recordData);
+ }
+
+ @Override
+ public void updateAssociatedRecords(RecordChangedMessages message, Map<String, List<String>> upsertKindIds, Map<String, List<String>> deleteKindIds) {
+ if (upsertKindIds == null) {
+ upsertKindIds = new HashMap<>();
+ }
+ if (deleteKindIds == null) {
+ deleteKindIds = new HashMap<>();
+ }
+
+ Map<String, String> attributes = message.getAttributes();
+ String ancestors = attributes.containsKey(Constants.ANCESTRY_KINDS) ? attributes.get(Constants.ANCESTRY_KINDS) : "";
+ Map<String, List<RecordChangeInfo>> recordChangeInfoMap = createRecordChangeInfoMap(upsertKindIds, deleteKindIds);
+ for (Map.Entry<String, List<RecordChangeInfo>> entry : recordChangeInfoMap.entrySet()) {
+ String kind = entry.getKey();
+ List<RecordChangeInfo> recordChangeInfoList = entry.getValue();
+ String updatedAncestors = Strings.isNullOrEmpty(ancestors) ? kind : ancestors + ANCESTRY_KINDS_DELIMITER + kind;
+
+ updateAssociatedParentRecords(updatedAncestors, kind, recordChangeInfoList);
+ updateAssociatedChildrenRecords(updatedAncestors, kind, recordChangeInfoList);
+ }
+ }
+
+ /******************************************************** Private methods **************************************************************/
+ private boolean isNullOrEmptyConfigurations(PropertyConfigurations configuration) {
+ return configuration == null || Strings.isNullOrEmpty(configuration.getCode());
+ }
+
+ private SchemaItem createAssociatedIdentitiesSchemaItem() {
+ SchemaItem extendedSchemaItem = new SchemaItem();
+ extendedSchemaItem.setPath(ASSOCIATED_IDENTITIES_PROPERTY);
+ extendedSchemaItem.setKind(ASSOCIATED_IDENTITIES_PROPERTY_STORAGE_FORMAT_TYPE);
+ return extendedSchemaItem;
+ }
+
+ private String createIdsFilter(List<String> ids) {
+ StringBuilder idsBuilder = new StringBuilder();
+ for (String id : ids) {
+ if (idsBuilder.length() > 0) {
+ idsBuilder.append(" OR ");
+ }
+ idsBuilder.append("\"");
+ idsBuilder.append(PropertyUtil.removeIdPostfix(id));
+ idsBuilder.append("\"");
+ }
+ return idsBuilder.toString();
+ }
+
+ private void createWorkerTask(String ancestors, List<RecordInfo> recordInfos) {
+ Map<String, String> attributes = new HashMap<>();
+ DpsHeaders headers = this.requestInfo.getHeadersWithDwdAuthZ();
+ attributes.put(DpsHeaders.ACCOUNT_ID, headers.getAccountId());
+ attributes.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
+ attributes.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
+ attributes.put(Constants.ANCESTRY_KINDS, ancestors);
+
+ RecordChangedMessages recordChangedMessages = RecordChangedMessages.builder().data(gson.toJson(recordInfos)).attributes(attributes).build();
+ String recordChangedMessagePayload = gson.toJson(recordChangedMessages);
+ this.indexerQueueTaskBuilder.createWorkerTask(recordChangedMessagePayload, 0L, this.requestInfo.getHeadersWithDwdAuthZ());
+ }
+
+ private Map<String, Object> getRelatedObjectData(String relatedObjectKind, String relatedObjectId) {
+ String key = PropertyUtil.removeIdPostfix(relatedObjectId);
+ RecordData recordData = relatedObjectCache.get(key);
+ Map<String, Object> relatedObject = (recordData != null)? recordData.getData() : null;
+ if (relatedObject == null) {
+ SearchRecord searchRecord = searchRelatedRecord(relatedObjectKind, relatedObjectId);
+ if (searchRecord != null) {
+ relatedObject = searchRecord.getData();
+
+ recordData = new RecordData();
+ recordData.setData(relatedObject);
+ relatedObjectCache.put(key, recordData);
+ }
+ }
+
+ return relatedObject;
+ }
+
+ private Map<String, List<RecordChangeInfo>> createRecordChangeInfoMap(Map<String, List<String>> upsertKindIds, Map<String, List<String>> deleteKindIds) {
+ Map<String, List<RecordChangeInfo>> recordChangeInfoMap = new HashMap<>();
+ for (Map.Entry<String, List<String>> entry : upsertKindIds.entrySet()) {
+ List<RecordChangeInfo> recordChangeInfoList = getOrCreateRecordChangeInfoList(entry.getKey(), recordChangeInfoMap);
+ for (String id : entry.getValue()) {
+ RecordChangeInfo changeInfo = recordChangeInfoCache.get(id);
+ if (changeInfo == null) {
+ changeInfo = new RecordChangeInfo();
+ changeInfo.setRecordInfo(this.createRecordInfo(entry.getKey(), id, OperationType.create));
+ }
+ recordChangeInfoList.add(changeInfo);
+ }
+ }
+ for (Map.Entry<String, List<String>> entry : deleteKindIds.entrySet()) {
+ List<RecordChangeInfo> recordChangeInfoList = getOrCreateRecordChangeInfoList(entry.getKey(), recordChangeInfoMap);
+ for (String id : entry.getValue()) {
+ RecordChangeInfo changeInfo = new RecordChangeInfo();
+ changeInfo.setRecordInfo(this.createRecordInfo(entry.getKey(), id, OperationType.delete));
+ recordChangeInfoList.add(changeInfo);
+ }
+ }
+
+ return recordChangeInfoMap;
+ }
+
+ private List<RecordChangeInfo> getOrCreateRecordChangeInfoList(String kind, Map<String, List<RecordChangeInfo>> recordChangeInfoMap) {
+ List<RecordChangeInfo> recordChangeInfoList;
+ if (recordChangeInfoMap.containsKey(kind)) {
+ recordChangeInfoList = recordChangeInfoMap.get(kind);
+ } else {
+ recordChangeInfoList = new ArrayList<>();
+ recordChangeInfoMap.put(kind, recordChangeInfoList);
+ }
+ return recordChangeInfoList;
+ }
+
+ private RecordInfo createRecordInfo(String kind, String id, OperationType operationType) {
+ RecordInfo recordInfo = new RecordInfo();
+ recordInfo.setKind(kind);
+ recordInfo.setId(id);
+ recordInfo.setOp(operationType.getValue());
+ return recordInfo;
+ }
+
+ private List<SchemaItem> getExtendedSchemaItems(Schema schema, PropertyConfiguration configuration, PropertyPath propertyPath) {
+ String relatedPropertyPath = PropertyUtil.removeDataPrefix(propertyPath.getValueExtraction().getValuePath());
+ if (relatedPropertyPath.contains(ARRAY_SYMBOL)) { // Nested
+ List<SchemaItem> extendedSchemaItems = new ArrayList<>();
+ extendedSchemaItems = cloneExtendedSchemaItemsFromNestedSchema(Arrays.asList(schema.getSchema()), configuration, relatedPropertyPath);
+ if (extendedSchemaItems.isEmpty()) {
+ // It is possible that the format of the source property is not defined
+ // In this case, we assume that the format of property is string in order to make its value(s) searchable
+ SchemaItem extendedSchemaItem = new SchemaItem();
+ extendedSchemaItem.setPath(configuration.getName());
+ if (configuration.isExtractFirstMatch()) {
+ extendedSchemaItem.setKind("string");
+ } else {
+ extendedSchemaItem.setKind("[]string");
+ }
+ extendedSchemaItems.add(extendedSchemaItem);
+ }
+ return extendedSchemaItems;
+ } else {// Flatten
+ List<SchemaItem> schemaItems = Arrays.asList(schema.getSchema());
+ return cloneExtendedSchemaItems(schemaItems, configuration, relatedPropertyPath);
+ }
+ }
+
+ private List<SchemaItem> cloneExtendedSchemaItems(List<SchemaItem> schemaItems, PropertyConfiguration configuration, String relatedPropertyPath) {
+ List<SchemaItem> extendedSchemaItems = new ArrayList<>();
+ for (SchemaItem schemaItem : schemaItems) {
+ if (PropertyUtil.isPropertyPathMatched(schemaItem.getPath(), relatedPropertyPath)) {
+ String path = schemaItem.getPath();
+ path = path.replace(relatedPropertyPath, configuration.getName());
+ SchemaItem extendedSchemaItem = new SchemaItem();
+ extendedSchemaItem.setPath(path);
+ if (configuration.isExtractFirstMatch()) {
+ extendedSchemaItem.setKind(schemaItem.getKind());
+ } else {
+ extendedSchemaItem.setKind("[]" + schemaItem.getKind());
+ }
+ extendedSchemaItems.add(extendedSchemaItem);
+ }
+ }
+ return extendedSchemaItems;
+ }
+
+ private List<SchemaItem> cloneExtendedSchemaItemsFromNestedSchema(List<SchemaItem> schemaItems, PropertyConfiguration configuration, String relatedPropertyPath) {
+ if (relatedPropertyPath.contains(ARRAY_SYMBOL)) {
+ List<SchemaItem> extendedSchemaItems = new ArrayList<>();
+ int idx = relatedPropertyPath.indexOf(NESTED_OBJECT_DELIMITER);
+ String prePath = relatedPropertyPath.substring(0, idx);
+ String postPath = relatedPropertyPath.substring(idx + NESTED_OBJECT_DELIMITER.length());
+ for (SchemaItem schemaItem : schemaItems) {
+ if (schemaItem.getPath().equals(prePath)) {
+ if (schemaItem.getKind().equals(SCHEMA_NESTED_KIND) && schemaItem.getProperties() != null) {
+ schemaItems = Arrays.asList(schemaItem.getProperties());
+ extendedSchemaItems = cloneExtendedSchemaItemsFromNestedSchema(schemaItems, configuration, postPath);
+ }
+ break;
+ }
+ }
+ return extendedSchemaItems;
+ } else {
+ return cloneExtendedSchemaItems(schemaItems, configuration, relatedPropertyPath);
+ }
+ }
+
+ private List<String> getRelatedObjectIds(Map<String, Object> dataMap, RelatedObjectsSpec relatedObjectsSpec) {
+ if (dataMap == null || dataMap.isEmpty() || relatedObjectsSpec == null || !relatedObjectsSpec.isValid())
+ return new ArrayList<>();
+
+ Map<String, Object> propertyValues = getPropertyValues(dataMap, relatedObjectsSpec.getRelatedObjectID(), relatedObjectsSpec, relatedObjectsSpec.hasValidCondition(), false);
+ List<String> relatedObjectIds = new ArrayList<>();
+ for (Object value : propertyValues.values()) {
+ if (value instanceof List) {
+ for (Object obj : (List) value) {
+ relatedObjectIds.add(obj.toString());
+ }
+ } else {
+ relatedObjectIds.add(value.toString());
+ }
+ }
+ return relatedObjectIds;
+ }
+
+ private Map<String, Object> getExtendedPropertyValues(String extendedPropertyName, Map<String, Object> dataMap, ValueExtraction valueExtraction, boolean isExtractFirstMatch) {
+ if (dataMap == null || dataMap.isEmpty() || valueExtraction == null || !valueExtraction.isValid())
+ return new HashMap<>();
+ Map<String, Object> propertyValues = getPropertyValues(dataMap, valueExtraction.getValuePath(), valueExtraction, valueExtraction.hasValidCondition(), isExtractFirstMatch);
+ return PropertyUtil.replacePropertyPaths(extendedPropertyName, valueExtraction.getValuePath(), propertyValues);
+ }
+
+ private Map<String, Object> getPropertyValues(Map<String, Object> dataMap, String valuePath, RelatedCondition relatedCondition, boolean hasValidCondition, boolean isExtractFirstMatch) {
+ valuePath = PropertyUtil.removeDataPrefix(valuePath);
+ Map<String, Object> propertyValues = new HashMap<>();
+ if (valuePath.contains(ARRAY_SYMBOL)) { // Nested
+ String conditionProperty = null;
+ List<String> conditionMatches = null;
+ if (hasValidCondition) {
+ int idx = relatedCondition.getRelatedConditionProperty().lastIndexOf(NESTED_OBJECT_DELIMITER);
+ conditionProperty = relatedCondition.getRelatedConditionProperty().substring(idx + NESTED_OBJECT_DELIMITER.length());
+ conditionMatches = relatedCondition.getRelatedConditionMatches();
+ }
+
+ List<Object> valueList = getPropertyValuesFromNestedObjects(dataMap, valuePath, conditionProperty, conditionMatches, hasValidCondition, isExtractFirstMatch);
+ if (!valueList.isEmpty()) {
+ if (isExtractFirstMatch) {
+ propertyValues.put(valuePath, valueList.get(0));
+ } else {
+ propertyValues.put(valuePath, valueList);
+ }
+ }
+ } else { // Flatten
+ for (Map.Entry<String, Object> entry : dataMap.entrySet()) {
+ String key = entry.getKey();
+ if (key.equals(valuePath) || key.startsWith(valuePath + PROPERTY_DELIMITER)) {
+ if (isExtractFirstMatch) {
+ propertyValues.put(key, entry.getValue());
+ } else {
+ List<Object> values = new ArrayList<>();
+ values.add(entry.getValue());
+ propertyValues.put(key, values);
+ }
+ }
+ }
+ }
+
+ return propertyValues;
+ }
+
+ private List<Object> getPropertyValuesFromNestedObjects(Map<String, Object> dataMap, String valuePath, String conditionProperty, List<String> conditionMatches, boolean hasCondition, boolean isExtractFirstMatch) {
+ Set<Object> propertyValues = new HashSet<>();
+
+ if (valuePath.contains(ARRAY_SYMBOL)) {
+ int idx = valuePath.indexOf(NESTED_OBJECT_DELIMITER);
+ String prePath = valuePath.substring(0, idx);
+ String postPath = valuePath.substring(idx + NESTED_OBJECT_DELIMITER.length());
+ try {
+ if (dataMap.containsKey(prePath) && dataMap.get(prePath) != null) {
+ List<Map<String, Object>> nestedObjects = (List<Map<String, Object>>) dataMap.get(prePath);
+ for (Map<String, Object> nestedObject : nestedObjects) {
+ List<Object> valueList = getPropertyValuesFromNestedObjects(nestedObject, postPath, conditionProperty, conditionMatches, hasCondition, isExtractFirstMatch);
+ if (valueList != null && !valueList.isEmpty()) {
+ propertyValues.addAll(valueList);
+ if (isExtractFirstMatch)
+ break;
+ }
+ }
+ }
+ } catch (Exception ex) {
+ //Ignore cast exception
+ }
+ } else if (dataMap.containsKey(valuePath) && dataMap.get(valuePath) != null) {
+ Object extractPropertyValue = dataMap.get(valuePath);
+ if (hasCondition) {
+ if (dataMap.containsKey(conditionProperty) && dataMap.get(conditionProperty) != null) {
+ String conditionPropertyValue = dataMap.get(conditionProperty).toString();
+ if (conditionMatches.contains(conditionPropertyValue) && extractPropertyValue != null) {
+ propertyValues.add(extractPropertyValue);
+ }
+ }
+ } else {
+ propertyValues.add(extractPropertyValue);
+ }
+ }
+
+ List<Object> propertyValueList = new ArrayList<>(propertyValues);
+ Collections.sort(propertyValueList, Comparator.comparing(Object::toString));
+ return propertyValueList;
+ }
+
+ private List<String> getChildrenKinds(String parentKind) {
+ final String parentKindWithMajor = PropertyUtil.getKindWithMajor(parentKind);
+ ChildrenKinds childrenKinds = childrenKindsCache.get(parentKindWithMajor);
+ if(childrenKinds == null) {
+ childrenKinds = new ChildrenKinds();
+ Set<String> kinds = new HashSet<>();
+ for (PropertyConfigurations propertyConfigurations: searchChildrenKindConfigurations(parentKindWithMajor)) {
+ kinds.add(propertyConfigurations.getCode());
+ }
+ childrenKinds.setKinds(new ArrayList<>(kinds));
+ childrenKindsCache.put(parentKindWithMajor, childrenKinds);
+ }
+
+ return childrenKinds.getKinds();
+ }
+
+ private ParentChildRelationshipSpecs getParentChildRelatedObjectsSpecs(String childKind) {
+ final String childKindWithMajor = PropertyUtil.getKindWithMajor(childKind);
+
+ ParentChildRelationshipSpecs specs = parentChildRelationshipSpecsCache.get(childKindWithMajor);
+ if (specs == null) {
+ List<ParentChildRelationshipSpec> specsList = new ArrayList<>();
+ specs = new ParentChildRelationshipSpecs();
+ specs.setSpecList(specsList);
+
+ List<PropertyConfigurations> configurationsList = searchParentKindConfigurations((childKindWithMajor));
+ for (PropertyConfigurations configurations : configurationsList) {
+ for (PropertyConfiguration configuration : configurations.getConfigurations()) {
+ List<PropertyPath> matchedPropertyPaths = configuration.getPaths().stream().filter(p ->
+ p.hasValidRelatedObjectsSpec() &&
+ p.getRelatedObjectsSpec().isParentToChildren() &&
+ p.getRelatedObjectsSpec().getRelatedObjectKind().contains(childKindWithMajor))
+ .collect(Collectors.toList());
+ for(PropertyPath propertyPath: matchedPropertyPaths) {
+ ParentChildRelationshipSpec spec = toParentChildRelationshipSpec(propertyPath, configurations.getCode(), childKindWithMajor);
+ boolean merged = false;
+ for(ParentChildRelationshipSpec sp: specsList) {
+ if(sp.equals(spec)) {
+ List<String> childValuePaths = sp.getChildValuePaths();
+ if(!childValuePaths.contains(spec.getChildValuePaths().get(0))) {
+ childValuePaths.add(spec.getChildValuePaths().get(0));
+ }
+ merged = true;
+ break;
+ }
+ }
+ if(!merged) {
+ specsList.add(spec);
+ }
+ }
+ }
+ }
+
+ parentChildRelationshipSpecsCache.put(childKindWithMajor, specs);
+ }
+
+ return specs;
+ }
+
+ private ParentChildRelationshipSpec toParentChildRelationshipSpec(PropertyPath propertyPath, String parentKind, String childKind) {
+ ParentChildRelationshipSpec spec = new ParentChildRelationshipSpec();
+ spec.setParentKind(parentKind);
+ spec.setParentObjectIdPath(propertyPath.getRelatedObjectsSpec().getRelatedObjectID());
+ spec.setChildKind(childKind);
+ String valuePath = PropertyUtil.removeDataPrefix(propertyPath.getValueExtraction().getValuePath());
+ spec.getChildValuePaths().add(valuePath);
+ return spec;
+ }
+
+ private void updateAssociatedParentRecords(String ancestors, String childKind, List<RecordChangeInfo> childRecordChangeInfos) {
+ ParentChildRelationshipSpecs specs = getParentChildRelatedObjectsSpecs(childKind);
+ Set ancestorSet = new HashSet<>(Arrays.asList(ancestors.split(ANCESTRY_KINDS_DELIMITER)));
+ for (ParentChildRelationshipSpec spec : specs.getSpecList()) {
+ List childRecordIds = getChildRecordIdsWithExtendedPropertiesChanged(spec, childRecordChangeInfos);
+ if (childRecordIds.isEmpty())
+ continue;
+
+ List<String> parentIds = searchUniqueParentIds(childKind, childRecordIds, spec.getParentObjectIdPath());
+ if (parentIds.isEmpty())
+ continue;
+
+ final int limit = configurationProperties.getStorageRecordsByKindBatchSize();
+ Map<String, List<String>> parentKindIds = searchKindIds(spec.getParentKind(), parentIds);
+ List<RecordInfo> recordInfos = new ArrayList<>();
+ for (Map.Entry<String, List<String>> entry : parentKindIds.entrySet()) {
+ if (ancestorSet.contains(entry.getKey()))
+ continue; // circular indexing found.
+
+ for (String id : entry.getValue()) {
+ RecordInfo recordInfo = new RecordInfo();
+ recordInfo.setKind(entry.getKey());
+ recordInfo.setId(id);
+ recordInfo.setOp(OperationType.update.getValue());
+ recordInfos.add(recordInfo);
+
+ if (recordInfos.size() >= limit) {
+ createWorkerTask(ancestors, recordInfos);
+ recordInfos = new ArrayList<>();
+ }
+ }
+ }
+ if (!recordInfos.isEmpty()) {
+ createWorkerTask(ancestors, recordInfos);
+ }
+ }
+ }
+
+ private List<String> getChildRecordIdsWithExtendedPropertiesChanged(ParentChildRelationshipSpec spec, List<RecordChangeInfo> childRecordChangeInfos) {
+ List<String> childRecordIds = new ArrayList<>();
+ for (RecordChangeInfo recordChangeInfo : childRecordChangeInfos) {
+ if (recordChangeInfo.getRecordInfo().getOp().equals(OperationType.update.getValue())) {
+ String updatedExtendedProperty = recordChangeInfo.getUpdatedProperties().stream().filter(p -> {
+ for (String valuePath : spec.getChildValuePaths()) {
+ if (PropertyUtil.isPropertyPathMatched(valuePath, p) ||
+ PropertyUtil.isPropertyPathMatched(p, valuePath)) {
+ return true;
+ }
+ }
+ return false;
+ }).findFirst().orElse(null);
+
+ if (updatedExtendedProperty != null) {
+ // The parent property that is extended by the children was updated
+ childRecordIds.add(recordChangeInfo.getRecordInfo().getId());
+ }
+ }
+ else {
+ childRecordIds.add(recordChangeInfo.getRecordInfo().getId());
+ }
+ }
+ return childRecordIds;
+ }
+
+ private boolean areExtendedPropertiesChanged(String childKind, List<RecordChangeInfo> parentRecordChangeInfos) {
+ if (parentRecordChangeInfos.stream().filter(info -> !info.getRecordInfo().getOp().equals(OperationType.update.getValue())).findFirst().orElse(null) != null) {
+ // If there is any OP of the parent record(s) that is not OperationType.update. It must be OperationType.delete in this case. Then the child record should be updated
+ return true;
+ }
+
+ PropertyConfigurations propertyConfigurations = this.getPropertyConfigurations(childKind);
+ if(propertyConfigurations != null) {
+ for (PropertyConfiguration propertyConfiguration : propertyConfigurations.getConfigurations()) {
+ for (PropertyPath propertyPath : propertyConfiguration.getPaths().stream().filter(
+ p -> p.hasValidValueExtraction() && p.hasValidRelatedObjectsSpec()).collect(Collectors.toList())) {
+ String relatedObjectKind = propertyPath.getRelatedObjectsSpec().getRelatedObjectKind();
+ String valuePath = PropertyUtil.removeDataPrefix(propertyPath.getValueExtraction().getValuePath());
+
+ // Find any parent record which has changed property that is extended by the child (kind)
+ RecordChangeInfo parentRecordChangeInfo = parentRecordChangeInfos.stream().filter(info -> {
+ if (PropertyUtil.hasSameMajorKind(info.getRecordInfo().getKind(), relatedObjectKind)) {
+ List<String> matchedProperties = info.getUpdatedProperties().stream().filter(
+ p -> PropertyUtil.isPropertyPathMatched(p, valuePath) || PropertyUtil.isPropertyPathMatched(valuePath, p)).collect(Collectors.toList());
+ return !matchedProperties.isEmpty();
+ }
+ return false;
+ }).findFirst().orElse(null);
+ if (parentRecordChangeInfo != null) {
+ return true;
+ }
+ }
+ }
+ }
+
+ return false;
+ }
+
+ private void updateAssociatedChildrenRecords(String ancestors, String parentKind, List<RecordChangeInfo> recordChangeInfos) {
+ List<String> processedIds = recordChangeInfos.stream().map(recordChangeInfo -> recordChangeInfo.getRecordInfo().getId()).collect(Collectors.toList());
+ String query = String.format("data.%s:(%s)", ASSOCIATED_IDENTITIES_PROPERTY, createIdsFilter(processedIds));
+
+ List<String> childrenKinds = getChildrenKinds(parentKind);
+ for (String ancestryKind : ancestors.split(ANCESTRY_KINDS_DELIMITER)) {
+ // Exclude the kinds in the ancestryKinds to prevent circular chasing
+ childrenKinds.removeIf(k -> ancestryKind.contains(k));
+ }
+ if(childrenKinds.isEmpty()) {
+ return;
+ }
+
+ List<String> multiKinds = new ArrayList<>();
+ for(String kind: childrenKinds) {
+ String kindWithMajor = PropertyUtil.getKindWithMajor(kind) + "*.*";
+ multiKinds.add(kindWithMajor);
+ }
+ SearchRequest searchRequest = new SearchRequest();
+ searchRequest.setKind(multiKinds);
+ searchRequest.setQuery(query);
+ searchRequest.setReturnedFields(Arrays.asList("kind", "id", "data." + ASSOCIATED_IDENTITIES_PROPERTY));
+ List<RecordInfo> recordInfos = new ArrayList<>();
+ for (SearchRecord record : searchAllRecords(searchRequest)) {
+ Map<String, Object> data = record.getData();
+ if (!data.containsKey(ASSOCIATED_IDENTITIES_PROPERTY) || data.get(ASSOCIATED_IDENTITIES_PROPERTY) == null)
+ continue;
+
+ List<String> associatedParentIds = (List<String>) data.get(ASSOCIATED_IDENTITIES_PROPERTY);
+ List<RecordChangeInfo> associatedParentRecordChangeInfos = recordChangeInfos.stream().filter(
+ info -> associatedParentIds.contains(info.getRecordInfo().getId())).collect(Collectors.toList());
+ if (areExtendedPropertiesChanged(record.getKind(), associatedParentRecordChangeInfos)) {
+ RecordInfo recordInfo = new RecordInfo();
+ recordInfo.setKind(record.getKind());
+ recordInfo.setId(record.getId());
+ recordInfo.setOp(OperationType.update.getValue());
+ recordInfos.add(recordInfo);
+
+ if (recordInfos.size() >= configurationProperties.getStorageRecordsByKindBatchSize()) {
+ createWorkerTask(ancestors, recordInfos);
+ recordInfos = new ArrayList<>();
+ }
+ }
+ }
+ if (!recordInfos.isEmpty()) {
+ createWorkerTask(ancestors, recordInfos);
+ }
+ }
+
+ private String getLatestVersionOfKind(String kindWithMajor) {
+ Kind kind = new Kind(kindWithMajor);
+ String version = kind.getVersion();
+ String[] subVersions = version.split("\\.");
+ String majorVersion = subVersions[0];
+ String latestKind = null;
+ try {
+ SchemaInfoResponse response = schemaService.getSchemaInfos(kind.getAuthority(), kind.getSource(), kind.getType(), majorVersion, null, null, true);
+ if (response != null && response.getSchemaInfos() != null && response.getSchemaInfos().size() > 0) {
+ SchemaInfo schemaInfo = response.getSchemaInfos().get(0);
+ SchemaIdentity schemaIdentity = schemaInfo.getSchemaIdentity();
+ latestKind = schemaIdentity.getAuthority() + ":" +
+ schemaIdentity.getSource() + ":" +
+ schemaIdentity.getEntityType() + ":" +
+ schemaIdentity.getSchemaVersionMajor() + "." +
+ schemaIdentity.getSchemaVersionMinor() + "." +
+ schemaIdentity.getSchemaVersionPatch();
+ }
+ } catch (URISyntaxException e) {
+ jaxRsDpsLog.error("failed to get schema info", e);
+ } catch (UnsupportedEncodingException e) {
+ jaxRsDpsLog.error("failed to get schema info", e);
+ }
+
+ return latestKind;
+ }
+
+ /****************************** search methods that use search service to get the data **************************************/
+ private SearchRequest createSearchRequest(String kind, String query) {
+ SearchRequest searchRequest = new SearchRequest();
+ searchRequest.setKind(kind);
+ searchRequest.setQuery(query);
+ return searchRequest;
+ }
+
+ private PropertyConfigurations searchConfigurations(String kind) {
+ String query = String.format("data.Code: \"%s\"", kind);
+ SearchRequest searchRequest = createSearchRequest(INDEX_PROPERTY_PATH_CONFIGURATION_KIND, query);
+ for(PropertyConfigurations configurations : searchConfigurations(searchRequest)) {
+ if (kind.equals(configurations.getCode())) {
+ return configurations;
+ }
+ }
+ return null;
+ }
+
+ private List<PropertyConfigurations> searchParentKindConfigurations(String childKind) {
+ String query = String.format(PARENT_CHILDREN_CONFIGURATION_QUERY_FORMAT, childKind);
+ SearchRequest searchRequest = createSearchRequest(INDEX_PROPERTY_PATH_CONFIGURATION_KIND, query);
+ return searchConfigurations(searchRequest);
+ }
+
+ private List<PropertyConfigurations> searchChildrenKindConfigurations(String parentKind) {
+ String query = String.format(CHILDREN_PARENT_CONFIGURATION_QUERY_FORMAT, parentKind);
+ SearchRequest searchRequest = createSearchRequest(INDEX_PROPERTY_PATH_CONFIGURATION_KIND, query);
+ return searchConfigurations(searchRequest);
+ }
+
+ private List<PropertyConfigurations> searchConfigurations(SearchRequest searchRequest) {
+ List<PropertyConfigurations> configurationsList = new ArrayList<>();
+ for (SearchRecord searchRecord : searchAllRecords(searchRequest)) {
+ try {
+ String data = objectMapper.writeValueAsString(searchRecord.getData());
+ PropertyConfigurations configurations = objectMapper.readValue(data, PropertyConfigurations.class);
+ String kind = PropertyUtil.getKindWithMajor(configurations.getCode());
+ propertyConfigurationCache.put(kind, configurations);
+ configurationsList.add(configurations);
+ } catch (JsonProcessingException e) {
+ jaxRsDpsLog.error("failed to deserialize PropertyConfigurations object", e);
+ }
+ }
+ return configurationsList;
+ }
+
+ private SearchRecord searchRelatedRecord(String relatedObjectKind, String relatedObjectId) {
+ String kind = PropertyUtil.isConcreteKind(relatedObjectKind) ? relatedObjectKind : relatedObjectKind + "*";
+ String id = PropertyUtil.removeIdPostfix(relatedObjectId);
+ SearchRequest searchRequest = new SearchRequest();
+ searchRequest.setKind(kind);
+ String query = String.format("id: \"%s\"", id);
+ searchRequest.setQuery(query);
+ return searchFirstRecord(searchRequest);
+ }
+
+ private Map<String, List<String>> searchKindIds(String majorKind, List<String> ids) {
+ Map<String, List<String>> kindIds = new HashMap<>();
+ SearchRequest searchRequest = new SearchRequest();
+ String kind = PropertyUtil.isConcreteKind(majorKind) ? majorKind : majorKind + "*";
+ searchRequest.setKind(kind);
+ String query = String.format("id: (%s)", createIdsFilter(ids));
+ searchRequest.setReturnedFields(Arrays.asList("kind", "id"));
+ searchRequest.setQuery(query);
+ for (SearchRecord record : searchAllRecords(searchRequest)) {
+ if (kindIds.containsKey(record.getKind())) {
+ kindIds.get(record.getKind()).add(record.getId());
+ } else {
+ List<String> idList = new ArrayList<>();
+ idList.add(record.getId());
+ kindIds.put(record.getKind(), idList);
+ }
+ }
+ return kindIds;
+ }
+
+ private List<String> searchUniqueParentIds(String childKind, List<String> childRecordIds, String parentObjectIdPath) {
+ Set<String> parentIds = new HashSet<>();
+ SearchRequest searchRequest = new SearchRequest();
+ searchRequest.setKind(childKind);
+ String query = String.format("id: (%s)", createIdsFilter(childRecordIds));
+ searchRequest.setReturnedFields(Arrays.asList(parentObjectIdPath));
+ searchRequest.setQuery(query);
+ parentObjectIdPath = PropertyUtil.removeDataPrefix(parentObjectIdPath);
+ for (SearchRecord record : searchAllRecords(searchRequest)) {
+ if (record.getData().containsKey(parentObjectIdPath)) {
+ Object id = record.getData().get(parentObjectIdPath);
+ if (id != null && !parentIds.contains(id)) {
+ parentIds.add(id.toString());
+ }
+ }
+ }
+ return new ArrayList<>(parentIds);
+ }
+
+ private List<SearchRecord> searchChildrenRecords(String childrenObjectKind, String childrenObjectField, String parentId) {
+ String kind = PropertyUtil.isConcreteKind(childrenObjectKind) ? childrenObjectKind : childrenObjectKind + "*";
+ SearchRequest searchRequest = new SearchRequest();
+ searchRequest.setKind(kind);
+ String query = String.format("%s: \"%s\"", childrenObjectField, parentId);
+ searchRequest.setQuery(query);
+ return searchAllRecords(searchRequest);
+ }
+
+ private List<SearchRecord> searchAllRecords(SearchRequest searchRequest) {
+ searchRequest.setLimit(MAX_SEARCH_LIMIT);
+ List<SearchRecord> allRecords = new ArrayList<>();
+ boolean done = false;
+ try {
+ while (!done) {
+ SearchResponse searchResponse = searchService.queryWithCursor(searchRequest);
+ List<SearchRecord> results = searchResponse.getResults();
+ if (results != null && results.size() > 0) {
+ allRecords.addAll(results);
+ }
+
+ if (!Strings.isNullOrEmpty(searchResponse.getCursor())) {
+ searchRequest.setCursor(searchResponse.getCursor());
+ } else {
+ done = true;
+ }
+ }
+ } catch (URISyntaxException e) {
+ jaxRsDpsLog.error("Failed to call search service.", e);
+ }
+ return allRecords;
+ }
+
+ private SearchRecord searchFirstRecord(SearchRequest searchRequest) {
+ searchRequest.setLimit(1);
+ try {
+ SearchResponse searchResponse = searchService.query(searchRequest);
+ List<SearchRecord> results = searchResponse.getResults();
+ if (results != null && !results.isEmpty()) {
+ return results.get(0);
+ }
+ } catch (URISyntaxException e) {
+ jaxRsDpsLog.error("Failed to call search service.", e);
+ }
+ return null;
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexService.java
index 6569743e49fd39a381eec56123e14ea7b96c7232..231cbf2ad089e0839efe8e5141a1ea4ae7078f09 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexService.java
@@ -16,10 +16,15 @@ package org.opengroup.osdu.indexer.service;
import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
+import org.opengroup.osdu.core.common.model.indexer.Records;
+
+import java.util.List;
public interface ReindexService {
- String reindexRecords(RecordReindexRequest recordReindexRequest, boolean forceClean);
+ String reindexKind(RecordReindexRequest recordReindexRequest, boolean forceClean);
+
+ Records reindexRecords(List<String> recordIds);
void fullReindex(boolean forceClean);
-}
\ No newline at end of file
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java
index af15d671f8f1cbc4ad1907daf99be0cf0e9a0297..062c48da5c31506fd443d9551191d1ac86cf6af0 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java
@@ -17,27 +17,21 @@ package org.opengroup.osdu.indexer.service;
import com.google.common.base.Strings;
import com.google.gson.Gson;
-import java.util.Objects;
+import java.util.*;
import lombok.SneakyThrows;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.indexer.OperationType;
-import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
-import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
+import org.opengroup.osdu.core.common.model.indexer.*;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
-import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
import org.springframework.stereotype.Component;
import javax.inject.Inject;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
import java.util.stream.Collectors;
@Component
@@ -58,7 +52,7 @@ public class ReindexServiceImpl implements ReindexService {
@SneakyThrows
@Override
- public String reindexRecords(RecordReindexRequest recordReindexRequest, boolean forceClean) {
+ public String reindexKind(RecordReindexRequest recordReindexRequest, boolean forceClean) {
Long initialDelayMillis = 0l;
@@ -75,20 +69,12 @@ public class ReindexServiceImpl implements ReindexService {
List<RecordInfo> msgs = recordQueryResponse.getResults().stream()
.map(record -> RecordInfo.builder().id(record).kind(recordReindexRequest.getKind()).op(OperationType.create.name()).build()).collect(Collectors.toList());
-
- Map<String, String> attributes = new HashMap<>();
- attributes.put(DpsHeaders.ACCOUNT_ID, headers.getAccountId());
- attributes.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
- attributes.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
-
- Gson gson = new Gson();
- RecordChangedMessages recordChangedMessages = RecordChangedMessages.builder().data(gson.toJson(msgs)).attributes(attributes).build();
- String recordChangedMessagePayload = gson.toJson(recordChangedMessages);
- this.indexerQueueTaskBuilder.createWorkerTask(recordChangedMessagePayload, initialDelayMillis, headers);
+ String recordChangedMessagePayload = this.replayReindexMsg(msgs, initialDelayMillis, headers);
// don't call reindex-worker endpoint if it's the last batch
// previous storage query result size will be less then requested (limit param)
if (!Strings.isNullOrEmpty(recordQueryResponse.getCursor()) && recordQueryResponse.getResults().size() == configurationProperties.getStorageRecordsByKindBatchSize()) {
+ Gson gson = new Gson();
String newPayLoad = gson.toJson(RecordReindexRequest.builder().cursor(recordQueryResponse.getCursor()).kind(recordReindexRequest.getKind()).build());
this.indexerQueueTaskBuilder.createReIndexTask(newPayLoad, initialDelayMillis, headers);
return newPayLoad;
@@ -101,6 +87,18 @@ public class ReindexServiceImpl implements ReindexService {
return null;
}
+ @SneakyThrows
+ @Override
+ public Records reindexRecords(List<String> recordIds) {
+ Records records = this.storageService.getStorageRecords(recordIds);
+ if (records.getRecords().size() > 0) {
+ List<RecordInfo> msgs = records.getRecords().stream()
+ .map(record -> RecordInfo.builder().id(record.getId()).kind(record.getKind()).op(OperationType.create.name()).build()).collect(Collectors.toList());
+ this.replayReindexMsg(msgs, 0L, null);
+ }
+ return records;
+ }
+
@Override
public void fullReindex(boolean forceClean) {
List<String> allKinds = null;
@@ -115,11 +113,26 @@ public class ReindexServiceImpl implements ReindexService {
}
for (String kind : allKinds) {
try {
- reindexRecords(new RecordReindexRequest(kind, ""), forceClean);
+ reindexKind(new RecordReindexRequest(kind, ""), forceClean);
} catch (Exception e) {
jaxRsDpsLog.warning(String.format("kind: %s cannot be re-indexed", kind));
continue;
}
}
}
-}
\ No newline at end of file
+
+ private String replayReindexMsg(List<RecordInfo> msgs, Long initialDelayMillis, DpsHeaders headers) {
+ Map<String, String> attributes = new HashMap<>();
+ if (headers == null) {
+ headers = this.requestInfo.getHeadersWithDwdAuthZ();
+ }
+ attributes.put(DpsHeaders.ACCOUNT_ID, headers.getAccountId());
+ attributes.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
+ attributes.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
+ Gson gson = new Gson();
+ RecordChangedMessages recordChangedMessages = RecordChangedMessages.builder().data(gson.toJson(msgs)).attributes(attributes).build();
+ String recordChangedMessagePayload = gson.toJson(recordChangedMessages);
+ this.indexerQueueTaskBuilder.createWorkerTask(recordChangedMessagePayload, initialDelayMillis, headers);
+ return recordChangedMessagePayload;
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SchemaProviderImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SchemaProviderImpl.java
index 6a688858371e14480c9a090b9a66fa75108553ec..8a9fd0cac07e83c46e5e9e3a48076076bfc2a455 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SchemaProviderImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SchemaProviderImpl.java
@@ -15,12 +15,15 @@
package org.opengroup.osdu.indexer.service;
import com.google.api.client.http.HttpMethods;
+
import java.io.UnsupportedEncodingException;
import java.net.URISyntaxException;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
-import java.util.Objects;
import javax.inject.Inject;
+
+import com.google.api.client.util.Strings;
+import com.google.gson.Gson;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.http.FetchServiceHttpRequest;
import org.opengroup.osdu.core.common.http.IUrlFetchService;
@@ -29,6 +32,7 @@ import org.opengroup.osdu.core.common.model.http.HttpResponse;
import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
import org.opengroup.osdu.indexer.logging.AuditLogger;
+import org.opengroup.osdu.indexer.model.SchemaInfoResponse;
import org.opengroup.osdu.indexer.schema.converter.interfaces.SchemaToStorageFormat;
import org.springframework.stereotype.Component;
@@ -37,6 +41,8 @@ import org.springframework.stereotype.Component;
*/
@Component
public class SchemaProviderImpl implements SchemaService {
+ private final Gson gson = new Gson();
+ private final int MAX_NUMBER_OF_SCHEMA_INFOS = 1000;
@Inject
private JaxRsDpsLog log;
@@ -65,6 +71,20 @@ public class SchemaProviderImpl implements SchemaService {
return schemaServiceSchema;
}
+ @Override
+ public SchemaInfoResponse getSchemaInfos(String authority, String source, String entityType, String majorVersion, String minorVersion, String patchVersion, boolean latestVersion) throws URISyntaxException, UnsupportedEncodingException {
+ String queryParams = buildQueryString(authority, source, entityType, majorVersion, minorVersion, patchVersion, latestVersion);
+ String url = String.format("%s?%s", configurationProperties.getSchemaHost(), queryParams);
+ FetchServiceHttpRequest request = FetchServiceHttpRequest.builder()
+ .httpMethod(HttpMethods.GET)
+ .headers(this.requestInfo.getHeadersMapWithDwdAuthZ())
+ .url(url)
+ .build();
+
+ HttpResponse response = this.urlFetchService.sendRequest(request);
+ return gson.fromJson(response.getBody(), SchemaInfoResponse.class);
+ }
+
protected String getFromSchemaService(String kind) throws UnsupportedEncodingException, URISyntaxException {
HttpResponse response = getSchemaServiceResponse(kind);
@@ -99,4 +119,28 @@ public class SchemaProviderImpl implements SchemaService {
return this.urlFetchService.sendRequest(request);
}
+
+ private String buildQueryString(String authority, String source, String entityType, String majorVersion, String minorVersion, String patchVersion, boolean latestVersion) throws UnsupportedEncodingException {
+ StringBuilder stringBuilder = new StringBuilder();
+ addQueryParam(stringBuilder, "authority", authority);
+ addQueryParam(stringBuilder, "source", source);
+ addQueryParam(stringBuilder, "entityType", entityType);
+ addQueryParam(stringBuilder, "schemaVersionMajor", majorVersion);
+ addQueryParam(stringBuilder, "schemaVersionMinor", minorVersion);
+ addQueryParam(stringBuilder, "schemaVersionPatch", patchVersion);
+ addQueryParam(stringBuilder, "latestVersion", String.valueOf(latestVersion));
+ addQueryParam(stringBuilder, "limit", String.valueOf(MAX_NUMBER_OF_SCHEMA_INFOS));
+ return stringBuilder.toString();
+ }
+
+ private StringBuilder addQueryParam(StringBuilder stringBuilder, String paramName, String paramValue) {
+ if(!Strings.isNullOrEmpty(paramName) && !Strings.isNullOrEmpty(paramValue)) {
+ if (stringBuilder.length() > 0)
+ stringBuilder.append("&");
+ stringBuilder.append(paramName);
+ stringBuilder.append("=");
+ stringBuilder.append(paramValue);
+ }
+ return stringBuilder;
+ }
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SchemaService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SchemaService.java
index cb161958c76d27981321707df057b17c03cf3317..ccea60df40cb1593e65d56286c302626cf15b260 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SchemaService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SchemaService.java
@@ -14,15 +14,18 @@
package org.opengroup.osdu.indexer.service;
+import org.opengroup.osdu.indexer.model.SchemaInfo;
+import org.opengroup.osdu.indexer.model.SchemaInfoResponse;
+
import java.io.UnsupportedEncodingException;
import java.net.URISyntaxException;
+import java.util.List;
/**
* Interface to consume schemas from the Schema Service
*/
public interface SchemaService {
/**
- *
* @param kind key to retrieve schema
* @return obtained schema
* @throws URISyntaxException
@@ -30,4 +33,12 @@ public interface SchemaService {
*/
String getSchema(String kind) throws URISyntaxException, UnsupportedEncodingException;
+ /**
+ * @param authority
+ * @param source
+ * @param entityType
+ * @param majorVersion
+ * @return The latest version of the kind
+ */
+ SchemaInfoResponse getSchemaInfos(String authority, String source, String entityType, String majorVersion, String minorVersion, String patchVersion, boolean latestVersion) throws URISyntaxException, UnsupportedEncodingException;
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SearchService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SearchService.java
new file mode 100644
index 0000000000000000000000000000000000000000..6617e4b90bb8fccce1f0d0e2cc8ae6de941a5dcc
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SearchService.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.service;
+
+import org.opengroup.osdu.indexer.model.SearchRequest;
+import org.opengroup.osdu.indexer.model.SearchResponse;
+
+import java.net.URISyntaxException;
+
+/**
+ * Interface to consume schemas from the Schema Service
+ */
+public interface SearchService {
+ /**
+ *
+ * @param searchRequest
+ * @return
+ * @throws URISyntaxException
+ */
+ SearchResponse query(SearchRequest searchRequest) throws URISyntaxException;
+
+ /**
+ *
+ * @param searchRequest
+ * @return
+ * @throws URISyntaxException
+ */
+ SearchResponse queryWithCursor(SearchRequest searchRequest) throws URISyntaxException;
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SearchServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SearchServiceImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..51f39344987c7b6ff700bc59b3ae10ed8aa5335f
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SearchServiceImpl.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.service;
+
+import com.google.api.client.http.HttpMethods;
+import com.google.common.base.Strings;
+import com.google.gson.Gson;
+import org.opengroup.osdu.core.common.http.FetchServiceHttpRequest;
+import org.opengroup.osdu.core.common.http.IUrlFetchService;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.HttpResponse;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.opengroup.osdu.indexer.model.SearchRequest;
+import org.opengroup.osdu.indexer.model.SearchResponse;
+import org.springframework.stereotype.Component;
+
+import javax.inject.Inject;
+import java.net.URISyntaxException;
+
+@Component
+public class SearchServiceImpl implements SearchService {
+ private static final String QUERY_PATH = "query";
+ private static final String QUERY_WITH_CURSOR_PATH = "query_with_cursor";
+ private final Gson gson = new Gson();
+ private static final int OK_CODE = 200;
+
+ @Inject
+ private IUrlFetchService urlFetchService;
+ @Inject
+ private IRequestInfo requestInfo;
+ @Inject
+ private IndexerConfigurationProperties configurationProperties;
+ @Inject
+ private JaxRsDpsLog jaxRsDpsLog;
+
+ @Override
+ public SearchResponse query(SearchRequest searchRequest) throws URISyntaxException {
+ return searchRecords(searchRequest, QUERY_PATH);
+ }
+
+ @Override
+ public SearchResponse queryWithCursor(SearchRequest searchRequest) throws URISyntaxException {
+ return searchRecords(searchRequest, QUERY_WITH_CURSOR_PATH);
+ }
+
+ private SearchResponse searchRecords(SearchRequest searchRequest, String path) throws URISyntaxException {
+ if(Strings.isNullOrEmpty(configurationProperties.getSearchHost())) {
+ jaxRsDpsLog.error("SEARCH_HOST", "The environment variable SEARCH_HOST is not setup");
+ return new SearchResponse();
+ }
+
+ try {
+ String body = this.gson.toJson(searchRequest);
+ String url = String.format("%s/%s", configurationProperties.getSearchHost(), path);
+ FetchServiceHttpRequest request = FetchServiceHttpRequest.builder()
+ .httpMethod(HttpMethods.POST)
+ .url(url)
+ .headers(this.requestInfo.getHeaders())
+ .body(body)
+ .build();
+ HttpResponse response = this.urlFetchService.sendRequest(request);
+
+ if (response != null && response.getResponseCode() == OK_CODE) {
+ return gson.fromJson(response.getBody(), SearchResponse.class);
+ } else {
+ if (response != null)
+ jaxRsDpsLog.error(String.format("Search service: failed to call the search service: %d", response.getResponseCode()));
+ else
+ jaxRsDpsLog.error(String.format("Search service: failed to call the search service. The response is null."));
+ return new SearchResponse();
+ }
+ }
+ catch(URISyntaxException ex) {
+ throw ex;
+ }
+ catch(Exception ex) {
+ jaxRsDpsLog.error(String.format("Search service: failed to call the search service", ex));
+ throw new URISyntaxException(ex.getMessage(), "Unexpected exception type: " + ex.getClass().getName());
+ }
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java
index 91d120a692b15f4fec2fa0d7de8ba4f4ac64ee1c..bba17ba3e2748270af062067688ef4378880e433 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapper.java
@@ -30,10 +30,9 @@ import org.opengroup.osdu.indexer.schema.converter.interfaces.IVirtualProperties
import org.opengroup.osdu.indexer.schema.converter.tags.Priority;
import org.opengroup.osdu.indexer.schema.converter.tags.VirtualProperties;
import org.opengroup.osdu.indexer.schema.converter.tags.VirtualProperty;
-import org.opengroup.osdu.indexer.util.VirtualPropertyUtil;
+import org.opengroup.osdu.indexer.util.PropertyUtil;
import org.opengroup.osdu.indexer.util.geo.decimator.DecimatedResult;
import org.opengroup.osdu.indexer.util.geo.decimator.GeoShapeDecimator;
-import org.opengroup.osdu.indexer.util.geo.decimator.GeoShapeDecimationSetting;
import org.springframework.stereotype.Component;
import javax.inject.Inject;
@@ -58,8 +57,6 @@ public class StorageIndexerPayloadMapper {
private IVirtualPropertiesSchemaCache virtualPropertiesSchemaCache;
@Inject
private GeoShapeDecimator decimator;
- @Inject
- private GeoShapeDecimationSetting decimationSetting;
public Map<String, Object> mapDataPayload(IndexSchema storageSchema, Map<String, Object> storageRecordData,
String recordId) {
@@ -201,7 +198,7 @@ public class StorageIndexerPayloadMapper {
} catch (NoSuchMethodException e) {
this.log.warning(String.format("record-id: %s | error fetching property: %s | error: %s", recordId, propertyKey, e.getMessage()));
} catch (IllegalArgumentException | IllegalAccessException | InvocationTargetException e) {
- this.log.warning(String.format("record-id: %s | error fetching property: %s | error: %s", recordId, propertyKey, e.getMessage()), e);
+ this.log.warning(String.format("record-id: %s | error fetching property: %s | error: %s", recordId, propertyKey, e.getMessage()));
}
return null;
}
@@ -223,21 +220,21 @@ public class StorageIndexerPayloadMapper {
continue;
}
Priority priority = chooseOriginalProperty(entry.getKey(), entry.getValue().getPriorities(), dataCollectorMap);
- String virtualPropertyPath = VirtualPropertyUtil.removeDataPrefix(entry.getKey());
- String originalPropertyPath = VirtualPropertyUtil.removeDataPrefix(priority.getPath());
+ String virtualPropertyPath = PropertyUtil.removeDataPrefix(entry.getKey());
+ String originalPropertyPath = PropertyUtil.removeDataPrefix(priority.getPath());
// Populate the virtual property values from the chosen original property
List<String> originalPropertyNames = dataCollectorMap.keySet().stream()
- .filter(originalPropertyName -> VirtualPropertyUtil.isPropertyPathMatched(originalPropertyName, originalPropertyPath))
+ .filter(originalPropertyName -> PropertyUtil.isPropertyPathMatched(originalPropertyName, originalPropertyPath))
.collect(Collectors.toList());
originalPropertyNames.forEach(originalPropertyName -> {
String virtualPropertyName = virtualPropertyPath + originalPropertyName.substring(originalPropertyPath.length());
dataCollectorMap.put(virtualPropertyName, dataCollectorMap.get(originalPropertyName));
});
- if(virtualPropertyPath.equals(VirtualPropertyUtil.VIRTUAL_DEFAULT_LOCATION) &&
- dataCollectorMap.containsKey(VirtualPropertyUtil.VIRTUAL_DEFAULT_LOCATION_WGS84_PATH)) {
- originalGeoShapeProperty = originalPropertyPath + VirtualPropertyUtil.FIELD_WGS84_COORDINATES;
+ if(virtualPropertyPath.equals(PropertyUtil.VIRTUAL_DEFAULT_LOCATION) &&
+ dataCollectorMap.containsKey(PropertyUtil.VIRTUAL_DEFAULT_LOCATION_WGS84_PATH)) {
+ originalGeoShapeProperty = originalPropertyPath + PropertyUtil.FIELD_WGS84_COORDINATES;
}
}
}
@@ -245,7 +242,7 @@ public class StorageIndexerPayloadMapper {
// No VirtualProperties.DefaultLocation.Wgs84Coordinates defined, use the default geo-shape property
if (originalGeoShapeProperty == null)
originalGeoShapeProperty = getDefaultGeoShapeProperty(dataCollectorMap);
- if(originalGeoShapeProperty != null && decimationSetting.isDecimationEnabled()) {
+ if(originalGeoShapeProperty != null) {
try {
decimateGeoShape(originalGeoShapeProperty, dataCollectorMap);
} catch (JsonProcessingException ex) {
@@ -273,30 +270,30 @@ public class StorageIndexerPayloadMapper {
DecimatedResult result = decimator.decimateShapeObj(shapeObj);
if(result.isDecimated()) {
dataCollectorMap.put(originalGeoShapeProperty, result.getDecimatedShapeObj());
- if(dataCollectorMap.containsKey(VirtualPropertyUtil.VIRTUAL_DEFAULT_LOCATION_WGS84_PATH)) {
- dataCollectorMap.put(VirtualPropertyUtil.VIRTUAL_DEFAULT_LOCATION_WGS84_PATH, result.getDecimatedShapeObj());
+ if(dataCollectorMap.containsKey(PropertyUtil.VIRTUAL_DEFAULT_LOCATION_WGS84_PATH)) {
+ dataCollectorMap.put(PropertyUtil.VIRTUAL_DEFAULT_LOCATION_WGS84_PATH, result.getDecimatedShapeObj());
}
}
- if(dataCollectorMap.containsKey(VirtualPropertyUtil.VIRTUAL_DEFAULT_LOCATION_WGS84_PATH)) {
- dataCollectorMap.put(VirtualPropertyUtil.VIRTUAL_DEFAULT_LOCATION_IS_DECIMATED_PATH, result.isDecimated());
+ if(dataCollectorMap.containsKey(PropertyUtil.VIRTUAL_DEFAULT_LOCATION_WGS84_PATH)) {
+ dataCollectorMap.put(PropertyUtil.VIRTUAL_DEFAULT_LOCATION_IS_DECIMATED_PATH, result.isDecimated());
}
}
private Priority chooseOriginalProperty(String virtualPropertyPath, List<Priority> priorities, Map<String, Object> dataCollectorMap) {
- if (VirtualPropertyUtil.VIRTUAL_DEFAULT_LOCATION.equals(virtualPropertyPath) || VirtualPropertyUtil.DATA_VIRTUAL_DEFAULT_LOCATION.equals(virtualPropertyPath)) {
+ if (PropertyUtil.VIRTUAL_DEFAULT_LOCATION.equals(virtualPropertyPath) || PropertyUtil.DATA_VIRTUAL_DEFAULT_LOCATION.equals(virtualPropertyPath)) {
// Specially handle "data.VirtualProperties.DefaultLocation" -- check the value of the field "wgs84Coordinates"
for (Priority priority : priorities) {
- String originalPropertyPath = VirtualPropertyUtil.removeDataPrefix(priority.getPath());
- String wgs84PropertyField = originalPropertyPath + VirtualPropertyUtil.FIELD_WGS84_COORDINATES;
+ String originalPropertyPath = PropertyUtil.removeDataPrefix(priority.getPath());
+ String wgs84PropertyField = originalPropertyPath + PropertyUtil.FIELD_WGS84_COORDINATES;
if (dataCollectorMap.containsKey(wgs84PropertyField) && dataCollectorMap.get(wgs84PropertyField) != null)
return priority;
}
}
for (Priority priority : priorities) {
- String originalPropertyPath = VirtualPropertyUtil.removeDataPrefix(priority.getPath());
+ String originalPropertyPath = PropertyUtil.removeDataPrefix(priority.getPath());
List<String> originalPropertyNames = dataCollectorMap.keySet().stream()
- .filter(name -> VirtualPropertyUtil.isPropertyPathMatched(name, originalPropertyPath))
+ .filter(name -> PropertyUtil.isPropertyPathMatched(name, originalPropertyPath))
.collect(Collectors.toList());
for (String originalPropertyName : originalPropertyNames) {
if (dataCollectorMap.containsKey(originalPropertyName) && dataCollectorMap.get(originalPropertyName) != null)
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageService.java
index 54a19e807079d6076b855e89ea2707f88308e038..d2833e9748c3ad8dcb98a0efb913d01ca68c090b 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageService.java
@@ -28,9 +28,11 @@ public interface StorageService {
Records getStorageRecords(List<String> ids, List<RecordInfo> recordChangedInfos) throws AppException, URISyntaxException;
+ Records getStorageRecords(List<String> ids) throws URISyntaxException;
+
RecordQueryResponse getRecordsByKind(RecordReindexRequest request) throws URISyntaxException;
String getStorageSchema(String kind) throws URISyntaxException, UnsupportedEncodingException;
List<String> getAllKinds() throws URISyntaxException;
-}
\ No newline at end of file
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java
index 1b20cfb14cd9128dcb1595d9a608d6c4ed1da4b1..cfcf8ce96187e5918a133e31800ea6731f609263 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/StorageServiceImpl.java
@@ -24,7 +24,11 @@ import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import org.apache.http.HttpStatus;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.entity.StringEntity;
import org.opengroup.osdu.core.common.http.FetchServiceHttpRequest;
+import org.opengroup.osdu.core.common.http.IHttpClientHandler;
import org.opengroup.osdu.core.common.http.IUrlFetchService;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
@@ -50,11 +54,7 @@ import java.io.UnsupportedEncodingException;
import java.net.URISyntaxException;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import java.util.stream.Collectors;
import static org.opengroup.osdu.core.common.Constants.SLB_FRAME_OF_REFERENCE_VALUE;
@@ -70,6 +70,8 @@ public class StorageServiceImpl implements StorageService {
@Inject
private IUrlFetchService urlFetchService;
@Inject
+ private IHttpClientHandler httpClientHandler;
+ @Inject
private JobStatus jobStatus;
@Inject
private IRequestInfo requestInfo;
@@ -98,6 +100,24 @@ public class StorageServiceImpl implements StorageService {
return Records.builder().records(valid).notFound(notFound).conversionStatuses(conversionStatuses).missingRetryRecords(missingRetryRecordIds).build();
}
+ @Override
+ public Records getStorageRecords(List<String> ids) throws URISyntaxException {
+ List<Records.Entity> valid = new ArrayList<>();
+ List<String> notFound = new ArrayList<>();
+ List<ConversionStatus> conversionStatuses = new ArrayList<>();
+ List<String> missingRetryRecordIds = new ArrayList<>();
+
+ List<List<String>> batch = Lists.partition(ids, configurationProperties.getStorageRecordsBatchSize());
+ for (List<String> recordsBatch : batch) {
+ Records storageOut = this.getRecords(recordsBatch);
+ valid.addAll(storageOut.getRecords());
+ notFound.addAll(storageOut.getNotFound());
+ conversionStatuses.addAll(storageOut.getConversionStatuses());
+ missingRetryRecordIds.addAll(storageOut.getMissingRetryRecords());
+ }
+ return Records.builder().records(valid).notFound(notFound).conversionStatuses(conversionStatuses).missingRetryRecords(missingRetryRecordIds).build();
+ }
+
protected Records getRecords(List<String> ids, Map<String, String> recordChangedMap, Map<String, String> validRecordKindPatchMap) throws URISyntaxException {
// e.g. {"records":["test:10"]}
String body = this.gson.toJson(RecordIds.builder().records(ids).build());
@@ -114,6 +134,27 @@ public class StorageServiceImpl implements StorageService {
return this.validateStorageResponse(response, ids, recordChangedMap, validRecordKindPatchMap);
}
+ protected Records getRecords(List<String> ids) throws URISyntaxException {
+ String body = this.gson.toJson(RecordIds.builder().records(ids).build());
+ DpsHeaders headers = this.requestInfo.getHeaders();
+ URIBuilder builder = new URIBuilder(configurationProperties.getStorageQueryRecordHost());
+ HttpPost request = new HttpPost(builder.build());
+ request.setEntity(new StringEntity(body, StandardCharsets.UTF_8));
+ // we do not need retry on storage based on not found record
+ HttpResponse response = httpClientHandler.sendRequest(request, headers);
+ if (response.getResponseCode() > 299) {
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Internal Error", response.getBody());
+ }
+ try {
+ Records records = this.objectMapper.readValue(response.getBody(), Records.class);
+ ids.removeAll(records.getRecords().stream().map(Records.Entity::getId).collect(Collectors.toList()));
+ records.setNotFound(ids);
+ return records;
+ } catch (JsonProcessingException e) {
+ throw new AppException(RequestStatus.INVALID_RECORD, "Invalid request", "Successful Storage service response with wrong json", e);
+ }
+ }
+
private Records validateStorageResponse(HttpResponse response, List<String> ids, Map<String, String> recordChangedMap, Map<String, String> validRecordKindPatchMap) {
String bulkStorageData = response.getBody();
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/AugmenterSetting.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/AugmenterSetting.java
new file mode 100644
index 0000000000000000000000000000000000000000..d11d41cc38f5b3c7aaf1cd63cfbafd38ca4250eb
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/AugmenterSetting.java
@@ -0,0 +1,16 @@
+package org.opengroup.osdu.indexer.util;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class AugmenterSetting {
+ private static final String PROPERTY_NAME = "index-augmenter-enabled";
+
+ @Autowired
+ private BooleanFeatureFlagClient booleanFeatureFlagClient;
+
+ public boolean isEnabled() {
+ return booleanFeatureFlagClient.isEnabled(PROPERTY_NAME, false);
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/geo/decimator/GeoShapeDecimationSetting.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/BooleanFeatureFlagClient.java
similarity index 54%
rename from indexer-core/src/main/java/org/opengroup/osdu/indexer/util/geo/decimator/GeoShapeDecimationSetting.java
rename to indexer-core/src/main/java/org/opengroup/osdu/indexer/util/BooleanFeatureFlagClient.java
index a062c1ac6e645e3e5bdcec4d9c066e0d0230d199..f21e0968d976de22c1f2f35daf4e33b17cbe7406 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/geo/decimator/GeoShapeDecimationSetting.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/BooleanFeatureFlagClient.java
@@ -1,37 +1,21 @@
-/*
- * Copyright © Schlumberger
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0 *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+package org.opengroup.osdu.indexer.util;
-package org.opengroup.osdu.indexer.util.geo.decimator;
-
-import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.partition.*;
import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient;
+import org.opengroup.osdu.indexer.cache.FeatureFlagCache;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
@Component
-public class GeoShapeDecimationSetting {
- private static final String PROPERTY_NAME = "indexer-decimation-enabled";
+public class BooleanFeatureFlagClient {
+ private static final String TOKEN_PREFIX = "Bearer ";
@Lazy
@Autowired
- private DecimationSettingCache cache;
+ private FeatureFlagCache cache;
@Autowired
private JaxRsDpsLog logger;
@@ -45,24 +29,24 @@ public class GeoShapeDecimationSetting {
@Autowired
private IServiceAccountJwtClient tokenService;
- public boolean isDecimationEnabled() {
+ public boolean isEnabled(String featureName, boolean defaultValue) {
String dataPartitionId = headers.getPartitionId();
- String cacheKey = String.format("%s-%s", dataPartitionId, PROPERTY_NAME);
+ String cacheKey = String.format("%s-%s", dataPartitionId, featureName);
if (cache != null && cache.containsKey(cacheKey))
return cache.get(cacheKey);
- boolean decimationEnabled = true;
+ boolean isEnabled = defaultValue;
try {
PartitionInfo partitionInfo = getPartitionInfo(dataPartitionId);
- decimationEnabled = getDecimationSetting(partitionInfo);
+ isEnabled = getFeatureValue(partitionInfo, featureName, defaultValue);
} catch (Exception e) {
- this.logger.error(String.format("PartitionService: Error getting %s for dataPartition with Id: %s. Turn on the feature flag by default.", PROPERTY_NAME, dataPartitionId), e);
+ this.logger.error(String.format("PartitionService: Error getting %s for dataPartition with Id: %s. Turn on the feature flag by default.", featureName, dataPartitionId), e);
}
- this.cache.put(cacheKey, decimationEnabled);
- return decimationEnabled;
+ this.cache.put(cacheKey, isEnabled);
+ return isEnabled;
}
- private PartitionInfo getPartitionInfo(String dataPartitionId) {
+ private PartitionInfo getPartitionInfo(String dataPartitionId) throws PartitionException {
try {
DpsHeaders partitionHeaders = DpsHeaders.createFromMap(headers.getHeaders());
partitionHeaders.put(DpsHeaders.AUTHORIZATION, this.tokenService.getIdToken(dataPartitionId));
@@ -71,18 +55,19 @@ public class GeoShapeDecimationSetting {
PartitionInfo partitionInfo = partitionProvider.get(dataPartitionId);
return partitionInfo;
} catch (PartitionException e) {
- throw new AppException(HttpStatus.SC_FORBIDDEN, "Service unavailable", String.format("Error getting partition info for data-partition: %s", dataPartitionId), e);
+ logger.error(String.format("Error getting partition info for data-partition: %s", dataPartitionId), e);
+ throw e;
}
}
- private boolean getDecimationSetting(PartitionInfo partitionInfo) {
+ private boolean getFeatureValue(PartitionInfo partitionInfo, String featureName, boolean defaultValue) {
if(partitionInfo == null || partitionInfo.getProperties() == null)
- return true;
+ return defaultValue;
- if(partitionInfo.getProperties().containsKey(PROPERTY_NAME)) {
- Property property = partitionInfo.getProperties().get(PROPERTY_NAME);
+ if(partitionInfo.getProperties().containsKey(featureName)) {
+ Property property = partitionInfo.getProperties().get(featureName);
return Boolean.parseBoolean((String)property.getValue());
}
- return true;
+ return defaultValue;
}
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/PropertyUtil.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/PropertyUtil.java
new file mode 100644
index 0000000000000000000000000000000000000000..49a7175ee43287395f3bfc28fe1a3f2d2acba903
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/PropertyUtil.java
@@ -0,0 +1,261 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.util;
+
+import com.google.api.client.util.Strings;
+import com.google.common.collect.MapDifference;
+import com.google.common.collect.Maps;
+import org.opengroup.osdu.indexer.model.Kind;
+
+import java.util.*;
+
+public class PropertyUtil {
+ public static final String DATA_VIRTUAL_DEFAULT_LOCATION = "data.VirtualProperties.DefaultLocation";
+ public static final String VIRTUAL_DEFAULT_LOCATION = "VirtualProperties.DefaultLocation";
+ public static final String FIELD_WGS84_COORDINATES = ".Wgs84Coordinates";
+ public static final String VIRTUAL_DEFAULT_LOCATION_WGS84_PATH = VIRTUAL_DEFAULT_LOCATION + FIELD_WGS84_COORDINATES;
+ public static final String VIRTUAL_DEFAULT_LOCATION_IS_DECIMATED_PATH = VIRTUAL_DEFAULT_LOCATION + ".IsDecimated";
+
+
+ private static final String PROPERTY_DELIMITER = ".";
+ private static final String NESTED_OBJECT_DELIMITER = "[].";
+ private static final String ARRAY_SYMBOL = "[]";
+ private static final String DATA_PREFIX = "data" + PROPERTY_DELIMITER;
+
+ public static boolean isPropertyPathMatched(String propertyPath, String parentPropertyPath) {
+ // We should not just use propertyPath.startsWith(parentPropertyPath)
+ // For example, if parentPropertyPath is "data.FacilityName" and propertyPath.startsWith(parentPropertyPath) is used,
+ // then the property "data.FacilityNameAlias" will be matched and unexpected result will be returned.
+ return !Strings.isNullOrEmpty(propertyPath) && (propertyPath.startsWith(parentPropertyPath + PROPERTY_DELIMITER) || propertyPath.equals(parentPropertyPath));
+ }
+
+ public static boolean hasSameMajorKind(String left, String right) {
+ try {
+ Kind leftKind = new Kind(left);
+ Kind rightKind = new Kind(right);
+
+ String[] leftVersions = leftKind.getVersion().split("\\.");
+ String[] rightVersions = rightKind.getVersion().split("\\.");
+ return leftKind.getAuthority().equals(rightKind.getAuthority()) &&
+ leftKind.getSource().equals(rightKind.getSource()) &&
+ leftKind.getType().equals(rightKind.getType()) &&
+ leftVersions[0].equals(rightVersions[0]);
+ }
+ catch(Exception ex) {
+ // catch exception in case either left kind or right kind is an invalid kind
+ return false;
+ }
+ }
+
+ public static String removeDataPrefix(String path) {
+ if (!Strings.isNullOrEmpty(path) && path.startsWith(DATA_PREFIX))
+ return path.substring(DATA_PREFIX.length());
+ return path;
+ }
+
+ public static String removeIdPostfix(String objectId) {
+ if (objectId != null && objectId.endsWith(":")) {
+ objectId = objectId.substring(0, objectId.length() - 1);
+ }
+ return objectId;
+ }
+
+ public static Map<String, Object> combineObjectMap(Map<String, Object> to, Map<String, Object> from) {
+ if((to == null || to.isEmpty()) && (from == null || from.isEmpty())) {
+ return new HashMap<>();
+ }
+ else if(to == null || to.isEmpty()) {
+ return from;
+ }
+ else if(from == null || from.isEmpty()) {
+ return to;
+ }
+
+ for (Map.Entry<String, Object> entry : from.entrySet()) {
+ if (to.containsKey(entry.getKey())) {
+ Object toObject = to.get(entry.getKey());
+ Object fromObject = entry.getValue();
+ if (toObject instanceof List && fromObject instanceof List) {
+ Set<Object> objectSet = new HashSet<>();
+ objectSet.addAll((List) toObject);
+ objectSet.addAll((List) fromObject);
+ List<Object> propertyValueList = new ArrayList<>(objectSet);
+ Collections.sort(propertyValueList, Comparator.comparing(Object::toString));
+ to.put(entry.getKey(), propertyValueList);
+ }
+ else if(toObject instanceof Map && fromObject instanceof Map) {
+ Object objectMap = combineObjectMap((Map<String, Object>) toObject, (Map<String, Object>) fromObject);
+ to.put(entry.getKey(), objectMap);
+ }
+ else if(!toObject.equals(fromObject)) {
+ if(toObject.getClass().equals(fromObject.getClass())) {
+ List<Object> propertyValueList = new ArrayList<>();
+ propertyValueList.add(toObject);
+ propertyValueList.add(fromObject);
+ Collections.sort(propertyValueList, Comparator.comparing(Object::toString));
+ to.put(entry.getKey(), propertyValueList);
+ }
+ else if(toObject instanceof List || fromObject instanceof List) {
+ List<Object> propertyValueList = toObject instanceof List? (List)toObject : (List)fromObject;
+ Object object = toObject instanceof List? fromObject : toObject;
+ if(!propertyValueList.isEmpty() && propertyValueList.get(0).getClass().equals(object.getClass())) {
+ propertyValueList.add(object);
+ Collections.sort(propertyValueList, Comparator.comparing(Object::toString));
+ to.put(entry.getKey(), propertyValueList);
+ }
+ else if(propertyValueList.isEmpty()) {
+ to.put(entry.getKey(), object);
+ }
+ }
+ }
+ } else {
+ to.put(entry.getKey(), entry.getValue());
+ }
+ }
+
+ return to;
+ }
+
+ public static Map<String, Object> replacePropertyPaths(String newPathPrefix, String valuePath, Map<String, Object> objectMap) {
+ if(Strings.isNullOrEmpty(newPathPrefix) || Strings.isNullOrEmpty(valuePath) || objectMap == null || objectMap.isEmpty()) {
+ return new HashMap<>();
+ }
+
+ newPathPrefix = removeDataPrefix(newPathPrefix);
+ valuePath = removeDataPrefix(valuePath);
+
+ Map<String, Object> values = new HashMap<>();
+ for (Map.Entry<String, Object> entry : objectMap.entrySet()) {
+ String key = entry.getKey();
+ if (key.equals(valuePath) || key.startsWith(valuePath + PROPERTY_DELIMITER)) {
+ key = key.replace(valuePath, newPathPrefix);
+ values.put(key, entry.getValue());
+ }
+ }
+ return values;
+ }
+
+ public static boolean isConcreteKind(String kind) {
+ if(Strings.isNullOrEmpty(kind)) {
+ return false;
+ }
+
+ String[] parts = kind.split(":");
+ if(parts.length != 4)
+ return false;
+ String[] subVersions = parts[3].split("\\.");
+ return (subVersions.length == 3);
+ }
+
+ public static String getKindWithMajor(String kind) {
+ if(Strings.isNullOrEmpty(kind)) {
+ return kind;
+ }
+ String[] parts = kind.split(":");
+ if(parts.length != 4)
+ return "";
+
+ int index = kind.lastIndexOf(":");
+ String[] subVersions = parts[3].split("\\.");
+ String kindWithMajor = kind.substring(0, index) + ":" + subVersions[0] + ".";
+ return kindWithMajor;
+ }
+
+ public static List<String> getChangedProperties(Map<String, Object> leftMap, Map<String, Object> rightMap) {
+ if(leftMap == null && rightMap == null) {
+ return new ArrayList<>();
+ }
+
+ if(leftMap == null) {
+ leftMap = new HashMap<>();
+ }
+ if(rightMap == null) {
+ rightMap = new HashMap<>();
+ }
+ MapDifference<String, Object> difference = Maps.difference(leftMap, rightMap);
+ if(difference.areEqual()) {
+ return new ArrayList<>();
+ }
+
+ Set<String> changedProperties = new HashSet<>();
+ if (difference.entriesOnlyOnLeft().size() > 0) {
+ difference.entriesOnlyOnLeft().forEach((key, value) -> changedProperties.add(key));
+ }
+ if (difference.entriesOnlyOnRight().size() > 0) {
+ difference.entriesOnlyOnRight().forEach((key, value) -> changedProperties.add(key));
+ }
+ if (difference.entriesDiffering().size() > 0) {
+ for(Map.Entry<String, MapDifference.ValueDifference<Object>> entry : difference.entriesDiffering().entrySet()) {
+ try {
+ MapDifference.ValueDifference<Object> valueDifference = entry.getValue();
+ Object left = valueDifference.leftValue();
+ Object right = valueDifference.rightValue();
+ if(left == null && right == null) {
+ continue;
+ }
+ else if(left == null || right == null) {
+ changedProperties.add(entry.getKey());
+ }
+ else if(left instanceof Map) {
+ Map<String, Object> innerLeftMap = (Map<String, Object>)left;
+ Map<String, Object> innerRightMap = (Map<String, Object>)right;
+ List<String> nestedChangedProperties = getChangedProperties(innerLeftMap, innerRightMap);
+ for (String nestedProperty: nestedChangedProperties) {
+ String p = entry.getKey() + PROPERTY_DELIMITER + nestedProperty;
+ changedProperties.add(p);
+ }
+ }
+ else if(left instanceof List) {
+ List<Object> innerLeftList = (List<Object>)left;
+ List<Object> innerRightList = (List<Object>)right;
+ if(innerLeftList.size() != innerRightList.size()) {
+ String p = entry.getKey() + ARRAY_SYMBOL;
+ changedProperties.add(p);
+ }
+ else {
+ for(int i = 0; i < innerLeftList.size(); i++) {
+ Map<String, Object> innerLeftMap = (Map<String, Object>)innerLeftList.get(i);
+ Map<String, Object> innerRightMap = (Map<String, Object>)innerRightList.get(i);
+ List<String> nestedChangedProperties = getChangedProperties(innerLeftMap, innerRightMap);
+ for (String nestedProperty: nestedChangedProperties) {
+ String p = entry.getKey() + NESTED_OBJECT_DELIMITER + nestedProperty;
+ changedProperties.add(p);
+ }
+ }
+ }
+ }
+ else {
+ // Special handle of number. The integer value from the search result could be converted to double
+ if(left instanceof Double || right instanceof Double) {
+ double leftValue = Double.parseDouble(left.toString());
+ double rightValue = Double.parseDouble(right.toString());
+ if(Double.compare(leftValue, rightValue) == 0) {
+ continue; // The left/right values are the same in this case
+ }
+ }
+ changedProperties.add(entry.getKey());
+ }
+ }
+ catch (Exception ex) {
+ // assume there is difference in this case
+ changedProperties.add(entry.getKey());
+ }
+ }
+ }
+
+ return new ArrayList<>(changedProperties);
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/VirtualPropertyUtil.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/VirtualPropertyUtil.java
deleted file mode 100644
index ba9cea6085728ede4305fdacc63a568ab32d63c0..0000000000000000000000000000000000000000
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/VirtualPropertyUtil.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Copyright © Schlumberger
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0 *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.util;
-
-import com.google.api.client.util.Strings;
-
-public class VirtualPropertyUtil {
- public static final String DATA_VIRTUAL_DEFAULT_LOCATION = "data.VirtualProperties.DefaultLocation";
- public static final String VIRTUAL_DEFAULT_LOCATION = "VirtualProperties.DefaultLocation";
- public static final String FIELD_WGS84_COORDINATES = ".Wgs84Coordinates";
- public static final String VIRTUAL_DEFAULT_LOCATION_WGS84_PATH = VIRTUAL_DEFAULT_LOCATION + FIELD_WGS84_COORDINATES;
- public static final String VIRTUAL_DEFAULT_LOCATION_IS_DECIMATED_PATH = VIRTUAL_DEFAULT_LOCATION + ".IsDecimated";
-
-
- private static final String PROPERTY_DELIMITER = ".";
- private static final String DATA_PREFIX = "data" + PROPERTY_DELIMITER;
-
- public static boolean isPropertyPathMatched(String propertyPath, String parentPropertyPath) {
- // We should not just use propertyPath.startsWith(parentPropertyPath)
- // For example, if parentPropertyPath is "data.FacilityName" and propertyPath.startsWith(parentPropertyPath) is used,
- // then the property "data.FacilityNameAlias" will be matched and unexpected result will be returned.
- return !Strings.isNullOrEmpty(propertyPath) && (propertyPath.startsWith(parentPropertyPath + PROPERTY_DELIMITER) || propertyPath.equals(parentPropertyPath));
- }
-
- public static String removeDataPrefix(String path) {
- if (!Strings.isNullOrEmpty(path) && path.startsWith(DATA_PREFIX))
- return path.substring(DATA_PREFIX.length());
- return path;
- }
-}
diff --git a/indexer-core/src/main/resources/swagger.properties b/indexer-core/src/main/resources/swagger.properties
index 12506ebeb2e9aa1af8fdb396baaa543879cbcba0..cdfffe670e8476df7aca32c71f1dea065eea33ac 100644
--- a/indexer-core/src/main/resources/swagger.properties
+++ b/indexer-core/src/main/resources/swagger.properties
@@ -31,6 +31,9 @@ partitionSetupApi.provisionPartition.description=Provision partition. Required r
reindexApi.reindex.summary=Re-index given 'kind'
reindexApi.reindex.description=This API allows users to re-index a 'kind' without re-ingesting the records via storage API. \
Required roles: `service.search.admin`
+reindexApi.reindexRecords.summary=Re-index given records
+reindexApi.reindexRecords.description=This API allows users to re-index the given records by providing record ids without \
+re-ingesting the records via storage API. Required roles: `service.search.admin`
reindexApi.fullReindex.summary=Full Re-index by data partition
reindexApi.fullReindex.description=This API allows users to re-index an entire partition without re-ingesting the records via storage API.\
Required roles: `service.search.admin`
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/PartitionSetupApiTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/PartitionSetupApiTest.java
index 03c6d718f261f8e4d7b9f7fa7120d5a11fff86be..46c7a775dc8c0d2c9a3b05b6b81dafc38850bd7f 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/PartitionSetupApiTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/PartitionSetupApiTest.java
@@ -18,9 +18,11 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.indexer.logging.AuditLogger;
import org.opengroup.osdu.indexer.service.IClusterConfigurationService;
+import org.opengroup.osdu.indexer.service.IndexAliasService;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.test.context.junit4.SpringRunner;
@@ -38,6 +40,10 @@ public class PartitionSetupApiTest {
private AuditLogger auditLogger;
@Mock
private IClusterConfigurationService clusterConfigurationService;
+ @Mock
+ private IndexAliasService indexAliasService;
+ @Mock
+ private JaxRsDpsLog jaxRsDpsLog;
@InjectMocks
private PartitionSetupApi sut;
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/ReindexApiTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/ReindexApiTest.java
index 23513bc5d19a682f3d549bbc18485982d3655e8c..f5437fbd963bcaf622406c2ebde8be6b90374ac8 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/ReindexApiTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/ReindexApiTest.java
@@ -21,7 +21,10 @@ import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
+import org.opengroup.osdu.core.common.model.indexer.Records;
import org.opengroup.osdu.indexer.logging.AuditLogger;
+import org.opengroup.osdu.indexer.model.ReindexRecordsRequest;
+import org.opengroup.osdu.indexer.model.ReindexRecordsResponse;
import org.opengroup.osdu.indexer.service.IndexSchemaService;
import org.opengroup.osdu.indexer.service.ReindexService;
import org.springframework.http.HttpStatus;
@@ -29,14 +32,19 @@ import org.springframework.http.ResponseEntity;
import org.springframework.test.context.junit4.SpringRunner;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.when;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.*;
@RunWith(SpringRunner.class)
public class ReindexApiTest {
private RecordReindexRequest recordReindexRequest;
+ private List<String> recordIds;
@Mock
private ReindexService reIndexService;
@@ -50,11 +58,13 @@ public class ReindexApiTest {
@Before
public void setup() {
recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor("100").build();
+ recordIds = new ArrayList<>();
+ recordIds.add("id1");
}
@Test
public void should_return200_when_valid_kind_provided() throws IOException {
- when(this.reIndexService.reindexRecords(recordReindexRequest, false)).thenReturn("something");
+ when(this.reIndexService.reindexKind(recordReindexRequest, false)).thenReturn("something");
ResponseEntity<?> response = sut.reindex(recordReindexRequest, false);
@@ -63,15 +73,49 @@ public class ReindexApiTest {
@Test(expected = AppException.class)
public void should_throwAppException_ifUnknownExceptionCaught_reindexTest() throws IOException {
- when(this.reIndexService.reindexRecords(recordReindexRequest, false)).thenThrow(new AppException(500, "", ""));
+ when(this.reIndexService.reindexKind(recordReindexRequest, false)).thenThrow(new AppException(500, "", ""));
sut.reindex(recordReindexRequest, false);
}
@Test(expected = NullPointerException.class)
public void should_throwAppException_ifNullPointerExceptionCaught_ReindexTest() throws IOException {
- when(this.reIndexService.reindexRecords(recordReindexRequest, false)).thenThrow(new NullPointerException(""));
+ when(this.reIndexService.reindexKind(recordReindexRequest, false)).thenThrow(new NullPointerException(""));
sut.reindex(recordReindexRequest, false);
}
+
+ @Test
+ public void should_return200_when_valid_record_id_list_provided() {
+ when(this.reIndexService.reindexRecords(recordIds)).thenReturn(Records.builder().records(new ArrayList<>()).records(Collections.singletonList(Records.Entity.builder().id("id1").build())).notFound(recordIds).build());
+
+ ResponseEntity<?> response = sut.reindexRecords(new ReindexRecordsRequest(recordIds));
+
+ assertEquals(HttpStatus.ACCEPTED, response.getStatusCode());
+ verify(auditLogger).getReindexRecords(any());
+ }
+
+ @Test
+ public void should_notWriteAuditLog_when_no_valid_record_id_list_provided() {
+ when(this.reIndexService.reindexRecords(recordIds)).thenReturn(Records.builder().records(new ArrayList<>()).records(Collections.emptyList()).notFound(recordIds).build());
+
+ ResponseEntity<?> response = sut.reindexRecords(new ReindexRecordsRequest(recordIds));
+
+ assertEquals(HttpStatus.ACCEPTED, response.getStatusCode());
+ verify(auditLogger, never()).getReindex(any());
+ }
+
+ @Test(expected = AppException.class)
+ public void should_throwAppException_ifUnknownExceptionCaught_reindexRecordsTest() {
+ when(this.reIndexService.reindexRecords(recordIds)).thenThrow(new AppException(500, "", ""));
+
+ sut.reindexRecords(new ReindexRecordsRequest(recordIds));
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void should_throwAppException_ifNullPointerExceptionCaught_ReindexRecordsTest() {
+ when(this.reIndexService.reindexRecords(recordIds)).thenThrow(new NullPointerException(""));
+
+ sut.reindexRecords(new ReindexRecordsRequest(recordIds));
+ }
}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/geo/decimator/DecimationSettingCacheTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/cache/FeatureFlagCacheTest.java
similarity index 86%
rename from indexer-core/src/test/java/org/opengroup/osdu/indexer/util/geo/decimator/DecimationSettingCacheTest.java
rename to indexer-core/src/test/java/org/opengroup/osdu/indexer/cache/FeatureFlagCacheTest.java
index e850126fe2880d457bd3fcd15ede9e0953b1d779..b440c049d04c7575df51732a1c8f83aad8fe5f1a 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/geo/decimator/DecimationSettingCacheTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/cache/FeatureFlagCacheTest.java
@@ -13,23 +13,24 @@
* limitations under the License.
*/
-package org.opengroup.osdu.indexer.util.geo.decimator;
+package org.opengroup.osdu.indexer.cache;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
+import org.opengroup.osdu.indexer.cache.FeatureFlagCache;
import org.springframework.test.context.junit4.SpringRunner;
@RunWith(SpringRunner.class)
-public class DecimationSettingCacheTest {
+public class FeatureFlagCacheTest {
private static final String VALID_KEY = "Tenant1-indexer-decimation-enabled";
private static final String INVALID_KEY = "Tenant2-indexer-decimation-enabled";
- DecimationSettingCache cache;
+ FeatureFlagCache cache;
@Before
public void setup() {
- cache = new DecimationSettingCache();
+ cache = new FeatureFlagCache();
cache.put(VALID_KEY, true);
}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/indexproperty/RelatedObjectsSpecTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/indexproperty/RelatedObjectsSpecTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..63a53c37d41db25c2ae8284fe6947016ad40906c
--- /dev/null
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/indexproperty/RelatedObjectsSpecTest.java
@@ -0,0 +1,121 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model.indexproperty;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+@RunWith(SpringRunner.class)
+public class RelatedObjectsSpecTest {
+
+ @Test
+ public void hasValidCondition_return_true() {
+ RelatedObjectsSpec spec = new RelatedObjectsSpec();
+ spec.setRelatedObjectKind("osdu:wks:master-data--GeoPoliticalEntity:1.");
+ spec.setRelationshipDirection(RelatedObjectsSpec.CHILD_TO_PARENT);
+ spec.setRelatedObjectID("data.GeoContexts[].GeoPoliticalEntityID");
+ spec.setRelatedConditionProperty("data.GeoContexts[].GeoTypeID");
+ List<String> matches = Arrays.asList("opendes:reference-data--GeoPoliticalEntityType:Country:");
+ spec.setRelatedConditionMatches(matches);
+
+ Assert.assertTrue(spec.hasValidCondition());
+ }
+
+ @Test
+ public void hasValidCondition_return_true_with_multi_nested() {
+ RelatedObjectsSpec spec = new RelatedObjectsSpec();
+ spec.setRelatedObjectKind("osdu:wks:master-data--Organisation:1.");
+ spec.setRelationshipDirection(RelatedObjectsSpec.CHILD_TO_PARENT);
+ spec.setRelatedObjectID("data.TechnicalAssurances[].Reviewers[].OrganisationID");
+ spec.setRelatedConditionProperty("data.TechnicalAssurances[].Reviewers[].RoleTypeID");
+ List<String> matches = Arrays.asList("opendes:reference-data--ContactRoleType:AccountOwner:");
+ spec.setRelatedConditionMatches(matches);
+
+ Assert.assertTrue(spec.hasValidCondition());
+ }
+
+ @Test
+ public void hasValidCondition_return_false_with_unmatched_multi_nested() {
+ RelatedObjectsSpec spec = new RelatedObjectsSpec();
+ spec.setRelatedObjectKind("osdu:wks:master-data--Organisation:1.");
+ spec.setRelatedObjectID("data.TechnicalAssurances[].Reviewers[].OrganisationID");
+ spec.setRelatedConditionProperty("data.TechnicalAssurances[].Auditors[].RoleTypeID");
+ List<String> matches = Arrays.asList("opendes:reference-data--ContactRoleType:AccountOwner:");
+ spec.setRelatedConditionMatches(matches);
+
+ Assert.assertFalse(spec.hasValidCondition());
+ }
+
+ @Test
+ public void hasValidCondition_return_false_with_null_propertyPath() {
+ RelatedObjectsSpec spec = new RelatedObjectsSpec();
+ spec.setRelatedObjectKind("osdu:wks:master-data--GeoPoliticalEntity:1.");
+ spec.setRelatedObjectID("data.GeoContexts[].GeoPoliticalEntityID");
+ Assert.assertFalse(spec.hasValidCondition());
+ }
+
+ @Test
+ public void hasValidCondition_return_false_with_empty_matches() {
+ RelatedObjectsSpec spec = new RelatedObjectsSpec();
+ spec.setRelatedObjectKind("osdu:wks:master-data--GeoPoliticalEntity:1.");
+ spec.setRelatedObjectID("data.GeoContexts[].GeoPoliticalEntityID");
+ spec.setRelatedConditionProperty("data.GeoContexts[].GeoTypeID");
+ spec.setRelatedConditionMatches(new ArrayList<>());
+
+ Assert.assertFalse(spec.hasValidCondition());
+ }
+
+ @Test
+ public void hasValidCondition_return_false_with_unmatched_propertyPath() {
+ RelatedObjectsSpec spec = new RelatedObjectsSpec();
+ spec.setRelatedObjectKind("osdu:wks:master-data--GeoPoliticalEntity:1.");
+ spec.setRelatedObjectID("data.GeoContexts[].GeoPoliticalEntityID");
+ spec.setRelatedConditionProperty("data.VerticalMeasurementID[].VerticalMeasurementID");
+ List<String> matches = Arrays.asList("KB");
+ spec.setRelatedConditionMatches(matches);
+
+ Assert.assertFalse(spec.hasValidCondition());
+ }
+
+ @Test
+ public void hasValidCondition_return_false_without_nested_property() {
+ RelatedObjectsSpec spec = new RelatedObjectsSpec();
+ spec.setRelatedObjectKind("osdu:wks:master-data--GeoPoliticalEntity:1.");
+ spec.setRelatedObjectID("data.GeoContexts[]");
+ spec.setRelatedConditionProperty("data.GeoContexts[]");
+ List<String> matches = Arrays.asList("opendes:reference-data--GeoPoliticalEntityType:Country:");
+ spec.setRelatedConditionMatches(matches);
+ Assert.assertFalse(spec.hasValidCondition());
+ }
+
+ @Test
+ public void hasValidCondition_return_false_for_none_nested_property() {
+ RelatedObjectsSpec spec = new RelatedObjectsSpec();
+ spec.setRelatedObjectKind("osdu:wks:master-data--GeoPoliticalEntity:1.");
+ spec.setRelatedObjectID("data.GeoContexts.GeoPoliticalEntityID");
+ spec.setRelatedConditionProperty("data.GeoContexts.GeoTypeID");
+ List<String> matches = Arrays.asList("opendes:reference-data--GeoPoliticalEntityType:Country:");
+ spec.setRelatedConditionMatches(matches);
+ Assert.assertFalse(spec.hasValidCondition());
+ }
+
+}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/indexproperty/ValueExtractionTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/indexproperty/ValueExtractionTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..4c578c9a400250d5c69cb5259d8cd0ee53031281
--- /dev/null
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/indexproperty/ValueExtractionTest.java
@@ -0,0 +1,109 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model.indexproperty;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+@RunWith(SpringRunner.class)
+public class ValueExtractionTest {
+ @Test
+ public void hasValidCondition_return_true() {
+ ValueExtraction valueExtraction = new ValueExtraction();
+ valueExtraction.setValuePath("data.NameAliases[].AliasName");
+ valueExtraction.setRelatedConditionProperty("data.NameAliases[].AliasNameTypeID");
+ List<String> matches = Arrays.asList("opendes:reference-data--AliasNameType:UniqueIdentifier:","reference-data--AliasNameType:RegulatoryName:");
+ valueExtraction.setRelatedConditionMatches(matches);
+
+ Assert.assertTrue(valueExtraction.hasValidCondition());
+ }
+
+ @Test
+ public void hasValidCondition_return_true_with_multi_nested() {
+ ValueExtraction valueExtraction = new ValueExtraction();
+ valueExtraction.setValuePath("data.TechnicalAssurances[].Reviewers[].Name");
+ valueExtraction.setRelatedConditionProperty("data.TechnicalAssurances[].Reviewers[].RoleTypeID");
+ List<String> matches = Arrays.asList("opendes:reference-data--ContactRoleType:AccountOwner:");
+ valueExtraction.setRelatedConditionMatches(matches);
+
+ Assert.assertTrue(valueExtraction.hasValidCondition());
+ }
+
+ @Test
+ public void hasValidCondition_return_false_with_unmatched_multi_nested() {
+ ValueExtraction valueExtraction = new ValueExtraction();
+ valueExtraction.setValuePath("data.TechnicalAssurances[].Reviewers[].Name");
+ valueExtraction.setRelatedConditionProperty("data.TechnicalAssurances[].Auditors[].RoleTypeID");
+ List<String> matches = Arrays.asList("opendes:reference-data--ContactRoleType:AccountOwner:");
+ valueExtraction.setRelatedConditionMatches(matches);
+
+ Assert.assertFalse(valueExtraction.hasValidCondition());
+ }
+
+ @Test
+ public void hasValidCondition_return_false_with_null_propertyPath() {
+ ValueExtraction valueExtraction = new ValueExtraction();
+ valueExtraction.setValuePath("data.NameAliases[].AliasName");
+ Assert.assertFalse(valueExtraction.hasValidCondition());
+ }
+
+ @Test
+ public void hasValidCondition_return_false_with_empty_matches() {
+ ValueExtraction valueExtraction = new ValueExtraction();
+ valueExtraction.setValuePath("data.NameAliases[].AliasName");
+ valueExtraction.setRelatedConditionProperty("data.NameAliases[].AliasNameTypeID");
+ valueExtraction.setRelatedConditionMatches(new ArrayList<>());
+
+ Assert.assertFalse(valueExtraction.hasValidCondition());
+ }
+
+ @Test
+ public void hasValidCondition_return_false_with_unmatched_propertyPath() {
+ ValueExtraction valueExtraction = new ValueExtraction();
+ valueExtraction.setValuePath("data.NameAliases[].AliasName");
+ valueExtraction.setRelatedConditionProperty("data.VerticalMeasurementID[].VerticalMeasurementID");
+ List<String> matches = Arrays.asList("KB");
+ valueExtraction.setRelatedConditionMatches(matches);
+
+ Assert.assertFalse(valueExtraction.hasValidCondition());
+ }
+
+ @Test
+ public void hasValidCondition_return_false_without_nested_property() {
+ ValueExtraction valueExtraction = new ValueExtraction();
+ valueExtraction.setValuePath("data.NameAliases[]");
+ valueExtraction.setRelatedConditionProperty("data.NameAliases[]");
+ List<String> matches = Arrays.asList("opendes:reference-data--AliasNameType:UniqueIdentifier:","reference-data--AliasNameType:RegulatoryName:");
+ valueExtraction.setRelatedConditionMatches(matches);
+ Assert.assertFalse(valueExtraction.hasValidCondition());
+ }
+
+ @Test
+ public void hasValidCondition_return_false_for_none_nested_property() {
+ ValueExtraction valueExtraction = new ValueExtraction();
+ valueExtraction.setValuePath("data.NameAliases.AliasName");
+ valueExtraction.setRelatedConditionProperty("data.NameAliases.AliasNameTypeID");
+ List<String> matches = Arrays.asList("opendes:reference-data--AliasNameType:UniqueIdentifier:","reference-data--AliasNameType:RegulatoryName:");
+ valueExtraction.setRelatedConditionMatches(matches);
+ Assert.assertFalse(valueExtraction.hasValidCondition());
+ }
+}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/indexproperty/jackson/PropertyPathDeserializerTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/indexproperty/jackson/PropertyPathDeserializerTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..d149703ab706598c8667ac354d9e6598fa7148a9
--- /dev/null
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/model/indexproperty/jackson/PropertyPathDeserializerTest.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.model.indexproperty.jackson;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.SneakyThrows;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.opengroup.osdu.indexer.model.indexproperty.PropertyConfiguration;
+import org.opengroup.osdu.indexer.model.indexproperty.PropertyConfigurations;
+import org.opengroup.osdu.indexer.model.indexproperty.PropertyPath;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+
+@RunWith(SpringRunner.class)
+public class PropertyPathDeserializerTest {
+ @Test
+ public void deserialize_configurations_test() throws JsonProcessingException {
+ String jsonText = getJsonFromFile("well_configuration_record.json");
+ ObjectMapper objectMapper = new ObjectMapper();
+ PropertyConfigurations configurations = objectMapper.readValue(jsonText, PropertyConfigurations.class);
+ Assert.assertNotNull(configurations);
+ Assert.assertEquals(2, configurations.getConfigurations().size());
+ PropertyConfiguration countryNameConfiguration = configurations.getConfigurations().get(0);
+ PropertyConfiguration wellUWIConfiguration = configurations.getConfigurations().get(1);
+
+ Assert.assertEquals("CountryNames", countryNameConfiguration.getName());
+ Assert.assertEquals(1, countryNameConfiguration.getPaths().size());
+ PropertyPath path1 = countryNameConfiguration.getPaths().get(0);
+ Assert.assertTrue(path1.hasValidRelatedObjectsSpec());
+ Assert.assertTrue(path1.getRelatedObjectsSpec().hasValidCondition());
+ Assert.assertEquals(1, path1.getRelatedObjectsSpec().getRelatedConditionMatches().size());
+ Assert.assertTrue(path1.hasValidValueExtraction());
+ Assert.assertFalse(path1.getValueExtraction().hasValidCondition());
+
+ Assert.assertEquals("WellUWI", wellUWIConfiguration.getName());
+ Assert.assertEquals(1, wellUWIConfiguration.getPaths().size());
+ PropertyPath path2 = wellUWIConfiguration.getPaths().get(0);
+ Assert.assertFalse(path2.hasValidRelatedObjectsSpec());
+ Assert.assertTrue(path2.hasValidValueExtraction());
+ Assert.assertTrue(path2.getValueExtraction().hasValidCondition());
+ Assert.assertEquals(5, path2.getValueExtraction().getRelatedConditionMatches().size());
+
+ }
+
+ @SneakyThrows
+ private String getJsonFromFile(String file) {
+ InputStream inStream = this.getClass().getResourceAsStream("/indexproperty/" + file);
+ BufferedReader br = new BufferedReader(new InputStreamReader(inStream));
+ StringBuilder stringBuilder = new StringBuilder();
+ String sCurrentLine;
+ while ((sCurrentLine = br.readLine()) != null)
+ {
+ stringBuilder.append(sCurrentLine).append("\n");
+ }
+ return stringBuilder.toString();
+ }
+
+}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexAliasServiceImplTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexAliasServiceImplTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..df1e5d4397aa8209fafe272747c1fe9f5b3972f9
--- /dev/null
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexAliasServiceImplTest.java
@@ -0,0 +1,191 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.service;
+
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.GetAliasesResponse;
+import org.elasticsearch.client.IndicesClient;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.cluster.metadata.AliasMetadata;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
+import org.opengroup.osdu.indexer.model.IndexAliasesResult;
+import org.opengroup.osdu.indexer.service.mock.BucketMock;
+import org.opengroup.osdu.indexer.service.mock.TermMock;
+import org.opengroup.osdu.indexer.util.ElasticClientHandler;
+import org.powermock.api.mockito.PowerMockito;
+import org.springframework.context.annotation.Lazy;
+
+import java.io.IOException;
+import java.util.*;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.when;
+import static org.mockito.MockitoAnnotations.initMocks;
+
+@RunWith(MockitoJUnitRunner.class)
+public class IndexAliasServiceImplTest {
+ @Mock
+ private ElasticClientHandler elasticClientHandler;
+ @Mock
+ private ElasticIndexNameResolver elasticIndexNameResolver;
+ @Mock
+ @Lazy
+ private JaxRsDpsLog log;
+ @InjectMocks
+ private IndexAliasServiceImpl sut;
+
+ private RestHighLevelClient restHighLevelClient;
+ private IndicesClient indicesClient;
+ private GetAliasesResponse getAliasesResponse, getAliasesNotFoundResponse;
+
+
+ private static String kind = "common:welldb:wellbore:1.2.0";
+ private static String index = "common-welldb-wellbore-1.2.0";
+ private static String alias = "a1234567890";
+
+ @Before
+ public void setup() {
+ initMocks(this);
+ indicesClient = PowerMockito.mock(IndicesClient.class);
+ restHighLevelClient = PowerMockito.mock(RestHighLevelClient.class);
+ getAliasesResponse = PowerMockito.mock(GetAliasesResponse.class);
+ getAliasesNotFoundResponse = PowerMockito.mock(GetAliasesResponse.class);
+
+ }
+
+ @Test
+ public void createIndexAlias_test_when_index_name_is_not_alias() throws IOException {
+ AcknowledgedResponse updateAliasesResponse = new AcknowledgedResponse(true);
+ when(elasticIndexNameResolver.getIndexNameFromKind(any())).thenReturn(index);
+ when(elasticIndexNameResolver.getIndexAliasFromKind(any())).thenReturn(alias);
+ when(elasticIndexNameResolver.isIndexAliasSupported(any())).thenReturn(true);
+ when(restHighLevelClient.indices()).thenReturn(indicesClient);
+ when(indicesClient.getAlias(any(GetAliasesRequest.class), any(RequestOptions.class))).thenReturn(getAliasesNotFoundResponse);
+ when(getAliasesNotFoundResponse.status()).thenReturn(RestStatus.NOT_FOUND);
+ when(indicesClient.updateAliases(any(IndicesAliasesRequest.class), any(RequestOptions.class))).thenReturn(updateAliasesResponse);
+
+ boolean ok = sut.createIndexAlias(restHighLevelClient, kind);
+ Assert.assertTrue(ok);
+ }
+
+ @Test
+ public void createIndexAlias_test_when_index_name_is_alias() throws IOException {
+ Map<String, Set<AliasMetadata>> aliases = new HashMap<>();
+ Set<AliasMetadata> aliasMetadataSet = new HashSet<>();
+ aliasMetadataSet.add(AliasMetadata.builder(index).build());
+ aliases.put(index + "_123456789", aliasMetadataSet);
+
+ AcknowledgedResponse updateAliasesResponse = new AcknowledgedResponse(true);
+ when(elasticIndexNameResolver.getIndexNameFromKind(any())).thenReturn(index);
+ when(elasticIndexNameResolver.getIndexAliasFromKind(any())).thenReturn(alias);
+ when(elasticIndexNameResolver.isIndexAliasSupported(any())).thenReturn(true);
+ when(restHighLevelClient.indices()).thenReturn(indicesClient);
+ when(indicesClient.getAlias(any(GetAliasesRequest.class), any(RequestOptions.class))).thenReturn(getAliasesResponse);
+ when(getAliasesResponse.status()).thenReturn(RestStatus.OK);
+ when(getAliasesResponse.getAliases()).thenReturn(aliases);
+ when(indicesClient.updateAliases(any(IndicesAliasesRequest.class), any(RequestOptions.class))).thenReturn(updateAliasesResponse);
+
+ boolean ok = sut.createIndexAlias(restHighLevelClient, kind);
+ Assert.assertTrue(ok);
+ }
+
+ @Test
+ public void createIndexAlias_test_when_updateAliases_fails() throws IOException {
+ AcknowledgedResponse updateAliasesResponse = new AcknowledgedResponse(false);
+ when(elasticIndexNameResolver.getIndexNameFromKind(any())).thenReturn(index);
+ when(elasticIndexNameResolver.getIndexAliasFromKind(any())).thenReturn(alias);
+ when(elasticIndexNameResolver.isIndexAliasSupported(any())).thenReturn(true);
+ when(restHighLevelClient.indices()).thenReturn(indicesClient);
+ when(indicesClient.getAlias(any(GetAliasesRequest.class), any(RequestOptions.class))).thenReturn(getAliasesNotFoundResponse);
+ when(getAliasesNotFoundResponse.status()).thenReturn(RestStatus.NOT_FOUND);
+ when(indicesClient.updateAliases(any(IndicesAliasesRequest.class), any(RequestOptions.class))).thenReturn(updateAliasesResponse);
+
+ boolean ok = sut.createIndexAlias(restHighLevelClient, kind);
+ Assert.assertFalse(ok);
+ }
+
+ @Test
+ public void createIndexAlias_test_when_updateAliases_throws_exception() throws IOException {
+ when(elasticIndexNameResolver.getIndexNameFromKind(any())).thenReturn(index);
+ when(elasticIndexNameResolver.getIndexAliasFromKind(any())).thenReturn(alias);
+ when(elasticIndexNameResolver.isIndexAliasSupported(any())).thenReturn(true);
+ when(restHighLevelClient.indices()).thenReturn(indicesClient);
+ when(indicesClient.getAlias(any(GetAliasesRequest.class), any(RequestOptions.class))).thenReturn(getAliasesNotFoundResponse);
+ when(getAliasesNotFoundResponse.status()).thenReturn(RestStatus.NOT_FOUND);
+ when(indicesClient.updateAliases(any(IndicesAliasesRequest.class), any(RequestOptions.class))).thenThrow(IOException.class);
+
+ boolean ok = sut.createIndexAlias(restHighLevelClient, kind);
+ Assert.assertFalse(ok);
+ }
+
+ @Test
+ public void createIndexAliasesForAll_test() throws IOException {
+ String unsupportedKind = "common:welldb:wellbore:1";
+ String unsupportedIndex = unsupportedKind.replace(":", "-");
+
+ SearchResponse searchResponse = PowerMockito.mock(SearchResponse.class);
+ Aggregations aggregations = PowerMockito.mock(Aggregations.class);
+ TermMock terms = PowerMockito.mock(TermMock.class);
+ BucketMock bucket = PowerMockito.mock(BucketMock.class);
+ BucketMock bucket2 = PowerMockito.mock(BucketMock.class);
+ List<BucketMock> bucketList = Arrays.asList(bucket, bucket, bucket2);
+ AcknowledgedResponse updateAliasesResponse = new AcknowledgedResponse(true);
+ when(elasticIndexNameResolver.getIndexNameFromKind(any()))
+ .thenAnswer(invocation ->{
+ String argument = invocation.getArgument(0);
+ return argument.replace(":", "-");
+ });
+ when(elasticIndexNameResolver.getIndexAliasFromKind(any())).thenReturn(alias);
+ when(elasticIndexNameResolver.isIndexAliasSupported(any()))
+ .thenAnswer(invocation ->{
+ String argument = invocation.getArgument(0);
+ return !unsupportedKind.equals(argument);
+ });
+ when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ when(restHighLevelClient.indices()).thenReturn(indicesClient);
+ when(restHighLevelClient.search(any(SearchRequest.class), any(RequestOptions.class))).thenReturn(searchResponse);
+ when(searchResponse.getAggregations()).thenReturn(aggregations);
+ when(aggregations.get(anyString())).thenReturn(terms);
+ when(terms.getBuckets()).thenReturn(bucketList);
+ when(bucket.getKey()).thenReturn(kind);
+ when(bucket2.getKey()).thenReturn(unsupportedKind);
+ when(indicesClient.getAlias(any(GetAliasesRequest.class), any(RequestOptions.class))).thenReturn(getAliasesNotFoundResponse);
+ when(getAliasesNotFoundResponse.status()).thenReturn(RestStatus.NOT_FOUND);
+ when(indicesClient.updateAliases(any(IndicesAliasesRequest.class), any(RequestOptions.class))).thenReturn(updateAliasesResponse);
+
+ IndexAliasesResult result = sut.createIndexAliasesForAll();
+ Assert.assertEquals(2,result.getIndicesWithAliases().size());
+ Assert.assertEquals(index, result.getIndicesWithAliases().get(0));
+ Assert.assertEquals(1,result.getIndicesWithoutAliases().size());
+ Assert.assertEquals(unsupportedIndex, result.getIndicesWithoutAliases().get(0));
+ }
+}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java
index d1ea8650ed05dac34ead0ad8f88dbc0f57b6de47..4c488fb13c7b1d66e4a7f7113edf003e40b5bc1e 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java
@@ -14,6 +14,9 @@
package org.opengroup.osdu.indexer.service;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
import org.apache.http.HttpStatus;
import org.elasticsearch.client.RestHighLevelClient;
import org.junit.Assert;
@@ -27,35 +30,30 @@ import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.RequestStatus;
import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
import org.opengroup.osdu.core.common.model.indexer.OperationType;
+import org.opengroup.osdu.core.common.model.storage.SchemaItem;
import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
-import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache;
+import org.opengroup.osdu.indexer.cache.PartitionSafeFlattenedSchemaCache;
+import org.opengroup.osdu.indexer.cache.PartitionSafeSchemaCache;
+import org.opengroup.osdu.indexer.model.indexproperty.PropertyConfigurations;
import org.opengroup.osdu.indexer.schema.converter.exeption.SchemaProcessingException;
import org.opengroup.osdu.indexer.schema.converter.interfaces.IVirtualPropertiesSchemaCache;
+import org.opengroup.osdu.indexer.util.AugmenterSetting;
import org.opengroup.osdu.indexer.util.ElasticClientHandler;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.springframework.test.context.junit4.SpringRunner;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
+import java.lang.reflect.Type;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyBoolean;
-import static org.mockito.ArgumentMatchers.anyString;
-import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.junit.Assert.*;
+import static org.mockito.ArgumentMatchers.*;
+import static org.mockito.Mockito.*;
import static org.mockito.MockitoAnnotations.initMocks;
import static org.powermock.api.mockito.PowerMockito.mock;
import static org.powermock.api.mockito.PowerMockito.when;
@@ -81,9 +79,15 @@ public class IndexerSchemaServiceTest {
@Mock
private SchemaService schemaService;
@Mock
- private ISchemaCache schemaCache;
+ private PartitionSafeSchemaCache schemaCache;
+ @Mock
+ private PartitionSafeFlattenedSchemaCache flattenedSchemaCache;
@Mock
private IVirtualPropertiesSchemaCache virtualPropertiesSchemaCache;
+ @Mock
+ private PropertyConfigurationsService propertyConfigurationsService;
+ @Mock
+ private AugmenterSetting augmenterSetting;
@InjectMocks
private IndexSchemaServiceImpl sut;
@@ -92,6 +96,7 @@ public class IndexerSchemaServiceTest {
initMocks(this);
RestHighLevelClient restHighLevelClient = mock(RestHighLevelClient.class);
when(elasticClientHandler.createRestClient()).thenReturn(restHighLevelClient);
+ when(augmenterSetting.isEnabled()).thenReturn(true);
}
@Test
@@ -221,6 +226,151 @@ public class IndexerSchemaServiceTest {
verifyNoMoreInteractions(this.mappingService);
}
+ @Test
+ public void should_merge_schema_without_invalidateCache_when_kind_has_property_configuration() throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"startDate\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ String extendedProperties = "[{\n" +
+ " \"path\": \"ext_p1\",\n" +
+ " \"kind\": \"string\"\n" +
+ " }, {\n" +
+ " \"path\": \"ext_p2\",\n" +
+ " \"kind\": \"string\"\n" +
+ " }\n" +
+ "]";
+ Gson gson = new Gson();
+ Type listOfSchemaItems = new TypeToken<ArrayList<SchemaItem>>() {}.getType();
+ List<SchemaItem> extendedSchemaItems = gson.fromJson(extendedProperties, listOfSchemaItems);
+
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.schemaService.getSchema(kind)).thenReturn(storageSchema);
+ when(this.propertyConfigurationsService.getPropertyConfigurations(kind)).thenReturn(new PropertyConfigurations());
+ when(this.propertyConfigurationsService.getExtendedSchemaItems(any(), any(), any())).thenReturn(extendedSchemaItems);
+
+ IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false);
+ assertNotNull(indexSchema);
+ assertEquals(4, indexSchema.getDataSchema().size());
+ verify(this.schemaCache, times(0)).delete(any());
+ verify(this.flattenedSchemaCache, times(0)).delete(any());
+ verify(this.virtualPropertiesSchemaCache, times(0)).delete(any());
+ }
+
+ @Test
+ public void should_merge_schema_with_invalidateCache_when_kind_has_property_configuration() throws IOException, URISyntaxException {
+ String kind = "tenant1:avocet:completion:1.0.0";
+ String storageSchema = "{" +
+ " \"kind\": \"tenant1:avocet:completion:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"startDate\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ String extendedProperties = "[{\n" +
+ " \"path\": \"ext_p1\",\n" +
+ " \"kind\": \"string\"\n" +
+ " }, {\n" +
+ " \"path\": \"ext_p2\",\n" +
+ " \"kind\": \"string\"\n" +
+ " }\n" +
+ "]";
+ Gson gson = new Gson();
+ Type listOfSchemaItems = new TypeToken<ArrayList<SchemaItem>>() {}.getType();
+ List<SchemaItem> extendedSchemaItems = gson.fromJson(extendedProperties, listOfSchemaItems);
+
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.schemaService.getSchema(kind)).thenReturn(storageSchema);
+ when(this.propertyConfigurationsService.getPropertyConfigurations(kind)).thenReturn(new PropertyConfigurations());
+ when(this.propertyConfigurationsService.getExtendedSchemaItems(any(), any(), any())).thenReturn(extendedSchemaItems);
+
+ IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, true);
+ assertNotNull(indexSchema);
+ assertEquals(4, indexSchema.getDataSchema().size());
+ verify(this.schemaCache, times(1)).delete(any());
+ verify(this.flattenedSchemaCache, times(1)).delete(any());
+ verify(this.virtualPropertiesSchemaCache, times(1)).delete(any());
+ }
+
+ @Test
+ public void should_get_schema_of_related_object_kinds_when__kind_has_property_configuration() throws IOException, URISyntaxException {
+ String kind = "osdu:wks:work-product-component--WellLog:1.0.0";
+ String storageSchema = "{" +
+ " \"kind\": \"osdu:wks:work-product-component--WellLog:1.0.0\"," +
+ " \"schema\": [" +
+ " {" +
+ " \"path\": \"status\"," +
+ " \"kind\": \"string\"" +
+ " }," +
+ " {" +
+ " \"path\": \"startDate\"," +
+ " \"kind\": \"string\"" +
+ " }" +
+ " ]" +
+ "}";
+ String propertyConfigurations = "{\n" +
+ " \"Name\": \"WellLogIndex-PropertyPathConfiguration\",\n" +
+ " \"Code\": \"osdu:wks:work-product-component--WellLog:1.\",\n" +
+ " \"AttributionAuthority\": \"OSDU\",\n" +
+ " \"Configurations\": [{\n" +
+ " \"Name\": \"WellboreName\",\n" +
+ " \"Policy\": \"ExtractFirstMatch\",\n" +
+ " \"Paths\": [{\n" +
+ " \"RelatedObjectsSpec.RelationshipDirection\": \"ChildToParent\",\n" +
+ " \"RelatedObjectsSpec.RelatedObjectKind\": \"osdu:wks:master-data--Wellbore:1.\",\n" +
+ " \"RelatedObjectsSpec.RelatedObjectID\": \"data.WellboreID\",\n" +
+ " \"ValueExtraction.ValuePath\": \"data.FacilityName\"\n" +
+ " }\n" +
+ " ]\n" +
+ " }, {\n" +
+ " \"Name\": \"TechnicalAssuranceReviewerOrganisationNames\",\n" +
+ " \"Policy\": \"ExtractAllMatches\",\n" +
+ " \"Paths\": [{\n" +
+ " \"RelatedObjectsSpec.RelationshipDirection\": \"ChildToParent\",\n" +
+ " \"RelatedObjectsSpec.RelatedObjectKind\": \"osdu:wks:master-data--Organisation:1.\",\n" +
+ " \"RelatedObjectsSpec.RelatedObjectID\": \"data.TechnicalAssurances[].Reviewers[].OrganisationID\",\n" +
+ " \"ValueExtraction.ValuePath\": \"data.OrganisationName\"\n" +
+ " }\n" +
+ " ]\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+ ObjectMapper objectMapper = new ObjectMapper();
+ PropertyConfigurations configurations = objectMapper.readValue(propertyConfigurations, PropertyConfigurations.class);
+
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
+ when(this.schemaCache.get(kind)).thenReturn(null);
+ when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
+ when(this.schemaService.getSchema(kind)).thenReturn(storageSchema);
+ when(this.propertyConfigurationsService.getPropertyConfigurations(kind)).thenReturn(configurations);
+ when(this.propertyConfigurationsService.resolveConcreteKind(anyString())).thenAnswer(invocation -> {
+ String relatedObjectKind = invocation.getArgument(0);
+ return relatedObjectKind + "0.0";
+ });
+
+ IndexSchema indexSchema = this.sut.getIndexerInputSchema(kind, false);
+ assertEquals(2, indexSchema.getDataSchema().size());
+ verify(this.propertyConfigurationsService, times(2)).resolveConcreteKind(any());
+ verify(this.schemaCache, times(3)).get(any());
+ verify(this.schemaService, times(3)).getSchema(any());
+ }
+
@Test
public void should_throw_mapping_conflict_when_elastic_backend_cannot_process_schema_changes() throws IOException, URISyntaxException {
String kind = "tenant1:avocet:completion:1.0.0";
@@ -315,13 +465,12 @@ public class IndexerSchemaServiceTest {
when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(kind.replace(":", "-"));
when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
- when(this.schemaCache.get(kind)).thenReturn("schema");
- when(this.schemaCache.get(kind + "_flattened")).thenReturn("flattened schema");
this.sut.processSchemaMessages(schemaMessages);
- verify(this.schemaCache, times(2)).get(anyString());
- verify(this.schemaCache, times(2)).delete(anyString());
+ verify(this.schemaCache, times(1)).delete(anyString());
+ verify(this.flattenedSchemaCache, times(1)).delete(anyString());
+ verify(this.virtualPropertiesSchemaCache, times(1)).delete(anyString());
}
@Test
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceImplTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceImplTest.java
index 194fdab46f77aa9ca67834f640a3ba13337e25c7..5ab9d69ec9d284b08c8dfde688021832d45d8c9d 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceImplTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceImplTest.java
@@ -1,3 +1,18 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.opengroup.osdu.indexer.service;
import com.google.common.reflect.TypeToken;
@@ -6,9 +21,11 @@ import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.rest.RestStatus;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Spy;
@@ -22,8 +39,11 @@ import org.opengroup.osdu.core.common.model.storage.ConversionStatus;
import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
import org.opengroup.osdu.indexer.logging.AuditLogger;
+import org.opengroup.osdu.indexer.model.indexproperty.PropertyConfigurations;
import org.opengroup.osdu.indexer.provider.interfaces.IPublisher;
+import org.opengroup.osdu.indexer.util.AugmenterSetting;
import org.opengroup.osdu.indexer.util.ElasticClientHandler;
+import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -34,8 +54,7 @@ import java.net.URISyntaxException;
import java.util.*;
import static java.util.Collections.singletonList;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
+import static org.junit.Assert.*;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@@ -77,22 +96,34 @@ public class IndexerServiceImplTest {
private IMappingService mappingService;
@Mock
private IPublisher progressPublisher;
+ @Mock
+ private PropertyConfigurationsService propertyConfigurationsService;
+ @Mock
+ private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
+ @Mock
+ private AugmenterSetting augmenterSetting;
private List<RecordInfo> recordInfos = new ArrayList<>();
private final String pubsubMsg = "[{\"id\":\"opendes:doc:test1\",\"kind\":\"opendes:testindexer1:well:1.0.0\",\"op\":\"update\"}," +
- "{\"id\":\"opendes:doc:test2\",\"kind\":\"opendes:testindexer2:well:1.0.0\",\"op\":\"create\"}]";
+ "{\"id\":\"opendes:doc:test2\",\"kind\":\"opendes:testindexer2:well:1.0.0\",\"op\":\"create\"}, {\"id\":\"opendes:doc:test3\",\"kind\":\"opendes:testindexer2:well:1.0.0\",\"op\":\"create\"}]";
+ private final String pubsubMsgForDeletion = "[{\"id\":\"opendes:doc:test1\",\"kind\":\"opendes:testindexer1:well:1.0.0\",\"op\":\"delete\"}," +
+ "{\"id\":\"opendes:doc:test2\",\"kind\":\"opendes:testindexer2:well:1.0.0\",\"op\":\"delete\"}, {\"id\":\"opendes:doc:test3\",\"kind\":\"opendes:testindexer2:well:1.0.0\",\"op\":\"delete\"}]";
+
private final String kind1 = "opendes:testindexer1:well:1.0.0";
private final String kind2 = "opendes:testindexer2:well:1.0.0";
private final String recordId1 = "opendes:doc:test1";
private final String recordId2 = "opendes:doc:test2";
+ private final String recordId3 = "opendes:doc:test3";
private final String failureMassage = "test failure";
+ private final String badRequestMessage = "Elasticsearch exception [type=mapper_parsing_exception, reason=failed to parse field [data.SpatialLocation.Wgs84Coordinates] of type [geo_shape]]";
private DpsHeaders dpsHeaders;
private RecordChangedMessages recordChangedMessages;
@Before
public void setup() throws IOException {
+ when(augmenterSetting.isEnabled()).thenReturn(true);
}
@Test
@@ -115,57 +146,17 @@ public class IndexerServiceImplTest {
@Test
public void should_properlyUpdateAuditLogs_givenValidCreateAndUpdateRecords() {
try {
- mockStatic(Acl.class);
-
- // setup headers
- this.dpsHeaders = new DpsHeaders();
- this.dpsHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
- when(this.requestInfo.getHeaders()).thenReturn(dpsHeaders);
- when(this.requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(dpsHeaders.getHeaders());
-
- // setup message
- Type listType = new TypeToken<List<RecordInfo>>() {}.getType();
- this.recordInfos = (new Gson()).fromJson(this.pubsubMsg, listType);
- Map<String, String> messageAttributes = new HashMap<>();
- messageAttributes.put(DpsHeaders.DATA_PARTITION_ID, "opendes");
- this.recordChangedMessages = RecordChangedMessages.builder().attributes(messageAttributes).messageId("xxxx").publishTime("2000-01-02T10:10:44+0000").data("{}").build();
-
- // setup schema
- Map<String, Object> schema = createSchema();
- indexSchemaServiceMock(kind2, schema);
- indexSchemaServiceMock(kind1, null);
-
- // setup storage records
- Map<String, Object> storageData = new HashMap<>();
- storageData.put("schema1", "test-value");
- List<Records.Entity> validRecords = new ArrayList<>();
- validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build());
- List<ConversionStatus> conversionStatus = new LinkedList<>();
- Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build();
- when(this.storageService.getStorageRecords(any(), any())).thenReturn(storageRecords);
-
- // setup elastic, index and mapped document
- when(this.indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true);
- when(this.mappingService.getIndexMappingFromRecordSchema(any())).thenReturn(new HashMap<>());
-
- when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient);
- when(this.restHighLevelClient.bulk(any(), any(RequestOptions.class))).thenReturn(this.bulkResponse);
-
- Map<String, Object> indexerMappedPayload = new HashMap<>();
- indexerMappedPayload.put("id", "keyword");
- when(this.storageIndexerPayloadMapper.mapDataPayload(any(), any(), any())).thenReturn(indexerMappedPayload);
-
- BulkItemResponse[] responses = new BulkItemResponse[]{prepareFailedResponse(), prepareSuccessfulResponse()};
- when(this.bulkResponse.getItems()).thenReturn(responses);
+ prepareTestDataAndEnv(this.pubsubMsg);
// test
JobStatus jobStatus = this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
// validate
- assertEquals(2, jobStatus.getStatusesList().size());
- assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
+ assertEquals(3, jobStatus.getStatusesList().size());
+ assertEquals(2, jobStatus.getIdsByIndexingStatus(IndexingStatus.FAIL).size());
assertEquals(1, jobStatus.getIdsByIndexingStatus(IndexingStatus.SUCCESS).size());
+ verify(restHighLevelClient, times(2)).bulk(any(), any());
verify(this.auditLogger).indexCreateRecordSuccess(singletonList("RecordStatus(id=opendes:doc:test2, kind=opendes:testindexer2:well:1.0.0, operationType=create, status=SUCCESS)"));
verify(this.auditLogger).indexUpdateRecordFail(singletonList("RecordStatus(id=opendes:doc:test1, kind=opendes:testindexer1:well:1.0.0, operationType=update, status=FAIL, message=test failure)"));
} catch (Exception e) {
@@ -173,6 +164,125 @@ public class IndexerServiceImplTest {
}
}
+ @Test
+ public void should_updateAssociatedRecords_givenValidCreateAndUpdateRecords() {
+ try {
+ prepareTestDataAndEnv(this.pubsubMsg);
+
+ // setup property configuration
+ when(this.propertyConfigurationsService.isPropertyConfigurationsEnabled(any())).thenReturn(true);
+ ArgumentCaptor<Map<String, List<String>>> upsertArgumentCaptor = ArgumentCaptor.forClass(Map.class);
+ ArgumentCaptor<Map<String, List<String>>> deleteArgumentCaptor = ArgumentCaptor.forClass(Map.class);
+
+ // test
+ this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
+
+ // validate
+ verify(this.propertyConfigurationsService, times(1)).updateAssociatedRecords(any(), upsertArgumentCaptor.capture(), deleteArgumentCaptor.capture());
+ Map<String, List<String>> upsertKindIds = upsertArgumentCaptor.getValue();
+ Map<String, List<String>> deleteKindIds = deleteArgumentCaptor.getValue();
+ assertEquals(2, upsertKindIds.size());
+ assertEquals(1, upsertKindIds.get(kind1).size());
+ assertEquals(2, upsertKindIds.get(kind2).size());
+ assertEquals(0, deleteKindIds.size());
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_updateAssociatedRecords_givenValidDeleteRecords() {
+ try {
+ prepareTestDataAndEnv(this.pubsubMsgForDeletion);
+
+ // setup property configuration
+ when(this.propertyConfigurationsService.isPropertyConfigurationsEnabled(any())).thenReturn(true);
+ ArgumentCaptor<Map<String, List<String>>> upsertArgumentCaptor = ArgumentCaptor.forClass(Map.class);
+ ArgumentCaptor<Map<String, List<String>>> deleteArgumentCaptor = ArgumentCaptor.forClass(Map.class);
+
+ // test
+ this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
+
+ // validate
+ verify(this.propertyConfigurationsService, times(1)).updateAssociatedRecords(any(), upsertArgumentCaptor.capture(), deleteArgumentCaptor.capture());
+ Map<String, List<String>> upsertKindIds = upsertArgumentCaptor.getValue();
+ Map<String, List<String>> deleteKindIds = deleteArgumentCaptor.getValue();
+ assertEquals(0, upsertKindIds.size());
+ assertEquals(2, deleteKindIds.size());
+ assertEquals(1, deleteKindIds.get(kind1).size());
+ assertEquals(2, deleteKindIds.get(kind2).size());
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_mergeExtendedProperties_givenValidCreateAndUpdateRecords_and_kindsHavingPropertyConfigurations() {
+ try {
+ prepareTestDataAndEnv(this.pubsubMsg);
+
+ // setup property configuration
+ when(this.propertyConfigurationsService.isPropertyConfigurationsEnabled(any())).thenReturn(true);
+ when(this.propertyConfigurationsService.getPropertyConfigurations(any())).thenReturn(new PropertyConfigurations());
+
+ // test
+ this.sut.processRecordChangedMessages(recordChangedMessages, recordInfos);
+
+ // validate
+ verify(this.propertyConfigurationsService, times(2)).getPropertyConfigurations(any());
+ verify(this.propertyConfigurationsService, times(2)).getExtendedProperties(any(), any(), any());
+ verify(this.propertyConfigurationsService, times(2)).cacheDataRecord(any(), any(), any());
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ private void prepareTestDataAndEnv(String pubsubMsg) throws IOException, URISyntaxException {
+ mockStatic(Acl.class);
+
+ // setup headers
+ this.dpsHeaders = new DpsHeaders();
+ this.dpsHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
+ when(this.requestInfo.getHeaders()).thenReturn(dpsHeaders);
+ when(this.requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(dpsHeaders.getHeaders());
+
+ // setup message
+ Type listType = new TypeToken<List<RecordInfo>>() {}.getType();
+ this.recordInfos = (new Gson()).fromJson(pubsubMsg, listType);
+ Map<String, String> messageAttributes = new HashMap<>();
+ messageAttributes.put(DpsHeaders.DATA_PARTITION_ID, "opendes");
+ this.recordChangedMessages = RecordChangedMessages.builder().attributes(messageAttributes).messageId("xxxx").publishTime("2000-01-02T10:10:44+0000").data("{}").build();
+
+ // setup schema
+ Map<String, Object> schema = createSchema();
+ indexSchemaServiceMock(kind2, schema);
+ indexSchemaServiceMock(kind1, null);
+
+ // setup storage records
+ Map<String, Object> storageData = new HashMap<>();
+ storageData.put("schema1", "test-value");
+ List<Records.Entity> validRecords = new ArrayList<>();
+ validRecords.add(Records.Entity.builder().id(recordId2).kind(kind2).data(storageData).build());
+ validRecords.add(Records.Entity.builder().id(recordId3).kind(kind2).data(storageData).build());
+ List<ConversionStatus> conversionStatus = new LinkedList<>();
+ Records storageRecords = Records.builder().records(validRecords).conversionStatuses(conversionStatus).build();
+ when(this.storageService.getStorageRecords(any(), any())).thenReturn(storageRecords);
+
+ // setup elastic, index and mapped document
+ when(this.indicesService.createIndex(any(), any(), any(), any(), any())).thenReturn(true);
+ when(this.mappingService.getIndexMappingFromRecordSchema(any())).thenReturn(new HashMap<>());
+
+ when(this.elasticClientHandler.createRestClient()).thenReturn(this.restHighLevelClient);
+ when(this.restHighLevelClient.bulk(any(), any(RequestOptions.class))).thenReturn(this.bulkResponse);
+
+ Map<String, Object> indexerMappedPayload = new HashMap<>();
+ indexerMappedPayload.put("id", "keyword");
+ when(this.storageIndexerPayloadMapper.mapDataPayload(any(), any(), any())).thenReturn(indexerMappedPayload);
+
+ BulkItemResponse[] responses = new BulkItemResponse[]{prepareFailedResponse(), prepareSuccessfulResponse(), prepare400Response()};
+ when(this.bulkResponse.getItems()).thenReturn(responses);
+ }
+
private BulkItemResponse prepareFailedResponse() {
BulkItemResponse responseFail = mock(BulkItemResponse.class);
when(responseFail.isFailed()).thenReturn(true);
@@ -182,6 +292,14 @@ public class IndexerServiceImplTest {
return responseFail;
}
+ private BulkItemResponse prepare400Response() {
+ BulkItemResponse responseFail = mock(BulkItemResponse.class);
+ when(responseFail.isFailed()).thenReturn(true);
+ when(responseFail.getId()).thenReturn(recordId3);
+ when(responseFail.getFailure()).thenReturn(new BulkItemResponse.Failure("failure index", "failure type", "failure id", new Exception(badRequestMessage), RestStatus.BAD_REQUEST));
+ return responseFail;
+ }
+
private BulkItemResponse prepareSuccessfulResponse() {
BulkItemResponse responseSuccess = mock(BulkItemResponse.class);
when(responseSuccess.getId()).thenReturn(recordId2);
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndicesServiceTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndicesServiceTest.java
index 7f22b619a9364b308791473adbae5324a0687d9a..7bc3fe17d8a060337c7b624fe7d7c76036ff3d7f 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndicesServiceTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndicesServiceTest.java
@@ -77,6 +77,8 @@ public class IndicesServiceTest {
private Response response;
@Mock
private HttpEntity httpEntity;
+ @Mock
+ private IndexAliasService indexAliasService;
@InjectMocks
private IndicesServiceImpl sut;
@@ -94,22 +96,19 @@ public class IndicesServiceTest {
@Test
public void create_elasticIndex() throws Exception {
- String kind = "common:welldb:wellbore:1.2.0";
String index = "common-welldb-wellbore-1.2.0";
CreateIndexResponse indexResponse = new CreateIndexResponse(true, true, index);
AcknowledgedResponse acknowledgedResponse = new AcknowledgedResponse(true);
- when(elasticIndexNameResolver.getKindFromIndexName(any())).thenReturn(kind);
when(elasticIndexNameResolver.getIndexNameFromKind(any())).thenReturn(index);
- when(elasticIndexNameResolver.getIndexAliasFromKind(any())).thenReturn("a12345678");
- when(elasticIndexNameResolver.isIndexAliasSupported(any())).thenReturn(true);
when(restHighLevelClient.indices()).thenReturn(indicesClient);
when(indicesClient.create(any(CreateIndexRequest.class), any(RequestOptions.class))).thenReturn(indexResponse);
when(indicesClient.updateAliases(any(IndicesAliasesRequest.class), any(RequestOptions.class))).thenReturn(acknowledgedResponse);
+
boolean response = this.sut.createIndex(restHighLevelClient, index, null, "anytype", new HashMap<>());
assertTrue(response);
when(this.indicesExistCache.get(index)).thenReturn(true);
- verify(this.indicesClient, times(2)).updateAliases(any(IndicesAliasesRequest.class), any(RequestOptions.class));
+ verify(this.indexAliasService, times(1)).createIndexAlias(any(), any());
}
@Test
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/PropertyConfigurationsServiceImplTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/PropertyConfigurationsServiceImplTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..9a11d7a791463b0668619ccad412d7fc90d403e3
--- /dev/null
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/PropertyConfigurationsServiceImplTest.java
@@ -0,0 +1,900 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.service;
+
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Strings;
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import lombok.SneakyThrows;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.indexer.OperationType;
+import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
+import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
+import org.opengroup.osdu.core.common.model.storage.RecordData;
+import org.opengroup.osdu.core.common.model.storage.Schema;
+import org.opengroup.osdu.core.common.model.storage.SchemaItem;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.opengroup.osdu.indexer.cache.*;
+import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.opengroup.osdu.indexer.model.*;
+import org.opengroup.osdu.indexer.model.indexproperty.PropertyConfigurations;
+import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.UnsupportedEncodingException;
+import java.lang.reflect.Type;
+import java.net.URISyntaxException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.mockito.ArgumentMatchers.*;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@RunWith(SpringRunner.class)
+public class PropertyConfigurationsServiceImplTest {
+ private final Gson gson = new Gson();
+
+ @InjectMocks
+ private PropertyConfigurationsServiceImpl sut;
+
+ @Mock
+ private IndexerConfigurationProperties configurationProperties;
+ @Mock
+ private PartitionSafePropertyConfigurationsCache propertyConfigurationCache;
+ @Mock
+ private PartitionSafePropertyConfigurationsEnabledCache propertyConfigurationsEnabledCache;
+ @Mock
+ private PartitionSafeParentChildRelationshipSpecsCache parentChildRelationshipSpecsCache;
+ @Mock
+ private PartitionSafeChildrenKindsCache childrenKindsCache;
+ @Mock
+ private PartitionSafeKindCache kindCache;
+ @Mock
+ private IRelatedObjectCache relatedObjectCache;
+ @Mock
+ private IRecordChangeInfoCache recordChangeInfoCache;
+ @Mock
+ private SearchService searchService;
+ @Mock
+ private SchemaService schemaService;
+ @Mock
+ private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
+ @Mock
+ private IRequestInfo requestInfo;
+ @Mock
+ private JaxRsDpsLog jaxRsDpsLog;
+
+ private final String propertyConfigurationKind = "osdu:wks:reference-data--IndexPropertyPathConfiguration:*";
+ private String childKind;
+ private String childId;
+ private String parentKind;
+ private String parentId;
+
+ @Test
+ public void isPropertyConfigurationsEnabled_invalid_kind() {
+ Assert.assertFalse(sut.isPropertyConfigurationsEnabled(null));
+ Assert.assertFalse(sut.isPropertyConfigurationsEnabled(""));
+ Assert.assertFalse(sut.isPropertyConfigurationsEnabled("anyAuth:anySource:anyEntity"));
+ }
+
+ @Test
+ public void isPropertyConfigurationsEnabled_with_value_true_in_cache() {
+ String kind = "anyAuth:anySource:anyEntity:1.";
+ when(this.propertyConfigurationsEnabledCache.get(any())).thenReturn(true);
+ Assert.assertTrue(sut.isPropertyConfigurationsEnabled(kind));
+ verify(this.propertyConfigurationsEnabledCache, times(0)).put(any(), any());
+ }
+
+ @Test
+ public void isPropertyConfigurationsEnabled_with_value_false_in_cache() {
+ String kind = "anyAuth:anySource:anyEntity:1.";
+ when(this.propertyConfigurationsEnabledCache.get(any())).thenReturn(false);
+ Assert.assertFalse(sut.isPropertyConfigurationsEnabled(kind));
+ verify(this.propertyConfigurationsEnabledCache, times(0)).put(any(), any());
+ }
+
+ @Test
+ public void isPropertyConfigurationsEnabled_with_result_from_search() throws URISyntaxException {
+ String kind = "anyAuth:anySource:anyEntity:1.";
+ SearchResponse response = new SearchResponse();
+ response.setResults(Arrays.asList(new SearchRecord()));
+ when(this.propertyConfigurationsEnabledCache.get(any())).thenReturn(null);
+ when(this.searchService.query(any())).thenReturn(response);
+ Assert.assertTrue(sut.isPropertyConfigurationsEnabled(kind));
+ verify(this.propertyConfigurationsEnabledCache, times(1)).put(any(), any());
+ }
+
+ @Test
+ public void isPropertyConfigurationsEnabled_without_result_from_search() throws URISyntaxException {
+ String kind = "anyAuth:anySource:anyEntity:1.";
+ SearchResponse response = new SearchResponse();
+ when(this.propertyConfigurationsEnabledCache.get(any())).thenReturn(null);
+ when(this.searchService.query(any())).thenReturn(response);
+ Assert.assertFalse(sut.isPropertyConfigurationsEnabled(kind));
+ verify(this.propertyConfigurationsEnabledCache, times(1)).put(any(), any());
+ }
+
+ @Test
+ public void getPropertyConfigurations_invalid_kind() {
+ Assert.assertNull(sut.getPropertyConfigurations(null));
+ Assert.assertNull(sut.getPropertyConfigurations(""));
+ Assert.assertNull(sut.getPropertyConfigurations("anyAuth:anySource:anyEntity"));
+ }
+
+ @Test
+ public void getPropertyConfigurations_with_configuration_in_cache() {
+ String code = "anyAuth:anySource:anyEntity:1.";
+ String kind = "anyAuth:anySource:anyEntity:1.0.0";
+ PropertyConfigurations configuration = new PropertyConfigurations();
+ configuration.setCode(code);
+ when(this.propertyConfigurationCache.get(eq(code))).thenReturn(configuration);
+ PropertyConfigurations configuration2 = sut.getPropertyConfigurations(kind);
+
+ Assert.assertNotNull(configuration2);
+ Assert.assertEquals(code, configuration2.getCode());
+ }
+
+ @Test
+ public void getPropertyConfigurations_with_empty_configuration_in_cache() {
+ String code = "anyAuth:anySource:anyEntity:1.";
+ String kind = "anyAuth:anySource:anyEntity:1.0.0";
+ PropertyConfigurations configuration = new PropertyConfigurations();
+ when(this.propertyConfigurationCache.get(eq(code))).thenReturn(configuration);
+ PropertyConfigurations configuration2 = sut.getPropertyConfigurations(kind);
+
+ Assert.assertNull(configuration2);
+ }
+
+ @Test
+ public void getPropertyConfigurations_with_result_from_search() throws URISyntaxException {
+ Map<String, Object> data = this.getDataMap("well_configuration_record.json");
+ SearchRecord searchRecord = new SearchRecord();
+ searchRecord.setData(data);
+ List<SearchRecord> results = Arrays.asList(searchRecord);
+ SearchResponse searchResponse = new SearchResponse();
+ searchResponse.setResults(results);
+ searchResponse.setTotalCount(results.size());
+ when(this.searchService.queryWithCursor(any())).thenReturn(searchResponse);
+ String kind = "osdu:wks:master-data--Well:1.0.0";
+ String code = "osdu:wks:master-data--Well:1.";
+ PropertyConfigurations configuration = sut.getPropertyConfigurations(kind);
+
+ ArgumentCaptor<PropertyConfigurations> argumentCaptor = ArgumentCaptor.forClass(PropertyConfigurations.class);
+ // If we mock the implementation of propertyConfigurationCache, it should be called once
+ verify(this.propertyConfigurationCache, times(2)).put(any(), argumentCaptor.capture());
+ Assert.assertNotNull(configuration);
+ Assert.assertEquals(code, configuration.getCode());
+ Assert.assertEquals(code, argumentCaptor.getValue().getCode());
+ }
+
+ @Test
+ public void getPropertyConfigurations_without_result_from_search() throws URISyntaxException {
+ when(this.searchService.queryWithCursor(any())).thenReturn(new SearchResponse());
+
+ String kind = "osdu:wks:master-data--Well:1.0.0";
+ PropertyConfigurations configuration = sut.getPropertyConfigurations(kind);
+
+ ArgumentCaptor<PropertyConfigurations> argumentCaptor = ArgumentCaptor.forClass(PropertyConfigurations.class);
+ verify(this.propertyConfigurationCache, times(1)).put(any(), argumentCaptor.capture());
+ Assert.assertNull(configuration);
+ Assert.assertNull(argumentCaptor.getValue().getCode());
+ }
+
+ @Test
+ public void getExtendedProperties_from_children_objects() throws JsonProcessingException, URISyntaxException {
+ PropertyConfigurations propertyConfigurations = getConfigurations("wellbore_configuration_record.json");
+ Map<String, Object> originalDataMap = getDataMap("wellbore_data.json");
+ String jsonText = getJsonFromFile("welllog_search_records.json");
+ Type type = new TypeToken<List<SearchRecord>>() {}.getType();
+ List<SearchRecord> childrenRecords = gson.fromJson(jsonText, type);
+ SearchResponse response = new SearchResponse();
+ response.setResults(childrenRecords);
+ when(this.searchService.queryWithCursor(any())).thenReturn(response);
+
+ Map<String, Object> extendedProperties = this.sut.getExtendedProperties("anyId", originalDataMap, propertyConfigurations);
+ Map<String, Object> expectedExtendedProperties = getDataMap("wellbore_extended_data.json");
+ verifyMap(expectedExtendedProperties, extendedProperties);
+ }
+
+ @Test
+ public void getExtendedProperties_from_self_and_parent_objects() throws JsonProcessingException, URISyntaxException {
+ PropertyConfigurations propertyConfigurations = getConfigurations("welllog_configuration_record.json");
+ Map<String, Object> originalDataMap = getDataMap("welllog_original_data.json");
+ Map<String, Object> relatedObjectData;
+ Map<String, Map<String, Object>> relatedObjects = new HashMap<>();
+ relatedObjectData = getDataMap("wellbore_data.json");
+ relatedObjects.put("opendes:master-data--Wellbore:nz-100000113552", relatedObjectData);
+ relatedObjectData = getDataMap("organisation_data1.json");
+ relatedObjects.put("opendes:master-data--Organisation:BigOil-Department-SeismicInterpretation", relatedObjectData);
+ relatedObjectData = getDataMap("organisation_data2.json");
+ relatedObjects.put("opendes:master-data--Organisation:BigOil-Department-SeismicProcessing", relatedObjectData);
+
+ // Setup search response for searchService.queryWithCursor(...)
+ when(this.searchService.query(any())).thenAnswer(invocation -> {
+ SearchRequest searchRequest = invocation.getArgument(0);
+ String query = searchRequest.getQuery();
+ Map<String, Object> data = null;
+ for(Map.Entry<String, Map<String, Object>> entry: relatedObjects.entrySet()) {
+ if(query.contains(entry.getKey())) {
+ data = entry.getValue();
+ break;
+ }
+ }
+ if(data == null)
+ throw new Exception("Unexpected search");
+ SearchResponse searchResponse = new SearchResponse();
+ SearchRecord record = new SearchRecord();
+ record.setData(data);
+ searchResponse.setResults(Arrays.asList(record));
+ return searchResponse;
+ });
+
+ Map<String, Object> extendedProperties = this.sut.getExtendedProperties("anyId", originalDataMap, propertyConfigurations);
+ Map<String, Object> expectedExtendedProperties = getDataMap("welllog_extended_data.json");
+ verifyMap(expectedExtendedProperties, extendedProperties);
+ }
+
+ private void verifyMap(Map<String, Object> expectedExtendedProperties, Map<String, Object> extendedProperties) {
+ Assert.assertEquals(expectedExtendedProperties.size(), extendedProperties.size());
+
+ for(Map.Entry<String, Object> entry: expectedExtendedProperties.entrySet()) {
+ String name = entry.getKey();
+ Object value = entry.getValue();
+ Assert.assertTrue(extendedProperties.containsKey(name));
+ if(value instanceof String) {
+ Assert.assertEquals(value, extendedProperties.get(name));
+ }
+ else if(value instanceof List) {
+ List<String> expectedValues = (List<String>)value;
+ List<String> values = (List<String>)extendedProperties.get(name);
+ Assert.assertEquals(expectedValues.size(), values.size());
+ for(int i = 0; i < expectedValues.size(); i++) {
+ Assert.assertEquals(expectedValues.get(i), values.get(i));
+ }
+ }
+ else {
+ Assert.assertEquals(value, extendedProperties.get(name));
+ }
+ }
+ }
+
+ @Test
+ public void getExtendedSchemaItems_from_self_and_parent_object_kind() throws JsonProcessingException {
+ PropertyConfigurations propertyConfigurations = getConfigurations("well_configuration_record.json");
+ Schema originalSchema = getSchema("well_storage_schema.json");
+ Schema geoPoliticalEntitySchema = getSchema("geo_political_entity_storage_schema.json");
+ String relatedObjectKind = "osdu:wks:master-data--GeoPoliticalEntity:1.";
+ Map<String, Schema> relatedObjectKindSchemas = new HashMap<>();
+ relatedObjectKindSchemas.put(relatedObjectKind, geoPoliticalEntitySchema);
+
+ List<SchemaItem> extendedSchemaItems = this.sut.getExtendedSchemaItems(originalSchema, relatedObjectKindSchemas, propertyConfigurations);
+ Assert.assertEquals(3, extendedSchemaItems.size());
+ SchemaItem countryNameItem = extendedSchemaItems.stream().filter(item -> item.getPath().equals("CountryNames")).findFirst().orElse(null);
+ Assert.assertNotNull(countryNameItem);
+ Assert.assertEquals("[]string", countryNameItem.getKind());
+
+ SchemaItem wellUWIItem = extendedSchemaItems.stream().filter(item -> item.getPath().equals("WellUWI")).findFirst().orElse(null);
+ Assert.assertNotNull(wellUWIItem);
+ Assert.assertEquals("string", wellUWIItem.getKind());
+
+ SchemaItem associatedIdentitiesItem = extendedSchemaItems.stream().filter(item -> item.getPath().equals("AssociatedIdentities")).findFirst().orElse(null);
+ Assert.assertNotNull(associatedIdentitiesItem);
+ Assert.assertEquals("[]string", associatedIdentitiesItem.getKind());
+ }
+
+ @Test
+ public void getExtendedSchemaItems_from_multiple_object_kinds() throws JsonProcessingException {
+ PropertyConfigurations propertyConfigurations = getConfigurations("welllog_configuration_record.json");
+ Schema originalSchema = getSchema("welllog_storage_schema.json");
+ Map<String, Schema> relatedObjectKindSchemas = new HashMap<>();
+ Schema wellboreSchema = getSchema("wellbore_storage_schema.json");
+ relatedObjectKindSchemas.put("osdu:wks:master-data--Wellbore:1.", wellboreSchema);
+ Schema organisationSchema = getSchema("organisation_storage_schema.json");
+ relatedObjectKindSchemas.put("osdu:wks:master-data--Organisation:1.", organisationSchema);
+
+ String jsonText = getJsonFromFile("welllog_extended_schema_items.json");
+ Type type = new TypeToken<List<SchemaItem>>() {}.getType();
+ List<SchemaItem> expectedExtendedSchemaItems = gson.fromJson(jsonText, type);
+
+ List<SchemaItem> extendedSchemaItems = this.sut.getExtendedSchemaItems(originalSchema, relatedObjectKindSchemas, propertyConfigurations);
+ Assert.assertEquals(expectedExtendedSchemaItems.size(), extendedSchemaItems.size());
+ for(int i = 0; i < expectedExtendedSchemaItems.size(); i++) {
+ SchemaItem expectedExtendedSchemaItem = expectedExtendedSchemaItems.get(i);
+ SchemaItem extendedSchemaItem = extendedSchemaItems.get(i);
+ Assert.assertEquals(expectedExtendedSchemaItem.getKind(), extendedSchemaItem.getKind());
+ Assert.assertEquals(expectedExtendedSchemaItem.getPath(), extendedSchemaItem.getPath());
+ }
+ }
+
+ @Test
+ public void resolveConcreteKind_with_concreteKind() {
+ String kind = "osdu:wks:master-data--Well:1.0.0";
+ Assert.assertEquals(kind, sut.resolveConcreteKind(kind));
+ }
+
+ @Test
+ public void resolveConcreteKind_with_null_empty_kind() {
+ Assert.assertTrue(Strings.isNullOrEmpty(sut.resolveConcreteKind(null)));
+ Assert.assertTrue(Strings.isNullOrEmpty(sut.resolveConcreteKind("")));
+ }
+
+ @Test
+ public void resolveConcreteKind_with_value_in_cache() {
+ String kind = "osdu:wks:master-data--Well:1.";
+ String expectedKind = kind + "2.3";
+
+ when(this.kindCache.get(any())).thenReturn(expectedKind);
+ Assert.assertEquals(expectedKind, sut.resolveConcreteKind(kind));
+ }
+
+ @Test
+ public void resolveConcreteKind_with_result_from_schemaService() throws UnsupportedEncodingException, URISyntaxException {
+ String kind = "osdu:wks:master-data--Well:1.";
+ String expectedKind = kind + "2.3";
+
+ SchemaIdentity schemaIdentity = new SchemaIdentity();
+ schemaIdentity.setAuthority("osdu");
+ schemaIdentity.setSource("wks");
+ schemaIdentity.setEntityType("master-data--Well");
+ schemaIdentity.setSchemaVersionMajor(1);
+ schemaIdentity.setSchemaVersionMinor(2);
+ schemaIdentity.setSchemaVersionPatch(3);
+ SchemaInfo schemaInfo = new SchemaInfo();
+ schemaInfo.setSchemaIdentity(schemaIdentity);
+ List<SchemaInfo> schemaInfos = Arrays.asList(schemaInfo);
+ SchemaInfoResponse response = new SchemaInfoResponse();
+ response.setSchemaInfos(schemaInfos);
+ response.setTotalCount(schemaInfos.size());
+ when(this.schemaService.getSchemaInfos(any(), any(), any(), any(), eq(null), eq(null), eq(true))).thenReturn(response);
+ String latestKind = sut.resolveConcreteKind(kind);
+
+ ArgumentCaptor<String> argumentCaptor = ArgumentCaptor.forClass(String.class);
+ verify(this.kindCache, times(1)).put(any(), argumentCaptor.capture());
+ Assert.assertEquals(expectedKind, latestKind);
+ Assert.assertEquals(expectedKind, argumentCaptor.getValue());
+ }
+
+ @Test
+ public void resolveConcreteKind_without_result_from_schemaService() throws UnsupportedEncodingException, URISyntaxException {
+ String kind = "osdu:wks:master-data--Well:1.";
+
+ SchemaInfoResponse response = new SchemaInfoResponse();
+ when(this.schemaService.getSchemaInfos(any(), any(), any(), any(), eq(null), eq(null), eq(true))).thenReturn(response);
+ String latestKind = sut.resolveConcreteKind(kind);
+
+ verify(this.kindCache, times(0)).put(any(), any());
+ Assert.assertNull(latestKind);
+ }
+
+ @Test
+ public void cacheDataRecord_create_record() throws URISyntaxException {
+ ArgumentCaptor<RecordChangeInfo> recordInfoArgumentCaptor = ArgumentCaptor.forClass(RecordChangeInfo.class);
+ ArgumentCaptor<RecordData> dataMapArgumentCaptor = ArgumentCaptor.forClass(RecordData.class);
+ String recordId = "anyId";
+ String kind = "anyKind";
+ Map<String, Object> dataMap = new HashMap<>();
+ dataMap.put("p1", "v1");
+
+ when(this.searchService.query(any())).thenReturn(new SearchResponse());
+
+ this.sut.cacheDataRecord(recordId, kind, dataMap);
+
+ verify(this.recordChangeInfoCache, times(1)).put(any(), recordInfoArgumentCaptor.capture());
+ verify(this.relatedObjectCache, times(1)).put(any(), dataMapArgumentCaptor.capture());
+
+ Assert.assertEquals(OperationType.create.getValue(), recordInfoArgumentCaptor.getValue().getRecordInfo().getOp());
+ Assert.assertEquals(1, dataMapArgumentCaptor.getValue().getData().size());
+ }
+
+ @Test
+ public void cacheDataRecord_update_record() throws URISyntaxException {
+ ArgumentCaptor<RecordChangeInfo> recordInfoArgumentCaptor = ArgumentCaptor.forClass(RecordChangeInfo.class);
+ ArgumentCaptor<RecordData> dataMapArgumentCaptor = ArgumentCaptor.forClass(RecordData.class);
+ String recordId = "anyId";
+ String kind = "anyKind";
+ Map<String, Object> dataMap = new HashMap<>();
+ dataMap.put("p1", "v1");
+ Map<String, Object> previousDataMap = new HashMap<>();
+ previousDataMap.put("p1", "v10");
+ previousDataMap.put("p2", "v2");
+
+ SearchRecord searchRecord = new SearchRecord();
+ searchRecord.setKind(kind);
+ searchRecord.setId(recordId);
+ searchRecord.setData(previousDataMap);
+ SearchResponse searchResponse = new SearchResponse();
+ searchResponse.setResults(Arrays.asList(searchRecord));
+ searchResponse.setTotalCount(1);
+ when(this.searchService.query(any())).thenReturn(searchResponse);
+
+ this.sut.cacheDataRecord(recordId, kind, dataMap);
+
+ verify(this.recordChangeInfoCache, times(1)).put(any(), recordInfoArgumentCaptor.capture());
+ verify(this.relatedObjectCache, times(2)).put(any(), dataMapArgumentCaptor.capture());
+
+ RecordChangeInfo changedInfo = recordInfoArgumentCaptor.getValue();
+ Assert.assertEquals(OperationType.update.getValue(), changedInfo.getRecordInfo().getOp());
+ Assert.assertEquals(2, changedInfo.getUpdatedProperties().size());
+ Assert.assertTrue(changedInfo.getUpdatedProperties().contains("p1"));
+ Assert.assertTrue(changedInfo.getUpdatedProperties().contains("p2"));
+ Assert.assertEquals(1, dataMapArgumentCaptor.getValue().getData().size());
+ Assert.assertEquals("v1", dataMapArgumentCaptor.getValue().getData().get("p1"));
+ }
+
+ @Test
+ public void cacheDataRecord_update_record_merge_previous_UpdateChangedInfo() throws URISyntaxException {
+ ArgumentCaptor<RecordChangeInfo> recordInfoArgumentCaptor = ArgumentCaptor.forClass(RecordChangeInfo.class);
+ ArgumentCaptor<RecordData> dataMapArgumentCaptor = ArgumentCaptor.forClass(RecordData.class);
+ String recordId = "anyId";
+ String kind = "anyKind";
+ Map<String, Object> dataMap = new HashMap<>();
+ dataMap.put("p1", "v1");
+ Map<String, Object> previousDataMap = new HashMap<>();
+ previousDataMap.put("p1", "v1");
+ previousDataMap.put("p2", "v2");
+
+ RecordChangeInfo previousChangedInfo = new RecordChangeInfo();
+ RecordInfo recordInfo = new RecordInfo();
+ recordInfo.setId(recordId);
+ recordInfo.setKind(kind);
+ recordInfo.setOp(OperationType.update.getValue());
+ previousChangedInfo.setRecordInfo(recordInfo);
+ previousChangedInfo.setUpdatedProperties(Arrays.asList("p1"));
+
+ SearchRecord searchRecord = new SearchRecord();
+ searchRecord.setKind(kind);
+ searchRecord.setId(recordId);
+ searchRecord.setData(previousDataMap);
+ SearchResponse searchResponse = new SearchResponse();
+ searchResponse.setResults(Arrays.asList(searchRecord));
+ searchResponse.setTotalCount(1);
+
+ when(this.searchService.query(any())).thenReturn(searchResponse);
+ when(this.recordChangeInfoCache.get(any())).thenReturn(previousChangedInfo);
+
+ this.sut.cacheDataRecord(recordId, kind, dataMap);
+
+ verify(this.recordChangeInfoCache, times(1)).put(any(), recordInfoArgumentCaptor.capture());
+ verify(this.relatedObjectCache, times(2)).put(any(), dataMapArgumentCaptor.capture());
+
+ RecordChangeInfo changedInfo = recordInfoArgumentCaptor.getValue();
+ Assert.assertEquals(OperationType.update.getValue(), changedInfo.getRecordInfo().getOp());
+ Assert.assertEquals(2, changedInfo.getUpdatedProperties().size());
+ Assert.assertTrue(changedInfo.getUpdatedProperties().contains("p1"));
+ Assert.assertTrue(changedInfo.getUpdatedProperties().contains("p2"));
+ Assert.assertEquals(1, dataMapArgumentCaptor.getValue().getData().size());
+ Assert.assertEquals("v1", dataMapArgumentCaptor.getValue().getData().get("p1"));
+ }
+
+ @Test
+ public void cacheDataRecord_update_record_merge_previous_CreateChangedInfo() throws URISyntaxException {
+ ArgumentCaptor<RecordChangeInfo> recordInfoArgumentCaptor = ArgumentCaptor.forClass(RecordChangeInfo.class);
+ ArgumentCaptor<RecordData> dataMapArgumentCaptor = ArgumentCaptor.forClass(RecordData.class);
+ String recordId = "anyId";
+ String kind = "anyKind";
+ Map<String, Object> dataMap = new HashMap<>();
+ dataMap.put("p1", "v1");
+ Map<String, Object> previousDataMap = new HashMap<>();
+ previousDataMap.put("p1", "v1");
+ previousDataMap.put("p2", "v2");
+
+ RecordChangeInfo previousChangedInfo = new RecordChangeInfo();
+ RecordInfo recordInfo = new RecordInfo();
+ recordInfo.setId(recordId);
+ recordInfo.setKind(kind);
+ recordInfo.setOp(OperationType.create.getValue());
+ previousChangedInfo.setRecordInfo(recordInfo);
+
+ SearchRecord searchRecord = new SearchRecord();
+ searchRecord.setKind(kind);
+ searchRecord.setId(recordId);
+ searchRecord.setData(previousDataMap);
+ SearchResponse searchResponse = new SearchResponse();
+ searchResponse.setResults(Arrays.asList(searchRecord));
+ searchResponse.setTotalCount(1);
+
+ when(this.searchService.query(any())).thenReturn(searchResponse);
+ when(this.recordChangeInfoCache.get(any())).thenReturn(previousChangedInfo);
+
+ this.sut.cacheDataRecord(recordId, kind, dataMap);
+
+ verify(this.recordChangeInfoCache, times(1)).put(any(), recordInfoArgumentCaptor.capture());
+ verify(this.relatedObjectCache, times(2)).put(any(), dataMapArgumentCaptor.capture());
+
+ RecordChangeInfo changedInfo = recordInfoArgumentCaptor.getValue();
+ Assert.assertEquals(OperationType.create.getValue(), changedInfo.getRecordInfo().getOp());
+ Assert.assertNull(changedInfo.getUpdatedProperties());
+ Assert.assertEquals(1, dataMapArgumentCaptor.getValue().getData().size());
+ Assert.assertEquals("v1", dataMapArgumentCaptor.getValue().getData().get("p1"));
+ }
+
+ @Test
+ public void updateAssociatedRecords_updateAssociatedParentRecords_for_created_childRecord() throws URISyntaxException {
+ updateAssociatedRecords_updateAssociatedParentRecords_for_created_delete(OperationType.create);
+ }
+
+ @Test
+ public void updateAssociatedRecords_updateAssociatedParentRecords_for_deleted_childRecord() throws URISyntaxException {
+ updateAssociatedRecords_updateAssociatedParentRecords_for_created_delete(OperationType.delete);
+ }
+
+ private void updateAssociatedRecords_updateAssociatedParentRecords_for_created_delete(OperationType operationType) throws URISyntaxException {
+ updateAssociatedRecords_updateAssociatedParentRecords_baseSetup();
+
+ // Test
+ RecordChangedMessages recordChangedMessages = new RecordChangedMessages();
+ recordChangedMessages.setAttributes(new HashMap<>());
+ Map<String, List<String>> upsertKindIds = new HashMap<>();
+ Map<String, List<String>> deleteKindIds = new HashMap<>();
+ if(operationType == OperationType.create)
+ upsertKindIds.put(childKind, Arrays.asList(childId));
+ else
+ deleteKindIds.put(childKind, Arrays.asList(childId));
+ this.sut.updateAssociatedRecords(recordChangedMessages, upsertKindIds, deleteKindIds);
+
+ // Verify
+ ArgumentCaptor<String> payloadArgumentCaptor = ArgumentCaptor.forClass(String.class);
+ verify(this.indexerQueueTaskBuilder,times(1)).createWorkerTask(payloadArgumentCaptor.capture(), any(), any());
+
+ RecordChangedMessages newMessages = gson.fromJson(payloadArgumentCaptor.getValue(), RecordChangedMessages.class);
+ Type type = new TypeToken<List<RecordInfo>>() {}.getType();
+ List<RecordInfo> infoList = gson.fromJson(newMessages.getData(), type);
+ Assert.assertEquals(childKind, newMessages.getAttributes().get(Constants.ANCESTRY_KINDS));
+ Assert.assertEquals(1, infoList.size());
+ Assert.assertEquals(parentKind, infoList.get(0).getKind());
+ Assert.assertEquals(parentId, infoList.get(0).getId());
+ }
+
+ @Test
+ public void updateAssociatedRecords_updateAssociatedParentRecords_for_updated_childRecord_with_extendedPropertyChanged() throws URISyntaxException {
+ updateAssociatedRecords_updateAssociatedParentRecords_baseSetup();
+
+ RecordChangeInfo recordChangeInfo = new RecordChangeInfo();
+ RecordInfo recordInfo = new RecordInfo();
+ recordInfo.setKind(childKind);
+ recordInfo.setId(childId);
+ recordInfo.setOp(OperationType.update.getValue());
+ recordChangeInfo.setRecordInfo(recordInfo);
+ recordChangeInfo.setUpdatedProperties(Arrays.asList("Curves[].Mnemonic", "Name"));
+ when(this.recordChangeInfoCache.get(any())).thenReturn(recordChangeInfo);
+
+ // Test
+ RecordChangedMessages recordChangedMessages = new RecordChangedMessages();
+ recordChangedMessages.setAttributes(new HashMap<>());
+ Map<String, List<String>> upsertKindIds = new HashMap<>();
+ Map<String, List<String>> deleteKindIds = new HashMap<>();
+ upsertKindIds.put(childKind, Arrays.asList(childId));
+ this.sut.updateAssociatedRecords(recordChangedMessages, upsertKindIds, deleteKindIds);
+
+ // Verify
+ ArgumentCaptor<String> payloadArgumentCaptor = ArgumentCaptor.forClass(String.class);
+ verify(this.indexerQueueTaskBuilder,times(1)).createWorkerTask(payloadArgumentCaptor.capture(), any(), any());
+
+ RecordChangedMessages newMessages = gson.fromJson(payloadArgumentCaptor.getValue(), RecordChangedMessages.class);
+ Type type = new TypeToken<List<RecordInfo>>() {}.getType();
+ List<RecordInfo> infoList = gson.fromJson(newMessages.getData(), type);
+ Assert.assertEquals(childKind, newMessages.getAttributes().get(Constants.ANCESTRY_KINDS));
+ Assert.assertEquals(1, infoList.size());
+ Assert.assertEquals(parentKind, infoList.get(0).getKind());
+ Assert.assertEquals(parentId, infoList.get(0).getId());
+ }
+
+ @Test
+ public void updateAssociatedRecords_updateAssociatedParentRecords_for_updated_childRecord_without_extendedPropertyChanged() throws URISyntaxException {
+ updateAssociatedRecords_updateAssociatedParentRecords_baseSetup();
+
+ RecordChangeInfo recordChangeInfo = new RecordChangeInfo();
+ RecordInfo recordInfo = new RecordInfo();
+ recordInfo.setKind(childKind);
+ recordInfo.setId(childId);
+ recordInfo.setOp(OperationType.update.getValue());
+ recordChangeInfo.setRecordInfo(recordInfo);
+ recordChangeInfo.setUpdatedProperties(Arrays.asList("Name"));
+ when(this.recordChangeInfoCache.get(any())).thenReturn(recordChangeInfo);
+
+ // Test
+ RecordChangedMessages recordChangedMessages = new RecordChangedMessages();
+ recordChangedMessages.setAttributes(new HashMap<>());
+ Map<String, List<String>> upsertKindIds = new HashMap<>();
+ Map<String, List<String>> deleteKindIds = new HashMap<>();
+ upsertKindIds.put(childKind, Arrays.asList(childId));
+ this.sut.updateAssociatedRecords(recordChangedMessages, upsertKindIds, deleteKindIds);
+
+ // Verify
+ verify(this.indexerQueueTaskBuilder,times(0)).createWorkerTask(any(), any(), any());
+ }
+
+ @Test
+ public void updateAssociatedRecords_updateAssociatedParentRecords_circularIndexing() throws URISyntaxException {
+ updateAssociatedRecords_updateAssociatedParentRecords_baseSetup();
+
+ // Test
+ RecordChangedMessages recordChangedMessages = new RecordChangedMessages();
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(Constants.ANCESTRY_KINDS, parentKind);
+ recordChangedMessages.setAttributes(attributes);
+ Map<String, List<String>> upsertKindIds = new HashMap<>();
+ Map<String, List<String>> deleteKindIds = new HashMap<>();
+ upsertKindIds.put(childKind, Arrays.asList(childId));
+ this.sut.updateAssociatedRecords(recordChangedMessages, upsertKindIds, deleteKindIds);
+
+ // Verify
+ verify(this.indexerQueueTaskBuilder,times(0)).createWorkerTask(any(), any(), any());
+ }
+
+ private void updateAssociatedRecords_updateAssociatedParentRecords_baseSetup() throws URISyntaxException {
+ childKind = "osdu:wks:work-product-component--WellLog:1.0.0";
+ childId = "anyChildId";
+ parentKind = "osdu:wks:master-data--Wellbore:1.0.0";
+ parentId = "anyParentId";
+
+ // Setup search response for searchService.queryWithCursor(...)
+ when(this.searchService.queryWithCursor(any())).thenAnswer(invocation -> {
+ SearchRequest searchRequest = invocation.getArgument(0);
+ SearchResponse searchResponse = new SearchResponse();
+ if (searchRequest.getKind().toString().equals(propertyConfigurationKind)) {
+ if (searchRequest.getQuery().contains("ParentToChildren")) {
+ // Return of getParentChildRelatedObjectsSpecs(...)
+ Map<String, Object> dataMap = getDataMap("wellbore_configuration_record.json");
+ SearchRecord searchRecord = new SearchRecord();
+ searchRecord.setData(dataMap);
+ searchResponse.setResults(Arrays.asList(searchRecord));
+ } else {
+ // search ChildToParent.
+ // NO result
+ }
+ } else {
+ if(searchRequest.getKind().toString().equals(childKind)) {
+ // Return of searchUniqueParentIds(...)
+ SearchRecord searchRecord = new SearchRecord();
+ Map<String, Object> childDataMap = new HashMap<>();
+ childDataMap.put("WellboreID", parentId);
+ searchRecord.setKind(childKind);
+ searchRecord.setData(childDataMap);
+ searchResponse.setResults(Arrays.asList(searchRecord));
+ }
+ else if(searchRequest.getKind().toString().equals("osdu:wks:master-data--Wellbore:1.*")) {
+ // Return of searchKindIds(...)
+ SearchRecord searchRecord = new SearchRecord();
+ searchRecord.setKind(parentKind);
+ searchRecord.setId(parentId);
+ searchResponse.setResults(Arrays.asList(searchRecord));
+ }
+ }
+ return searchResponse;
+ });
+
+ // setup headers
+ DpsHeaders dpsHeaders = new DpsHeaders();
+ dpsHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
+ dpsHeaders.put(DpsHeaders.DATA_PARTITION_ID, "opendes");
+ dpsHeaders.put(DpsHeaders.CORRELATION_ID, "123");
+ when(this.requestInfo.getHeadersWithDwdAuthZ()).thenReturn(dpsHeaders);
+ }
+
+ @Test
+ public void updateAssociatedRecords_updateAssociatedChildrenRecords_for_created_parentRecord() throws URISyntaxException {
+ updateAssociatedRecords_updateAssociatedChildrenRecords_for_created_delete(OperationType.create);
+ }
+
+ @Test
+ public void updateAssociatedRecords_updateAssociatedChildrenRecords_for_deleted_parentRecord() throws URISyntaxException {
+ updateAssociatedRecords_updateAssociatedChildrenRecords_for_created_delete(OperationType.delete);
+ }
+
+ private void updateAssociatedRecords_updateAssociatedChildrenRecords_for_created_delete(OperationType op) throws URISyntaxException {
+ updateAssociatedRecords_updateAssociatedChildrenRecords_baseSetup();
+
+ // Test
+ RecordChangedMessages recordChangedMessages = new RecordChangedMessages();
+ recordChangedMessages.setAttributes(new HashMap<>());
+ Map<String, List<String>> upsertKindIds = new HashMap<>();
+ Map<String, List<String>> deleteKindIds = new HashMap<>();
+ if(op == OperationType.create)
+ upsertKindIds.put(parentKind, Arrays.asList(parentId));
+ else
+ deleteKindIds.put(parentKind, Arrays.asList(parentId));
+ this.sut.updateAssociatedRecords(recordChangedMessages, upsertKindIds, deleteKindIds);
+
+ // Verify
+ ArgumentCaptor<String> payloadArgumentCaptor = ArgumentCaptor.forClass(String.class);
+ verify(this.indexerQueueTaskBuilder,times(1)).createWorkerTask(payloadArgumentCaptor.capture(), any(), any());
+
+ RecordChangedMessages newMessages = gson.fromJson(payloadArgumentCaptor.getValue(), RecordChangedMessages.class);
+ Type type = new TypeToken<List<RecordInfo>>() {}.getType();
+ List<RecordInfo> infoList = gson.fromJson(newMessages.getData(), type);
+ Assert.assertEquals(parentKind, newMessages.getAttributes().get(Constants.ANCESTRY_KINDS));
+ Assert.assertEquals(1, infoList.size());
+ Assert.assertEquals(childKind, infoList.get(0).getKind());
+ Assert.assertEquals(childId, infoList.get(0).getId());
+ }
+
+ @Test
+ public void updateAssociatedRecords_updateAssociatedChildrenRecords_for_updated_parentRecord_with_extendedPropertyChanged() throws URISyntaxException {
+ updateAssociatedRecords_updateAssociatedChildrenRecords_baseSetup();
+
+ RecordChangeInfo recordChangeInfo = new RecordChangeInfo();
+ RecordInfo recordInfo = new RecordInfo();
+ recordInfo.setKind(parentKind);
+ recordInfo.setId(parentId);
+ recordInfo.setOp(OperationType.update.getValue());
+ recordChangeInfo.setRecordInfo(recordInfo);
+ recordChangeInfo.setUpdatedProperties(Arrays.asList("GeoPoliticalEntityName"));
+ when(this.recordChangeInfoCache.get(any())).thenReturn(recordChangeInfo);
+
+ // Test
+ RecordChangedMessages recordChangedMessages = new RecordChangedMessages();
+ recordChangedMessages.setAttributes(new HashMap<>());
+ Map<String, List<String>> upsertKindIds = new HashMap<>();
+ Map<String, List<String>> deleteKindIds = new HashMap<>();
+ upsertKindIds.put(parentKind, Arrays.asList(parentId));
+ this.sut.updateAssociatedRecords(recordChangedMessages, upsertKindIds, deleteKindIds);
+
+ // Verify
+ ArgumentCaptor<String> payloadArgumentCaptor = ArgumentCaptor.forClass(String.class);
+ verify(this.indexerQueueTaskBuilder,times(1)).createWorkerTask(payloadArgumentCaptor.capture(), any(), any());
+
+ RecordChangedMessages newMessages = gson.fromJson(payloadArgumentCaptor.getValue(), RecordChangedMessages.class);
+ Type type = new TypeToken<List<RecordInfo>>() {}.getType();
+ List<RecordInfo> infoList = gson.fromJson(newMessages.getData(), type);
+ Assert.assertEquals(parentKind, newMessages.getAttributes().get(Constants.ANCESTRY_KINDS));
+ Assert.assertEquals(1, infoList.size());
+ Assert.assertEquals(childKind, infoList.get(0).getKind());
+ Assert.assertEquals(childId, infoList.get(0).getId());
+ }
+
+ @Test
+ public void updateAssociatedRecords_updateAssociatedChildrenRecords_for_updated_parentRecord_without_extendedPropertyChanged() throws URISyntaxException {
+ updateAssociatedRecords_updateAssociatedChildrenRecords_baseSetup();
+
+ RecordChangeInfo recordChangeInfo = new RecordChangeInfo();
+ RecordInfo recordInfo = new RecordInfo();
+ recordInfo.setKind(parentKind);
+ recordInfo.setId(parentId);
+ recordInfo.setOp(OperationType.update.getValue());
+ recordChangeInfo.setRecordInfo(recordInfo);
+ recordChangeInfo.setUpdatedProperties(Arrays.asList("abc"));
+ when(this.recordChangeInfoCache.get(any())).thenReturn(recordChangeInfo);
+
+ // Test
+ RecordChangedMessages recordChangedMessages = new RecordChangedMessages();
+ recordChangedMessages.setAttributes(new HashMap<>());
+ Map<String, List<String>> upsertKindIds = new HashMap<>();
+ Map<String, List<String>> deleteKindIds = new HashMap<>();
+ upsertKindIds.put(parentKind, Arrays.asList(parentId));
+ this.sut.updateAssociatedRecords(recordChangedMessages, upsertKindIds, deleteKindIds);
+
+ // Verify
+ verify(this.indexerQueueTaskBuilder,times(0)).createWorkerTask(any(), any(), any());
+ }
+
+ @Test
+ public void updateAssociatedRecords_updateAssociatedChildrenRecords_circularIndexing() throws URISyntaxException {
+ updateAssociatedRecords_updateAssociatedChildrenRecords_baseSetup();
+
+ // Test
+ RecordChangedMessages recordChangedMessages = new RecordChangedMessages();
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(Constants.ANCESTRY_KINDS, childKind);
+ recordChangedMessages.setAttributes(attributes);
+ Map<String, List<String>> upsertKindIds = new HashMap<>();
+ Map<String, List<String>> deleteKindIds = new HashMap<>();
+ upsertKindIds.put(parentKind, Arrays.asList(parentId));
+ this.sut.updateAssociatedRecords(recordChangedMessages, upsertKindIds, deleteKindIds);
+
+ // Verify
+ verify(this.indexerQueueTaskBuilder,times(0)).createWorkerTask(any(), any(), any());
+ }
+
+ private void updateAssociatedRecords_updateAssociatedChildrenRecords_baseSetup() throws URISyntaxException {
+ childKind = "osdu:wks:master-data--Well:1.0.0";
+ childId = "anyChildId";
+ parentKind = "osdu:wks:master-data--GeoPoliticalEntity:1.0.0";
+ parentId = "anyParentId";
+
+ // Setup search response for searchService.queryWithCursor(...)
+ when(this.searchService.queryWithCursor(any())).thenAnswer(invocation -> {
+ SearchRequest searchRequest = invocation.getArgument(0);
+ SearchResponse searchResponse = new SearchResponse();
+ if (searchRequest.getKind().toString().equals(propertyConfigurationKind)) {
+ if (searchRequest.getQuery().contains("ChildToParent") || searchRequest.getQuery().contains("data.Code:")) {
+ // Return of getParentChildRelatedObjectsSpecs(...) or
+ // getPropertyConfigurations(...)
+ Map<String, Object> dataMap = getDataMap("well_configuration_record.json");
+ SearchRecord searchRecord = new SearchRecord();
+ searchRecord.setData(dataMap);
+ searchResponse.setResults(Arrays.asList(searchRecord));
+ } else {
+ // Search ParentToChildren
+ // No result
+ }
+ } else {
+ if(searchRequest.getKind().toString().contains("osdu:wks:master-data--Well:1.")) {
+ // Return of searchUniqueParentIds(...)
+ SearchRecord searchRecord = new SearchRecord();
+ Map<String, Object> childDataMap = new HashMap<>();
+ childDataMap.put("AssociatedIdentities", Arrays.asList(parentId));
+ searchRecord.setKind(childKind);
+ searchRecord.setId(childId);
+ searchRecord.setData(childDataMap);
+ searchResponse.setResults(Arrays.asList(searchRecord));
+ }
+ else {
+ // This branch is a setup for test case:
+ // updateAssociatedRecords_updateAssociatedChildrenRecords_circularIndexing
+ throw new Exception("Unexpected search");
+ }
+ }
+ return searchResponse;
+ });
+
+ // setup headers
+ DpsHeaders dpsHeaders = new DpsHeaders();
+ dpsHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
+ dpsHeaders.put(DpsHeaders.DATA_PARTITION_ID, "opendes");
+ dpsHeaders.put(DpsHeaders.CORRELATION_ID, "123");
+ when(this.requestInfo.getHeadersWithDwdAuthZ()).thenReturn(dpsHeaders);
+ }
+
+ private Schema getSchema(String file) {
+ String jsonText = getJsonFromFile(file);
+ return gson.fromJson(jsonText, Schema.class);
+ }
+
+ private PropertyConfigurations getConfigurations(String file) throws JsonProcessingException {
+ Map<String, Object> dataMap = getDataMap(file);
+ ObjectMapper objectMapper = new ObjectMapper();
+ String data = objectMapper.writeValueAsString(dataMap);
+ PropertyConfigurations configurations = objectMapper.readValue(data, PropertyConfigurations.class);
+ return configurations;
+ }
+
+ private Map<String, Object> getDataMap(String file) {
+ String jsonText = getJsonFromFile(file);
+ Type type = new TypeToken<Map<String, Object>>() {}.getType();
+ return gson.fromJson(jsonText, type);
+ }
+
+ @SneakyThrows
+ private String getJsonFromFile(String file) {
+ InputStream inStream = this.getClass().getResourceAsStream("/indexproperty/" + file);
+ BufferedReader br = new BufferedReader(new InputStreamReader(inStream));
+ StringBuilder stringBuilder = new StringBuilder();
+ String sCurrentLine;
+ while ((sCurrentLine = br.readLine()) != null)
+ {
+ stringBuilder.append(sCurrentLine).append("\n");
+ }
+ return stringBuilder.toString();
+ }
+}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/SchemaProviderImplTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/SchemaProviderImplTest.java
index fe91e5ffbca0e444ee037753a66341fe31d56cca..68a35ad4494d33acadfc7c11bbf634ce78aab17d 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/SchemaProviderImplTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/SchemaProviderImplTest.java
@@ -14,41 +14,36 @@
package org.opengroup.osdu.indexer.service;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.inOrder;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.powermock.api.mockito.PowerMockito.when;
-
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
-import java.io.UnsupportedEncodingException;
-import java.net.URISyntaxException;
-import java.util.Map;
import org.apache.http.HttpStatus;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
-import org.mockito.InOrder;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.mockito.Mockito;
-import org.mockito.Spy;
+import org.mockito.*;
+import org.opengroup.osdu.core.common.http.FetchServiceHttpRequest;
import org.opengroup.osdu.core.common.http.IUrlFetchService;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.HttpResponse;
import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
import org.opengroup.osdu.indexer.logging.AuditLogger;
+import org.opengroup.osdu.indexer.model.SchemaInfoResponse;
import org.opengroup.osdu.indexer.schema.converter.SchemaToStorageFormatImpl;
import org.powermock.api.mockito.PowerMockito;
import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder;
import org.springframework.test.context.junit4.SpringRunner;
+import java.io.UnsupportedEncodingException;
+import java.net.URISyntaxException;
+import java.util.Map;
+
+import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.*;
+import static org.powermock.api.mockito.PowerMockito.when;
+
@RunWith(SpringRunner.class)
public class SchemaProviderImplTest {
@@ -167,4 +162,74 @@ public class SchemaProviderImplTest {
verify(schemaService, times(0)).getFromStorageService(any());
}
+ @Test
+ public void getSchemaInfos() throws URISyntaxException, UnsupportedEncodingException {
+ HttpResponse httpResponse = createSchemaInfoResponse();
+ PowerMockito.when(this.urlFetchService.sendRequest(any())).thenReturn(httpResponse);
+
+ SchemaInfoResponse schemaInfoResponse = sut.getSchemaInfos("osdu", "wks", "master-data--Wellbore", "1", null, null, true);
+ assertEquals(1, schemaInfoResponse.getCount());
+ assertEquals(1, schemaInfoResponse.getSchemaInfos().size());
+ assertEquals("osdu:wks:master-data--Wellbore:1.3.0", schemaInfoResponse.getSchemaInfos().get(0).getSchemaIdentity().getId());
+ }
+
+ @Test
+ public void getSchemaInfos_latestSchemaInfo_url() throws URISyntaxException, UnsupportedEncodingException {
+ String schemaHost = "http://localhost/api/schema-service/v1/schema";
+ ArgumentCaptor<FetchServiceHttpRequest> argumentCaptor = ArgumentCaptor.forClass(FetchServiceHttpRequest.class);
+ HttpResponse httpResponse = createSchemaInfoResponse();
+ PowerMockito.when(this.configurationProperties.getSchemaHost()).thenReturn(schemaHost);
+ PowerMockito.when(this.urlFetchService.sendRequest(any())).thenReturn(httpResponse);
+
+ sut.getSchemaInfos("osdu", "wks", "master-data--Wellbore", "1", null, null, true);
+ verify(this.urlFetchService).sendRequest(argumentCaptor.capture());
+ FetchServiceHttpRequest request = argumentCaptor.getValue();
+ String url = request.getUrl();
+ String expectedUrl = "http://localhost/api/schema-service/v1/schema?authority=osdu&source=wks&entityType=master-data--Wellbore&schemaVersionMajor=1&latestVersion=true&limit=1000";
+ assertEquals(expectedUrl, url);
+ }
+
+ @Test
+ public void getSchemaInfos_allSchemaInfo_url() throws URISyntaxException, UnsupportedEncodingException {
+ String schemaHost = "http://localhost/api/schema-service/v1/schema";
+ ArgumentCaptor<FetchServiceHttpRequest> argumentCaptor = ArgumentCaptor.forClass(FetchServiceHttpRequest.class);
+ HttpResponse httpResponse = createSchemaInfoResponse();
+ PowerMockito.when(this.configurationProperties.getSchemaHost()).thenReturn(schemaHost);
+ PowerMockito.when(this.urlFetchService.sendRequest(any())).thenReturn(httpResponse);
+
+ sut.getSchemaInfos("osdu", "wks", "master-data--Wellbore", "1", "2", null, false);
+ verify(this.urlFetchService).sendRequest(argumentCaptor.capture());
+ FetchServiceHttpRequest request = argumentCaptor.getValue();
+ String url = request.getUrl();
+ String expectedUrl = "http://localhost/api/schema-service/v1/schema?authority=osdu&source=wks&entityType=master-data--Wellbore&schemaVersionMajor=1&schemaVersionMinor=2&latestVersion=false&limit=1000";
+ assertEquals(expectedUrl, url);
+ }
+
+ private HttpResponse createSchemaInfoResponse() {
+ String schemaInfos = "{\n" +
+ " \"schemaInfos\": [{\n" +
+ " \"schemaIdentity\": {\n" +
+ " \"authority\": \"osdu\",\n" +
+ " \"source\": \"wks\",\n" +
+ " \"entityType\": \"master-data--Wellbore\",\n" +
+ " \"schemaVersionMajor\": 1,\n" +
+ " \"schemaVersionMinor\": 3,\n" +
+ " \"schemaVersionPatch\": 0,\n" +
+ " \"id\": \"osdu:wks:master-data--Wellbore:1.3.0\"\n" +
+ " },\n" +
+ " \"createdBy\": \"ServiceAdminUser\",\n" +
+ " \"dateCreated\": \"2023-03-27T12:49:13.822+00:00\",\n" +
+ " \"status\": \"PUBLISHED\",\n" +
+ " \"scope\": \"SHARED\"\n" +
+ " }\n" +
+ " ],\n" +
+ " \"offset\": 0,\n" +
+ " \"count\": 1,\n" +
+ " \"totalCount\": 1\n" +
+ "}";
+ HttpResponse httpResponse = new HttpResponse();
+ httpResponse.setBody(schemaInfos);
+ return httpResponse;
+ }
+
}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/SearchServiceImplTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/SearchServiceImplTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..93ee08260785537a679529039d0966dbbee88fb7
--- /dev/null
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/SearchServiceImplTest.java
@@ -0,0 +1,156 @@
+package org.opengroup.osdu.indexer.service;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.opengroup.osdu.core.common.http.IUrlFetchService;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.http.HttpResponse;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.opengroup.osdu.indexer.model.SearchRequest;
+import org.opengroup.osdu.indexer.model.SearchResponse;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.ArgumentMatchers.any;
+
+import java.net.URISyntaxException;
+
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@RunWith(PowerMockRunner.class)
+public class SearchServiceImplTest {
+ @InjectMocks
+ private SearchServiceImpl sut;
+
+ @Mock
+ private IUrlFetchService urlFetchService;
+ @Mock
+ private IRequestInfo requestInfo;
+ @Mock
+ private IndexerConfigurationProperties configurationProperties;
+ @Mock
+ private JaxRsDpsLog jaxRsDpsLog;
+
+ private String searchHost = "http://localhost";
+
+ @Test
+ public void query_without_searchHostSetting() throws URISyntaxException {
+ when(this.configurationProperties.getSearchHost()).thenReturn(null);
+ SearchResponse response = sut.query(new SearchRequest());
+ Assert.assertNotNull(response);
+ Assert.assertNull(response.getResults());
+ }
+
+ @Test
+ public void query_with_responseCode_OK() throws URISyntaxException {
+ String bodyJson = "{\n" +
+ " \"results\": [\n" +
+ " {\n" +
+ " \"data\": {\n" +
+ " \"FacilityName\": \"A123\"\n" +
+ " },\n" +
+ " \"kind\": \"osdu:wks:master-data--Wellbore:1.0.0\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"data\": {\n" +
+ " \"FacilityName\": \"B123\"\n" +
+ " },\n" +
+ " \"kind\": \"osdu:wks:master-data--Wellbore:1.0.0\"\n" +
+ " }\n" +
+ " ],\n" +
+ " \"aggregations\": null,\n" +
+ " \"totalCount\": 10000\n" +
+ "}";
+ HttpResponse response = new HttpResponse();
+ response.setResponseCode(200);
+ response.setBody(bodyJson);
+ when(this.configurationProperties.getSearchHost()).thenReturn(searchHost);
+ when(this.urlFetchService.sendRequest(any())).thenReturn(response);
+ SearchResponse searchResponse = sut.query(new SearchRequest());
+ Assert.assertNotNull(searchResponse);
+ Assert.assertEquals(10000,searchResponse.getTotalCount());
+ Assert.assertEquals(2,searchResponse.getResults().size());
+ }
+
+ @Test
+ public void query_with_cursor_with_responseCode_OK() throws URISyntaxException {
+ String bodyJson = "{\n" +
+ " \"cursor\": \"509E144E7F9B81F8148327D6CB73BB6F\",\n" +
+ " \"results\": [\n" +
+ " {\n" +
+ " \"kind\": \"osdu:wks:master-data--Wellbore:1.0.0\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"kind\": \"osdu:wks:master-data--Wellbore:1.0.1\"\n" +
+ " }\n" +
+ " ],\n" +
+ " \"totalCount\": 1000\n" +
+ "}";
+ HttpResponse response = new HttpResponse();
+ response.setResponseCode(200);
+ response.setBody(bodyJson);
+ when(this.configurationProperties.getSearchHost()).thenReturn(searchHost);
+ when(this.urlFetchService.sendRequest(any())).thenReturn(response);
+ SearchResponse searchResponse = sut.query(new SearchRequest());
+ Assert.assertNotNull(searchResponse);
+ Assert.assertNotNull(searchResponse.getCursor());
+ Assert.assertEquals(1000,searchResponse.getTotalCount());
+ Assert.assertEquals(2,searchResponse.getResults().size());
+ }
+
+ @Test
+ public void query_with_responseCode_OK_EmptyResult() throws URISyntaxException {
+ String bodyJson = "{\n" +
+ " \"results\": [],\n" +
+ " \"aggregations\": [],\n" +
+ " \"totalCount\": 0\n" +
+ "}";
+ HttpResponse response = new HttpResponse();
+ response.setResponseCode(200);
+ response.setBody(bodyJson);
+ when(this.configurationProperties.getSearchHost()).thenReturn(searchHost);
+ when(this.urlFetchService.sendRequest(any())).thenReturn(response);
+ SearchResponse searchResponse = sut.query(new SearchRequest());
+ Assert.assertNotNull(searchResponse);
+ Assert.assertEquals(0,searchResponse.getTotalCount());
+ Assert.assertEquals(0,searchResponse.getResults().size());
+ }
+
+ @Test
+ public void query_with_responseCode_BadRequest() throws URISyntaxException {
+ String bodyJson = "{\n" +
+ " \"code\": 400,\n" +
+ " \"reason\": \"Bad Request\",\n" +
+ " \"message\": \"Invalid parameters were given on search request\"\n" +
+ "}";
+ HttpResponse response = new HttpResponse();
+ response.setResponseCode(200);
+ response.setBody(bodyJson);
+ when(this.configurationProperties.getSearchHost()).thenReturn(searchHost);
+ when(this.urlFetchService.sendRequest(any())).thenReturn(response);
+ SearchResponse searchResponse = sut.query(new SearchRequest());
+ Assert.assertNotNull(searchResponse);
+ Assert.assertNull(searchResponse.getResults());
+ }
+
+ @Test
+ public void query_with_null_response() throws URISyntaxException {
+ when(this.configurationProperties.getSearchHost()).thenReturn(searchHost);
+ when(this.urlFetchService.sendRequest(any())).thenReturn(null);
+ SearchResponse searchResponse = sut.query(new SearchRequest());
+ Assert.assertNotNull(searchResponse);
+ Assert.assertNull(searchResponse.getResults());
+ }
+
+ @Test
+ public void query_with_exception() throws URISyntaxException {
+ when(this.configurationProperties.getSearchHost()).thenReturn(searchHost);
+ when(this.urlFetchService.sendRequest(any())).thenThrow(new AppException(415, "upstream server responded with unsupported media type: text/plain", "Unsupported media type" ));
+ assertThrows(URISyntaxException.class, () -> sut.query(new SearchRequest()));
+ }
+}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapperTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapperTest.java
index a8ede574da14ec07ad905907c98d5854e132a6e0..b56339a96f294e8c8035d7844354cf885a7f1f2d 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapperTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageIndexerPayloadMapperTest.java
@@ -12,13 +12,12 @@ import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
import org.opengroup.osdu.core.common.model.indexer.JobStatus;
+import org.opengroup.osdu.indexer.cache.FeatureFlagCache;
import org.opengroup.osdu.indexer.schema.converter.config.SchemaConverterPropertiesConfig;
import org.opengroup.osdu.indexer.schema.converter.exeption.SchemaProcessingException;
import org.opengroup.osdu.indexer.schema.converter.interfaces.IVirtualPropertiesSchemaCache;
import org.opengroup.osdu.indexer.schema.converter.tags.SchemaRoot;
import org.opengroup.osdu.indexer.schema.converter.tags.VirtualProperties;
-import org.opengroup.osdu.indexer.service.mock.PartitionFactoryMock;
-import org.opengroup.osdu.indexer.service.mock.PartitionProviderMock;
import org.opengroup.osdu.indexer.service.mock.ServiceAccountJwtClientMock;
import org.opengroup.osdu.indexer.service.mock.VirtualPropertiesSchemaCacheMock;
import org.opengroup.osdu.indexer.util.geo.decimator.*;
@@ -31,7 +30,6 @@ import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder;
import org.springframework.test.context.junit4.SpringRunner;
-import javax.inject.Inject;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
@@ -45,9 +43,9 @@ import static org.junit.Assert.*;
@RunWith(SpringRunner.class)
@SpringBootTest(classes = {StorageIndexerPayloadMapper.class, AttributeParsingServiceImpl.class, NumberParser.class,
BooleanParser.class, DateTimeParser.class, GeoShapeParser.class, DouglasPeuckerReducer.class, GeoShapeDecimator.class,
- GeometryDecimator.class, GeometryConversionService.class, DecimationSettingCache.class,
- GeoShapeDecimationSetting.class, DpsHeaders.class, JobStatus.class, SchemaConverterPropertiesConfig.class, JaxRsDpsLog.class,
- PartitionFactoryMock.class, PartitionProviderMock.class, ServiceAccountJwtClientMock.class, VirtualPropertiesSchemaCacheMock.class, })
+ GeometryDecimator.class, GeometryConversionService.class, FeatureFlagCache.class,
+ DpsHeaders.class, JobStatus.class, SchemaConverterPropertiesConfig.class, JaxRsDpsLog.class,
+ ServiceAccountJwtClientMock.class, VirtualPropertiesSchemaCacheMock.class, })
public class StorageIndexerPayloadMapperTest {
public static final String FIRST_OBJECT_INNER_PROPERTY = "FirstObjectInnerProperty";
@@ -77,9 +75,6 @@ public class StorageIndexerPayloadMapperTest {
@Autowired
private IVirtualPropertiesSchemaCache virtualPropertiesSchemaCache;
- @Inject
- private GeoShapeDecimationSetting decimationSetting;
-
@BeforeClass
public static void setUp() {
HashMap<String, Object> dataMap = new HashMap<>();
@@ -205,11 +200,6 @@ public class StorageIndexerPayloadMapperTest {
assertNull(dataCollectorMap.get("VirtualProperties.DefaultName"));
}
- @Test
- public void geoshape_decimation_is_enabled_by_default() {
- assertTrue(decimationSetting.isDecimationEnabled());
- }
-
@Test
public void geoshape_decimation_is_executed_with_virtual_spatial_location() {
final String kind = "osdu:wks:master-data--SeismicAcquisitionSurvey:1.0.0";
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceImplTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceImplTest.java
index e64574d7a6452f704361f80ea66884232b506c0a..437f8714f53d299f09f6da9d80e5c3169edd77e6 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceImplTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceImplTest.java
@@ -24,6 +24,7 @@ import org.mockito.ArgumentMatchers;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Spy;
+import org.opengroup.osdu.core.common.http.IHttpClientHandler;
import org.opengroup.osdu.core.common.http.IUrlFetchService;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
@@ -43,11 +44,7 @@ import org.springframework.test.context.junit4.SpringRunner;
import java.lang.reflect.Type;
import java.net.URISyntaxException;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import static java.util.Collections.singletonList;
import static org.junit.Assert.assertEquals;
@@ -64,6 +61,8 @@ public class StorageServiceImplTest {
@Mock
private IUrlFetchService urlFetchService;
@Mock
+ private IHttpClientHandler httpClientHandler;
+ @Mock
private JobStatus jobStatus;
@Mock
private JaxRsDpsLog log;
@@ -188,6 +187,24 @@ public class StorageServiceImplTest {
verify(this.log).warning("stale records found with older kind, skipping indexing | record ids: testid");
}
+ @Test
+ public void should_returnStorageRecords_givenRecordIds_getValidStorageRecordsTest() throws URISyntaxException {
+
+ String validDataFromStorage = "{\"records\":[{\"id\":\"tenant1:doc:1dbf528e0e0549cab7a08f29fbfc8465\", \"version\":1, \"kind\":\"tenant:test:test:1.0.0\"}]}";
+
+ HttpResponse httpResponse = mock(HttpResponse.class);
+ when(httpResponse.getBody()).thenReturn(validDataFromStorage);
+
+ when(configurationProperties.getStorageQueryRecordHost()).thenReturn("storageUrl");
+ when(this.httpClientHandler.sendRequest(any(), any())).thenReturn(httpResponse);
+ List<String> idsCopy = new ArrayList<>();
+ idsCopy.addAll(ids);
+ Records storageRecords = this.sut.getStorageRecords(idsCopy);
+
+ assertEquals(1, storageRecords.getRecords().size());
+ assertEquals(1, storageRecords.getNotFound().size());
+ }
+
@Test
public void should_logMissingRecord_given_storageMissedRecords() throws URISyntaxException {
@@ -310,4 +327,4 @@ public class StorageServiceImplTest {
fail("Should not throw this exception" + e.getMessage());
}
}
-}
\ No newline at end of file
+}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/mock/BucketMock.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/mock/BucketMock.java
new file mode 100644
index 0000000000000000000000000000000000000000..df778acd8c8a4213bd5acae54e2d1155e1c55698
--- /dev/null
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/mock/BucketMock.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.service.mock;
+
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.aggregations.bucket.terms.Terms;
+
+import java.io.IOException;
+
+public class BucketMock implements Terms.Bucket {
+ @Override
+ public Number getKeyAsNumber() {
+ return null;
+ }
+
+ @Override
+ public long getDocCountError() {
+ return 0;
+ }
+
+ @Override
+ public Object getKey() {
+ return null;
+ }
+
+ @Override
+ public String getKeyAsString() {
+ return null;
+ }
+
+ @Override
+ public long getDocCount() {
+ return 0;
+ }
+
+ @Override
+ public Aggregations getAggregations() {
+ return null;
+ }
+
+ @Override
+ public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException {
+ return null;
+ }
+}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/mock/PartitionProviderMock.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/mock/PartitionProviderMock.java
deleted file mode 100644
index 55767e8fd7841b2e1e9063fc240dab0cae6b8f22..0000000000000000000000000000000000000000
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/mock/PartitionProviderMock.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Copyright © Schlumberger
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0 *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.service.mock;
-
-import org.opengroup.osdu.core.common.partition.IPartitionProvider;
-import org.opengroup.osdu.core.common.partition.PartitionException;
-import org.opengroup.osdu.core.common.partition.PartitionInfo;
-import org.opengroup.osdu.core.common.partition.Property;
-
-import java.util.List;
-
-public class PartitionProviderMock implements IPartitionProvider {
- private static final String PROPERTY_NAME = "indexer-decimation-enabled";
-
- @Override
- public PartitionInfo get(String s) throws PartitionException {
- PartitionInfo partitionInfo = new PartitionInfo();
- Property property = new Property();
- property.setSensitive(false);
- property.setValue("true");
- partitionInfo.getProperties().put(PROPERTY_NAME, property);
- return partitionInfo;
- }
-
- @Override
- public PartitionInfo create(String s, PartitionInfo partitionInfo) throws PartitionException {
- return null;
- }
-
- @Override
- public void update(String s, PartitionInfo partitionInfo) throws PartitionException {
-
- }
-
- @Override
- public void delete(String s) throws PartitionException {
-
- }
-
- @Override
- public List<String> list() throws PartitionException {
- return null;
- }
-}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/mock/TermMock.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/mock/TermMock.java
new file mode 100644
index 0000000000000000000000000000000000000000..c0250cf4dee80b5224f6df8014090fc90abc491b
--- /dev/null
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/mock/TermMock.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.service.mock;
+
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.search.aggregations.bucket.terms.Terms;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+public class TermMock implements Terms {
+ @Override
+ public List<BucketMock> getBuckets() {
+ return null;
+ }
+
+ @Override
+ public Bucket getBucketByKey(String s) {
+ return null;
+ }
+
+ @Override
+ public long getDocCountError() {
+ return 0;
+ }
+
+ @Override
+ public long getSumOfOtherDocCounts() {
+ return 0;
+ }
+
+ @Override
+ public String getName() {
+ return null;
+ }
+
+ @Override
+ public String getType() {
+ return null;
+ }
+
+ @Override
+ public Map<String, Object> getMetadata() {
+ return null;
+ }
+
+ @Override
+ public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException {
+ return null;
+ }
+}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/geo/decimator/GeoShapeDecimationSettingTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/BooleanFeatureFlagClientTest.java
similarity index 73%
rename from indexer-core/src/test/java/org/opengroup/osdu/indexer/util/geo/decimator/GeoShapeDecimationSettingTest.java
rename to indexer-core/src/test/java/org/opengroup/osdu/indexer/util/BooleanFeatureFlagClientTest.java
index 82fc83db7ccd0a520eabb90ea3a0f414ca0338c2..8b5fb420f4e1269ddcb0bd255a535a172869056d 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/geo/decimator/GeoShapeDecimationSettingTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/BooleanFeatureFlagClientTest.java
@@ -13,7 +13,7 @@
* limitations under the License.
*/
-package org.opengroup.osdu.indexer.util.geo.decimator;
+package org.opengroup.osdu.indexer.util;
import org.junit.Assert;
import org.junit.Before;
@@ -25,6 +25,7 @@ import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.partition.*;
import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient;
+import org.opengroup.osdu.indexer.cache.FeatureFlagCache;
import org.springframework.test.context.junit4.SpringRunner;
import java.util.HashMap;
@@ -34,14 +35,14 @@ import static org.mockito.ArgumentMatchers.anyString;
import static org.powermock.api.mockito.PowerMockito.when;
@RunWith(SpringRunner.class)
-public class GeoShapeDecimationSettingTest {
+public class BooleanFeatureFlagClientTest {
private static final String PROPERTY_NAME = "indexer-decimation-enabled";
@InjectMocks
- private GeoShapeDecimationSetting sut;
+ private BooleanFeatureFlagClient sut;
@Mock
- private DecimationSettingCache cache;
+ private FeatureFlagCache cache;
@Mock
private JaxRsDpsLog logger;
@@ -74,7 +75,12 @@ public class GeoShapeDecimationSettingTest {
property.setValue("true");
partitionInfo.getProperties().put(PROPERTY_NAME, property);
when(this.partitionProvider.get(anyString())).thenReturn(partitionInfo);
- boolean enabled = sut.isDecimationEnabled();
+
+ // Default value won't take any effect
+ boolean enabled = sut.isEnabled(PROPERTY_NAME, true);
+ Assert.assertTrue(enabled);
+
+ enabled = sut.isEnabled(PROPERTY_NAME, false);
Assert.assertTrue(enabled);
}
@@ -86,25 +92,36 @@ public class GeoShapeDecimationSettingTest {
property.setValue("false");
partitionInfo.getProperties().put(PROPERTY_NAME, property);
when(this.partitionProvider.get(anyString())).thenReturn(partitionInfo);
- boolean enabled = sut.isDecimationEnabled();
+
+ // Default value won't take any effect
+ boolean enabled = sut.isEnabled(PROPERTY_NAME, true);
+ Assert.assertFalse(enabled);
+
+ enabled = sut.isEnabled(PROPERTY_NAME, false);
Assert.assertFalse(enabled);
}
@Test
- public void isDecimationEnabled_return_true_when_property_does_not_exist() throws PartitionException {
+ public void isDecimationEnabled_return_default_value_when_property_does_not_exist() throws PartitionException {
// The feature flag is enabled by default
PartitionInfo partitionInfo = new PartitionInfo();
when(this.partitionProvider.get(anyString())).thenReturn(partitionInfo);
- boolean enabled = sut.isDecimationEnabled();
+ boolean enabled = sut.isEnabled(PROPERTY_NAME, true);;
Assert.assertTrue(enabled);
+
+ enabled = sut.isEnabled(PROPERTY_NAME, false);;
+ Assert.assertFalse(enabled);
}
@Test
- public void isDecimationEnabled_return_true_when_partitionProvider_throws_exception() throws PartitionException {
+ public void isDecimationEnabled_return_default_value_when_partitionProvider_throws_exception() throws PartitionException {
// The feature flag is enabled by default
when(this.partitionProvider.get(anyString())).thenThrow(PartitionException.class);
- boolean enabled = sut.isDecimationEnabled();
+ boolean enabled = sut.isEnabled(PROPERTY_NAME, true);;
Assert.assertTrue(enabled);
+
+ enabled = sut.isEnabled(PROPERTY_NAME, false);;
+ Assert.assertFalse(enabled);
}
}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/PropertyUtilTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/PropertyUtilTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..683953925ea222178228f6ed4b4b348148b9b476
--- /dev/null
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/PropertyUtilTest.java
@@ -0,0 +1,294 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.util;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import lombok.SneakyThrows;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.lang.reflect.Type;
+import java.util.*;
+
+@RunWith(SpringRunner.class)
+public class PropertyUtilTest {
+ private static final Gson gson = new Gson();
+
+ @Test
+ public void isPropertyPathMatched() {
+ Assert.assertTrue(PropertyUtil.isPropertyPathMatched("data.FacilityName", "data.FacilityName"));
+ Assert.assertTrue(PropertyUtil.isPropertyPathMatched("data.ProjectedBottomHoleLocation.Wgs84Coordinates", "data.ProjectedBottomHoleLocation"));
+
+ Assert.assertFalse(PropertyUtil.isPropertyPathMatched("data.FacilityName", "data.FacilityNameAliase"));
+ Assert.assertFalse(PropertyUtil.isPropertyPathMatched("data.ProjectedBottomHoleLocation.Wgs84Coordinates", "data.ProjectedBottomHole"));
+ Assert.assertFalse(PropertyUtil.isPropertyPathMatched("", "data.ProjectedBottomHole"));
+ Assert.assertFalse(PropertyUtil.isPropertyPathMatched(null, "data.ProjectedBottomHole"));
+ }
+
+ @Test
+ public void hasSameMajorVersion() {
+ Assert.assertTrue(PropertyUtil.hasSameMajorKind("osdu:wks:master-data--Well:1.0.0", "osdu:wks:master-data--Well:1.0.0"));
+ Assert.assertTrue(PropertyUtil.hasSameMajorKind("osdu:wks:master-data--Well:1.1.0", "osdu:wks:master-data--Well:1.0.0"));
+ Assert.assertTrue(PropertyUtil.hasSameMajorKind("osdu:wks:master-data--Well:1.0.2", "osdu:wks:master-data--Well:1.0.0"));
+ Assert.assertFalse(PropertyUtil.hasSameMajorKind("osdu:wks:master-data--Well:2.0.0", "osdu:wks:master-data--Well:1.0.0"));
+ Assert.assertFalse(PropertyUtil.hasSameMajorKind("osdu:wks:master-data--Well:2.0.0", null));
+ }
+
+ @Test
+ public void removeDataPrefix() {
+ Assert.assertEquals("FacilityName", PropertyUtil.removeDataPrefix("data.FacilityName"));
+ Assert.assertEquals("FacilityName", PropertyUtil.removeDataPrefix("FacilityName"));
+ Assert.assertEquals("ProjectedBottomHoleLocation", PropertyUtil.removeDataPrefix("data.ProjectedBottomHoleLocation"));
+ Assert.assertEquals("ProjectedBottomHoleLocation", PropertyUtil.removeDataPrefix("ProjectedBottomHoleLocation"));
+ Assert.assertEquals("", PropertyUtil.removeDataPrefix(""));
+ Assert.assertNull(PropertyUtil.removeDataPrefix(null));
+ }
+
+ @Test
+ public void removeIdPostfix() {
+ Assert.assertEquals("data-partition:entity:12345", PropertyUtil.removeIdPostfix("data-partition:entity:12345"));
+ Assert.assertEquals("data-partition:entity:12345", PropertyUtil.removeIdPostfix("data-partition:entity:12345:"));
+ }
+
+ @Test
+ public void combineObjectMap_listObject() {
+ Map<String, Object> leftObjectMap = new HashMap<>();
+ List<Object> leftList = new ArrayList<>(Arrays.asList("v1", "v3"));
+ leftObjectMap.put("key", leftList);
+ Map<String, Object> rightObjectMap = new HashMap<>();
+ List<Object> rightList = new ArrayList<>(Arrays.asList("v1", "v2", "v4"));
+ rightObjectMap.put("key", rightList);
+
+ Map<String, Object> combinedObjectMap = PropertyUtil.combineObjectMap(leftObjectMap, new HashMap<>());
+ Assert.assertEquals(1, combinedObjectMap.size());
+ Assert.assertEquals(2, ((List)combinedObjectMap.get("key")).size());
+
+ combinedObjectMap = PropertyUtil.combineObjectMap(new HashMap<>(), rightObjectMap);
+ Assert.assertEquals(1, combinedObjectMap.size());
+ Assert.assertEquals(3, ((List)combinedObjectMap.get("key")).size());
+
+ combinedObjectMap = PropertyUtil.combineObjectMap(leftObjectMap, rightObjectMap);
+ Assert.assertEquals(1, combinedObjectMap.size());
+ Assert.assertEquals(4, ((List)combinedObjectMap.get("key")).size());
+ }
+
+ @Test
+ public void combineObjectMap_mapObject() {
+ Map<String, Object> leftObjectMap = new HashMap<>();
+ Map<String, Object> leftInnerMap = new HashMap<>();
+ leftInnerMap.put("key2_1", "value1");
+ leftObjectMap.put("key", leftInnerMap);
+
+ Map<String, Object> rightObjectMap = new HashMap<>();
+ Map<String, Object> rightInnerMap = new HashMap<>();
+ rightInnerMap.put("key2_2", "value2");
+ rightObjectMap.put("key", rightInnerMap);
+
+ Map<String, Object> combinedObjectMap = PropertyUtil.combineObjectMap(leftObjectMap, new HashMap<>());
+ Assert.assertEquals(1, combinedObjectMap.size());
+ Assert.assertEquals(1, ((Map)combinedObjectMap.get("key")).size());
+
+ combinedObjectMap = PropertyUtil.combineObjectMap(new HashMap<>(), rightObjectMap);
+ Assert.assertEquals(1, combinedObjectMap.size());
+ Assert.assertEquals(1, ((Map)combinedObjectMap.get("key")).size());
+
+ combinedObjectMap = PropertyUtil.combineObjectMap(leftObjectMap, rightObjectMap);
+ Assert.assertEquals(1, combinedObjectMap.size());
+ Assert.assertEquals(2, ((Map)combinedObjectMap.get("key")).size());
+ }
+
+ @Test
+ public void combineObjectMap_sameStringValue() {
+ Map<String, Object> leftObjectMap = new HashMap<>();
+ leftObjectMap.put("key", "value");
+
+ Map<String, Object> rightObjectMap = new HashMap<>();
+ rightObjectMap.put("key", "value");
+
+ Map<String, Object> combinedObjectMap = PropertyUtil.combineObjectMap(leftObjectMap, new HashMap<>());
+ Assert.assertEquals(1, combinedObjectMap.size());
+ Assert.assertEquals("value", combinedObjectMap.get("key"));
+
+ combinedObjectMap = PropertyUtil.combineObjectMap(new HashMap<>(), rightObjectMap);
+ Assert.assertEquals(1, combinedObjectMap.size());
+ Assert.assertEquals("value", combinedObjectMap.get("key"));
+
+ combinedObjectMap = PropertyUtil.combineObjectMap(leftObjectMap, rightObjectMap);
+ Assert.assertEquals(1, combinedObjectMap.size());
+ Assert.assertEquals("value", combinedObjectMap.get("key"));
+
+ }
+
+ @Test
+ public void combineObjectMap_differentStringValue() {
+ Map<String, Object> leftObjectMap = new HashMap<>();
+ leftObjectMap.put("key", "value1");
+
+ Map<String, Object> rightObjectMap = new HashMap<>();
+ rightObjectMap.put("key", "value2");
+
+ Map<String, Object> combinedObjectMap = PropertyUtil.combineObjectMap(leftObjectMap, new HashMap<>());
+ Assert.assertEquals(1, combinedObjectMap.size());
+ Assert.assertEquals("value1", combinedObjectMap.get("key"));
+
+ combinedObjectMap = PropertyUtil.combineObjectMap(new HashMap<>(), rightObjectMap);
+ Assert.assertEquals(1, combinedObjectMap.size());
+ Assert.assertEquals("value2", combinedObjectMap.get("key"));
+
+ combinedObjectMap = PropertyUtil.combineObjectMap(leftObjectMap, rightObjectMap);
+ Assert.assertEquals(1, combinedObjectMap.size());
+ Assert.assertEquals(2, ((List)combinedObjectMap.get("key")).size());
+ }
+
+ @Test
+ public void combineObjectMap_ObjectList() {
+ Map<String, Object> leftObjectMap = new HashMap<>();
+ leftObjectMap.put("key", "value1");
+
+ Map<String, Object> rightObjectMap = new HashMap<>();
+ List<Object> rightList = new ArrayList<>(Arrays.asList("v1", "v2", "v4"));
+ rightObjectMap.put("key", rightList);
+
+ Map<String, Object> combinedObjectMap = PropertyUtil.combineObjectMap(leftObjectMap, rightObjectMap);
+ Assert.assertEquals(1, combinedObjectMap.size());
+ Assert.assertEquals(4, ((List)combinedObjectMap.get("key")).size());
+
+ leftObjectMap = new HashMap<>();
+ leftObjectMap.put("key", "value1");
+ rightObjectMap = new HashMap<>();
+ rightObjectMap.put("key", new ArrayList<>());
+ combinedObjectMap = PropertyUtil.combineObjectMap(leftObjectMap, rightObjectMap);
+ Assert.assertEquals(1, combinedObjectMap.size());
+ Assert.assertEquals("value1", combinedObjectMap.get("key"));
+ }
+
+ @Test
+ public void combineObjectMap_misMatchTypeValue() {
+ Map<String, Object> leftObjectMap = new HashMap<>();
+ leftObjectMap.put("key", "value1");
+
+ Map<String, Object> rightObjectMap = new HashMap<>();
+ rightObjectMap.put("key", new HashMap<>());
+
+ // The object values in rightObjectMap will be ignored in this case
+ Map<String, Object> combinedObjectMap = PropertyUtil.combineObjectMap(leftObjectMap, rightObjectMap);
+ Assert.assertEquals(1, combinedObjectMap.size());
+ Assert.assertTrue(combinedObjectMap.get("key") instanceof String);
+ Assert.assertEquals("value1", combinedObjectMap.get("key"));
+
+ combinedObjectMap = PropertyUtil.combineObjectMap(rightObjectMap, leftObjectMap);
+ Assert.assertEquals(1, combinedObjectMap.size());
+ Assert.assertTrue(combinedObjectMap.get("key") instanceof Map);
+ }
+
+ @Test
+ public void replacePropertyPaths() {
+ Map<String, Object> map = new HashMap<>();
+ map.put("aaa", "value1");
+ map.put("bbb.ccc", "value2");
+ map.put("bbb.ddd", "value3");
+
+ Map<String, Object> newMap;
+ newMap = PropertyUtil.replacePropertyPaths("111", "aaa", map);
+ Assert.assertEquals("value1", newMap.get("111"));
+ newMap = PropertyUtil.replacePropertyPaths("111", "data.aaa", map);
+ Assert.assertEquals("value1", newMap.get("111"));
+ newMap = PropertyUtil.replacePropertyPaths("data.111", "aaa", map);
+ Assert.assertEquals("value1", newMap.get("111"));
+
+ newMap = PropertyUtil.replacePropertyPaths("222", "bbb", map);
+ Assert.assertEquals("value2", newMap.get("222.ccc"));
+ Assert.assertEquals("value3", newMap.get("222.ddd"));
+ newMap = PropertyUtil.replacePropertyPaths("222", "data.bbb", map);
+ Assert.assertEquals("value2", newMap.get("222.ccc"));
+ Assert.assertEquals("value3", newMap.get("222.ddd"));
+ newMap = PropertyUtil.replacePropertyPaths("data.222", "bbb", map);
+ Assert.assertEquals("value2", newMap.get("222.ccc"));
+ Assert.assertEquals("value3", newMap.get("222.ddd"));
+ }
+
+ @Test
+ public void replacePropertyPaths_emptyValues() {
+ Map<String, Object> map = new HashMap<>();
+ map.put("abc", "value1");
+ Assert.assertTrue(PropertyUtil.replacePropertyPaths(null, "abc", map).isEmpty());
+ Assert.assertTrue(PropertyUtil.replacePropertyPaths("abc", null, map).isEmpty());
+ Assert.assertTrue(PropertyUtil.replacePropertyPaths("a12", "abc", new HashMap<>()).isEmpty());
+ Assert.assertTrue(PropertyUtil.replacePropertyPaths("a12", "abc", null).isEmpty());
+ }
+
+ @Test
+ public void isConcreteKind() {
+ Assert.assertTrue(PropertyUtil.isConcreteKind("osdu:wks:master-data--Well:1.0.0"));
+ Assert.assertFalse(PropertyUtil.isConcreteKind("osdu:wks:master-data--Well:1.0."));
+ Assert.assertFalse(PropertyUtil.isConcreteKind("osdu:master-data--Well:1.0.0"));
+ Assert.assertFalse(PropertyUtil.isConcreteKind(null));
+ Assert.assertFalse(PropertyUtil.isConcreteKind(""));
+ }
+
+ @Test
+ public void getKindWithMajor() {
+ Assert.assertEquals("osdu:wks:master-data--Well:1.", PropertyUtil.getKindWithMajor("osdu:wks:master-data--Well:1.0.0"));
+ Assert.assertEquals("osdu:wks:master-data--Well:1.", PropertyUtil.getKindWithMajor("osdu:wks:master-data--Well:1."));
+ Assert.assertEquals("osdu:wks:master-data--Well:1.", PropertyUtil.getKindWithMajor("osdu:wks:master-data--Well:1"));
+
+ Assert.assertEquals("", PropertyUtil.getKindWithMajor("osdu:wks:master-data--Well"));
+ Assert.assertEquals("", PropertyUtil.getKindWithMajor(""));
+ Assert.assertNull(null, PropertyUtil.getKindWithMajor(null));
+ }
+
+ @Test
+ public void getChangedProperties() {
+ Map<String, Object> dataMapLeft = getDataMap("well.json");
+ Map<String, Object> dataMapRight = getDataMap("well2.json");
+ List<String> changedProperties = PropertyUtil.getChangedProperties(dataMapLeft, dataMapRight);
+ Assert.assertEquals(3, changedProperties.size());
+ List<String> expectedChangedWellProperties = Arrays.asList("VirtualProperties.DefaultName", "VerticalMeasurements[].VerticalMeasurementID", "FacilityName");
+ changedProperties.forEach(p -> Assert.assertTrue(expectedChangedWellProperties.contains(p)));
+
+ dataMapLeft = getDataMap("wellLog.json");
+ dataMapRight = getDataMap("wellLog2.json");
+ changedProperties = PropertyUtil.getChangedProperties(dataMapLeft, dataMapRight);
+ List<String> expectedChangedWellLogProperties = Arrays.asList("Curves[].CurveID");
+ changedProperties.forEach(p -> Assert.assertTrue(expectedChangedWellLogProperties.contains(p)));
+ }
+
+ private Map<String, Object> getDataMap(String file) {
+ String jsonText = getJsonFromFile(file);
+ Type type = new TypeToken<Map<String, Object>>() {}.getType();
+ return gson.fromJson(jsonText, type);
+ }
+
+ @SneakyThrows
+ private String getJsonFromFile(String file) {
+ InputStream inStream = this.getClass().getResourceAsStream("/indexproperty/" + file);
+ BufferedReader br = new BufferedReader(new InputStreamReader(inStream));
+ StringBuilder stringBuilder = new StringBuilder();
+ String sCurrentLine;
+ while ((sCurrentLine = br.readLine()) != null)
+ {
+ stringBuilder.append(sCurrentLine).append("\n");
+ }
+ return stringBuilder.toString();
+ }
+
+}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/VirtualPropertyUtilTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/VirtualPropertyUtilTest.java
deleted file mode 100644
index 63c33460fbfed619231f2b529e2596482966e23b..0000000000000000000000000000000000000000
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/VirtualPropertyUtilTest.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Copyright © Schlumberger
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0 *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.util;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.springframework.test.context.junit4.SpringRunner;
-
-@RunWith(SpringRunner.class)
-public class VirtualPropertyUtilTest {
-
- @Test
- public void isPropertyPathMatched() {
- Assert.assertTrue(VirtualPropertyUtil.isPropertyPathMatched("data.FacilityName", "data.FacilityName"));
- Assert.assertTrue(VirtualPropertyUtil.isPropertyPathMatched("data.ProjectedBottomHoleLocation.Wgs84Coordinates", "data.ProjectedBottomHoleLocation"));
-
- Assert.assertFalse(VirtualPropertyUtil.isPropertyPathMatched("data.FacilityName", "data.FacilityNameAliase"));
- Assert.assertFalse(VirtualPropertyUtil.isPropertyPathMatched("data.ProjectedBottomHoleLocation.Wgs84Coordinates", "data.ProjectedBottomHole"));
- Assert.assertFalse(VirtualPropertyUtil.isPropertyPathMatched("", "data.ProjectedBottomHole"));
- Assert.assertFalse(VirtualPropertyUtil.isPropertyPathMatched(null, "data.ProjectedBottomHole"));
- }
-
- @Test
- public void removeDataPrefix() {
- Assert.assertEquals("FacilityName", VirtualPropertyUtil.removeDataPrefix("data.FacilityName"));
- Assert.assertEquals("FacilityName", VirtualPropertyUtil.removeDataPrefix("FacilityName"));
- Assert.assertEquals("ProjectedBottomHoleLocation", VirtualPropertyUtil.removeDataPrefix("data.ProjectedBottomHoleLocation"));
- Assert.assertEquals("ProjectedBottomHoleLocation", VirtualPropertyUtil.removeDataPrefix("ProjectedBottomHoleLocation"));
- Assert.assertEquals("", VirtualPropertyUtil.removeDataPrefix(""));
- Assert.assertNull(VirtualPropertyUtil.removeDataPrefix(null));
- }
-}
diff --git a/indexer-core/src/test/resources/indexproperty/geo_political_entity_storage_schema.json b/indexer-core/src/test/resources/indexproperty/geo_political_entity_storage_schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..9021c6eabe177343a0a220bc8e4d67466adaedd6
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/geo_political_entity_storage_schema.json
@@ -0,0 +1,186 @@
+{
+ "kind": "osdu:wks:master-data--GeoPoliticalEntity:1.0.0",
+ "schema": [{
+ "path": "ResourceHomeRegionID",
+ "kind": "string"
+ }, {
+ "path": "ResourceHostRegionIDs",
+ "kind": "[]string"
+ }, {
+ "path": "ResourceLifecycleStatus",
+ "kind": "string"
+ }, {
+ "path": "ResourceSecurityClassification",
+ "kind": "string"
+ }, {
+ "path": "ResourceCurationStatus",
+ "kind": "string"
+ }, {
+ "path": "ExistenceKind",
+ "kind": "string"
+ }, {
+ "path": "TechnicalAssuranceID",
+ "kind": "string"
+ }, {
+ "path": "Source",
+ "kind": "string"
+ }, {
+ "path": "NameAliases",
+ "kind": "nested",
+ "properties": [{
+ "path": "AliasName",
+ "kind": "string"
+ }, {
+ "path": "TerminationDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "AliasNameTypeID",
+ "kind": "string"
+ }, {
+ "path": "EffectiveDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "DefinitionOrganisationID",
+ "kind": "string"
+ }
+ ]
+ }, {
+ "path": "SpatialLocation.SpatialParameterTypeID",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.QuantitativeAccuracyBandID",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.CoordinateQualityCheckRemarks",
+ "kind": "[]string"
+ }, {
+ "path": "SpatialLocation.AppliedOperations",
+ "kind": "[]string"
+ }, {
+ "path": "SpatialLocation.QualitativeSpatialAccuracyTypeID",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.CoordinateQualityCheckPerformedBy",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.SpatialLocationCoordinatesDate",
+ "kind": "datetime"
+ }, {
+ "path": "SpatialLocation.CoordinateQualityCheckDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "SpatialLocation.Wgs84Coordinates",
+ "kind": "core:dl:geoshape:1.0.0"
+ }, {
+ "path": "SpatialLocation.SpatialGeometryTypeID",
+ "kind": "string"
+ }, {
+ "path": "VersionCreationReason",
+ "kind": "string"
+ }, {
+ "path": "TechnicalAssuranceTypeID",
+ "kind": "string"
+ }, {
+ "path": "GeoContexts",
+ "kind": "nested",
+ "properties": [{
+ "path": "GeoPoliticalEntityID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "BasinID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "FieldID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "PlayID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "ProspectID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }
+ ]
+ }, {
+ "path": "ParentGeoPoliticalEntityID",
+ "kind": "string"
+ }, {
+ "path": "GeoPoliticalEntityTypeID",
+ "kind": "string"
+ }, {
+ "path": "TerminationDate",
+ "kind": "datetime"
+ }, {
+ "path": "DisputedIndicator",
+ "kind": "bool"
+ }, {
+ "path": "DaylightSavingTimeStartDate",
+ "kind": "datetime"
+ }, {
+ "path": "GeoPoliticalEntityName",
+ "kind": "string"
+ }, {
+ "path": "GeoPoliticalEntityID",
+ "kind": "string"
+ }, {
+ "path": "DaylightSavingTimeEndDate",
+ "kind": "datetime"
+ }, {
+ "path": "GeoPoliticalEntityNameAliases",
+ "kind": "[]object"
+ }, {
+ "path": "EffectiveDate",
+ "kind": "datetime"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.SpatialParameterTypeID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.QuantitativeAccuracyBandID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.CoordinateQualityCheckRemarks",
+ "kind": "[]string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.AppliedOperations",
+ "kind": "[]string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.QualitativeSpatialAccuracyTypeID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.CoordinateQualityCheckPerformedBy",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.SpatialLocationCoordinatesDate",
+ "kind": "datetime"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.CoordinateQualityCheckDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.Wgs84Coordinates",
+ "kind": "core:dl:geoshape:1.0.0"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.SpatialGeometryTypeID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultName",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.IsDecimated",
+ "kind": "boolean"
+ }
+ ]
+}
diff --git a/indexer-core/src/test/resources/indexproperty/organisation_data1.json b/indexer-core/src/test/resources/indexproperty/organisation_data1.json
new file mode 100644
index 0000000000000000000000000000000000000000..dad4b5c3da7d61ededf41066b2c0bc5dfe3c523c
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/organisation_data1.json
@@ -0,0 +1,8 @@
+{
+ "InternalOrganisationIndicator": true,
+ "VirtualProperties.DefaultName": "BigOil SeismicInterpretation Department (original)",
+ "VersionCreationReason": "Index testing",
+ "OrganisationName": "BigOil SeismicInterpretation Department (original)",
+ "TechnicalAssuranceTypeID": "opendes:reference-data--TechnicalAssuranceType:Certified:",
+ "OrganisationTypeID": "opendes:reference-data--OrganisationType:OrganizationUnit:"
+}
diff --git a/indexer-core/src/test/resources/indexproperty/organisation_data2.json b/indexer-core/src/test/resources/indexproperty/organisation_data2.json
new file mode 100644
index 0000000000000000000000000000000000000000..e486b811c3f5c12390b127321bd42e395919751c
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/organisation_data2.json
@@ -0,0 +1,8 @@
+{
+ "InternalOrganisationIndicator": true,
+ "VirtualProperties.DefaultName": "BigOil SeismicProcessing Department (original)",
+ "VersionCreationReason": "Index testing",
+ "OrganisationName": "BigOil SeismicProcessing Department (original)",
+ "TechnicalAssuranceTypeID": "opendes:reference-data--TechnicalAssuranceType:Certified:",
+ "OrganisationTypeID": "opendes:reference-data--OrganisationType:OrganizationUnit:"
+}
diff --git a/indexer-core/src/test/resources/indexproperty/organisation_storage_schema.json b/indexer-core/src/test/resources/indexproperty/organisation_storage_schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..ba6563b82bdfc59e8511c0298e430f1214ebd626
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/organisation_storage_schema.json
@@ -0,0 +1,186 @@
+{
+ "kind": "osdu:wks:master-data--Organisation:1.1.0",
+ "schema": [{
+ "path": "ResourceHomeRegionID",
+ "kind": "string"
+ }, {
+ "path": "ResourceHostRegionIDs",
+ "kind": "[]string"
+ }, {
+ "path": "ResourceLifecycleStatus",
+ "kind": "string"
+ }, {
+ "path": "ResourceSecurityClassification",
+ "kind": "string"
+ }, {
+ "path": "ResourceCurationStatus",
+ "kind": "string"
+ }, {
+ "path": "ExistenceKind",
+ "kind": "string"
+ }, {
+ "path": "TechnicalAssuranceID",
+ "kind": "string"
+ }, {
+ "path": "Source",
+ "kind": "string"
+ }, {
+ "path": "NameAliases",
+ "kind": "nested",
+ "properties": [{
+ "path": "AliasName",
+ "kind": "string"
+ }, {
+ "path": "TerminationDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "AliasNameTypeID",
+ "kind": "string"
+ }, {
+ "path": "EffectiveDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "DefinitionOrganisationID",
+ "kind": "string"
+ }
+ ]
+ }, {
+ "path": "SpatialLocation.SpatialParameterTypeID",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.QuantitativeAccuracyBandID",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.CoordinateQualityCheckRemarks",
+ "kind": "[]string"
+ }, {
+ "path": "SpatialLocation.AppliedOperations",
+ "kind": "[]string"
+ }, {
+ "path": "SpatialLocation.QualitativeSpatialAccuracyTypeID",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.CoordinateQualityCheckPerformedBy",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.SpatialLocationCoordinatesDate",
+ "kind": "datetime"
+ }, {
+ "path": "SpatialLocation.CoordinateQualityCheckDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "SpatialLocation.Wgs84Coordinates",
+ "kind": "core:dl:geoshape:1.0.0"
+ }, {
+ "path": "SpatialLocation.SpatialGeometryTypeID",
+ "kind": "string"
+ }, {
+ "path": "VersionCreationReason",
+ "kind": "string"
+ }, {
+ "path": "TechnicalAssuranceTypeID",
+ "kind": "string"
+ }, {
+ "path": "GeoContexts",
+ "kind": "nested",
+ "properties": [{
+ "path": "GeoPoliticalEntityID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "BasinID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "FieldID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "PlayID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "ProspectID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }
+ ]
+ }, {
+ "path": "OrganisationPurposeDescription",
+ "kind": "string"
+ }, {
+ "path": "TerminationDate",
+ "kind": "datetime"
+ }, {
+ "path": "OrganisationTypeID",
+ "kind": "string"
+ }, {
+ "path": "OrganisationID",
+ "kind": "string"
+ }, {
+ "path": "InternalOrganisationIndicator",
+ "kind": "bool"
+ }, {
+ "path": "OrganisationName",
+ "kind": "string"
+ }, {
+ "path": "OrganisationNameAliases",
+ "kind": "[]object"
+ }, {
+ "path": "ParentOrganisationID",
+ "kind": "string"
+ }, {
+ "path": "OrganisationDescription",
+ "kind": "string"
+ }, {
+ "path": "EffectiveDate",
+ "kind": "datetime"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.SpatialParameterTypeID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.QuantitativeAccuracyBandID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.CoordinateQualityCheckRemarks",
+ "kind": "[]string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.AppliedOperations",
+ "kind": "[]string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.QualitativeSpatialAccuracyTypeID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.CoordinateQualityCheckPerformedBy",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.SpatialLocationCoordinatesDate",
+ "kind": "datetime"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.CoordinateQualityCheckDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.Wgs84Coordinates",
+ "kind": "core:dl:geoshape:1.0.0"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.SpatialGeometryTypeID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultName",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.IsDecimated",
+ "kind": "boolean"
+ }
+ ]
+}
diff --git a/indexer-core/src/test/resources/indexproperty/well.json b/indexer-core/src/test/resources/indexproperty/well.json
new file mode 100644
index 0000000000000000000000000000000000000000..918b86432c450ac5d85409a4f7c99aea9815dd9e
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/well.json
@@ -0,0 +1,136 @@
+{
+ "VirtualProperties.DefaultLocation.QuantitativeAccuracyBandID": null,
+ "ResourceLifecycleStatus": null,
+ "DefaultVerticalMeasurementID": "KB",
+ "WasBusinessInterestObligatory": null,
+ "SpatialLocation.Wgs84Coordinates": {
+ "type": "geometrycollection",
+ "geometries": [{
+ "type": "point",
+ "coordinates": [171.8745361, -41.2456122]
+ }
+ ]
+ },
+ "ResourceCurationStatus": null,
+ "TechnicalAssuranceID": "opendes:reference-data--TechnicalAssuranceType:Unsuitable:",
+ "VirtualProperties.DefaultLocation.SpatialGeometryTypeID": "opendes:reference-data--SpatialGeometryType:Point:",
+ "RoleID": null,
+ "FacilityName": "Kongahu-1",
+ "Source": null,
+ "FacilityID": null,
+ "OutcomeID": null,
+ "VirtualProperties.DefaultName": "Kongahu-1",
+ "VerticalMeasurements": [{
+ "WellboreTVDTrajectoryID": null,
+ "VerticalCRSID": null,
+ "VerticalReferenceID": null,
+ "VerticalMeasurementSourceID": null,
+ "VerticalMeasurementID": "KB",
+ "VerticalMeasurementPathID": "opendes:reference-data--VerticalMeasurementPath:ELEV:",
+ "VerticalMeasurement": 26.0,
+ "VerticalMeasurementTypeID": "opendes:reference-data--VerticalMeasurementType:KB:",
+ "VerticalMeasurementDescription": null,
+ "VerticalMeasurementUnitOfMeasureID": "opendes:reference-data--UnitOfMeasure:m:",
+ "VerticalReferenceEntityID": null,
+ "RigID": null
+ }, {
+ "WellboreTVDTrajectoryID": null,
+ "VerticalCRSID": null,
+ "VerticalReferenceID": null,
+ "VerticalMeasurementSourceID": null,
+ "VerticalMeasurementID": "Well Head Elevation",
+ "VerticalMeasurementPathID": "opendes:reference-data--VerticalMeasurementPath:ELEV:",
+ "VerticalMeasurement": 26.0,
+ "VerticalMeasurementTypeID": null,
+ "VerticalMeasurementDescription": null,
+ "VerticalMeasurementUnitOfMeasureID": "opendes:reference-data--UnitOfMeasure:m:",
+ "VerticalReferenceEntityID": null,
+ "RigID": null
+ }, {
+ "WellboreTVDTrajectoryID": null,
+ "VerticalCRSID": null,
+ "VerticalReferenceID": null,
+ "VerticalMeasurementSourceID": null,
+ "VerticalMeasurementID": "Water Depth",
+ "VerticalMeasurementPathID": "opendes:reference-data--VerticalMeasurementPath:ELEV:",
+ "VerticalMeasurement": 94.0,
+ "VerticalMeasurementTypeID": null,
+ "VerticalMeasurementDescription": null,
+ "VerticalMeasurementUnitOfMeasureID": "opendes:reference-data--UnitOfMeasure:m:",
+ "VerticalReferenceEntityID": null,
+ "RigID": null
+ }
+ ],
+ "VirtualProperties.DefaultLocation.CoordinateQualityCheckPerformedBy": null,
+ "VersionCreationReason": null,
+ "ResourceSecurityClassification": null,
+ "InterestTypeID": null,
+ "DataSourceOrganisationID": null,
+ "SpatialLocation.SpatialParameterTypeID": null,
+ "ExistenceKind": null,
+ "SpatialLocation.CoordinateQualityCheckPerformedBy": null,
+ "FacilityOperators": [{
+ "FacilityOperatorID": "Operator",
+ "FacilityOperatorOrganisationID": "opendes:master-data--Organisation:HOME%20ENERGY%20NZ%20LTD:"
+ }
+ ],
+ "FacilityTypeID": "opendes:reference-data--FacilityType:Well:",
+ "BusinessIntentionID": null,
+ "NameAliases": [{
+ "AliasName": "100000113552",
+ "AliasNameTypeID": "opendes:reference-data--AliasNameType:UniqueIdentifier:",
+ "DefinitionOrganisationID": null
+ }, {
+ "AliasName": "Well1",
+ "AliasNameTypeID": "opendes:reference-data--AliasNameType:CommonName:",
+ "DefinitionOrganisationID": null
+ }
+ ],
+ "DefaultVerticalCRSID": null,
+ "FacilityEvents": [{
+ "EffectiveDateTime": "1984-06-21T00:00:00+0000",
+ "FacilityEventTypeID": "opendes:reference-data--FacilityEventType:Spud:"
+ }
+ ],
+ "VirtualProperties.DefaultLocation.IsDecimated": false,
+ "TechnicalAssuranceTypeID": null,
+ "GeoContexts": [{
+ "BasinID": null,
+ "FieldID": null,
+ "PlayID": null,
+ "GeoPoliticalEntityID": "opendes:master-data--GeoPoliticalEntity:New%20Zealand:",
+ "GeoTypeID": "opendes:reference-data--GeoPoliticalEntityType:Country:",
+ "ProspectID": null
+ }, {
+ "BasinID": null,
+ "FieldID": null,
+ "PlayID": null,
+ "GeoPoliticalEntityID": "opendes:master-data--GeoPoliticalEntity:38058:",
+ "GeoTypeID": "opendes:reference-data--GeoPoliticalEntityType:LicenseBlock:",
+ "ProspectID": null
+ }
+ ],
+ "FakeGeoContexts": ["opendes:reference-data--GeoPoliticalEntityType:Country:","opendes:reference-data--GeoPoliticalEntityType:LicenseBlock:","opendes:reference-data--GeoPoliticalEntityType:Country:"],
+ "WasBusinessInterestFinancialNonOperated": null,
+ "CurrentOperatorID": null,
+ "SpatialLocation.QualitativeSpatialAccuracyTypeID": null,
+ "WasBusinessInterestTechnical": null,
+ "SpatialLocation.SpatialGeometryTypeID": "opendes:reference-data--SpatialGeometryType:Point:",
+ "OperatingEnvironmentID": "opendes:reference-data--OperatingEnvironment:Offshore:",
+ "VirtualProperties.DefaultLocation.SpatialParameterTypeID": null,
+ "ResourceHomeRegionID": null,
+ "StatusSummaryID": null,
+ "VirtualProperties.DefaultLocation.QualitativeSpatialAccuracyTypeID": null,
+ "ConditionID": null,
+ "InitialOperatorID": null,
+ "WasBusinessInterestFinancialOperated": null,
+ "VirtualProperties.DefaultLocation.Wgs84Coordinates": {
+ "type": "geometrycollection",
+ "geometries": [{
+ "type": "point",
+ "coordinates": [171.8745361, -41.2456122]
+ }
+ ]
+ },
+ "SpatialLocation.QuantitativeAccuracyBandID": null
+}
diff --git a/indexer-core/src/test/resources/indexproperty/well2.json b/indexer-core/src/test/resources/indexproperty/well2.json
new file mode 100644
index 0000000000000000000000000000000000000000..76fce04a10dfdc6a1b09661d0879fa4465f7f620
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/well2.json
@@ -0,0 +1,136 @@
+{
+ "VirtualProperties.DefaultLocation.QuantitativeAccuracyBandID": null,
+ "ResourceLifecycleStatus": null,
+ "DefaultVerticalMeasurementID": "KB",
+ "WasBusinessInterestObligatory": null,
+ "SpatialLocation.Wgs84Coordinates": {
+ "type": "geometrycollection",
+ "geometries": [{
+ "type": "point",
+ "coordinates": [171.8745361, -41.2456122]
+ }
+ ]
+ },
+ "ResourceCurationStatus": null,
+ "TechnicalAssuranceID": "opendes:reference-data--TechnicalAssuranceType:Unsuitable:",
+ "VirtualProperties.DefaultLocation.SpatialGeometryTypeID": "opendes:reference-data--SpatialGeometryType:Point:",
+ "RoleID": null,
+ "FacilityName": "Kongahu-2",
+ "Source": null,
+ "FacilityID": null,
+ "OutcomeID": null,
+ "VirtualProperties.DefaultName": "Kongahu-2",
+ "VerticalMeasurements": [{
+ "WellboreTVDTrajectoryID": null,
+ "VerticalCRSID": null,
+ "VerticalReferenceID": null,
+ "VerticalMeasurementSourceID": null,
+ "VerticalMeasurementID": "MB",
+ "VerticalMeasurementPathID": "opendes:reference-data--VerticalMeasurementPath:ELEV:",
+ "VerticalMeasurement": 26.0,
+ "VerticalMeasurementTypeID": "opendes:reference-data--VerticalMeasurementType:KB:",
+ "VerticalMeasurementDescription": null,
+ "VerticalMeasurementUnitOfMeasureID": "opendes:reference-data--UnitOfMeasure:m:",
+ "VerticalReferenceEntityID": null,
+ "RigID": null
+ }, {
+ "WellboreTVDTrajectoryID": null,
+ "VerticalCRSID": null,
+ "VerticalReferenceID": null,
+ "VerticalMeasurementSourceID": null,
+ "VerticalMeasurementID": "Well Head Elevation",
+ "VerticalMeasurementPathID": "opendes:reference-data--VerticalMeasurementPath:ELEV:",
+ "VerticalMeasurement": 26.0,
+ "VerticalMeasurementTypeID": null,
+ "VerticalMeasurementDescription": null,
+ "VerticalMeasurementUnitOfMeasureID": "opendes:reference-data--UnitOfMeasure:m:",
+ "VerticalReferenceEntityID": null,
+ "RigID": null
+ }, {
+ "WellboreTVDTrajectoryID": null,
+ "VerticalCRSID": null,
+ "VerticalReferenceID": null,
+ "VerticalMeasurementSourceID": null,
+ "VerticalMeasurementID": "Water Depth",
+ "VerticalMeasurementPathID": "opendes:reference-data--VerticalMeasurementPath:ELEV:",
+ "VerticalMeasurement": 94.0,
+ "VerticalMeasurementTypeID": null,
+ "VerticalMeasurementDescription": null,
+ "VerticalMeasurementUnitOfMeasureID": "opendes:reference-data--UnitOfMeasure:m:",
+ "VerticalReferenceEntityID": null,
+ "RigID": null
+ }
+ ],
+ "VirtualProperties.DefaultLocation.CoordinateQualityCheckPerformedBy": null,
+ "VersionCreationReason": null,
+ "ResourceSecurityClassification": null,
+ "InterestTypeID": null,
+ "DataSourceOrganisationID": null,
+ "SpatialLocation.SpatialParameterTypeID": null,
+ "ExistenceKind": null,
+ "SpatialLocation.CoordinateQualityCheckPerformedBy": null,
+ "FacilityOperators": [{
+ "FacilityOperatorID": "Operator",
+ "FacilityOperatorOrganisationID": "opendes:master-data--Organisation:HOME%20ENERGY%20NZ%20LTD:"
+ }
+ ],
+ "FacilityTypeID": "opendes:reference-data--FacilityType:Well:",
+ "BusinessIntentionID": null,
+ "NameAliases": [{
+ "AliasName": "100000113552",
+ "AliasNameTypeID": "opendes:reference-data--AliasNameType:UniqueIdentifier:",
+ "DefinitionOrganisationID": null
+ }, {
+ "AliasName": "Well1",
+ "AliasNameTypeID": "opendes:reference-data--AliasNameType:CommonName:",
+ "DefinitionOrganisationID": null
+ }
+ ],
+ "DefaultVerticalCRSID": null,
+ "FacilityEvents": [{
+ "EffectiveDateTime": "1984-06-21T00:00:00+0000",
+ "FacilityEventTypeID": "opendes:reference-data--FacilityEventType:Spud:"
+ }
+ ],
+ "VirtualProperties.DefaultLocation.IsDecimated": false,
+ "TechnicalAssuranceTypeID": null,
+ "GeoContexts": [{
+ "BasinID": null,
+ "FieldID": null,
+ "PlayID": null,
+ "GeoPoliticalEntityID": "opendes:master-data--GeoPoliticalEntity:New%20Zealand:",
+ "GeoTypeID": "opendes:reference-data--GeoPoliticalEntityType:Country:",
+ "ProspectID": null
+ }, {
+ "BasinID": null,
+ "FieldID": null,
+ "PlayID": null,
+ "GeoPoliticalEntityID": "opendes:master-data--GeoPoliticalEntity:38058:",
+ "GeoTypeID": "opendes:reference-data--GeoPoliticalEntityType:LicenseBlock:",
+ "ProspectID": null
+ }
+ ],
+ "FakeGeoContexts": ["opendes:reference-data--GeoPoliticalEntityType:Country:","opendes:reference-data--GeoPoliticalEntityType:LicenseBlock:","opendes:reference-data--GeoPoliticalEntityType:Country:"],
+ "WasBusinessInterestFinancialNonOperated": null,
+ "CurrentOperatorID": null,
+ "SpatialLocation.QualitativeSpatialAccuracyTypeID": null,
+ "WasBusinessInterestTechnical": null,
+ "SpatialLocation.SpatialGeometryTypeID": "opendes:reference-data--SpatialGeometryType:Point:",
+ "OperatingEnvironmentID": "opendes:reference-data--OperatingEnvironment:Offshore:",
+ "VirtualProperties.DefaultLocation.SpatialParameterTypeID": null,
+ "ResourceHomeRegionID": null,
+ "StatusSummaryID": null,
+ "VirtualProperties.DefaultLocation.QualitativeSpatialAccuracyTypeID": null,
+ "ConditionID": null,
+ "InitialOperatorID": null,
+ "WasBusinessInterestFinancialOperated": null,
+ "VirtualProperties.DefaultLocation.Wgs84Coordinates": {
+ "type": "geometrycollection",
+ "geometries": [{
+ "type": "point",
+ "coordinates": [171.8745361, -41.2456122]
+ }
+ ]
+ },
+ "SpatialLocation.QuantitativeAccuracyBandID": null
+}
diff --git a/indexer-core/src/test/resources/indexproperty/wellLog.json b/indexer-core/src/test/resources/indexproperty/wellLog.json
new file mode 100644
index 0000000000000000000000000000000000000000..0caafe08589d987a025b07393236a236f77d683d
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/wellLog.json
@@ -0,0 +1,352 @@
+{
+ "LogRemark": null,
+ "SpatialArea.QuantitativeAccuracyBandID": null,
+ "CompanyID": null,
+ "VirtualProperties.DefaultLocation.QuantitativeAccuracyBandID": null,
+ "SpatialArea.SpatialParameterTypeID": null,
+ "ResourceCurationStatus": null,
+ "SpatialArea.SpatialGeometryTypeID": null,
+ "IsExtendedLoad": null,
+ "Name": "EC_CURVES",
+ "VerticalMeasurement.VerticalCRSID": null,
+ "VirtualProperties.DefaultName": "EC_CURVES",
+ "SeismicReferenceElevation.VerticalMeasurementUnitOfMeasureID": null,
+ "VirtualProperties.DefaultLocation.CoordinateQualityCheckPerformedBy": null,
+ "ResourceSecurityClassification": null,
+ "SeismicReferenceElevation.VerticalMeasurementDescription": null,
+ "VerticalMeasurement.VerticalMeasurementPathID": null,
+ "SpatialLocation.SpatialParameterTypeID": null,
+ "ExistenceKind": null,
+ "HoleTypeLogging": null,
+ "LoggingDirection": null,
+ "VerticalMeasurementID": null,
+ "SeismicReferenceElevation.VerticalMeasurementPathID": null,
+ "SpatialArea.QualitativeSpatialAccuracyTypeID": null,
+ "VerticalMeasurement.VerticalReferenceEntityID": null,
+ "SpatialPoint.SpatialGeometryTypeID": null,
+ "SpatialLocation.SpatialGeometryTypeID": "opendes:reference-data--SpatialGeometryType:Point:",
+ "VerticalMeasurement.VerticalMeasurementSourceID": null,
+ "WellLogTypeID": null,
+ "IsDiscoverable": null,
+ "SeismicReferenceElevation.VerticalMeasurementTypeID": null,
+ "LoggingService": null,
+ "VirtualProperties.DefaultLocation.QualitativeSpatialAccuracyTypeID": null,
+ "VerticalMeasurement.VerticalReferenceID": null,
+ "SubmitterName": null,
+ "LogRun": null,
+ "SpatialPoint.QualitativeSpatialAccuracyTypeID": null,
+ "LogActivity": null,
+ "Description": null,
+ "ResourceLifecycleStatus": null,
+ "ServiceCompanyID": null,
+ "SpatialLocation.Wgs84Coordinates": {
+ "geometries": [{
+ "coordinates": [171.8745361, -41.2456122],
+ "type": "point"
+ }
+ ],
+ "type": "geometrycollection"
+ },
+ "TechnicalAssuranceID": null,
+ "SeismicReferenceElevation.WellboreTVDTrajectoryID": null,
+ "VirtualProperties.DefaultLocation.SpatialGeometryTypeID": null,
+ "Source": null,
+ "AssociatedIdentities": ["opendes:master-data--Wellbore:nz-100000113552"],
+ "LogVersion": null,
+ "SeismicReferenceElevation.VerticalCRSID": null,
+ "VerticalMeasurement.VerticalMeasurementTypeID": null,
+ "ToolStringDescription": null,
+ "LogSource": null,
+ "SpatialPoint.CoordinateQualityCheckPerformedBy": null,
+ "IsRegular": null,
+ "DrillingFluidProperty": null,
+ "SeismicReferenceElevation.VerticalReferenceEntityID": null,
+ "VerticalMeasurement.WellboreTVDTrajectoryID": null,
+ "SpatialLocation.CoordinateQualityCheckPerformedBy": null,
+ "VerticalMeasurement.VerticalMeasurementDescription": null,
+ "SpatialPoint.SpatialParameterTypeID": null,
+ "WellboreID": "opendes:master-data--Wellbore:nz-100000113552:",
+ "ActivityType": null,
+ "VerticalMeasurement.VerticalMeasurementUnitOfMeasureID": null,
+ "SpatialPoint.QuantitativeAccuracyBandID": null,
+ "SamplingDomainTypeID": null,
+ "SpatialArea.CoordinateQualityCheckPerformedBy": null,
+ "ReferenceCurveID": "DEPTH",
+ "SpatialLocation.IsDecimated": false,
+ "WellboreName": "Kongahu-5",
+ "SpatialLocation.QualitativeSpatialAccuracyTypeID": null,
+ "VirtualProperties.DefaultLocation.SpatialParameterTypeID": null,
+ "ResourceHomeRegionID": null,
+ "FrameIdentifier": null,
+ "SeismicReferenceElevation.VerticalReferenceID": null,
+ "SeismicReferenceElevation.VerticalMeasurementSourceID": null,
+ "Curves": [{
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Measured%20Depth:",
+ "CurveID": "DEPTH",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "DEPTH",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:M:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": null,
+ "CurveID": "CT",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "CT",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:MH%2FM:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Conductivity%20-%20Flushed%20Zone:",
+ "CurveID": "CXO",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "CXO",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:MH%2FM:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Invasion%20Diameter:",
+ "CurveID": "DI",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "DI",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Fracture%20Pressure:",
+ "CurveID": "FPRESS",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "FPRESS",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:PSI:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Formation%20Temperature%20%28Estimate%29:",
+ "CurveID": "FTEMP",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "FTEMP",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:DEGC:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": null,
+ "CurveID": "HMC_POR",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "HMC_POR",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": null,
+ "CurveID": "HMC_RES",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "HMC_RES",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Mud%20Resistivity:",
+ "CurveID": "RM",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "RM",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Mudcake%20Resistivity:",
+ "CurveID": "RMC",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "RMC",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Mud%20Filtrate%20Resistivity:",
+ "CurveID": "RMF",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "RMF",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20True%20Formation:",
+ "CurveID": "RT",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "RT",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Flushed%20Zone:",
+ "CurveID": "RXO",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "RXO",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Volumetric%20Photoelectric%20Factor:",
+ "CurveID": "U",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "U",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:B%2FC3:",
+ "CurveDescription": null
+ }
+ ],
+ "SpatialLocation.QuantitativeAccuracyBandID": null
+}
diff --git a/indexer-core/src/test/resources/indexproperty/wellLog2.json b/indexer-core/src/test/resources/indexproperty/wellLog2.json
new file mode 100644
index 0000000000000000000000000000000000000000..7a071091f7a76dc83ccc2f5fb80a8438cd0a25a9
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/wellLog2.json
@@ -0,0 +1,352 @@
+{
+ "LogRemark": null,
+ "SpatialArea.QuantitativeAccuracyBandID": null,
+ "CompanyID": null,
+ "VirtualProperties.DefaultLocation.QuantitativeAccuracyBandID": null,
+ "SpatialArea.SpatialParameterTypeID": null,
+ "ResourceCurationStatus": null,
+ "SpatialArea.SpatialGeometryTypeID": null,
+ "IsExtendedLoad": null,
+ "Name": "EC_CURVES",
+ "VerticalMeasurement.VerticalCRSID": null,
+ "VirtualProperties.DefaultName": "EC_CURVES",
+ "SeismicReferenceElevation.VerticalMeasurementUnitOfMeasureID": null,
+ "VirtualProperties.DefaultLocation.CoordinateQualityCheckPerformedBy": null,
+ "ResourceSecurityClassification": null,
+ "SeismicReferenceElevation.VerticalMeasurementDescription": null,
+ "VerticalMeasurement.VerticalMeasurementPathID": null,
+ "SpatialLocation.SpatialParameterTypeID": null,
+ "ExistenceKind": null,
+ "HoleTypeLogging": null,
+ "LoggingDirection": null,
+ "VerticalMeasurementID": null,
+ "SeismicReferenceElevation.VerticalMeasurementPathID": null,
+ "SpatialArea.QualitativeSpatialAccuracyTypeID": null,
+ "VerticalMeasurement.VerticalReferenceEntityID": null,
+ "SpatialPoint.SpatialGeometryTypeID": null,
+ "SpatialLocation.SpatialGeometryTypeID": "opendes:reference-data--SpatialGeometryType:Point:",
+ "VerticalMeasurement.VerticalMeasurementSourceID": null,
+ "WellLogTypeID": null,
+ "IsDiscoverable": null,
+ "SeismicReferenceElevation.VerticalMeasurementTypeID": null,
+ "LoggingService": null,
+ "VirtualProperties.DefaultLocation.QualitativeSpatialAccuracyTypeID": null,
+ "VerticalMeasurement.VerticalReferenceID": null,
+ "SubmitterName": null,
+ "LogRun": null,
+ "SpatialPoint.QualitativeSpatialAccuracyTypeID": null,
+ "LogActivity": null,
+ "Description": null,
+ "ResourceLifecycleStatus": null,
+ "ServiceCompanyID": null,
+ "SpatialLocation.Wgs84Coordinates": {
+ "geometries": [{
+ "coordinates": [171.8745361, -41.2456122],
+ "type": "point"
+ }
+ ],
+ "type": "geometrycollection"
+ },
+ "TechnicalAssuranceID": null,
+ "SeismicReferenceElevation.WellboreTVDTrajectoryID": null,
+ "VirtualProperties.DefaultLocation.SpatialGeometryTypeID": null,
+ "Source": null,
+ "AssociatedIdentities": ["opendes:master-data--Wellbore:nz-100000113552"],
+ "LogVersion": null,
+ "SeismicReferenceElevation.VerticalCRSID": null,
+ "VerticalMeasurement.VerticalMeasurementTypeID": null,
+ "ToolStringDescription": null,
+ "LogSource": null,
+ "SpatialPoint.CoordinateQualityCheckPerformedBy": null,
+ "IsRegular": null,
+ "DrillingFluidProperty": null,
+ "SeismicReferenceElevation.VerticalReferenceEntityID": null,
+ "VerticalMeasurement.WellboreTVDTrajectoryID": null,
+ "SpatialLocation.CoordinateQualityCheckPerformedBy": null,
+ "VerticalMeasurement.VerticalMeasurementDescription": null,
+ "SpatialPoint.SpatialParameterTypeID": null,
+ "WellboreID": "opendes:master-data--Wellbore:nz-100000113552:",
+ "ActivityType": null,
+ "VerticalMeasurement.VerticalMeasurementUnitOfMeasureID": null,
+ "SpatialPoint.QuantitativeAccuracyBandID": null,
+ "SamplingDomainTypeID": null,
+ "SpatialArea.CoordinateQualityCheckPerformedBy": null,
+ "ReferenceCurveID": "DEPTH",
+ "SpatialLocation.IsDecimated": false,
+ "WellboreName": "Kongahu-5",
+ "SpatialLocation.QualitativeSpatialAccuracyTypeID": null,
+ "VirtualProperties.DefaultLocation.SpatialParameterTypeID": null,
+ "ResourceHomeRegionID": null,
+ "FrameIdentifier": null,
+ "SeismicReferenceElevation.VerticalReferenceID": null,
+ "SeismicReferenceElevation.VerticalMeasurementSourceID": null,
+ "Curves": [{
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Measured%20Depth:",
+ "CurveID": "DEPTH",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "DEPTH",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:M:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": null,
+ "CurveID": "ABC",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "CT",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:MH%2FM:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Conductivity%20-%20Flushed%20Zone:",
+ "CurveID": "CXO",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "CXO",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:MH%2FM:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Invasion%20Diameter:",
+ "CurveID": "DI",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "DI",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Fracture%20Pressure:",
+ "CurveID": "FPRESS",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "FPRESS",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:PSI:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Formation%20Temperature%20%28Estimate%29:",
+ "CurveID": "FTEMP",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "FTEMP",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:DEGC:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": null,
+ "CurveID": "HMC_POR",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "HMC_POR",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": null,
+ "CurveID": "HMC_RES",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "HMC_RES",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Mud%20Resistivity:",
+ "CurveID": "RM",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "RM",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Mudcake%20Resistivity:",
+ "CurveID": "RMC",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "RMC",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Mud%20Filtrate%20Resistivity:",
+ "CurveID": "RMF",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "RMF",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20True%20Formation:",
+ "CurveID": "RT",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "RT",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Flushed%20Zone:",
+ "CurveID": "RXO",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "RXO",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:",
+ "CurveDescription": null
+ }, {
+ "IsProcessed": null,
+ "LogCurveMainFamilyID": null,
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Volumetric%20Photoelectric%20Factor:",
+ "CurveID": "U",
+ "CurveVersion": null,
+ "CurveSampleTypeID": null,
+ "InterpreterName": null,
+ "CurveQuality": null,
+ "NullValue": null,
+ "Interpolate": null,
+ "DepthUnit": null,
+ "DepthCoding": null,
+ "Mnemonic": "U",
+ "LogCurveTypeID": null,
+ "LogCurveBusinessValueID": null,
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:B%2FC3:",
+ "CurveDescription": null
+ }
+ ],
+ "SpatialLocation.QuantitativeAccuracyBandID": null
+}
diff --git a/indexer-core/src/test/resources/indexproperty/well_configuration_record.json b/indexer-core/src/test/resources/indexproperty/well_configuration_record.json
new file mode 100644
index 0000000000000000000000000000000000000000..0cf4995d3cdd40b537d685b2fd7af394605df845
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/well_configuration_record.json
@@ -0,0 +1,56 @@
+{
+ "AttributionPublication": null,
+ "InactiveIndicator": null,
+ "Description": "The index property list for master-data--Well:1., valid for all master-data--Well kinds for major version 1.",
+ "ResourceLifecycleStatus": null,
+ "ResourceCurationStatus": null,
+ "TechnicalAssuranceID": null,
+ "Code": "osdu:wks:master-data--Well:1.",
+ "Source": null,
+ "Name": "Well-IndexPropertyPathConfiguration",
+ "AttributionAuthority": "OSDU",
+ "ResourceHomeRegionID": null,
+ "VirtualProperties.DefaultName": "Well-IndexPropertyPathConfiguration",
+ "AttributionRevision": null,
+ "ResourceSecurityClassification": null,
+ "ID": null,
+ "ExistenceKind": null,
+ "Configurations": [{
+ "Policy": "ExtractAllMatches",
+ "UseCase": "As a user I want to find objects by a country name, with the understanding that an object may extend over country boundaries.",
+ "Paths": [{
+ "RelatedObjectsSpec.RelatedObjectID": "data.GeoContexts[].GeoPoliticalEntityID",
+ "RelatedObjectsSpec.RelatedConditionMatches": [
+ "opendes:reference-data--GeoPoliticalEntityType:Country:"
+ ],
+ "ValueExtraction.ValuePath": "data.GeoPoliticalEntityName",
+ "RelatedObjectsSpec.RelatedObjectKind": "osdu:wks:master-data--GeoPoliticalEntity:1.",
+ "RelatedObjectsSpec.RelatedConditionProperty": "data.GeoContexts[].GeoTypeID",
+ "RelatedObjectsSpec.RelationshipDirection": "ChildToParent",
+ "ValueExtraction.RelatedConditionProperty": null
+ }
+ ],
+ "Name": "CountryNames"
+ }, {
+ "Policy": "ExtractFirstMatch",
+ "UseCase": "As a user I want to discover and match Wells by their UWI. I am aware that this is not globally reliable, however, I am able to specify a prioritized AliasNameType list to look up value in the NameAliases array.",
+ "Paths": [{
+ "RelatedObjectsSpec.RelatedObjectID": null,
+ "ValueExtraction.ValuePath": "data.NameAliases[].AliasName",
+ "RelatedObjectsSpec.RelatedObjectKind": null,
+ "RelatedObjectsSpec.RelatedConditionProperty": null,
+ "RelatedObjectsSpec.RelationshipDirection": null,
+ "ValueExtraction.RelatedConditionMatches": [
+ "opendes:reference-data--AliasNameType:UniqueIdentifier:",
+ "opendes:reference-data--AliasNameType:RegulatoryName:",
+ "opendes:reference-data--AliasNameType:PreferredName:",
+ "opendes:reference-data--AliasNameType:CommonName:",
+ "opendes:reference-data--AliasNameType:ShortName:"
+ ],
+ "ValueExtraction.RelatedConditionProperty": "data.NameAliases[].AliasNameTypeID"
+ }
+ ],
+ "Name": "WellUWI"
+ }
+ ]
+}
diff --git a/indexer-core/src/test/resources/indexproperty/well_storage_schema.json b/indexer-core/src/test/resources/indexproperty/well_storage_schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..ee6178a3868902da8681181b0bf2016ef8e93394
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/well_storage_schema.json
@@ -0,0 +1,314 @@
+{
+ "kind": "osdu:wks:master-data--Well:1.1.0",
+ "schema": [{
+ "path": "ResourceHomeRegionID",
+ "kind": "string"
+ }, {
+ "path": "ResourceHostRegionIDs",
+ "kind": "[]string"
+ }, {
+ "path": "ResourceLifecycleStatus",
+ "kind": "string"
+ }, {
+ "path": "ResourceSecurityClassification",
+ "kind": "string"
+ }, {
+ "path": "ResourceCurationStatus",
+ "kind": "string"
+ }, {
+ "path": "ExistenceKind",
+ "kind": "string"
+ }, {
+ "path": "TechnicalAssuranceID",
+ "kind": "string"
+ }, {
+ "path": "Source",
+ "kind": "string"
+ }, {
+ "path": "NameAliases",
+ "kind": "nested",
+ "properties": [{
+ "path": "AliasNameTypeID",
+ "kind": "string"
+ }, {
+ "path": "EffectiveDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "AliasName",
+ "kind": "string"
+ }, {
+ "path": "TerminationDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "DefinitionOrganisationID",
+ "kind": "string"
+ }
+ ]
+ }, {
+ "path": "SpatialLocation.SpatialParameterTypeID",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.QuantitativeAccuracyBandID",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.CoordinateQualityCheckRemarks",
+ "kind": "[]string"
+ }, {
+ "path": "SpatialLocation.AppliedOperations",
+ "kind": "[]string"
+ }, {
+ "path": "SpatialLocation.QualitativeSpatialAccuracyTypeID",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.CoordinateQualityCheckPerformedBy",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.SpatialLocationCoordinatesDate",
+ "kind": "datetime"
+ }, {
+ "path": "SpatialLocation.CoordinateQualityCheckDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "SpatialLocation.Wgs84Coordinates",
+ "kind": "core:dl:geoshape:1.0.0"
+ }, {
+ "path": "SpatialLocation.SpatialGeometryTypeID",
+ "kind": "string"
+ }, {
+ "path": "VersionCreationReason",
+ "kind": "string"
+ }, {
+ "path": "TechnicalAssuranceTypeID",
+ "kind": "string"
+ }, {
+ "path": "GeoContexts",
+ "kind": "nested",
+ "properties": [{
+ "path": "GeoPoliticalEntityID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "BasinID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "FieldID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "PlayID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "ProspectID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }
+ ]
+ }, {
+ "path": "FacilityStates",
+ "kind": "nested",
+ "properties": [{
+ "path": "EffectiveDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "FacilityStateTypeID",
+ "kind": "string"
+ }, {
+ "path": "TerminationDateTime",
+ "kind": "datetime"
+ }
+ ]
+ }, {
+ "path": "FacilityID",
+ "kind": "string"
+ }, {
+ "path": "OperatingEnvironmentID",
+ "kind": "string"
+ }, {
+ "path": "FacilityNameAliases",
+ "kind": "[]object"
+ }, {
+ "path": "FacilityEvents",
+ "kind": "nested",
+ "properties": [{
+ "path": "EffectiveDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "TerminationDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "FacilityEventTypeID",
+ "kind": "string"
+ }
+ ]
+ }, {
+ "path": "FacilitySpecifications",
+ "kind": "flattened"
+ }, {
+ "path": "DataSourceOrganisationID",
+ "kind": "string"
+ }, {
+ "path": "InitialOperatorID",
+ "kind": "string"
+ }, {
+ "path": "CurrentOperatorID",
+ "kind": "string"
+ }, {
+ "path": "FacilityOperators",
+ "kind": "nested",
+ "properties": [{
+ "path": "FacilityOperatorID",
+ "kind": "string"
+ }, {
+ "path": "EffectiveDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "FacilityOperatorOrganisationID",
+ "kind": "string"
+ }, {
+ "path": "TerminationDateTime",
+ "kind": "datetime"
+ }
+ ]
+ }, {
+ "path": "FacilityName",
+ "kind": "string"
+ }, {
+ "path": "FacilityTypeID",
+ "kind": "string"
+ }, {
+ "path": "BusinessIntentionID",
+ "kind": "string"
+ }, {
+ "path": "DefaultVerticalCRSID",
+ "kind": "string"
+ }, {
+ "path": "WasBusinessInterestFinancialNonOperated",
+ "kind": "bool"
+ }, {
+ "path": "DefaultVerticalMeasurementID",
+ "kind": "string"
+ }, {
+ "path": "WasBusinessInterestObligatory",
+ "kind": "bool"
+ }, {
+ "path": "RoleID",
+ "kind": "string"
+ }, {
+ "path": "WasBusinessInterestTechnical",
+ "kind": "bool"
+ }, {
+ "path": "OutcomeID",
+ "kind": "string"
+ }, {
+ "path": "VerticalMeasurements",
+ "kind": "nested",
+ "properties": [{
+ "path": "VerticalMeasurementID",
+ "kind": "string"
+ }, {
+ "path": "RigID",
+ "kind": "string"
+ }, {
+ "path": "WellboreTVDTrajectoryID",
+ "kind": "string"
+ }, {
+ "path": "VerticalCRSID",
+ "kind": "string"
+ }, {
+ "path": "VerticalMeasurementSourceID",
+ "kind": "string"
+ }, {
+ "path": "VerticalReferenceID",
+ "kind": "string"
+ }, {
+ "path": "TerminationDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "VerticalMeasurementPathID",
+ "kind": "string"
+ }, {
+ "path": "EffectiveDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "VerticalMeasurement",
+ "kind": "double"
+ }, {
+ "path": "VerticalMeasurementTypeID",
+ "kind": "string"
+ }, {
+ "path": "VerticalMeasurementDescription",
+ "kind": "string"
+ }, {
+ "path": "VerticalMeasurementUnitOfMeasureID",
+ "kind": "string"
+ }, {
+ "path": "VerticalReferenceEntityID",
+ "kind": "string"
+ }
+ ]
+ }, {
+ "path": "HistoricalInterests",
+ "kind": "[]object"
+ }, {
+ "path": "StatusSummaryID",
+ "kind": "string"
+ }, {
+ "path": "InterestTypeID",
+ "kind": "string"
+ }, {
+ "path": "ConditionID",
+ "kind": "string"
+ }, {
+ "path": "WasBusinessInterestFinancialOperated",
+ "kind": "bool"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.SpatialParameterTypeID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.QuantitativeAccuracyBandID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.CoordinateQualityCheckRemarks",
+ "kind": "[]string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.AppliedOperations",
+ "kind": "[]string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.QualitativeSpatialAccuracyTypeID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.CoordinateQualityCheckPerformedBy",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.SpatialLocationCoordinatesDate",
+ "kind": "datetime"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.CoordinateQualityCheckDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.Wgs84Coordinates",
+ "kind": "core:dl:geoshape:1.0.0"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.SpatialGeometryTypeID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultName",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.IsDecimated",
+ "kind": "boolean"
+ }
+ ]
+}
diff --git a/indexer-core/src/test/resources/indexproperty/wellbore_configuration_record.json b/indexer-core/src/test/resources/indexproperty/wellbore_configuration_record.json
new file mode 100644
index 0000000000000000000000000000000000000000..0eb75a54ad5af31e9d7e3143e8689754af495ce1
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/wellbore_configuration_record.json
@@ -0,0 +1,33 @@
+{
+ "AttributionPublication": null,
+ "InactiveIndicator": null,
+ "Description": "The index property list for master-data--Wellbore:1., valid for all master-data--Wellbore kinds for major version 1.",
+ "ResourceLifecycleStatus": null,
+ "ResourceCurationStatus": null,
+ "TechnicalAssuranceID": null,
+ "Code": "osdu:wks:master-data--Wellbore:1.",
+ "Source": null,
+ "Name": "Well-IndexPropertyPathConfiguration",
+ "AttributionAuthority": "OSDU",
+ "ResourceHomeRegionID": null,
+ "VirtualProperties.DefaultName": "Well-IndexPropertyPathConfiguration",
+ "AttributionRevision": null,
+ "ResourceSecurityClassification": null,
+ "ID": null,
+ "ExistenceKind": null,
+ "Configurations": [{
+ "Policy": "ExtractAllMatches",
+ "UseCase": "As a user I want to find wellbores by well log type(s).",
+ "Paths": [{
+ "RelatedObjectsSpec.RelatedObjectID": "data.WellboreID",
+ "ValueExtraction.ValuePath": "data.Curves[].Mnemonic",
+ "RelatedObjectsSpec.RelatedObjectKind": "osdu:wks:work-product-component--WellLog:1.",
+ "RelatedObjectsSpec.RelatedConditionProperty": null,
+ "RelatedObjectsSpec.RelationshipDirection": "ParentToChildren",
+ "ValueExtraction.RelatedConditionProperty": null
+ }
+ ],
+ "Name": "WellLogs"
+ }
+ ]
+}
diff --git a/indexer-core/src/test/resources/indexproperty/wellbore_data.json b/indexer-core/src/test/resources/indexproperty/wellbore_data.json
new file mode 100644
index 0000000000000000000000000000000000000000..1150d64100d254247eea99305faeb131b79a5437
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/wellbore_data.json
@@ -0,0 +1,65 @@
+{
+ "DefaultVerticalMeasurementID": "KB",
+ "FacilityName": "Kongahu-1",
+ "VirtualProperties.DefaultName": "Kongahu-1",
+ "FacilityTypeID": "opendes:reference-data--FacilityType:Wellbore:",
+ "FacilityEvents": [{
+ "EffectiveDateTime": "1984-06-21T00:00:00+0000",
+ "FacilityEventTypeID": "opendes:reference-data--FacilityEventType:Spud:"
+ }
+ ],
+ "VirtualProperties.DefaultLocation.IsDecimated": false,
+ "CurrentOperatorID": "opendes:master-data--Organisation:HOME%20ENERGY%20NZ%20LTD:",
+ "SpatialLocation.SpatialGeometryTypeID": "opendes:reference-data--SpatialGeometryType:Point:",
+ "WellID": "opendes:master-data--Well:1816bd81",
+ "SpatialLocation.Wgs84Coordinates": {
+ "type": "geometrycollection",
+ "geometries": [{
+ "type": "point",
+ "coordinates": [171.8745361, -41.2456122]
+ }
+ ]
+ },
+ "TechnicalAssuranceID": "opendes:reference-data--TechnicalAssuranceType:Unsuitable:",
+ "VirtualProperties.DefaultLocation.SpatialGeometryTypeID": "opendes:reference-data--SpatialGeometryType:Point:",
+ "DefinitiveTrajectoryID": "opendes:work-product-component--WellboreTrajectory:nz-100000113552:",
+ "VerticalMeasurements": [{
+ "VerticalMeasurementID": "KB",
+ "VerticalMeasurementPathID": "opendes:reference-data--VerticalMeasurementPath:ELEV:",
+ "VerticalMeasurement": 26.0,
+ "VerticalMeasurementTypeID": "opendes:reference-data--VerticalMeasurementType:KB:",
+ "VerticalMeasurementUnitOfMeasureID": "opendes:reference-data--UnitOfMeasure:m:"
+ }, {
+ "VerticalMeasurementID": "Total Depth MD",
+ "VerticalMeasurementPathID": "opendes:reference-data--VerticalMeasurementPath:MD:",
+ "VerticalMeasurement": 2015.5,
+ "VerticalMeasurementUnitOfMeasureID": "opendes:reference-data--UnitOfMeasure:m:"
+ }, {
+ "VerticalMeasurementID": "Well Head Elevation",
+ "VerticalMeasurementPathID": "opendes:reference-data--VerticalMeasurementPath:ELEV:",
+ "VerticalMeasurement": 26.0,
+ "VerticalMeasurementUnitOfMeasureID": "opendes:reference-data--UnitOfMeasure:m:"
+ }
+ ],
+ "NameAliases": [{
+ "AliasName": "100000113552",
+ "AliasNameTypeID": "opendes:reference-data--AliasNameType:UniqueIdentifier:"
+ }
+ ],
+ "GeoContexts": [{
+ "GeoPoliticalEntityID": "opendes:master-data--GeoPoliticalEntity:New%20Zealand:",
+ "GeoTypeID": "opendes:reference-data--GeoPoliticalEntityType:Country:"
+ }, {
+ "GeoPoliticalEntityID": "opendes:master-data--GeoPoliticalEntity:38058:",
+ "GeoTypeID": "opendes:reference-data--GeoPoliticalEntityType:LicenseBlock:"
+ }
+ ],
+ "VirtualProperties.DefaultLocation.Wgs84Coordinates": {
+ "type": "geometrycollection",
+ "geometries": [{
+ "type": "point",
+ "coordinates": [171.8745361, -41.2456122]
+ }
+ ]
+ }
+}
diff --git a/indexer-core/src/test/resources/indexproperty/wellbore_extended_data.json b/indexer-core/src/test/resources/indexproperty/wellbore_extended_data.json
new file mode 100644
index 0000000000000000000000000000000000000000..0ed1e2b9b88266ccd9fe24960b7281454efa094d
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/wellbore_extended_data.json
@@ -0,0 +1,3 @@
+{
+ "WellLogs": ["BS", "BS_2", "BS_3", "BS_4", "BS_5", "BS_6", "CALI", "CALI_2", "CALI_3", "CALI_4", "CALI_5", "CALI_6", "CT", "CXO", "DENS", "DENS_CORR", "DEPTH", "DI", "DRHO", "DT", "DTC", "DT_2", "DT_3", "DT_4", "FPRESS", "FTEMP", "GR", "GR_10", "GR_2", "GR_3", "GR_4", "GR_5", "GR_6", "GR_7", "GR_8", "GR_9", "GR_CORR", "HMC_POR", "HMC_RES", "LLD", "LLD_2", "LLD_3", "LLD_4", "LLS", "LLS_2", "LLS_3", "LLS_4", "MINV", "MINV_2", "MINV_3", "MINV_4", "MNOR", "MNOR_2", "MNOR_3", "MNOR_4", "MSFL", "MSFL_2", "MSFL_3", "MSFL_4", "NEUT", "NEUT_CORR", "NPHI", "PEF", "RESD", "RESD_CORR", "RESS", "RESS_CORR", "RHOB", "RM", "RMC", "RMF", "RT", "RXO", "SP", "SP_2", "SP_3", "SP_4", "TENS", "TENS_10", "TENS_2", "TENS_3", "TENS_4", "TENS_5", "TENS_6", "TENS_7", "TENS_8", "TENS_9", "U"]
+}
diff --git a/indexer-core/src/test/resources/indexproperty/wellbore_storage_schema.json b/indexer-core/src/test/resources/indexproperty/wellbore_storage_schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..3d52ae29a04d50b3bc48a27b778ed944f3e4666f
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/wellbore_storage_schema.json
@@ -0,0 +1,442 @@
+{
+ "kind": "osdu:wks:master-data--Wellbore:1.3.0",
+ "schema": [{
+ "path": "ResourceHomeRegionID",
+ "kind": "string"
+ }, {
+ "path": "ResourceHostRegionIDs",
+ "kind": "[]string"
+ }, {
+ "path": "ResourceLifecycleStatus",
+ "kind": "string"
+ }, {
+ "path": "ResourceSecurityClassification",
+ "kind": "string"
+ }, {
+ "path": "ResourceCurationStatus",
+ "kind": "string"
+ }, {
+ "path": "ExistenceKind",
+ "kind": "string"
+ }, {
+ "path": "TechnicalAssuranceID",
+ "kind": "string"
+ }, {
+ "path": "Source",
+ "kind": "string"
+ }, {
+ "path": "NameAliases",
+ "kind": "nested",
+ "properties": [{
+ "path": "AliasNameTypeID",
+ "kind": "string"
+ }, {
+ "path": "EffectiveDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "AliasName",
+ "kind": "string"
+ }, {
+ "path": "TerminationDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "DefinitionOrganisationID",
+ "kind": "string"
+ }
+ ]
+ }, {
+ "path": "SpatialLocation.SpatialParameterTypeID",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.QuantitativeAccuracyBandID",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.CoordinateQualityCheckRemarks",
+ "kind": "[]string"
+ }, {
+ "path": "SpatialLocation.AppliedOperations",
+ "kind": "[]string"
+ }, {
+ "path": "SpatialLocation.QualitativeSpatialAccuracyTypeID",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.CoordinateQualityCheckPerformedBy",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.SpatialLocationCoordinatesDate",
+ "kind": "datetime"
+ }, {
+ "path": "SpatialLocation.CoordinateQualityCheckDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "SpatialLocation.Wgs84Coordinates",
+ "kind": "core:dl:geoshape:1.0.0"
+ }, {
+ "path": "SpatialLocation.SpatialGeometryTypeID",
+ "kind": "string"
+ }, {
+ "path": "VersionCreationReason",
+ "kind": "string"
+ }, {
+ "path": "TechnicalAssuranceTypeID",
+ "kind": "string"
+ }, {
+ "path": "GeoContexts",
+ "kind": "nested",
+ "properties": [{
+ "path": "GeoPoliticalEntityID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "BasinID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "FieldID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "PlayID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "ProspectID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }
+ ]
+ }, {
+ "path": "FacilityStates",
+ "kind": "nested",
+ "properties": [{
+ "path": "EffectiveDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "FacilityStateTypeID",
+ "kind": "string"
+ }, {
+ "path": "TerminationDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "Remark",
+ "kind": "string"
+ }
+ ]
+ }, {
+ "path": "FacilityNameAliases",
+ "kind": "[]object"
+ }, {
+ "path": "FacilityEvents",
+ "kind": "nested",
+ "properties": [{
+ "path": "EffectiveDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "FacilityEventTypeID",
+ "kind": "string"
+ }, {
+ "path": "TerminationDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "Remark",
+ "kind": "string"
+ }
+ ]
+ }, {
+ "path": "CurrentOperatorID",
+ "kind": "string"
+ }, {
+ "path": "FacilityName",
+ "kind": "string"
+ }, {
+ "path": "FacilityID",
+ "kind": "string"
+ }, {
+ "path": "OperatingEnvironmentID",
+ "kind": "string"
+ }, {
+ "path": "FacilitySpecifications",
+ "kind": "flattened"
+ }, {
+ "path": "DataSourceOrganisationID",
+ "kind": "string"
+ }, {
+ "path": "FacilityDescription",
+ "kind": "string"
+ }, {
+ "path": "InitialOperatorID",
+ "kind": "string"
+ }, {
+ "path": "FacilityOperators",
+ "kind": "nested",
+ "properties": [{
+ "path": "FacilityOperatorID",
+ "kind": "string"
+ }, {
+ "path": "EffectiveDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "FacilityOperatorOrganisationID",
+ "kind": "string"
+ }, {
+ "path": "TerminationDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "Remark",
+ "kind": "string"
+ }
+ ]
+ }, {
+ "path": "FacilityTypeID",
+ "kind": "string"
+ }, {
+ "path": "FluidDirectionID",
+ "kind": "string"
+ }, {
+ "path": "TargetFormation",
+ "kind": "string"
+ }, {
+ "path": "FormationNameAtTotalDepth",
+ "kind": "string"
+ }, {
+ "path": "DefaultVerticalMeasurementID",
+ "kind": "string"
+ }, {
+ "path": "WasBusinessInterestObligatory",
+ "kind": "bool"
+ }, {
+ "path": "RoleID",
+ "kind": "string"
+ }, {
+ "path": "DefinitiveTrajectoryID",
+ "kind": "string"
+ }, {
+ "path": "GeographicBottomHoleLocation.SpatialParameterTypeID",
+ "kind": "string"
+ }, {
+ "path": "GeographicBottomHoleLocation.QuantitativeAccuracyBandID",
+ "kind": "string"
+ }, {
+ "path": "GeographicBottomHoleLocation.CoordinateQualityCheckRemarks",
+ "kind": "[]string"
+ }, {
+ "path": "GeographicBottomHoleLocation.AppliedOperations",
+ "kind": "[]string"
+ }, {
+ "path": "GeographicBottomHoleLocation.QualitativeSpatialAccuracyTypeID",
+ "kind": "string"
+ }, {
+ "path": "GeographicBottomHoleLocation.CoordinateQualityCheckPerformedBy",
+ "kind": "string"
+ }, {
+ "path": "GeographicBottomHoleLocation.SpatialLocationCoordinatesDate",
+ "kind": "datetime"
+ }, {
+ "path": "GeographicBottomHoleLocation.CoordinateQualityCheckDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "GeographicBottomHoleLocation.Wgs84Coordinates",
+ "kind": "core:dl:geoshape:1.0.0"
+ }, {
+ "path": "GeographicBottomHoleLocation.SpatialGeometryTypeID",
+ "kind": "string"
+ }, {
+ "path": "DrillingReasons",
+ "kind": "[]object"
+ }, {
+ "path": "OutcomeID",
+ "kind": "string"
+ }, {
+ "path": "VerticalMeasurements",
+ "kind": "nested",
+ "properties": [{
+ "path": "VerticalMeasurementID",
+ "kind": "string"
+ }, {
+ "path": "RigID",
+ "kind": "string"
+ }, {
+ "path": "WellboreTVDTrajectoryID",
+ "kind": "string"
+ }, {
+ "path": "VerticalCRSID",
+ "kind": "string"
+ }, {
+ "path": "VerticalMeasurementSourceID",
+ "kind": "string"
+ }, {
+ "path": "VerticalReferenceID",
+ "kind": "string"
+ }, {
+ "path": "TerminationDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "VerticalMeasurementPathID",
+ "kind": "string"
+ }, {
+ "path": "EffectiveDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "VerticalMeasurement",
+ "kind": "double"
+ }, {
+ "path": "VerticalMeasurementTypeID",
+ "kind": "string"
+ }, {
+ "path": "VerticalMeasurementDescription",
+ "kind": "string"
+ }, {
+ "path": "VerticalMeasurementUnitOfMeasureID",
+ "kind": "string"
+ }, {
+ "path": "VerticalReferenceEntityID",
+ "kind": "string"
+ }
+ ]
+ }, {
+ "path": "WellboreCosts",
+ "kind": "nested",
+ "properties": [{
+ "path": "ActivityTypeID",
+ "kind": "string"
+ }, {
+ "path": "Cost",
+ "kind": "double"
+ }
+ ]
+ }, {
+ "path": "PrimaryProductTypeID",
+ "kind": "string"
+ }, {
+ "path": "SequenceNumber",
+ "kind": "int"
+ }, {
+ "path": "InterestTypeID",
+ "kind": "string"
+ }, {
+ "path": "KickOffWellbore",
+ "kind": "string"
+ }, {
+ "path": "TertiaryProductTypeID",
+ "kind": "string"
+ }, {
+ "path": "ShowProductTypeID",
+ "kind": "string"
+ }, {
+ "path": "BusinessIntentionID",
+ "kind": "string"
+ }, {
+ "path": "WasBusinessInterestFinancialNonOperated",
+ "kind": "bool"
+ }, {
+ "path": "ProjectedBottomHoleLocation.SpatialParameterTypeID",
+ "kind": "string"
+ }, {
+ "path": "ProjectedBottomHoleLocation.QuantitativeAccuracyBandID",
+ "kind": "string"
+ }, {
+ "path": "ProjectedBottomHoleLocation.CoordinateQualityCheckRemarks",
+ "kind": "[]string"
+ }, {
+ "path": "ProjectedBottomHoleLocation.AppliedOperations",
+ "kind": "[]string"
+ }, {
+ "path": "ProjectedBottomHoleLocation.QualitativeSpatialAccuracyTypeID",
+ "kind": "string"
+ }, {
+ "path": "ProjectedBottomHoleLocation.CoordinateQualityCheckPerformedBy",
+ "kind": "string"
+ }, {
+ "path": "ProjectedBottomHoleLocation.SpatialLocationCoordinatesDate",
+ "kind": "datetime"
+ }, {
+ "path": "ProjectedBottomHoleLocation.CoordinateQualityCheckDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "ProjectedBottomHoleLocation.Wgs84Coordinates",
+ "kind": "core:dl:geoshape:1.0.0"
+ }, {
+ "path": "ProjectedBottomHoleLocation.SpatialGeometryTypeID",
+ "kind": "string"
+ }, {
+ "path": "WasBusinessInterestTechnical",
+ "kind": "bool"
+ }, {
+ "path": "WellboreReasonID",
+ "kind": "string"
+ }, {
+ "path": "WellboreTrajectoryTypeID",
+ "kind": "string"
+ }, {
+ "path": "PrimaryMaterialID",
+ "kind": "string"
+ }, {
+ "path": "HistoricalInterests",
+ "kind": "[]object"
+ }, {
+ "path": "StatusSummaryID",
+ "kind": "string"
+ }, {
+ "path": "ConditionID",
+ "kind": "string"
+ }, {
+ "path": "WasBusinessInterestFinancialOperated",
+ "kind": "bool"
+ }, {
+ "path": "SecondaryProductTypeID",
+ "kind": "string"
+ }, {
+ "path": "WellID",
+ "kind": "string"
+ }, {
+ "path": "TrajectoryTypeID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.SpatialParameterTypeID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.QuantitativeAccuracyBandID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.CoordinateQualityCheckRemarks",
+ "kind": "[]string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.AppliedOperations",
+ "kind": "[]string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.QualitativeSpatialAccuracyTypeID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.CoordinateQualityCheckPerformedBy",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.SpatialLocationCoordinatesDate",
+ "kind": "datetime"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.CoordinateQualityCheckDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.Wgs84Coordinates",
+ "kind": "core:dl:geoshape:1.0.0"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.SpatialGeometryTypeID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultName",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.IsDecimated",
+ "kind": "boolean"
+ }
+ ]
+}
diff --git a/indexer-core/src/test/resources/indexproperty/welllog_configuration_record.json b/indexer-core/src/test/resources/indexproperty/welllog_configuration_record.json
new file mode 100644
index 0000000000000000000000000000000000000000..2807dd86384ffad35ce5d75e09e94e414b20bd07
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/welllog_configuration_record.json
@@ -0,0 +1,100 @@
+{
+ "AttributionPublication": null,
+ "InactiveIndicator": null,
+ "Description": "The index property list for WellLog:1., valid for all WellLog kinds for major version 1.",
+ "ResourceLifecycleStatus": null,
+ "ResourceCurationStatus": null,
+ "TechnicalAssuranceID": null,
+ "Code": "osdu:wks:work-product-component--WellLog:1.",
+ "Source": null,
+ "Name": "WellLogIndex-PropertyPathConfiguration",
+ "AttributionAuthority": "OSDU",
+ "ResourceHomeRegionID": null,
+ "VirtualProperties.DefaultName": "WellLogIndex-PropertyPathConfiguration",
+ "AttributionRevision": null,
+ "ResourceSecurityClassification": null,
+ "ID": null,
+ "ExistenceKind": null,
+ "Configurations": [{
+ "Policy": "ExtractFirstMatch",
+ "UseCase": "As a user I want to discover WellLog instances by the wellbore's name value.",
+ "Paths": [{
+ "RelatedObjectsSpec.RelatedObjectID": "data.WellboreID",
+ "ValueExtraction.ValuePath": "data.VirtualProperties.DefaultName",
+ "RelatedObjectsSpec.RelatedObjectKind": "osdu:wks:master-data--Wellbore:1.",
+ "RelatedObjectsSpec.RelatedConditionProperty": null,
+ "RelatedObjectsSpec.RelationshipDirection": "ChildToParent",
+ "ValueExtraction.RelatedConditionProperty": null
+ }, {
+ "RelatedObjectsSpec.RelatedObjectID": "data.WellboreID",
+ "ValueExtraction.ValuePath": "data.FacilityName",
+ "RelatedObjectsSpec.RelatedObjectKind": "osdu:wks:master-data--Wellbore:1.",
+ "RelatedObjectsSpec.RelatedConditionProperty": null,
+ "RelatedObjectsSpec.RelationshipDirection": "ChildToParent",
+ "ValueExtraction.RelatedConditionProperty": null
+ }
+ ],
+ "Name": "WellboreName"
+ }, {
+ "Policy": "ExtractFirstMatch",
+ "UseCase": "As a user I want to discover WellLog instances by spatial location.",
+ "Paths": [{
+ "RelatedObjectsSpec.RelatedObjectID": "data.WellboreID",
+ "ValueExtraction.ValuePath": "data.VirtualProperties.DefaultLocation",
+ "RelatedObjectsSpec.RelatedObjectKind": "osdu:wks:master-data--Wellbore:1.",
+ "RelatedObjectsSpec.RelatedConditionProperty": null,
+ "RelatedObjectsSpec.RelationshipDirection": "ChildToParent",
+ "ValueExtraction.RelatedConditionProperty": null
+ }, {
+ "RelatedObjectsSpec.RelatedObjectID": "data.WellboreID",
+ "ValueExtraction.ValuePath": "data.ProjectedBottomHoleLocation",
+ "RelatedObjectsSpec.RelatedObjectKind": "osdu:wks:master-data--Wellbore:1.",
+ "RelatedObjectsSpec.RelatedConditionProperty": null,
+ "RelatedObjectsSpec.RelationshipDirection": "ChildToParent",
+ "ValueExtraction.RelatedConditionProperty": null
+ }, {
+ "RelatedObjectsSpec.RelatedObjectID": "data.WellboreID",
+ "ValueExtraction.ValuePath": "data.GeographicBottomHoleLocation",
+ "RelatedObjectsSpec.RelatedObjectKind": "osdu:wks:master-data--Wellbore:1.",
+ "RelatedObjectsSpec.RelatedConditionProperty": null,
+ "RelatedObjectsSpec.RelationshipDirection": "ChildToParent",
+ "ValueExtraction.RelatedConditionProperty": null
+ }, {
+ "RelatedObjectsSpec.RelatedObjectID": "data.WellboreID",
+ "ValueExtraction.ValuePath": "data.SpatialLocation",
+ "RelatedObjectsSpec.RelatedObjectKind": "osdu:wks:master-data--Wellbore:1.",
+ "RelatedObjectsSpec.RelatedConditionProperty": null,
+ "RelatedObjectsSpec.RelationshipDirection": "ChildToParent",
+ "ValueExtraction.RelatedConditionProperty": null
+ }
+ ],
+ "Name": "SpatialLocation"
+ }, {
+ "Policy": "ExtractAllMatches",
+ "UseCase": "As a user I want to discover WellLog instances by the Technical Assurance reviewer name.",
+ "Paths": [{
+ "RelatedObjectsSpec.RelatedObjectID": null,
+ "ValueExtraction.ValuePath": "data.TechnicalAssurances[].Reviewers[].Name",
+ "RelatedObjectsSpec.RelatedObjectKind": null,
+ "RelatedObjectsSpec.RelatedConditionProperty": null,
+ "RelatedObjectsSpec.RelationshipDirection": null,
+ "ValueExtraction.RelatedConditionProperty": null
+ }
+ ],
+ "Name": "TechnicalAssuranceReviewerNames"
+ }, {
+ "Policy": "ExtractAllMatches",
+ "UseCase": "As a user I want to discover WellLog instances by the Technical Assurance Organisation name.",
+ "Paths": [{
+ "RelatedObjectsSpec.RelatedObjectID": "data.TechnicalAssurances[].Reviewers[].OrganisationID",
+ "ValueExtraction.ValuePath": "data.OrganisationName",
+ "RelatedObjectsSpec.RelatedObjectKind": "osdu:wks:master-data--Organisation:1.",
+ "RelatedObjectsSpec.RelatedConditionProperty": null,
+ "RelatedObjectsSpec.RelationshipDirection": "ChildToParent",
+ "ValueExtraction.RelatedConditionProperty": null
+ }
+ ],
+ "Name": "TechnicalAssuranceReviewerOrganisationNames"
+ }
+ ]
+}
diff --git a/indexer-core/src/test/resources/indexproperty/welllog_extended_data.json b/indexer-core/src/test/resources/indexproperty/welllog_extended_data.json
new file mode 100644
index 0000000000000000000000000000000000000000..a4e6bece65b88730b2503aeadf8e457e404ae16a
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/welllog_extended_data.json
@@ -0,0 +1,16 @@
+{
+ "SpatialLocation.SpatialGeometryTypeID": "opendes:reference-data--SpatialGeometryType:Point:",
+ "TechnicalAssuranceReviewerOrganisationNames": ["BigOil SeismicInterpretation Department (original)", "BigOil SeismicProcessing Department (original)"],
+ "SpatialLocation.Wgs84Coordinates": {
+ "geometries": [{
+ "coordinates": [171.8745361, -41.2456122],
+ "type": "point"
+ }
+ ],
+ "type": "geometrycollection"
+ },
+ "SpatialLocation.IsDecimated": false,
+ "TechnicalAssuranceReviewerNames": ["BigOil Department SeismicInterpretation (denormalized)", "BigOil Department SeismicProcessing (denormalized)"],
+ "WellboreName": "Kongahu-1",
+ "AssociatedIdentities": ["opendes:master-data--Organisation:BigOil-Department-SeismicProcessing", "opendes:master-data--Wellbore:nz-100000113552", "opendes:master-data--Organisation:BigOil-Department-SeismicInterpretation"]
+}
diff --git a/indexer-core/src/test/resources/indexproperty/welllog_extended_schema_items.json b/indexer-core/src/test/resources/indexproperty/welllog_extended_schema_items.json
new file mode 100644
index 0000000000000000000000000000000000000000..db795359e48669299f07c0a7287ab7a440fdead1
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/welllog_extended_schema_items.json
@@ -0,0 +1,47 @@
+[{
+ "path": "WellboreName",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.SpatialParameterTypeID",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.QuantitativeAccuracyBandID",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.CoordinateQualityCheckRemarks",
+ "kind": "[]string"
+ }, {
+ "path": "SpatialLocation.AppliedOperations",
+ "kind": "[]string"
+ }, {
+ "path": "SpatialLocation.QualitativeSpatialAccuracyTypeID",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.CoordinateQualityCheckPerformedBy",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.SpatialLocationCoordinatesDate",
+ "kind": "datetime"
+ }, {
+ "path": "SpatialLocation.CoordinateQualityCheckDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "SpatialLocation.Wgs84Coordinates",
+ "kind": "core:dl:geoshape:1.0.0"
+ }, {
+ "path": "SpatialLocation.SpatialGeometryTypeID",
+ "kind": "string"
+ }, {
+ "path": "SpatialLocation.IsDecimated",
+ "kind": "boolean"
+ }, {
+ "path": "TechnicalAssuranceReviewerNames",
+ "kind": "[]string"
+ }, {
+ "path": "TechnicalAssuranceReviewerOrganisationNames",
+ "kind": "[]string"
+ }, {
+ "path": "AssociatedIdentities",
+ "kind": "[]string"
+ }
+]
diff --git a/indexer-core/src/test/resources/indexproperty/welllog_original_data.json b/indexer-core/src/test/resources/indexproperty/welllog_original_data.json
new file mode 100644
index 0000000000000000000000000000000000000000..047bce6d5ca99a1bedfb180479afffcf69804c9a
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/welllog_original_data.json
@@ -0,0 +1,142 @@
+{
+ "Name": "A0_FINAL_INTER_REF_D",
+ "VirtualProperties.DefaultName": "A0_FINAL_INTER_REF_D",
+ "WellboreID": "opendes:master-data--Wellbore:nz-100000113552:",
+ "ReferenceCurveID": "DEPTH",
+ "TechnicalAssurances": [{
+ "Comment": "This is free form text from reviewer, e.g. restrictions on use",
+ "UnacceptableUsage": [{
+ "WorkflowUsage": "opendes:reference-data--WorkflowUsageType:SeismicInterpretation:",
+ "WorkflowPersona": "opendes:reference-data--WorkflowPersonaType:SeismicInterpreter:"
+ }
+ ],
+ "TechnicalAssuranceTypeID": "opendes:reference-data--TechnicalAssuranceType:Trusted:",
+ "Reviewers": [{
+ "RoleTypeID": "opendes:reference-data--ContactRoleType:ProjectManager:AccountOwner:",
+ "OrganisationID": "opendes:master-data--Organisation:BigOil-Department-SeismicProcessing:",
+ "Name": "BigOil Department SeismicProcessing (denormalized)"
+ }, {
+ "RoleTypeID": "opendes:reference-data--ContactRoleType:AccountOwner:",
+ "OrganisationID": "opendes:master-data--Organisation:BigOil-Department-SeismicInterpretation:",
+ "Name": "BigOil Department SeismicInterpretation (denormalized)"
+ }
+ ],
+ "AcceptableUsage": [{
+ "WorkflowUsage": "opendes:reference-data--WorkflowUsageType:SeismicProcessing:",
+ "WorkflowPersona": "opendes:reference-data--WorkflowPersonaType:SeismicProcessor:"
+ }
+ ],
+ "EffectiveDate": "2020-02-13T00:00:00+0000"
+ }
+ ],
+ "Curves": [{
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Measured%20Depth:",
+ "CurveID": "DEPTH",
+ "Mnemonic": "DEPTH",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:M:"
+ }, {
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Bit%20Size:",
+ "CurveID": "BS",
+ "Mnemonic": "BS",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Caliper:",
+ "CurveID": "CALI",
+ "Mnemonic": "CALI",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Bulk%20Density:",
+ "CurveID": "DENS",
+ "Mnemonic": "DENS",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:G%2FC3:"
+ }, {
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Bulk%20Density:",
+ "CurveID": "DENS_CORR",
+ "Mnemonic": "DENS_CORR",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:G%2FC3:"
+ }, {
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Bulk%20Density%20Correction:",
+ "CurveID": "DRHO",
+ "Mnemonic": "DRHO",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:G%2FC3:"
+ }, {
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Compressional%20Slowness:",
+ "CurveID": "DTC",
+ "Mnemonic": "DTC",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:US%2FF:"
+ }, {
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Gamma%20Ray:",
+ "CurveID": "GR",
+ "Mnemonic": "GR",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"
+ }, {
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Gamma%20Ray:",
+ "CurveID": "GR_CORR",
+ "Mnemonic": "GR_CORR",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"
+ }, {
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Neutron%20Porosity:",
+ "CurveID": "NEUT",
+ "Mnemonic": "NEUT",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:V%2FV:"
+ }, {
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Neutron%20Porosity:",
+ "CurveID": "NEUT_CORR",
+ "Mnemonic": "NEUT_CORR",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:V%2FV:"
+ }, {
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Photoelectric%20Factor:",
+ "CurveID": "PEF",
+ "Mnemonic": "PEF",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:B%2FE:"
+ }, {
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Deep:",
+ "CurveID": "RESD",
+ "Mnemonic": "RESD",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Deep:",
+ "CurveID": "RESD_CORR",
+ "Mnemonic": "RESD_CORR",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Tool%20Status:",
+ "CurveID": "RESS",
+ "Mnemonic": "RESS",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Tool%20Status:",
+ "CurveID": "RESS_CORR",
+ "Mnemonic": "RESS_CORR",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Spontaneous%20Potential:",
+ "CurveID": "SP",
+ "Mnemonic": "SP",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:MV:"
+ }, {
+ "NumberOfColumns": 1,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Tension:",
+ "CurveID": "TENS",
+ "Mnemonic": "TENS",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:KGF:"
+ }
+ ]
+}
diff --git a/indexer-core/src/test/resources/indexproperty/welllog_search_records.json b/indexer-core/src/test/resources/indexproperty/welllog_search_records.json
new file mode 100644
index 0000000000000000000000000000000000000000..f2abecb03c274744dfcb3e64bddad49794b3ab6d
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/welllog_search_records.json
@@ -0,0 +1,678 @@
+[{
+ "id": "opendes:work-product-component--WellLog:9b579416f23c4f36af4a00c10657babe",
+ "kind": "osdu:wks:work-product-component--WellLog:1.2.0",
+ "data": {
+ "Name": "EC_CURVES",
+ "VirtualProperties.DefaultName": "EC_CURVES",
+ "SpatialLocation.SpatialGeometryTypeID": "opendes:reference-data--SpatialGeometryType:Point:",
+ "SpatialLocation.Wgs84Coordinates": {
+ "geometries": [{
+ "coordinates": [171.8745361, -41.2456122],
+ "type": "point"
+ }
+ ],
+ "type": "geometrycollection"
+ },
+ "AssociatedIdentities": ["opendes:master-data--Wellbore:nz-100000113552"],
+ "WellboreID": "opendes:master-data--Wellbore:nz-100000113552:",
+ "ReferenceCurveID": "DEPTH",
+ "SpatialLocation.IsDecimated": false,
+ "WellboreName": "SLB",
+ "Curves": [{
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Measured%20Depth:",
+ "CurveID": "DEPTH",
+ "Mnemonic": "DEPTH",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:M:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "CurveID": "CT",
+ "Mnemonic": "CT",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:MH%2FM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Conductivity%20-%20Flushed%20Zone:",
+ "CurveID": "CXO",
+ "Mnemonic": "CXO",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:MH%2FM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Invasion%20Diameter:",
+ "CurveID": "DI",
+ "Mnemonic": "DI",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Fracture%20Pressure:",
+ "CurveID": "FPRESS",
+ "Mnemonic": "FPRESS",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:PSI:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Formation%20Temperature%20%28Estimate%29:",
+ "CurveID": "FTEMP",
+ "Mnemonic": "FTEMP",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:DEGC:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "CurveID": "HMC_POR",
+ "Mnemonic": "HMC_POR",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "CurveID": "HMC_RES",
+ "Mnemonic": "HMC_RES",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Mud%20Resistivity:",
+ "CurveID": "RM",
+ "Mnemonic": "RM",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Mudcake%20Resistivity:",
+ "CurveID": "RMC",
+ "Mnemonic": "RMC",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Mud%20Filtrate%20Resistivity:",
+ "CurveID": "RMF",
+ "Mnemonic": "RMF",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20True%20Formation:",
+ "CurveID": "RT",
+ "Mnemonic": "RT",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Flushed%20Zone:",
+ "CurveID": "RXO",
+ "Mnemonic": "RXO",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Volumetric%20Photoelectric%20Factor:",
+ "CurveID": "U",
+ "Mnemonic": "U",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:B%2FC3:"
+ }
+ ]
+ }
+ }, {
+ "id": "opendes:work-product-component--WellLog:bee5994052564bb0ac9db531601a96aa",
+ "kind": "osdu:wks:work-product-component--WellLog:1.2.0",
+ "data": {
+ "Name": "RAW_INTER_REF_D",
+ "VirtualProperties.DefaultName": "RAW_INTER_REF_D",
+ "SpatialLocation.SpatialGeometryTypeID": "opendes:reference-data--SpatialGeometryType:Point:",
+ "SpatialLocation.Wgs84Coordinates": {
+ "geometries": [{
+ "coordinates": [171.8745361, -41.2456122],
+ "type": "point"
+ }
+ ],
+ "type": "geometrycollection"
+ },
+ "AssociatedIdentities": ["opendes:master-data--Wellbore:nz-100000113552"],
+ "WellboreID": "opendes:master-data--Wellbore:nz-100000113552:",
+ "ReferenceCurveID": "DEPTH",
+ "SpatialLocation.IsDecimated": false,
+ "WellboreName": "SLB",
+ "Curves": [{
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Measured%20Depth:",
+ "CurveID": "DEPTH",
+ "Mnemonic": "DEPTH",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:M:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Bit%20Size:",
+ "CurveID": "BS",
+ "Mnemonic": "BS",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Bit%20Size:",
+ "CurveID": "BS_2",
+ "Mnemonic": "BS_2",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Bit%20Size:",
+ "CurveID": "BS_3",
+ "Mnemonic": "BS_3",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Bit%20Size:",
+ "CurveID": "BS_4",
+ "Mnemonic": "BS_4",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Bit%20Size:",
+ "CurveID": "BS_5",
+ "Mnemonic": "BS_5",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Bit%20Size:",
+ "CurveID": "BS_6",
+ "Mnemonic": "BS_6",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Caliper:",
+ "CurveID": "CALI",
+ "Mnemonic": "CALI",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Caliper:",
+ "CurveID": "CALI_2",
+ "Mnemonic": "CALI_2",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Caliper:",
+ "CurveID": "CALI_3",
+ "Mnemonic": "CALI_3",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Caliper:",
+ "CurveID": "CALI_4",
+ "Mnemonic": "CALI_4",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Caliper:",
+ "CurveID": "CALI_5",
+ "Mnemonic": "CALI_5",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Caliper:",
+ "CurveID": "CALI_6",
+ "Mnemonic": "CALI_6",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Bulk%20Density%20Correction:",
+ "CurveID": "DRHO",
+ "Mnemonic": "DRHO",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:G%2FC3:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Compressional%20Slowness:",
+ "CurveID": "DT",
+ "Mnemonic": "DT",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:US%2FF:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Acoustic%20Slowness:",
+ "CurveID": "DT_2",
+ "Mnemonic": "DT_2",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:US%2FF:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Acoustic%20Slowness:",
+ "CurveID": "DT_3",
+ "Mnemonic": "DT_3",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:US%2FF:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Acoustic%20Slowness:",
+ "CurveID": "DT_4",
+ "Mnemonic": "DT_4",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:US%2FF:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Gamma%20Ray:",
+ "CurveID": "GR",
+ "Mnemonic": "GR",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Gamma%20Ray:",
+ "CurveID": "GR_2",
+ "Mnemonic": "GR_2",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Gamma%20Ray:",
+ "CurveID": "GR_3",
+ "Mnemonic": "GR_3",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Gamma%20Ray:",
+ "CurveID": "GR_4",
+ "Mnemonic": "GR_4",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Gamma%20Ray:",
+ "CurveID": "GR_5",
+ "Mnemonic": "GR_5",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Gamma%20Ray:",
+ "CurveID": "GR_6",
+ "Mnemonic": "GR_6",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Gamma%20Ray:",
+ "CurveID": "GR_7",
+ "Mnemonic": "GR_7",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Gamma%20Ray:",
+ "CurveID": "GR_8",
+ "Mnemonic": "GR_8",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Gamma%20Ray:",
+ "CurveID": "GR_9",
+ "Mnemonic": "GR_9",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Gamma%20Ray:",
+ "CurveID": "GR_10",
+ "Mnemonic": "GR_10",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Deep%20Laterolog:",
+ "CurveID": "LLD",
+ "Mnemonic": "LLD",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Deep%20Laterolog:",
+ "CurveID": "LLD_2",
+ "Mnemonic": "LLD_2",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Deep%20Laterolog:",
+ "CurveID": "LLD_3",
+ "Mnemonic": "LLD_3",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Deep%20Laterolog:",
+ "CurveID": "LLD_4",
+ "Mnemonic": "LLD_4",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Shallow%20Laterolog:",
+ "CurveID": "LLS",
+ "Mnemonic": "LLS",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Shallow%20Laterolog:",
+ "CurveID": "LLS_2",
+ "Mnemonic": "LLS_2",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Shallow%20Laterolog:",
+ "CurveID": "LLS_3",
+ "Mnemonic": "LLS_3",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Shallow%20Laterolog:",
+ "CurveID": "LLS_4",
+ "Mnemonic": "LLS_4",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Micro%20Inverse:",
+ "CurveID": "MINV",
+ "Mnemonic": "MINV",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Micro%20Inverse:",
+ "CurveID": "MINV_2",
+ "Mnemonic": "MINV_2",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Micro%20Inverse:",
+ "CurveID": "MINV_3",
+ "Mnemonic": "MINV_3",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Micro%20Inverse:",
+ "CurveID": "MINV_4",
+ "Mnemonic": "MINV_4",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Micro%20Normal:",
+ "CurveID": "MNOR",
+ "Mnemonic": "MNOR",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Micro%20Normal:",
+ "CurveID": "MNOR_2",
+ "Mnemonic": "MNOR_2",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Micro%20Normal:",
+ "CurveID": "MNOR_3",
+ "Mnemonic": "MNOR_3",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Micro%20Normal:",
+ "CurveID": "MNOR_4",
+ "Mnemonic": "MNOR_4",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Micro%20Laterolog:",
+ "CurveID": "MSFL",
+ "Mnemonic": "MSFL",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Micro%20Laterolog:",
+ "CurveID": "MSFL_2",
+ "Mnemonic": "MSFL_2",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Micro%20Laterolog:",
+ "CurveID": "MSFL_3",
+ "Mnemonic": "MSFL_3",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Micro%20Laterolog:",
+ "CurveID": "MSFL_4",
+ "Mnemonic": "MSFL_4",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Thermal%20Neutron%20Porosity:",
+ "CurveID": "NPHI",
+ "Mnemonic": "NPHI",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:V%2FV:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Photoelectric%20Factor:",
+ "CurveID": "PEF",
+ "Mnemonic": "PEF",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:B%2FE:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Bulk%20Density:",
+ "CurveID": "RHOB",
+ "Mnemonic": "RHOB",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:G%2FC3:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Spontaneous%20Potential:",
+ "CurveID": "SP",
+ "Mnemonic": "SP",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:MV:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Spontaneous%20Potential:",
+ "CurveID": "SP_2",
+ "Mnemonic": "SP_2",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:MV:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Spontaneous%20Potential:",
+ "CurveID": "SP_3",
+ "Mnemonic": "SP_3",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:MV:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Spontaneous%20Potential:",
+ "CurveID": "SP_4",
+ "Mnemonic": "SP_4",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:MV:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Tension:",
+ "CurveID": "TENS",
+ "Mnemonic": "TENS",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:KGF:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Tension:",
+ "CurveID": "TENS_2",
+ "Mnemonic": "TENS_2",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:KGF:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Tension:",
+ "CurveID": "TENS_3",
+ "Mnemonic": "TENS_3",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:KGF:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Tension:",
+ "CurveID": "TENS_4",
+ "Mnemonic": "TENS_4",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:KGF:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Tension:",
+ "CurveID": "TENS_5",
+ "Mnemonic": "TENS_5",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:KGF:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Tension:",
+ "CurveID": "TENS_6",
+ "Mnemonic": "TENS_6",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:KGF:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Tension:",
+ "CurveID": "TENS_7",
+ "Mnemonic": "TENS_7",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:KGF:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Tension:",
+ "CurveID": "TENS_8",
+ "Mnemonic": "TENS_8",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:KGF:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Tension:",
+ "CurveID": "TENS_9",
+ "Mnemonic": "TENS_9",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:KGF:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Tension:",
+ "CurveID": "TENS_10",
+ "Mnemonic": "TENS_10",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:KGF:"
+ }
+ ]
+ }
+ }, {
+ "id": "opendes:work-product-component--WellLog:23b7dfde2c1349d58a0f97ae78bff9df",
+ "kind": "osdu:wks:work-product-component--WellLog:1.2.0",
+ "data": {
+ "Name": "A0_FINAL_INTER_REF_D",
+ "VirtualProperties.DefaultName": "A0_FINAL_INTER_REF_D",
+ "TechnicalAssuranceReviewerNames": ["BigOil Department SeismicInterpretation (denormalized)", "BigOil Department SeismicProcessing (denormalized)"],
+ "SpatialLocation.SpatialGeometryTypeID": "opendes:reference-data--SpatialGeometryType:Point:",
+ "SpatialLocation.Wgs84Coordinates": {
+ "geometries": [{
+ "coordinates": [171.8745361, -41.2456122],
+ "type": "point"
+ }
+ ],
+ "type": "geometrycollection"
+ },
+ "AssociatedIdentities": ["opendes:master-data--Organisation:BigOil-Department-SeismicProcessing", "opendes:master-data--Wellbore:nz-100000113552", "opendes:master-data--Organisation:BigOil-Department-SeismicInterpretation"],
+ "WellboreID": "opendes:master-data--Wellbore:nz-100000113552:",
+ "ReferenceCurveID": "DEPTH",
+ "SpatialLocation.IsDecimated": false,
+ "WellboreName": "Kongahu-1",
+ "TechnicalAssurances": [{
+ "Comment": "This is free form text from reviewer, e.g. restrictions on use",
+ "UnacceptableUsage": [{
+ "WorkflowUsage": "opendes:reference-data--WorkflowUsageType:SeismicInterpretation:",
+ "WorkflowPersona": "opendes:reference-data--WorkflowPersonaType:SeismicInterpreter:"
+ }
+ ],
+ "TechnicalAssuranceTypeID": "opendes:reference-data--TechnicalAssuranceType:Trusted:",
+ "Reviewers": [{
+ "RoleTypeID": "opendes:reference-data--ContactRoleType:ProjectManager:AccountOwner:",
+ "OrganisationID": "opendes:master-data--Organisation:BigOil-Department-SeismicProcessing:",
+ "Name": "BigOil Department SeismicProcessing (denormalized)"
+ }, {
+ "RoleTypeID": "opendes:reference-data--ContactRoleType:AccountOwner:",
+ "OrganisationID": "opendes:master-data--Organisation:BigOil-Department-SeismicInterpretation:",
+ "Name": "BigOil Department SeismicInterpretation (denormalized)"
+ }
+ ],
+ "AcceptableUsage": [{
+ "WorkflowUsage": "opendes:reference-data--WorkflowUsageType:SeismicProcessing:",
+ "WorkflowPersona": "opendes:reference-data--WorkflowPersonaType:SeismicProcessor:"
+ }
+ ],
+ "EffectiveDate": "2020-02-13T00:00:00+0000"
+ }
+ ],
+ "TechnicalAssuranceReviewerOrganisationNames": ["BigOil SeismicInterpretation Department (original)", "BigOil SeismicProcessing Department (original)"],
+ "Curves": [{
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Measured%20Depth:",
+ "CurveID": "DEPTH",
+ "Mnemonic": "DEPTH",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:M:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Bit%20Size:",
+ "CurveID": "BS",
+ "Mnemonic": "BS",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Caliper:",
+ "CurveID": "CALI",
+ "Mnemonic": "CALI",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:IN:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Bulk%20Density:",
+ "CurveID": "DENS",
+ "Mnemonic": "DENS",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:G%2FC3:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Bulk%20Density:",
+ "CurveID": "DENS_CORR",
+ "Mnemonic": "DENS_CORR",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:G%2FC3:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Bulk%20Density%20Correction:",
+ "CurveID": "DRHO",
+ "Mnemonic": "DRHO",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:G%2FC3:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Compressional%20Slowness:",
+ "CurveID": "DTC",
+ "Mnemonic": "DTC",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:US%2FF:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Gamma%20Ray:",
+ "CurveID": "GR",
+ "Mnemonic": "GR",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Gamma%20Ray:",
+ "CurveID": "GR_CORR",
+ "Mnemonic": "GR_CORR",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:GAPI:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Neutron%20Porosity:",
+ "CurveID": "NEUT",
+ "Mnemonic": "NEUT",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:V%2FV:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Neutron%20Porosity:",
+ "CurveID": "NEUT_CORR",
+ "Mnemonic": "NEUT_CORR",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:V%2FV:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Photoelectric%20Factor:",
+ "CurveID": "PEF",
+ "Mnemonic": "PEF",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:B%2FE:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Deep:",
+ "CurveID": "RESD",
+ "Mnemonic": "RESD",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Resistivity%20-%20Deep:",
+ "CurveID": "RESD_CORR",
+ "Mnemonic": "RESD_CORR",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Tool%20Status:",
+ "CurveID": "RESS",
+ "Mnemonic": "RESS",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Tool%20Status:",
+ "CurveID": "RESS_CORR",
+ "Mnemonic": "RESS_CORR",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:OHMM:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Spontaneous%20Potential:",
+ "CurveID": "SP",
+ "Mnemonic": "SP",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:MV:"
+ }, {
+ "NumberOfColumns": 1.0,
+ "LogCurveFamilyID": "opendes:reference-data--LogCurveFamily:Tension:",
+ "CurveID": "TENS",
+ "Mnemonic": "TENS",
+ "CurveUnit": "opendes:reference-data--UnitOfMeasure:KGF:"
+ }
+ ]
+ }
+ }
+]
diff --git a/indexer-core/src/test/resources/indexproperty/welllog_storage_schema.json b/indexer-core/src/test/resources/indexproperty/welllog_storage_schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..734ecd5c6e3954ec57ee15db09478623519bab4e
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/welllog_storage_schema.json
@@ -0,0 +1,449 @@
+{
+ "kind": "osdu:wks:work-product-component--WellLog:1.2.0",
+ "schema": [{
+ "path": "ResourceHomeRegionID",
+ "kind": "string"
+ }, {
+ "path": "ResourceHostRegionIDs",
+ "kind": "[]string"
+ }, {
+ "path": "ResourceLifecycleStatus",
+ "kind": "string"
+ }, {
+ "path": "ResourceSecurityClassification",
+ "kind": "string"
+ }, {
+ "path": "ResourceCurationStatus",
+ "kind": "string"
+ }, {
+ "path": "ExistenceKind",
+ "kind": "string"
+ }, {
+ "path": "TechnicalAssuranceID",
+ "kind": "string"
+ }, {
+ "path": "Source",
+ "kind": "string"
+ }, {
+ "path": "Datasets",
+ "kind": "[]string"
+ }, {
+ "path": "IsDiscoverable",
+ "kind": "bool"
+ }, {
+ "path": "TechnicalAssurances",
+ "kind": "nested",
+ "properties": [{
+ "path": "Comment",
+ "kind": "string"
+ }, {
+ "path": "Reviewers",
+ "kind": "[]object"
+ }, {
+ "path": "UnacceptableUsage",
+ "kind": "flattened"
+ }, {
+ "path": "AcceptableUsage",
+ "kind": "flattened"
+ }, {
+ "path": "TechnicalAssuranceTypeID",
+ "kind": "string"
+ }, {
+ "path": "EffectiveDate",
+ "kind": "datetime"
+ }
+ ]
+ }, {
+ "path": "Artefacts",
+ "kind": "flattened"
+ }, {
+ "path": "IsExtendedLoad",
+ "kind": "bool"
+ }, {
+ "path": "SpatialArea.SpatialParameterTypeID",
+ "kind": "string"
+ }, {
+ "path": "SpatialArea.QuantitativeAccuracyBandID",
+ "kind": "string"
+ }, {
+ "path": "SpatialArea.CoordinateQualityCheckRemarks",
+ "kind": "[]string"
+ }, {
+ "path": "SpatialArea.AppliedOperations",
+ "kind": "[]string"
+ }, {
+ "path": "SpatialArea.QualitativeSpatialAccuracyTypeID",
+ "kind": "string"
+ }, {
+ "path": "SpatialArea.CoordinateQualityCheckPerformedBy",
+ "kind": "string"
+ }, {
+ "path": "SpatialArea.SpatialLocationCoordinatesDate",
+ "kind": "datetime"
+ }, {
+ "path": "SpatialArea.CoordinateQualityCheckDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "SpatialArea.Wgs84Coordinates",
+ "kind": "core:dl:geoshape:1.0.0"
+ }, {
+ "path": "SpatialArea.SpatialGeometryTypeID",
+ "kind": "string"
+ }, {
+ "path": "Description",
+ "kind": "string"
+ }, {
+ "path": "CreationDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "BusinessActivities",
+ "kind": "[]string"
+ }, {
+ "path": "SpatialPoint.SpatialParameterTypeID",
+ "kind": "string"
+ }, {
+ "path": "SpatialPoint.QuantitativeAccuracyBandID",
+ "kind": "string"
+ }, {
+ "path": "SpatialPoint.CoordinateQualityCheckRemarks",
+ "kind": "[]string"
+ }, {
+ "path": "SpatialPoint.AppliedOperations",
+ "kind": "[]string"
+ }, {
+ "path": "SpatialPoint.QualitativeSpatialAccuracyTypeID",
+ "kind": "string"
+ }, {
+ "path": "SpatialPoint.CoordinateQualityCheckPerformedBy",
+ "kind": "string"
+ }, {
+ "path": "SpatialPoint.SpatialLocationCoordinatesDate",
+ "kind": "datetime"
+ }, {
+ "path": "SpatialPoint.CoordinateQualityCheckDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "SpatialPoint.Wgs84Coordinates",
+ "kind": "core:dl:geoshape:1.0.0"
+ }, {
+ "path": "SpatialPoint.SpatialGeometryTypeID",
+ "kind": "string"
+ }, {
+ "path": "GeoContexts",
+ "kind": "nested",
+ "properties": [{
+ "path": "GeoPoliticalEntityID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "BasinID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "FieldID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "PlayID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }, {
+ "path": "ProspectID",
+ "kind": "string"
+ }, {
+ "path": "GeoTypeID",
+ "kind": "string"
+ }
+ ]
+ }, {
+ "path": "AuthorIDs",
+ "kind": "[]string"
+ }, {
+ "path": "SubmitterName",
+ "kind": "string"
+ }, {
+ "path": "LineageAssertions",
+ "kind": "flattened"
+ }, {
+ "path": "Tags",
+ "kind": "[]string"
+ }, {
+ "path": "Name",
+ "kind": "string"
+ }, {
+ "path": "LogRemark",
+ "kind": "string"
+ }, {
+ "path": "SamplingStop",
+ "kind": "double"
+ }, {
+ "path": "CompanyID",
+ "kind": "string"
+ }, {
+ "path": "TopMeasuredDepth",
+ "kind": "double"
+ }, {
+ "path": "ServiceCompanyID",
+ "kind": "string"
+ }, {
+ "path": "SamplingInterval",
+ "kind": "double"
+ }, {
+ "path": "LogVersion",
+ "kind": "string"
+ }, {
+ "path": "LogSource",
+ "kind": "string"
+ }, {
+ "path": "ToolStringDescription",
+ "kind": "string"
+ }, {
+ "path": "DrillingFluidProperty",
+ "kind": "string"
+ }, {
+ "path": "IsRegular",
+ "kind": "bool"
+ }, {
+ "path": "LogServiceDateInterval.StartDate",
+ "kind": "datetime"
+ }, {
+ "path": "LogServiceDateInterval.EndDate",
+ "kind": "datetime"
+ }, {
+ "path": "HoleTypeLogging",
+ "kind": "string"
+ }, {
+ "path": "LoggingDirection",
+ "kind": "string"
+ }, {
+ "path": "SamplingStart",
+ "kind": "double"
+ }, {
+ "path": "WellboreID",
+ "kind": "string"
+ }, {
+ "path": "PassNumber",
+ "kind": "int"
+ }, {
+ "path": "ActivityType",
+ "kind": "string"
+ }, {
+ "path": "VerticalMeasurementID",
+ "kind": "string"
+ }, {
+ "path": "SamplingDomainTypeID",
+ "kind": "string"
+ }, {
+ "path": "ReferenceCurveID",
+ "kind": "string"
+ }, {
+ "path": "CandidateReferenceCurveIDs",
+ "kind": "[]string"
+ }, {
+ "path": "SeismicReferenceElevation.WellboreTVDTrajectoryID",
+ "kind": "string"
+ }, {
+ "path": "SeismicReferenceElevation.VerticalCRSID",
+ "kind": "string"
+ }, {
+ "path": "SeismicReferenceElevation.VerticalMeasurementSourceID",
+ "kind": "string"
+ }, {
+ "path": "SeismicReferenceElevation.VerticalReferenceID",
+ "kind": "string"
+ }, {
+ "path": "SeismicReferenceElevation.TerminationDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "SeismicReferenceElevation.VerticalMeasurementPathID",
+ "kind": "string"
+ }, {
+ "path": "SeismicReferenceElevation.EffectiveDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "SeismicReferenceElevation.VerticalMeasurement",
+ "kind": "double"
+ }, {
+ "path": "SeismicReferenceElevation.VerticalMeasurementTypeID",
+ "kind": "string"
+ }, {
+ "path": "SeismicReferenceElevation.VerticalMeasurementDescription",
+ "kind": "string"
+ }, {
+ "path": "SeismicReferenceElevation.VerticalMeasurementUnitOfMeasureID",
+ "kind": "string"
+ }, {
+ "path": "SeismicReferenceElevation.VerticalReferenceEntityID",
+ "kind": "string"
+ }, {
+ "path": "WellLogTypeID",
+ "kind": "string"
+ }, {
+ "path": "FrameIdentifier",
+ "kind": "string"
+ }, {
+ "path": "LoggingService",
+ "kind": "string"
+ }, {
+ "path": "BottomMeasuredDepth",
+ "kind": "double"
+ }, {
+ "path": "VerticalMeasurement.WellboreTVDTrajectoryID",
+ "kind": "string"
+ }, {
+ "path": "VerticalMeasurement.VerticalCRSID",
+ "kind": "string"
+ }, {
+ "path": "VerticalMeasurement.VerticalMeasurementSourceID",
+ "kind": "string"
+ }, {
+ "path": "VerticalMeasurement.VerticalReferenceID",
+ "kind": "string"
+ }, {
+ "path": "VerticalMeasurement.TerminationDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "VerticalMeasurement.VerticalMeasurementPathID",
+ "kind": "string"
+ }, {
+ "path": "VerticalMeasurement.EffectiveDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "VerticalMeasurement.VerticalMeasurement",
+ "kind": "double"
+ }, {
+ "path": "VerticalMeasurement.VerticalMeasurementTypeID",
+ "kind": "string"
+ }, {
+ "path": "VerticalMeasurement.VerticalMeasurementDescription",
+ "kind": "string"
+ }, {
+ "path": "VerticalMeasurement.VerticalMeasurementUnitOfMeasureID",
+ "kind": "string"
+ }, {
+ "path": "VerticalMeasurement.VerticalReferenceEntityID",
+ "kind": "string"
+ }, {
+ "path": "LogRun",
+ "kind": "string"
+ }, {
+ "path": "ZeroTime",
+ "kind": "datetime"
+ }, {
+ "path": "LogActivity",
+ "kind": "string"
+ }, {
+ "path": "Curves",
+ "kind": "nested",
+ "properties": [{
+ "path": "IsProcessed",
+ "kind": "bool"
+ }, {
+ "path": "LogCurveMainFamilyID",
+ "kind": "string"
+ }, {
+ "path": "DateStamp",
+ "kind": "datetime"
+ }, {
+ "path": "NumberOfColumns",
+ "kind": "int"
+ }, {
+ "path": "LogCurveFamilyID",
+ "kind": "string"
+ }, {
+ "path": "CurveID",
+ "kind": "string"
+ }, {
+ "path": "TopDepth",
+ "kind": "double"
+ }, {
+ "path": "CurveVersion",
+ "kind": "string"
+ }, {
+ "path": "CurveSampleTypeID",
+ "kind": "string"
+ }, {
+ "path": "InterpreterName",
+ "kind": "string"
+ }, {
+ "path": "CurveQuality",
+ "kind": "string"
+ }, {
+ "path": "NullValue",
+ "kind": "bool"
+ }, {
+ "path": "Interpolate",
+ "kind": "bool"
+ }, {
+ "path": "DepthUnit",
+ "kind": "string"
+ }, {
+ "path": "DepthCoding",
+ "kind": "string"
+ }, {
+ "path": "Mnemonic",
+ "kind": "string"
+ }, {
+ "path": "BaseDepth",
+ "kind": "double"
+ }, {
+ "path": "LogCurveTypeID",
+ "kind": "string"
+ }, {
+ "path": "LogCurveBusinessValueID",
+ "kind": "string"
+ }, {
+ "path": "CurveUnit",
+ "kind": "string"
+ }, {
+ "path": "CurveDescription",
+ "kind": "string"
+ }
+ ]
+ }, {
+ "path": "VirtualProperties.DefaultLocation.SpatialParameterTypeID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.QuantitativeAccuracyBandID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.CoordinateQualityCheckRemarks",
+ "kind": "[]string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.AppliedOperations",
+ "kind": "[]string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.QualitativeSpatialAccuracyTypeID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.CoordinateQualityCheckPerformedBy",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.SpatialLocationCoordinatesDate",
+ "kind": "datetime"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.CoordinateQualityCheckDateTime",
+ "kind": "datetime"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.Wgs84Coordinates",
+ "kind": "core:dl:geoshape:1.0.0"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.SpatialGeometryTypeID",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultName",
+ "kind": "string"
+ }, {
+ "path": "VirtualProperties.DefaultLocation.IsDecimated",
+ "kind": "boolean"
+ }
+ ]
+}
diff --git a/pom.xml b/pom.xml
index 41919837141fd99c52bad08bf487f436c68e4721..7cebb9ac54ed9261424dd7df186b3e47e0e04479 100644
--- a/pom.xml
+++ b/pom.xml
@@ -5,7 +5,7 @@
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-service</artifactId>
<packaging>pom</packaging>
- <version>0.21.0-SNAPSHOT</version>
+ <version>0.22.0-SNAPSHOT</version>
<description>Indexer Service</description>
<properties>
@@ -13,7 +13,7 @@
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
<spring-cloud.version>Greenwich.SR2</spring-cloud.version>
- <os-core-common.version>0.21.0-rc4</os-core-common.version>
+ <os-core-common.version>0.21.0</os-core-common.version>
<snakeyaml.version>2.0</snakeyaml.version>
<hibernate-validator.version>6.1.5.Final</hibernate-validator.version>
<jackson-databind.version>2.14.1</jackson-databind.version>
diff --git a/provider/indexer-aws/README.md b/provider/indexer-aws/README.md
index 3c2ad5d3ac00b155f30c9cf2fcd2bc7b77b2d9b9..01b9d492c12801e3b6adfb002eabff7b3ae2a3d2 100644
--- a/provider/indexer-aws/README.md
+++ b/provider/indexer-aws/README.md
@@ -123,27 +123,28 @@ You should see in the logs that pop up what url and port it runs on. By default
export ELASTIC_PASSWORD=$ELASTIC_PASSWORD
export ELASTIC_USER_NAME=$ELASTIC_USERNAME
- | name | example value | description | sensitive?
- | --- | --- | --- | --- |
+ | name | example value | description | sensitive?
+ | --- |----------------------------------------------------------------------------------------| --- | --- |
| `AWS_ACCESS_KEY_ID` | `ASIAXXXXXXXXXXXXXX` | The AWS Access Key for a user with access to Backend Resources required by the service | yes |
| `AWS_SECRET_ACCESS_KEY` | `super-secret-key==` | The AWS Secret Key for a user with access to Backend Resources required by the service | yes |
- | `AWS_SESSION_TOKEN` | `session-token-xxxxxxxxx` | AWS Session token needed if using an SSO user session to authenticate | yes |
- | `AWS_COGNITO_USER_POOL_ID` | `us-east-1_xxxxxxxx` | User Pool Id for the reference cognito | no |
- | `AWS_COGNITO_CLIENT_ID` | `xxxxxxxxxxxx` | Client ID for the Auth Flow integrated with the Cognito User Pool | no |
- | `AWS_COGNITO_AUTH_FLOW` | `USER_PASSWORD_AUTH` | Auth flow used by reference cognito deployment | no |
- | `DEFAULT_DATA_PARTITION_ID_TENANT1` | `opendes` | Partition used to create and index record | no |
- | `DEFAULT_DATA_PARTITION_ID_TENANT2` | `common` | Another needed partition| no |
- | `AWS_COGNITO_AUTH_PARAMS_USER` | `int-test-user@testing.com` | Int Test Username | no |
- | `AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS` | `noaccess@testing.com` | No Access Username | no |
- | `AWS_COGNITO_AUTH_PARAMS_PASSWORD` | `some-secure-password` | Int Test User/NoAccessUser Password | yes |
- | `ENTITLEMENTS_DOMAIN` | `example.com` | Domain for user's groups | no |
- | `OTHER_RELEVANT_DATA_COUNTRIES` | `US` | Used to create demo legal tag | no |
- | `STORAGE_HOST` | `http://localhost:8080/api/storage/v2/` | The url where the storage API is hosted | no |
- | `HOST` | `http://localhost:8080` | Base url for deployment | no |
- | `ELASTIC_HOST` | `localhost` | Url for elasticsearch | no |
- | `ELASTIC_PORT` | `9300` | Port for elasticsearch | no |
- | `ELASTIC_PASSWORD` | `xxxxxxxxxxxxxxx` | Password for user to access elasticsearch | yes |
- | `ELASTIC_USER_NAME` | `xxxxxxxxxxxxxxxx` | Username for user to access elasticsearch | yes |
+ | `AWS_SESSION_TOKEN` | `session-token-xxxxxxxxx` | AWS Session token needed if using an SSO user session to authenticate | yes |
+ | `AWS_COGNITO_USER_POOL_ID` | `us-east-1_xxxxxxxx` | User Pool Id for the reference cognito | no |
+ | `AWS_COGNITO_CLIENT_ID` | `xxxxxxxxxxxx` | Client ID for the Auth Flow integrated with the Cognito User Pool | no |
+ | `AWS_COGNITO_AUTH_FLOW` | `USER_PASSWORD_AUTH` | Auth flow used by reference cognito deployment | no |
+ | `DEFAULT_DATA_PARTITION_ID_TENANT1` | `opendes` | Partition used to create and index record | no |
+ | `DEFAULT_DATA_PARTITION_ID_TENANT2` | `common` | Another needed partition | no |
+ | `AWS_COGNITO_AUTH_PARAMS_USER` | `int-test-user@testing.com` | Int Test Username | no |
+ | `AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS` | `noaccess@testing.com` | No Access Username | no |
+ | `AWS_COGNITO_AUTH_PARAMS_PASSWORD` | `some-secure-password` | Int Test User/NoAccessUser Password | yes |
+ | `ENTITLEMENTS_DOMAIN` | `example.com` | Domain for user's groups | no |
+ | `OTHER_RELEVANT_DATA_COUNTRIES` | `US` | Used to create demo legal tag | no |
+ | `STORAGE_HOST` | `http://localhost:8080/api/storage/v2/` | The url where the storage API is hosted | no |
+ | `HOST` | `http://localhost:8080` | Base url for deployment | no |
+ | `ELASTIC_HOST` | `localhost` | Url for elasticsearch | no |
+ | `ELASTIC_PORT` | `9300` | Port for elasticsearch | no |
+ | `ELASTIC_PASSWORD` | `xxxxxxxxxxxxxxx` | Password for user to access elasticsearch | yes |
+ | `ELASTIC_USER_NAME` | `xxxxxxxxxxxxxxxx` | Username for user to access elasticsearch | yes |
+ | `CUCUMBER_OPTIONS` | `--tags '~@indexer-extended'` OR `--tags '~@* and @indexer-extended'` | By default `--tags '~@* and @indexer-extended'` to enable experimental feature testing | no |
**Creating a new user to use for integration tests**
@@ -199,4 +200,4 @@ Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
-limitations under the License.
\ No newline at end of file
+limitations under the License.
diff --git a/provider/indexer-aws/pom.xml b/provider/indexer-aws/pom.xml
index d6e2c591ae472b8fb64b96741e2d73e64bcf14dc..24c1abaa57c3e9d1036b81fc4df9f473bb36564c 100644
--- a/provider/indexer-aws/pom.xml
+++ b/provider/indexer-aws/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-service</artifactId>
- <version>0.21.0-SNAPSHOT</version>
+ <version>0.22.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
@@ -26,7 +26,7 @@
<artifactId>indexer-aws</artifactId>
<description>Indexer service on AWS</description>
<packaging>jar</packaging>
- <version>0.21.0-SNAPSHOT</version>
+ <version>0.22.0-SNAPSHOT</version>
<properties>
<aws.version>1.11.1018</aws.version>
@@ -66,12 +66,12 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-core</artifactId>
- <version>0.21.0-SNAPSHOT</version>
+ <version>0.22.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
<artifactId>os-core-lib-aws</artifactId>
- <version>0.21.0-SNAPSHOT</version>
+ <version>0.21.0</version>
</dependency>
<!-- AWS managed packages -->
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java
index 0944a9091c2ac6533ac5b26c88d06fec12808c70..dbc4e36fb84a4f8b2d4a2a449e61174610f94aea 100644
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/util/IndexerQueueTaskBuilderAws.java
@@ -23,6 +23,7 @@ import org.opengroup.osdu.core.aws.ssm.K8sLocalParameterProvider;
import org.opengroup.osdu.core.aws.ssm.K8sParameterNotFoundException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
+import org.opengroup.osdu.indexer.model.Constants;
import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Primary;
@@ -134,6 +135,7 @@ public class IndexerQueueTaskBuilderAws extends IndexerQueueTaskBuilder {
retryCount = 1;
retryDelay = INITIAL_RETRY_DELAY_SECONDS;
}
+
System.out.println("Re-queuing for retry attempt #: " + retryCount);
System.out.println("Delay (in seconds) before next retry: " + retryDelay);
@@ -142,6 +144,12 @@ public class IndexerQueueTaskBuilderAws extends IndexerQueueTaskBuilder {
.withDataType("String")
.withStringValue(String.valueOf(retryCount))
);
+ // Append the ancestry kinds used to prevent circular chasing
+ if(message.getAttributes().containsKey(Constants.ANCESTRY_KINDS)) {
+ messageAttributes.put(Constants.ANCESTRY_KINDS, new MessageAttributeValue()
+ .withDataType("String")
+ .withStringValue(message.getAttributes().get(Constants.ANCESTRY_KINDS)));
+ }
// Send a message with an attribute and a delay
final SendMessageRequest sendMessageRequest ;
diff --git a/provider/indexer-aws/src/main/resources/application.properties b/provider/indexer-aws/src/main/resources/application.properties
index 6529bf15accf2f4b6045d51c5d76de14f7620f7a..a0d1a419f6a6aa71128963495390ae748e7e9649 100644
--- a/provider/indexer-aws/src/main/resources/application.properties
+++ b/provider/indexer-aws/src/main/resources/application.properties
@@ -24,6 +24,7 @@ GAE_SERVICE=indexer
#reusing STORAGE_BASE_URL variable here as the base url to point to schema service
SCHEMA_HOST=${SCHEMA_BASE_URL}/api/schema-service/v1/schema
+SEARCH_HOST=${SEARCH_BASE_URL}/api/search/v2
PARTITION_PATH=/api/partition/v1
PARTITION_API=${PARTITION_BASE_URL}${PARTITION_PATH}
diff --git a/provider/indexer-azure/README.md b/provider/indexer-azure/README.md
index cb13417cb40902bdfa6cea76f058604ad25bae21..03a4a114ea9dad4923f4289cba4b59f152c0acd8 100644
--- a/provider/indexer-azure/README.md
+++ b/provider/indexer-azure/README.md
@@ -43,6 +43,7 @@ az keyvault secret show --vault-name $KEY_VAULT_NAME --name $KEY_VAULT_SECRET_NA
| `server.servlet.contextPath` | `/api/indexer/v2/` | Servlet context path | no | - |
| `schema_service_url` | ex `https://schema.azurewebsites.net` | Endpoint of schema service | no | output of infrastructure deployments |
| `SCHEMA_HOST` | `${schema_service_url}/schema` | Endpoint of schema API | no | - |
+| `SEARCH_HOST` | `${search_service_endpoint}` | Endpoint of search API | no | - |
| `storage_service_url` | ex `https://storage.azurewebsites.net` | Endpoint of storage service | no | output of infrastructure deployments |
| `STORAGE_SCHEMA_HOST` | `${storage_service_url}/schemas` | Endpoint of schema API | no | - |
| `STORAGE_QUERY_RECORD_HOST` | `${storage_service_url}/query/records` | Endpoint of records API | no | - |
@@ -79,10 +80,12 @@ az keyvault secret show --vault-name $KEY_VAULT_NAME --name $KEY_VAULT_SECRET_NA
| `DEFAULT_DATA_PARTITION_ID_TENANT2` | ex `common` | Secondary data partition for queries | no | Data in search index |
| `STORAGE_HOST` | ex `https://storage.azurewebsites.net/` | Storage service endpoint | no | output of infrastructure deployment |
| `SCHEMA_HOST` | ex `https://schema.azurewebsites.net/` | Endpoint of schema API | no | - |
+| `SEARCH_HOST` | ex `https://search.azurewebsites.net/` | Endpoint of search API | no | - |
| `ENVIRONMENT` | `CLOUD` | Deployment environment | no | - |
| `ENTITLEMENTS_DOMAIN` | `contoso.com` | OSDU R2 service domain | no | - |
| `LEGAL_TAG` | `opendes-public-usa-dataset-7643990` | Legal tag used for test records | no | Needs to be in DB. The referenced tag should already exist. |
| `OTHER_RELEVANT_DATA_COUNTRIES` | `US` | ? | no | - |
+| `CUCUMBER_OPTIONS` | `--tags '~@indexer-extended'` OR `--tags '~@* and @indexer-extended'` | By default `--tags '~@* and @indexer-extended'` to enable experimental feature testing | no | - |
### Configure Maven
diff --git a/provider/indexer-azure/pom.xml b/provider/indexer-azure/pom.xml
index 9533622b4fc072ad626e2559dd4844c0121b4a8c..225148b55b43d4b8da69cb1bc0919a49b11300da 100644
--- a/provider/indexer-azure/pom.xml
+++ b/provider/indexer-azure/pom.xml
@@ -21,12 +21,12 @@
<parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-service</artifactId>
- <version>0.21.0-SNAPSHOT</version>
+ <version>0.22.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>indexer-azure</artifactId>
- <version>0.21.0-SNAPSHOT</version>
+ <version>0.22.0-SNAPSHOT</version>
<name>indexer-azure</name>
<description>Indexer Service Azure</description>
<packaging>jar</packaging>
@@ -39,10 +39,10 @@
<azure.appservice.subscription />
<log4j.version>2.17.1</log4j.version>
<nimbus-jose-jwt.version>8.20.2</nimbus-jose-jwt.version>
- <indexer-core.version>0.21.0-SNAPSHOT</indexer-core.version>
+ <indexer-core.version>0.22.0-SNAPSHOT</indexer-core.version>
<spring-security-jwt.version>1.1.1.RELEASE</spring-security-jwt.version>
<osdu.corelibazure.version>0.20.0-rc5</osdu.corelibazure.version>
- <os-core-common.version>0.19.0-rc6</os-core-common.version>
+ <os-core-common.version>0.21.0</os-core-common.version>
<reactor-netty.version>0.9.12.RELEASE</reactor-netty.version>
<java-jwt.version>3.8.1</java-jwt.version>
<powermock.version>2.0.2</powermock.version>
@@ -320,6 +320,12 @@
<version>${cobertura-maven-plugin.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>xerces</groupId>
+ <artifactId>xercesImpl</artifactId>
+ <version>2.12.2</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.codehaus.plexus</groupId>
<artifactId>plexus-utils</artifactId>
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java
index 639f6e4a0e7fbf9b2bd56656d68e593df70b18c7..cf24317df9db9a6316ec7a399c6fd0b515728175 100644
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java
+++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/IndexerQueueTaskBuilderAzure.java
@@ -33,6 +33,7 @@ import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
import org.opengroup.osdu.indexer.azure.di.PublisherConfig;
import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.opengroup.osdu.indexer.model.Constants;
import org.opengroup.osdu.indexer.service.StorageService;
import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
import org.springframework.beans.factory.annotation.Autowired;
@@ -161,6 +162,7 @@ public class IndexerQueueTaskBuilderAzure extends IndexerQueueTaskBuilder {
// data
List<RecordInfo> recordInfos = parseRecordsAsJSON(receivedPayload.getData());
+ Map<String, String> attributes = receivedPayload.getAttributes();
// add all to body {"message": {"data":[], "id":...}}
JsonObject jo = new JsonObject();
@@ -168,6 +170,10 @@ public class IndexerQueueTaskBuilderAzure extends IndexerQueueTaskBuilder {
jo.addProperty(DpsHeaders.ACCOUNT_ID, headers.getPartitionIdWithFallbackToAccountId());
jo.addProperty(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
jo.addProperty(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
+ // Append the ancestry kinds used to prevent circular chasing
+ if(attributes != null && attributes.containsKey(Constants.ANCESTRY_KINDS)) {
+ jo.addProperty(Constants.ANCESTRY_KINDS, attributes.get(Constants.ANCESTRY_KINDS));
+ }
JsonObject jomsg = new JsonObject();
jomsg.add("message", jo);
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/RequestInfoImpl.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/RequestInfoImpl.java
index 11dc1340220e413f32d8af2e7d6e28dfa322dd57..4ff0343363a9f903ca82b0c976d34ed0d357d24e 100644
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/RequestInfoImpl.java
+++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/RequestInfoImpl.java
@@ -97,7 +97,7 @@ public class RequestInfoImpl implements IRequestInfo {
}
return authHeader;
} else {
- return "Bearer " + this.serviceAccountJwtClient.getIdToken(tenantInfo.getName());
+ return this.serviceAccountJwtClient.getIdToken(tenantInfo.getName());
}
}
}
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/ServiceAccountJwtClientImpl.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/ServiceAccountJwtClientImpl.java
index 12ff2ac7990717e41ff96496bc48e65accdcb055..ab4a559577720d6c9d27181fb6c0a2ff134dd495 100644
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/ServiceAccountJwtClientImpl.java
+++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/util/ServiceAccountJwtClientImpl.java
@@ -14,6 +14,7 @@
package org.opengroup.osdu.indexer.azure.util;
+import com.google.common.base.Strings;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.azure.util.AzureServicePrincipleTokenService;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
@@ -31,6 +32,7 @@ import javax.inject.Inject;
@Component
@RequestScope
public class ServiceAccountJwtClientImpl implements IServiceAccountJwtClient {
+ private final String BEARER = "Bearer";
@Inject
private ITenantFactory tenantInfoServiceProvider;
@@ -55,6 +57,10 @@ public class ServiceAccountJwtClientImpl implements IServiceAccountJwtClient {
this.dpsHeaders.put(DpsHeaders.USER_EMAIL, tenant.getServiceAccount());
- return this.tokenService.getAuthorizationToken();
+ String token = this.tokenService.getAuthorizationToken();
+ if(!Strings.isNullOrEmpty(token) && !token.startsWith(BEARER)) {
+ token = BEARER + " " + token;
+ }
+ return token;
}
}
diff --git a/provider/indexer-azure/src/main/resources/application.properties b/provider/indexer-azure/src/main/resources/application.properties
index d2dd19ac5bc19b00222901d71b828f970198f90a..2e870669aa046452a3d94e847578089fd3a3afe6 100644
--- a/provider/indexer-azure/src/main/resources/application.properties
+++ b/provider/indexer-azure/src/main/resources/application.properties
@@ -48,6 +48,7 @@ STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST=${storage_service_url}/query/records:ba
STORAGE_RECORDS_BATCH_SIZE=20
STORAGE_RECORDS_BY_KIND_BATCH_SIZE=1000
+SEARCH_HOST=${search_service_url}
INDEXER_QUEUE_HOST=http://127.0.0.1:9000
diff --git a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/ReindexServiceTest.java b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/ReindexServiceTest.java
index 5165084ac11832bc7b60e3faa6fffb8f8b54fa9a..4cca1d431c0374d040ab8945f15ca4fb35b57c2c 100644
--- a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/ReindexServiceTest.java
+++ b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/ReindexServiceTest.java
@@ -27,6 +27,7 @@ import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.indexer.Records;
import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
import org.opengroup.osdu.indexer.service.ReindexServiceImpl;
import org.opengroup.osdu.indexer.service.StorageService;
@@ -36,9 +37,11 @@ import org.powermock.modules.junit4.PowerMockRunner;
import org.powermock.modules.junit4.PowerMockRunnerDelegate;
import org.springframework.test.context.junit4.SpringRunner;
+import java.net.URISyntaxException;
import java.util.*;
import static org.junit.Assert.fail;
+import static org.mockito.Mockito.verify;
import static org.mockito.MockitoAnnotations.initMocks;
import static org.powermock.api.mockito.PowerMockito.mockStatic;
import static org.powermock.api.mockito.PowerMockito.when;
@@ -95,7 +98,7 @@ public class ReindexServiceTest {
recordQueryResponse.setResults(null);
when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
- String response = sut.reindexRecords(recordReindexRequest, false);
+ String response = sut.reindexKind(recordReindexRequest, false);
Assert.assertNull(response);
} catch (Exception e) {
@@ -109,7 +112,7 @@ public class ReindexServiceTest {
recordQueryResponse.setResults(new ArrayList<>());
when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
- String response = sut.reindexRecords(recordReindexRequest, false);
+ String response = sut.reindexKind(recordReindexRequest, false);
Assert.assertNull(response);
} catch (Exception e) {
@@ -129,7 +132,7 @@ public class ReindexServiceTest {
when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
- String taskQueuePayload = sut.reindexRecords(recordReindexRequest, false);
+ String taskQueuePayload = sut.reindexKind(recordReindexRequest, false);
Assert.assertEquals("{\"kind\":\"tenant:test:test:1.0.0\",\"cursor\":\"100\"}", taskQueuePayload);
} catch (Exception e) {
@@ -145,11 +148,26 @@ public class ReindexServiceTest {
recordQueryResponse.setResults(results);
when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
- String taskQueuePayload = sut.reindexRecords(recordReindexRequest, false);
+ String taskQueuePayload = sut.reindexKind(recordReindexRequest, false);
Assert.assertEquals(String.format("{\"data\":\"[{\\\"id\\\":\\\"test1\\\",\\\"kind\\\":\\\"tenant:test:test:1.0.0\\\",\\\"op\\\":\\\"create\\\"}]\",\"attributes\":{\"slb-correlation-id\":\"%s\"}}", correlationId), taskQueuePayload);
} catch (Exception e) {
fail("Should not throw exception" + e.getMessage());
}
}
+
+ @Test
+ public void should_createReindexTaskForValidRecords_givenValidRecordIds_reIndexRecordsTest() throws URISyntaxException {
+ DpsHeaders headers = new DpsHeaders();
+ when(requestInfo.getHeadersWithDwdAuthZ()).thenReturn(headers);
+ when(configurationProperties.getStorageRecordsBatchSize()).thenReturn(2);
+ List<String> recordIds = Arrays.asList("id1", "id2");
+ when(storageService.getStorageRecords(recordIds)).thenReturn(
+ Records.builder().records(Collections.singletonList(Records.Entity.builder().id("id1").kind("kind1").build())).notFound(Collections.singletonList("id2")).build()
+ );
+ Records records = sut.reindexRecords(recordIds);
+ Assert.assertEquals(1, records.getRecords().size());
+ Assert.assertEquals(1, records.getNotFound().size());
+ verify(indexerQueueTaskBuilder).createWorkerTask("{\"data\":\"[{\\\"id\\\":\\\"id1\\\",\\\"kind\\\":\\\"kind1\\\",\\\"op\\\":\\\"create\\\"}]\",\"attributes\":{}}", 0L, headers);
+ }
}
diff --git a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/util/RequestInfoImplTest.java b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/util/RequestInfoImplTest.java
index 8b2c66becc92c96dcd12825a5347af9cc39acdfa..b9abb87b68e61f9c3078e25f4abec6dfde23e042 100644
--- a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/util/RequestInfoImplTest.java
+++ b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/util/RequestInfoImplTest.java
@@ -24,7 +24,7 @@ public class RequestInfoImplTest {
private static String deploymentEnvironmentValue = "LOCAL";
private static String deploymentEnvironmentValueCloud = "CLOUD";
private static String tenant = "tenant1";
- private static String bearerToken = "bearerToken";
+ private static String bearerToken = "Bearer bearerToken";
private static String expectedToken = "Bearer bearerToken";
private static String partitionId = "opendes";
private static String owner = "owner";
diff --git a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/util/ServiceAccountJwtClientImplTest.java b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/util/ServiceAccountJwtClientImplTest.java
index f55c96dc2a34306fc6e0d4beb4cbed79513263f6..9478a58bb08390b0572a711f50ac56006139b25b 100644
--- a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/util/ServiceAccountJwtClientImplTest.java
+++ b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/util/ServiceAccountJwtClientImplTest.java
@@ -26,7 +26,7 @@ import static org.mockito.Mockito.*;
public class ServiceAccountJwtClientImplTest {
private String partitionId="opendes";
- private static String authorizationToken = "authorizationToken";
+ private static String authorizationToken = "Bearer authorizationToken";
@Mock
private ITenantFactory tenantInfoServiceProvider;
diff --git a/provider/indexer-gc/README.md b/provider/indexer-gc/README.md
index 40f0a3b857e18748efcf064cb99357ff2ddc97fb..22e77c33bbd094573845b7d5ad974792a0812ab0 100644
--- a/provider/indexer-gc/README.md
+++ b/provider/indexer-gc/README.md
@@ -21,9 +21,9 @@ These instructions will get you a copy of the project up and running on your loc
## Service Configuration
-### Anthos
+### Baremetal
-[Anthos service configuration](docs/anthos/README.md)
+[Baremetal service configuration](docs/baremetal/README.md)
### Google Cloud
@@ -57,9 +57,9 @@ In the current version, the mappers are equipped with several drivers to the sto
* GCloud SDK with java (latest version)
-### Anthos Service Configuration
+### Baremetal Service Configuration
-[Anthos service configuration](docs/anthos/README.md)
+[Baremetal service configuration](docs/baremetal/README.md)
### Google Cloud Service Configuration
@@ -161,9 +161,9 @@ $ (cd testing/indexer-test-core/ && mvn clean install)
This section describes how to run cloud OSDU E2E tests.
-### Anthos test configuration
+### Baremetal test configuration
-[Anthos service configuration](docs/anthos/README.md)
+[Baremetal service configuration](docs/baremetal/README.md)
### Google Cloud test configuration
diff --git a/provider/indexer-gc/cloudbuild/Dockerfile.cloudbuild b/provider/indexer-gc/cloudbuild/Dockerfile.cloudbuild
index a0ff9348ea3a81c12eb4379e5bad31c246fa1e71..3ebc13d397e1af9ee4e2cb1324d904c7017123fe 100644
--- a/provider/indexer-gc/cloudbuild/Dockerfile.cloudbuild
+++ b/provider/indexer-gc/cloudbuild/Dockerfile.cloudbuild
@@ -1,13 +1,22 @@
-# Use the official AdoptOpenJDK for a base image.
-# https://hub.docker.com/_/openjdk
-FROM openjdk:8-slim
+FROM azul/zulu-openjdk:8-latest
+
WORKDIR /app
+
ARG PROVIDER_NAME
ENV PROVIDER_NAME $PROVIDER_NAME
+
ARG PORT
ENV PORT $PORT
+
# Copy the jar to the production image from the builder stage.
COPY provider/indexer-${PROVIDER_NAME}/target/indexer-${PROVIDER_NAME}-*-spring-boot.jar indexer-${PROVIDER_NAME}.jar
+
+# Add a non-root user
+RUN groupadd -g 10001 -r nonroot \
+ && useradd -g 10001 -r -u 10001 nonroot
+
+# Run as non-root user
+USER 10001:10001
+
# Run the web service on container startup.
CMD java -Djava.security.egd=indexer:/dev/./urandom -Dserver.port=${PORT} -Dlog4j.formatMsgNoLookups=true -jar /app/indexer-${PROVIDER_NAME}.jar
-
diff --git a/provider/indexer-gc/docs/anthos/README.md b/provider/indexer-gc/docs/anthos/README.md
index 9800216a3e1cdb6ccc86f5537f108206e179dfd2..e8755d005d7663280058ff087e742b09a12e415a 100644
--- a/provider/indexer-gc/docs/anthos/README.md
+++ b/provider/indexer-gc/docs/anthos/README.md
@@ -32,7 +32,7 @@ Defined in default application property file but possible to override:
| `STORAGE_HOST` | ex `https://storage.com` | Storage host | no | output of infrastructure deployment |
| `SCHEMA_BASE_HOST` | ex `https://schema.com` | Schema service host | no | output of infrastructure deployment |
-These variables define service behavior, and are used to switch between `anthos` or `gcp` environments, their overriding and usage in mixed mode was not tested.
+These variables define service behavior, and are used to switch between `baremetal` or `gcp` environments, their overriding and usage in mixed mode was not tested.
Usage of spring profiles is preferred.
| name | value | description | sensitive? | source |
@@ -201,17 +201,21 @@ curl -L -X PATCH 'https://dev.osdu.club/api/partition/v1/partitions/opendes' -H
#### Exchanges and queues configuration
+
+
RabbitMq should have exchanges and queues with names and configs:
-| EXCHANGE NAME | EXCHANGE CONFIG | Target queue name | Target queue config |
-|----------------------------------|-----------------------------------------------------------------------------|---------------------------|----------------------------------------------------------------------|
+| EXCHANGE NAME | EXCHANGE CONFIG | Target queue name | Target queue config |
+|----------------------------------|-----------------------------------------------------------------------------|----------------------------|----------------------------------------------------------------------|
| indexing-progress | `Type fanout` <br/>`durable: true` | (Consumer not implemented) | (Consumer not implemented) |
-| records-changed | `Type fanout` <br/>`durable: true` | indexer-records-changed | `x-delivery-limit: 5`<br/>`x-queue-type: quorum`<br/>`durable: true` |
-| indexer-records-changed-exchange | `Type x-delayed-message` <br/>`durable: true`<br/>`x-delayed-type: fanout` | indexer-records-changed | `x-delivery-limit: 5`<br/>`x-queue-type: quorum`<br/>`durable: true` |
-| reprocess | `Type fanout` <br/>`durable: true` | indexer-reprocess | `x-delivery-limit: 5`<br/>`x-queue-type: quorum`<br/>`durable: true` |
-| indexer-reprocess-exchange | `Type x-delayed-message` <br/>`durable: true`<br/>`x-delayed-type: fanout` | indexer-reprocess | `x-delivery-limit: 5`<br/>`x-queue-type: quorum`<br/>`durable: true` |
-| schema-changed | `Type fanout` <br/>`durable: true` | indexer-schema-changed | `x-delivery-limit: 5`<br/>`x-queue-type: quorum`<br/>`durable: true` |
-| indexer-schema-changed-exchange | `Type x-delayed-message` <br/>`durable: true`<br/>`x-delayed-type: fanout` | indexer-schema-changed | `x-delivery-limit: 5`<br/>`x-queue-type: quorum`<br/>`durable: true` |
+| records-changed | `Type fanout` <br/>`durable: true` | indexer-records-changed | `x-delivery-limit: 5`<br/>`x-queue-type: quorum`<br/>`durable: true` |
+| indexer-records-changed-exchange | `Type x-delayed-message` <br/>`durable: true`<br/>`x-delayed-type: fanout` | indexer-records-changed | `x-delivery-limit: 5`<br/>`x-queue-type: quorum`<br/>`durable: true` |
+| reprocess | `Type fanout` <br/>`durable: true` | indexer-reprocess | `x-delivery-limit: 5`<br/>`x-queue-type: quorum`<br/>`durable: true` |
+| indexer-reprocess-exchange | `Type x-delayed-message` <br/>`durable: true`<br/>`x-delayed-type: fanout` | indexer-reprocess | `x-delivery-limit: 5`<br/>`x-queue-type: quorum`<br/>`durable: true` |
+| schema-changed | `Type fanout` <br/>`durable: true` | indexer-schema-changed | `x-delivery-limit: 5`<br/>`x-queue-type: quorum`<br/>`durable: true` |
+| indexer-schema-changed-exchange | `Type x-delayed-message` <br/>`durable: true`<br/>`x-delayed-type: fanout` | indexer-schema-changed | `x-delivery-limit: 5`<br/>`x-queue-type: quorum`<br/>`durable: true` |
+| reindex | `Type fanout` <br/>`durable: true` | indexer-reindex | `x-delivery-limit: 5`<br/>`x-queue-type: quorum`<br/>`durable: true` |
+| indexer-reindex-exchange | `Type x-delayed-message` <br/>`durable: true`<br/>`x-delayed-type: fanout` | indexer-reindex | `x-delivery-limit: 5`<br/>`x-queue-type: quorum`<br/>`durable: true` |
## Keycloak configuration
@@ -233,35 +237,36 @@ Give `client-id` and `client-secret` to services, which should be authorized wit
You will need to have the following environment variables defined.
-| name | value | description | sensitive? | source |
-|--------------------------------------|-----------------------------------------------------------------|---------------------------------------------------------------------------------------------------|-----------------------------------------|-------------------------------------|
-| `ELASTIC_PASSWORD` | `********` | Password for Elasticsearch | yes | output of infrastructure deployment |
-| `ELASTIC_USER_NAME` | `********` | User name for Elasticsearch | yes | output of infrastructure deployment |
-| `ELASTIC_HOST` | ex `elastic.domain.com` | Host Elasticsearch | yes | output of infrastructure deployment |
-| `ELASTIC_PORT` | ex `9243` | Port Elasticsearch | yes | output of infrastructure deployment |
-| `INDEXER_HOST` | ex `https://os-indexer-dot-opendes.appspot.com/api/indexer/v2/` | Indexer API endpoint | no | output of infrastructure deployment |
-| `ENTITLEMENTS_DOMAIN` | ex `opendes-gcp.projects.com` | OSDU R2 to run tests under | no | - |
-| `OTHER_RELEVANT_DATA_COUNTRIES` | ex `US` | valid legal tag with a other relevant data countries | no | - |
-| `LEGAL_TAG` | ex `opendes-demo-legaltag` | valid legal tag with a other relevant data countries from `DEFAULT_OTHER_RELEVANT_DATA_COUNTRIES` | no | - |
-| `DEFAULT_DATA_PARTITION_ID_TENANT1` | ex `opendes` | HTTP Header 'Data-Partition-ID' | no | - |
-| `DEFAULT_DATA_PARTITION_ID_TENANT2` | ex `opendes` | HTTP Header 'Data-Partition-ID' | no | - |
-| `SEARCH_HOST` | ex `http://localhost:8080/api/search/v2/` | Endpoint of search service | no | - |
-| `STORAGE_HOST` | ex `http://os-storage-dot-opendes.appspot.com/api/storage/v2/` | Storage API endpoint | no | output of infrastructure deployment |
-| `SECURITY_HTTPS_CERTIFICATE_TRUST` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment |
-| `TEST_OPENID_PROVIDER_CLIENT_ID` | `********` | Client Id for `$INTEGRATION_TESTER` | yes | -- |
-| `TEST_OPENID_PROVIDER_CLIENT_SECRET` | `********` | | Client secret for `$INTEGRATION_TESTER` | -- |
-| `TEST_OPENID_PROVIDER_URL` | `https://keycloak.com/auth/realms/osdu` | OpenID provider url | yes | -- |
+| name | value | description | sensitive? | source |
+|--------------------------------------|-----------------------------------------------------------------------|---------------------------------------------------------------------------------------------------|-----------------------------------------|-------------------------------------|
+| `ELASTIC_PASSWORD` | `********` | Password for Elasticsearch | yes | output of infrastructure deployment |
+| `ELASTIC_USER_NAME` | `********` | User name for Elasticsearch | yes | output of infrastructure deployment |
+| `ELASTIC_HOST` | ex `elastic.domain.com` | Host Elasticsearch | yes | output of infrastructure deployment |
+| `ELASTIC_PORT` | ex `9243` | Port Elasticsearch | yes | output of infrastructure deployment |
+| `INDEXER_HOST` | ex `https://os-indexer-dot-opendes.appspot.com/api/indexer/v2/` | Indexer API endpoint | no | output of infrastructure deployment |
+| `GROUP_ID` | ex `opendes-gcp.projects.com` | OSDU R2 to run tests under | no | - |
+| `OTHER_RELEVANT_DATA_COUNTRIES` | ex `US` | valid legal tag with a other relevant data countries | no | - |
+| `LEGAL_TAG` | ex `opendes-demo-legaltag` | valid legal tag with a other relevant data countries from `DEFAULT_OTHER_RELEVANT_DATA_COUNTRIES` | no | - |
+| `DEFAULT_DATA_PARTITION_ID_TENANT1` | ex `opendes` | HTTP Header 'Data-Partition-ID' | no | - |
+| `DEFAULT_DATA_PARTITION_ID_TENANT2` | ex `opendes` | HTTP Header 'Data-Partition-ID' | no | - |
+| `SEARCH_HOST` | ex `http://localhost:8080/api/search/v2/` | Endpoint of search service | no | - |
+| `STORAGE_HOST` | ex `http://os-storage-dot-opendes.appspot.com/api/storage/v2/` | Storage API endpoint | no | output of infrastructure deployment |
+| `SECURITY_HTTPS_CERTIFICATE_TRUST` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment |
+| `TEST_OPENID_PROVIDER_CLIENT_ID` | `********` | Client Id for `$INTEGRATION_TESTER` | yes | -- |
+| `TEST_OPENID_PROVIDER_CLIENT_SECRET` | `********` | | Client secret for `$INTEGRATION_TESTER` | -- |
+| `TEST_OPENID_PROVIDER_URL` | `https://keycloak.com/auth/realms/osdu` | OpenID provider url | yes | -- |
+| `CUCUMBER_OPTIONS` | `--tags '~@indexer-extended'` OR `--tags '~@* and @indexer-extended'` | By default `--tags '~@indexer-extended'` to disable experimental feature testing | no | -- |
**Entitlements configuration for integration accounts**
-| INTEGRATION_TESTER | NO_DATA_ACCESS_TESTER |
-|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------|
-| users<br/>users.datalake.ops<br/>service.storage.creator<br/>service.entitlements.user<br/>service.search.user<br/>service.search.admin<br/>data.test1<br/>data.integration.test<br/>users@{tenant1}@{domain}.com | |
+| INTEGRATION_TESTER | NO_DATA_ACCESS_TESTER |
+|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------|
+| users<br/>users.datalake.ops<br/>service.storage.creator<br/>service.entitlements.user<br/>service.search.user<br/>service.search.admin<br/>data.test1<br/>data.integration.test<br/>users@{tenant1}@{groupId}.com | |
Execute following command to build code and run all the integration tests:
```bash
# Note: this assumes that the environment variables for integration tests as outlined
# above are already exported in your environment.
-$ (cd testing/indexer-test-anthos/ && mvn clean test)
+$ (cd testing/indexer-test-baremetal/ && mvn clean test)
```
diff --git a/provider/indexer-gc/docs/anthos/pics/indexer.png b/provider/indexer-gc/docs/anthos/pics/indexer.png
new file mode 100644
index 0000000000000000000000000000000000000000..711fc590e19dab1bbcb1837521ad58d96fff81da
Binary files /dev/null and b/provider/indexer-gc/docs/anthos/pics/indexer.png differ
diff --git a/provider/indexer-gc/docs/gc/README.md b/provider/indexer-gc/docs/gc/README.md
index 8edb09a7137f11a0a93a89ffab304faa2a81e472..cac8344500587a8fbcf827ba972527f54f280499 100644
--- a/provider/indexer-gc/docs/gc/README.md
+++ b/provider/indexer-gc/docs/gc/README.md
@@ -14,21 +14,22 @@ Must have:
Defined in default application property file but possible to override:
-| name | value | description | sensitive? | source |
-|------------------------------------|---------------------------------------------------------------------------|---------------------------------------------------------------------------|------------|--------------------------------------------------------------|
-| `LOG_PREFIX` | `service` | Logging prefix | no | - |
-| `LOG_LEVEL` | `****` | Logging level | no | - |
-| `SECURITY_HTTPS_CERTIFICATE_TRUST` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment |
-| `REDIS_SEARCH_HOST` | ex `127.0.0.1` | Redis host | no | |
-| `REDIS_SEARCH_PORT` | ex `6379` | Redis host port | no | |
-| `REDIS_SEARCH_PASSWORD` | ex `*****` | Redis host password | yes | |
-| `REDIS_SEARCH_WITH_SSL` | ex `true` or `false` | Redis host ssl config | no | |
-| `REDIS_SEARCH_EXPIRATION` | ex `30` | Redis cache expiration in seconds | no | |
-| `PARTITION_HOST` | ex `https://partition.com` | Partition host | no | output of infrastructure deployment |
-| `ENTITLEMENTS_HOST` | ex `https://entitlements.com` | Entitlements host | no | output of infrastructure deployment |
-| `STORAGE_HOST` | ex `https://storage.com` | Storage host | no | output of infrastructure deployment |
-| `SCHEMA_BASE_HOST` | ex `https://schema.com` | Schema service host | no | output of infrastructure deployment |
-| `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | <https://console.cloud.google.com/iam-admin/serviceaccounts> |
+| name | value | description | sensitive? | source |
+|------------------------------------|------------------------------------------|---------------------------------------------------------------------------------------------------------------------|------------|--------------------------------------------------------------|
+| `LOG_PREFIX` | `service` | Logging prefix | no | - |
+| `LOG_LEVEL` | `****` | Logging level | no | - |
+| `SECURITY_HTTPS_CERTIFICATE_TRUST` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment |
+| `REDIS_SEARCH_HOST` | ex `127.0.0.1` | Redis host | no | |
+| `REDIS_SEARCH_PORT` | ex `6379` | Redis host port | no | |
+| `REDIS_SEARCH_PASSWORD` | ex `*****` | Redis host password | yes | |
+| `REDIS_SEARCH_WITH_SSL` | ex `true` or `false` | Redis host ssl config | no | |
+| `REDIS_SEARCH_EXPIRATION` | ex `30` | Redis cache expiration in seconds | no | |
+| `PARTITION_HOST` | ex `https://partition.com` | Partition host | no | output of infrastructure deployment |
+| `ENTITLEMENTS_HOST` | ex `https://entitlements.com` | Entitlements host | no | output of infrastructure deployment |
+| `STORAGE_HOST` | ex `https://storage.com` | Storage host | no | output of infrastructure deployment |
+| `SCHEMA_BASE_HOST` | ex `https://schema.com` | Schema service host | no | output of infrastructure deployment |
+| `GOOGLE_APPLICATION_CREDENTIALS` | ex `/path/to/directory/service-key.json` | Service account credentials, you only need this if running locally | yes | <https://console.cloud.google.com/iam-admin/serviceaccounts> |
+| `DEAD_LETTERING_ENABLED` | ex `true` or `false` | Dead lettering configuration validation, if enabled, then service will require configured dead lettering in Pubsub. | no | <https://console.cloud.google.com/cloudpubsub/topic/list> |
These variables define service behavior, and are used to switch between `Reference` or `Google Cloud` environments, their overriding and usage in mixed mode was not tested.
Usage of spring profiles is preferred.
@@ -41,17 +42,21 @@ Usage of spring profiles is preferred.
## Pubsub configuration
+
+
Pubsub should have topics and subscribers with names and configs:
-| TOPIC NAME | Subscription name | Subscription config |
-|-----------------------------|----------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| indexing-progress | (Consumer not implemented) | (Consumer not implemented) |
-| records-changed | indexer-records-changed | `Maximum delivery attempts: 10`<br/>`Retry policy: Retry after exponential backoff delay`<br/>`Minimum backoff duration: 0 seconds`<br/>`Maximum backoff duration: 30 seconds`<br/>`Grant forwarding permissions for dead letter` |
-| records-changed-dead-letter | (Consumer not implemented) | (Consumer not implemented) |
-| reprocess | indexer-reprocess | `Maximum delivery attempts: 5`<br/>`Retry policy: Retry after exponential backoff delay`<br/>`Minimum backoff duration: 10 seconds`<br/>`Maximum backoff duration: 600 seconds`<br/>`Grant forwarding permissions for dead letter` |
-| reprocess-dead-letter | (Consumer not implemented) | (Consumer not implemented) |
-| schema-changed | indexer-schema-changed | `Maximum delivery attempts: 5`<br/>`Retry policy: Retry after exponential backoff delay`<br/>`Minimum backoff duration: 10 seconds`<br/>`Maximum backoff duration: 600 seconds`<br/>`Grant forwarding permissions for dead letter` |
-| schema-changed-dead-letter | (Consumer not implemented) | (Consumer not implemented) |
+| TOPIC NAME | Subscription name | Subscription config |
+|----------------------------------|----------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| indexing-progress | (Consumer not implemented) | (Consumer not implemented) |
+| records-changed | indexer-records-changed | `Maximum delivery attempts: 10`<br/>`Retry policy: Retry after exponential backoff delay`<br/>`Minimum backoff duration: 0 seconds`<br/>`Maximum backoff duration: 30 seconds`<br/>`Grant forwarding permissions for dead letter` |
+| records-changed-dead-lettering | (Consumer not implemented) | (Consumer not implemented) |
+| reprocess | indexer-reprocess | `Maximum delivery attempts: 5`<br/>`Retry policy: Retry after exponential backoff delay`<br/>`Minimum backoff duration: 10 seconds`<br/>`Maximum backoff duration: 600 seconds`<br/>`Grant forwarding permissions for dead letter` |
+| indexer-reprocess-dead-lettering | (Consumer not implemented) | (Consumer not implemented) |
+| schema-changed | indexer-schema-changed | `Maximum delivery attempts: 5`<br/>`Retry policy: Retry after exponential backoff delay`<br/>`Minimum backoff duration: 10 seconds`<br/>`Maximum backoff duration: 600 seconds`<br/>`Grant forwarding permissions for dead letter` |
+| schema-changed-dead-lettering | (Consumer not implemented) | (Consumer not implemented) |
+| reindex | indexer-reindex | `Maximum delivery attempts: 5`<br/>`Retry policy: Retry after exponential backoff delay`<br/>`Minimum backoff duration: 10 seconds`<br/>`Maximum backoff duration: 600 seconds`<br/>`Grant forwarding permissions for dead letter` |
+| reindex-dead-lettering | (Consumer not implemented) | (Consumer not implemented) |
### Additional throughput configuration for PubSub subscription consumer via Partition service
@@ -167,29 +172,30 @@ TBD
You will need to have the following environment variables defined.
-| name | value | description | sensitive? | source |
-|-------------------------------------|----------------------------------------------------------------|---------------------------------------------------------------------------------------------------|------------|------------------------------------------------------------|
-| `ELASTIC_PASSWORD` | `********` | Password for Elasticsearch | yes | output of infrastructure deployment |
-| `ELASTIC_USER_NAME` | `********` | User name for Elasticsearch | yes | output of infrastructure deployment |
-| `ELASTIC_HOST` | ex `elastic.domain.com` | Host Elasticsearch | yes | output of infrastructure deployment |
-| `ELASTIC_PORT` | ex `9243` | Port Elasticsearch | yes | output of infrastructure deployment |
-| `GCLOUD_PROJECT` | ex `opendes` | Google Cloud Project Id | no | output of infrastructure deployment |
-| `INDEXER_HOST` | ex `https://os-indexer-dot-opendes.appspot.com/api/indexer/v2/` | Indexer API endpoint | no | output of infrastructure deployment |
-| `ENTITLEMENTS_DOMAIN` | ex `opendes-gc.projects.com` | OSDU R2 to run tests under | no | - |
-| `OTHER_RELEVANT_DATA_COUNTRIES` | ex `US` | valid legal tag with a other relevant data countries | no | - |
-| `LEGAL_TAG` | ex `opendes-demo-legaltag` | valid legal tag with a other relevant data countries from `DEFAULT_OTHER_RELEVANT_DATA_COUNTRIES` | no | - |
-| `DEFAULT_DATA_PARTITION_ID_TENANT1` | ex `opendes` | HTTP Header 'Data-Partition-ID' | no | - |
-| `DEFAULT_DATA_PARTITION_ID_TENANT2` | ex `opendes` | HTTP Header 'Data-Partition-ID' | no | - |
-| `SEARCH_INTEGRATION_TESTER` | `********` | Service account for API calls. Note: this user must have entitlements configured already | yes | <https://console.cloud.google.com/iam-admin/serviceaccounts> |
-| `SEARCH_HOST` | ex `http://localhost:8080/api/search/v2/` | Endpoint of search service | no | - |
-| `STORAGE_HOST` | ex `http://os-storage-dot-opendes.appspot.com/api/storage/v2/` | Storage API endpoint | no | output of infrastructure deployment |
-| `SECURITY_HTTPS_CERTIFICATE_TRUST` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment |
+| name | value | description | sensitive? | source |
+|-------------------------------------|-----------------------------------------------------------------------|---------------------------------------------------------------------------------------------------|------------|--------------------------------------------------------------|
+| `ELASTIC_PASSWORD` | `********` | Password for Elasticsearch | yes | output of infrastructure deployment |
+| `ELASTIC_USER_NAME` | `********` | User name for Elasticsearch | yes | output of infrastructure deployment |
+| `ELASTIC_HOST` | ex `elastic.domain.com` | Host Elasticsearch | yes | output of infrastructure deployment |
+| `ELASTIC_PORT` | ex `9243` | Port Elasticsearch | yes | output of infrastructure deployment |
+| `GCLOUD_PROJECT` | ex `opendes` | Google Cloud Project Id | no | output of infrastructure deployment |
+| `INDEXER_HOST` | ex `https://os-indexer-dot-opendes.appspot.com/api/indexer/v2/` | Indexer API endpoint | no | output of infrastructure deployment |
+| `GROUP_ID` | ex `opendes-gc.projects.com` | OSDU R2 to run tests under | no | - |
+| `OTHER_RELEVANT_DATA_COUNTRIES` | ex `US` | valid legal tag with a other relevant data countries | no | - |
+| `LEGAL_TAG` | ex `opendes-demo-legaltag` | valid legal tag with a other relevant data countries from `DEFAULT_OTHER_RELEVANT_DATA_COUNTRIES` | no | - |
+| `DEFAULT_DATA_PARTITION_ID_TENANT1` | ex `opendes` | HTTP Header 'Data-Partition-ID' | no | - |
+| `DEFAULT_DATA_PARTITION_ID_TENANT2` | ex `opendes` | HTTP Header 'Data-Partition-ID' | no | - |
+| `SEARCH_INTEGRATION_TESTER` | `********` | Service account for API calls. Note: this user must have entitlements configured already | yes | <https://console.cloud.google.com/iam-admin/serviceaccounts> |
+| `SEARCH_HOST` | ex `http://localhost:8080/api/search/v2/` | Endpoint of search service | no | - |
+| `STORAGE_HOST` | ex `http://os-storage-dot-opendes.appspot.com/api/storage/v2/` | Storage API endpoint | no | output of infrastructure deployment |
+| `SECURITY_HTTPS_CERTIFICATE_TRUST` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment |
+| `CUCUMBER_OPTIONS` | `--tags '~@indexer-extended'` OR `--tags '~@* and @indexer-extended'` | By default `--tags '~@indexer-extended'` to disable experimental feature testing | no | -- |
**Entitlements configuration for integration accounts**
-| INTEGRATION_TESTER | NO_DATA_ACCESS_TESTER |
-|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------|
-| users<br/>users.datalake.ops<br/>service.storage.creator<br/>service.entitlements.user<br/>service.search.user<br/>service.search.admin<br/>data.test1<br/>data.integration.test<br/>users@{tenant1}@{domain}.com |
+| INTEGRATION_TESTER | NO_DATA_ACCESS_TESTER |
+|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------|
+| users<br/>users.datalake.ops<br/>service.storage.creator<br/>service.entitlements.user<br/>service.search.user<br/>service.search.admin<br/>data.test1<br/>data.integration.test<br/>users@{tenant1}@{groupId}.com |
Execute following command to build code and run all the integration tests:
diff --git a/provider/indexer-gc/pom.xml b/provider/indexer-gc/pom.xml
index 634032280d30488dc829882408020011869d670d..dc9d9b3103031b91c3802d5ccc31efedfa6f1760 100644
--- a/provider/indexer-gc/pom.xml
+++ b/provider/indexer-gc/pom.xml
@@ -5,12 +5,12 @@
<parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-service</artifactId>
- <version>0.21.0-SNAPSHOT</version>
+ <version>0.22.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>indexer-gc</artifactId>
- <version>0.21.0-SNAPSHOT</version>
+ <version>0.22.0-SNAPSHOT</version>
<name>indexer-gc</name>
<description>Indexer Service Google Cloud</description>
<packaging>jar</packaging>
@@ -21,7 +21,7 @@
<dependency>
<groupId>com.fasterxml.jackson</groupId>
<artifactId>jackson-bom</artifactId>
- <version>2.14.2</version>
+ <version>2.15.0</version>
<type>pom</type>
<scope>import</scope>
</dependency>
@@ -34,17 +34,18 @@
</dependency>
</dependencies>
</dependencyManagement>
+
<dependencies>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-gc</artifactId>
- <version>0.21.0-rc4</version>
+ <version>0.21.0</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-core</artifactId>
- <version>0.21.0-SNAPSHOT</version>
- <!-- excluded due to runtime conflict with latest core-lib-gcp transient dependencies -->
+ <version>0.22.0-SNAPSHOT</version>
+ <!-- excluded due to runtime conflict with latest core-lib-gc transient dependencies -->
<exclusions>
<exclusion>
<groupId>com.google.api-client</groupId>
@@ -156,10 +157,11 @@
<version>5.3.22</version>
</dependency>
+ <!-- Mappers -->
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>oqm</artifactId>
- <version>0.21.0-rc3</version>
+ <version>0.21.0</version>
</dependency>
</dependencies>
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/publish/ReprocessingTaskPublisher.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/publish/ReprocessingTaskPublisher.java
index 720628c8a042376ddf4d2ac549a428918674c95c..85a824f6241add7fd46f74f7bbabe365ba2c5ee4 100644
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/publish/ReprocessingTaskPublisher.java
+++ b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/publish/ReprocessingTaskPublisher.java
@@ -26,7 +26,8 @@ import org.opengroup.osdu.core.gcp.oqm.driver.OqmDriver;
import org.opengroup.osdu.core.gcp.oqm.model.OqmDestination;
import org.opengroup.osdu.core.gcp.oqm.model.OqmMessage;
import org.opengroup.osdu.core.gcp.oqm.model.OqmTopic;
-import org.opengroup.osdu.indexer.provider.gcp.indexing.processing.IndexerMessagingConfigProperties;
+import org.opengroup.osdu.indexer.model.Constants;
+import org.opengroup.osdu.indexer.provider.gcp.indexing.config.MessagingConfigProperties;
import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
@@ -51,16 +52,16 @@ public class ReprocessingTaskPublisher extends IndexerQueueTaskBuilder {
private final OqmDriver driver;
- private final IndexerMessagingConfigProperties properties;
+ private final MessagingConfigProperties properties;
private OqmTopic reprocessOqmTopic;
- private OqmTopic recordsChangedTopic;
+ private OqmTopic reindexTopic;
@PostConstruct
public void setUp() {
reprocessOqmTopic = OqmTopic.builder().name(properties.getReprocessTopicName()).build();
- recordsChangedTopic = OqmTopic.builder().name(properties.getRecordsChangedTopicName()).build();
+ reindexTopic = OqmTopic.builder().name(properties.getReindexTopicName()).build();
}
public void createWorkerTask(String payload, DpsHeaders headers) {
@@ -111,15 +112,6 @@ public class ReprocessingTaskPublisher extends IndexerQueueTaskBuilder {
);
}
- private void publishReindexTask(String payload, DpsHeaders headers) {
- OqmDestination oqmDestination = OqmDestination.builder().partitionId(headers.getPartitionId())
- .build();
- Map<String, String> attributes = getAttributesFromHeaders(headers);
- OqmMessage oqmMessage = OqmMessage.builder().data(payload).attributes(attributes).build();
- log.info("Reprocessing task: {} ,has been published.", oqmMessage);
- driver.publish(oqmMessage, reprocessOqmTopic, oqmDestination);
- }
-
private void publishRecordsChangedTask(String payload, DpsHeaders headers) {
OqmDestination oqmDestination = OqmDestination.builder()
.partitionId(headers.getPartitionId())
@@ -128,14 +120,29 @@ public class ReprocessingTaskPublisher extends IndexerQueueTaskBuilder {
RecordChangedMessages recordChangedMessages = gson.fromJson(payload,
RecordChangedMessages.class);
+ Map<String, String> attributes = getAttributesFromHeaders(headers);
+ // Append the ancestry kinds used to prevent circular chasing
+ if(recordChangedMessages.getAttributes().containsKey(Constants.ANCESTRY_KINDS)) {
+ attributes.put(Constants.ANCESTRY_KINDS, recordChangedMessages.getAttributes().get(Constants.ANCESTRY_KINDS));
+ }
+
OqmMessage oqmMessage = OqmMessage.builder()
.id(headers.getCorrelationId())
.data(recordChangedMessages.getData())
- .attributes(getAttributesFromHeaders(headers))
+ .attributes(attributes)
.build();
+ log.info("Reindex task: {} ,has been published.", oqmMessage);
+ driver.publish(oqmMessage, reindexTopic, oqmDestination);
+ }
+
+ private void publishReindexTask(String payload, DpsHeaders headers) {
+ OqmDestination oqmDestination = OqmDestination.builder().partitionId(headers.getPartitionId())
+ .build();
+ Map<String, String> attributes = getAttributesFromHeaders(headers);
+ OqmMessage oqmMessage = OqmMessage.builder().data(payload).attributes(attributes).build();
log.info("Reprocessing task: {} ,has been published.", oqmMessage);
- driver.publish(oqmMessage, recordsChangedTopic, oqmDestination);
+ driver.publish(oqmMessage, reprocessOqmTopic, oqmDestination);
}
@NotNull
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/publish/StatusPublisherImpl.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/publish/StatusPublisherImpl.java
index 4551656f75da06d7f66f58004de9f0c4262c1352..2cad9bb1dc87599c5a7408c194013ff8ec44cab9 100644
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/publish/StatusPublisherImpl.java
+++ b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/publish/StatusPublisherImpl.java
@@ -31,7 +31,7 @@ import org.opengroup.osdu.core.gcp.oqm.driver.OqmDriver;
import org.opengroup.osdu.core.gcp.oqm.model.OqmDestination;
import org.opengroup.osdu.core.gcp.oqm.model.OqmMessage;
import org.opengroup.osdu.core.gcp.oqm.model.OqmTopic;
-import org.opengroup.osdu.indexer.provider.gcp.indexing.processing.IndexerMessagingConfigProperties;
+import org.opengroup.osdu.indexer.provider.gcp.indexing.config.MessagingConfigProperties;
import org.opengroup.osdu.indexer.provider.interfaces.IPublisher;
import org.springframework.stereotype.Component;
@@ -41,7 +41,7 @@ import org.springframework.stereotype.Component;
public class StatusPublisherImpl implements IPublisher {
private final OqmDriver driver;
- private final IndexerMessagingConfigProperties properties;
+ private final MessagingConfigProperties properties;
private final JsonSerializer<JobStatus> statusJsonSerializer;
private OqmTopic oqmTopic;
private Gson gson;
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/IndexerMessagingConfigProperties.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/config/MessagingConfigProperties.java
similarity index 81%
rename from provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/IndexerMessagingConfigProperties.java
rename to provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/config/MessagingConfigProperties.java
index 04dd82069daa7e9210566136573f81cc583bc1e1..384768395976742f72174fae415e52e4bbd46302 100644
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/IndexerMessagingConfigProperties.java
+++ b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/config/MessagingConfigProperties.java
@@ -1,6 +1,6 @@
/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
+ * Copyright 2020-2023 Google LLC
+ * Copyright 2020-2023 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.opengroup.osdu.indexer.provider.gcp.indexing.processing;
+package org.opengroup.osdu.indexer.provider.gcp.indexing.config;
import lombok.Getter;
import lombok.Setter;
@@ -26,12 +26,12 @@ import org.springframework.context.annotation.Configuration;
@Getter
@ConfigurationProperties
@Configuration
-public class IndexerMessagingConfigProperties {
-
+public class MessagingConfigProperties {
+ @Deprecated
+ private String defaultRelativeIndexerWorkerUrl;
private String recordsChangedTopicName;
private String schemaChangedTopicName;
- private String defaultRelativeIndexerWorkerUrl;
- private String reprocessTopicName;
private String statusChangedTopicName;
-
+ private String reprocessTopicName;
+ private String reindexTopicName;
}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/initialization/OqmSubscriberManager.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/initialization/OqmSubscriberManager.java
new file mode 100644
index 0000000000000000000000000000000000000000..716dc5a80e5690e3d8d620e09e39683268200e40
--- /dev/null
+++ b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/initialization/OqmSubscriberManager.java
@@ -0,0 +1,116 @@
+/*
+ * Copyright 2020-2023 Google LLC
+ * Copyright 2020-2023 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.provider.gcp.indexing.initialization;
+
+import javax.annotation.Nullable;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.gcp.oqm.driver.OqmDriver;
+import org.opengroup.osdu.core.gcp.oqm.model.OqmDestination;
+import org.opengroup.osdu.core.gcp.oqm.model.OqmMessageReceiver;
+import org.opengroup.osdu.core.gcp.oqm.model.OqmSubscriber;
+import org.opengroup.osdu.core.gcp.oqm.model.OqmSubscriberThroughput;
+import org.opengroup.osdu.core.gcp.oqm.model.OqmSubscription;
+import org.opengroup.osdu.core.gcp.oqm.model.OqmSubscriptionQuery;
+import org.opengroup.osdu.core.gcp.oqm.model.OqmTopic;
+import org.springframework.http.HttpStatus;
+import org.springframework.stereotype.Service;
+
+@Service
+@Slf4j
+@RequiredArgsConstructor
+public class OqmSubscriberManager {
+
+ private final OqmDriver driver;
+
+ public void registerSubscriber(String dataPartitionId, String topicName, String subscriptionName,
+ OqmMessageReceiver messageReceiver, OqmSubscriberThroughput throughput) {
+ OqmSubscription subscriptionForTenant = getSubscriptionForTenant(dataPartitionId, topicName, subscriptionName);
+ log.info("OQM: registering Subscriber for subscription {}", subscriptionName);
+
+ OqmDestination destination = getDestination(dataPartitionId);
+ OqmSubscriber subscriber = OqmSubscriber.builder()
+ .subscription(subscriptionForTenant)
+ .messageReceiver(messageReceiver)
+ .throughput(throughput)
+ .build();
+
+ driver.subscribe(subscriber, destination);
+ log.info("OQM: provisioning subscription {}: Subscriber REGISTERED.", subscriptionName);
+ }
+
+ private OqmSubscription getSubscriptionForTenant(String dataPartitionId, String topicName, String subscriptionName) {
+ log.info("OQM: provisioning tenant {}:", dataPartitionId);
+ log.info("OQM: check for topic {} existence:", topicName);
+ OqmTopic topic = driver.getTopic(topicName, getDestination(dataPartitionId)).orElse(null);
+
+ if (topic == null) {
+ log.error("OQM: check for topic: {}, tenant: {} existence: ABSENT.", topicName, dataPartitionId);
+ throw new AppException(
+ HttpStatus.INTERNAL_SERVER_ERROR.value(),
+ "Required topic not exists.",
+ String.format(
+ "Required topic not exists. Create topic: %s for tenant: %s and restart service.",
+ topicName, dataPartitionId
+ )
+ );
+ }
+
+ log.info("OQM: check for topic {} existence: PRESENT", topicName);
+ OqmSubscription subscription = getSubscription(dataPartitionId, topic, subscriptionName);
+
+ if (subscription == null) {
+ log.error(
+ "OQM: check for subscription {}, tenant: {} existence: ABSENT.",
+ subscriptionName,
+ dataPartitionId
+ );
+ throw new AppException(
+ HttpStatus.INTERNAL_SERVER_ERROR.value(),
+ "Required subscription not exists.",
+ String.format(
+ "Required subscription not exists. Create subscription: %s for tenant: %s and restart service.",
+ subscriptionName,
+ dataPartitionId
+ )
+ );
+ }
+ log.info("OQM: provisioning tenant {}: COMPLETED.", dataPartitionId);
+ return subscription;
+ }
+
+ @Nullable
+ private OqmSubscription getSubscription(String dataPartitionId, OqmTopic topic, String subscriptionName) {
+ log.info("OQM: check for subscription {} existence:", subscriptionName);
+ OqmSubscriptionQuery query = OqmSubscriptionQuery.builder()
+ .namePrefix(subscriptionName)
+ .subscriberable(true)
+ .build();
+ return driver
+ .listSubscriptions(topic, query, getDestination(dataPartitionId)).stream()
+ .findAny()
+ .orElse(null);
+ }
+
+ private OqmDestination getDestination(String dataPartitionId) {
+ return OqmDestination.builder()
+ .partitionId(dataPartitionId)
+ .build();
+ }
+}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/initialization/TenantSubscriberConfiguration.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/initialization/TenantSubscriberConfiguration.java
new file mode 100644
index 0000000000000000000000000000000000000000..10c7722bf9c56a265bf45be14ba9d8fe681ea855
--- /dev/null
+++ b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/initialization/TenantSubscriberConfiguration.java
@@ -0,0 +1,104 @@
+/*
+ * Copyright 2020-2023 Google LLC
+ * Copyright 2020-2023 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.provider.gcp.indexing.initialization;
+
+import javax.annotation.PostConstruct;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.opengroup.osdu.core.auth.TokenProvider;
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
+import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
+import org.opengroup.osdu.core.gcp.oqm.model.OqmSubscriberThroughput;
+import org.opengroup.osdu.indexer.api.RecordIndexerApi;
+import org.opengroup.osdu.indexer.api.ReindexApi;
+import org.opengroup.osdu.indexer.provider.gcp.indexing.config.MessagingConfigProperties;
+import org.opengroup.osdu.indexer.provider.gcp.indexing.processing.RecordsChangedMessageReceiver;
+import org.opengroup.osdu.indexer.provider.gcp.indexing.processing.ReindexMessageReceiver;
+import org.opengroup.osdu.indexer.provider.gcp.indexing.processing.ReprocessorMessageReceiver;
+import org.opengroup.osdu.indexer.provider.gcp.indexing.processing.SchemaChangedMessageReceiver;
+import org.opengroup.osdu.indexer.provider.gcp.indexing.scope.ThreadDpsHeaders;
+import org.springframework.stereotype.Component;
+
+/**
+ * Subscription configuration class.
+ */
+@Slf4j
+@Component
+@RequiredArgsConstructor
+public class TenantSubscriberConfiguration {
+
+ private static final String SUBSCRIPTION_PREFIX = "indexer-";
+ private final MessagingConfigProperties properties;
+ private final OqmSubscriberManager subscriberManager;
+ private final ITenantFactory tenantInfoFactory;
+ private final TokenProvider tokenProvider;
+ private final ThreadDpsHeaders headers;
+ private final RecordIndexerApi recordIndexerApi;
+ private final ReindexApi reindexApi;
+
+ /**
+ * Tenant configurations provided by the Partition service will be used to configure subscribers. If tenants use the
+ * same message broker(The same RabbitMQ instance, or the same GCP project Pub/Sub) then only one subscriber in this
+ * broker will be used.
+ */
+ @PostConstruct
+ void postConstruct() {
+ log.info("OqmSubscriberManager provisioning STARTED");
+ String recordsChangedTopicName = properties.getRecordsChangedTopicName();
+ String schemaChangedTopicName = properties.getSchemaChangedTopicName();
+ String reprocessTopicName = properties.getReprocessTopicName();
+ String reindexTopicName = properties.getReindexTopicName();
+
+ for (TenantInfo tenantInfo : tenantInfoFactory.listTenantInfo()) {
+ String dataPartitionId = tenantInfo.getDataPartitionId();
+ subscriberManager.registerSubscriber(
+ dataPartitionId,
+ recordsChangedTopicName,
+ getSubscriptionName(recordsChangedTopicName),
+ new RecordsChangedMessageReceiver(headers, tokenProvider, recordIndexerApi),
+ OqmSubscriberThroughput.MAX
+ );
+ subscriberManager.registerSubscriber(
+ dataPartitionId,
+ schemaChangedTopicName,
+ getSubscriptionName(schemaChangedTopicName),
+ new SchemaChangedMessageReceiver(headers, tokenProvider, recordIndexerApi),
+ OqmSubscriberThroughput.MIN
+ );
+ subscriberManager.registerSubscriber(
+ dataPartitionId,
+ reprocessTopicName,
+ getSubscriptionName(reprocessTopicName),
+ new ReprocessorMessageReceiver(headers, tokenProvider, reindexApi),
+ OqmSubscriberThroughput.MIN
+ );
+ subscriberManager.registerSubscriber(
+ dataPartitionId,
+ reindexTopicName,
+ getSubscriptionName(reindexTopicName),
+ new ReindexMessageReceiver(headers, tokenProvider, recordIndexerApi),
+ OqmSubscriberThroughput.MAX
+ );
+ }
+ log.info("OqmSubscriberManager provisioning COMPLETED");
+ }
+
+ private String getSubscriptionName(String topicName) {
+ return SUBSCRIPTION_PREFIX + topicName;
+ }
+}
\ No newline at end of file
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/IndexerOqmMessageReceiver.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/IndexerOqmMessageReceiver.java
index c14b4df4e59f4ce235d8eca7803de135fcea98aa..39d85831b63730b927d238d407f51cb5a555fd75 100644
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/IndexerOqmMessageReceiver.java
+++ b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/IndexerOqmMessageReceiver.java
@@ -23,7 +23,6 @@ import lombok.extern.slf4j.Slf4j;
import org.opengroup.osdu.core.auth.TokenProvider;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.http.RequestStatus;
import org.opengroup.osdu.core.gcp.oqm.model.OqmAckReplier;
import org.opengroup.osdu.core.gcp.oqm.model.OqmMessage;
import org.opengroup.osdu.core.gcp.oqm.model.OqmMessageReceiver;
@@ -57,7 +56,7 @@ public abstract class IndexerOqmMessageReceiver implements OqmMessageReceiver {
oqmAckReplier.ack();
} catch (AppException appException) {
int statusCode = appException.getError().getCode();
- if (statusCode > 199 && statusCode < 300 && statusCode != RequestStatus.INVALID_RECORD) {
+ if (statusCode > 199 && statusCode < 300) {
skipMessage(oqmMessage, dpsHeaders, oqmAckReplier, appException);
} else {
rescheduleMessage(oqmMessage, dpsHeaders, oqmAckReplier, getException(appException));
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/OqmSubscriberManager.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/OqmSubscriberManager.java
deleted file mode 100644
index acc4ecd7a9fa8fec98cf531077e993f44a95f919..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/OqmSubscriberManager.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Copyright 2020-2023 Google LLC
- * Copyright 2020-2023 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.provider.gcp.indexing.processing;
-
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
-import org.opengroup.osdu.core.gcp.oqm.driver.OqmDriver;
-import org.opengroup.osdu.core.gcp.oqm.model.*;
-import org.springframework.stereotype.Service;
-
-import javax.annotation.Nullable;
-
-@Service
-@Slf4j
-@RequiredArgsConstructor
-public class OqmSubscriberManager {
-
- private final OqmDriver driver;
-
- private OqmSubscription getOrCreateSubscriptionForTenant(TenantInfo tenantInfo, String topicName, String subscriptionName) {
- log.info("OQM: provisioning tenant {}:", tenantInfo.getDataPartitionId());
- log.info("OQM: check for topic {} existence:", topicName);
- OqmTopic topic = driver.getTopic(topicName, getDestination(tenantInfo))
- .orElse(null);
- if (topic == null) {
- log.info("OQM: check for topic {} existence: ABSENT. Skipped", topicName);
- throw new RuntimeException();
- }
-
- log.info("OQM: check for topic {} existence: PRESENT", topicName);
- OqmSubscription subscription = getSubscription(tenantInfo, topic, subscriptionName);
-
- if (subscription == null) {
- subscription = createSubscription(tenantInfo, topic, subscriptionName);
- } else {
- log.info("OQM: check for subscription {} existence: PRESENT", subscriptionName);
- }
- log.info("OQM: provisioning tenant {}: COMPLETED.", tenantInfo.getDataPartitionId());
- return subscription;
- }
-
- @Nullable
- private OqmSubscription getSubscription(TenantInfo tenantInfo, OqmTopic topic, String subscriptionName) {
- log.info("OQM: check for subscription {} existence:", subscriptionName);
- OqmSubscriptionQuery query = OqmSubscriptionQuery.builder()
- .namePrefix(subscriptionName)
- .subscriberable(true)
- .build();
- return driver
- .listSubscriptions(topic, query, getDestination(tenantInfo)).stream()
- .findAny()
- .orElse(null);
- }
-
- private OqmSubscription createSubscription(TenantInfo tenantInfo, OqmTopic topic, String subscriptionName) {
- log.info("OQM: check for subscription {} existence: ABSENT. Will create.", subscriptionName);
- OqmSubscription request = OqmSubscription.builder()
- .topic(topic)
- .name(subscriptionName)
- .build();
- return driver.createAndGetSubscription(request, getDestination(tenantInfo));
- }
-
- public void registerSubscriber(TenantInfo tenantInfo, String topicName, String subscriptionName, OqmMessageReceiver messageReceiver, OqmSubscriberThroughput throughput) {
- OqmSubscription subscriptionForTenant = getOrCreateSubscriptionForTenant(tenantInfo, topicName, subscriptionName);
- log.info("OQM: registering Subscriber for subscription {}", subscriptionName);
- OqmDestination destination = getDestination(tenantInfo);
-
- OqmSubscriber subscriber = OqmSubscriber.builder()
- .subscription(subscriptionForTenant)
- .messageReceiver(messageReceiver)
- .throughput(throughput)
- .build();
- driver.subscribe(subscriber, destination);
- log.info("OQM: provisioning subscription {}: Subscriber REGISTERED.", subscriptionName);
- }
-
- private OqmDestination getDestination(TenantInfo tenantInfo) {
- return OqmDestination.builder()
- .partitionId(tenantInfo.getDataPartitionId())
- .build();
- }
-}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/ReindexMessageReceiver.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/ReindexMessageReceiver.java
new file mode 100644
index 0000000000000000000000000000000000000000..e00c9c0e767fbe018618cfa9b59c7a988ce1a1cf
--- /dev/null
+++ b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/ReindexMessageReceiver.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2020-2023 Google LLC
+ * Copyright 2020-2023 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.provider.gcp.indexing.processing;
+
+import java.time.LocalDateTime;
+import lombok.extern.slf4j.Slf4j;
+import org.opengroup.osdu.core.auth.TokenProvider;
+import org.opengroup.osdu.core.common.model.indexer.JobStatus;
+import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
+import org.opengroup.osdu.core.gcp.oqm.model.OqmMessage;
+import org.opengroup.osdu.indexer.api.RecordIndexerApi;
+import org.opengroup.osdu.indexer.provider.gcp.indexing.scope.ThreadDpsHeaders;
+import org.springframework.http.ResponseEntity;
+
+@Slf4j
+public class ReindexMessageReceiver extends IndexerOqmMessageReceiver {
+
+ private final RecordIndexerApi recordIndexerApi;
+
+ public ReindexMessageReceiver(ThreadDpsHeaders dpsHeaders, TokenProvider tokenProvider, RecordIndexerApi recordIndexerApi) {
+ super(dpsHeaders, tokenProvider);
+ this.recordIndexerApi = recordIndexerApi;
+ }
+
+ @Override
+ protected void sendMessage(OqmMessage oqmMessage) throws Exception {
+ RecordChangedMessages indexWorkerRequestBody = getIndexWorkerRequestBody(oqmMessage);
+ log.debug("Reindex job message body: {}", indexWorkerRequestBody);
+ ResponseEntity<JobStatus> jobStatusResponse = recordIndexerApi.indexWorker(indexWorkerRequestBody);
+ log.debug("Job status: {}", jobStatusResponse);
+ }
+
+ private RecordChangedMessages getIndexWorkerRequestBody(OqmMessage request) {
+ RecordChangedMessages recordChangedMessages = new RecordChangedMessages();
+ recordChangedMessages.setMessageId(dpsHeaders.getCorrelationId());
+ recordChangedMessages.setData(request.getData());
+ recordChangedMessages.setAttributes(request.getAttributes());
+ recordChangedMessages.setPublishTime(LocalDateTime.now().toString());
+ return recordChangedMessages;
+ }
+}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/RepressorMessageReceiver.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/ReprocessorMessageReceiver.java
similarity index 86%
rename from provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/RepressorMessageReceiver.java
rename to provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/ReprocessorMessageReceiver.java
index ccfe7a214004595ffdc9b29026a095d65db33015..c5d71abf72d70f3765dda5c2999b00b0c7870969 100644
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/RepressorMessageReceiver.java
+++ b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/ReprocessorMessageReceiver.java
@@ -1,6 +1,6 @@
/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
+ * Copyright 2020-2023 Google LLC
+ * Copyright 2020-2023 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -27,13 +27,12 @@ import org.opengroup.osdu.indexer.provider.gcp.indexing.scope.ThreadDpsHeaders;
import org.springframework.http.ResponseEntity;
@Slf4j
-public class RepressorMessageReceiver extends IndexerOqmMessageReceiver {
+public class ReprocessorMessageReceiver extends IndexerOqmMessageReceiver {
private final Gson gson = new Gson();
private final ReindexApi reindexApi;
- public RepressorMessageReceiver(ThreadDpsHeaders dpsHeaders, TokenProvider tokenProvider,
- ReindexApi reindexApi) {
+ public ReprocessorMessageReceiver(ThreadDpsHeaders dpsHeaders, TokenProvider tokenProvider, ReindexApi reindexApi) {
super(dpsHeaders, tokenProvider);
this.reindexApi = reindexApi;
}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/TenantSubscriberConfiguration.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/TenantSubscriberConfiguration.java
deleted file mode 100644
index 9d3a7789d585a6575073719655c6bb2251e343a2..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/TenantSubscriberConfiguration.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.provider.gcp.indexing.processing;
-
-import java.util.Collection;
-import javax.annotation.PostConstruct;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.opengroup.osdu.core.auth.TokenProvider;
-import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
-import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmSubscriberThroughput;
-import org.opengroup.osdu.indexer.api.RecordIndexerApi;
-import org.opengroup.osdu.indexer.api.ReindexApi;
-import org.opengroup.osdu.indexer.provider.gcp.indexing.scope.ThreadDpsHeaders;
-import org.springframework.stereotype.Component;
-
-/**
- * Subscription configuration class.
- */
-@Slf4j
-@Component
-@RequiredArgsConstructor
-public class TenantSubscriberConfiguration {
-
- private static final String SUBSCRIPTION_PREFIX = "indexer-";
- private final IndexerMessagingConfigProperties properties;
- private final OqmSubscriberManager subscriberManager;
- private final ITenantFactory tenantInfoFactory;
- private final TokenProvider tokenProvider;
- private final ThreadDpsHeaders headers;
- private final RecordIndexerApi recordIndexerApi;
- private final ReindexApi reindexApi;
-
- /**
- * Tenant configurations provided by the Partition service will be used to configure subscribers. If tenants use the same message broker(The same RabbitMQ
- * instance, or the same GCP project Pub/Sub) then only one subscriber in this broker will be used.
- */
- @PostConstruct
- void postConstruct() {
- log.info("OqmSubscriberManager provisioning STARTED");
- String recordsChangedTopicName = properties.getRecordsChangedTopicName();
- String reprocessTopicName = properties.getReprocessTopicName();
- String schemaChangedTopicName = properties.getSchemaChangedTopicName();
-
- Collection<TenantInfo> tenantInfos = tenantInfoFactory.listTenantInfo();
-
- for (TenantInfo tenantInfo : tenantInfos) {
- subscriberManager.registerSubscriber(
- tenantInfo,
- recordsChangedTopicName,
- getSubscriptionName(recordsChangedTopicName),
- new RecordsChangedMessageReceiver(headers, tokenProvider, recordIndexerApi),
- OqmSubscriberThroughput.MAX
- );
- subscriberManager.registerSubscriber(
- tenantInfo,
- reprocessTopicName,
- getSubscriptionName(reprocessTopicName),
- new RepressorMessageReceiver(headers, tokenProvider, reindexApi),
- OqmSubscriberThroughput.MIN
- );
- subscriberManager.registerSubscriber(
- tenantInfo,
- schemaChangedTopicName,
- getSubscriptionName(schemaChangedTopicName),
- new SchemaChangedMessageReceiver(headers, tokenProvider, recordIndexerApi),
- OqmSubscriberThroughput.MIN
- );
- }
- log.info("OqmSubscriberManager provisioning COMPLETED");
- }
-
- private String getSubscriptionName(String topicName) {
- return SUBSCRIPTION_PREFIX + topicName;
- }
-}
diff --git a/provider/indexer-gc/src/main/resources/application-gcp.properties b/provider/indexer-gc/src/main/resources/application-gcp.properties
index 8ac15c3f94a0bb7addf3ffcd4fead8bcbd2ffa5e..2e6be18d1a00c96e97bb3567da10699f5201428e 100644
--- a/provider/indexer-gc/src/main/resources/application-gcp.properties
+++ b/provider/indexer-gc/src/main/resources/application-gcp.properties
@@ -1,3 +1,4 @@
oqmDriver=pubsub
service.token.provider=GCP
partition-auth-enabled=true
+dead-lettering-required=true
diff --git a/provider/indexer-gc/src/main/resources/application.properties b/provider/indexer-gc/src/main/resources/application.properties
index 1a2613ffe5df6a87e20724b7fcfb6dd1f027c239..7259834e06820469324fabe055f78d305118979f 100644
--- a/provider/indexer-gc/src/main/resources/application.properties
+++ b/provider/indexer-gc/src/main/resources/application.properties
@@ -20,13 +20,15 @@ kinds-redis-database=1
cron-index-cleanup-threshold-days=3
cron-empty-index-cleanup-threshold-days=7
-#indexer service config
+# Indexer service config
+propertyResolver.strategy=partition
DEFAULT_DATA_COUNTRY=US
gae-service=indexer
security.https.certificate.trust=false
storage-records-by-kind-batch-size=1000
storage-records-batch-size=20
+# External services config
REDIS_SEARCH_PORT=6379
REDIS_SEARCH_HOST=redis-cache-search
@@ -49,10 +51,11 @@ STORAGE_SCHEMA_HOST=${STORAGE_API}/schemas
SCHEMA_BASE_HOST=http://schema
SCHEMA_PATH=/api/schema-service/v1/schema
SCHEMA_HOST=${SCHEMA_BASE_HOST}${SCHEMA_PATH}
+SEARCH_HOST=${SEARCH_BASE_HOST}/api/search/v2
+# OQM config
records-changed-topic-name=records-changed
schema-changed-topic-name=schema-changed
-reprocess-topic-name=reprocess
status-changed-topic-name=indexing-progress
-
-propertyResolver.strategy=partition
\ No newline at end of file
+reprocess-topic-name=reprocess
+reindex-topic-name=reindex
\ No newline at end of file
diff --git a/provider/indexer-gc/src/test/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/RepressorMessageReceiverTest.java b/provider/indexer-gc/src/test/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/ReprocessorMessageReceiverTest.java
similarity index 92%
rename from provider/indexer-gc/src/test/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/RepressorMessageReceiverTest.java
rename to provider/indexer-gc/src/test/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/ReprocessorMessageReceiverTest.java
index be752140196c8601cd17d2271dc1b2baef4411ef..ebddfb3c23686379ca347e2699a7c00063df2d09 100644
--- a/provider/indexer-gc/src/test/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/RepressorMessageReceiverTest.java
+++ b/provider/indexer-gc/src/test/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/ReprocessorMessageReceiverTest.java
@@ -35,7 +35,7 @@ import org.opengroup.osdu.indexer.api.ReindexApi;
import org.opengroup.osdu.indexer.provider.gcp.indexing.scope.ThreadDpsHeaders;
@RunWith(Theories.class)
-public class RepressorMessageReceiverTest {
+public class ReprocessorMessageReceiverTest {
protected ThreadDpsHeaders dpsHeaders = Mockito.mock(ThreadDpsHeaders.class);
@@ -45,11 +45,11 @@ public class RepressorMessageReceiverTest {
private ReindexApi reindexApi = Mockito.mock(ReindexApi.class);
- private RepressorMessageReceiver receiver;
+ private ReprocessorMessageReceiver receiver;
@Before
public void setUp() {
- receiver = new RepressorMessageReceiver(dpsHeaders, tokenProvider, reindexApi);
+ receiver = new ReprocessorMessageReceiver(dpsHeaders, tokenProvider, reindexApi);
}
@DataPoints("VALID_EVENTS")
diff --git a/provider/indexer-gc/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java b/provider/indexer-gc/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java
index 403199cb3231d84c8b072b6588dee7a1389ab447..aab33b30c1eb11aa41b810022472ac46ba1d2329 100644
--- a/provider/indexer-gc/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java
+++ b/provider/indexer-gc/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java
@@ -90,7 +90,7 @@ public class ReindexServiceTest {
recordQueryResponse.setResults(null);
when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
- String response = sut.reindexRecords(recordReindexRequest, false);
+ String response = sut.reindexKind(recordReindexRequest, false);
Assert.assertNull(response);
} catch (Exception e) {
@@ -104,7 +104,7 @@ public class ReindexServiceTest {
recordQueryResponse.setResults(new ArrayList<>());
when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
- String response = sut.reindexRecords(recordReindexRequest, false);
+ String response = sut.reindexKind(recordReindexRequest, false);
Assert.assertNull(response);
} catch (Exception e) {
@@ -124,7 +124,7 @@ public class ReindexServiceTest {
when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
- String taskQueuePayload = sut.reindexRecords(recordReindexRequest, false);
+ String taskQueuePayload = sut.reindexKind(recordReindexRequest, false);
Assert.assertEquals("{\"kind\":\"tenant:test:test:1.0.0\",\"cursor\":\"100\"}", taskQueuePayload);
} catch (Exception e) {
@@ -140,7 +140,7 @@ public class ReindexServiceTest {
recordQueryResponse.setResults(results);
when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
- String taskQueuePayload = sut.reindexRecords(recordReindexRequest, false);
+ String taskQueuePayload = sut.reindexKind(recordReindexRequest, false);
Assert.assertEquals(String.format("{\"data\":\"[{\\\"id\\\":\\\"test1\\\",\\\"kind\\\":\\\"tenant:test:test:1.0.0\\\",\\\"op\\\":\\\"create\\\"}]\",\"attributes\":{\"correlation-id\":\"%s\"}}", correlationId), taskQueuePayload);
} catch (Exception e) {
diff --git a/provider/indexer-ibm/pom.xml b/provider/indexer-ibm/pom.xml
index 82d58bd58e8364de9c639bb2ae4b24433ae9b1d1..4ae8be12c5f7d879d936634dbcae0d78425925e5 100644
--- a/provider/indexer-ibm/pom.xml
+++ b/provider/indexer-ibm/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-service</artifactId>
- <version>0.21.0-SNAPSHOT</version>
+ <version>0.22.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
@@ -57,7 +57,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-core</artifactId>
- <version>0.21.0-SNAPSHOT</version>
+ <version>0.22.0-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>io.netty</groupId>
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/RequestInfoImpl.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/RequestInfoImpl.java
index 5f52f26ee0f01d225c64a97ff8bd608dcc2d2b8b..3e7bd6c60ba3395a3cc8277751972d142b299e18 100644
--- a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/RequestInfoImpl.java
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/RequestInfoImpl.java
@@ -115,7 +115,7 @@ public class RequestInfoImpl implements IRequestInfo {
}
return "Bearer " + authHeader;
} else {
- return "Bearer " + this.serviceAccountJwtClient.getIdToken(tenantInfo.getName());
+ return this.serviceAccountJwtClient.getIdToken(tenantInfo.getName());
}
}
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/ServiceAccountJwtClientImpl.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/ServiceAccountJwtClientImpl.java
index 18d3b7f5011408cdf49b7e9e9553be6a26340aa4..6a6fbfc4662bf23121463dbdb654d3f5fe210aee 100644
--- a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/ServiceAccountJwtClientImpl.java
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/util/ServiceAccountJwtClientImpl.java
@@ -4,6 +4,7 @@ package org.opengroup.osdu.indexer.ibm.util;
import javax.inject.Inject;
+import com.google.common.base.Strings;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
@@ -18,6 +19,7 @@ import org.springframework.web.context.annotation.RequestScope;
@Component
@RequestScope
public class ServiceAccountJwtClientImpl implements IServiceAccountJwtClient {
+ private final String BEARER = "Bearer";
@Inject
private ITenantFactory tenantInfoServiceProvider;
@@ -65,6 +67,9 @@ public class ServiceAccountJwtClientImpl implements IServiceAccountJwtClient {
throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Persistence error", "Error generating token", e);
}
+ if(!Strings.isNullOrEmpty(ACCESS_TOKEN) && !ACCESS_TOKEN.startsWith(BEARER)) {
+ ACCESS_TOKEN = BEARER + " " + ACCESS_TOKEN;
+ }
return ACCESS_TOKEN;
}
diff --git a/provider/indexer-ibm/src/main/resources/application.properties b/provider/indexer-ibm/src/main/resources/application.properties
index aaf4ceb6cf42aaa7a1fcaa8d72e71830700d4b27..e22d699633b9017f30a20d39d6862d214f6ef89c 100644
--- a/provider/indexer-ibm/src/main/resources/application.properties
+++ b/provider/indexer-ibm/src/main/resources/application.properties
@@ -27,6 +27,7 @@ CRON_INDEX_CLEANUP_THRESHOLD_DAYS=3
CRON_EMPTY_INDEX_CLEANUP_THRESHOLD_DAYS=7
SCHEMA_HOST=${HOST}/api/schema-service/v1/schema
+SEARCH_HOST=${search_service_url}/api/search/v2
storage_service_url=http://localhost:8082
#storage_service_url=https://os-storage-ibm-osdu-r2.osduadev-a1c3eaf78a86806e299f5f3f207556f0-0000.us-south.containers.appdomain.cloud
diff --git a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/ReindexServiceTest.java b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/ReindexServiceTest.java
index acb116a762ba85ce6373ad0e8fa3814de66a4937..2067d75f0a8b4ef7bc5c7cb5351d0ef443e09282 100644
--- a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/ReindexServiceTest.java
+++ b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/ReindexServiceTest.java
@@ -84,7 +84,7 @@ public class ReindexServiceTest {
recordQueryResponse.setResults(null);
when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
- String response = sut.reindexRecords(recordReindexRequest, false);
+ String response = sut.reindexKind(recordReindexRequest, false);
Assert.assertNull(response);
} catch (Exception e) {
@@ -98,7 +98,7 @@ public class ReindexServiceTest {
recordQueryResponse.setResults(new ArrayList<>());
when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
- String response = sut.reindexRecords(recordReindexRequest, false);
+ String response = sut.reindexKind(recordReindexRequest, false);
Assert.assertNull(response);
} catch (Exception e) {
@@ -118,7 +118,7 @@ public class ReindexServiceTest {
when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
- String taskQueuePayload = sut.reindexRecords(recordReindexRequest, false);
+ String taskQueuePayload = sut.reindexKind(recordReindexRequest, false);
Assert.assertEquals("{\"kind\":\"tenant:test:test:1.0.0\",\"cursor\":\"100\"}", taskQueuePayload);
} catch (Exception e) {
@@ -134,7 +134,7 @@ public class ReindexServiceTest {
recordQueryResponse.setResults(results);
when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
- String taskQueuePayload = sut.reindexRecords(recordReindexRequest, false);
+ String taskQueuePayload = sut.reindexKind(recordReindexRequest, false);
Assert.assertEquals(String.format("{\"data\":\"[{\\\"id\\\":\\\"test1\\\",\\\"kind\\\":\\\"tenant:test:test:1.0.0\\\",\\\"op\\\":\\\"create\\\"}]\",\"attributes\":{\"slb-correlation-id\":\"%s\"}}", correlationId), taskQueuePayload);
} catch (Exception e) {
diff --git a/testing/indexer-test-aws/pom.xml b/testing/indexer-test-aws/pom.xml
index 4873e4dd208a357a1edd026e1c49b299ca5dded0..08d214bd7005e05155ba3cad35dd3351b8f3103e 100644
--- a/testing/indexer-test-aws/pom.xml
+++ b/testing/indexer-test-aws/pom.xml
@@ -21,13 +21,13 @@
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>indexer-test</artifactId>
- <version>0.21.0-SNAPSHOT</version>
+ <version>0.22.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-aws</artifactId>
- <version>0.21.0-SNAPSHOT</version>
+ <version>0.22.0-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
@@ -44,14 +44,14 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
- <version>0.21.0-SNAPSHOT</version>
+ <version>0.22.0-SNAPSHOT</version>
</dependency>
<!-- AWS specific packages -->
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
<artifactId>os-core-lib-aws</artifactId>
- <version>0.19.0-rc3</version>
+ <version>0.21.0</version>
</dependency>
<!-- Testing -->
diff --git a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
index 37dd8f26ea74440a4c9affde1844a74d4168ec0b..62d0edc5586490e71850ac197614560e367e81c1 100644
--- a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
+++ b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -75,6 +75,16 @@ public class Steps extends SchemaServiceRecordSteps {
super.the_schema_is_created_with_the_following_kind(dataTable);
}
+ @Then("^I set starting stateful scenarios$")
+ public void i_set_starting_stateful_scenarios() throws Throwable {
+ super.i_set_scenarios_as_stateful(true);
+ }
+
+ @Then("^I set ending stateful scenarios$")
+ public void i_set_ending_stateful_scenarios() throws Throwable {
+ super.i_set_scenarios_as_stateful(false);
+ }
+
@When("^I ingest records with the \"(.*?)\" with \"(.*?)\" for a given \"(.*?)\"$")
public void i_ingest_records_with_the_for_a_given(String record, String dataGroup, String kind) {
super.i_ingest_records_with_the_for_a_given(record, dataGroup, kind);
@@ -115,6 +125,16 @@ public class Steps extends SchemaServiceRecordSteps {
super.iShouldBeAbleToSearchRecordByTagKeyAndTagValue(index, tagKey, tagValue, expectedNumber);
}
+ @Then("^I clean up the index of the extended kinds \"([^\"]*)\" in the Elastic Search$")
+ public void iShouldCleanupIndicesOfExtendedKinds(String extendedKinds) throws Throwable {
+ super.iShouldCleanupIndicesOfExtendedKinds(extendedKinds);
+ }
+
+ @Then("^I should be able to search (\\d+) record with index \"([^\"]*)\" by extended data field \"([^\"]*)\" and value \"([^\"]*)\"$")
+ public void iShouldBeAbleToSearchRecordByFieldAndFieldValue(int expectedNumber, String index, String fieldKey, String fieldValue) throws Throwable {
+ super.iShouldBeAbleToSearchRecordByFieldAndFieldValue(index, fieldKey, fieldValue, expectedNumber);
+ }
+
@Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by bounding box query with points \\((-?\\d+), (-?\\d+)\\) and \\((-?\\d+), (-?\\d+)\\) on field \"(.*?)\"$")
public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery (
int expectedCount, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field) throws Throwable {
@@ -141,4 +161,4 @@ public class Steps extends SchemaServiceRecordSteps {
throws Throwable {
super.i_should_get_object_in_search_response_without_hints_in_schema(objectInnerField ,index, recordFile, acl, kind);
}
-}
\ No newline at end of file
+}
diff --git a/testing/indexer-test-aws/src/test/resources/cucumber.properties b/testing/indexer-test-aws/src/test/resources/cucumber.properties
new file mode 100644
index 0000000000000000000000000000000000000000..4b0c14c43ba6053662d611f07881939ef39e4110
--- /dev/null
+++ b/testing/indexer-test-aws/src/test/resources/cucumber.properties
@@ -0,0 +1,2 @@
+# tag indexer-extended disabled by default
+cucumber.options=--tags ~@indexer-extended
diff --git a/testing/indexer-test-azure/pom.xml b/testing/indexer-test-azure/pom.xml
index c636dad6ad94d6143ef73dbef89a7bcc0e753fb2..be6efd9ae228f9eacfccc671caef7e673e445afb 100644
--- a/testing/indexer-test-azure/pom.xml
+++ b/testing/indexer-test-azure/pom.xml
@@ -21,13 +21,13 @@
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>indexer-test</artifactId>
- <version>0.21.0-SNAPSHOT</version>
+ <version>0.22.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-azure</artifactId>
- <version>0.21.0-SNAPSHOT</version>
+ <version>0.22.0-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
@@ -46,7 +46,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
- <version>0.21.0-SNAPSHOT</version>
+ <version>0.22.0-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
diff --git a/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
index ae5e82f842d377236fccbedf449f30d25167f811..3cad1b4b8634a1786e9c620a1e18d7d9a49d29ee 100644
--- a/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
+++ b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -43,6 +43,16 @@ public class Steps extends SchemaServiceRecordSteps {
super.the_schema_is_created_with_the_following_kind(dataTable);
}
+ @Then("^I set starting stateful scenarios$")
+ public void i_set_starting_stateful_scenarios() throws Throwable {
+ super.i_set_scenarios_as_stateful(true);
+ }
+
+ @Then("^I set ending stateful scenarios$")
+ public void i_set_ending_stateful_scenarios() throws Throwable {
+ super.i_set_scenarios_as_stateful(false);
+ }
+
@When("^I ingest records with the \"(.*?)\" with \"(.*?)\" for a given \"(.*?)\"$")
public void i_ingest_records_with_the_for_a_given(String record, String dataGroup, String kind) {
super.i_ingest_records_with_the_for_a_given(record, dataGroup, kind);
@@ -83,6 +93,16 @@ public class Steps extends SchemaServiceRecordSteps {
super.iShouldBeAbleToSearchRecordByTagKeyAndTagValue(index, tagKey, tagValue, expectedNumber);
}
+ @Then("^I clean up the index of the extended kinds \"([^\"]*)\" in the Elastic Search$")
+ public void iShouldCleanupIndicesOfExtendedKinds(String extendedKinds) throws Throwable {
+ super.iShouldCleanupIndicesOfExtendedKinds(extendedKinds);
+ }
+
+ @Then("^I should be able to search (\\d+) record with index \"([^\"]*)\" by extended data field \"([^\"]*)\" and value \"([^\"]*)\"$")
+ public void iShouldBeAbleToSearchRecordByFieldAndFieldValue(int expectedNumber, String index, String fieldKey, String fieldValue) throws Throwable {
+ super.iShouldBeAbleToSearchRecordByFieldAndFieldValue(index, fieldKey, fieldValue, expectedNumber);
+ }
+
@Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by bounding box query with points \\((-?\\d+), (-?\\d+)\\) and \\((-?\\d+), (-?\\d+)\\) on field \"([^\"]*)\"$")
public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery(
int expectedCount, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field) throws Throwable {
@@ -109,4 +129,4 @@ public class Steps extends SchemaServiceRecordSteps {
throws Throwable {
super.i_should_get_object_in_search_response_without_hints_in_schema(objectInnerField ,index, recordFile, acl, kind);
}
-}
\ No newline at end of file
+}
diff --git a/testing/indexer-test-azure/src/test/resources/cucumber.properties b/testing/indexer-test-azure/src/test/resources/cucumber.properties
new file mode 100644
index 0000000000000000000000000000000000000000..2ca785a2f55f5f35f2c0f4b755c2a6cd072669c0
--- /dev/null
+++ b/testing/indexer-test-azure/src/test/resources/cucumber.properties
@@ -0,0 +1,2 @@
+# tag indexer-extended enabled by default
+cucumber.options=--tags '~@* and @indexer-extended'
diff --git a/testing/indexer-test-anthos/pom.xml b/testing/indexer-test-baremetal/pom.xml
similarity index 95%
rename from testing/indexer-test-anthos/pom.xml
rename to testing/indexer-test-baremetal/pom.xml
index c1c1191aa79fadaff4e64c4c9deb6e6a91fbce22..de7cc1a6dc31ff4d6ddef8e1538bd3b9162fadef 100644
--- a/testing/indexer-test-anthos/pom.xml
+++ b/testing/indexer-test-baremetal/pom.xml
@@ -20,11 +20,11 @@
<parent>
<artifactId>indexer-test</artifactId>
<groupId>org.opengroup.osdu</groupId>
- <version>0.21.0-SNAPSHOT</version>
+ <version>0.22.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <artifactId>indexer-test-anthos</artifactId>
+ <artifactId>indexer-test-baremetal</artifactId>
<properties>
<maven.compiler.target>1.8</maven.compiler.target>
@@ -36,7 +36,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
- <version>0.21.0-SNAPSHOT</version>
+ <version>0.22.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.nimbusds</groupId>
diff --git a/testing/indexer-test-anthos/src/test/java/org/opengroup/osdu/step_definitions/info/RunTest.java b/testing/indexer-test-baremetal/src/test/java/org/opengroup/osdu/step_definitions/info/RunTest.java
similarity index 100%
rename from testing/indexer-test-anthos/src/test/java/org/opengroup/osdu/step_definitions/info/RunTest.java
rename to testing/indexer-test-baremetal/src/test/java/org/opengroup/osdu/step_definitions/info/RunTest.java
diff --git a/testing/indexer-test-anthos/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java b/testing/indexer-test-baremetal/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java
similarity index 100%
rename from testing/indexer-test-anthos/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java
rename to testing/indexer-test-baremetal/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java
diff --git a/testing/indexer-test-anthos/src/test/java/org/opengroup/osdu/step_definitions/record/RunTest.java b/testing/indexer-test-baremetal/src/test/java/org/opengroup/osdu/step_definitions/record/RunTest.java
similarity index 100%
rename from testing/indexer-test-anthos/src/test/java/org/opengroup/osdu/step_definitions/record/RunTest.java
rename to testing/indexer-test-baremetal/src/test/java/org/opengroup/osdu/step_definitions/record/RunTest.java
diff --git a/testing/indexer-test-anthos/src/test/java/org/opengroup/osdu/step_definitions/record/Steps.java b/testing/indexer-test-baremetal/src/test/java/org/opengroup/osdu/step_definitions/record/Steps.java
similarity index 85%
rename from testing/indexer-test-anthos/src/test/java/org/opengroup/osdu/step_definitions/record/Steps.java
rename to testing/indexer-test-baremetal/src/test/java/org/opengroup/osdu/step_definitions/record/Steps.java
index 40620fbf4a626a1eb279f28fae937a55ffdc8469..c88b45c901145e00567c752fea0770a31177a66e 100644
--- a/testing/indexer-test-anthos/src/test/java/org/opengroup/osdu/step_definitions/record/Steps.java
+++ b/testing/indexer-test-baremetal/src/test/java/org/opengroup/osdu/step_definitions/record/Steps.java
@@ -27,6 +27,7 @@ import lombok.extern.java.Log;
import org.opengroup.osdu.common.SchemaServiceRecordSteps;
import org.opengroup.osdu.util.AnthosHTTPClient;
import org.opengroup.osdu.util.ElasticUtils;
+import org.opengroup.osdu.util.conf.AnthosConfig;
@Log
public class Steps extends SchemaServiceRecordSteps {
@@ -37,6 +38,7 @@ public class Steps extends SchemaServiceRecordSteps {
@Before
public void before(Scenario scenario) {
+ AnthosConfig.updateEntitlementsDomainVariable();
this.scenario = scenario;
this.httpClient = new AnthosHTTPClient();
}
@@ -46,6 +48,16 @@ public class Steps extends SchemaServiceRecordSteps {
super.the_schema_is_created_with_the_following_kind(dataTable);
}
+ @Then("^I set starting stateful scenarios$")
+ public void i_set_starting_stateful_scenarios() throws Throwable {
+ super.i_set_scenarios_as_stateful(true);
+ }
+
+ @Then("^I set ending stateful scenarios$")
+ public void i_set_ending_stateful_scenarios() throws Throwable {
+ super.i_set_scenarios_as_stateful(false);
+ }
+
@When("^I ingest records with the \"(.*?)\" with \"(.*?)\" for a given \"(.*?)\"$")
public void i_ingest_records_with_the_for_a_given(String record, String dataGroup, String kind) {
super.i_ingest_records_with_the_for_a_given(record, dataGroup, kind);
@@ -88,6 +100,16 @@ public class Steps extends SchemaServiceRecordSteps {
super.iShouldBeAbleToSearchRecordByTagKeyAndTagValue(index, tagKey, tagValue, expectedNumber);
}
+ @Then("^I clean up the index of the extended kinds \"([^\"]*)\" in the Elastic Search$")
+ public void iShouldCleanupIndicesOfExtendedKinds(String extendedKinds) throws Throwable {
+ super.iShouldCleanupIndicesOfExtendedKinds(extendedKinds);
+ }
+
+ @Then("^I should be able to search (\\d+) record with index \"([^\"]*)\" by extended data field \"([^\"]*)\" and value \"([^\"]*)\"$")
+ public void iShouldBeAbleToSearchRecordByFieldAndFieldValue(int expectedNumber, String index, String fieldKey, String fieldValue) throws Throwable {
+ super.iShouldBeAbleToSearchRecordByFieldAndFieldValue(index, fieldKey, fieldValue, expectedNumber);
+ }
+
@Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by bounding box query with points \\((-?\\d+), (-?\\d+)\\) and \\((-?\\d+), (-?\\d+)\\) on field \"([^\"]*)\"$")
public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery(
int expectedCount, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field)
@@ -121,4 +143,4 @@ public class Steps extends SchemaServiceRecordSteps {
String actualName = generateActualName(index, null);
super.i_should_get_object_in_search_response_without_hints_in_schema(objectInnerField, actualName, recordFile, acl, kind);
}
-}
\ No newline at end of file
+}
diff --git a/testing/indexer-test-anthos/src/test/java/org/opengroup/osdu/util/AnthosHTTPClient.java b/testing/indexer-test-baremetal/src/test/java/org/opengroup/osdu/util/AnthosHTTPClient.java
similarity index 100%
rename from testing/indexer-test-anthos/src/test/java/org/opengroup/osdu/util/AnthosHTTPClient.java
rename to testing/indexer-test-baremetal/src/test/java/org/opengroup/osdu/util/AnthosHTTPClient.java
diff --git a/testing/indexer-test-anthos/src/test/java/org/opengroup/osdu/util/OpenIDTokenProvider.java b/testing/indexer-test-baremetal/src/test/java/org/opengroup/osdu/util/OpenIDTokenProvider.java
similarity index 100%
rename from testing/indexer-test-anthos/src/test/java/org/opengroup/osdu/util/OpenIDTokenProvider.java
rename to testing/indexer-test-baremetal/src/test/java/org/opengroup/osdu/util/OpenIDTokenProvider.java
diff --git a/testing/indexer-test-baremetal/src/test/java/org/opengroup/osdu/util/conf/AnthosConfig.java b/testing/indexer-test-baremetal/src/test/java/org/opengroup/osdu/util/conf/AnthosConfig.java
new file mode 100644
index 0000000000000000000000000000000000000000..b7e332b5c7354f56d141f6880296c4cf55e88c7e
--- /dev/null
+++ b/testing/indexer-test-baremetal/src/test/java/org/opengroup/osdu/util/conf/AnthosConfig.java
@@ -0,0 +1,15 @@
+package org.opengroup.osdu.util.conf;
+
+import java.util.Optional;
+
+public class AnthosConfig {
+
+ private static final String GROUP_ID_VARIABLE = "GROUP_ID";
+ private static final String ENTITLEMENTS_DOMAIN_VARIABLE = "ENTITLEMENTS_DOMAIN";
+
+ public static void updateEntitlementsDomainVariable() {
+ String groupId = Optional.ofNullable(System.getProperty(GROUP_ID_VARIABLE, System.getenv(GROUP_ID_VARIABLE)))
+ .orElse("group");
+ System.setProperty(ENTITLEMENTS_DOMAIN_VARIABLE, groupId);
+ }
+}
diff --git a/testing/indexer-test-anthos/src/test/java/org/opengroup/osdu/util/conf/OpenIDProviderConfig.java b/testing/indexer-test-baremetal/src/test/java/org/opengroup/osdu/util/conf/OpenIDProviderConfig.java
similarity index 100%
rename from testing/indexer-test-anthos/src/test/java/org/opengroup/osdu/util/conf/OpenIDProviderConfig.java
rename to testing/indexer-test-baremetal/src/test/java/org/opengroup/osdu/util/conf/OpenIDProviderConfig.java
diff --git a/testing/indexer-test-baremetal/src/test/resources/cucumber.properties b/testing/indexer-test-baremetal/src/test/resources/cucumber.properties
new file mode 100644
index 0000000000000000000000000000000000000000..6884be422aa64d5c1c7a2dfb832c803f77371e43
--- /dev/null
+++ b/testing/indexer-test-baremetal/src/test/resources/cucumber.properties
@@ -0,0 +1,18 @@
+#
+# Copyright 2020-2023 Google LLC
+# Copyright 2020-2023 EPAM Systems, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# tag indexer-extended disabled by default
+cucumber.options=--tags ~@indexer-extended
diff --git a/testing/indexer-test-anthos/src/test/java/resources/logback-test.xml b/testing/indexer-test-baremetal/src/test/resources/logback-test.xml
similarity index 100%
rename from testing/indexer-test-anthos/src/test/java/resources/logback-test.xml
rename to testing/indexer-test-baremetal/src/test/resources/logback-test.xml
diff --git a/testing/indexer-test-core/pom.xml b/testing/indexer-test-core/pom.xml
index c5e856334d3d9cb01b22885714b3841db7668a3e..da0787021778e02c2ce55dbf9b90acd9dd29b06f 100644
--- a/testing/indexer-test-core/pom.xml
+++ b/testing/indexer-test-core/pom.xml
@@ -5,13 +5,13 @@
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>indexer-test</artifactId>
- <version>0.21.0-SNAPSHOT</version>
+ <version>0.22.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
- <version>0.21.0-SNAPSHOT</version>
+ <version>0.22.0-SNAPSHOT</version>
<properties>
<maven.compiler.target>1.8</maven.compiler.target>
@@ -85,7 +85,7 @@
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
- <version>1.18.2</version>
+ <version>1.18.26</version>
<scope>provided</scope>
</dependency>
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
index ebfaba6b5ef99eaf3a5429799057d459172c5063..da0676cf610850a43b80e0f31cc6e465525b2de1 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/RecordSteps.java
@@ -21,6 +21,7 @@ import org.springframework.util.CollectionUtils;
import javax.ws.rs.HttpMethod;
import java.io.IOException;
import java.lang.reflect.Type;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -101,6 +102,14 @@ public class RecordSteps extends TestsBase {
String createTime = java.time.Instant.now().toString();
for (Map<String, Object> testRecord : records) {
+ if(testRecord.containsKey("data")) {
+ Map<String, Object> data = (Map<String, Object>)testRecord.get("data");
+ if(data != null && data.size() > 0) {
+ data = replaceValues(data, timeStamp);
+ testRecord.put("data", data);
+ }
+ }
+
testRecord.put("kind", actualKind);
testRecord.put("id", generateRecordId(testRecord));
testRecord.put("legal", generateLegalTag());
@@ -184,6 +193,22 @@ public class RecordSteps extends TestsBase {
assertEquals(expectedNumber, actualNumberOfRecords);
}
+ public void iShouldCleanupIndicesOfExtendedKinds(String extendedKinds) throws Throwable {
+ String[] kinds = extendedKinds.split(",");
+ for(String kind : kinds) {
+ String actualKind = this.generateActualName(kind.trim(), timeStamp);
+ TestIndex testIndex = this.getInputIndexMap().get(actualKind);
+ testIndex.cleanupIndex(actualKind);
+ }
+ }
+
+ public void iShouldBeAbleToSearchRecordByFieldAndFieldValue(String index, String fieldKey, String fieldValue, int expectedNumber) throws Throwable {
+ TimeUnit.SECONDS.sleep(60);
+ index = generateActualName(index, timeStamp);
+ long actualNumberOfRecords = elasticUtils.fetchRecordsByFieldAndFieldValue(index, fieldKey, fieldValue);
+ assertEquals(expectedNumber, actualNumberOfRecords);
+ }
+
public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery (
int expectedNumber, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field) throws Throwable {
index = generateActualName(index, timeStamp);
@@ -232,6 +257,46 @@ public class RecordSteps extends TestsBase {
testIndex.addIndex();
}
+ private Map<String, Object> replaceValues(Map<String, Object> data, String timeStamp) {
+ for(String key : data.keySet()) {
+ Object value = data.get(key);
+ Object replacedValue = replaceValue(value, timeStamp);
+ data.put(key, replacedValue);
+ }
+ return data;
+ }
+
+ private List<Object> replaceValues(List<Object> values, String timeStamp) {
+ List<Object> replacedValues = new ArrayList<>();
+ for(Object value : values) {
+ Object replacedValue = replaceValue(value, timeStamp);
+ replacedValues.add(replacedValue);
+ }
+
+ return replacedValues;
+ }
+
+ private Object replaceValue(Object value, String timeStamp) {
+ Object replacedValue = value;
+
+ if(value instanceof String) {
+ String rawValue = (String) value;
+ for (Map.Entry<String, String> tenant : tenantMap.entrySet()) {
+ rawValue = rawValue.replaceAll(tenant.getKey() + ":", tenant.getValue() + ":");
+ }
+ replacedValue = rawValue.replaceAll("<timestamp>", timeStamp);
+ }
+ else if(value instanceof List) {
+ replacedValue = replaceValues((List)value, timeStamp);
+ }
+ else if(value instanceof Map) {
+ replacedValue = replaceValues((Map<String, Object>) value, timeStamp);
+ }
+
+ return replacedValue;
+ }
+
+
private long createIndex(String index) throws InterruptedException, IOException {
long numOfIndexedDocuments = 0;
int iterator;
@@ -320,4 +385,4 @@ public class RecordSteps extends TestsBase {
shutDownHookAdded = true;
}
}
-}
\ No newline at end of file
+}
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/SchemaServiceRecordSteps.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/SchemaServiceRecordSteps.java
index af48584f1b8e315f0435c65d78442913b8cbfd6f..53e19fb874a5c27f07e30af25fe85bb9c1172aa5 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/SchemaServiceRecordSteps.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/SchemaServiceRecordSteps.java
@@ -10,15 +10,22 @@ import java.util.List;
import java.util.Map;
public class SchemaServiceRecordSteps extends RecordSteps {
+ private static boolean runStatefulScenario = false;
public SchemaServiceRecordSteps(HTTPClient httpClient, ElasticUtils elasticUtils) {
super(httpClient, elasticUtils);
}
public void the_schema_is_created_with_the_following_kind(DataTable dataTable) {
- List<Setup> inputList = dataTable.asList(Setup.class);
- inputList.forEach(this::setup);
- super.addShutDownHook();
+ if(!SchemaServiceRecordSteps.runStatefulScenario) {
+ List<Setup> inputList = dataTable.asList(Setup.class);
+ inputList.forEach(this::setup);
+ super.addShutDownHook();
+ }
+ }
+
+ public void i_set_scenarios_as_stateful(boolean stateful) throws Throwable {
+ SchemaServiceRecordSteps.runStatefulScenario = stateful;
}
private void setup(Setup input) {
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
index b37e7daa324d2f73d3c85598dcb64a5317abaf5b..de2b99f64cf6857596062d614281dfcd345321a0 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
@@ -244,6 +244,24 @@ public class ElasticUtils {
}
}
+ public long fetchRecordsByFieldAndFieldValue(String index, String fieldKey, String fieldValue) throws IOException {
+ try {
+ try (RestHighLevelClient client = this.createClient(username, password, host)) {
+ SearchRequest request = new SearchRequest(index);
+ if(!Strings.isNullOrEmpty(fieldKey)) {
+ SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
+ sourceBuilder.query(boolQuery().must(matchQuery(fieldKey, fieldValue)));
+ request.source(sourceBuilder);
+ }
+ SearchResponse searchResponse = client.search(request, RequestOptions.DEFAULT);
+ return searchResponse.getHits().getTotalHits().value;
+ }
+ } catch (ElasticsearchStatusException e) {
+ log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage()));
+ return -1;
+ }
+ }
+
public List<Map<String, Object>> fetchRecordsByAttribute(String index, String attributeKey, String attributeValue) throws IOException {
List<Map<String, Object>> out = new ArrayList<>();
try {
@@ -499,4 +517,4 @@ public class ElasticUtils {
return false;
}
-}
\ No newline at end of file
+}
diff --git a/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature b/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
index d11cd33d5bf4076f0881a0966f8e25d573394644..556a969a6d9aca64f28bf7d4aa544e3bf7c034bb 100644
--- a/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
+++ b/testing/indexer-test-core/src/main/resources/features/indexrecord/indexRecord-schema-service.feature
@@ -13,7 +13,45 @@ Feature: Indexing of the documents
| tenant1:indexer:test-update-data--Integration:1.0.1 | tenant1-indexer-test-update-data--integration-1.0.1 | index_update_records_kind_v1 |
| tenant1:indexer:test-update-data--Integration:2.0.1 | tenant1-indexer-test-update-data--integration-2.0.1 | index_update_records_kind_v2 |
| tenant1:indexer:virtual-properties-Integration:1.0.0 | tenant1-indexer-virtual-properties-integration-1.0.0 | index_record_virtual_properties |
- | tenant1:indexer:decimation-Integration:1.0.0 | tenant1-indexer-decimation-integration-1.0.0 | index_record_seismic_survey |
+ | tenant1:indexer:decimation-Integration:1.0.0 | tenant1-indexer-decimation-integration-1.0.0 | index_record_seismic_survey |
+ | osdu:wks:reference-data--IndexPropertyPathConfiguration:1.0.0 | osdu-wks-reference-data--indexpropertypathconfiguration-1.0.0 | osdu_wks_IndexPropertyPathConfiguration_v1 |
+ | test:indexer:index-property--Wellbore:1.0.0 | test-indexer-index-property--wellbore-1.0.0 | index-property-wellbore_v1 |
+ | test:indexer:index-property--WellLog:1.0.0 | test-indexer-index-property--welllog-1.0.0 | index-property-welllog_v1 |
+
+ @indexer-extended
+ Scenario Outline: Prepare the index property configuration records and clean up index of the extended kinds in the Elastic Search
+ When I ingest records with the <recordFile> with <acl> for a given <kind>
+ Then I should get the <number> documents for the <index> in the Elastic Search
+ Then I clean up the index of the extended kinds <extendedKinds> in the Elastic Search
+ Then I set starting stateful scenarios
+
+ Examples:
+ | kind | recordFile | number | index | acl | extendedKinds |
+ | "osdu:wks:reference-data--IndexPropertyPathConfiguration:1.0.0" | "osdu_wks_IndexPropertyPathConfiguration_v1" | 2 | "osdu-wks-reference-data--indexpropertypathconfiguration-1.0.0" | "data.default.viewers@tenant1" | "test:indexer:index-property--Wellbore:1.0.0,test:indexer:index-property--WellLog:1.0.0" |
+
+ @indexer-extended
+ Scenario Outline: Ingest the records of the extended kinds, Index in the Elastic Search and Search string field
+ When I ingest records with the <recordFile> with <acl> for a given <kind>
+ Then I should be able to search <number> record with index <index> by extended data field <field> and value <value>
+
+ Examples:
+ | kind | recordFile | number | index | acl | field | value |
+ | "test:indexer:index-property--Wellbore:1.0.0" | "index-property-wellbore_v1" | 1 | "test-indexer-index-property--wellbore-1.0.0" | "data.default.viewers@tenant1" | "data.WellUWI" | "123454321" |
+ | "test:indexer:index-property--WellLog:1.0.0" | "index-property-welllog_v1" | 1 | "test-indexer-index-property--welllog-1.0.0" | "data.default.viewers@tenant1" | "data.WellboreName" | "Facility_123" |
+
+ @indexer-extended
+ Scenario Outline: Ingest the records of the extended kinds, Index in the Elastic Search and Search spatial field
+ When I ingest records with the <recordFile> with <acl> for a given <kind>
+ Then I should be able search <number> documents for the <index> by bounding box query with points (<top_left_latitude>, <top_left_longitude>) and (<bottom_right_latitude>, <bottom_right_longitude>) on field <field>
+
+ Examples:
+ | kind | recordFile | number | index | acl | field | top_left_latitude | top_left_longitude | bottom_right_latitude | bottom_right_longitude |
+ | "test:indexer:index-property--Wellbore:1.0.0" | "index-property-wellbore_v1" | 1 | "test-indexer-index-property--wellbore-1.0.0" | "data.default.viewers@tenant1" | "data.Location" | 30 | -96 | 29 | -95 |
+ | "test:indexer:index-property--WellLog:1.0.0" | "index-property-welllog_v1" | 1 | "test-indexer-index-property--welllog-1.0.0" | "data.default.viewers@tenant1" | "data.SpatialLocation" | 30 | -96 | 29 | -95 |
+
+ @indexer-extended
+ Scenario: End Stateful Scenarios
+ Then I set ending stateful scenarios
Scenario Outline: Ingest the record and Index in the Elastic Search
When I ingest records with the <recordFile> with <acl> for a given <kind>
diff --git a/testing/indexer-test-core/src/main/resources/testData/index-property-wellbore_v1.json b/testing/indexer-test-core/src/main/resources/testData/index-property-wellbore_v1.json
new file mode 100644
index 0000000000000000000000000000000000000000..469a0d22c7637a8fd867bbf2ed9aad733b87b839
--- /dev/null
+++ b/testing/indexer-test-core/src/main/resources/testData/index-property-wellbore_v1.json
@@ -0,0 +1,16 @@
+[{
+ "id": "tenant1:index-property--Wellbore:testIngest1",
+ "data": {
+ "Location": {
+ "latitude": 29.170640,
+ "longitude": -95.002875
+ },
+ "FacilityName": "Facility_123",
+ "NameAliases": [{
+ "AliasName": "123454321",
+ "AliasNameTypeID": "tenant1:reference-data--AliasNameType:UniqueIdentifier:"
+ }
+ ]
+ }
+ }
+]
diff --git a/testing/indexer-test-core/src/main/resources/testData/index-property-wellbore_v1.schema.json b/testing/indexer-test-core/src/main/resources/testData/index-property-wellbore_v1.schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..93d616f44e393c27e176724c2a97c80028f613bb
--- /dev/null
+++ b/testing/indexer-test-core/src/main/resources/testData/index-property-wellbore_v1.schema.json
@@ -0,0 +1,118 @@
+{
+ "schemaInfo": {
+ "schemaIdentity": {
+ "authority": "test",
+ "source": "indexer",
+ "entityType": "index-property--Wellbore",
+ "schemaVersionMajor": 1,
+ "schemaVersionMinor": 0,
+ "schemaVersionPatch": 0,
+ "id": "test:indexer:index-property--Wellbore:1.0.0"
+ },
+ "status": "DEVELOPMENT"
+ },
+ "schema": {
+ "definitions": {
+ "opendes:wks:core_dl_geopoint:1.0.0": {
+ "description": "A 2D point location in latitude and longitude referenced to WGS 84 if not specified otherwise.",
+ "properties": {
+ "latitude": {
+ "description": "The latitude value in degrees of arc (dega). Value range [-90, 90].",
+ "maximum": 90,
+ "minimum": -90,
+ "title": "Latitude",
+ "type": "number"
+ },
+ "longitude": {
+ "description": "The longitude value in degrees of arc (dega). Value range [-180, 180]",
+ "maximum": 180,
+ "minimum": -180,
+ "title": "Longitude",
+ "type": "number"
+ }
+ },
+ "required": [
+ "latitude",
+ "longitude"
+ ],
+ "title": "2D Map Location",
+ "type": "object"
+ },
+ "opendes:wks:AbstractAliasNames:1.0.0": {
+ "title": "AbstractAliasNames",
+ "type": "object",
+ "properties": {
+ "AliasNameTypeID": {
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-AliasNameType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "A classification of alias names such as by role played or type of source, such as regulatory name, regulatory code, company code, international standard name, etc.",
+ "x-osdu-relationship": [{
+ "EntityType": "AliasNameType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "type": "string"
+ },
+ "EffectiveDateTime": {
+ "format": "date-time",
+ "type": "string",
+ "description": "The date and time when an alias name becomes effective."
+ },
+ "AliasName": {
+ "type": "string",
+ "description": "Alternative Name value of defined name type for an object."
+ },
+ "TerminationDateTime": {
+ "format": "date-time",
+ "type": "string",
+ "description": "The data and time when an alias name is no longer in effect."
+ },
+ "DefinitionOrganisationID": {
+ "pattern": "^[\\w\\-\\.]+:(reference-data\\-\\-StandardsOrganisation|master-data\\-\\-Organisation):[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The StandardsOrganisation (reference-data) or Organisation (master-data) that provided the name (the source).",
+ "x-osdu-relationship": [{
+ "EntityType": "StandardsOrganisation",
+ "GroupType": "reference-data"
+ }, {
+ "EntityType": "Organisation",
+ "GroupType": "master-data"
+ }
+ ],
+ "type": "string"
+ }
+ }
+ }
+ },
+ "properties": {
+ "data": {
+ "allOf": [
+ {
+ "type": "object",
+ "properties": {
+ "Location": {
+ "$ref": "#/definitions/opendes:wks:core_dl_geopoint:1.0.0",
+ "description": "The wellbore's position .",
+ "format": "core:dl:geopoint:1.0.0",
+ "title": "WGS 84 Position",
+ "type": "object"
+ },
+ "NameAliases": {
+ "x-osdu-indexing": {
+ "type": "nested"
+ },
+ "description": "Alternative names, including historical, by which this master data is/has been known (it should include all the identifiers).",
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/opendes:wks:AbstractAliasNames:1.0.0"
+ }
+ },
+ "FacilityName": {
+ "description": "Name of the Facility.",
+ "type": "string"
+ }
+ }
+ }
+ ]
+ }
+ }
+ }
+}
diff --git a/testing/indexer-test-core/src/main/resources/testData/index-property-welllog_v1.json b/testing/indexer-test-core/src/main/resources/testData/index-property-welllog_v1.json
new file mode 100644
index 0000000000000000000000000000000000000000..b22a36722b78de757f1631809f9f4ea51fa02f25
--- /dev/null
+++ b/testing/indexer-test-core/src/main/resources/testData/index-property-welllog_v1.json
@@ -0,0 +1,8 @@
+[{
+ "id": "tenant1:index-property--WellLog:testIngest2",
+ "data": {
+ "Name": "Log_ABC",
+ "WellboreID": "tenant1:index-property--Wellbore:testIngest1"
+ }
+ }
+]
diff --git a/testing/indexer-test-core/src/main/resources/testData/index-property-welllog_v1.schema.json b/testing/indexer-test-core/src/main/resources/testData/index-property-welllog_v1.schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..1b9549b616e5cc889ed200f08decc0b08d4a481b
--- /dev/null
+++ b/testing/indexer-test-core/src/main/resources/testData/index-property-welllog_v1.schema.json
@@ -0,0 +1,36 @@
+{
+ "schemaInfo": {
+ "schemaIdentity": {
+ "authority": "test",
+ "source": "indexer",
+ "entityType": "index-property--WellLog",
+ "schemaVersionMajor": 1,
+ "schemaVersionMinor": 0,
+ "schemaVersionPatch": 0,
+ "id": "test:indexer:index-property--WellLog:1.0.0"
+ },
+ "status": "DEVELOPMENT"
+ },
+ "schema": {
+ "properties": {
+ "data": {
+ "allOf": [
+ {
+ "type": "object",
+ "properties": {
+ "WellboreID": {
+ "description": "Id of parent wellbore.",
+ "type": "string"
+ },
+ "Name": {
+ "description": "Name of welllog.",
+ "type": "string"
+ }
+ }
+ }
+ ]
+ }
+ }
+ }
+
+}
diff --git a/testing/indexer-test-core/src/main/resources/testData/osdu_wks_IndexPropertyPathConfiguration_v1.json b/testing/indexer-test-core/src/main/resources/testData/osdu_wks_IndexPropertyPathConfiguration_v1.json
new file mode 100644
index 0000000000000000000000000000000000000000..f8ba28eb63ab5e68676610c029c12defbd962e2e
--- /dev/null
+++ b/testing/indexer-test-core/src/main/resources/testData/osdu_wks_IndexPropertyPathConfiguration_v1.json
@@ -0,0 +1,70 @@
+[{
+ "id": "tenant1:reference-data--IndexPropertyPathConfiguration:index-property--WellLog:1.",
+ "data": {
+ "Name": "WellLogIndex-PropertyPathConfiguration",
+ "Description": "The index property list for WellLog:1., valid for all WellLog kinds for major version 1.",
+ "Code": "test:indexer:index-property--WellLog:1.",
+ "AttributionAuthority": "OSDU",
+ "Configurations": [{
+ "Name": "WellboreName",
+ "Policy": "ExtractFirstMatch",
+ "Paths": [{
+ "RelatedObjectsSpec": {
+ "RelationshipDirection": "ChildToParent",
+ "RelatedObjectKind": "test:indexer:index-property--Wellbore:1.",
+ "RelatedObjectID": "data.WellboreID"
+ },
+ "ValueExtraction": {
+ "ValuePath": "data.FacilityName"
+ }
+ }
+ ],
+ "UseCase": "As a user I want to discover WellLog instances by the wellbore's name value."
+ }, {
+ "Name": "SpatialLocation",
+ "Policy": "ExtractFirstMatch",
+ "Paths": [{
+ "RelatedObjectsSpec": {
+ "RelationshipDirection": "ChildToParent",
+ "RelatedObjectKind": "test:indexer:index-property--Wellbore:1.",
+ "RelatedObjectID": "data.WellboreID"
+ },
+ "ValueExtraction": {
+ "ValuePath": "data.Location"
+ }
+ }
+ ],
+ "UseCase": "As a user I want to discover WellLog instances by spatial location."
+ }
+ ]
+ }
+ }, {
+ "id": "tenant1:reference-data--IndexPropertyPathConfiguration:index-property--Wellbore:1.",
+ "data": {
+ "Name": "Wellbore-IndexPropertyPathConfiguration",
+ "Description": "The index property list for index-property--Wellbore:1., valid for all index-property--Wellbore kinds for major version 1.",
+ "Code": "test:indexer:index-property--Wellbore:1.",
+ "AttributionAuthority": "OSDU",
+ "Configurations": [{
+ "Name": "WellUWI",
+ "Policy": "ExtractFirstMatch",
+ "Paths": [{
+ "ValueExtraction": {
+ "RelatedConditionMatches": [
+ "tenant1:reference-data--AliasNameType:UniqueIdentifier:",
+ "tenant1:reference-data--AliasNameType:RegulatoryName:",
+ "tenant1:reference-data--AliasNameType:PreferredName:",
+ "tenant1:reference-data--AliasNameType:CommonName:",
+ "tenant1:reference-data--AliasNameType:ShortName:"
+ ],
+ "RelatedConditionProperty": "data.NameAliases[].AliasNameTypeID",
+ "ValuePath": "data.NameAliases[].AliasName"
+ }
+ }
+ ],
+ "UseCase": "As a user I want to discover and match Wells by their UWI. I am aware that this is not globally reliable, however, I am able to specify a prioritized AliasNameType list to look up value in the NameAliases array."
+ }
+ ]
+ }
+ }
+]
diff --git a/testing/indexer-test-core/src/main/resources/testData/osdu_wks_IndexPropertyPathConfiguration_v1.schema.json b/testing/indexer-test-core/src/main/resources/testData/osdu_wks_IndexPropertyPathConfiguration_v1.schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..7cf15888121f227c11303d934c9af6340f384798
--- /dev/null
+++ b/testing/indexer-test-core/src/main/resources/testData/osdu_wks_IndexPropertyPathConfiguration_v1.schema.json
@@ -0,0 +1,284 @@
+{
+ "schemaInfo": {
+ "schemaIdentity": {
+ "authority": "osdu",
+ "source": "wks",
+ "entityType": "reference-data--IndexPropertyPathConfiguration",
+ "schemaVersionMajor": 1,
+ "schemaVersionMinor": 0,
+ "schemaVersionPatch": 0,
+ "id": "osdu:wks:reference-data--IndexPropertyPathConfiguration:1.0.0"
+ },
+ "status": "DEVELOPMENT"
+ },
+ "schema": {
+ "x-osdu-license": "Copyright 2023, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$id": "https://schema.osdu.opengroup.org/json/reference-data/IndexPropertyPathConfiguration.1.0.0.json",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:reference-data--IndexPropertyPathConfiguration:1.0.0",
+ "title": "IndexPropertyPathConfiguration",
+ "description": "IndexPropertyPathConfiguration contains the de-normalization configuration settings for the Search index. The record id contains the kind so that there is a unique relationship between the kind and the kind's index extensions.",
+ "type": "object",
+ "properties": {
+ "id": {
+ "description": "Previously called ResourceID or SRN which identifies this OSDU resource object without version.",
+ "title": "Entity ID",
+ "type": "string",
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-IndexPropertyPathConfiguration:[\\w\\-\\.\\:\\%]+$",
+ "example": "namespace:reference-data--IndexPropertyPathConfiguration:c9d84708-2b1b-5e0b-954e-9621132f7154"
+ },
+ "kind": {
+ "description": "The schema identification for the OSDU resource object following the pattern {Namespace}:{Source}:{Type}:{VersionMajor}.{VersionMinor}.{VersionPatch}. The versioning scheme follows the semantic versioning, https://semver.org/.",
+ "title": "Entity Kind",
+ "type": "string",
+ "pattern": "^[\\w\\-\\.]+:[\\w\\-\\.]+:[\\w\\-\\.]+:[0-9]+.[0-9]+.[0-9]+$",
+ "example": "osdu:wks:reference-data--IndexPropertyPathConfiguration:1.0.0"
+ },
+ "version": {
+ "description": "The version number of this OSDU resource; set by the framework.",
+ "title": "Version Number",
+ "type": "integer",
+ "format": "int64",
+ "example": 1562066009929332
+ },
+ "acl": {
+ "description": "The access control tags associated with this entity.",
+ "title": "Access Control List",
+ "$ref": "osdu:wks:AbstractAccessControlList:1.0.0"
+ },
+ "legal": {
+ "description": "The entity's legal tags and compliance status. The actual contents associated with the legal tags is managed by the Compliance Service.",
+ "title": "Legal Tags",
+ "$ref": "osdu:wks:AbstractLegalTags:1.0.0"
+ },
+ "tags": {
+ "title": "Tag Dictionary",
+ "description": "A generic dictionary of string keys mapping to string value. Only strings are permitted as keys and values.",
+ "type": "object",
+ "additionalProperties": {
+ "type": "string"
+ },
+ "example": {
+ "NameOfKey": "String value"
+ }
+ },
+ "createTime": {
+ "description": "Timestamp of the time at which initial version of this OSDU resource object was created. Set by the System. The value is a combined date-time string in ISO-8601 given in UTC.",
+ "title": "Resource Object Creation DateTime",
+ "type": "string",
+ "format": "date-time",
+ "example": "2020-12-16T11:46:20.163Z"
+ },
+ "createUser": {
+ "title": "Resource Object Creation User Reference",
+ "description": "The user reference, which created the first version of this resource object. Set by the System.",
+ "type": "string",
+ "example": "some-user@some-company-cloud.com"
+ },
+ "modifyTime": {
+ "description": "Timestamp of the time at which this version of the OSDU resource object was created. Set by the System. The value is a combined date-time string in ISO-8601 given in UTC.",
+ "title": "Resource Object Version Creation DateTime",
+ "type": "string",
+ "format": "date-time",
+ "example": "2020-12-16T11:52:24.477Z"
+ },
+ "modifyUser": {
+ "title": "Resource Object Version Creation User Reference",
+ "description": "The user reference, which created this version of this resource object. Set by the System.",
+ "type": "string",
+ "example": "some-user@some-company-cloud.com"
+ },
+ "ancestry": {
+ "description": "The links to data, which constitute the inputs, from which this record instance is derived.",
+ "title": "Ancestry",
+ "$ref": "osdu:wks:AbstractLegalParentList:1.0.0"
+ },
+ "meta": {
+ "description": "The Frame of Reference meta data section linking the named properties to self-contained definitions.",
+ "title": "Frame of Reference Meta Data",
+ "type": "array",
+ "items": {
+ "$ref": "osdu:wks:AbstractMetaItem:1.0.0"
+ }
+ },
+ "data": {
+ "allOf": [
+ {
+ "$ref": "osdu:wks:AbstractCommonResources:1.0.0"
+ },
+ {
+ "$ref": "osdu:wks:AbstractReferenceType:1.0.0"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "Configurations": {
+ "type": "array",
+ "title": "Configurations",
+ "description": "The list of index property configurations for the specific kind.",
+ "x-osdu-indexing": {
+ "type": "nested"
+ },
+ "items": {
+ "type": "object",
+ "title": "Configuration",
+ "description": "One single configuration to derive an Search index property value and assign it to the index 'column' with Name.",
+ "properties": {
+ "Name": {
+ "type": "string",
+ "title": "Name",
+ "description": "The name of the indexed property, i.e., this is the property name used in Search.",
+ "example": "CountryNames"
+ },
+ "UseCase": {
+ "type": "string",
+ "title": "Use Case",
+ "description": "The use case description this configuration satisfies.",
+ "example": "As a user I want to find objects by a country name, with the understanding that an object may extend over country boundaries."
+ },
+ "Policy": {
+ "type": "string",
+ "title": "Extraction Policy",
+ "description": "Current supported policies are 'ExtractAllMatches' resulting in an array of values or 'ExtractFirstMatch' single value. The policy applies only to the Paths[].ValueExtraction.",
+ "example": "ExtractAllMatches",
+ "pattern": "^(ExtractFirstMatch|ExtractAllMatches)$"
+ },
+ "Paths": {
+ "type": "array",
+ "title": "Paths",
+ "description": "The list of path definitions to derive the property value from.",
+ "x-osdu-indexing": {
+ "type": "nested"
+ },
+ "items": {
+ "type": "object",
+ "title": "Path",
+ "description": "A single path definition to derive a property value from.",
+ "properties": {
+ "RelatedObjectsSpec": {
+ "type": "object",
+ "title": "Related Objects Specification",
+ "description": "The specification to extract related objects, from which to derive the ValueExtraction. If this property is empty or absent, the ValueExtraction is done on the current object to be indexed.",
+ "properties": {
+ "RelatedObjectID": {
+ "type": "string",
+ "title": "Related Object ID",
+ "description": "The path to the property containing the ID of the target record to chase. This property is only populated if the property is extracted from a related object, which must be chased. If the property is derived from 'within' the same record, which triggered the indexing, the RelatedObjectID is left absent.",
+ "example": "GeoContexts[].GeoPoliticalEntityID"
+ },
+ "RelatedObjectKind": {
+ "type": "string",
+ "title": "Related Object Kind",
+ "description": "The kind or schema id expected as the target object type. This property is only populated if the property is extracted from a related object, which must be chased. If the property is derived from 'within' the same record, which triggered the indexing, the RelatedObjectKind is left absent.",
+ "example": "osdu:wks:master-data--GeoPoliticalEntity:1.",
+ "pattern": "^[\\w\\-\\.]+:[\\w\\-\\.]+:[\\w\\-\\.]+:[0-9]+.$"
+ },
+ "RelationshipDirection": {
+ "type": "string",
+ "title": "Relationship Direction",
+ "description": "The direction of the relationship definition seen from the object being indexed. 'ChildToParent' assumes an outgoing relationship with the target record defined in the object being indexed. 'ParentToChildren' assumes that the related objects have a relationship by RelatedObjectID to the id of the record being indexed.",
+ "example": "ChildToParent",
+ "pattern": "^(ChildToParent|ParentToChildren)$"
+ },
+ "RelatedConditionProperty": {
+ "type": "string",
+ "title": "Related Condition Property",
+ "description": "The property path of the target record data block, which needs subjected to the conditional matching. The data prefix is not required.",
+ "example": "GeoContexts[].GeoTypeID"
+ },
+ "RelatedConditionMatches": {
+ "type": "array",
+ "title": "Related Condition Matches",
+ "description": "The RelatedConditionProperty values, which need to match in order to be accepted as de-normalized value(s). If the Policy is ExtractFirstMatch, the list is prioritized and the first match is accepted as final value. Policy ExtractAllMatches collects all matching values as array.",
+ "example": [
+ "namespace:reference-data--GeoPoliticalEntityType:Country:"
+ ],
+ "items": {
+ "type": "string"
+ }
+ }
+ }
+ },
+ "ValueExtraction": {
+ "type": "object",
+ "title": "Value Extraction",
+ "description": "The instructions from where to derive the value.",
+ "properties": {
+ "RelatedConditionProperty": {
+ "type": "string",
+ "title": "Related Condition Property",
+ "description": "The property path of the target record data block, which needs to be subjected to the conditional matching. The data prefix is not required in the path."
+ },
+ "RelatedConditionMatches": {
+ "type": "array",
+ "title": "Related Condition Matches",
+ "description": "The RelatedConditionProperty values, which need to match in order to be accepted as de-normalized value(s). If the Policy is ExtractFirstMatch, the list is prioritized and the first match is accepted as final value. Policy ExtractAllMatches collects all matching values as array.",
+ "items": {
+ "type": "string"
+ }
+ },
+ "ValuePath": {
+ "type": "string",
+ "title": "Value Path",
+ "description": "The path to the property from where to extract the de-normalized value. The data prefix is not required in the path.",
+ "example": "GeoPoliticalEntityName"
+ }
+ },
+ "required": [
+ "ValuePath"
+ ]
+ }
+ },
+ "required": [
+ "ValueExtraction"
+ ]
+ }
+ }
+ }
+ }
+ }
+ },
+ "title": "IndividualProperties"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "ExtensionProperties": {
+ "type": "object"
+ }
+ },
+ "title": "ExtensionProperties"
+ }
+ ]
+ }
+ },
+ "required": [
+ "kind",
+ "acl",
+ "legal"
+ ],
+ "additionalProperties": false,
+ "x-osdu-review-status": "Accepted",
+ "x-osdu-governance-model": "OPEN",
+ "x-osdu-governance-authorities": [
+ "Energistics",
+ "OSDU"
+ ],
+ "x-osdu-virtual-properties": {
+ "data.VirtualProperties.DefaultName": {
+ "type": "string",
+ "priority": [
+ {
+ "path": "data.Name"
+ }
+ ]
+ }
+ },
+ "x-osdu-inheriting-from-kind": [
+ {
+ "name": "ReferenceType",
+ "kind": "osdu:wks:AbstractReferenceType:1.0.0"
+ }
+ ]
+ }
+}
diff --git a/testing/indexer-test-gc/pom.xml b/testing/indexer-test-gc/pom.xml
index db3c4f41e71c5524572f9f433a4d7288532370e1..5518883c91c18ac2825493006e0143f0c33a3700 100644
--- a/testing/indexer-test-gc/pom.xml
+++ b/testing/indexer-test-gc/pom.xml
@@ -6,13 +6,13 @@
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>indexer-test</artifactId>
- <version>0.21.0-SNAPSHOT</version>
+ <version>0.22.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-gc</artifactId>
- <version>0.21.0-SNAPSHOT</version>
+ <version>0.22.0-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
@@ -37,7 +37,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
- <version>0.21.0-SNAPSHOT</version>
+ <version>0.22.0-SNAPSHOT</version>
</dependency>
<!-- Cucumber -->
@@ -87,7 +87,7 @@
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
- <version>1.18.2</version>
+ <version>1.18.26</version>
<scope>provided</scope>
</dependency>
diff --git a/testing/indexer-test-gc/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-gc/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
index a01ca063d369d546a19d52e69470784459f5b70c..591f8a68f097bff0a0e4ed53d32b1246b95d858a 100644
--- a/testing/indexer-test-gc/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
+++ b/testing/indexer-test-gc/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -9,6 +9,7 @@ import cucumber.api.java.en.When;
import lombok.extern.java.Log;
import org.opengroup.osdu.common.SchemaServiceRecordSteps;
import org.opengroup.osdu.util.ElasticUtils;
+import org.opengroup.osdu.util.GCConfig;
import org.opengroup.osdu.util.GCPHTTPClient;
@Log
@@ -20,6 +21,7 @@ public class Steps extends SchemaServiceRecordSteps {
@Before
public void before(Scenario scenario) {
+ GCConfig.updateEntitlementsDomainVariable();
this.scenario = scenario;
this.httpClient = new GCPHTTPClient();
}
@@ -29,6 +31,16 @@ public class Steps extends SchemaServiceRecordSteps {
super.the_schema_is_created_with_the_following_kind(dataTable);
}
+ @Then("^I set starting stateful scenarios$")
+ public void i_set_starting_stateful_scenarios() throws Throwable {
+ super.i_set_scenarios_as_stateful(true);
+ }
+
+ @Then("^I set ending stateful scenarios$")
+ public void i_set_ending_stateful_scenarios() throws Throwable {
+ super.i_set_scenarios_as_stateful(false);
+ }
+
@When("^I ingest records with the \"(.*?)\" with \"(.*?)\" for a given \"(.*?)\"$")
public void i_ingest_records_with_the_for_a_given(String record, String dataGroup, String kind) {
super.i_ingest_records_with_the_for_a_given(record, dataGroup, kind);
@@ -70,6 +82,16 @@ public class Steps extends SchemaServiceRecordSteps {
super.iShouldBeAbleToSearchRecordByTagKeyAndTagValue(index, tagKey, tagValue, expectedNumber);
}
+ @Then("^I clean up the index of the extended kinds \"([^\"]*)\" in the Elastic Search$")
+ public void iShouldCleanupIndicesOfExtendedKinds(String extendedKinds) throws Throwable {
+ super.iShouldCleanupIndicesOfExtendedKinds(extendedKinds);
+ }
+
+ @Then("^I should be able to search (\\d+) record with index \"([^\"]*)\" by extended data field \"([^\"]*)\" and value \"([^\"]*)\"$")
+ public void iShouldBeAbleToSearchRecordByFieldAndFieldValue(int expectedNumber, String index, String fieldKey, String fieldValue) throws Throwable {
+ super.iShouldBeAbleToSearchRecordByFieldAndFieldValue(index, fieldKey, fieldValue, expectedNumber);
+ }
+
@Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by bounding box query with points \\((-?\\d+), (-?\\d+)\\) and \\((-?\\d+), (-?\\d+)\\) on field \"([^\"]*)\"$")
public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery(
int expectedCount, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field)
@@ -102,4 +124,4 @@ public class Steps extends SchemaServiceRecordSteps {
String actualName = generateActualName(index, null);
super.i_should_get_object_in_search_response_without_hints_in_schema(objectInnerField ,actualName, recordFile, acl, kind);
}
-}
\ No newline at end of file
+}
diff --git a/testing/indexer-test-gc/src/test/java/org/opengroup/osdu/util/GCConfig.java b/testing/indexer-test-gc/src/test/java/org/opengroup/osdu/util/GCConfig.java
new file mode 100644
index 0000000000000000000000000000000000000000..eb78bdee6a753ba195843b86fe4fc55c2ea957ca
--- /dev/null
+++ b/testing/indexer-test-gc/src/test/java/org/opengroup/osdu/util/GCConfig.java
@@ -0,0 +1,15 @@
+package org.opengroup.osdu.util;
+
+import java.util.Optional;
+
+public class GCConfig {
+
+ private static final String GROUP_ID_VARIABLE = "GROUP_ID";
+ private static final String ENTITLEMENTS_DOMAIN_VARIABLE = "ENTITLEMENTS_DOMAIN";
+
+ public static void updateEntitlementsDomainVariable() {
+ String groupId = Optional.ofNullable(System.getProperty(GROUP_ID_VARIABLE, System.getenv(GROUP_ID_VARIABLE)))
+ .orElse("group");
+ System.setProperty(ENTITLEMENTS_DOMAIN_VARIABLE, groupId);
+ }
+}
diff --git a/testing/indexer-test-gc/src/test/resources/cucumber.properties b/testing/indexer-test-gc/src/test/resources/cucumber.properties
new file mode 100644
index 0000000000000000000000000000000000000000..6884be422aa64d5c1c7a2dfb832c803f77371e43
--- /dev/null
+++ b/testing/indexer-test-gc/src/test/resources/cucumber.properties
@@ -0,0 +1,18 @@
+#
+# Copyright 2020-2023 Google LLC
+# Copyright 2020-2023 EPAM Systems, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# tag indexer-extended disabled by default
+cucumber.options=--tags ~@indexer-extended
diff --git a/testing/indexer-test-ibm/pom.xml b/testing/indexer-test-ibm/pom.xml
index 6d27a4904bff803ef284949fd99b9730e0bd6f28..d9ef6e862725d616bf7b16d746fd02c48222d58d 100644
--- a/testing/indexer-test-ibm/pom.xml
+++ b/testing/indexer-test-ibm/pom.xml
@@ -6,13 +6,13 @@
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>indexer-test</artifactId>
- <version>0.21.0-SNAPSHOT</version>
+ <version>0.22.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-ibm</artifactId>
- <version>0.21.0-SNAPSHOT</version>
+ <version>0.22.0-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
@@ -38,7 +38,7 @@
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
<artifactId>indexer-test-core</artifactId>
- <version>0.21.0-SNAPSHOT</version>
+ <version>0.22.0-SNAPSHOT</version>
</dependency>
<dependency>
diff --git a/testing/indexer-test-ibm/src/main/resources/cucumber.properties b/testing/indexer-test-ibm/src/main/resources/cucumber.properties
new file mode 100644
index 0000000000000000000000000000000000000000..2ca785a2f55f5f35f2c0f4b755c2a6cd072669c0
--- /dev/null
+++ b/testing/indexer-test-ibm/src/main/resources/cucumber.properties
@@ -0,0 +1,2 @@
+# tag indexer-extended enabled by default
+cucumber.options=--tags '~@* and @indexer-extended'
diff --git a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
index 624f994deeeded56f15e7acd660e8873e7ed0518..f7eb42a590c9f9faca81b2e36d3b9b7d4067855c 100644
--- a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
+++ b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/index/record/Steps.java
@@ -48,6 +48,16 @@ public class Steps extends SchemaServiceRecordSteps {
super.the_schema_is_created_with_the_following_kind(dataTable);
}
+ @Then("^I set starting stateful scenarios$")
+ public void i_set_starting_stateful_scenarios() throws Throwable {
+ super.i_set_scenarios_as_stateful(true);
+ }
+
+ @Then("^I set ending stateful scenarios$")
+ public void i_set_ending_stateful_scenarios() throws Throwable {
+ super.i_set_scenarios_as_stateful(false);
+ }
+
@When("^I ingest records with the \"(.*?)\" with \"(.*?)\" for a given \"(.*?)\"$")
public void i_ingest_records_with_the_for_a_given(String record, String dataGroup, String kind) {
super.i_ingest_records_with_the_for_a_given(record, dataGroup, kind);
@@ -88,6 +98,16 @@ public class Steps extends SchemaServiceRecordSteps {
super.iShouldBeAbleToSearchRecordByTagKeyAndTagValue(index, tagKey, tagValue, expectedNumber);
}
+ @Then("^I clean up the index of the extended kinds \"([^\"]*)\" in the Elastic Search$")
+ public void iShouldCleanupIndicesOfExtendedKinds(String extendedKinds) throws Throwable {
+ super.iShouldCleanupIndicesOfExtendedKinds(extendedKinds);
+ }
+
+ @Then("^I should be able to search (\\d+) record with index \"([^\"]*)\" by extended data field \"([^\"]*)\" and value \"([^\"]*)\"$")
+ public void iShouldBeAbleToSearchRecordByFieldAndFieldValue(int expectedNumber, String index, String fieldKey, String fieldValue) throws Throwable {
+ super.iShouldBeAbleToSearchRecordByFieldAndFieldValue(index, fieldKey, fieldValue, expectedNumber);
+ }
+
@Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by bounding box query with points \\((-?\\d+), (-?\\d+)\\) and \\((-?\\d+), (-?\\d+)\\) on field \"(.*?)\"$")
public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery (
int expectedCount, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field) throws Throwable {
@@ -156,4 +176,4 @@ public class Steps extends SchemaServiceRecordSteps {
}
}
-}
\ No newline at end of file
+}
diff --git a/testing/pom.xml b/testing/pom.xml
index 0af155f036c2d6b01b7720d850d3555e49beef36..b16bb64b6bbd3047e482a31d492582735da64685 100644
--- a/testing/pom.xml
+++ b/testing/pom.xml
@@ -18,7 +18,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.opengroup.osdu</groupId>
<artifactId>indexer-test</artifactId>
- <version>0.21.0-SNAPSHOT</version>
+ <version>0.22.0-SNAPSHOT</version>
<description>Indexer Service Integration Test Root Project</description>
<properties>
<spring.version>5.3.23</spring.version>
@@ -39,7 +39,7 @@
<module>indexer-test-aws</module>
<module>indexer-test-azure</module>
<module>indexer-test-ibm</module>
- <module>indexer-test-anthos</module>
+ <module>indexer-test-baremetal</module>
</modules>
<repositories>