diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 0b189ff5fe1e543ea43b90b9150735a8e2ca4aef..25b54f08fe6e97ca6abfd83e02af323df0fc10b4 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -29,6 +29,8 @@ variables:
AZURE_TEST_SUBDIR: testing/indexer-test-azure
SERVICE_JAVA_VERSION: "17"
+ ACCEPTANCE_TEST_DIR: "indexer-acceptance-test"
+
include:
- project: "osdu/platform/ci-cd-pipelines"
file: "standard-setup.yml"
@@ -65,103 +67,6 @@ include:
- local: "devops/core-plus/pipeline/override-stages.yml"
- local: "/publish.yml"
-.maven:
- image: maven:3.9.3-eclipse-temurin-17
- tags: ["osdu-medium"]
- variables:
- MAVEN_REPO_PATH: "$CI_PROJECT_DIR/.m2/repository"
- MAVEN_CLI_OPTS: "--batch-mode --settings=$CI_PROJECT_DIR/.mvn/community-maven.settings.xml"
-
- cache:
- paths:
- - $MAVEN_REPO_PATH
-
- artifacts:
- paths:
- - ./**/target/*.jar
- - ./**/maven-*-output.txt
- when: always
- expire_in: 2 days
-
- before_script:
- - | # Check for the presence of a maven wrapper script
- apt-get update && apt-get install -y python3
- if [ -e "$CI_PROJECT_DIR/mvnw" ]; then
- export MAVEN_BINARY="$CI_PROJECT_DIR/mvnw"
- unset MAVEN_CONFIG
- mkdir -pv .mvn/wrapper
- else
- export MAVEN_BINARY="mvn"
- fi
- - export MAVEN="$MAVEN_BINARY $MAVEN_CLI_OPTS -Dmaven.repo.local=$MAVEN_REPO_PATH"
- - echo $MAVEN_BINARY
- - echo $MAVEN_CLI_OPTS
- - echo $MAVEN_REPO_PATH
- - echo $MAVEN
- - mkdir -pv $MAVEN_REPO_PATH
- - export ALL_MAVEN_BUILD_DIRS_Z=$(mktemp)
- - python3 --version
-
- # This scans for all pom.xml files that aren't listed as a child pom (as
- # determine by the presence of a <parent> tag).
- - |
- (python3 | sort -z > $ALL_MAVEN_BUILD_DIRS_Z) <<EOF
- from xml.dom.minidom import parse
- import os
- import sys
-
- allPomFiles = set()
- submoduleFiles = set()
-
- for root, dirnames, filenames in os.walk(os.environ['CI_PROJECT_DIR']):
- if 'pom.xml' in filenames:
- pomFile = os.path.join(root, 'pom.xml')
- allPomFiles.add(pomFile)
-
- dom = parse(pomFile)
- for modules in dom.getElementsByTagName('modules'):
- for module in modules.getElementsByTagName('module'):
- submoduleFiles.add(os.path.join(root, module.firstChild.nodeValue, "pom.xml"))
-
- for pomFile in allPomFiles:
- if pomFile not in submoduleFiles:
- dir = os.path.dirname(pomFile)
- sys.stdout.write(dir + '\0')
- EOF
- - xargs -0rn 1 echo < $ALL_MAVEN_BUILD_DIRS_Z
- # This creates a script for running maven, capturing output to a file.
- # - First argument is the directory to build from
- # - Second argument is the name to use for the logfile
- # - The remaining arguments are the maven command to run
- - export MAVEN_BUILD=$(tempfile -m 755 -p build -s .sh)
- - echo "#!/bin/bash" > $MAVEN_BUILD
- - |
- cat >> $MAVEN_BUILD <<EOF
- id=maven_build_\$(echo "\$@" | sha1sum | head -c 7)
- cd "\$1"; shift
- outputFile="\$1"; shift
-
- echo "********************************************************************************"
- echo -e "\e[0Ksection_start:\$(date +%s):\${id}\r\e[0K\e[1m\$(pwd)\e[0m"
- echo -e "\e[32;1m\$MAVEN" "\$@" "\e[0m"
-
- if \$MAVEN "\$@" > "\$outputFile" 2>&1; then
- tail -n 25 "\$outputFile";
- echo -e "\e[36m**** Showing the last 25 lines of the passed build (above). See Job artifacts for full log details *****\e[0m";
- echo -e "\e[0Ksection_end:\$(date +%s):\${id}\r\e[0K"
- else
- tail -n 200 "\$outputFile";
- echo -e "\e[31m**** Showing the last 200 lines of the failed build (above). See Job artifacts for full log details *****\e[0m";
- echo -e "\e[0Ksection_end:\$(date +%s):\${id}\r\e[0K"
- exit 255;
- fi
- EOF
-
- # This creates a script for applying the $MAVEN_BUILD script to all build directories
- - export MAVEN_FOREACH=$(tempfile -m 755 -p fr-ec -s .sh)
- - echo "#!/bin/sh" > $MAVEN_FOREACH
- - echo 'xargs -0rI {} $MAVEN_BUILD {} "$@" < $ALL_MAVEN_BUILD_DIRS_Z' >> $MAVEN_FOREACH
-
download_plugins:
image: maven:3.8.3-openjdk-17-slim
stage: build
@@ -185,22 +90,6 @@ download_plugins:
variables:
- $CORE == '1'
-gemnasium-maven-dependency_scanning:
- variables:
- DS_JAVA_VERSION: 17
-
-azure_containerize:
- variables:
- AZURE_CONTAINERIZE_REPOSITORY: "-b jdk-17 https://gitlab-ci-token:${CI_JOB_TOKEN}@community.opengroup.org/osdu/platform/deployment-and-operations/base-containers-azure/service-base-image"
-
-azure_test:
- image: community.opengroup.org:5555/osdu/platform/deployment-and-operations/base-containers-azure/azure-maven17:v0.0.1
-
aws-test-java:
image: $CI_REGISTRY/osdu/platform/deployment-and-operations/base-containers-aws/aws-maven/aws-maven:v2.1
tags: ["aws-internal-test-ubuntu22.04"]
-
-fossa-analyze:
- image: $CI_REGISTRY/divido/fossa-with-cache:v0.9-jdk17
-fossa-check-notice:
- image: $CI_REGISTRY/divido/fossa-with-cache:v0.9-jdk17
diff --git a/NOTICE b/NOTICE
index 993c3f001083fdbe6f88622d5de739c89d246ed3..322ddeb2d199e9e78889adcac5fdb810bfd74b69 100644
--- a/NOTICE
+++ b/NOTICE
@@ -3,22 +3,12 @@ Generated by fossa-cli (https://github.com/fossas/fossa-cli).
Formatted by fossa-with-cache (https://community.opengroup.org/divido/fossa-with-cache).
This software includes the following software and licenses:
-========================================================================
-Android-Sdk
-========================================================================
-The following software have components provided under the terms of this license:
-
-- Android SDK (from https://www.android.com/)
-
========================================================================
Apache-1.1
========================================================================
The following software have components provided under the terms of this license:
- AspectJ Weaver (from http://www.aspectj.org, https://www.eclipse.org/aspectj/)
-- Microsoft Application Insights Java Agent (from https://github.com/Microsoft/ApplicationInsights-Java)
-- Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java)
-- Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils, https://codehaus-plexus.github.io/plexus-utils/, https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-utils)
- Proton-J (from https://repo1.maven.org/maven2/org/apache/qpid/proton-j)
@@ -29,7 +19,6 @@ The following software have components provided under the terms of this license:
- AMQP 1.0 JMS Spring Boot AutoConfiguration (from https://repo1.maven.org/maven2/org/amqphub/spring/amqp-10-jms-spring-boot-autoconfigure)
- AMQP 1.0 JMS Spring Boot Starter (from https://repo1.maven.org/maven2/org/amqphub/spring/amqp-10-jms-spring-boot-starter)
-- API Common (from https://github.com/googleapis, https://github.com/googleapis/api-common-java, https://repo1.maven.org/maven2/com/google/api/api-common)
- ASM Commons (from http://asm.ow2.io/, https://repo1.maven.org/maven2/org/ow2/asm/asm-commons)
- ASM based accessors helper used by json-smart (from https://urielch.github.io/)
- AWS Java SDK for AWS KMS (from https://aws.amazon.com/sdkforjava)
@@ -44,15 +33,11 @@ The following software have components provided under the terms of this license:
- AWS Java SDK for the AWS Simple Systems Management (SSM) Service (from https://aws.amazon.com/sdkforjava)
- AWS SDK for Java - Core (from https://aws.amazon.com/sdkforjava)
- Adapter: RxJava (from https://github.com/square/retrofit)
-- Animal Sniffer Annotations (from https://repo1.maven.org/maven2/org/codehaus/mojo/animal-sniffer-annotations)
-- Apache Commons BeanUtils (from http://commons.apache.org/proper/commons-beanutils/, https://commons.apache.org/proper/commons-beanutils/, https://repo1.maven.org/maven2/commons-beanutils/commons-beanutils)
+- Apache Commons BeanUtils (from http://commons.apache.org/proper/commons-beanutils/, https://commons.apache.org/proper/commons-beanutils, https://repo1.maven.org/maven2/commons-beanutils/commons-beanutils)
- Apache Commons Codec (from http://commons.apache.org/proper/commons-codec/, https://commons.apache.org/proper/commons-codec/)
- Apache Commons IO (from http://commons.apache.org/io/, https://commons.apache.org/proper/commons-io/, https://repo1.maven.org/maven2/commons-io/commons-io)
- Apache Commons Lang (from https://commons.apache.org/proper/commons-lang/)
- Apache Commons Logging (from http://commons.apache.org/logging/, http://commons.apache.org/proper/commons-logging/, https://commons.apache.org/proper/commons-logging/)
-- Apache Commons Validator (from http://commons.apache.org/proper/commons-validator/, http://jakarta.apache.org/commons/${pom.artifactId.substring(8)}/, https://repo1.maven.org/maven2/commons-validator/commons-validator)
-- Apache Groovy (from http://groovy-lang.org, http://groovy.codehaus.org/, https://groovy-lang.org)
-- Apache HTTP transport v2 for the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-apache-v2)
- Apache HttpAsyncClient (from http://hc.apache.org/httpcomponents-asyncclient)
- Apache HttpClient (from http://hc.apache.org/httpcomponents-client, http://hc.apache.org/httpcomponents-client-ga)
- Apache HttpClient Cache (from http://hc.apache.org/httpcomponents-client, http://hc.apache.org/httpcomponents-client-ga)
@@ -60,12 +45,12 @@ The following software have components provided under the terms of this license:
- Apache HttpCore NIO (from http://hc.apache.org/httpcomponents-core-ga, http://hc.apache.org/httpcomponents-core/)
- Apache Log4j API (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-api)
- Apache Log4j Core (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core)
-- Apache Log4j JUL Adapter (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-jul)
+- Apache Log4j JUL Handler (from https://logging.apache.org/log4j/3.x/)
+- Apache Log4j SLF4J 2.0 Binding (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-slf4j2-impl)
- Apache Log4j SLF4J Binding (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-slf4j-impl)
- Apache Log4j to SLF4J Adapter (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-to-slf4j)
- AssertJ Core (from https://assertj.github.io/doc/#assertj-core)
- Asynchronous Http Client (from https://repo1.maven.org/maven2/org/asynchttpclient/async-http-client)
-- Asynchronous Http Client Netty Utils (from https://repo1.maven.org/maven2/org/asynchttpclient/async-http-client-netty-utils)
- AutoValue Annotations (from https://github.com/google/auto/tree/main/value, https://github.com/google/auto/tree/master/value, https://repo1.maven.org/maven2/com/google/auto/value/auto-value-annotations)
- Awaitility (from http://awaitility.org)
- BSON (from http://bsonspec.org, https://bsonspec.org)
@@ -73,41 +58,27 @@ The following software have components provided under the terms of this license:
- Byte Buddy (without dependencies) (from https://repo1.maven.org/maven2/net/bytebuddy/byte-buddy)
- Byte Buddy Java agent (from https://repo1.maven.org/maven2/net/bytebuddy/byte-buddy-agent)
- ClassMate (from http://github.com/cowtowncoder/java-classmate)
-- Cloud Key Management Service (KMS) API (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-cloudkms)
- Collections (from https://repo1.maven.org/maven2/commons-collections/commons-collections)
-- Commons Digester (from http://commons.apache.org/digester/)
- Converter: Jackson (from https://github.com/square/retrofit, https://repo1.maven.org/maven2/com/squareup/retrofit2/converter-jackson)
- Core functionality for the Reactor Netty library (from https://github.com/reactor/reactor-netty)
- Elasticsearch Java API Client (from https://github.com/elastic/elasticsearch-java/)
- FindBugs-jsr305 (from http://findbugs.sourceforge.net/)
-- GAX (Google Api eXtensions) for Java (Core) (from https://github.com/googleapis, https://github.com/googleapis/gax-java, https://repo1.maven.org/maven2/com/google/api/gax)
-- GAX (Google Api eXtensions) for Java (HTTP JSON) (from <https://repo1.maven.org/maven2/com/google/api/gax-httpjson>, https://repo1.maven.org/maven2/com/google/api/gax-httpjson)
-- GAX (Google Api eXtensions) for Java (gRPC) (from <https://repo1.maven.org/maven2/com/google/api/gax-grpc>, https://repo1.maven.org/maven2/com/google/api/gax-grpc)
- GSON extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-gson)
-- Google APIs Client Library for Java (from https://repo1.maven.org/maven2/com/google/api-client/google-api-client)
- Google Auth Library for Java - Credentials (from https://repo1.maven.org/maven2/com/google/auth/google-auth-library-credentials)
- Google Auth Library for Java - OAuth2 HTTP (from https://repo1.maven.org/maven2/com/google/auth/google-auth-library-oauth2-http)
-- Google Cloud Core (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-core, https://github.com/googleapis/java-core, https://repo1.maven.org/maven2/com/google/cloud/google-cloud-core)
-- Google Cloud Core gRPC (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-core-grpc, https://github.com/googleapis/java-core, https://repo1.maven.org/maven2/com/google/cloud/google-cloud-core-grpc)
-- Google Cloud Logging (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-logging, https://github.com/googleapis/java-logging)
-- Google Cloud Pub/Sub (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-pubsub, https://github.com/googleapis/java-pubsub)
- Google HTTP Client Library for Java (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client)
-- Google OAuth Client Library for Java (from https://repo1.maven.org/maven2/com/google/oauth-client/google-oauth-client)
- Gson (from http://code.google.com/p/google-gson/, https://repo1.maven.org/maven2/com/google/code/gson/gson)
- Guava InternalFutureFailureAccess and InternalFutures (from https://repo1.maven.org/maven2/com/google/guava/failureaccess)
- Guava ListenableFuture only (from https://repo1.maven.org/maven2/com/google/guava/listenablefuture)
- Guava: Google Core Libraries for Java (from http://code.google.com/p/guava-libraries, https://github.com/google/guava, https://repo1.maven.org/maven2/com/google/guava/guava)
- HTTP functionality for the Reactor Netty library (from https://github.com/reactor/reactor-netty)
-- Hibernate Validator (from http://hibernate.org/validator, https://repo1.maven.org/maven2/org/hibernate/hibernate-validator, https://repo1.maven.org/maven2/org/hibernate/validator/hibernate-validator)
-- Hop (from https://github.com/rabbitmq/hop, https://www.rabbitmq.com)
+- Hibernate Validator (from https://hibernate.org/validator, https://repo1.maven.org/maven2/org/hibernate/hibernate-validator, https://repo1.maven.org/maven2/org/hibernate/validator/hibernate-validator)
- IBM COS Java SDK for Amazon S3 (from https://github.com/ibm/ibm-cos-sdk-java)
- IBM COS Java SDK for COS KMS (from https://github.com/ibm/ibm-cos-sdk-java)
- IBM COS SDK For Java (from https://github.com/ibm/ibm-cos-sdk-java)
- IBM COS SDK for Java - Core (from https://github.com/ibm/ibm-cos-sdk-java)
- J2ObjC Annotations (from https://github.com/google/j2objc/)
- JBoss Logging 3 (from http://www.jboss.org)
-- JBoss Marshalling API (from https://repo1.maven.org/maven2/org/jboss/marshalling/jboss-marshalling)
-- JBoss Marshalling River (from https://repo1.maven.org/maven2/org/jboss/marshalling/jboss-marshalling-river)
- JCIP Annotations under Apache License (from http://stephenc.github.com/jcip-annotations)
- JJWT :: API (from https://repo1.maven.org/maven2/io/jsonwebtoken/jjwt-api)
- JJWT :: Extensions :: Jackson (from https://repo1.maven.org/maven2/io/jsonwebtoken/jjwt-jackson)
@@ -119,16 +90,14 @@ The following software have components provided under the terms of this license:
- JSON.simple (from http://code.google.com/p/json-simple/)
- JSONassert (from http://github.com/skyscreamer/yoga, https://github.com/skyscreamer/JSONassert)
- JSR107 API and SPI (from https://github.com/jsr107/jsr107spec)
-- Jackson 2 extensions to the Google APIs Client Library for Java (from https://repo1.maven.org/maven2/com/google/api-client/google-api-client-jackson2)
+- JSpecify annotations (from http://jspecify.org/)
- Jackson dataformat: CBOR (from http://github.com/FasterXML/jackson-dataformats-binary)
- Jackson datatype: JSR310 (from http://wiki.fasterxml.com/JacksonModuleJSR310, https://repo1.maven.org/maven2/com/fasterxml/jackson/datatype/jackson-datatype-jsr310)
- Jackson datatype: Joda (from http://wiki.fasterxml.com/JacksonModuleJoda, https://github.com/FasterXML/jackson-datatype-joda)
- Jackson datatype: jdk8 (from https://repo1.maven.org/maven2/com/fasterxml/jackson/datatype/jackson-datatype-jdk8)
-- Jackson module: Afterburner (from http://wiki.fasterxml.com/JacksonHome, https://github.com/FasterXML/jackson-modules-base)
- Jackson module: Jakarta XML Bind Annotations (jakarta.xml.bind) (from https://github.com/FasterXML/jackson-modules-base)
- Jackson-annotations (from http://github.com/FasterXML/jackson, http://wiki.fasterxml.com/JacksonHome, https://github.com/FasterXML/jackson)
- Jackson-core (from http://wiki.fasterxml.com/JacksonHome, https://github.com/FasterXML/jackson-core)
-- Jackson-dataformat-XML (from http://wiki.fasterxml.com/JacksonExtensionXmlDataBinding, https://github.com/FasterXML/jackson-dataformat-xml)
- Jackson-dataformat-YAML (from https://github.com/FasterXML/jackson, https://github.com/FasterXML/jackson-dataformats-text)
- Jackson-module-parameter-names (from https://repo1.maven.org/maven2/com/fasterxml/jackson/module/jackson-module-parameter-names)
- Jakarta Dependency Injection (from https://github.com/eclipse-ee4j/injection-api)
@@ -137,7 +106,6 @@ The following software have components provided under the terms of this license:
- Jakarta Validation API (from https://beanvalidation.org)
- Java Native Access (from https://github.com/java-native-access/jna, https://github.com/twall/jna)
- Java Native Access Platform (from https://github.com/java-native-access/jna)
-- JavaBeans Activation Framework (from <http://java.sun.com/javase/technologies/desktop/javabeans/jaf/index.jsp>, http://java.sun.com/javase/technologies/desktop/javabeans/jaf/index.jsp, https://repo1.maven.org/maven2/com/sun/activation/javax.activation)
- Javassist (from http://www.javassist.org/, https://www.javassist.org/)
- JetBrains Java Annotations (from https://github.com/JetBrains/java-annotations)
- Jetty :: Utilities (from http://jetty.mortbay.org, http://www.eclipse.org/jetty, https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-util, https://repo1.maven.org/maven2/org/mortbay/jetty/jetty-util)
@@ -149,17 +117,12 @@ The following software have components provided under the terms of this license:
- Kotlin Stdlib Jdk8 (from <https://kotlinlang.org/>, https://kotlinlang.org/)
- Lettuce (from http://github.com/lettuce-io/lettuce-core, http://github.com/mp911de/lettuce/wiki, https://github.com/lettuce-io/lettuce-core/wiki)
- Metrics Core (from https://repo1.maven.org/maven2/io/dropwizard/metrics/metrics-core)
-- Microsoft Application Insights Java Agent (from https://github.com/Microsoft/ApplicationInsights-Java)
-- Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java)
-- Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Azure Java Core Library (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure Netty HTTP Client Library (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java)
-- Microsoft Azure client library for Identity (from https://github.com/Azure/azure-sdk-for-java)
- Mockito (from http://mockito.org, https://github.com/mockito/mockito)
- MongoDB Driver (from https://www.mongodb.com/)
- MongoDB Java Driver (from http://mongodb.org/, http://www.mongodb.org, https://www.mongodb.com/)
-- Netty Reactive Streams Implementation (from https://repo1.maven.org/maven2/com/typesafe/netty/netty-reactive-streams)
- Netty/Buffer (from https://repo1.maven.org/maven2/io/netty/netty-buffer)
- Netty/Codec (from https://repo1.maven.org/maven2/io/netty/netty-codec)
- Netty/Codec/DNS (from https://repo1.maven.org/maven2/io/netty/netty-codec-dns)
@@ -169,7 +132,6 @@ The following software have components provided under the terms of this license:
- Netty/Common (from https://repo1.maven.org/maven2/io/netty/netty-common)
- Netty/Handler (from https://repo1.maven.org/maven2/io/netty/netty-handler)
- Netty/Handler/Proxy (from https://repo1.maven.org/maven2/io/netty/netty-handler-proxy)
-- Netty/Incubator/Codec/Classes/Quic (from <https://repo1.maven.org/maven2/io/netty/incubator/netty-incubator-codec-classes-quic>, https://repo1.maven.org/maven2/io/netty/incubator/netty-incubator-codec-classes-quic)
- Netty/Resolver (from https://repo1.maven.org/maven2/io/netty/netty-resolver)
- Netty/Resolver/DNS (from https://repo1.maven.org/maven2/io/netty/netty-resolver-dns)
- Netty/Resolver/DNS/Classes/MacOS (from https://repo1.maven.org/maven2/io/netty/netty-resolver-dns-classes-macos)
@@ -190,25 +152,15 @@ The following software have components provided under the terms of this license:
- OkHttp Logging Interceptor (from https://github.com/square/okhttp, https://repo1.maven.org/maven2/com/squareup/okhttp3/logging-interceptor, https://square.github.io/okhttp/)
- OkHttp URLConnection (from https://repo1.maven.org/maven2/com/squareup/okhttp3/okhttp-urlconnection, https://square.github.io/okhttp/)
- Okio (from https://github.com/square/okio/, https://repo1.maven.org/maven2/com/squareup/okio/okio)
-- OpenCensus (from https://github.com/census-instrumentation/opencensus-java, https://github.com/census-instrumentation/opencensus-proto)
+- OpenCensus (from https://github.com/census-instrumentation/opencensus-java)
- OpenTelemetry Java (from https://github.com/open-telemetry/opentelemetry-java)
-- OpenTelemetry Semantic Conventions Java (from https://github.com/open-telemetry/semantic-conventions-java)
- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils, https://codehaus-plexus.github.io/plexus-utils/, https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-utils)
-- Prometheus Metrics Config (from https://repo1.maven.org/maven2/io/prometheus/prometheus-metrics-config)
-- Prometheus Metrics Core (from https://repo1.maven.org/maven2/io/prometheus/prometheus-metrics-core)
-- Prometheus Metrics Exposition Formats (from https://repo1.maven.org/maven2/io/prometheus/prometheus-metrics-exposition-formats)
-- Prometheus Metrics Model (from https://repo1.maven.org/maven2/io/prometheus/prometheus-metrics-model)
-- Prometheus Metrics Tracer Common (from https://repo1.maven.org/maven2/io/prometheus/prometheus-metrics-tracer-common)
- Proton-J (from https://repo1.maven.org/maven2/org/apache/qpid/proton-j)
-- QUIC functionality for the Reactor Netty library (from https://github.com/reactor/reactor-netty)
- QpidJMS Client (from https://repo1.maven.org/maven2/org/apache/qpid/qpid-jms-client)
-- RabbitMQ Java Client (from http://www.rabbitmq.com, https://www.rabbitmq.com)
-- Reactive Streams Netty driver (from https://github.com/reactor/reactor-netty)
-- Redisson (from http://redisson.org)
+- Redisson (from http://redisson.org, https://redisson.pro)
- Retrofit (from https://github.com/square/retrofit, https://repo1.maven.org/maven2/com/squareup/retrofit2/retrofit)
- RxJava (from https://github.com/ReactiveX/RxJava)
- Servlet API (from https://repo1.maven.org/maven2/org/mortbay/jetty/servlet-api)
-- Shaded Protobuf (from https://repo1.maven.org/maven2/io/prometheus/prometheus-metrics-shaded-protobuf)
- SnakeYAML (from http://code.google.com/p/snakeyaml/, http://www.snakeyaml.org, https://bitbucket.org/snakeyaml/snakeyaml)
- Spatial4J (from https://projects.eclipse.org/projects/locationtech.spatial4j)
- Spring AOP (from http://www.springframework.org, https://github.com/spring-projects/spring-framework, https://repo1.maven.org/maven2/org/springframework/spring-aop)
@@ -219,7 +171,6 @@ The following software have components provided under the terms of this license:
- Spring Boot Actuator AutoConfigure (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-actuator-autoconfigure, https://spring.io/projects/spring-boot)
- Spring Boot Actuator Starter (from http://projects.spring.io/spring-boot/, https://spring.io/projects/spring-boot)
- Spring Boot AutoConfigure (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-autoconfigure, https://spring.io/projects/spring-boot)
-- Spring Boot Configuration Processor (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-tools/spring-boot-configuration-processor, https://spring.io/projects/spring-boot)
- Spring Boot Jersey Starter (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-jersey, https://spring.io/projects/spring-boot)
- Spring Boot Json Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-json, https://spring.io/projects/spring-boot)
- Spring Boot Log4j 2 Starter (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-log4j2, https://spring.io/projects/spring-boot)
@@ -245,31 +196,17 @@ The following software have components provided under the terms of this license:
- Spring Security - Core (from http://spring.io/spring-security, https://repo1.maven.org/maven2/org/springframework/security/spring-security-core, https://spring.io/projects/spring-security, https://spring.io/spring-security)
- Spring Security - Namespace Configuration Module (from http://spring.io/spring-security, https://repo1.maven.org/maven2/org/springframework/security/spring-security-config, https://spring.io/projects/spring-security, https://spring.io/spring-security)
- Spring Security - Web Application Security Module (from http://spring.io/spring-security, https://repo1.maven.org/maven2/org/springframework/security/spring-security-web, https://spring.io/projects/spring-security, https://spring.io/spring-security)
-- Spring Security JWT Library (from http://github.com/spring-projects/spring-security-oauth)
+- Spring Security JWT Library (from https://github.com/spring-projects/spring-security-oauth)
- Spring TestContext Framework (from https://github.com/SpringSource/spring-framework, https://github.com/spring-projects/spring-framework)
- Spring Transaction (from https://github.com/SpringSource/spring-framework, https://github.com/spring-projects/spring-framework)
- Spring Web (from http://www.springframework.org, https://github.com/spring-projects/spring-framework, https://repo1.maven.org/maven2/org/springframework/spring-web)
- Spring Web MVC (from https://github.com/spring-projects/spring-framework, https://repo1.maven.org/maven2/org/springframework/spring-webmvc)
- Standard Uri Template (from https://std-uritemplate.github.io/)
- Swagger UI (from <http://webjars.org>, http://webjars.org, https://www.webjars.org)
-- Woodstox (from https://github.com/FasterXML/woodstox)
- Xerces2-j (from https://repo1.maven.org/maven2/xerces/xercesImpl, https://xerces.apache.org/xerces2-j/)
- error-prone annotations (from https://repo1.maven.org/maven2/com/google/errorprone/error_prone_annotations)
-- io.grpc:grpc-alts (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-api (from https://github.com/grpc/grpc-java)
-- io.grpc:grpc-auth (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-context (from https://github.com/grpc/grpc-java)
-- io.grpc:grpc-core (from https://github.com/grpc/grpc-java)
-- io.grpc:grpc-googleapis (from https://github.com/grpc/grpc-java)
-- io.grpc:grpc-grpclb (from https://github.com/grpc/grpc-java)
-- io.grpc:grpc-inprocess (from https://github.com/grpc/grpc-java)
-- io.grpc:grpc-netty-shaded (from https://github.com/grpc/grpc-java)
-- io.grpc:grpc-protobuf (from https://github.com/grpc/grpc-java)
-- io.grpc:grpc-protobuf-lite (from https://github.com/grpc/grpc-java)
-- io.grpc:grpc-services (from https://github.com/grpc/grpc-java)
-- io.grpc:grpc-stub (from https://github.com/grpc/grpc-java)
-- io.grpc:grpc-util (from https://github.com/grpc/grpc-java)
-- io.grpc:grpc-xds (from https://github.com/grpc/grpc-java)
- ion-java (from https://github.com/amzn/ion-java/, https://github.com/amznlabs/ion-java/)
- jackson-databind (from http://github.com/FasterXML/jackson, http://wiki.fasterxml.com/JacksonHome, https://github.com/FasterXML/jackson)
- java-cloudant (from https://cloudant.com)
@@ -292,16 +229,9 @@ The following software have components provided under the terms of this license:
- micrometer-jakarta9 (from https://github.com/micrometer-metrics/micrometer)
- micrometer-observation (from https://github.com/micrometer-metrics/micrometer)
- micrometer-registry-azure-monitor (from https://github.com/micrometer-metrics/micrometer)
-- micrometer-registry-prometheus (from <https://github.com/micrometer-metrics/micrometer>, https://github.com/micrometer-metrics/micrometer)
- org.apiguardian:apiguardian-api (from https://github.com/apiguardian-team/apiguardian)
-- org.conscrypt:conscrypt-openjdk-uber (from https://conscrypt.org/)
- org.opentest4j:opentest4j (from https://github.com/ota4j-team/opentest4j)
- org.xmlunit:xmlunit-core (from http://www.xmlunit.org/, https://www.xmlunit.org/)
-- perfmark:perfmark-api (from https://github.com/perfmark/perfmark)
-- proto-google-cloud-logging-v2 (from https://github.com/googleapis/java-logging/proto-google-cloud-logging-v2, https://repo1.maven.org/maven2/com/google/api/grpc/proto-google-cloud-logging-v2)
-- proto-google-cloud-pubsub-v1 (from https://github.com/googleapis/googleapis, https://github.com/googleapis/java-pubsub/proto-google-cloud-pubsub-v1)
-- proto-google-common-protos (from https://github.com/googleapis/api-client-staging, https://github.com/googleapis/googleapis, https://github.com/googleapis/java-iam/proto-google-common-protos, https://github.com/googleapis/sdk-platform-java)
-- proto-google-iam-v1 (from https://github.com/googleapis/googleapis, https://github.com/googleapis/java-iam/proto-google-iam-v1, https://github.com/googleapis/sdk-platform-java)
- resilience4j (from https://github.com/resilience4j/resilience4j, https://resilience4j.readme.io, ttps://resilience4j.readme.io)
- rest (from https://github.com/elastic/elasticsearch, https://github.com/elastic/elasticsearch.git)
- spring-boot-loader (from https://spring.io/projects/spring-boot)
@@ -320,6 +250,7 @@ The following software have components provided under the terms of this license:
- tomcat-embed-core (from http://tomcat.apache.org/)
- tomcat-embed-el (from http://tomcat.apache.org/, https://tomcat.apache.org/)
- tomcat-embed-websocket (from http://tomcat.apache.org/, https://tomcat.apache.org/)
+- webjars-locator-lite (from https://webjars.org)
- xml-apis (from https://repo1.maven.org/maven2/xml-apis/xml-apis)
========================================================================
@@ -328,14 +259,13 @@ BSD-2-Clause
The following software have components provided under the terms of this license:
- Apache Log4j Core (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core)
-- Hamcrest (from http://hamcrest.org/JavaHamcrest/)
- Hamcrest Core (from http://hamcrest.org/, http://hamcrest.org/JavaHamcrest/, https://repo1.maven.org/maven2/org/hamcrest/hamcrest-core)
- HdrHistogram (from http://hdrhistogram.github.io/HdrHistogram/)
-- Jodd BeanUtil (from http://jodd.org)
-- Jodd Core (from http://jodd.org)
+- Jodd Util (from https://util.jodd.org)
- LatencyUtils (from http://latencyutils.github.io/LatencyUtils/)
+- MinLog (from https://github.com/EsotericSoftware/minlog)
- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils, https://codehaus-plexus.github.io/plexus-utils/, https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-utils)
-- Stax2 API (from http://github.com/FasterXML/stax2-api)
+- ReflectASM (from https://github.com/EsotericSoftware/reflectasm)
- jersey-container-servlet (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet)
- jersey-container-servlet-core (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet-core)
- jersey-core-client (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-client)
@@ -352,44 +282,32 @@ BSD-3-Clause
========================================================================
The following software have components provided under the terms of this license:
-- API Common (from https://github.com/googleapis, https://github.com/googleapis/api-common-java, https://repo1.maven.org/maven2/com/google/api/api-common)
- ASM Commons (from http://asm.ow2.io/, https://repo1.maven.org/maven2/org/ow2/asm/asm-commons)
- ASM Core (from http://asm.ow2.io/, http://asm.ow2.org/)
- AspectJ Weaver (from http://www.aspectj.org, https://www.eclipse.org/aspectj/)
- Class Model for Hk2 (from https://repo1.maven.org/maven2/org/glassfish/hk2/class-model)
- Eclipse Parsson (from https://repo1.maven.org/maven2/org/eclipse/parsson/parsson)
- FindBugs-jsr305 (from http://findbugs.sourceforge.net/)
-- GAX (Google Api eXtensions) for Java (Core) (from https://github.com/googleapis, https://github.com/googleapis/gax-java, https://repo1.maven.org/maven2/com/google/api/gax)
-- GAX (Google Api eXtensions) for Java (HTTP JSON) (from <https://repo1.maven.org/maven2/com/google/api/gax-httpjson>, https://repo1.maven.org/maven2/com/google/api/gax-httpjson)
-- GAX (Google Api eXtensions) for Java (gRPC) (from <https://repo1.maven.org/maven2/com/google/api/gax-grpc>, https://repo1.maven.org/maven2/com/google/api/gax-grpc)
-- Google APIs Client Library for Java (from https://repo1.maven.org/maven2/com/google/api-client/google-api-client)
- Google Auth Library for Java - Credentials (from https://repo1.maven.org/maven2/com/google/auth/google-auth-library-credentials)
- Google Auth Library for Java - OAuth2 HTTP (from https://repo1.maven.org/maven2/com/google/auth/google-auth-library-oauth2-http)
-- Google OAuth Client Library for Java (from https://repo1.maven.org/maven2/com/google/oauth-client/google-oauth-client)
- HK2 Implementation Utilities (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-utils)
- HK2 core module (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-core)
- Hamcrest (from http://hamcrest.org/JavaHamcrest/)
- Hamcrest Core (from http://hamcrest.org/, http://hamcrest.org/JavaHamcrest/, https://repo1.maven.org/maven2/org/hamcrest/hamcrest-core)
-- Jackson module: Afterburner (from http://wiki.fasterxml.com/JacksonHome, https://github.com/FasterXML/jackson-modules-base)
- Jakarta Activation API (from https://github.com/eclipse-ee4j/jaf, https://github.com/jakartaee/jaf-api, https://repo1.maven.org/maven2/jakarta/activation/jakarta.activation-api)
- Jakarta Annotations API (from https://projects.eclipse.org/projects/ee4j.ca)
- Jakarta JSON Processing API (from https://github.com/eclipse-ee4j/jsonp, https://javaee.github.io/jsonp)
- Jakarta Messaging API (from https://projects.eclipse.org/projects/ee4j.jms)
- Jakarta RESTful WS API (from https://github.com/eclipse-ee4j/jaxrs-api, https://repo1.maven.org/maven2/jakarta/ws/rs/jakarta.ws.rs-api)
- Jakarta XML Binding API (from https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api, https://repo1.maven.org/maven2/org/jboss/spec/javax/xml/bind/jboss-jaxb-api_2.3_spec)
-- Microsoft Application Insights Java Agent (from https://github.com/Microsoft/ApplicationInsights-Java)
-- Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java)
-- Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
+- Kryo (from https://repo1.maven.org/maven2/com/esotericsoftware/kryo)
+- MinLog (from https://github.com/EsotericSoftware/minlog)
- Netty/Codec/HTTP (from https://repo1.maven.org/maven2/io/netty/netty-codec-http)
- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils, https://codehaus-plexus.github.io/plexus-utils/, https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-utils)
-- Protocol Buffer Java API (from http://code.google.com/p/protobuf, https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java)
-- Protocol Buffers [Util] (from https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java-util)
-- RE2/J (from http://github.com/google/re2j)
-- Redisson (from http://redisson.org)
+- Redisson (from http://redisson.org, https://redisson.pro)
+- ReflectASM (from https://github.com/EsotericSoftware/reflectasm)
- ServiceLocator Default Implementation (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-locator)
-- Shaded Protobuf (from https://repo1.maven.org/maven2/io/prometheus/prometheus-metrics-shaded-protobuf)
- Spring Core (from http://www.springframework.org, https://github.com/spring-projects/spring-framework, https://repo1.maven.org/maven2/org/springframework/spring-core)
-- ThreeTen backport (from https://github.com/ThreeTen/threetenbp, https://www.threeten.org/threetenbp)
- aopalliance-repackaged (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/aopalliance-repackaged)
- jersey-container-servlet (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet)
- jersey-container-servlet-core (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet-core)
@@ -404,15 +322,6 @@ The following software have components provided under the terms of this license:
- jts-core (from https://repo1.maven.org/maven2/org/locationtech/jts/jts-core)
- org.locationtech.jts.io:jts-io-common (from https://repo1.maven.org/maven2/org/locationtech/jts/io/jts-io-common)
-========================================================================
-Beerware
-========================================================================
-The following software have components provided under the terms of this license:
-
-- Microsoft Application Insights Java Agent (from https://github.com/Microsoft/ApplicationInsights-Java)
-- Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java)
-- Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
-
========================================================================
BouncyCastle
========================================================================
@@ -420,7 +329,6 @@ The following software have components provided under the terms of this license:
- Bouncy Castle PKIX, CMS, EAC, TSP, PKCS, OCSP, CMP, and CRMF APIs (from http://www.bouncycastle.org/java.html, https://www.bouncycastle.org/java.html)
- Bouncy Castle Provider (from http://www.bouncycastle.org/java.html, https://www.bouncycastle.org/java.html)
-- Microsoft Azure client library for Identity (from https://github.com/Azure/azure-sdk-for-java)
========================================================================
CC-BY-2.5
@@ -430,15 +338,6 @@ The following software have components provided under the terms of this license:
- FindBugs-jsr305 (from http://findbugs.sourceforge.net/)
- MongoDB Java Driver (from http://mongodb.org/, http://www.mongodb.org, https://www.mongodb.com/)
-========================================================================
-CC-BY-4.0
-========================================================================
-The following software have components provided under the terms of this license:
-
-- Microsoft Application Insights Java Agent (from https://github.com/Microsoft/ApplicationInsights-Java)
-- Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java)
-- Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
-
========================================================================
CC0-1.0
========================================================================
@@ -446,10 +345,10 @@ The following software have components provided under the terms of this license:
- Guava: Google Core Libraries for Java (from http://code.google.com/p/guava-libraries, https://github.com/google/guava, https://repo1.maven.org/maven2/com/google/guava/guava)
- HdrHistogram (from http://hdrhistogram.github.io/HdrHistogram/)
-- Hibernate Validator (from http://hibernate.org/validator, https://repo1.maven.org/maven2/org/hibernate/hibernate-validator, https://repo1.maven.org/maven2/org/hibernate/validator/hibernate-validator)
+- Hibernate Validator (from https://hibernate.org/validator, https://repo1.maven.org/maven2/org/hibernate/hibernate-validator, https://repo1.maven.org/maven2/org/hibernate/validator/hibernate-validator)
- LatencyUtils (from http://latencyutils.github.io/LatencyUtils/)
- Netty/Common (from https://repo1.maven.org/maven2/io/netty/netty-common)
-- RabbitMQ Java Client (from http://www.rabbitmq.com, https://www.rabbitmq.com)
+- Redisson (from http://redisson.org, https://redisson.pro)
- jersey-container-servlet (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet)
- jersey-container-servlet-core (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet-core)
- jersey-core-client (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-client)
@@ -468,7 +367,6 @@ CDDL-1.0
The following software have components provided under the terms of this license:
- Apache Log4j Core (from https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core)
-- JavaBeans Activation Framework (from <http://java.sun.com/javase/technologies/desktop/javabeans/jaf/index.jsp>, http://java.sun.com/javase/technologies/desktop/javabeans/jaf/index.jsp, https://repo1.maven.org/maven2/com/sun/activation/javax.activation)
- javax.annotation API (from http://jcp.org/en/jsr/detail?id=250)
========================================================================
@@ -478,8 +376,6 @@ The following software have components provided under the terms of this license:
- HK2 Spring Bridge (from https://repo1.maven.org/maven2/org/glassfish/hk2/spring-bridge)
- HK2 module of HK2 itself (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2)
-- JavaBeans Activation Framework (from <http://java.sun.com/javase/technologies/desktop/javabeans/jaf/index.jsp>, http://java.sun.com/javase/technologies/desktop/javabeans/jaf/index.jsp, https://repo1.maven.org/maven2/com/sun/activation/javax.activation)
-- JavaMail API (from https://repo1.maven.org/maven2/com/sun/mail/javax.mail)
- Run Level Service (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-runlevel)
- javax.annotation API (from http://jcp.org/en/jsr/detail?id=250)
- tomcat-embed-core (from http://tomcat.apache.org/)
@@ -497,6 +393,7 @@ EDL-1.0
========================================================================
The following software have components provided under the terms of this license:
+- Jakarta Activation (from https://repo1.maven.org/maven2/com/sun/activation/jakarta.activation)
- Jakarta Activation API (from https://github.com/eclipse-ee4j/jaf, https://github.com/jakartaee/jaf-api, https://repo1.maven.org/maven2/jakarta/activation/jakarta.activation-api)
========================================================================
@@ -504,15 +401,13 @@ EPL-1.0
========================================================================
The following software have components provided under the terms of this license:
+- JUnit Vintage Engine (from http://junit.org/junit5/, https://junit.org/junit5/)
- Jetty :: Utilities (from http://jetty.mortbay.org, http://www.eclipse.org/jetty, https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-util, https://repo1.maven.org/maven2/org/mortbay/jetty/jetty-util)
- Jetty Server (from http://jetty.mortbay.org, https://repo1.maven.org/maven2/org/mortbay/jetty/jetty)
- Logback Contrib :: JSON :: Classic (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-json-classic)
- Logback Contrib :: JSON :: Core (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-json-core)
- Logback Contrib :: Jackson (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-jackson)
- Logback Core Module (from http://logback.qos.ch, https://repo1.maven.org/maven2/ch/qos/logback/logback-core)
-- Microsoft Application Insights Java Agent (from https://github.com/Microsoft/ApplicationInsights-Java)
-- Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java)
-- Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
- jts-core (from https://repo1.maven.org/maven2/org/locationtech/jts/jts-core)
- org.locationtech.jts.io:jts-io-common (from https://repo1.maven.org/maven2/org/locationtech/jts/io/jts-io-common)
@@ -527,6 +422,7 @@ The following software have components provided under the terms of this license:
- Eclipse Parsson (from https://repo1.maven.org/maven2/org/eclipse/parsson/parsson)
- HK2 API module (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-api)
- HK2 Implementation Utilities (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-utils)
+- HK2 OSGi resource locator (from https://repo1.maven.org/maven2/org/glassfish/hk2/osgi-resource-locator)
- HK2 core module (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-core)
- JUnit Jupiter (Aggregator) (from https://junit.org/junit5/)
- JUnit Jupiter API (from http://junit.org/junit5/, https://junit.org/junit5/)
@@ -534,15 +430,16 @@ The following software have components provided under the terms of this license:
- JUnit Jupiter Params (from http://junit.org/junit5/, https://junit.org/junit5/)
- JUnit Platform Commons (from http://junit.org/junit5/, https://junit.org/junit5/)
- JUnit Platform Engine API (from http://junit.org/junit5/, https://junit.org/junit5/)
+- JUnit Vintage Engine (from http://junit.org/junit5/, https://junit.org/junit5/)
- Jakarta Activation API (from https://github.com/eclipse-ee4j/jaf, https://github.com/jakartaee/jaf-api, https://repo1.maven.org/maven2/jakarta/activation/jakarta.activation-api)
- Jakarta Annotations API (from https://projects.eclipse.org/projects/ee4j.ca)
+- Jakarta Dependency Injection (from https://github.com/eclipse-ee4j/injection-api)
- Jakarta JSON Processing API (from https://github.com/eclipse-ee4j/jsonp, https://javaee.github.io/jsonp)
- Jakarta Messaging API (from https://projects.eclipse.org/projects/ee4j.jms)
- Jakarta RESTful WS API (from https://github.com/eclipse-ee4j/jaxrs-api, https://repo1.maven.org/maven2/jakarta/ws/rs/jakarta.ws.rs-api)
- Jakarta Servlet (from https://projects.eclipse.org/projects/ee4j.servlet)
- Jakarta Validation API (from https://beanvalidation.org)
- Jakarta XML Binding API (from https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api, https://repo1.maven.org/maven2/org/jboss/spec/javax/xml/bind/jboss-jaxb-api_2.3_spec)
-- OSGi resource locator (from https://repo1.maven.org/maven2/org/glassfish/hk2/osgi-resource-locator)
- ServiceLocator Default Implementation (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-locator)
- aopalliance-repackaged (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/aopalliance-repackaged)
- jersey-container-servlet (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet)
@@ -572,17 +469,17 @@ The following software have components provided under the terms of this license:
- Eclipse Expressly (from https://projects.eclipse.org/projects/ee4j.expressly)
- Eclipse Parsson (from https://repo1.maven.org/maven2/org/eclipse/parsson/parsson)
- HK2 Implementation Utilities (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-utils)
+- HK2 OSGi resource locator (from https://repo1.maven.org/maven2/org/glassfish/hk2/osgi-resource-locator)
- HK2 core module (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-core)
- Jakarta Activation API (from https://github.com/eclipse-ee4j/jaf, https://github.com/jakartaee/jaf-api, https://repo1.maven.org/maven2/jakarta/activation/jakarta.activation-api)
- Jakarta Annotations API (from https://projects.eclipse.org/projects/ee4j.ca)
+- Jakarta Dependency Injection (from https://github.com/eclipse-ee4j/injection-api)
- Jakarta JSON Processing API (from https://github.com/eclipse-ee4j/jsonp, https://javaee.github.io/jsonp)
- Jakarta Messaging API (from https://projects.eclipse.org/projects/ee4j.jms)
- Jakarta RESTful WS API (from https://github.com/eclipse-ee4j/jaxrs-api, https://repo1.maven.org/maven2/jakarta/ws/rs/jakarta.ws.rs-api)
- Jakarta Servlet (from https://projects.eclipse.org/projects/ee4j.servlet)
- Jakarta Validation API (from https://beanvalidation.org)
- Jakarta XML Binding API (from https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api, https://repo1.maven.org/maven2/org/jboss/spec/javax/xml/bind/jboss-jaxb-api_2.3_spec)
-- JavaBeans Activation Framework (from <http://java.sun.com/javase/technologies/desktop/javabeans/jaf/index.jsp>, http://java.sun.com/javase/technologies/desktop/javabeans/jaf/index.jsp, https://repo1.maven.org/maven2/com/sun/activation/javax.activation)
-- OSGi resource locator (from https://repo1.maven.org/maven2/org/glassfish/hk2/osgi-resource-locator)
- ServiceLocator Default Implementation (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-locator)
- aopalliance-repackaged (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/aopalliance-repackaged)
- javax.annotation API (from http://jcp.org/en/jsr/detail?id=250)
@@ -616,9 +513,6 @@ The following software have components provided under the terms of this license:
- Logback Contrib :: JSON :: Core (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-json-core)
- Logback Contrib :: Jackson (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-jackson)
- Logback Core Module (from http://logback.qos.ch, https://repo1.maven.org/maven2/ch/qos/logback/logback-core)
-- Microsoft Application Insights Java Agent (from https://github.com/Microsoft/ApplicationInsights-Java)
-- Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java)
-- Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
========================================================================
LGPL-2.1-or-later
@@ -634,7 +528,6 @@ MIT
========================================================================
The following software have components provided under the terms of this license:
-- Animal Sniffer Annotations (from https://repo1.maven.org/maven2/org/codehaus/mojo/animal-sniffer-annotations)
- Azure Java Client Authentication Library for AutoRest (from https://github.com/Azure/autorest-clientruntime-for-java)
- Azure Java Client Runtime for ARM (from https://github.com/Azure/autorest-clientruntime-for-java)
- Azure Java Client Runtime for AutoRest (from https://github.com/Azure/autorest-clientruntime-for-java)
@@ -645,12 +538,13 @@ The following software have components provided under the terms of this license:
- Jackson-core (from http://wiki.fasterxml.com/JacksonHome, https://github.com/FasterXML/jackson-core)
- Java Client Runtime for AutoRest (from https://github.com/Azure/autorest-clientruntime-for-java)
- Java JWT (from http://www.jwt.io, https://github.com/auth0/java-jwt)
+- Lettuce (from http://github.com/lettuce-io/lettuce-core, http://github.com/mp911de/lettuce/wiki, https://github.com/lettuce-io/lettuce-core/wiki)
- Microsoft Application Insights Java Agent (from https://github.com/Microsoft/ApplicationInsights-Java)
-- Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java)
-- Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
+- Microsoft Azure Identity Brokered Authentication Library (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure Java Core AMQP Library (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure Java Core Library (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure Java JSON Library (from https://github.com/Azure/azure-sdk-for-java)
+- Microsoft Azure Java XML Library (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure Management Java Core Library (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure Netty HTTP Client Library (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure SDK annotations (from https://github.com/Microsoft/java-api-annotations)
@@ -687,6 +581,7 @@ The following software have components provided under the terms of this license:
- Spring Data for Azure Cosmos DB SQL API (from https://github.com/Azure/azure-sdk-for-java/tree/master/sdk/cosmos/azure-spring-data-cosmos, https://github.com/Azure/azure-sdk-for-java/tree/master/sdk/spring/azure-spring-data-cosmos)
- System Stubs Core (from https://repo1.maven.org/maven2/uk/org/webcompere/system-stubs-core)
- adal4j (from https://github.com/AzureAD/azure-activedirectory-library-for-java)
+- javamsalruntime (from https://github.com/AzureAD/microsoft-authentication-library-for-cpp)
- jersey-container-servlet (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet)
- jersey-container-servlet-core (from https://repo1.maven.org/maven2/org/glassfish/jersey/containers/jersey-container-servlet-core)
- jersey-core-client (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-client)
@@ -702,14 +597,9 @@ The following software have components provided under the terms of this license:
- mockito-inline (from http://mockito.org, https://github.com/mockito/mockito)
- mockito-junit-jupiter (from https://github.com/mockito/mockito)
- msal4j (from https://github.com/AzureAD/microsoft-authentication-library-for-java)
+- msal4j-brokers (from https://github.com/AzureAD/microsoft-authentication-library-for-java)
- msal4j-persistence-extension (from https://github.com/AzureAD/microsoft-authentication-extensions-for-java, https://github.com/AzureAD/microsoft-authentication-library-for-java)
-
-========================================================================
-MPL-2.0
-========================================================================
-The following software have components provided under the terms of this license:
-
-- RabbitMQ Java Client (from http://www.rabbitmq.com, https://www.rabbitmq.com)
+- webjars-locator-lite (from https://webjars.org)
========================================================================
NCSA
diff --git a/devops/aws/chart/.helmignore b/devops/aws/chart/.helmignore
deleted file mode 100644
index ddd3d572395f8d2b36dbfce78b7d2218f37567f3..0000000000000000000000000000000000000000
--- a/devops/aws/chart/.helmignore
+++ /dev/null
@@ -1,40 +0,0 @@
-# Copyright © Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Patterns to ignore when building packages.
-# This supports shell glob matching, relative path matching, and
-# negation (prefixed with !). Only one pattern per line.
-.DS_Store
-# Common VCS dirs
-.git/
-.gitignore
-.bzr/
-.bzrignore
-.hg/
-.hgignore
-.svn/
-# Common backup files
-*.swp
-*.bak
-*.tmp
-*.orig
-*~
-# Various IDEs
-.project
-.idea/
-*.tmproj
-.vscode/
-# Build files
-*.html
-build-info.json
\ No newline at end of file
diff --git a/devops/aws/chart/Chart.yaml b/devops/aws/chart/Chart.yaml
deleted file mode 100644
index 70e19a55f27a868045da33a2cf2500cb37ba3968..0000000000000000000000000000000000000000
--- a/devops/aws/chart/Chart.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-apiVersion: v2
-name: "os-indexer"
-version: __CHART_VERSION__
-description: Indexer Helm Chart for Kubernetes
-type: application
-appVersion: __VERSION__
-dependencies:
- - name: osdu-aws-lib
- version: ~0
- repository: __HELM_REPO__
-deprecated: false
diff --git a/devops/aws/chart/README.md b/devops/aws/chart/README.md
deleted file mode 100644
index 7035aa7f5a1c3f65a3bfe135a9841678e35784e2..0000000000000000000000000000000000000000
--- a/devops/aws/chart/README.md
+++ /dev/null
@@ -1,92 +0,0 @@
-# OSDU on AWS Service Helm Chart
-
-## Introduction
-The following document outlines how to deploy and update the service application onto an existing Kubernetes deployment using the [Helm](https://helm.sh) package manager.
-
-## Prerequisites
-The below software must be installed before continuing:
-* [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html)
-* [kubectl](https://kubernetes.io/docs/tasks/tools/)
-* [Helm](https://helm.sh/docs/intro/install/)
-* [Helm S3 Plugin](https://github.com/hypnoglow/helm-s3)
-
-Additionally, an OSDU on AWS environment must be deployed.
-
-## Installation/Updating
-To install or update the service application by executing the following command in the CHART folder:
-
-```bash
-helm upgrade [RELEASE_NAME] . -i -n [NAMESPACE]
-```
-
-To observe the Kubernetes resources before deploying them using the command:
-```bash
-helm upgrade [RELEASE_NAME] . -i -n [NAMESPACE] --dry-run --debug
-```
-
-To observe the history of the current release, use the following command:
-```bash
-helm history [RELEASE_NAME] -n [NAMESPACE]
-```
-
-To revert to a previous release, use the following command:
-```bash
-helm rollback [RELEASE] [REVISION] -n [NAMESPACE]
-```
-
-Refer to the [Helm CLI guide](https://helm.sh/docs/helm/helm/) for additional commands.
-
-## Customizing the Deployment
-It is possible to modify the default values specified in the **values.yaml** file using the --set option. The below parameters can be modified by advanced users to customize the deployment configuration:
-
-### Globals
-Global Helm values apply to all services within the parent chart deployment. Global values will not override service defaults or locally set values.
-| Name | Example Value | Description | Type | Required |
-| --- | ------------- | ----------- | ---- | -------- |
-| `global.allowOrigins` | `{http://localhost,https://www.osdu.aws}` | A list of domains that are permitted by CORS policy. An empty list permits all origins. | array[str] | no |
-| `global.metricsServerAddress` | `http://prometheus-service.monitoring:8080` | The URL of the accessible metrics server for evaluating autoscaling decisions. | str | no |
-| `global.podAnnotations` | `podAnnotations.version=v1.0.0` | Additional annotations on the service pod | dict | no |
-| `global.podSecurityContext` | `fsGroup: 1337` | The [pod security context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) apply to all containers in the pod | str | no |
-| `global.securityContext` | `fsGroup: 1337` | The security context is the container specific security context. Will inherit [pod security context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) | str | no |
-
-### Local
-Local Helm values apply to specific services. Local Helm values will override global values and default presets.
-| Name | Example Value | Description | Type | Required |
-| --- | ------------- | ----------- | ---- | -------- |
-| `image` | `registry.repo.osdu.aws/service:0.21.0` | The custom image of the service deployment. | str | no |
-| `imagePullPolicy` | `IfNotPresent` | The service image pull policy | str | no |
-| `resources.limits.cpu` | `500M` | [CPU resource management limit for pods](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) | str | no |
-| `resources.limits.memory` | `900M` | [Memory resource management limit for pods](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) | str | no |
-| `resources.requests.cpu` | `500M` | [MemoCPUry resource management for pods](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) | str | no |
-| `resources.requests.memory` | `900M` | [Memory resource management for pods](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) | str | no |
-| `replicaCount` | `1` | The number of pod replicas to be initially deployed | int | no |
-| `autoscaling.minReplicas` | `1` | Minimum number of pod replicas | int | no |
-| `autoscaling.maxReplicas` | `100` | Maximum number of pod replicas | int | no |
-| `autoscaling.targetCPUUtilizationPercentage` | `80` | CPU utilization target | int | no |
-| `autoscaling.targetMemoryUtilizationPercentage` | `80` | Memory utilization target | int | no |
-| `autoscaling.ServiceRequestCountThreshold` | `25` | The number of requests per second threshold averaged over a minute to trigger a scaling event. | int | no |
-| `autoscaling.ServiceRequestDurationAverage` | `300` | The response time measured in miliseconds averaged over 3 minutes to trigger a scaling event. | int | no |
-| `autoscaling.coolDownPeriod` | `120` | The period to wait after the last trigger reported active before scaling the resource back to 0. Managed by Keda. | int | no |
-| `autoscaling.pollingInterval` | `1` | This is the interval to check each trigger on. | int | no |
-| `livenessProbe.failureThreshold` | `3` | [Kubernetes probe configuration](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes). | int | no |
-| `livenessProbe.periodSeconds` | `10` | [Kubernetes probe configuration](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes). | int | no |
-| `livenessProbe.successThreshold` | `1` | [Kubernetes probe configuration](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes). | int | no |
-| `livenessProbe.timeoutSeconds` | `1` | [Kubernetes probe configuration](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes). | int | no |
-| `readinessProbe.initialDelaySeconds` | `30` | [Kubernetes probe configuration](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes). | int | no |
-| `readinessProbe.failureThreshold` | `3` | [Kubernetes probe configuration](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes). | int | no |
-| `readinessProbe.periodSeconds` | `10` | [Kubernetes probe configuration](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes). | int | no |
-| `readinessProbe.successThreshold` | `1` | [Kubernetes probe configuration](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes). | int | no |
-| `readinessProbe.timeoutSeconds` | `1` | [Kubernetes probe configuration](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes). | int | no |
-| `maxPendingRequests` | `10000` | Maximum number of requests that will be queued while waiting for a ready connection pool connection. Used for circuit breaking. Used for [circuit breaking.](https://istio.io/latest/docs/tasks/traffic-management/circuit-breaking/). | int | no |
-| `maxRequestsPerConnection` | `100` | Maximum number of active requests to a destination. Used for [circuit breaking.](https://istio.io/latest/docs/tasks/traffic-management/circuit-breaking/). | int | no |
-| `maxConnections` | `0` | Maximum number of HTTP1 /TCP connections to a destination host. Used for [circuit breaking.](https://istio.io/latest/docs/tasks/traffic-management/circuit-breaking/). | int | no |
-| `podAnnotations` | `podAnnotations.version=v1.0.0` | Additional annotations on the service pod | dict | no |
-| `podSecurityContext` | `fsGroup: 1337` | The [pod security context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) apply to all containers in the pod | str | no |
-| `securityContext` | `fsGroup: 1337` | The security context is the container specific security context. Will inherit [pod security context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) | str | no |
-
-## Uninstalling the Chart
-To uninstall the helm release:
-
-```bash
-helm uninstall [RELEASE] -n [NAMESPACE] --keep-history
-```
\ No newline at end of file
diff --git a/devops/aws/chart/templates/NOTES.txt b/devops/aws/chart/templates/NOTES.txt
deleted file mode 100644
index f15afa919efb6cf0dc53f0d70283f9ae0dbbcbbb..0000000000000000000000000000000000000000
--- a/devops/aws/chart/templates/NOTES.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-{{ .Chart.Name }} deployed to {{ .Release.Namespace }} namespace
-{{- if (.Values.service).apiPath }}
-Application URL available at path {{ .Values.service.apiPath }}
-{{- end }}
\ No newline at end of file
diff --git a/devops/aws/chart/templates/authorizationpolicy.yaml b/devops/aws/chart/templates/authorizationpolicy.yaml
deleted file mode 100644
index 6ccd988de6762ca11448628b0ac5abb697c83cb1..0000000000000000000000000000000000000000
--- a/devops/aws/chart/templates/authorizationpolicy.yaml
+++ /dev/null
@@ -1,19 +0,0 @@
-{{- /* Copyright 2020 Amazon Web Services
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.*/}}
-
-{{- template "common.authzpolicy" (list . "os-indexer.authzpolicy") -}}
-{{- define "os-indexer.authzpolicy" -}}
-## Define overrides for the service authorization policy resource here
-{{- end -}}
-
diff --git a/devops/aws/chart/templates/deployment.yaml b/devops/aws/chart/templates/deployment.yaml
deleted file mode 100644
index 2f9aeb488289f705a913bb7ae23c8ec119defd40..0000000000000000000000000000000000000000
--- a/devops/aws/chart/templates/deployment.yaml
+++ /dev/null
@@ -1,18 +0,0 @@
-{{- /* Copyright 2020 Amazon Web Services
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.*/}}
-
-{{- template "common.deployment" (list . "os-indexer.deployment") -}}
-{{- define "os-indexer.deployment" -}}
-## Define overrides for the service deployment resource here
-{{- end -}}
\ No newline at end of file
diff --git a/devops/aws/chart/templates/destinationrule.yaml b/devops/aws/chart/templates/destinationrule.yaml
deleted file mode 100644
index 423bbac05f296987b905409e838f7f929ffafc9d..0000000000000000000000000000000000000000
--- a/devops/aws/chart/templates/destinationrule.yaml
+++ /dev/null
@@ -1,18 +0,0 @@
-{{- /* Copyright 2020 Amazon Web Services
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.*/}}
-
-{{- template "common.destinationrule" (list . "os-indexer.destinationrule") -}}
-{{- define "os-indexer.destinationrule" -}}
-## Define overrides for the service destination rule resource here
-{{- end -}}
\ No newline at end of file
diff --git a/devops/aws/chart/templates/scaledobject.yaml b/devops/aws/chart/templates/scaledobject.yaml
deleted file mode 100644
index e69fc79757e3d044dc42dd01ec46d2e95a689986..0000000000000000000000000000000000000000
--- a/devops/aws/chart/templates/scaledobject.yaml
+++ /dev/null
@@ -1,18 +0,0 @@
-{{- /* Copyright 2020 Amazon Web Services
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.*/}}
-
-{{- template "common.scaledobject" (list . "os-indexer.scaledobject") -}}
-{{- define "os-indexer.scaledobject" -}}
-## Define overrides for the service's scaled object resource here
-{{- end -}}
\ No newline at end of file
diff --git a/devops/aws/chart/templates/service.yaml b/devops/aws/chart/templates/service.yaml
deleted file mode 100644
index 1da8cc565560b605fc04bcff04ca619f30b47d23..0000000000000000000000000000000000000000
--- a/devops/aws/chart/templates/service.yaml
+++ /dev/null
@@ -1,18 +0,0 @@
-{{- /* Copyright 2020 Amazon Web Services
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.*/}}
-
-{{- template "common.service" (list . "os-indexer.service") -}}
-{{- define "os-indexer.service" -}}
-## Define overrides for the service resource here
-{{- end -}}
\ No newline at end of file
diff --git a/devops/aws/chart/templates/serviceaccount.yaml b/devops/aws/chart/templates/serviceaccount.yaml
deleted file mode 100644
index 9c67b27ea9a073f0e2e8d08419b0b1672985fa1e..0000000000000000000000000000000000000000
--- a/devops/aws/chart/templates/serviceaccount.yaml
+++ /dev/null
@@ -1,18 +0,0 @@
-{{- /* Copyright 2020 Amazon Web Services
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.*/}}
-
-{{- template "common.serviceaccount" (list . "os-indexer.serviceaccount") -}}
-{{- define "os-indexer.serviceaccount" -}}
-## Define overrides for the service's service account resource here
-{{- end -}}
\ No newline at end of file
diff --git a/devops/aws/chart/templates/serviceparameters.yaml b/devops/aws/chart/templates/serviceparameters.yaml
deleted file mode 100644
index 7e06462a913006cbece0ddbbbfea29692d96e585..0000000000000000000000000000000000000000
--- a/devops/aws/chart/templates/serviceparameters.yaml
+++ /dev/null
@@ -1,18 +0,0 @@
-{{- /* Copyright 2020 Amazon Web Services
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.*/}}
-
-{{- template "common.serviceparameters" (list . "os-indexer.serviceparameters") -}}
-{{- define "os-indexer.serviceparameters" -}}
-## Define overrides for the service parameters here
-{{- end -}}
\ No newline at end of file
diff --git a/devops/aws/chart/templates/virtualservice.yaml b/devops/aws/chart/templates/virtualservice.yaml
deleted file mode 100644
index f33da9e51c609eedb56c7c4caca1a4cc223ef15a..0000000000000000000000000000000000000000
--- a/devops/aws/chart/templates/virtualservice.yaml
+++ /dev/null
@@ -1,18 +0,0 @@
-{{- /* Copyright 2020 Amazon Web Services
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.*/}}
-
-{{- template "common.virtualservice" (list . "os-indexer.virtualservice") -}}
-{{- define "os-indexer.virtualservice" -}}
-## Define overrides for the service's virtual service resource here
-{{- end -}}
\ No newline at end of file
diff --git a/devops/aws/chart/values.schema.json b/devops/aws/chart/values.schema.json
deleted file mode 100644
index 105edd0d286f491b2bec56f07a65afa6f14bc047..0000000000000000000000000000000000000000
--- a/devops/aws/chart/values.schema.json
+++ /dev/null
@@ -1,264 +0,0 @@
-{
- "$schema": "osdu-aws-services-public",
- "type": "object",
- "default": {},
- "required": [
- "image",
- "service",
- "serviceAccountRole"
- ],
- "properties": {
- "image": {
- "type": "string",
- "title": "The service container image URL"
- },
- "imagePullPolicy": {
- "type": "string",
- "pattern": "^(Always|Never|IfNotPresent)$",
- "default": "Always"
- },
- "service": {
- "type": "object",
- "required": [
- "apiPath"
- ],
- "properties": {
- "type": {
- "type": "string",
- "default": "ClusterIP"
- },
- "port": {
- "type": "integer",
- "default": 8080,
- "title": "The port the service is hosted on",
- "examples": [
- 8080,
- 443,
- 80
- ]
- },
- "apiPath": {
- "type": "string",
- "title": "The main API path",
- "examples": [
- "/api/service/v1/"
- ]
- },
- "readinessProbeHttpPath": {
- "type": "string",
- "examples": [
- "/api/service/v1/actuator/health"
- ]
- },
- "livenessProbeHttpPath": {
- "type": "string",
- "examples": [
- "/api/service/v1/actuator/health"
- ]
- },
- "startupProbeHttpPath": {
- "type": "string",
- "examples": [
- "/api/service/v1/actuator/health"
- ]
- }
- },
- "examples": [{
- "type": "ClusterIP",
- "port": 8080,
- "apiPath": "/api/service/v1/",
- "readinessProbeHttpPath": "/api/service/v1/actuator/health",
- "livenessProbeHttpPath": "/api/service/v1/actuator/health",
- "startupProbeHttpPath": "/api/service/v1/actuator/health"
- }]
- },
- "serviceParameters": {
- "type": "array",
- "title": "The secret parameters consumed by the service",
- "items": {
- "type": "object",
- "required": [
- "objectName",
- "objectType",
- "objectAlias"
- ],
- "properties": {
- "objectName": {
- "type": "string",
- "title": "The SSM or secrets parameter name",
- "examples": [
- "/osdu/resource-prefix/redis/redisauthtoken",
- "/osdu/resource-prefix/redis-core/end-point"
- ]
- },
- "objectType": {
- "type": "string",
- "title": "The type of parameter",
- "pattern": "^(secretsmanager|ssmparameter|)$"
- },
- "objectAlias": {
- "type": "string",
- "title": "The name of the environment variable consumed by the service",
- "examples": [
- "CACHE_CLUSTER_KEY",
- "CACHE_CLUSTER_ENDPOINT"
- ]
- }
- },
- "examples": [{
- "objectName": "/osdu/resource-prefix/redis/redisauthtoken",
- "objectType": "secretsmanager",
- "objectAlias": "CACHE_CLUSTER_KEY"
- },
- {
- "objectName": "/osdu/resource-prefix/redis-core/end-point",
- "objectType": "ssmparameter",
- "objectAlias": "CACHE_CLUSTER_ENDPOINT"
- }]
- }
- },
- "environmentVariables": {
- "type": "array",
- "title": "The environment variables consumed by the service",
- "items": {
- "type": "object",
- "title": "An environment variable definition",
- "required": [
- "name",
- "value"
- ],
- "properties": {
- "name": {
- "type": "string",
- "title": "The name of the environment variable consumed by the service",
- "examples": [
- "APPLICATION_PORT",
- "AWS_REGION"
- ]
- },
- "value": {
- "type": "string",
- "title": "The value of the environment variable"
- }
- },
- "examples": [{
- "name": "APPLICATION_PORT",
- "value": "8080"
- },
- {
- "name": "AWS_REGION",
- "value": "us-east-1"
- }]
- }
- },
- "podAnnotations": {
- "type": "object",
- "default": {}
- },
- "replicaCount": {
- "type": "integer",
- "default": 1,
- "title": "The number of pod replicas"
- },
- "maxConnections": {
- "type": "integer",
- "default": 200,
- "title": "The max number of service connections"
- },
- "resources": {
- "type": "object"
- },
- "autoscaling": {
- "type": "object"
- },
- "serviceAccountRole": {
- "type": "string",
- "default": "",
- "title": "The service account IAM role"
- },
- "cors": {
- "type": "object",
- "default": {},
- "title": "The cors Schema",
- "required": [
- "maxAge",
- "allowCredentials",
- "allowMethods",
- "allowHeaders"
- ],
- "properties": {
- "maxAge": {
- "type": "string",
- "default": "60m",
- "title": "The max age of the request"
- },
- "allowCredentials": {
- "type": "boolean",
- "default": true,
- "title": "The allowCredentials Schema"
- },
- "allowMethods": {
- "type": "array",
- "title": "A list of supported API request methods",
- "items": {
- "type": "string",
- "title": "Supported API request methods",
- "pattern": "^(GET|POST|PUT|PATCH|OPTIONS|DELETE)$"
- },
- "examples": [
- ["POST",
- "GET",
- "PATCH",
- "DELETE"
- ]
- ]
- },
- "allowHeaders": {
- "type": "array",
- "title": "A list of supported headers",
- "items": {
- "type": "string"
- },
- "examples": [
- ["Authorization",
- "Data-Partition-Id",
- "Correlation-Id",
- "Content-Type"
- ]
- ]
- }
- },
- "examples": [{
- "maxAge": "60m",
- "allowCredentials": true,
- "allowMethods": [
- "POST",
- "GET",
- "PATCH",
- "DELETE"
- ],
- "allowHeaders": [
- "Authorization",
- "Data-Partition-Id",
- "Correlation-Id",
- "Content-Type"
- ]
- }]
- },
- "securityContext": {
- "type": "object",
- "default": {}
- },
- "allowedPrincipals": {
- "type": "array",
- "title": "A list of allowed service principals",
- "items": {
- "type": "string",
- "title": "Allowed principal",
- "examples": [
- "cluster.local/ns/istio-system/sa/istio-ingressgateway"
- ]
- }
- }
- }
-}
diff --git a/devops/aws/chart/values.yaml b/devops/aws/chart/values.yaml
deleted file mode 100644
index 39693d046d96742733b37451d383e494eaac8d0a..0000000000000000000000000000000000000000
--- a/devops/aws/chart/values.yaml
+++ /dev/null
@@ -1,16 +0,0 @@
-# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Service Config
-image: __CONTAINER__
diff --git a/devops/core-plus/deploy/templates/configmap.yaml b/devops/core-plus/deploy/templates/configmap.yaml
index 7a23a68af03cdca043983b13b5d9876699a674d2..7cb7c68773321856a844eb29346d647386e8dd80 100644
--- a/devops/core-plus/deploy/templates/configmap.yaml
+++ b/devops/core-plus/deploy/templates/configmap.yaml
@@ -21,4 +21,3 @@ data:
SCHEMA_BASE_HOST: {{ .Values.data.schemaHost | quote }}
SECURITY_HTTPS_CERTIFICATE_TRUST: {{ .Values.data.securityHttpsCertificateTrust | quote }}
STORAGE_HOST: {{ .Values.data.storageHost | quote }}
- SEARCH_BASE_HOST: {{ .Values.data.searchHost | quote }}
diff --git a/devops/core-plus/deploy/values.yaml b/devops/core-plus/deploy/values.yaml
index cbfdd4fa5447e322dc79755805afc5f7804f5200..893118a04a8bc016d5d36d87c2068137b3e235ab 100644
--- a/devops/core-plus/deploy/values.yaml
+++ b/devops/core-plus/deploy/values.yaml
@@ -14,7 +14,6 @@ data:
schemaHost: "http://schema"
securityHttpsCertificateTrust: "true"
storageHost: "http://storage"
- searchHost: "http://search"
# Deployment
requestsCpu: "35m"
requestsMemory: "640Mi"
diff --git a/devops/gc/deploy/README.md b/devops/gc/deploy/README.md
index 9fe6daf2f0edef9c1d4e9850f61a64e47f5c52c7..711c492e38710427201f2a5b070563ebdaab8370 100644
--- a/devops/gc/deploy/README.md
+++ b/devops/gc/deploy/README.md
@@ -27,58 +27,85 @@ First you need to set variables in **values.yaml** file using any code editor. S
### Global variables
-| Name | Description | Type | Default |Required |
-|------|-------------|------|---------|---------|
-**global.domain** | your domain for the external endpoint, ex `example.com` | string | - | yes
-**global.onPremEnabled** | whether on-prem is enabled | boolean | false | yes
-**global.limitsEnabled** | whether CPU and memory limits are enabled | boolean | true | yes
+| Name | Description | Type | Default | Required |
+|--------------------------|---------------------------------------------------------|---------|---------|----------|
+| **global.domain** | your domain for the external endpoint, ex `example.com` | string | - | yes |
+| **global.limitsEnabled** | whether CPU and memory limits are enabled | boolean | `true` | yes |
+| **global.logLevel** | severity of logging level | string | `ERROR` | yes |
+| **global.tier** | Only PROD must be used to enable autoscaling | string | "" | no |
+| **global.autoscaling** | enables horizontal pod autoscaling, when tier=PROD | boolean | true | yes |
### Configmap variables
-| Name | Description | Type | Default |Required |
-|------|-------------|------|---------|---------|
-**data.entitlementsHost** | entitlements host | string | "http://entitlements" | yes
-**data.logLevel** | logging level | string | INFO | yes
-**data.partitionHost** | partition host | string | "http://partition" | yes
-**data.schemaHost** | schema host | string | "http://schema" | yes
-**data.securityHttpsCertificateTrust** | whether https is enabled | boolean | true | yes
-**data.storageHost** | storage host | string | "http://storage" | yes
-**data.redisIndexerHost** | The host for redis instance. If empty (by default), helm installs an internal redis instance | string | - | yes
-**data.redisIndexerPort** | The port for redis instance | digit | 6379 | yes
+| Name | Description | Type | Default | Required |
+|----------------------------------------|----------------------------------------------------------------------------------------------|---------|-----------------------|-------------------------------------------------|
+| **data.entitlementsHost** | entitlements host | string | `http://entitlements` | yes |
+| **data.logLevel** | logging severity level for this service only | string | - | yes, only if differs from the `global.logLevel` |
+| **data.partitionHost** | partition host | string | `http://partition` | yes |
+| **data.schemaHost** | schema host | string | `http://schema` | yes |
+| **data.securityHttpsCertificateTrust** | whether https is enabled | boolean | `true` | yes |
+| **data.storageHost** | storage host | string | `http://storage` | yes |
+| **data.redisIndexerHost** | The host for redis instance. If empty (by default), helm installs an internal redis instance | string | - | yes |
+| **data.redisIndexerPort** | The port for redis instance | digit | `6379` | yes |
### Deploy variables
-| Name | Description | Type | Default |Required |
-|------|-------------|------|---------|---------|
-**data.requestsCpu** | amount of requested CPU | string | 35m | yes
-**data.requestsMemory** | amount of requested memory| string | 640Mi | yes
-**data.limitsCpu** | CPU limit | string | 1 | only if `global.limitsEnabled` is true
-**data.limitsMemory** | memory limit | string | 1G | only if `global.limitsEnabled` is true
-**data.image** | service image | string | - | yes
-**data.imagePullPolicy** | when to pull image | string | IfNotPresent | yes
-**data.serviceAccountName** | name of your service account | string | indexer | yes
-**data.redisImage** | service image | string | `redis:7` | yes
+| Name | Description | Type | Default | Required |
+|-----------------------------|------------------------------|--------|----------------|----------------------------------------|
+| **data.requestsCpu** | amount of requested CPU | string | `35m` | yes |
+| **data.requestsMemory** | amount of requested memory | string | `640Mi` | yes |
+| **data.limitsCpu** | CPU limit | string | `1` | only if `global.limitsEnabled` is true |
+| **data.limitsMemory** | memory limit | string | `1G` | only if `global.limitsEnabled` is true |
+| **data.image** | service image | string | - | yes |
+| **data.imagePullPolicy** | when to pull image | string | `IfNotPresent` | yes |
+| **data.serviceAccountName** | name of your service account | string | `indexer` | yes |
+| **data.redisImage** | service image | string | `redis:7` | yes |
### Config variables
-| Name | Description | Type | Default |Required |
-|------|-------------|------|---------|---------|
-**conf.appName** | name of the app | string | `indexer` | yes
-**conf.configmap** | configmap to be used | string | `indexer-config` | yes
-**conf.elasticSecretName** | secret for elastic | string | `indexer-elastic-secret` | yes
-**conf.keycloakSecretName** | secret for keycloak | string | `indexer-keycloak-secret` | yes
-**conf.rabbitmqSecretName** | secret for rabbitmq | string | `rabbitmq-secret` | yes
-**conf.indexerRedisSecretName** | indexer Redis secret that contains redis password with REDIS_PASSWORD key | string | `indexer-redis-secret` | yes
+| Name | Description | Type | Default | Required |
+|---------------------------------|---------------------------------------------------------------------------|--------|---------------------------|----------|
+| **conf.appName** | name of the app | string | `indexer` | yes |
+| **conf.configmap** | configmap to be used | string | `indexer-config` | yes |
+| **conf.elasticSecretName** | secret for elastic | string | `indexer-elastic-secret` | yes |
+| **conf.indexerRedisSecretName** | indexer Redis secret that contains redis password with REDIS_PASSWORD key | string | `indexer-redis-secret` | yes |
### ISTIO variables
-| Name | Description | Type | Default |Required |
-|------|-------------|------|---------|---------|
-**istio.proxyCPU** | CPU request for Envoy sidecars | string | 10m | yes
-**istio.proxyCPULimit** | CPU limit for Envoy sidecars | string | 200m | yes
-**istio.proxyMemory** | memory request for Envoy sidecars | string | 100Mi | yes
-**istio.proxyMemoryLimit** | memory limit for Envoy sidecars | string | 256Mi | yes
-**istio.sidecarInject** | whether Istio sidecar will be injected. Setting to "false" reduces security, because disables authorization policy. | boolean | true | yes
+| Name | Description | Type | Default | Required |
+|----------------------------|---------------------------------------------------------------------------------------------------------------------|---------|---------|----------|
+| **istio.proxyCPU** | CPU request for Envoy sidecars | string | `10m` | yes |
+| **istio.proxyCPULimit** | CPU limit for Envoy sidecars | string | `200m` | yes |
+| **istio.proxyMemory** | memory request for Envoy sidecars | string | `100Mi` | yes |
+| **istio.proxyMemoryLimit** | memory limit for Envoy sidecars | string | `256Mi` | yes |
+| **istio.sidecarInject** | whether Istio sidecar will be injected. Setting to `false` reduces security, because disables authorization policy. | boolean | `true` | yes |
+
+### Horizontal Pod Autoscaling (HPA) variables (works only if tier=PROD and autoscaling=true)
+
+| Name | Description | Type | Default | Required |
+|-----------------------------------------------------|-------------------------------------------------------------------------------|---------|----------------|----------------------------------------------------------------|
+| **hpa.minReplicas** | minimum number of replicas | integer | 6 | only if `global.autoscaling` is true and `global.tier` is PROD |
+| **hpa.maxReplicas** | maximum number of replicas | integer | 15 | only if `global.autoscaling` is true and `global.tier` is PROD |
+| **hpa.pubsubTargetType** | type of measurements: AverageValue | string | "AverageValue" | only if `global.autoscaling` is true and `global.tier` is PROD |
+| **hpa.pubsubTargetValue** | threshold value to trigger the scaling up | integer | 50 | only if `global.autoscaling` is true and `global.tier` is PROD |
+| **hpa.cpuTargetType** | type of measurements: Utilization | string | "Utilization" | only if `global.autoscaling` is true and `global.tier` is PROD |
+| **hpa.cpuTargetValue** | threshold value to trigger the scaling up | integer | 70 | only if `global.autoscaling` is true and `global.tier` is PROD |
+| **hpa.memoryTargetType** | type of measurements: Utilization | string | "Utilization" | only if `global.autoscaling` is true and `global.tier` is PROD |
+| **hpa.memoryTargetValue** | threshold value to trigger the scaling up | integer | 70 | only if `global.autoscaling` is true and `global.tier` is PROD |
+| **hpa.behaviorScaleUpStabilizationWindowSeconds** | time to start implementing the scale up when it is triggered | integer | 60 | only if `global.autoscaling` is true and `global.tier` is PROD |
+| **hpa.behaviorScaleUpPoliciesValue** | the maximum number of new replicas to create (in percents from current state) | integer | 1 | only if `global.autoscaling` is true and `global.tier` is PROD |
+| **hpa.behaviorScaleUpPoliciesPeriodSeconds** | pause for every new scale up decision | integer | 120 | only if `global.autoscaling` is true and `global.tier` is PROD |
+| **hpa.behaviorScaleDownStabilizationWindowSeconds** | time to start implementing the scale down when it is triggered | integer | 60 | only if `global.autoscaling` is true and `global.tier` is PROD |
+| **hpa.behaviorScaleDownPoliciesValue** | the maximum number of replicas to destroy (in percents from current state) | integer | 1 | only if `global.autoscaling` is true and `global.tier` is PROD |
+| **hpa.behaviorScaleDownPoliciesPeriodSeconds** | pause for every new scale down decision | integer | 120 | only if `global.autoscaling` is true and `global.tier` is PROD |
+
+### Limits variables
+
+| Name | Description | Type | Default | Required |
+|--------------------------|-------------------------------------------------|---------|---------|----------------------------------------------------------------|
+| **limits.maxTokens** | maximum number of requests per fillInterval | integer | 50 | only if `global.autoscaling` is true and `global.tier` is PROD |
+| **limits.tokensPerFill** | number of new tokens allowed every fillInterval | integer | 50 | only if `global.autoscaling` is true and `global.tier` is PROD |
+| **limits.fillInterval** | time interval | string | "1s" | only if `global.autoscaling` is true and `global.tier` is PROD |
### Install the helm chart
diff --git a/devops/gc/deploy/templates/authorization-policy.yaml b/devops/gc/deploy/templates/authorization-policy.yaml
deleted file mode 100644
index f694999500c9a2e3318e61a0f633f8d13d56c11a..0000000000000000000000000000000000000000
--- a/devops/gc/deploy/templates/authorization-policy.yaml
+++ /dev/null
@@ -1,35 +0,0 @@
-# FIXME: remove it when migrate to istio 1.6 and use directResponse instead
-apiVersion: security.istio.io/v1beta1
-kind: AuthorizationPolicy
-metadata:
- name: {{ printf "%s-allow-policy" .Values.conf.appName | quote }}
- namespace: {{ .Release.Namespace | quote }}
-spec:
- selector:
- matchLabels:
- app: {{ .Values.conf.appName | quote }}
- action: ALLOW
- rules:
- - to:
- - operation:
- paths:
- - /api/indexer/v2/*
----
-apiVersion: security.istio.io/v1beta1
-kind: AuthorizationPolicy
-metadata:
- name: {{ printf "%s-deny-policy" .Values.conf.appName | quote }}
- namespace: {{ .Release.Namespace | quote }}
-spec:
- selector:
- matchLabels:
- app: {{ .Values.conf.appName | quote }}
- action: DENY
- rules:
- - from:
- - source:
- notNamespaces: [ {{ .Release.Namespace | quote }} ]
- to:
- - operation:
- paths:
- - /api/indexer/v2/_dps/*
diff --git a/devops/gc/deploy/templates/configmap.yaml b/devops/gc/deploy/templates/configmap.yaml
index 09308ce665fc9826356ee03bcd20a47ee8965f77..770ef0dac9778d40c4886c8843da42c4dd966c87 100644
--- a/devops/gc/deploy/templates/configmap.yaml
+++ b/devops/gc/deploy/templates/configmap.yaml
@@ -7,7 +7,7 @@ metadata:
namespace: {{ .Release.Namespace | quote }}
data:
ENTITLEMENTS_HOST: {{ .Values.data.entitlementsHost | quote }}
- LOG_LEVEL: {{ .Values.data.logLevel | quote }}
+ LOG_LEVEL: {{ .Values.data.logLevel | default .Values.global.logLevel | quote }}
PARTITION_HOST: {{ .Values.data.partitionHost | quote }}
{{- if .Values.data.redisIndexerHost }}
REDIS_GROUP_HOST: {{ .Values.data.redisIndexerHost | quote }}
@@ -20,10 +20,4 @@ data:
REDIS_GROUP_PORT: {{ .Values.data.redisIndexerPort | quote }}
SCHEMA_BASE_HOST: {{ .Values.data.schemaHost | quote }}
SECURITY_HTTPS_CERTIFICATE_TRUST: {{ .Values.data.securityHttpsCertificateTrust | quote }}
- {{- if .Values.global.onPremEnabled }}
- SPRING_PROFILES_ACTIVE: "anthos"
- {{- else }}
- SPRING_PROFILES_ACTIVE: "gcp"
- {{- end }}
STORAGE_HOST: {{ .Values.data.storageHost | quote }}
- SEARCH_BASE_HOST: {{ .Values.data.searchHost | quote }}
diff --git a/devops/gc/deploy/templates/deploy-redis.yaml b/devops/gc/deploy/templates/deploy-redis.yaml
index e930732a9b58570ddeb833433d7f9d816c27a174..c98f0ca1be31592900bd0329a793348969d8aecc 100644
--- a/devops/gc/deploy/templates/deploy-redis.yaml
+++ b/devops/gc/deploy/templates/deploy-redis.yaml
@@ -32,6 +32,7 @@ spec:
ports:
- containerPort: 6379
protocol: TCP
+ name: {{ printf "redis-%s" .Values.conf.appName | quote }}
resources:
requests:
cpu: 10m
diff --git a/devops/gc/deploy/templates/deployment.yaml b/devops/gc/deploy/templates/deployment.yaml
index 9f7a1ff89fbedcad8651c26b4883353dd330ca75..b33fa70f77480854060226989313048e6cef9bd2 100644
--- a/devops/gc/deploy/templates/deployment.yaml
+++ b/devops/gc/deploy/templates/deployment.yaml
@@ -51,12 +51,6 @@ spec:
name: {{ .Values.conf.configmap | quote }}
- secretRef:
name: {{ .Values.conf.elasticSecretName | quote }}
- {{- if .Values.global.onPremEnabled }}
- - secretRef:
- name: {{ .Values.conf.keycloakSecretName | quote }}
- - secretRef:
- name: {{ .Values.conf.rabbitmqSecretName | quote }}
- {{- end }}
securityContext:
allowPrivilegeEscalation: false
runAsNonRoot: true
@@ -87,3 +81,8 @@ spec:
memory: {{ .Values.data.limitsMemory | quote }}
{{- end }}
serviceAccountName: {{ .Values.data.serviceAccountName | quote }}
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxUnavailable: 1
+ maxSurge: 0
diff --git a/devops/gc/deploy/templates/hpa.yaml b/devops/gc/deploy/templates/hpa.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..0dca0a867da09bccfffb2ace44999bb23b89d186
--- /dev/null
+++ b/devops/gc/deploy/templates/hpa.yaml
@@ -0,0 +1,80 @@
+{{- if and (eq .Values.global.tier "PROD") (eq .Values.global.autoscaling true) }}
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+ name: {{ printf "%s-hpa" .Values.conf.appName | quote }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ scaleTargetRef:
+ apiVersion: apps/v1
+ kind: Deployment
+ name: {{ .Values.conf.appName | quote }}
+ minReplicas: {{ .Values.hpa.minReplicas }}
+ maxReplicas: {{ .Values.hpa.maxReplicas }}
+ metrics:
+ - external:
+ metric:
+ name: pubsub.googleapis.com|subscription|num_undelivered_messages
+ selector:
+ matchLabels:
+ resource.labels.subscription_id: indexer-records-changed
+ target:
+ type: {{ .Values.hpa.pubsubTargetType | quote }}
+ averageValue: {{ .Values.hpa.pubsubTargetValue }}
+ type: External
+ - external:
+ metric:
+ name: pubsub.googleapis.com|subscription|num_undelivered_messages
+ selector:
+ matchLabels:
+ resource.labels.subscription_id: indexer-reindex
+ target:
+ type: {{ .Values.hpa.pubsubTargetType | quote }}
+ averageValue: {{ .Values.hpa.pubsubTargetValue }}
+ type: External
+ - external:
+ metric:
+ name: pubsub.googleapis.com|subscription|num_undelivered_messages
+ selector:
+ matchLabels:
+ resource.labels.subscription_id: indexer-reprocess
+ target:
+ type: {{ .Values.hpa.pubsubTargetType | quote }}
+ averageValue: {{ .Values.hpa.pubsubTargetValue }}
+ type: External
+ - external:
+ metric:
+ name: pubsub.googleapis.com|subscription|num_messages_received
+ selector:
+ matchLabels:
+ resource.labels.subscription_id: indexer-schema-changed
+ target:
+ type: {{ .Values.hpa.pubsubTargetType | quote }}
+ averageValue: {{ .Values.hpa.pubsubTargetValue }}
+ type: External
+ - resource:
+ name: cpu
+ target:
+ type: {{ .Values.hpa.cpuTargetType | quote }}
+ averageUtilization: {{ .Values.hpa.cpuTargetValue }}
+ type: Resource
+ - resource:
+ name: memory
+ target:
+ type: {{ .Values.hpa.memoryTargetType | quote }}
+ averageUtilization: {{ .Values.hpa.memoryTargetValue }}
+ type: Resource
+ behavior:
+ scaleUp:
+ stabilizationWindowSeconds: {{ .Values.hpa.behaviorScaleUpStabilizationWindowSeconds }}
+ policies:
+ - type: Pods
+ value: {{ .Values.hpa.behaviorScaleUpPoliciesValue }}
+ periodSeconds: {{ .Values.hpa.behaviorScaleUpPoliciesPeriodSeconds }}
+ scaleDown:
+ stabilizationWindowSeconds: {{ .Values.hpa.behaviorScaleDownStabilizationWindowSeconds }}
+ policies:
+ - type: Pods
+ value: {{ .Values.hpa.behaviorScaleDownPoliciesValue }}
+ periodSeconds: {{ .Values.hpa.behaviorScaleDownPoliciesPeriodSeconds }}
+{{- end }}
diff --git a/devops/gc/deploy/templates/rate-limits.yaml b/devops/gc/deploy/templates/rate-limits.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..ddc911c8bc1c66f08dc28d8f0626668d98586ed8
--- /dev/null
+++ b/devops/gc/deploy/templates/rate-limits.yaml
@@ -0,0 +1,43 @@
+{{- if and (eq .Values.global.tier "PROD") (eq .Values.global.autoscaling true) }}
+apiVersion: networking.istio.io/v1alpha3
+kind: EnvoyFilter
+metadata:
+ name: {{ printf "%s-ratelimit" .Values.conf.appName | quote }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ workloadSelector:
+ labels:
+ app: {{ .Values.conf.appName | quote }}
+ configPatches:
+ - applyTo: "HTTP_FILTER"
+ match:
+ context: "SIDECAR_INBOUND"
+ listener:
+ filterChain:
+ filter:
+ name: "envoy.filters.network.http_connection_manager"
+ patch:
+ operation: INSERT_BEFORE
+ value:
+ name: "envoy.filters.http.local_ratelimit"
+ typed_config:
+ '@type': "type.googleapis.com/udpa.type.v1.TypedStruct"
+ type_url: "type.googleapis.com/envoy.extensions.filters.http.local_ratelimit.v3.LocalRateLimit"
+ value:
+ stat_prefix: "http_local_rate_limiter"
+ enable_x_ratelimit_headers: "DRAFT_VERSION_03"
+ token_bucket:
+ max_tokens: {{ .Values.limits.maxTokens }}
+ tokens_per_fill: {{ .Values.limits.tokensPerFill }}
+ fill_interval: {{ .Values.limits.fillInterval | quote}}
+ filter_enabled:
+ runtime_key: "local_rate_limit_enabled"
+ default_value:
+ numerator: 100
+ denominator: "HUNDRED"
+ filter_enforced:
+ runtime_key: "local_rate_limit_enforced"
+ default_value:
+ numerator: 100
+ denominator: "HUNDRED"
+{{- end }}
diff --git a/devops/gc/deploy/templates/service-account.yaml b/devops/gc/deploy/templates/service-account.yaml
deleted file mode 100644
index eec72f12889eec74d3de43ae6527a860bcd876ea..0000000000000000000000000000000000000000
--- a/devops/gc/deploy/templates/service-account.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-{{- if .Values.global.onPremEnabled }}
-apiVersion: v1
-kind: ServiceAccount
-metadata:
- name: {{ .Values.data.serviceAccountName | quote }}
- namespace: {{ .Release.Namespace | quote }}
-{{- end }}
diff --git a/devops/gc/deploy/templates/service-redis.yaml b/devops/gc/deploy/templates/service-redis.yaml
index abc86bf979808eec89b273afe2ea711e74995c84..327519a21096ccc4ba5d03242ab88abe3aafea96 100644
--- a/devops/gc/deploy/templates/service-redis.yaml
+++ b/devops/gc/deploy/templates/service-redis.yaml
@@ -9,6 +9,7 @@ spec:
- port: {{ .Values.data.redisIndexerPort }}
protocol: TCP
targetPort: 6379
+ name: {{ printf "redis-%s-port" .Values.conf.appName | quote }}
selector:
app: {{ printf "redis-%s" .Values.conf.appName | quote }}
{{ end }}
diff --git a/devops/gc/deploy/templates/service.yaml b/devops/gc/deploy/templates/service.yaml
index f985dd5f7a65eff8c6c0475c13d0298b9770ec00..c5b58d91c9e098da2ae09314885a55e1fdde5077 100644
--- a/devops/gc/deploy/templates/service.yaml
+++ b/devops/gc/deploy/templates/service.yaml
@@ -1,10 +1,8 @@
apiVersion: v1
kind: Service
metadata:
- {{- if not .Values.global.onPremEnabled }}
annotations:
cloud.google.com/neg: '{"ingress": true}'
- {{- end }}
name: {{ .Values.conf.appName | quote }}
namespace: {{ .Release.Namespace | quote }}
spec:
diff --git a/devops/gc/deploy/templates/virtual-service.yaml b/devops/gc/deploy/templates/virtual-service.yaml
index 0f882c05077190c1a18e59f6338eadada4354e42..208361f46387a545f21e5c8222e7ebf86d29c7ec 100644
--- a/devops/gc/deploy/templates/virtual-service.yaml
+++ b/devops/gc/deploy/templates/virtual-service.yaml
@@ -5,9 +5,7 @@ metadata:
namespace: {{ .Release.Namespace | quote }}
spec:
hosts:
- {{- if and .Values.global.domain .Values.global.onPremEnabled }}
- - {{ printf "osdu.%s" .Values.global.domain | quote }}
- {{- else if .Values.global.domain }}
+ {{- if .Values.global.domain }}
- {{ .Values.global.domain | quote }}
{{- else }}
- "*"
@@ -15,14 +13,13 @@ spec:
gateways:
- service-gateway
http:
- # FIXME: use it when migrate to istio 1.6
- # - match:
- # - uri:
- # prefix: "/api/indexer/v2/_dps"
- # directResponse:
- # status: 403
- # body:
- # string: "Forbidden"
+ - match:
+ - uri:
+ prefix: "/api/indexer/v2/_dps"
+ directResponse:
+ status: 403
+ body:
+ string: "Forbidden"
- match:
- uri:
prefix: "/api/indexer/v2"
diff --git a/devops/gc/deploy/values.yaml b/devops/gc/deploy/values.yaml
index c16f4a63e15d85c4b3c37352c0bd2b9d8d898929..e0865d492410f69161cd274f30bb461e6c7061e3 100644
--- a/devops/gc/deploy/values.yaml
+++ b/devops/gc/deploy/values.yaml
@@ -4,18 +4,19 @@
global:
domain: ""
- onPremEnabled: false
limitsEnabled: true
+ logLevel: "ERROR"
+ tier: ""
+ autoscaling: true
data:
# Configmap
entitlementsHost: "http://entitlements"
- logLevel: "ERROR"
+ logLevel: ""
partitionHost: "http://partition"
schemaHost: "http://schema"
securityHttpsCertificateTrust: "true"
storageHost: "http://storage"
- searchHost: "http://search"
# Deployment
requestsCpu: "100m"
requestsMemory: "640Mi"
@@ -33,8 +34,6 @@ conf:
appName: "indexer"
configmap: "indexer-config"
elasticSecretName: "indexer-elastic-secret"
- keycloakSecretName: "indexer-keycloak-secret"
- rabbitmqSecretName: "rabbitmq-secret"
indexerRedisSecretName: "indexer-redis-secret"
replicas: 2
@@ -44,3 +43,24 @@ istio:
proxyMemory: "100Mi"
proxyMemoryLimit: "256Mi"
sidecarInject: true
+
+hpa:
+ minReplicas: 6
+ maxReplicas: 15
+ pubsubTargetType: AverageValue
+ pubsubTargetValue: 50
+ cpuTargetType: Utilization
+ cpuTargetValue: 70
+ memoryTargetType: Utilization
+ memoryTargetValue: 70
+ behaviorScaleUpStabilizationWindowSeconds: 60
+ behaviorScaleUpPoliciesValue: 1
+ behaviorScaleUpPoliciesPeriodSeconds: 120
+ behaviorScaleDownStabilizationWindowSeconds: 60
+ behaviorScaleDownPoliciesValue: 1
+ behaviorScaleDownPoliciesPeriodSeconds: 120
+
+limits:
+ maxTokens: 200 #rps
+ tokensPerFill: 200
+ fillInterval: "1s"
diff --git a/devops/gc/pipeline/override-stages.yml b/devops/gc/pipeline/override-stages.yml
index bb8543728ebac52f7323a8dd3c94a4551cec18ab..3e15876937f911431d4aa012507a51f43dde56c0 100644
--- a/devops/gc/pipeline/override-stages.yml
+++ b/devops/gc/pipeline/override-stages.yml
@@ -5,12 +5,37 @@ variables:
gc-test:
variables:
CUCUMBER_OPTIONS: "--tags @as-ingested-coordinates,@default,@bag-of-words,@keyword-lower,@indexer-extended"
-
-gc-baremetal-test:
- variables:
- GC_VENDOR: baremetal
- CUCUMBER_OPTIONS: "--tags @default"
+ ELASTIC_8_PORT: $GC_ELASTIC_8_PORT
+ ELASTIC_8_USER_NAME: $GC_ELASTIC_8_USER_NAME
+ ELASTIC_8_PASSWORD: $GC_ELASTIC_8_PASSWORD
+ ELASTIC_8_HOST: $GC_ELASTIC_8_HOST
gc-dev2-test:
variables:
CUCUMBER_OPTIONS: "--tags @as-ingested-coordinates,@default,@bag-of-words,@keyword-lower,@indexer-extended"
+
+download_gc_plugins:
+ image: maven:3.8.3-openjdk-17-slim
+ stage: build
+ variables:
+ GC_OQM_PACKAGE_REGISTRY_URL: "https://community.opengroup.org/api/v4/projects/1477/packages/maven"
+ GC_OQM_VERSION: "0.27.2"
+ GC_APD_PACKAGE_REGISTRY_URL: "https://community.opengroup.org/api/v4/projects/1480/packages/maven"
+ GC_APD_VERSION: "0.27.2"
+ artifacts:
+ paths:
+ - ./tmp-gc/*.jar
+ when: always
+ expire_in: 1 days
+ script:
+ - mvn dependency:copy -DrepoUrl=$GC_OQM_PACKAGE_REGISTRY_URL -Dartifact="org.opengroup.osdu:gc-oqm-pubsub:$GC_OQM_VERSION:jar:plugin" -Dtransitive=false -DoutputDirectory="./tmp-gc"
+ - mvn dependency:copy -DrepoUrl=$GC_APD_PACKAGE_REGISTRY_URL -Dartifact="org.opengroup.osdu:gc-apd-acc:$GC_APD_VERSION:jar:plugin" -Dtransitive=false -DoutputDirectory="./tmp-gc"
+ only:
+ variables:
+ - $GC == '1'
+
+gc-containerize-gitlab:
+ needs: ["compile-and-unit-test", "download_gc_plugins"]
+
+gc-containerize-gcr:
+ needs: ["compile-and-unit-test", "download_gc_plugins"]
diff --git a/devops/gke-indexer-cicd.yml b/devops/gke-indexer-cicd.yml
index 951f9524c0faf028107d2947103cfce5b35736c1..337c1b290d8c516c40d7048bad9ce39109f3d42d 100644
--- a/devops/gke-indexer-cicd.yml
+++ b/devops/gke-indexer-cicd.yml
@@ -103,7 +103,7 @@ steps:
- task: Maven@3
inputs:
mavenPomFile: '$(integrationTestGcpPom)'
- options: '--settings $(mavenSettings) -Dsurefire.useFile=false -DVSTS_FEED_TOKEN=$(VSTS_FEED_TOKEN) -DDEFAULT_DATA_PARTITION_ID_TENANT1=$(DEFAULT_DATA_PARTITION_ID_TENANT1) -DDEFAULT_DATA_PARTITION_ID_TENANT2=$(DEFAULT_DATA_PARTITION_ID_TENANT2) -DELASTIC_HOST=$(ELASTIC_HOST) -DELASTIC_PASSWORD=$(ELASTIC_PASSWORD) -DELASTIC_USER_NAME=$(ELASTIC_USER_NAME) -DENTITLEMENTS_DOMAIN=$(ENTITLEMENTS_DOMAIN) -DINDEXER_HOST=$(INDEXER_HOST) -DLEGAL_TAG=$(LEGAL_TAG) -DOTHER_RELEVANT_DATA_COUNTRIES=$(OTHER_RELEVANT_DATA_COUNTRIES) -DSEARCH_HOST=$(SEARCH_HOST) -DSEARCH_INTEGRATION_TESTER=$(SEARCH_INTEGRATION_TESTER) -DSTORAGE_HOST=$(STORAGE_HOST) compile'
+ options: '--settings $(mavenSettings) -Dsurefire.useFile=false -DVSTS_FEED_TOKEN=$(VSTS_FEED_TOKEN) -DDEFAULT_DATA_PARTITION_ID_TENANT1=$(DEFAULT_DATA_PARTITION_ID_TENANT1) -DDEFAULT_DATA_PARTITION_ID_TENANT2=$(DEFAULT_DATA_PARTITION_ID_TENANT2) -DELASTIC_HOST=$(ELASTIC_HOST) -DELASTIC_PASSWORD=$(ELASTIC_PASSWORD) -DELASTIC_USER_NAME=$(ELASTIC_USER_NAME) -DENTITLEMENTS_DOMAIN=$(ENTITLEMENTS_DOMAIN) -DINDEXER_HOST=$(INDEXER_HOST) -DLEGAL_TAG=$(LEGAL_TAG) -DOTHER_RELEVANT_DATA_COUNTRIES=$(OTHER_RELEVANT_DATA_COUNTRIES) -DSEARCH_INTEGRATION_TESTER=$(SEARCH_INTEGRATION_TESTER) -DSTORAGE_HOST=$(STORAGE_HOST) compile'
publishJUnitResults: false
javaHomeOption: 'JDKVersion'
mavenVersionOption: 'Default'
diff --git a/docs/docs/IndexAugmenter.md b/docs/docs/IndexAugmenter.md
index 3ef5ff9d7c7f4f48e479bc616ce762cf2f6ff1bd..36dccc27e954b71ed845ad8d0a5024674fe2df07 100644
--- a/docs/docs/IndexAugmenter.md
+++ b/docs/docs/IndexAugmenter.md
@@ -352,6 +352,14 @@ It is not permitted to
implemented in the OSDU Data Definition pipeline to ensure that this reserved name does not appear as property in
the `data` block.
+* When the policy `ExtractAllMatches` is applied, the `Index Augmenter` will aggregate the `unique` values from all matched sources
+ to an array. The aggregated array could be large, especially when the relationship is `ParentToChildren`. There is a limitation
+ on the size of the array that ElasticSearch processes. A record will fail to be indexed if its property with array type
+ has more than 10,000 items. In order to balance the performance, the array size of the extended property should not be too
+ large. From M25, if the array size of the extended property exceeds the pre-defined limit, the extended property will be removed
+ and a message will be logged in the index of the record. The pre-defined limit is 2,000 by default. It can be
+ overridden by the setting in the Application.properties, e.g. `augmenter.extended_list_value.max_size=5000`.
+
## Deployment
@@ -381,4 +389,4 @@ the records from the `OSDU search` results, any one of the following mistakes ca
```
* If not all extended properties are missing, the `Configurations[]` of the missing extended properties could be invalid.
- The `Index Augmenter` engine can do basic syntax check on each configuration and only ignore the invalid ones.
\ No newline at end of file
+ The `Index Augmenter` engine can do basic syntax check on each configuration and only ignore the invalid ones.
diff --git a/docs/docs/PreviewFeatures.md b/docs/docs/PreviewFeatures.md
index 3c3979846ee93524012ae0eee2abb823a6f80374..3adaef94f0aa02218ab73b182f3e51dc5a5fb226 100644
--- a/docs/docs/PreviewFeatures.md
+++ b/docs/docs/PreviewFeatures.md
@@ -21,21 +21,59 @@ principles.
The configurations are encoded in OSDU reference-data records, one per each major schema version. The type name
is IndexPropertyPathConfiguration. With this, the extension properties can be defined as if they were provided by a schema.
-In order to reduce the risk when extended evaluation of the solution is still on going, a feature flag that is managed by
-the Partition Service is applied to the solution. Here is an example to enable this feature by setting the property
+The augmentation feature can be controlled by a feature flag that is managed via
+the `Partition Service`. The feature is `ON` by default since `M25`. The service provider can explicitly turn it off
+if the augmentation feature is not desirable for its clients.
+
+Here is an example to turn off augmentation feature by setting the property
"index-augmenter-enabled" in a given data partition:
```
{
"index-augmenter-enabled": {
"sensitive": false,
- "value": "true"
+ "value": "false"
}
}
```
-If the property "index-augmenter-enabled" is not created or the property value is set to "false" (String type) in the
+When the property "index-augmenter-enabled" is set to "false" (String type) via `Partition Service` for the
given data partition, the configurations defined as type IndexPropertyPathConfiguration will be ignored and index extension will be disabled.
+## Search text with special characters '_' and '.'
+
+OSDU indexer and search use Elasticsearch default analyzer (or called standard analyzer) to analyzes the unstructured
+text when they are indexed and searched. Due to the way Elasticsearch standard analzyer analyzes unstructured text,
+it is very difficult if not impossible to perform certain high-value searches on unstructured content. For example,
+users want to search for a file with file name `1-ABC_Seismic_Report.pdf`, it is impossible to use one or two keywords
+in the file name like "abc", "seismic", "report" to search the file or pdf extension to find search all pdf files.
+User can't even use wildcard like `*seismic*` to search the file as wildcard in prefix is not supported. The user would
+have to search using exact match or at a minimum ABC_Seismic* if they want to use wildcards.
+
+In the [ADR](doc:https://community.opengroup.org/osdu/platform/system/indexer-service/-/issues/186), we propose a
+change to extend the Elasticsearch Standard Analyzer to process two additional special characters as word delimiter:
+- underscore `_`
+- dot `.`. It will be handled like character `,`. Please note that Elasticsearch Standard Analyzer does not take the `,`
+ as word delimiter if it is part of number string, e.g. `1,663m`. In this proposal, the `.` will be processed in the
+ similar way, e.g. `-999.25` or `10.88` in which `.` won't be treated as word delimiter.
+
+In order to reduce risks (e.g. work interruption) on re-indexing, we will manage this solution with a feature flag that
+is set by the Partition Service. Here is an example to enable this feature by setting the property
+"custom-index-analyzer-enabled" in a given data partition:
+```
+{
+ "custom-index-analyzer-enabled": {
+ "sensitive": false,
+ "value": "true"
+ }
+}
+```
+
+If the property "custom-index-analyzer-enabled" is not created or the property value is set to "false" (String type) in the
+given data partition, the default index analyzer will be applied to indexing and search.
+
+After the feature is enabled, **it will require re-indexing kinds in the given partition in oder to adopt the custom analyzer.**
+
+
## Index AsIngestedCoordinates
Source: [issue 95](https://community.opengroup.org/osdu/platform/system/indexer-service/-/issues/95)
diff --git a/docs/docs/api.md b/docs/docs/api.md
index 68c9bb5e2c0eb2e17f906456373dedab67b62738..9af327a383342f92a6b19ad1eefa433fc40b4b96 100644
--- a/docs/docs/api.md
+++ b/docs/docs/api.md
@@ -268,7 +268,7 @@ Users must be a member of `users.datalake.ops` group.
### Schema change
-Schema change event listener endpoint.
+Schema change event (produced by [Schema Service](https://community.opengroup.org/osdu/platform/system/schema-service)) listener endpoint.
> __Note:__ This is internal API and shouldn't be exposed publicly.
@@ -304,6 +304,50 @@ POST /api/indexer/v2/_dps/task-handlers/schema-worker HTTP/1.1
`data` <br />
  (required, String) Schema change event message json string. Only `create` and `update` events are supported.
+### Record change
+
+Record change event (produced by [Storage service](https://community.opengroup.org/osdu/platform/system/storage)) listener endpoint.
+
+> __Note:__ This is internal API and shouldn't be exposed publicly.
+
+#### Request
+
+```http
+POST /api/indexer/v2/_dps/task-handlers/index-worker HTTP/1.1
+{
+ "messageId": "676895654",
+ "publishTime": "2024-03-19T00:00:00",
+ "attributes": {
+ "data-partition-id": "opendes",
+ "correlation-id": "b5a281bd-f59d-4db2-9939-b2d85036fc7e"
+ },
+ "data": "[{\"id\":\"opendes:master-data--Basin:0cfbffb72b2344aea5a8f92bbffd3953\",\"kind\":\"osdu:wks:master-data--Basin:1.0.0\",\"op\":\"create\"}]"
+}
+```
+
+#### Request body
+
+`messageId` <br />
+  (optional, String) Event message id.
+
+`publishTime` <br />
+  (optional, String) Event publish time.
+
+`attributes.data-partition-id` <br />
+  (required, String) Data partition id for which this message is targeted.
+
+`attributes.correlation-id` <br />
+  (optional, String) Correlation-id to enable tracing.
+
+`data` <br />
+  (required, String) Record change event message json string. Supported record-change event payload samples can be found [here](https://community.opengroup.org/osdu/platform/system/register/-/blame/master/register-core/src/main/resources/topics.json?ref_type=heads#L3).
+
+### Monitor indexing progress
+
+Once `index-worker` processes a record change event, it publishes event processing or indexing status to `indexing-progress` topic. Consumers can subscribe to this topic to monitor the indexing progress for a **data-partition**.
+
+Please be advised, Core services do not provide any additional message level filtering capabilities, so consumers will see status updates for all kinds for a given **data-partition**.
+
# Troubleshoot Indexing Issues
## Get indexing status
diff --git a/indexer-acceptance-test/docs/README.md b/indexer-acceptance-test/docs/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..e1362e9579205b3c8c9e27ff9a2ca5e7d7f00bef
--- /dev/null
+++ b/indexer-acceptance-test/docs/README.md
@@ -0,0 +1,76 @@
+### Running E2E Tests
+
+You will need to have the following environment variables defined.
+
+| name | value | description | sensitive? | source |
+|-------------------------------------|------------------------------------------------------------------|---------------------------------------------------------------------------------------------------|------------|-------------------------------------|
+| `HOST` | eg. `https://osdu.com` | - | no | - |
+| `INDEXER_HOST` | eg. `https://os-indexer-dot-opendes.appspot.com/api/indexer/v2/` | Indexer API endpoint | no | output of infrastructure deployment |
+| `SEARCH_HOST` | eg. `https://osdu.com/api/search/v2/` | - | no | - |
+| `STORAGE_HOST` | ex `http://os-storage-dot-opendes.appspot.com/api/storage/v2/` | Storage API endpoint | no | output of infrastructure deployment |
+| `SECURITY_HTTPS_CERTIFICATE_TRUST` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | no | output of infrastructure deployment |
+| `DEFAULT_DATA_PARTITION_ID_TENANT1` | ex `opendes` | HTTP Header 'Data-Partition-ID' | no | - |
+| `DEFAULT_DATA_PARTITION_ID_TENANT2` | ex `opendes` | HTTP Header 'Data-Partition-ID' | no | - |
+| `ENTITLEMENTS_DOMAIN` | eg. `group` | - | no | - |
+| `LEGAL_TAG` | ex `opendes-demo-legaltag` | valid legal tag with a other relevant data countries from `DEFAULT_OTHER_RELEVANT_DATA_COUNTRIES` | no | - |
+| `OTHER_RELEVANT_DATA_COUNTRIES` | ex `US` | valid legal tag with a other relevant data countries | no | - |
+
+Authentication can be provided as OIDC config:
+
+| name | value | description | sensitive? | source |
+|-------------------------------------------------|-----------------------------------------|-------------------------------------|------------|-------------------------------------|
+| `PRIVILEGED_USER_OPENID_PROVIDER_CLIENT_ID` | `********` | Client Id for `$INTEGRATION_TESTER` | yes | - |
+| `PRIVILEGED_USER_OPENID_PROVIDER_CLIENT_SECRET` | `********` | secret for `$INTEGRATION_TESTER` | yes | - |
+| `TEST_OPENID_PROVIDER_URL` | `https://keycloak.com/auth/realms/osdu` | OpenID provider url | yes | - |
+| `ELASTIC_HOST` | eg. `elastic.domain.com` | Host Elasticsearch | yes | output of infrastructure deployment |
+| `ELASTIC_USER_NAME` | `********` | User name for Elasticsearch | yes | output of infrastructure deployment |
+| `ELASTIC_PASSWORD` | `********` | Password for Elasticsearch | yes | output of infrastructure deployment |
+| `ELASTIC_PORT` | ex `9243` | Port Elasticsearch | yes | output of infrastructure deployment |
+Or tokens can be used directly from env variables:
+
+| name | value | description | sensitive? | source |
+|-------------------------|------------|-----------------------|------------|--------|
+| `PRIVILEGED_USER_TOKEN` | `********` | PRIVILEGED_USER Token | yes | - |
+
+#### Entitlements configuration for Integration Accounts
+
+| INTEGRATION_TESTER | NO_DATA_ACCESS_TESTER |
+|-------------------------------|-----------------------|
+| users | |
+| users.datalake.ops | |
+| service.storage.creator | |
+| service.entitlements.user | |
+| service.search.user | |
+| service.search.admin | |
+| data.test1 | |
+| data.integration.test | |
+| users@{tenant1}@{groupId}.com | |
+
+Execute following command to build code and run all the integration tests:
+
+ ```bash
+ # Note: this assumes that the environment variables for integration tests as outlined
+ # above are already exported in your environment.
+ # build + install integration test core
+ $ (cd indexer-acceptance-test && mvn clean verify)
+ ```
+
+## License
+
+Copyright © Google LLC
+
+Copyright © EPAM Systems
+
+Copyright © ExxonMobil
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+[http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0)
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
diff --git a/indexer-acceptance-test/pom.xml b/indexer-acceptance-test/pom.xml
new file mode 100644
index 0000000000000000000000000000000000000000..80b713978fd2999878930f6457b5ec7d44aecee8
--- /dev/null
+++ b/indexer-acceptance-test/pom.xml
@@ -0,0 +1,175 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.opengroup.osdu.indexer</groupId>
+ <artifactId>indexer-service</artifactId>
+ <version>0.28.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <groupId>org.opengroup.osdu.indexer</groupId>
+ <artifactId>indexer-acceptance-test</artifactId>
+ <version>1.0-SNAPSHOT</version>
+
+ <properties>
+ <maven.compiler.source>17</maven.compiler.source>
+ <maven.compiler.target>17</maven.compiler.target>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <cucumber.version>1.2.5</cucumber.version>
+ <argLine>
+ --add-opens java.base/java.lang=ALL-UNNAMED
+ --add-opens=java.base/java.util=ALL-UNNAMED
+ --add-opens java.base/java.lang.reflect=ALL-UNNAMED
+ --add-opens java.base/java.text=ALL-UNNAMED
+ --add-opens java.desktop/java.awt.font=ALL-UNNAMED
+ </argLine>
+ </properties>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>com.fasterxml.jackson</groupId>
+ <artifactId>jackson-bom</artifactId>
+ <version>2.17.2</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.opengroup.osdu</groupId>
+ <artifactId>os-core-common</artifactId>
+ <version>${os-core-common.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ <version>1.18.28</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.nimbusds</groupId>
+ <artifactId>oauth2-oidc-sdk</artifactId>
+ <version>9.15</version>
+ </dependency>
+
+ <!-- region Cucumber -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.13.2</version>
+ </dependency>
+ <dependency>
+ <groupId>info.cukes</groupId>
+ <artifactId>cucumber-java</artifactId>
+ <version>${cucumber.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>info.cukes</groupId>
+ <artifactId>cucumber-junit</artifactId>
+ <version>${cucumber.version}</version>
+ </dependency>
+ <!-- endregion -->
+
+ <!-- region Elasticsearch-->
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-client</artifactId>
+ <version>8.15.2</version>
+ </dependency>
+ <dependency>
+ <groupId>co.elastic.clients</groupId>
+ <artifactId>elasticsearch-java</artifactId>
+ <version>8.15.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.locationtech.jts.io</groupId>
+ <artifactId>jts-io-common</artifactId>
+ <version>1.15.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.locationtech.spatial4j</groupId>
+ <artifactId>spatial4j</artifactId>
+ <version>0.7</version>
+ </dependency>
+ <dependency>
+ <groupId>com.vividsolutions</groupId>
+ <artifactId>jts</artifactId>
+ <version>1.13</version>
+ <exclusions>
+ <exclusion>
+ <groupId>xerces</groupId>
+ <artifactId>xercesImpl</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <!-- endregion -->
+
+ <!-- region Gson: Java to Json conversion -->
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>2.9.1</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.jaxrs</groupId>
+ <artifactId>jackson-jaxrs-json-provider</artifactId>
+ <version>2.9.9</version>
+ </dependency>
+ <!-- endregion -->
+
+ <!--region Logging-->
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-to-slf4j</artifactId>
+ <version>2.11.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-jdk14</artifactId>
+ <version>1.8.0-beta4</version>
+ </dependency>
+ <!-- endregion -->
+ <!-- region Jackson -->
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </dependency>
+ <!-- endregion -->
+
+ <dependency>
+ <groupId>com.google.api-client</groupId>
+ <artifactId>google-api-client</artifactId>
+ <version>1.28.0</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-client</artifactId>
+ <version>1.19.4</version>
+ </dependency>
+ </dependencies>
+
+</project>
diff --git a/indexer-acceptance-test/src/test/java/org/opengroup/osdu/common/InfoBase.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/common/InfoBase.java
new file mode 100644
index 0000000000000000000000000000000000000000..9e83ec1c30e4bb6ea9c8d18905ab94b6aa1162d3
--- /dev/null
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/common/InfoBase.java
@@ -0,0 +1,73 @@
+package org.opengroup.osdu.common;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+import java.util.HashMap;
+import java.util.Map;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.logging.log4j.util.Strings;
+import org.opengroup.osdu.response.InfoResponseMock;
+import org.opengroup.osdu.util.Config;
+import org.opengroup.osdu.util.ElasticUtils;
+import org.opengroup.osdu.util.HTTPClient;
+
+@Slf4j
+public class InfoBase extends TestsBase {
+
+ protected Map<String, String> headers = new HashMap<>();
+ private InfoResponseMock response;
+
+ public InfoBase(HTTPClient httpClient) {
+ super(httpClient);
+ }
+
+ public InfoBase(HTTPClient httpClient, ElasticUtils elasticUtils) {
+ super(httpClient, elasticUtils);
+ }
+
+ @Override
+ protected String getApi() {
+ return Config.getIndexerBaseURL() + "info";
+ }
+
+ @Override
+ protected String getHttpMethod() {
+ return "GET";
+ }
+
+ public void i_send_get_request_to_version_info_endpoint() {
+ if (Config.getIndexerBaseURL().isEmpty()) {
+ log.warn("Env variable INDEXER_HOST is empty. Version info endpoint test is skipped");
+ return;
+ }
+
+ response =
+ executeQuery(
+ this.getApi(),
+ Strings.EMPTY,
+ headers,
+ httpClient.getAccessToken(),
+ InfoResponseMock.class);
+ }
+
+ public void i_send_get_request_to_version_info_endpoint_with_trailing_slash() {
+ response =
+ executeQuery(getApi()+"/", Strings.EMPTY, headers, httpClient.getAccessToken(), InfoResponseMock.class);
+ }
+
+ public void i_should_get_version_info_in_response() {
+ if (response != null) {
+ assertEquals(200, response.getResponseCode());
+ assertNotNull(response.getGroupId());
+ assertNotNull(response.getArtifactId());
+ assertNotNull(response.getVersion());
+ assertNotNull(response.getBuildTime());
+ assertNotNull(response.getBranch());
+ assertNotNull(response.getCommitId());
+ assertNotNull(response.getCommitMessage());
+ } else {
+ log.warn("Version info endpoint provided null response");
+ }
+ }
+}
diff --git a/indexer-acceptance-test/src/test/java/org/opengroup/osdu/common/RecordSteps.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/common/RecordSteps.java
new file mode 100644
index 0000000000000000000000000000000000000000..a4e1e7db44f35514a4321e0db8ce3ac8579374ae
--- /dev/null
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/common/RecordSteps.java
@@ -0,0 +1,554 @@
+package org.opengroup.osdu.common;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.opengroup.osdu.util.Config.getEntitlementsDomain;
+import static org.opengroup.osdu.util.Config.getStorageBaseURL;
+import static org.opengroup.osdu.util.HTTPClient.indentatedResponseBody;
+import static org.opengroup.osdu.util.JsonPathMatcher.FindArrayInJson;
+
+import co.elastic.clients.elasticsearch._types.Result;
+import co.elastic.clients.elasticsearch._types.mapping.TypeMapping;
+import co.elastic.clients.elasticsearch.core.DeleteResponse;
+import co.elastic.clients.elasticsearch.core.SearchResponse;
+import co.elastic.clients.elasticsearch.core.search.Hit;
+import co.elastic.clients.elasticsearch.indices.get_mapping.IndexMappingRecord;
+import co.elastic.clients.json.JsonpUtils;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.MapDifference;
+import com.google.common.collect.Maps;
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import com.sun.jersey.api.client.ClientResponse;
+import cucumber.api.DataTable;
+import java.io.IOException;
+import java.lang.reflect.Type;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
+import javax.ws.rs.HttpMethod;
+import lombok.extern.java.Log;
+import org.opengroup.osdu.core.common.http.CollaborationContextFactory;
+import org.opengroup.osdu.core.common.model.entitlements.Acl;
+import org.opengroup.osdu.core.common.model.http.CollaborationContext;
+import org.opengroup.osdu.core.common.model.storage.Record;
+import org.opengroup.osdu.core.common.model.storage.UpsertRecords;
+import org.opengroup.osdu.models.Setup;
+import org.opengroup.osdu.models.TestIndex;
+import org.opengroup.osdu.models.record.RecordData;
+import org.opengroup.osdu.util.ElasticUtils;
+import org.opengroup.osdu.util.FileHandler;
+import org.opengroup.osdu.util.HTTPClient;
+import org.springframework.util.CollectionUtils;
+
+@Log
+public class RecordSteps extends TestsBase {
+ private Map<String, TestIndex> inputIndexMap = new HashMap<>();
+ private ObjectMapper mapper = new ObjectMapper();
+ private boolean shutDownHookAdded = false;
+
+ private String timeStamp = String.valueOf(System.currentTimeMillis());
+ private List<Map<String, Object>> records;
+ private Map<String, String> headers = httpClient.getCommonHeader();
+
+ private UpsertRecords upsertedRecordsWithXcollab;
+ public static final String DpsHeaders_COLLABORATION = "x-collaboration";
+ public static final String X_COLLABORATION = "collaborationId";
+
+ public RecordSteps(HTTPClient httpClient) {
+ super(httpClient);
+ }
+
+ public RecordSteps(HTTPClient httpClient, ElasticUtils elasticUtils) {
+ super(httpClient, elasticUtils);
+ }
+
+ /******************One time cleanup for whole feature**************/
+ public void tearDown() {
+ for (String kind : inputIndexMap.keySet()) {
+ TestIndex testIndex = inputIndexMap.get(kind);
+ testIndex.cleanupIndex(kind);
+ testIndex.deleteSchema(kind);
+ }
+
+ if (!CollectionUtils.isEmpty(records)) {
+ cleanupRecords();
+ }
+ }
+
+ protected void cleanupRecords() {
+ for (Map<String, Object> testRecord : records) {
+ String id = testRecord.get("id").toString();
+ httpClient.send(HttpMethod.DELETE, getStorageBaseURL() + "records/" + id, null, headers, httpClient.getAccessToken());
+ log.info("Deleted the records");
+ }
+ }
+
+ public void the_schema_is_created_with_the_following_kind(DataTable dataTable) {
+
+ List<Setup> inputList = dataTable.asList(Setup.class);
+ for (Setup input : inputList) {
+ TestIndex testIndex = getTextIndex();
+ testIndex.setHttpClient(httpClient);
+ testIndex.setIndex(generateActualName(input.getIndex(), timeStamp));
+ testIndex.setKind(generateActualName(input.getKind(), timeStamp));
+ testIndex.setSchemaFile(input.getSchemaFile());
+ inputIndexMap.put(testIndex.getKind(), testIndex);
+ }
+
+ /******************One time setup for whole feature**************/
+ if (!shutDownHookAdded) {
+ for (String kind : inputIndexMap.keySet()) {
+ TestIndex testIndex = inputIndexMap.get(kind);
+ testIndex.setupSchema();
+ }
+ }
+ addShutDownHook();
+ }
+
+ public void i_ingest_records_with_the_for_a_given(String record, String dataGroup, String kind) {
+
+ String actualKind = generateActualName(kind, timeStamp);
+ try {
+ String fileContent = FileHandler.readFile(String.format("%s.%s", record, "json"));
+ records = new Gson().fromJson(fileContent, new TypeToken<List<Map<String, Object>>>() {}.getType());
+ String createTime = java.time.Instant.now().toString();
+
+ for (Map<String, Object> testRecord : records) {
+ if(testRecord.containsKey("data")) {
+ Map<String, Object> data = (Map<String, Object>)testRecord.get("data");
+ if(data != null && data.size() > 0) {
+ data = replaceValues(data, timeStamp);
+ testRecord.put("data", data);
+ }
+ }
+
+ testRecord.put("kind", actualKind);
+ testRecord.put("id", generateRecordId(testRecord));
+ testRecord.put("legal", generateLegalTag());
+ String[] x_acl = {generateActualName(dataGroup,timeStamp)+"."+getEntitlementsDomain()};
+ Acl acl = Acl.builder().viewers(x_acl).owners(x_acl).build();
+ testRecord.put("acl", acl);
+ String[] kindParts = kind.split(":");
+ String authority = tenantMap.get(kindParts[0]);
+ String source = kindParts[1];
+ testRecord.put("authority", authority);
+ testRecord.put("source", source);
+ testRecord.put("createUser", "TestUser");
+ testRecord.put("createTime", createTime);
+ }
+ String payLoad = new Gson().toJson(records);
+ log.log(Level.INFO, "Start ingesting records={0}", payLoad);
+ ClientResponse clientResponse = httpClient.send(HttpMethod.PUT, getStorageBaseURL() + "records", payLoad, headers, httpClient.getAccessToken());
+ log.info(String.format("Response body: %s\n Correlation id: %s\nResponse Status code: %s", indentatedResponseBody(clientResponse.getEntity(String.class)), clientResponse.getHeaders().get("correlation-id"), clientResponse.getStatus()));
+ assertEquals(201, clientResponse.getStatus());
+
+ } catch (Exception ex) {
+ throw new AssertionError(ex.getMessage());
+ }
+ }
+
+ protected String generateRecordId(Map<String, Object> testRecord) {
+ return generateActualId(testRecord.get("id").toString(), timeStamp, testRecord.get("kind").toString());
+ }
+
+ public void i_should_get_the_documents_for_the_in_the_Elastic_Search(int expectedCount, String index) throws Throwable {
+ index = generateActualName(index, timeStamp);
+ long numOfIndexedDocuments = createIndex(index);
+ assertEquals(expectedCount, numOfIndexedDocuments);
+ }
+
+ public void i_should_not_get_any_documents_for_the_index_in_the_Elastic_Search(String index) throws Throwable {
+ index = generateActualName(index, timeStamp);
+ getRecordsInIndex(index, 0);
+ }
+
+ public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String kind, String index) throws Throwable {
+ index = generateActualName(index, timeStamp);
+ Map<String, IndexMappingRecord> elasticMapping = elasticUtils.getMapping(index);
+ assertNotNull(elasticMapping);
+
+ String[] kindParts = kind.split(":");
+ String authority = tenantMap.get(kindParts[0]);
+ String source = kindParts[1];
+ expectedMapping = expectedMapping.replaceAll("<authority-id>", authority).replaceAll("<source-id>", source);
+ IndexMappingRecord typeMapping = elasticMapping.get(index);
+
+ StringBuilder collector = new StringBuilder();
+ TypeMapping mappings = typeMapping.mappings();
+ JsonpUtils.toString(mappings, collector);
+ Type type = new TypeToken<Map<String, Object>>() {}.getType();
+ Map<String, Object> mapping = new Gson().fromJson(collector.toString(), type);
+
+ assertNotNull(mapping);
+ assertTrue(areJsonEqual(expectedMapping, mapping.toString()));
+ }
+
+ public void i_can_validate_indexed_attributes(String index, String kind) throws Throwable {
+ String authority = tenantMap.get(kind.substring(0, kind.indexOf(":")));
+ index = generateActualName(index, timeStamp);
+ long numOfIndexedDocuments = createIndex(index);
+ List<Map<String, Object>> hits = elasticUtils.fetchRecordsByAttribute(index, "authority", authority);
+
+ assertTrue(hits.size() > 0);
+ for (Map<String, Object> result : hits) {
+ assertTrue(result.containsKey("authority"));
+ assertEquals(authority, result.get("authority"));
+ assertTrue(result.containsKey("source"));
+ assertTrue(result.containsKey("createUser"));
+ assertTrue(result.containsKey("createTime"));
+ }
+ }
+
+ public void iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(int expectedCount, String index, String skippedAttributes) throws Throwable {
+ index = generateActualName(index, timeStamp);
+ long numOfIndexedDocuments = createIndex(index);
+ long documentCountByQuery = elasticUtils.fetchRecordsByExistQuery(index, skippedAttributes);
+ assertEquals(expectedCount, documentCountByQuery);
+ }
+
+ public void iShouldBeAbleToSearchRecordByTagKeyAndTagValue(String index, String tagKey, String tagValue, int expectedNumber) throws Throwable {
+ TimeUnit.SECONDS.sleep(40);
+ index = generateActualName(index, timeStamp);
+ long actualNumberOfRecords = elasticUtils.fetchRecordsByTags(index, tagKey, tagValue);
+ assertEquals(expectedNumber, actualNumberOfRecords);
+ }
+
+ public void iShouldCleanupIndicesOfExtendedKinds(String extendedKinds) throws Throwable {
+ String[] kinds = extendedKinds.split(",");
+ for(String kind : kinds) {
+ String actualKind = this.generateActualName(kind.trim(), timeStamp);
+ TestIndex testIndex = this.getInputIndexMap().get(actualKind);
+ testIndex.cleanupIndex(actualKind);
+ }
+ }
+
+ public void iShouldBeAbleToSearchRecordByFieldAndFieldValue(String index, String fieldKey, String fieldValue, int expectedNumber) throws Throwable {
+ TimeUnit.SECONDS.sleep(60);
+ index = generateActualName(index, timeStamp);
+ long actualNumberOfRecords = elasticUtils.fetchRecordsByFieldAndFieldValue(index, fieldKey, fieldValue);
+ assertEquals(expectedNumber, actualNumberOfRecords);
+ }
+
+ public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery (
+ int expectedNumber, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field) throws Throwable {
+ index = generateActualName(index, timeStamp);
+ long numOfIndexedDocuments = createIndex(index);
+ long actualNumberOfRecords = elasticUtils.fetchRecordsByGeoWithinQuery(index, field, topLatitude, topLongitude, bottomLatitude, bottomLongitude);
+ assertEquals(expectedNumber, actualNumberOfRecords);
+ }
+
+ public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_AsIngestedCoordinates(
+ int expectedNumber, String index, Double topPointX, Double bottomPointX, String pointX, Double topPointY, Double bottomPointY, String pointY) throws Throwable {
+ index = generateActualName(index, timeStamp);
+ long numOfIndexedDocuments = createIndex(index);
+ long actualNumberOfRecords = elasticUtils.fetchRecordsByAsIngestedCoordinates(index, pointX, topPointX, bottomPointX, pointY, topPointY, bottomPointY);
+ assertEquals(expectedNumber, actualNumberOfRecords);
+ }
+
+ public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery(
+ int expectedNumber, String index, String path, String firstNestedField, String firstNestedValue, String secondNestedField, String secondNestedValue)
+ throws Throwable {
+ index = generateActualName(index, timeStamp);
+ long numOfIndexedDocuments = createIndex(index);
+ long actualNumberOfRecords = elasticUtils.fetchRecordsByNestedQuery(index, path, firstNestedField, firstNestedValue, secondNestedField, secondNestedValue);
+ assertEquals(expectedNumber, actualNumberOfRecords);
+ }
+
+ public void i_should_be_able_search_documents_for_the_by_flattened_inner_properties(int expectedCount, String index, String flattenedField,
+ String flattenedFieldValue) throws Throwable {
+ index = generateActualName(index, timeStamp);
+ long numOfIndexedDocuments = createIndex(index);
+ long actualNumberOfRecords = elasticUtils.fetchRecordsWithFlattenedFieldsQuery(index, flattenedField, flattenedFieldValue);
+ assertEquals(expectedCount, actualNumberOfRecords);
+ }
+
+ public void i_should_get_object_in_search_response_without_hints_in_schema(String objectField, String index, String recordFile, String acl, String kind)
+ throws Throwable {
+ index = generateActualName(index, timeStamp);
+
+ createIndex(index);
+
+ String expectedRecord = FileHandler.readFile(String.format("%s.%s", recordFile, "json"));
+
+ RecordData[] fileRecordData = mapper.readValue(expectedRecord, RecordData[].class);
+ RecordData expectedRecordData = fileRecordData[0];
+
+ RecordData actualRecordData = elasticUtils.fetchDataFromObjectsArrayRecords(index);
+
+ assertEquals(expectedRecordData.getData().get(objectField),actualRecordData.getData().get(objectField));
+ }
+
+ public void i_should_get_object_in_search_response(String innerField, String index)
+ throws Throwable {
+ index = generateActualName(index, timeStamp);
+
+ createIndex(index);
+
+ RecordData actualRecordData = elasticUtils.fetchDataFromObjectsArrayRecords(index);
+
+ assertTrue(actualRecordData.getData().containsKey(innerField));
+ }
+
+ public void i_should_get_string_array_in_search_response(String index, String field, String fieldValue, String arrayField, String desiredArrayValue)
+ throws Throwable {
+ TimeUnit.SECONDS.sleep(40);
+ final List<Map<String, Object>> elasticRecordData = elasticUtils.fetchRecordsByAttribute(index, field, fieldValue);
+ assertEquals(1, elasticRecordData.size());
+ final List<String> stringList = Arrays.asList(arrayField.split("\\."));
+ final Map<String, Object> jsonRecord = elasticRecordData.get(0);
+ assertEquals(String.join(",", (ArrayList<String>) FindArrayInJson(jsonRecord, stringList)), desiredArrayValue);
+ }
+
+ public void i_create_index_with_mapping_file_for_a_given_kind(String mappingFile, String index, String kind) throws Throwable {
+ String actualKind = generateActualName(kind, timeStamp);
+ TestIndex testIndex = getInputIndexMap().get(actualKind);
+ testIndex.setMappingFile(mappingFile);
+ this.getInputIndexMap().put(actualKind, testIndex);
+ testIndex.addIndex();
+ }
+
+ public void i_ingest_records_with_xcollab_value_included_with_the_with_for_a_given(String xCollab,
+ String record,
+ String dataGroup,
+ String kind) {
+ String actualKind = generateActualName(kind, timeStamp);
+ try {
+ String fileContent = FileHandler.readFile(String.format("%s.%s", record, "json"));
+ records = new Gson().fromJson(fileContent, new TypeToken<List<Map<String, Object>>>() {
+ }.getType());
+ String createTime = java.time.Instant.now().toString();
+
+ for (Map<String, Object> testRecord : records) {
+ if (testRecord.containsKey("data")) {
+ Map<String, Object> data = (Map<String, Object>) testRecord.get("data");
+ if (data != null && data.size() > 0) {
+ data = replaceValues(data, timeStamp);
+ testRecord.put("data", data);
+ }
+ }
+
+ testRecord.put("kind", actualKind);
+ testRecord.put("id", generateRecordId(testRecord));
+ testRecord.put("legal", generateLegalTag());
+ String[] x_acl = {generateActualName(dataGroup, timeStamp) + "." + getEntitlementsDomain()};
+ Acl acl = Acl.builder().viewers(x_acl).owners(x_acl).build();
+ testRecord.put("acl", acl);
+ String[] kindParts = kind.split(":");
+ String authority = tenantMap.get(kindParts[0]);
+ String source = kindParts[1];
+ testRecord.put("authority", authority);
+ testRecord.put("source", source);
+ testRecord.put("createUser", "TestUser");
+ testRecord.put("createTime", createTime);
+ }
+
+ // put record in WIP by the use of x-collaboration header
+ Map<String, String> headerXcollab = httpClient.getCommonHeader();
+ headerXcollab.put(DpsHeaders_COLLABORATION, xCollab);
+
+ String payLoad = new Gson().toJson(records);
+ log.log(Level.INFO, "Start ingesting records={0}", payLoad);
+ ClientResponse clientResponse = httpClient.send(HttpMethod.PUT,
+ getStorageBaseURL() + "records",
+ payLoad,
+ headerXcollab,
+ httpClient.getAccessToken());
+
+ String responseEntity = clientResponse.getEntity(String.class);
+ log.info(String.format("Response body with xcollab: %s\n Correlation id: %s\nResponse Status code: %s",
+ indentatedResponseBody(responseEntity),
+ clientResponse.getHeaders().get("correlation-id"),
+ clientResponse.getStatus()));
+ assertEquals(201, clientResponse.getStatus());
+
+ // remember record id for future tests
+ upsertedRecordsWithXcollab = mapper.readValue(responseEntity, UpsertRecords.class);
+ upsertedRecordsWithXcollab.getRecordIds()
+ .forEach(recordId -> log.info("Record ids with xcollab : " + recordId));
+
+ Optional<String> recordWithXcollab = upsertedRecordsWithXcollab.getRecordIds().stream().findAny();
+ assertTrue(recordWithXcollab.isPresent());
+
+ } catch (Exception ex) {
+ throw new AssertionError(ex.getMessage());
+ }
+ }
+
+ protected void i_should_get_the_documents_with_xcollab_value_included_for_the_in_the_Elastic_Search(
+ int expectedNumber, String xcollab, String index) throws Exception {
+
+ index = generateActualName(index, timeStamp);
+ // upsertedRecordsWithoutXcollab should have id received from previous steps
+ String id = upsertedRecordsWithXcollab.getRecordIds().stream().findAny().get();
+ log.log(Level.INFO, String.format("Try to find in Elastic a record with X collab with id : %s ", id));
+
+ SearchResponse<Record> searchResponse = null;
+ // should wait while Storage will publish Record into queue,
+ // then Index-queue should read message and pass it with http request to Indexer
+ // Indexer then will index the record into Elastic
+ CollaborationContextFactory collaborationContextFactory = new CollaborationContextFactory();
+ Optional<CollaborationContext> collaborationContext = collaborationContextFactory.create(xcollab);
+ String collaborationId = collaborationContext.orElseThrow().getId();
+
+ for (int i = 0; i < 20; i++) {
+ searchResponse = elasticUtils.fetchRecordsByIdAndMustHaveXcollab(index, id, collaborationId);
+ if (searchResponse.hits().total().value() == 0) {
+ log.log(Level.INFO, String.format("No records found with in index: %s, id: %s, collaborationId: %s,"
+ + " will try to wait up to 3 seconds.", index, id, collaborationId));
+ TimeUnit.SECONDS.sleep(3);
+ } else {
+ break;
+ }
+ }
+
+ log.log(Level.INFO,
+ String.format("xcollab feature: print searchResponse while being get a record by id and x-collab : %s",
+ searchResponse));
+ assertEquals(expectedNumber, searchResponse.hits().total().value());
+
+ // delete test record in namespace
+ List<Hit<Record>> hits = searchResponse.hits().hits();
+
+ String elasticId = hits.stream().findAny().get().id();
+ DeleteResponse deleteResponse = elasticUtils.deleteRecordsById(index, elasticId);
+ log.log(Level.INFO, String.format("Deleting record from Elasticsearch in index: %s with id: %s", index, elasticId));
+ assertEquals(deleteResponse.result(), Result.Deleted);
+ }
+
+ private Map<String, Object> replaceValues(Map<String, Object> data, String timeStamp) {
+ for(String key : data.keySet()) {
+ Object value = data.get(key);
+ Object replacedValue = replaceValue(value, timeStamp);
+ data.put(key, replacedValue);
+ }
+ return data;
+ }
+
+ private List<Object> replaceValues(List<Object> values, String timeStamp) {
+ List<Object> replacedValues = new ArrayList<>();
+ for(Object value : values) {
+ Object replacedValue = replaceValue(value, timeStamp);
+ replacedValues.add(replacedValue);
+ }
+
+ return replacedValues;
+ }
+
+ private Object replaceValue(Object value, String timeStamp) {
+ Object replacedValue = value;
+
+ if(value instanceof String) {
+ String rawValue = (String) value;
+ for (Map.Entry<String, String> tenant : tenantMap.entrySet()) {
+ rawValue = rawValue.replaceAll(tenant.getKey() + ":", tenant.getValue() + ":");
+ }
+ replacedValue = rawValue.replaceAll("<timestamp>", timeStamp);
+ }
+ else if(value instanceof List) {
+ replacedValue = replaceValues((List)value, timeStamp);
+ }
+ else if(value instanceof Map) {
+ replacedValue = replaceValues((Map<String, Object>) value, timeStamp);
+ }
+
+ return replacedValue;
+ }
+
+
+ private long createIndex(String index) throws InterruptedException, IOException {
+ long numOfIndexedDocuments = 0;
+ int iterator;
+
+ // index.refresh_interval is set to default 30s, wait for 40s initially
+ Thread.sleep(40000);
+
+ for (iterator = 0; iterator < 20; iterator++) {
+
+ numOfIndexedDocuments = elasticUtils.fetchRecords(index);
+ if (numOfIndexedDocuments > 0) {
+ log.info(String.format("index: %s | attempts: %s | documents acknowledged by elastic: %s", index, iterator, numOfIndexedDocuments));
+ break;
+ } else {
+ log.info(String.format("index: %s | documents acknowledged by elastic: %s", index, numOfIndexedDocuments));
+ Thread.sleep(5000);
+ }
+
+ if ((iterator + 1) % 5 == 0) elasticUtils.refreshIndex(index);
+ }
+ if (iterator >= 20) {
+ fail(String.format("index not created after waiting for %s seconds", ((40000 + iterator * 5000) / 1000)));
+ }
+ return numOfIndexedDocuments;
+ }
+
+ private long getRecordsInIndex(String index, int expectedCount) throws InterruptedException, IOException {
+ long numOfIndexedDocuments = 0;
+ int iterator;
+
+ // index.refresh_interval is set to default 30s, wait for 40s initially
+ Thread.sleep(40000);
+
+ for (iterator = 0; iterator < 20; iterator++) {
+
+ numOfIndexedDocuments = elasticUtils.fetchRecords(index);
+ if (expectedCount == numOfIndexedDocuments) {
+ log.info(String.format("index: %s | attempts: %s | documents acknowledged by elastic: %s", index, iterator, numOfIndexedDocuments));
+ break;
+ } else {
+ log.info(String.format("index: %s | documents acknowledged by elastic: %s", index, numOfIndexedDocuments));
+ Thread.sleep(5000);
+ }
+
+ if ((iterator + 1) % 5 == 0) elasticUtils.refreshIndex(index);
+ }
+ if (iterator >= 20) {
+ fail(String.format("index not created after waiting for %s seconds", ((40000 + iterator * 5000) / 1000)));
+ }
+ return numOfIndexedDocuments;
+ }
+
+ private Boolean areJsonEqual(String firstJson, String secondJson) {
+ Gson gson = new Gson();
+ Type mapType = new TypeToken<Map<String, Object>>() {}.getType();
+ Map<String, Object> firstMap = gson.fromJson(firstJson, mapType);
+ Map<String, Object> secondMap = gson.fromJson(secondJson, mapType);
+
+ MapDifference<String, Object> result = Maps.difference(firstMap, secondMap);
+ if (result != null && result.entriesDiffering().isEmpty()) return true;
+ log.info(String.format("difference: %s", result.entriesDiffering()));
+ return false;
+ }
+
+ @Override
+ protected String getApi() {
+ return null;
+ }
+
+ @Override
+ protected String getHttpMethod() {
+ return null;
+ }
+
+ public Map<String, TestIndex> getInputIndexMap() {
+ return inputIndexMap;
+ }
+
+ public String getTimeStamp() {
+ return timeStamp;
+ }
+
+ protected void addShutDownHook() {
+ if (!shutDownHookAdded) {
+ Runtime.getRuntime().addShutdownHook(new Thread(this::tearDown));
+ shutDownHookAdded = true;
+ }
+ }
+}
diff --git a/indexer-acceptance-test/src/test/java/org/opengroup/osdu/common/SchemaServiceRecordSteps.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/common/SchemaServiceRecordSteps.java
new file mode 100644
index 0000000000000000000000000000000000000000..c51b8aa23283fe029b7070e84c4416c32f3a8673
--- /dev/null
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/common/SchemaServiceRecordSteps.java
@@ -0,0 +1,68 @@
+package org.opengroup.osdu.common;
+
+import cucumber.api.DataTable;
+import org.opengroup.osdu.models.Setup;
+import org.opengroup.osdu.models.schema.PersistentSchemaTestIndex;
+import org.opengroup.osdu.util.ElasticUtils;
+import org.opengroup.osdu.util.HTTPClient;
+
+import java.util.List;
+import java.util.Map;
+
+public class SchemaServiceRecordSteps extends RecordSteps {
+ private static boolean runStatefulScenario = false;
+
+ public SchemaServiceRecordSteps(HTTPClient httpClient, ElasticUtils elasticUtils) {
+ super(httpClient, elasticUtils);
+ }
+
+ public void the_schema_is_created_with_the_following_kind(DataTable dataTable) {
+ if(!SchemaServiceRecordSteps.runStatefulScenario) {
+ List<Setup> inputList = dataTable.asList(Setup.class);
+ inputList.forEach(this::setup);
+ super.addShutDownHook();
+ }
+ }
+
+ public void i_set_scenarios_as_stateful(boolean stateful) throws Throwable {
+ SchemaServiceRecordSteps.runStatefulScenario = stateful;
+ }
+
+ private void setup(Setup input) {
+ PersistentSchemaTestIndex testIndex = new PersistentSchemaTestIndex(super.elasticUtils, super.httpClient, this);
+ testIndex.setIndex(generateActualName(input.getIndex(), super.getTimeStamp()));
+ testIndex.setSchemaFile(input.getSchemaFile());
+ testIndex.setHttpClient(super.httpClient);
+ testIndex.setupSchema();
+ testIndex.setKind(testIndex.getSchemaModel().getSchemaInfo().getSchemaIdentity().getId());
+
+ super.getInputIndexMap().put(testIndex.getKind(), testIndex);
+
+ deleteIndex(testIndex.getKind());
+ }
+
+ private void deleteIndex(String kind) {
+ this.indexerClientUtil.deleteIndex(kind);
+ }
+
+ @Override
+ protected String generateRecordId(Map<String, Object> testRecord) {
+ return generateActualIdWithoutTs(testRecord.get("id").toString(), testRecord.get("kind").toString());
+ }
+
+ @Override
+ public void i_ingest_records_with_the_for_a_given(String record, String dataGroup, String kind) {
+ super.i_ingest_records_with_the_for_a_given(record.replaceFirst("_schema", ""), dataGroup, kind);
+ }
+ @Override
+ public void i_should_get_object_in_search_response_without_hints_in_schema(String objectField, String index, String recordFile, String acl, String kind)
+ throws Throwable {
+ super.i_should_get_object_in_search_response_without_hints_in_schema(objectField, index, recordFile, acl, kind);
+ }
+ @Override
+ public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_AsIngestedCoordinates (
+ int expectedCount, String index, Double topPointX, Double bottomPointX, String pointX, Double topPointY, Double bottomPointY, String pointY) throws Throwable {
+ super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_AsIngestedCoordinates(expectedCount, index, topPointX, bottomPointX, pointX, topPointY, bottomPointY, pointY);
+ }
+
+}
diff --git a/indexer-acceptance-test/src/test/java/org/opengroup/osdu/common/TestsBase.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/common/TestsBase.java
new file mode 100644
index 0000000000000000000000000000000000000000..cb68e0d34f72b28f4acfc637eabadbe899a033f0
--- /dev/null
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/common/TestsBase.java
@@ -0,0 +1,174 @@
+package org.opengroup.osdu.common;
+
+import com.google.gson.Gson;
+import com.sun.jersey.api.client.ClientResponse;
+import cucumber.api.Scenario;
+import lombok.extern.java.Log;
+import org.opengroup.osdu.core.common.model.legal.Legal;
+import org.opengroup.osdu.models.Setup;
+import org.opengroup.osdu.models.TestIndex;
+import org.opengroup.osdu.response.ResponseBase;
+import org.opengroup.osdu.util.ElasticUtils;
+import org.opengroup.osdu.util.HTTPClient;
+import org.opengroup.osdu.util.IndexerClientUtil;
+
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.MultivaluedMap;
+import java.util.*;
+
+import static org.junit.Assert.assertTrue;
+import static org.opengroup.osdu.util.Config.*;
+import static org.opengroup.osdu.util.HTTPClient.indentatedResponseBody;
+
+
+@Log
+public abstract class TestsBase {
+ protected HTTPClient httpClient;
+ protected Scenario scenario;
+ protected Map<String, String> tenantMap = new HashMap<>();
+ protected Map<String, TestIndex> inputRecordMap = new HashMap<>();
+ protected ElasticUtils elasticUtils;
+ protected IndexerClientUtil indexerClientUtil;
+
+ public TestsBase(HTTPClient httpClient) {
+ this.httpClient = httpClient;
+ tenantMap.put("tenant1", getDataPartitionIdTenant1());
+ tenantMap.put("tenant2", getDataPartitionIdTenant2());
+ tenantMap.put("common", "common");
+
+ elasticUtils = new ElasticUtils();
+ this.indexerClientUtil = new IndexerClientUtil(this.httpClient);
+ }
+
+ public TestsBase(HTTPClient httpClient, ElasticUtils elasticUtils) {
+ this.httpClient = httpClient;
+ tenantMap.put("tenant1", getDataPartitionIdTenant1());
+ tenantMap.put("tenant2", getDataPartitionIdTenant2());
+ tenantMap.put("common", "common");
+
+ this.elasticUtils = elasticUtils;
+ this.indexerClientUtil = new IndexerClientUtil(this.httpClient);
+ }
+
+ protected TestIndex getTextIndex(){
+ return new TestIndex(elasticUtils);
+ }
+
+ protected void setUp(List<Setup> inputList, String timeStamp) {
+ for (Setup input : inputList) {
+ TestIndex testIndex = getTextIndex();
+ testIndex.setHttpClient(httpClient);
+ testIndex.setIndex(generateActualName(input.getIndex(), timeStamp));
+ testIndex.setKind(generateActualName(input.getKind(), timeStamp));
+ testIndex.setMappingFile(input.getMappingFile());
+ testIndex.setRecordFile(input.getRecordFile());
+ List<String> dataGroup = new ArrayList<>();
+ String[] viewerGroup = input.getViewerGroup().split(",");
+ for (int i = 0; i < viewerGroup.length; i++) {
+ viewerGroup[i] = generateActualName(viewerGroup[i], timeStamp) + "." + getEntitlementsDomain();
+ dataGroup.add(viewerGroup[i]);
+ }
+ String[] ownerGroup = input.getOwnerGroup().split(",");
+ for (int i = 0; i < ownerGroup.length; i ++) {
+ ownerGroup[i] = generateActualName(ownerGroup[i], timeStamp) + "." + getEntitlementsDomain();
+ if (dataGroup.indexOf(ownerGroup[i]) > 0) {
+ dataGroup.add(ownerGroup[i]);
+ }
+ }
+ testIndex.setViewerGroup(viewerGroup);
+ testIndex.setOwnerGroup(ownerGroup);
+ testIndex.setDataGroup(dataGroup.toArray(new String[dataGroup.size()]));
+ inputRecordMap.put(testIndex.getKind(), testIndex);
+ }
+ /******************One time setup for whole feature**************/
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ public void run() {
+ tearDown();
+ }
+ });
+ for (String kind : inputRecordMap.keySet()) {
+ TestIndex testIndex = inputRecordMap.get(kind);
+ testIndex.setupIndex();
+ }
+
+ }
+
+ /******************One time cleanup for whole feature**************/
+ public void tearDown() {
+ for (String kind : inputRecordMap.keySet()) {
+ TestIndex testIndex = inputRecordMap.get(kind);
+ testIndex.cleanupIndex(kind);
+ }
+ }
+
+ protected abstract String getApi();
+
+ protected abstract String getHttpMethod();
+
+ protected <T extends ResponseBase> T executeQuery(String api, String payLoad, Map<String, String> headers, String token, Class<T> typeParameterClass) {
+ ClientResponse clientResponse = httpClient.send(this.getHttpMethod(), api, payLoad, headers, token);
+ logCorrelationIdWithFunctionName(clientResponse.getHeaders());
+ return getResponse(clientResponse, typeParameterClass);
+ }
+
+ private <T extends ResponseBase> T getResponse(ClientResponse clientResponse, Class<T> typeParameterClass) {
+ log.info(String.format("Response status: %s, type: %s", clientResponse.getStatus(), clientResponse.getType().toString()));
+ assertTrue(clientResponse.getType().toString().contains(MediaType.APPLICATION_JSON));
+ String responseEntity = clientResponse.getEntity(String.class);
+
+ T response = new Gson().fromJson(responseEntity, typeParameterClass);
+ response.setHeaders(clientResponse.getHeaders());
+ response.setResponseCode(clientResponse.getStatus());
+ log.info(String.format("Response body: %s\nCorrelation id: %s\nResponse Status code: %s", indentatedResponseBody(responseEntity), response.getHeaders().get("correlation-id"), response.getResponseCode()));
+ return response;
+ }
+
+ private void logCorrelationIdWithFunctionName(MultivaluedMap<String, String> headers) {
+ log.info(String.format("Scenario Name: %s, Correlation-Id: %s", scenario.getId(), headers.get("correlation-id")));
+ }
+
+ protected String generateActualName(String rawName, String timeStamp) {
+ for (Map.Entry<String, String> tenant : tenantMap.entrySet()) {
+ rawName = rawName.replaceAll(tenant.getKey(), tenant.getValue());
+ }
+ return rawName.replaceAll("<timestamp>", timeStamp);
+ }
+
+ protected String generateActualId(String rawName, String timeStamp, String kind) {
+
+ rawName = generateActualName(rawName, timeStamp);
+
+ String kindSubType = kind.split(":")[2];
+
+ return rawName.replaceAll("<kindSubType>", kindSubType);
+
+ }
+
+
+ public String generateActualNameWithoutTs(String rawName) {
+ for (Map.Entry<String, String> tenant : tenantMap.entrySet()) {
+ rawName = rawName.replaceAll(tenant.getKey(), tenant.getValue());
+ }
+ return rawName.replaceAll("<timestamp>", "");
+ }
+
+ public String generateActualIdWithoutTs(String rawName, String kind) {
+ rawName = generateActualNameWithoutTs(rawName);
+
+ String kindSubType = kind.split(":")[2];
+
+ return rawName.replaceAll("<kindSubType>", kindSubType);
+ }
+
+ protected Legal generateLegalTag() {
+ Legal legal = new Legal();
+ Set<String> legalTags = new HashSet<>();
+ legalTags.add(getLegalTag());
+ legal.setLegaltags(legalTags);
+ Set<String> otherRelevantCountries = new HashSet<>();
+ otherRelevantCountries.add(getOtherRelevantDataCountries());
+ legal.setOtherRelevantDataCountries(otherRelevantCountries);
+ return legal;
+ }
+
+}
diff --git a/indexer-acceptance-test/src/test/java/org/opengroup/osdu/common/info/InfoSteps.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/common/info/InfoSteps.java
new file mode 100644
index 0000000000000000000000000000000000000000..2deee856b92d689b27c895bdc7f74feebef35493
--- /dev/null
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/common/info/InfoSteps.java
@@ -0,0 +1,16 @@
+package org.opengroup.osdu.common.info;
+
+import org.opengroup.osdu.common.InfoBase;
+import org.opengroup.osdu.util.ElasticUtils;
+import org.opengroup.osdu.util.HTTPClient;
+
+public class InfoSteps extends InfoBase {
+
+ public InfoSteps(HTTPClient httpClient) {
+ super(httpClient);
+ }
+
+ public InfoSteps(HTTPClient httpClient, ElasticUtils elasticUtils) {
+ super(httpClient, elasticUtils);
+ }
+}
diff --git a/indexer-acceptance-test/src/test/java/org/opengroup/osdu/models/FilterInfo.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/models/FilterInfo.java
new file mode 100644
index 0000000000000000000000000000000000000000..7981ad10f7ea50ee7f8c3d34e7277ee4d35e1ab5
--- /dev/null
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/models/FilterInfo.java
@@ -0,0 +1,6 @@
+package org.opengroup.osdu.models;
+
+public class FilterInfo {
+ public String name;
+ public String description;
+}
diff --git a/indexer-acceptance-test/src/test/java/org/opengroup/osdu/models/Setup.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/models/Setup.java
new file mode 100644
index 0000000000000000000000000000000000000000..190e1b233b79b40b6b767cd288adb1726c41e0b8
--- /dev/null
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/models/Setup.java
@@ -0,0 +1,22 @@
+package org.opengroup.osdu.models;
+
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.opengroup.osdu.util.HTTPClient;
+
+import java.util.Map;
+
+@Data
+@NoArgsConstructor
+public class Setup {
+ private String tenantId;
+ private String kind;
+ private String index;
+ private String viewerGroup;
+ private String ownerGroup;
+ private String mappingFile;
+ private String recordFile;
+ private String schemaFile;
+ private HTTPClient httpClient;
+ private Map<String, String> headers;
+}
diff --git a/indexer-acceptance-test/src/test/java/org/opengroup/osdu/models/TestIndex.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/models/TestIndex.java
new file mode 100644
index 0000000000000000000000000000000000000000..3353cda14a810e384586d1eebc7330618f3787d3
--- /dev/null
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/models/TestIndex.java
@@ -0,0 +1,145 @@
+package org.opengroup.osdu.models;
+
+import com.google.gson.Gson;
+import com.google.gson.JsonElement;
+import com.google.gson.reflect.TypeToken;
+import com.sun.jersey.api.client.ClientResponse;
+import lombok.Data;
+import org.opengroup.osdu.core.common.model.entitlements.Acl;
+import org.opengroup.osdu.core.common.model.legal.Legal;
+import org.opengroup.osdu.util.ElasticUtils;
+import org.opengroup.osdu.util.FileHandler;
+import org.opengroup.osdu.util.HTTPClient;
+import org.opengroup.osdu.util.IndexerClientUtil;
+
+import javax.ws.rs.HttpMethod;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.logging.Logger;
+
+import static org.junit.Assert.assertEquals;
+import static org.opengroup.osdu.util.Config.*;
+import static org.opengroup.osdu.util.HTTPClient.indentatedResponseBody;
+
+@Data
+public class TestIndex {
+ private static final Logger LOGGER = Logger.getLogger(TestIndex.class.getName());
+ private String kind;
+ private String index;
+ private String mappingFile;
+ private String recordFile;
+ private int recordCount;
+ private String schemaFile;
+ private String[] dataGroup;
+ private String[] viewerGroup;
+ private String[] ownerGroup;
+ private HTTPClient httpClient;
+ private Map<String, String> headers;
+ private ElasticUtils elasticUtils;
+ private IndexerClientUtil indexerClientUtil;
+ private Gson gson = new Gson();
+
+ public TestIndex(ElasticUtils elasticUtils){
+ this.elasticUtils = elasticUtils;
+ }
+
+ public void setHttpClient(HTTPClient httpClient) {
+ this.httpClient = httpClient;
+ headers = httpClient.getCommonHeader();
+ this.indexerClientUtil = new IndexerClientUtil(this.httpClient);
+ }
+
+ public void setupIndex() {
+ this.addIndex();
+ List<Map<String, Object>> records = getRecordsFromTestFile();
+ this.recordCount = this.elasticUtils.indexRecords(this.index, this.kind, records);
+ }
+
+ public void setupSchema() {
+ ClientResponse clientResponse = this.httpClient.send(HttpMethod.POST, getStorageBaseURL() + "schemas", this.getStorageSchemaFromJson(), headers, httpClient.getAccessToken());
+ if (clientResponse.getType() != null) {
+ LOGGER.info(String.format("Response status: %s, type: %s\nResponse body: %s", clientResponse.getStatus(), clientResponse.getType().toString(), indentatedResponseBody(clientResponse.getEntity(String.class))));
+ }
+ }
+
+ public void deleteSchema(String kind) {
+ ClientResponse clientResponse = this.httpClient.send(HttpMethod.DELETE, getStorageBaseURL() + "schemas/" + kind, null, headers, httpClient.getAccessToken());
+ assertEquals(204, clientResponse.getStatus());
+ if (clientResponse.getType() != null)
+ LOGGER.info(String.format("Response status: %s, type: %s", clientResponse.getStatus(), clientResponse.getType().toString()));
+ }
+
+ public void addIndex() {
+ this.elasticUtils.createIndex(this.index, this.getIndexMappingFromJson());
+ }
+
+ public void cleanupIndex(String kind) {
+ this.indexerClientUtil.deleteIndex(kind);
+ }
+
+ private String getRecordFile() {
+ return String.format("%s.json", this.recordFile);
+ }
+
+ private String getMappingFile() {
+ return String.format("%s.mapping", this.mappingFile);
+ }
+
+ protected String getSchemaFile() {
+ return String.format("%s.schema", this.schemaFile);
+ }
+
+ private List<Map<String, Object>> getRecordsFromTestFile() {
+ try {
+ String fileContent = FileHandler.readFile(getRecordFile());
+ List<Map<String, Object>> records = new Gson().fromJson(
+ fileContent, new TypeToken<List<Map<String,Object>>>() {}.getType());
+
+ for (Map<String, Object> testRecord : records) {
+ testRecord.put("kind", this.kind);
+ testRecord.put("legal", generateLegalTag());
+ testRecord.put("x-acl", dataGroup);
+ Acl acl = Acl.builder().viewers(viewerGroup).owners(ownerGroup).build();
+ testRecord.put("acl", acl);
+ }
+ return records;
+ } catch (Exception ex) {
+ throw new AssertionError(ex.getMessage());
+ }
+ }
+
+ private String getIndexMappingFromJson() {
+ try {
+ String fileContent = FileHandler.readFile(getMappingFile());
+ JsonElement json = gson.fromJson(fileContent, JsonElement.class);
+ return gson.toJson(json);
+ } catch (Exception e) {
+ throw new AssertionError(e.getMessage());
+ }
+ }
+
+ private String getStorageSchemaFromJson() {
+ try {
+ String fileContent = FileHandler.readFile(getSchemaFile());
+ fileContent = fileContent.replaceAll("KIND_VAL", this.kind);
+ JsonElement json = gson.fromJson(fileContent, JsonElement.class);
+ return gson.toJson(json);
+ } catch (Exception e) {
+ throw new AssertionError(e.getMessage());
+ }
+ }
+
+ private Legal generateLegalTag() {
+ Legal legal = new Legal();
+ Set<String> legalTags = new HashSet<>();
+ legalTags.add(getLegalTag());
+ legal.setLegaltags(legalTags);
+ Set<String> otherRelevantCountries = new HashSet<>();
+ otherRelevantCountries.add(getOtherRelevantDataCountries());
+ legal.setOtherRelevantDataCountries(otherRelevantCountries);
+ return legal;
+ }
+
+}
diff --git a/indexer-acceptance-test/src/test/java/org/opengroup/osdu/models/record/RecordData.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/models/record/RecordData.java
new file mode 100644
index 0000000000000000000000000000000000000000..c60265f31d84396015c7a6581afb703f0cbe7442
--- /dev/null
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/models/record/RecordData.java
@@ -0,0 +1,18 @@
+package org.opengroup.osdu.models.record;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.Map;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class RecordData {
+
+ @JsonProperty("data")
+ private Map<String, Object> data;
+
+ public Map<String, Object> getData() {
+ return data;
+ }
+
+}
diff --git a/indexer-acceptance-test/src/test/java/org/opengroup/osdu/models/schema/PersistentSchemaTestIndex.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/models/schema/PersistentSchemaTestIndex.java
new file mode 100644
index 0000000000000000000000000000000000000000..c930508d274ef8b79dc43dd32e36f6e65405beea
--- /dev/null
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/models/schema/PersistentSchemaTestIndex.java
@@ -0,0 +1,64 @@
+package org.opengroup.osdu.models.schema;
+
+import org.opengroup.osdu.common.SchemaServiceRecordSteps;
+import org.opengroup.osdu.models.TestIndex;
+import org.opengroup.osdu.util.ElasticUtils;
+import org.opengroup.osdu.util.FileHandler;
+import org.opengroup.osdu.util.HTTPClient;
+import org.opengroup.osdu.util.SchemaServiceClient;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+
+public class PersistentSchemaTestIndex extends TestIndex {
+
+ private static final Logger LOGGER = Logger.getLogger(PersistentSchemaTestIndex.class.getName());
+ private final SchemaServiceClient schemaServiceClient;
+ private final SchemaServiceRecordSteps recordSteps;
+ private SchemaModel schemaModel;
+
+ public PersistentSchemaTestIndex(ElasticUtils elasticUtils, HTTPClient client, SchemaServiceRecordSteps recordSteps) {
+ super(elasticUtils);
+ this.schemaServiceClient = new SchemaServiceClient(client);
+ this.recordSteps = recordSteps;
+ }
+
+ @Override
+ public void setupSchema() {
+ this.schemaModel = readSchemaFromJson();
+ SchemaIdentity schemaIdentity = schemaModel.getSchemaInfo().getSchemaIdentity();
+ LOGGER.log(Level.INFO, "Read the schema={0}", schemaIdentity);
+ schemaIdentity.setAuthority(recordSteps.generateActualNameWithoutTs(schemaIdentity.getAuthority()));
+ LOGGER.log(Level.INFO, "Updated the schema={0}", schemaIdentity);
+ schemaServiceClient.createIfNotExist(schemaModel);
+ LOGGER.log(Level.INFO, "Finished setting up the schema={0}", schemaIdentity);
+ }
+
+ @Override
+ public void deleteSchema(String kind) {
+ // The DELETE API is not supported in the Schema service.
+ // In order not to overwhelm a DB with a lots of test schemas
+ // the integration tests create/update a schema per schema file if the schema does not exists
+ // If a developer updates the schema manually, the developer is supposed to update its version as well
+ }
+
+ private SchemaModel readSchemaFromJson(){
+ try {
+ return FileHandler.readFile(getSchemaFile(), SchemaModel.class);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ public SchemaModel getSchemaModel() {
+ return schemaModel;
+ }
+
+ @Override
+ protected String getSchemaFile() {
+ return super.getSchemaFile() + ".json";
+ }
+}
diff --git a/indexer-acceptance-test/src/test/java/org/opengroup/osdu/models/schema/SchemaIdentity.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/models/schema/SchemaIdentity.java
new file mode 100644
index 0000000000000000000000000000000000000000..7a16af67fc6d644970d82aa025ab00addd502656
--- /dev/null
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/models/schema/SchemaIdentity.java
@@ -0,0 +1,81 @@
+package org.opengroup.osdu.models.schema;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+
+import java.util.StringJoiner;
+
+public class SchemaIdentity {
+
+ private String authority;
+ private String source;
+ private String entityType;
+ private String schemaVersionMajor;
+ private String schemaVersionMinor;
+ private String schemaVersionPatch;
+
+ public String getAuthority() {
+ return authority;
+ }
+
+ public void setAuthority(String authority) {
+ this.authority = authority;
+ }
+
+ public String getSource() {
+ return source;
+ }
+
+ public void setSource(String source) {
+ this.source = source;
+ }
+
+ public String getEntityType() {
+ return entityType;
+ }
+
+ public void setEntityType(String entityType) {
+ this.entityType = entityType;
+ }
+
+ public String getSchemaVersionMajor() {
+ return schemaVersionMajor;
+ }
+
+ public void setSchemaVersionMajor(String schemaVersionMajor) {
+ this.schemaVersionMajor = schemaVersionMajor;
+ }
+
+ public String getSchemaVersionMinor() {
+ return schemaVersionMinor;
+ }
+
+ public void setSchemaVersionMinor(String schemaVersionMinor) {
+ this.schemaVersionMinor = schemaVersionMinor;
+ }
+
+ public String getSchemaVersionPatch() {
+ return schemaVersionPatch;
+ }
+
+ public void setSchemaVersionPatch(String schemaVersionPatch) {
+ this.schemaVersionPatch = schemaVersionPatch;
+ }
+
+ @JsonIgnore
+ public String getId() {
+ return authority + ":" + source + ":" + entityType + ":" +
+ schemaVersionMajor + "." + schemaVersionMinor + "." + schemaVersionPatch;
+ }
+
+ @Override
+ public String toString() {
+ return new StringJoiner(", ", SchemaIdentity.class.getSimpleName() + "[", "]")
+ .add("AUTHORITY='" + authority + "'")
+ .add("SOURCE='" + source + "'")
+ .add("ENTITY_TYPE='" + entityType + "'")
+ .add("schemaVersionMajor='" + schemaVersionMajor + "'")
+ .add("schemaVersionMinor='" + schemaVersionMinor + "'")
+ .add("schemaVersionPatch='" + schemaVersionPatch + "'")
+ .toString();
+ }
+}
diff --git a/indexer-acceptance-test/src/test/java/org/opengroup/osdu/models/schema/SchemaInfo.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/models/schema/SchemaInfo.java
new file mode 100644
index 0000000000000000000000000000000000000000..88904aac2163db1215c35d53942d4b8cf5fea9ce
--- /dev/null
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/models/schema/SchemaInfo.java
@@ -0,0 +1,37 @@
+package org.opengroup.osdu.models.schema;
+
+import java.util.StringJoiner;
+
+public class SchemaInfo {
+
+ private SchemaIdentity schemaIdentity;
+ private SchemaStatus status;
+
+ public SchemaIdentity getSchemaIdentity() {
+ return schemaIdentity;
+ }
+
+ public void setSchemaIdentity(SchemaIdentity schemaIdentity) {
+ this.schemaIdentity = schemaIdentity;
+ }
+
+ public SchemaStatus getStatus() {
+ return status;
+ }
+
+ public void setStatus(SchemaStatus status) {
+ this.status = status;
+ }
+
+ @Override
+ public String toString() {
+ return new StringJoiner(", ", SchemaInfo.class.getSimpleName() + "[", "]")
+ .add("schemaIdentity=" + schemaIdentity)
+ .add("status=" + status)
+ .toString();
+ }
+
+ public enum SchemaStatus {
+ PUBLISHED, OBSOLETE, DEVELOPMENT
+ }
+}
diff --git a/indexer-acceptance-test/src/test/java/org/opengroup/osdu/models/schema/SchemaModel.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/models/schema/SchemaModel.java
new file mode 100644
index 0000000000000000000000000000000000000000..2a15bc3f903149f595fc0fd34dae8404401d140f
--- /dev/null
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/models/schema/SchemaModel.java
@@ -0,0 +1,33 @@
+package org.opengroup.osdu.models.schema;
+
+import java.util.StringJoiner;
+
+public class SchemaModel {
+
+ private SchemaInfo schemaInfo;
+ private Object schema;
+
+ public Object getSchema() {
+ return schema;
+ }
+
+ public void setSchema(Object schema) {
+ this.schema = schema;
+ }
+
+ public SchemaInfo getSchemaInfo() {
+ return schemaInfo;
+ }
+
+ public void setSchemaInfo(SchemaInfo schemaInfo) {
+ this.schemaInfo = schemaInfo;
+ }
+
+ @Override
+ public String toString() {
+ return new StringJoiner(", ", SchemaModel.class.getSimpleName() + "[", "]")
+ .add("schemaInfo=" + schemaInfo)
+ .add("schema=" + schema)
+ .toString();
+ }
+}
diff --git a/indexer-acceptance-test/src/test/java/org/opengroup/osdu/request/CursorQuery.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/request/CursorQuery.java
new file mode 100644
index 0000000000000000000000000000000000000000..52cb63fa9a2c25b20c4b07428ab70a1067636530
--- /dev/null
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/request/CursorQuery.java
@@ -0,0 +1,24 @@
+package org.opengroup.osdu.request;
+
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.List;
+
+@Data
+@NoArgsConstructor
+public class CursorQuery {
+ private String cursor;
+ private String kind;
+ private int limit;
+ private String query;
+ private List<String> returnedFields;
+ private SortQuery sort;
+ private Boolean queryAsOwner;
+ private SpatialFilter spatialFilter;
+
+ @Override
+ public String toString() {
+ return new com.google.gson.GsonBuilder().disableHtmlEscaping().create().toJson(this);
+ }
+}
diff --git a/indexer-acceptance-test/src/test/java/org/opengroup/osdu/request/Query.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/request/Query.java
new file mode 100644
index 0000000000000000000000000000000000000000..65962ce54833e905c98df16ae1a941f72379d999
--- /dev/null
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/request/Query.java
@@ -0,0 +1,26 @@
+package org.opengroup.osdu.request;
+
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.List;
+
+@Data
+@NoArgsConstructor
+public class Query {
+
+ private String kind;
+ private Integer offset;
+ private Integer limit;
+ private String query;
+ private SortQuery sort;
+ private Boolean queryAsOwner;
+ private String aggregateBy;
+ private List<String> returnedFields;
+ private SpatialFilter spatialFilter;
+
+ @Override
+ public String toString() {
+ return new com.google.gson.Gson().toJson(this);
+ }
+}
diff --git a/indexer-acceptance-test/src/test/java/org/opengroup/osdu/request/SortQuery.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/request/SortQuery.java
new file mode 100644
index 0000000000000000000000000000000000000000..1918014f440765717fcf6df6f2dd8edc732ab2f4
--- /dev/null
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/request/SortQuery.java
@@ -0,0 +1,14 @@
+package org.opengroup.osdu.request;
+
+import co.elastic.clients.elasticsearch._types.SortOrder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.List;
+
+@Data
+@NoArgsConstructor
+public class SortQuery {
+ private List<String> field;
+ private List<SortOrder> order;
+}
diff --git a/indexer-acceptance-test/src/test/java/org/opengroup/osdu/request/SpatialFilter.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/request/SpatialFilter.java
new file mode 100644
index 0000000000000000000000000000000000000000..c8d3eb68c7f4ba4e49bc70a34af00a1406135fa3
--- /dev/null
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/request/SpatialFilter.java
@@ -0,0 +1,40 @@
+package org.opengroup.osdu.request;
+
+import lombok.*;
+
+import java.util.List;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+@Setter
+public class SpatialFilter {
+ String field;
+ ByBoundingBox byBoundingBox;
+ ByDistance byDistance;
+ ByGeoPolygon byGeoPolygon;
+
+ @Builder
+ public static class ByDistance {
+ Coordinates point;
+ int distance;
+ }
+
+ @Builder
+ public static class ByBoundingBox {
+ Coordinates topLeft;
+ Coordinates bottomRight;
+ }
+
+ @Builder
+ public static class Coordinates {
+ Double latitude;
+ Double longitude;
+ }
+
+ @Builder
+ public static class ByGeoPolygon {
+ List<Coordinates> points;
+ }
+}
\ No newline at end of file
diff --git a/indexer-acceptance-test/src/test/java/org/opengroup/osdu/response/ErrorResponseMock.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/response/ErrorResponseMock.java
new file mode 100644
index 0000000000000000000000000000000000000000..20d0b943867335229ad846d60e6d99b3feceecfc
--- /dev/null
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/response/ErrorResponseMock.java
@@ -0,0 +1,15 @@
+package org.opengroup.osdu.response;
+
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+
+import java.util.List;
+
+@Data
+@EqualsAndHashCode(callSuper = true)
+public class ErrorResponseMock extends ResponseBase {
+ private List<String> errors;
+ private String code;
+ private String reason;
+ private String message;
+}
\ No newline at end of file
diff --git a/indexer-acceptance-test/src/test/java/org/opengroup/osdu/response/InfoResponseMock.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/response/InfoResponseMock.java
new file mode 100644
index 0000000000000000000000000000000000000000..682226c3d88f4b1a09195cddba665711c581c639
--- /dev/null
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/response/InfoResponseMock.java
@@ -0,0 +1,16 @@
+package org.opengroup.osdu.response;
+
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+
+@Data
+@EqualsAndHashCode(callSuper = true)
+public class InfoResponseMock extends ResponseBase {
+ private String groupId;
+ private String artifactId;
+ private String version;
+ private String buildTime;
+ private String branch;
+ private String commitId;
+ private String commitMessage;
+}
\ No newline at end of file
diff --git a/indexer-acceptance-test/src/test/java/org/opengroup/osdu/response/ResponseBase.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/response/ResponseBase.java
new file mode 100644
index 0000000000000000000000000000000000000000..7e981ba8f4edd467cac3a28f5a57ab98edfc51b8
--- /dev/null
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/response/ResponseBase.java
@@ -0,0 +1,11 @@
+package org.opengroup.osdu.response;
+
+import lombok.Data;
+
+import javax.ws.rs.core.MultivaluedMap;
+
+@Data
+public abstract class ResponseBase {
+ private int responseCode;
+ private MultivaluedMap<String, String> headers;
+}
\ No newline at end of file
diff --git a/indexer-acceptance-test/src/test/java/org/opengroup/osdu/response/ResponseMock.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/response/ResponseMock.java
new file mode 100644
index 0000000000000000000000000000000000000000..510701e438bcd113ff944253285a77e7bac9e2d0
--- /dev/null
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/response/ResponseMock.java
@@ -0,0 +1,16 @@
+package org.opengroup.osdu.response;
+
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+
+import java.util.List;
+import java.util.Map;
+
+@Data
+@EqualsAndHashCode(callSuper = true)
+public class ResponseMock extends ResponseBase {
+ private List<Map<String, Object>> results;
+ private List<Object> aggregations;
+ private long totalCount;
+ private String cursor;
+}
\ No newline at end of file
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/web/package-info.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/step_definitions/info/RunTest.java
similarity index 59%
rename from provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/web/package-info.java
rename to indexer-acceptance-test/src/test/java/org/opengroup/osdu/step_definitions/info/RunTest.java
index 45e225e75466a19c067498cbfb312e058771dcda..dff1d53fe02bb79e20ab8415a256e96f444c416b 100644
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/web/package-info.java
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/step_definitions/info/RunTest.java
@@ -14,8 +14,18 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- * This module is used to keep minimum required bean configuration in the web environment required for request processing. All beans will be placed in a
- * separate context.
- */
-package org.opengroup.osdu.indexer.provider.gcp.web;
+
+package org.opengroup.osdu.step_definitions.info;
+
+import cucumber.api.CucumberOptions;
+import cucumber.api.junit.Cucumber;
+import org.junit.runner.RunWith;
+
+@RunWith(Cucumber.class)
+@CucumberOptions(
+ features = "classpath:features/info/Info.feature",
+ glue = {"classpath:org.opengroup.osdu.step_definitions.info"},
+ plugin = {"pretty", "junit:target/cucumber-reports/TEST-info.xml"})
+public class RunTest {
+
+}
diff --git a/indexer-acceptance-test/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java
new file mode 100644
index 0000000000000000000000000000000000000000..c1d5ef080e47e293ffa004c02b659e484cbf0656
--- /dev/null
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright 2020-2022 Google LLC
+ * Copyright 2020-2022 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.step_definitions.info;
+
+import cucumber.api.Scenario;
+import cucumber.api.java.Before;
+import cucumber.api.java.en.Then;
+import cucumber.api.java.en.When;
+import org.opengroup.osdu.common.info.InfoSteps;
+import org.opengroup.osdu.util.AnthosHTTPClient;
+import org.opengroup.osdu.util.Config;
+
+public class Steps extends InfoSteps {
+
+ public Steps() {
+ super(new AnthosHTTPClient());
+ }
+
+ @Before
+ public void before(Scenario scenario) {
+ this.scenario = scenario;
+ this.httpClient = new AnthosHTTPClient();
+ }
+
+ /******************Inputs being set**************/
+
+ @When("^I send get request to version info endpoint$")
+ public void i_send_get_request_to_version_info_endpoint() {
+ super.i_send_get_request_to_version_info_endpoint();
+ }
+
+ @When("^I send get request to version info endpoint with trailing slash$")
+ public void i_send_get_request_to_version_info_endpoint_with_trailing_slash() {
+ super.i_send_get_request_to_version_info_endpoint_with_trailing_slash();
+ }
+
+ /******************Assert final response**************/
+
+ @Then("^I should get version info in response$")
+ public void i_should_get_version_info_in_response() {
+ super.i_should_get_version_info_in_response();
+ }
+
+ @Override
+ protected String getHttpMethod() {
+ return "GET";
+ }
+
+ @Override
+ protected String getApi() {
+ return Config.getIndexerBaseURL() + "info";
+ }
+}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/package-info.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/step_definitions/record/RunTest.java
similarity index 58%
rename from provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/package-info.java
rename to indexer-acceptance-test/src/test/java/org/opengroup/osdu/step_definitions/record/RunTest.java
index 391c0383a79f364f2ad3465bb3842abcc9d51dfc..b6a3467565aadd39849257e391f962702629daa7 100644
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/package-info.java
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/step_definitions/record/RunTest.java
@@ -14,8 +14,18 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- * This module is used to provide common bean configuration, for both web app context that serves user request, and non-web app context that process indexing
- * events.
- */
-package org.opengroup.osdu.indexer.provider.gcp.common;
+
+package org.opengroup.osdu.step_definitions.record;
+
+import cucumber.api.CucumberOptions;
+import cucumber.api.junit.Cucumber;
+import org.junit.runner.RunWith;
+
+@RunWith(Cucumber.class)
+@CucumberOptions(
+ features = "classpath:features/indexrecord/indexRecord-schema-service.feature",
+ glue = {"classpath:org.opengroup.osdu.step_definitions.record"},
+ plugin = {"pretty", "junit:target/cucumber-reports/TEST-indexrecord.xml"})
+public class RunTest {
+
+}
\ No newline at end of file
diff --git a/indexer-acceptance-test/src/test/java/org/opengroup/osdu/step_definitions/record/Steps.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/step_definitions/record/Steps.java
new file mode 100644
index 0000000000000000000000000000000000000000..b5afa828c1bf557082dbeb14641bbef5d24ef8d3
--- /dev/null
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/step_definitions/record/Steps.java
@@ -0,0 +1,164 @@
+/*
+ * Copyright 2020-2022 Google LLC
+ * Copyright 2020-2022 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.step_definitions.record;
+
+import cucumber.api.DataTable;
+import cucumber.api.Scenario;
+import cucumber.api.java.Before;
+import cucumber.api.java.en.Given;
+import cucumber.api.java.en.Then;
+import cucumber.api.java.en.When;
+import lombok.extern.java.Log;
+import org.opengroup.osdu.common.SchemaServiceRecordSteps;
+import org.opengroup.osdu.util.AnthosHTTPClient;
+import org.opengroup.osdu.util.ElasticUtils;
+import org.opengroup.osdu.util.conf.AnthosConfig;
+
+@Log
+public class Steps extends SchemaServiceRecordSteps {
+
+ public Steps() {
+ super(new AnthosHTTPClient(), new ElasticUtils());
+ }
+
+ @Before
+ public void before(Scenario scenario) {
+ AnthosConfig.updateEntitlementsDomainVariable();
+ this.scenario = scenario;
+ this.httpClient = new AnthosHTTPClient();
+ }
+
+ @Given("^the schema is created with the following kind$")
+ public void the_schema_is_created_with_the_following_kind(DataTable dataTable) {
+ super.the_schema_is_created_with_the_following_kind(dataTable);
+ }
+
+ @Then("^I set starting stateful scenarios$")
+ public void i_set_starting_stateful_scenarios() throws Throwable {
+ super.i_set_scenarios_as_stateful(true);
+ }
+
+ @Then("^I set ending stateful scenarios$")
+ public void i_set_ending_stateful_scenarios() throws Throwable {
+ super.i_set_scenarios_as_stateful(false);
+ }
+
+ @When("^I ingest records with the \"(.*?)\" with \"(.*?)\" for a given \"(.*?)\"$")
+ public void i_ingest_records_with_the_for_a_given(String record, String dataGroup, String kind) {
+ super.i_ingest_records_with_the_for_a_given(record, dataGroup, kind);
+ }
+
+ @When("^I create index with \"(.*?)\" for a given \"(.*?)\" and \"(.*?)\"$")
+ public void i_create_index_with_mapping_file_for_a_given_kind(String mappingFile, String index, String kind) throws Throwable {
+ super.i_create_index_with_mapping_file_for_a_given_kind(mappingFile, index, kind);
+ }
+
+ @Then("^I should get the (\\d+) documents for the \"([^\"]*)\" in the Elastic Search$")
+ public void i_should_get_the_documents_for_the_in_the_Elastic_Search(int expectedCount, String index) throws Throwable {
+ super.i_should_get_the_documents_for_the_in_the_Elastic_Search(expectedCount, index);
+ }
+
+ @Then("^I should not get any documents for the \"([^\"]*)\" in the Elastic Search$")
+ public void i_should_not_get_any_documents_for_the_index_in_the_Elastic_Search(String index) throws Throwable {
+ super.i_should_not_get_any_documents_for_the_index_in_the_Elastic_Search(index);
+ }
+
+ @Then("^I should get the elastic \"(.*?)\" for the \"([^\"]*)\" and \"([^\"]*)\" in the Elastic Search$")
+ public void i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(String expectedMapping, String kind, String index)
+ throws Throwable {
+ super.i_should_get_the_elastic_for_the_tenant_testindex_timestamp_well_in_the_Elastic_Search(expectedMapping, kind, index);
+ }
+
+ @Then("^I can validate indexed meta attributes for the \"([^\"]*)\" and given \"([^\"]*)\"$")
+ public void i_can_validate_indexed_meta_attributes(String index, String kind) throws Throwable {
+ super.i_can_validate_indexed_attributes(index, kind);
+ }
+
+ @Then("^I should get the (\\d+) documents for the \"([^\"]*)\" in the Elastic Search with out \"(.*?)\"$")
+ public void iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(int expectedCount, String index, String skippedAttributes)
+ throws Throwable {
+ super.iShouldGetTheNumberDocumentsForTheIndexInTheElasticSearchWithOutSkippedAttribute(expectedCount, index, skippedAttributes);
+ }
+
+ @Then("^I should be able to search (\\d+) record with index \"([^\"]*)\" by tag \"([^\"]*)\" and value \"([^\"]*)\"$")
+ public void iShouldBeAbleToSearchRecordByTagKeyAndTagValue(int expectedNumber, String index, String tagKey, String tagValue) throws Throwable {
+ super.iShouldBeAbleToSearchRecordByTagKeyAndTagValue(index, tagKey, tagValue, expectedNumber);
+ }
+
+ @Then("^I clean up the index of the extended kinds \"([^\"]*)\" in the Elastic Search$")
+ public void iShouldCleanupIndicesOfExtendedKinds(String extendedKinds) throws Throwable {
+ super.iShouldCleanupIndicesOfExtendedKinds(extendedKinds);
+ }
+
+ @Then("^I should be able to search (\\d+) record with index \"([^\"]*)\" by extended data field \"([^\"]*)\" and value \"([^\"]*)\"$")
+ public void iShouldBeAbleToSearchRecordByFieldAndFieldValue(int expectedNumber, String index, String fieldKey, String fieldValue) throws Throwable {
+ super.iShouldBeAbleToSearchRecordByFieldAndFieldValue(index, fieldKey, fieldValue, expectedNumber);
+ }
+
+ @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by bounding box query with points \\((-?\\d+), (-?\\d+)\\) and \\((-?\\d+), (-?\\d+)\\) on field \"([^\"]*)\"$")
+ public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery(
+ int expectedCount, String index, Double topLatitude, Double topLongitude, Double bottomLatitude, Double bottomLongitude, String field)
+ throws Throwable {
+ String actualName = generateActualName(index, null);
+ super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_geoQuery(expectedCount, actualName, topLatitude, topLongitude, bottomLatitude,
+ bottomLongitude, field);
+ }
+
+ @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by nested \"([^\"]*)\" and properties \\(\"([^\"]*)\", (\\d+)\\) and \\(\"([^\"]*)\", \"([^\"]*)\"\\)$")
+ public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery(
+ int expectedCount, String index, String path, String firstNestedProperty, String firstNestedValue, String secondNestedProperty,
+ String secondNestedValue) throws Throwable {
+ String actualName = generateActualName(index, null);
+ super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_nestedQuery(expectedCount, actualName, path, firstNestedProperty, firstNestedValue,
+ secondNestedProperty, secondNestedValue);
+ }
+
+ @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by flattened inner properties \\(\"([^\"]*)\", \"([^\"]*)\"\\)$")
+ public void i_should_be_able_search_documents_for_the_by_flattened_inner_properties(int expectedCount, String index, String flattenedField,
+ String flattenedFieldValue) throws Throwable {
+ String actualName = generateActualName(index, null);
+ super.i_should_be_able_search_documents_for_the_by_flattened_inner_properties(expectedCount, actualName, flattenedField, flattenedFieldValue);
+
+ }
+
+ @Then("^I should get \"([^\"]*)\" in response, without hints in schema for the \"([^\"]*)\" that present in the \"([^\"]*)\" with \"([^\"]*)\" for a given \"([^\"]*)\"$")
+ public void i_should_get_object_in_search_response_without_hints_in_schema(String objectInnerField, String index, String recordFile, String acl,
+ String kind)
+ throws Throwable {
+ String actualName = generateActualName(index, null);
+ super.i_should_get_object_in_search_response_without_hints_in_schema(objectInnerField, actualName, recordFile, acl, kind);
+ }
+
+ @Then("^I should be able to search for record from \"([^\"]*)\" by \"([^\"]*)\" for value \"([^\"]*)\" and find String arrays in \"([^\"]*)\" with \"([^\"]*)\"$")
+ public void i_should_get_string_array_in_search_response(String index, String field, String fieldValue, String arrayField, String arrayValue)
+ throws Throwable {
+ super.i_should_get_string_array_in_search_response(index, field, fieldValue, arrayField, arrayValue);
+ }
+
+ @Then("^I should get \"([^\"]*)\" in search response for the \"([^\"]*)\"$")
+ public void i_should_get_object_in_search_response(String innerField, String index)
+ throws Throwable {
+ super.i_should_get_object_in_search_response(innerField, index);
+ }
+
+ @Then("^I should be able search (\\d+) documents for the \"([^\"]*)\" by bounding box query with points \\((-?\\d+), (-?\\d+)\\) on field \"([^\"]*)\" and points \\((-?\\d+), (-?\\d+)\\) on field \"([^\"]*)\"$")
+ public void i_should_get_the_documents_for_the_in_the_Elastic_Search_by_AsIngestedCoordinates (
+ int expectedCount, String index, Double topPointX, Double bottomPointX, String pointX, Double topPointY, Double bottomPointY, String pointY) throws Throwable {
+ super.i_should_get_the_documents_for_the_in_the_Elastic_Search_by_AsIngestedCoordinates(expectedCount, index, topPointX, bottomPointX, pointX, topPointY, bottomPointY, pointY);
+ }
+}
diff --git a/indexer-acceptance-test/src/test/java/org/opengroup/osdu/util/AnthosHTTPClient.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/util/AnthosHTTPClient.java
new file mode 100644
index 0000000000000000000000000000000000000000..d74aafa32f463ef54a6364aa745aef372b286954
--- /dev/null
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/util/AnthosHTTPClient.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2020-2022 Google LLC
+ * Copyright 2020-2022 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.util;
+
+import com.google.common.base.Strings;
+import lombok.ToString;
+import lombok.extern.java.Log;
+
+@Log
+@ToString
+public class AnthosHTTPClient extends HTTPClient {
+
+ public static final String PRIVILEGED_USER_TOKEN = "ROOT_USER_TOKEN";
+ private static String token = null;
+ private static OpenIDTokenProvider openIDTokenProvider;
+
+ public AnthosHTTPClient() {
+ token = System.getProperty(PRIVILEGED_USER_TOKEN, System.getenv(PRIVILEGED_USER_TOKEN));
+
+ if (Strings.isNullOrEmpty(token)) {
+ openIDTokenProvider = new OpenIDTokenProvider();
+ }
+ }
+
+ @Override
+ public synchronized String getAccessToken() {
+ if (Strings.isNullOrEmpty(token)) {
+ token = openIDTokenProvider.getToken();
+ }
+ return "Bearer " + token;
+ }
+}
diff --git a/indexer-acceptance-test/src/test/java/org/opengroup/osdu/util/Config.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/util/Config.java
new file mode 100644
index 0000000000000000000000000000000000000000..fc9e92bd5af81b2d62e996858797cae93cd4c8b2
--- /dev/null
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/util/Config.java
@@ -0,0 +1,141 @@
+package org.opengroup.osdu.util;
+
+public class Config {
+
+ private static final String DEFAULT_ELASTIC_HOST = "";
+ private static final String DEFAULT_ELASTIC_USER_NAME = "";
+ private static final String DEFAULT_ELASTIC_PASSWORD = "";
+ private static final int PORT = 9243;
+ private static final String DEFAULT_ELASTIC_SSL_ENABLED = "true";
+
+ private static final String DEFAULT_INDEXER_HOST = "";
+ private static final String DEFAULT_SEARCH_HOST = "";
+ private static final String DEFAULT_STORAGE_HOST = "";
+ private static final String DEFAULT_HOST = "";
+ private static final String DEFAULT_DATA_PARTITION_ID_TENANT1 = "";
+ private static final String DEFAULT_DATA_PARTITION_ID_TENANT2 = "";
+ private static final String DEFAULT_SEARCH_INTEGRATION_TESTER = "";
+
+ private static final String DEFAULT_LEGAL_TAG = "";
+ private static final String DEFAULT_OTHER_RELEVANT_DATA_COUNTRIES = "";
+
+ private static final String DEFAULT_ENTITLEMENTS_DOMAIN = "";
+
+ private static final String SCHEMA_PATH = "/api/schema-service/v1";
+ private static final String DEFAULT_SECURITY_HTTPS_CERTIFICATE_TRUST = "false";
+
+
+ public static int getPort() {
+ return Integer.parseInt(getEnvironmentVariableOrDefaultValue("ELASTIC_PORT", String.valueOf(PORT)));
+ }
+
+ public static String getUserName() {
+ return getEnvironmentVariableOrDefaultValue("ELASTIC_USER_NAME", DEFAULT_ELASTIC_USER_NAME);
+ }
+
+ public static String getPassword() {
+ return getEnvironmentVariableOrDefaultValue("ELASTIC_PASSWORD", DEFAULT_ELASTIC_PASSWORD);
+ }
+
+ public static String getElasticHost() {
+ return getEnvironmentVariableOrDefaultValue("ELASTIC_HOST", DEFAULT_ELASTIC_HOST);
+ }
+
+ public static boolean isElasticSslEnabled() {
+ return Boolean.parseBoolean(getEnvironmentVariableOrDefaultValue("ELASTIC_SSL_ENABLED", DEFAULT_ELASTIC_SSL_ENABLED));
+ }
+
+ public static int getElastic8Port() {
+ return Integer.parseInt(getEnvironmentVariableOrDefaultValue("ELASTIC_8_PORT", String.valueOf(getPort())));
+ }
+
+ public static String getElastic8UserName() {
+ return getEnvironmentVariableOrDefaultValue("ELASTIC_8_USER_NAME", getUserName());
+ }
+
+ public static String getElastic8Password() {
+ return getEnvironmentVariableOrDefaultValue("ELASTIC_8_PASSWORD", getPassword());
+ }
+
+ public static String getElastic8Host() {
+ return getEnvironmentVariableOrDefaultValue("ELASTIC_8_HOST", getElasticHost());
+ }
+
+ public static boolean isElastic8SslEnabled() {
+ return Boolean.parseBoolean(getEnvironmentVariableOrDefaultValue("ELASTIC_8_SSL_ENABLED", String.valueOf(isElasticSslEnabled())));
+ }
+
+ public static String getOtherRelevantDataCountries() {
+ return getEnvironmentVariableOrDefaultValue("OTHER_RELEVANT_DATA_COUNTRIES", DEFAULT_OTHER_RELEVANT_DATA_COUNTRIES);
+ }
+
+ public static String getLegalTag() {
+ return getEnvironmentVariableOrDefaultValue("LEGAL_TAG", DEFAULT_LEGAL_TAG);
+ }
+
+ public static String getKeyValue() {
+ return getEnvironmentVariableOrDefaultValue("SEARCH_INTEGRATION_TESTER", DEFAULT_SEARCH_INTEGRATION_TESTER);
+ }
+
+ public static String getDataPartitionIdTenant1() {
+ return getEnvironmentVariableOrDefaultValue("DEFAULT_DATA_PARTITION_ID_TENANT1", DEFAULT_DATA_PARTITION_ID_TENANT1);
+ }
+
+ public static String getDataPartitionIdTenant2() {
+ return getEnvironmentVariableOrDefaultValue("DEFAULT_DATA_PARTITION_ID_TENANT2", DEFAULT_DATA_PARTITION_ID_TENANT2);
+ }
+
+ public static String getIndexerBaseURL() {
+ return getEnvironmentVariableOrDefaultValue("INDEXER_HOST", DEFAULT_INDEXER_HOST);
+ }
+
+ public static String getSearchBaseURL() {
+ return getEnvironmentVariableOrDefaultValue("SEARCH_HOST", DEFAULT_SEARCH_HOST);
+ }
+
+ public static String getStorageBaseURL() {
+ return getEnvironmentVariableOrDefaultValue("STORAGE_HOST", DEFAULT_STORAGE_HOST);
+ }
+
+ public static String getSchemaBaseURL() {
+ return getEnvironmentVariableOrDefaultValue("HOST", DEFAULT_HOST) + SCHEMA_PATH;
+ }
+
+ public static String getEntitlementsDomain() {
+ return getEnvironmentVariableOrDefaultValue("ENTITLEMENTS_DOMAIN", DEFAULT_ENTITLEMENTS_DOMAIN);
+ }
+
+ public static String getAWSCognitoClientId() {
+ return getEnvironmentVariableOrDefaultValue("AWS_COGNITO_CLIENT_ID", "");
+ }
+
+ public static String getAWSCognitoAuthFlow() {
+ return getEnvironmentVariableOrDefaultValue("AWS_COGNITO_AUTH_FLOW", "");
+ }
+
+ public static String getAWSCognitoUser() {
+ return getEnvironmentVariableOrDefaultValue("AWS_COGNITO_AUTH_PARAMS_USER", "");
+ }
+
+ public static String getAWSCognitoPassword() {
+ return getEnvironmentVariableOrDefaultValue("AWS_COGNITO_AUTH_PARAMS_PASSWORD", "");
+ }
+
+ public static boolean isSecurityHttpsCertificateTrust() {
+ return Boolean.parseBoolean(
+ getEnvironmentVariableOrDefaultValue("SECURITY_HTTPS_CERTIFICATE_TRUST",
+ DEFAULT_SECURITY_HTTPS_CERTIFICATE_TRUST));
+ }
+
+ private static String getEnvironmentVariableOrDefaultValue(String key, String defaultValue) {
+ String environmentVariable = getEnvironmentVariable(key);
+ if (environmentVariable == null) {
+ environmentVariable = defaultValue;
+ }
+ return environmentVariable;
+ }
+
+ private static String getEnvironmentVariable(String propertyKey) {
+ return System.getProperty(propertyKey, System.getenv(propertyKey));
+ }
+}
diff --git a/indexer-acceptance-test/src/test/java/org/opengroup/osdu/util/ElasticUtils.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/util/ElasticUtils.java
new file mode 100644
index 0000000000000000000000000000000000000000..0f767a8387058fc2e87cf3584e842b9f3fef78db
--- /dev/null
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/util/ElasticUtils.java
@@ -0,0 +1,681 @@
+/*
+ Copyright 2020 Google LLC
+ Copyright 2020 EPAM Systems, Inc
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+package org.opengroup.osdu.util;
+
+import co.elastic.clients.elasticsearch.ElasticsearchClient;
+import co.elastic.clients.elasticsearch._types.ElasticsearchException;
+import co.elastic.clients.elasticsearch._types.FieldValue;
+import co.elastic.clients.elasticsearch._types.GeoShapeRelation;
+import co.elastic.clients.elasticsearch._types.Time;
+import co.elastic.clients.elasticsearch._types.mapping.TypeMapping;
+import co.elastic.clients.elasticsearch._types.query_dsl.*;
+import co.elastic.clients.elasticsearch.core.*;
+import co.elastic.clients.elasticsearch.core.bulk.BulkOperation;
+import co.elastic.clients.elasticsearch.core.bulk.IndexOperation;
+import co.elastic.clients.elasticsearch.core.search.Hit;
+import co.elastic.clients.elasticsearch.core.search.HitsMetadata;
+import co.elastic.clients.elasticsearch.indices.ExistsRequest;
+import co.elastic.clients.elasticsearch.indices.*;
+import co.elastic.clients.elasticsearch.indices.get_mapping.IndexMappingRecord;
+import co.elastic.clients.json.JsonData;
+import co.elastic.clients.json.jackson.JacksonJsonpMapper;
+import co.elastic.clients.transport.rest_client.RestClientTransport;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.google.api.client.util.Strings;
+import com.google.gson.Gson;
+import lombok.extern.java.Log;
+import org.apache.http.Header;
+import org.apache.http.HttpHost;
+import org.apache.http.HttpStatus;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
+import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
+import org.apache.http.message.BasicHeader;
+import org.apache.http.ssl.SSLContextBuilder;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestClientBuilder;
+import org.opengroup.osdu.core.common.model.storage.Record;
+import org.opengroup.osdu.models.record.RecordData;
+
+import javax.net.ssl.SSLContext;
+import java.io.IOException;
+import java.io.StringReader;
+import java.lang.reflect.Type;
+import java.net.URI;
+import java.security.KeyManagementException;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
+
+import static org.opengroup.osdu.common.RecordSteps.X_COLLABORATION;
+
+
+/**
+ * All util methods to use elastic apis for tests
+ * It should be used only in the Setup or TearDown phase of the test
+ */
+@Log
+public class ElasticUtils {
+
+ private static final int REST_CLIENT_CONNECT_TIMEOUT = 5000;
+ private static final int REST_CLIENT_SOCKET_TIMEOUT = 60000;
+
+ private final Time REQUEST_TIMEOUT = Time.of(builder -> builder.time("1m"));
+ private final String username;
+ private final String password;
+ private final String host;
+ private final boolean sslEnabled;
+ private ElasticsearchClient elasticsearchClient;
+
+ public ElasticUtils() {
+ this.username = Config.getElastic8UserName();
+ this.password = Config.getElastic8Password();
+ this.host = Config.getElastic8Host();
+ this.sslEnabled = Config.isElastic8SslEnabled();
+ }
+
+ public void createIndex(String index, String mapping) {
+ try {
+ ElasticsearchClient client = this.getOrCreateClient(username, password, host);
+
+ IndexSettings indexSettings = IndexSettings.of(builder ->
+ builder.numberOfShards("1")
+ .numberOfReplicas("1")
+ );
+
+ // creating index + add mapping to the index
+ log.info("Creating index with name: " + index);
+ CreateIndexRequest request = CreateIndexRequest.of(builder ->
+ builder.index(index)
+ .settings(indexSettings)
+ .mappings(TypeMapping.of(
+ mappingBuilder -> mappingBuilder.withJson(new StringReader(mapping))))
+ .timeout(REQUEST_TIMEOUT)
+ );
+
+ CreateIndexResponse createIndexResponse = client.indices().create(request);
+
+ //wait for ack
+ for (int i = 0; ; i++) {
+ if (createIndexResponse.acknowledged() && createIndexResponse.shardsAcknowledged()) {
+ break;
+ } else {
+ log.info("Failed to get confirmation from elastic server, will sleep for 15 seconds");
+ Thread.sleep(15000);
+ if (i > 3) {
+ log.info("Failed to get confirmation from elastic server after 3 retries");
+ throw new AssertionError("Failed to get confirmation from Elastic cluster");
+ }
+ }
+ }
+
+ log.info("Done creating index with name: " + index);
+
+ } catch (ElasticsearchException e) {
+ if (e.status() == HttpStatus.SC_BAD_REQUEST &&
+ (e.getMessage().contains("resource_already_exists_exception") || e.getMessage().contains("IndexAlreadyExistsException"))) {
+ log.info("Index already exists. Ignoring error...");
+ }
+ } catch (Exception e) {
+ throw new AssertionError(e.getMessage());
+ }
+ }
+
+ public int indexRecords(String index, String kind, List<Map<String, Object>> testRecords) {
+ log.info("Creating records inside index with name: " + index);
+
+ BulkRequest.Builder bulkRequestBuilder = new BulkRequest.Builder();
+ bulkRequestBuilder.timeout(REQUEST_TIMEOUT);
+
+ List<IndexOperation<Map<String, Object>>> records = ElasticUtils.getIndexReqFromRecord(index, kind, testRecords);
+ for (IndexOperation<Map<String, Object>> operation : records) {
+ bulkRequestBuilder.operations(new BulkOperation.Builder().index(operation).build());
+ }
+
+ BulkResponse bulkResponse = null;
+ try {
+ ElasticsearchClient client = this.getOrCreateClient(username, password, host);
+ bulkResponse = client.bulk(bulkRequestBuilder.build());
+ log.info("Done creating records inside index with name: " + index);
+
+ } catch (IOException e) {
+ log.log(Level.SEVERE, "bulk indexing failed", e);
+ }
+
+ // Double check failures
+ if (bulkResponse != null && bulkResponse.errors()) {
+ throw new AssertionError("setup failed in data post to Index");
+ }
+
+ try {
+ ElasticsearchClient client = this.getOrCreateClient(username, password, host);
+
+ RefreshRequest request = RefreshRequest.of(builder -> builder.index(index));
+ RefreshResponse refreshResponse = client.indices().refresh(request);
+ log.info(String.format("refreshed index, acknowledged shards: %s | failed shards: %s | total shards: %s ",
+ refreshResponse.shards().successful(), refreshResponse.shards().failed(),
+ refreshResponse.shards().total()));
+
+ } catch (IOException | ElasticsearchException e) {
+ log.log(Level.SEVERE, "index refresh failed", e);
+ }
+
+ return records.size();
+ }
+
+ public void deleteIndex(String index) {
+ ElasticsearchClient client = this.getOrCreateClient(username, password, host);
+ try{
+ //retry if the elastic cluster is snapshotting and we cant delete it
+ for (int retries = 0; ; retries++) {
+ try {
+ log.info("Deleting index with name: " + index + ", retry count: " + retries);
+ DeleteIndexRequest request = DeleteIndexRequest.of(builder -> builder.index(index));
+ client.indices().delete(request);
+ log.info("Done deleting index with name: " + index);
+ return;
+ } catch (ElasticsearchException e) {
+ if (e.status() == HttpStatus.SC_NOT_FOUND) {
+ return;
+ } else if (e.getMessage().contains("Cannot delete indices that are being snapshotted")) {
+ closeIndex(client, index);
+ log.info(String.format("skipping %s index delete, as snapshot is being run, closing the index instead", index));
+ return;
+ } else if (retries < 4) {
+ log.info("Retrying to delete index due to following error: " + e.getMessage());
+ try {
+ Thread.sleep(12000);
+ } catch (InterruptedException e1) {
+ e1.printStackTrace();
+ }
+ } else {
+ closeIndex(client, index);
+ log.info(String.format("maximum retries: %s reached for index: %s delete, closing the index instead", retries, index));
+ }
+ }
+ }
+ } catch (IOException e) {
+ throw new AssertionError(e.getMessage());
+ }
+ }
+
+ public long fetchRecords(String index) throws IOException {
+ try {
+ ElasticsearchClient client = this.getOrCreateClient(username, password, host);
+
+ SearchRequest request = SearchRequest.of(builder -> builder.index(index));
+ SearchResponse<Void> searchResponse = client.search(request, Void.class);
+ return searchResponse.hits().total().value();
+
+ } catch (ElasticsearchException e) {
+ log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage()));
+ return -1;
+ }
+ }
+
+ public long fetchRecordsByTags(String index, String tagKey, String tagValue) throws IOException {
+ try {
+ ElasticsearchClient client = this.getOrCreateClient(username, password, host);
+
+ TermsQueryField termsQueryField = new TermsQueryField.Builder()
+ .value(List.of(FieldValue.of(tagValue)))
+ .build();
+
+ TermsQuery termsQuery = new TermsQuery.Builder()
+ .field("tags.%s".formatted(tagKey))
+ .terms(termsQueryField)
+ .build();
+
+ BoolQuery boolQuery = new BoolQuery.Builder()
+ .must(termsQuery._toQuery())
+ .build();
+
+ SearchRequest searchRequest = new SearchRequest.Builder()
+ .index(index)
+ .query(boolQuery._toQuery())
+ .build();
+
+ SearchResponse<Void> searchResponse = client.search(searchRequest, Void.class);
+ return searchResponse.hits().total().value();
+
+ } catch (ElasticsearchException e) {
+ log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage()));
+ return -1;
+ }
+ }
+
+ public long fetchRecordsByFieldAndFieldValue(String index, String fieldKey, String fieldValue) throws IOException {
+ try {
+ ElasticsearchClient client = this.getOrCreateClient(username, password, host);
+
+ SearchRequest.Builder searchRequestBuilder = new SearchRequest.Builder()
+ .index(index);
+
+ if (!Strings.isNullOrEmpty(fieldKey)) {
+ FieldValue elkFieldValue = new FieldValue.Builder()
+ .stringValue(fieldValue)
+ .build();
+
+ MatchQuery matchQuery = new MatchQuery.Builder()
+ .field(fieldKey)
+ .query(elkFieldValue)
+ .build();
+
+ BoolQuery boolQuery = new BoolQuery.Builder()
+ .must(matchQuery._toQuery())
+ .build();
+
+ searchRequestBuilder.query(Query.of(builder -> builder.bool(boolQuery)));
+ }
+ SearchResponse<Void> searchResponse = client.search(searchRequestBuilder.build(), Void.class);
+ return searchResponse.hits().total().value();
+ } catch (ElasticsearchException e) {
+ log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage()));
+ return -1;
+ }
+ }
+
+ public String[] fetchIndexList() throws IOException {
+ try {
+ ElasticsearchClient client = this.getOrCreateClient(username, password, host);
+ // List all index names
+ GetIndexRequest request = GetIndexRequest.of(builder -> builder.index("*"));
+ GetIndexResponse response = client.indices().get(request);
+ Set<String> keySet = response.result().keySet();
+ return keySet.toArray(new String[0]);
+ } catch (ElasticsearchException e) {
+ log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage()));
+ return new String[]{};
+ }
+ }
+
+ public List<Map<String, Object>> fetchRecordsByAttribute(String index, String attributeKey, String attributeValue) throws IOException {
+ List<Map<String, Object>> out = new ArrayList<>();
+ try {
+ ElasticsearchClient client = this.getOrCreateClient(username, password, host);
+
+ TermsQueryField termsQueryField = new TermsQueryField.Builder()
+ .value(List.of(FieldValue.of(attributeValue)))
+ .build();
+
+ TermsQuery termsQuery = new TermsQuery.Builder()
+ .field(attributeKey)
+ .terms(termsQueryField)
+ .build();
+
+ BoolQuery boolQuery = new BoolQuery.Builder()
+ .must(termsQuery._toQuery())
+ .build();
+
+ SearchRequest searchRequest = new SearchRequest.Builder()
+ .index(index)
+ .query(boolQuery._toQuery())
+ .build();
+
+ TypeReference<Map<String, Object>> typeReference = new TypeReference<>() {};
+ Type type = typeReference.getType();
+ SearchResponse<Map<String, Object>> searchResponse = client.search(searchRequest, type);
+ for (Hit<Map<String, Object>> searchHit : searchResponse.hits().hits()) {
+ Map<String, Object> source = searchHit.source();
+ out.add(source);
+ }
+ return out;
+ } catch (ElasticsearchException e) {
+ log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage()));
+ return out;
+ }
+ }
+
+ public long fetchRecordsByExistQuery(String index, String attributeName) throws Exception {
+ try {
+ TimeUnit.SECONDS.sleep(40);
+ ElasticsearchClient client = this.getOrCreateClient(username, password, host);
+ ExistsQuery existsQuery = new ExistsQuery.Builder()
+ .field(attributeName)
+ .build();
+
+ SearchRequest searchRequest = new SearchRequest.Builder()
+ .index(index)
+ .query(Query.of(builder -> builder.exists(existsQuery)))
+ .build();
+
+ SearchResponse searchResponse = client.search(searchRequest, Void.class);
+ return searchResponse.hits().total().value();
+ } catch (ElasticsearchException e) {
+ log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage()));
+ return -1;
+ }
+ }
+
+ public long fetchRecordsByGeoWithinQuery(String index, String field, Double topLatitude, Double topLongitude,
+ Double bottomLatitude, Double bottomLongitude) throws Exception {
+ ElasticsearchClient client = this.getOrCreateClient(username, password, host);
+ try {
+
+ StringReader jsonData = new StringReader("""
+ {
+ "type":"envelope",
+ "coordinates": [[%f, %f], [%f, %f] ]
+ }
+ """.formatted(topLongitude, topLatitude, bottomLongitude, bottomLatitude));
+
+ GeoShapeFieldQuery geoShapeFieldQuery = GeoShapeFieldQuery.of(
+ geoShapeFieldQueryBuilder ->
+ geoShapeFieldQueryBuilder.relation(GeoShapeRelation.Within)
+ .shape(JsonData.from(jsonData))
+ );
+
+ GeoShapeQuery geoShapeQuery = GeoShapeQuery.of(geoShapeQueryBuilder ->
+ geoShapeQueryBuilder.field(field)
+ .ignoreUnmapped(false)
+ .boost(1.0f)
+ .shape(geoShapeFieldQuery));
+
+ BoolQuery boolQuery = BoolQuery.of(boolQueryBuilder ->
+ boolQueryBuilder.must(queryBuilder -> queryBuilder
+ .geoShape(geoShapeQuery)).boost(1.0f));
+
+ SearchRequest searchRequest = new SearchRequest.Builder()
+ .index(index)
+ .query(boolQuery._toQuery())
+ .build();
+
+ SearchResponse searchResponse = client.search(searchRequest, Void.class);
+ return searchResponse.hits().total().value();
+ } catch (ElasticsearchException e) {
+ log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage()));
+ return -1;
+ }
+ }
+
+ public long fetchRecordsByAsIngestedCoordinates(String index, String pointX, Double topPointX, Double bottomPointX,
+ String pointY, Double topPointY, Double bottomPointY) throws Exception {
+ ElasticsearchClient client = this.getOrCreateClient(username, password, host);
+ try {
+ RangeQuery xRangeQuery = RangeQuery.of(builder -> builder.untyped(
+ untypedBuilder -> untypedBuilder.field(pointX).lte(JsonData.of(topPointX))
+ .gte(JsonData.of(bottomPointX))));
+
+ RangeQuery yRangeQuery = RangeQuery.of(builder -> builder.untyped(
+ untypedBuilder -> untypedBuilder.field(pointY).lte(JsonData.of(topPointY))
+ .gte(JsonData.of(bottomPointY))));
+
+ BoolQuery boolQuery = BoolQuery.of(
+ builder -> builder.must(xRangeQuery._toQuery()).must(yRangeQuery._toQuery()));
+
+ SearchRequest searchRequest = SearchRequest.of(builder -> builder.index(index).query(boolQuery._toQuery()));
+
+ SearchResponse searchResponse = client.search(searchRequest, Void.class);
+ return searchResponse.hits().total().value();
+ } catch (ElasticsearchException e) {
+ log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage()));
+ return -1;
+ }
+ }
+
+ public SearchResponse<Record> fetchRecordsByIdAndMustHaveXcollab(String index, String id, String xCollab) throws Exception {
+ ElasticsearchClient client = this.getOrCreateClient(username, password, host);
+ try{
+ TermQuery idTerm = TermQuery.of(builder -> builder.field("id").value(id));
+ TermQuery xCollabTerm = TermQuery.of(builder -> builder.field(X_COLLABORATION).value(xCollab));
+ BoolQuery of = BoolQuery.of(builder -> builder.must(idTerm._toQuery()).must(xCollabTerm._toQuery()));
+
+ SearchRequest searchRequest = SearchRequest.of(builder -> builder.index(index).query(of._toQuery()));
+
+ log.log(Level.INFO,
+ String.format("xcollab feature: print searchRequest to get record by id and x-collab value: %s",
+ searchRequest));
+
+ return client.search(searchRequest, Record.class);
+ } catch (ElasticsearchException e) {
+ log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage()));
+ throw e;
+ }
+ }
+
+
+ public DeleteResponse deleteRecordsById(String index, String id) throws Exception {
+ ElasticsearchClient client = this.getOrCreateClient(username, password, host);
+ try {
+ DeleteRequest request = DeleteRequest.of(builder -> builder.index(index).id(id));
+ return client.delete(request);
+ } catch (ElasticsearchException e) {
+ log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage()));
+ throw e;
+ }
+ }
+
+ public long fetchRecordsByNestedQuery(String index, String path, String firstNestedField, String firstNestedValue,
+ String secondNestedField, String secondNestedValue) throws Exception {
+ ElasticsearchClient client = this.getOrCreateClient(username, password, host);
+ try {
+ FieldValue firstFieldValue = FieldValue.of(firstNestedValue);
+ FieldValue secondFieldValue = FieldValue.of(secondNestedValue);
+ MatchQuery firstMatch = MatchQuery.of(
+ matchBuilder -> matchBuilder.field(firstNestedField).query(firstFieldValue));
+ MatchQuery secondMatch = MatchQuery.of(
+ matchBuilder -> matchBuilder.field(secondNestedField + ".keyword").query(secondFieldValue));
+ BoolQuery boolQuery = BoolQuery.of(
+ boolBuilder -> boolBuilder.must(firstMatch._toQuery()).must(secondMatch._toQuery()));
+
+ NestedQuery nestedQuery = NestedQuery.of(
+ builder -> builder.path(path).query(boolQuery._toQuery()).scoreMode(ChildScoreMode.Avg));
+
+ SearchRequest searchRequest = SearchRequest.of(
+ builder -> builder.index(index).query(nestedQuery._toQuery()));
+
+ SearchResponse searchResponse = client.search(searchRequest, Void.class);
+ return searchResponse.hits().total().value();
+ } catch (ElasticsearchException e) {
+ log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage()));
+ return -1;
+ }
+ }
+
+
+ public long fetchRecordsWithFlattenedFieldsQuery(String index, String flattenedField, String flattenedFieldValue)
+ throws IOException {
+ ElasticsearchClient client = this.getOrCreateClient(username, password, host);
+ try {
+ MatchQuery matchQuery = MatchQuery.of(
+ builder -> builder.field(flattenedField).query(FieldValue.of(flattenedFieldValue)));
+
+ BoolQuery boolQuery = BoolQuery.of(builder -> builder.must(matchQuery._toQuery()));
+
+ SearchRequest searchRequest = SearchRequest.of(builder -> builder.index(index).query(boolQuery._toQuery()));
+
+ SearchResponse searchResponse = client.search(searchRequest, Void.class);
+ return searchResponse.hits().total().value();
+ } catch (ElasticsearchException e) {
+ log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage()));
+ return -1;
+ }
+ }
+
+ public RecordData fetchDataFromObjectsArrayRecords(String index) throws IOException {
+ try {
+ ElasticsearchClient client = this.getOrCreateClient(username, password, host);
+
+ SearchRequest request = SearchRequest.of(builder -> builder.index(index));
+ SearchResponse<RecordData> searchResponse = client.search(request, RecordData.class);
+
+ HitsMetadata<RecordData> hits = searchResponse.hits();
+ if (hits.hits().size() != 0) {
+ return hits.hits().get(0).source();
+ }
+ return null;
+
+ } catch (ElasticsearchException e) {
+ log.log(Level.INFO, String.format("Elastic search threw exception: %s", e.getMessage()));
+ return null;
+ }
+ }
+
+ public Map<String, IndexMappingRecord> getMapping(String index) throws IOException {
+ ElasticsearchClient client = this.getOrCreateClient(username, password, host);
+ GetMappingRequest.Builder getMappingBuilder = new GetMappingRequest.Builder();
+ getMappingBuilder.index(index);
+ getMappingBuilder.masterTimeout(REQUEST_TIMEOUT);
+ GetMappingResponse mappingResponse = client.indices().getMapping(getMappingBuilder.build());
+ Map<String, IndexMappingRecord> result = mappingResponse.result();
+ return result;
+ }
+
+ public void refreshIndex(String index) throws IOException {
+ ElasticsearchClient client = this.getOrCreateClient(username, password, host);
+ try {
+ RefreshRequest refreshRequest = RefreshRequest.of(builder -> builder.index(index));
+ client.indices().refresh(refreshRequest);
+ } catch (ElasticsearchException exception) {
+ log.info(String.format("index: %s refresh failed. message: %s", index, exception.getMessage()));
+ }
+
+ }
+
+ private boolean closeIndex(ElasticsearchClient client, String index) {
+ try {
+ CloseIndexRequest request = CloseIndexRequest.of(builder -> builder.index(index).timeout(Time.of(timeBuilder -> timeBuilder.time("1m"))));
+ CloseIndexResponse closeIndexResponse = client.indices().close(request);
+ return closeIndexResponse.acknowledged();
+ } catch (ElasticsearchException | IOException exception) {
+ log.info(String.format("index: %s close failed. message: %s", index, exception.getMessage()));
+ }
+ return false;
+ }
+
+ @SuppressWarnings("unchecked")
+ private static List<IndexOperation<Map<String, Object>>> getIndexReqFromRecord(String index, String kind, List<Map<String, Object>> testRecords) {
+ List<IndexOperation<Map<String, Object>>> dataList = new ArrayList<>();
+ Gson gson = new Gson();
+ try {
+ for (Map<String, Object> record : testRecords) {
+ IndexOperation<Map<String, Object>> indexOperation = new IndexOperation.Builder<Map<String, Object>>()
+ .index(index)
+ .id((String) record.get("id"))
+ .document(gson.fromJson(gson.toJson(record), Map.class))
+ .build();
+
+ dataList.add(indexOperation);
+ }
+ } catch (Exception e) {
+ throw new AssertionError(e.getMessage());
+ }
+ return dataList;
+ }
+
+
+ private ElasticsearchClient getOrCreateClient(String username, String password, String host) {
+ if (this.elasticsearchClient == null) {
+ ElasticsearchClient restHighLevelClient;
+ int port = Config.getElastic8Port();
+ try {
+ String rawString = String.format("%s:%s", username, password);
+
+ RestClientBuilder builder = createClientBuilder(host, rawString, port);
+ RestClientTransport transport = new RestClientTransport(builder.build(), new JacksonJsonpMapper());
+ restHighLevelClient = new ElasticsearchClient(transport);
+
+ } catch (Exception e) {
+ throw new AssertionError("Setup elastic error: %s" + e.getMessage());
+ }
+ this.elasticsearchClient = restHighLevelClient;
+ }
+ return this.elasticsearchClient;
+ }
+
+ public RestClientBuilder createClientBuilder(String url, String usernameAndPassword, int port) throws Exception {
+ String scheme = this.sslEnabled ? "https" : "http";
+
+ url = url.trim().replaceAll("^(?i)(https?)://","");
+ URI uri = new URI(scheme + "://" + url);
+
+ RestClientBuilder builder = RestClient.builder(new HttpHost(uri.getHost(), port, uri.getScheme()));
+ if (!Strings.isNullOrEmpty(uri.getPath())) {
+ builder.setPathPrefix(uri.getPath());
+ }
+
+ builder.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(REST_CLIENT_CONNECT_TIMEOUT)
+ .setSocketTimeout(REST_CLIENT_SOCKET_TIMEOUT));
+
+ Header[] defaultHeaders = new Header[]{
+ new BasicHeader("client.transport.nodes_sampler_interval", "30s"),
+ new BasicHeader("client.transport.ping_timeout", "30s"),
+ new BasicHeader("client.transport.sniff", "false"),
+ new BasicHeader("request.headers.X-Found-Cluster", Config.getElastic8Host()),
+ new BasicHeader("cluster.name", Config.getElastic8Host()),
+ new BasicHeader("xpack.security.transport.ssl.enabled", Boolean.toString(true)),
+ new BasicHeader("Authorization", String.format("Basic %s", Base64.getEncoder().encodeToString(usernameAndPassword.getBytes()))),
+ };
+
+ boolean isSecurityHttpsCertificateTrust = Config.isSecurityHttpsCertificateTrust();
+ log.info(String.format(
+ "Elastic client connection uses protocolScheme = %s with a flag "
+ + "'security.https.certificate.trust' = %s",
+ scheme, isSecurityHttpsCertificateTrust));
+
+ if ("https".equals(scheme) && isSecurityHttpsCertificateTrust) {
+ log.warning("Elastic client connection uses TrustSelfSignedStrategy()");
+ SSLContext sslContext = createSSLContext();
+ builder.setHttpClientConfigCallback(httpClientBuilder ->
+ httpClientBuilder.setSSLContext(sslContext)
+ .setSSLHostnameVerifier(
+ NoopHostnameVerifier.INSTANCE));
+ }
+
+ builder.setDefaultHeaders(defaultHeaders);
+ return builder;
+ }
+
+ private SSLContext createSSLContext() {
+ SSLContextBuilder sslContextBuilder = new SSLContextBuilder();
+ try {
+ sslContextBuilder.loadTrustMaterial(null, new TrustSelfSignedStrategy());
+ return sslContextBuilder.build();
+ } catch (NoSuchAlgorithmException e) {
+ log.severe(e.getMessage());
+ } catch (KeyStoreException e) {
+ log.severe(e.getMessage());
+ } catch (KeyManagementException e) {
+ log.severe(e.getMessage());
+ }
+ return null;
+ }
+
+ public boolean isIndexExist(String index) {
+ boolean exists = false;
+ try {
+ exists = createRestClientAndCheckIndexExist(index);
+ } catch (ElasticsearchException e) {
+ log.log(Level.INFO, String.format("Error getting index: %s %s", index, e.getMessage()));
+ }
+ return exists;
+ }
+
+ private boolean createRestClientAndCheckIndexExist(String index) {
+ ElasticsearchClient client = this.getOrCreateClient(username, password, host);
+ try {
+ ExistsRequest request = ExistsRequest.of(builder -> builder.index(index));
+ return client.indices().exists(request).value();
+ } catch (IOException e) {
+ log.log(Level.INFO, String.format("Error getting index: %s %s", index, e.getMessage()));
+ }
+ return false;
+ }
+
+}
diff --git a/indexer-acceptance-test/src/test/java/org/opengroup/osdu/util/FileHandler.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/util/FileHandler.java
new file mode 100644
index 0000000000000000000000000000000000000000..38a5a507e776224a19ffc687e79f4c3deec16202
--- /dev/null
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/util/FileHandler.java
@@ -0,0 +1,38 @@
+package org.opengroup.osdu.util;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+
+public class FileHandler {
+
+ private static final ObjectMapper mapper = new ObjectMapper();
+
+ private FileHandler(){}
+
+ public static String readFile(String fileName) throws IOException {
+ InputStream inputStream = getFileStream(fileName);
+ if(inputStream == null) {
+ throw new IOException();
+ }
+ ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+ byte[] buffer = new byte[1024];
+ int length;
+ while ((length = inputStream.read(buffer)) != -1) {
+ outputStream.write(buffer, 0, length);
+ }
+ return outputStream.toString(StandardCharsets.UTF_8.toString());
+ }
+
+ public static <T> T readFile(String fileName, Class<T> targetClass) throws IOException {
+ InputStream is = getFileStream(fileName);
+ return mapper.readValue(is, targetClass);
+ }
+
+ private static InputStream getFileStream(String fileName) {
+ return FileHandler.class.getResourceAsStream(String.format("/testData/%s", fileName));
+ }
+}
diff --git a/indexer-acceptance-test/src/test/java/org/opengroup/osdu/util/HTTPClient.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/util/HTTPClient.java
new file mode 100644
index 0000000000000000000000000000000000000000..411a02f2fb9e50620bad6aedad99047412f6b291
--- /dev/null
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/util/HTTPClient.java
@@ -0,0 +1,107 @@
+package org.opengroup.osdu.util;
+
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonParser;
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import lombok.ToString;
+import lombok.extern.java.Log;
+
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
+import javax.ws.rs.core.MediaType;
+import java.security.SecureRandom;
+import java.security.cert.X509Certificate;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+
+@Log
+@ToString
+public abstract class HTTPClient {
+
+ private static Random random = new Random();
+ private final int MAX_ID_SIZE = 50;
+
+ protected static final String HEADER_CORRELATION_ID = "correlation-id";
+
+ public abstract String getAccessToken();
+
+ protected static Client getClient() {
+ TrustManager[] trustAllCerts = new TrustManager[]{new X509TrustManager() {
+ @Override
+ public X509Certificate[] getAcceptedIssuers() {
+ return null;
+ }
+
+ @Override
+ public void checkClientTrusted(X509Certificate[] certs, String authType) {
+ }
+
+ @Override
+ public void checkServerTrusted(X509Certificate[] certs, String authType) {
+ }
+ }};
+
+ try {
+ SSLContext sc = SSLContext.getInstance("TLS");
+ sc.init(null, trustAllCerts, new SecureRandom());
+ HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory());
+ } catch (Exception ignored) {
+ }
+ return Client.create();
+ }
+
+ public ClientResponse send(String httpMethod, String url, String payLoad, Map<String, String> headers, String token) {
+ ClientResponse response;
+ try {
+ String correlationId = java.util.UUID.randomUUID().toString();
+ log.info(String.format("Request correlation id: %s", correlationId));
+ headers.put(HEADER_CORRELATION_ID, correlationId);
+ Client client = getClient();
+ client.setReadTimeout(300000);
+ client.setConnectTimeout(300000);
+ log.info("URL: = " + url);
+ WebResource webResource = client.resource(url);
+ response = this.getClientResponse(httpMethod, payLoad, webResource, headers, token);
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw new AssertionError("Error: Send request error", e);
+ }
+ log.info("waiting on response");
+ return response;
+ }
+
+ protected ClientResponse getClientResponse(String httpMethod, String requestBody, WebResource webResource, Map<String, String> headers, String token) {
+ final WebResource.Builder builder = webResource.accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON).header("Authorization", token);
+ headers.forEach(builder::header);
+ log.info("making request to datalake api");
+ return builder.method(httpMethod, ClientResponse.class, requestBody);
+ }
+
+ public Map<String, String> getCommonHeader() {
+ Map<String, String> headers = new HashMap<>();
+ headers.put("data-partition-id", Config.getDataPartitionIdTenant1());
+ return headers;
+ }
+
+ public static Map<String, String> overrideHeader(Map<String, String> currentHeaders, String... partitions) {
+ String value = String.join(",", partitions);
+ currentHeaders.put("data-partition-id", value);
+ return currentHeaders;
+ }
+
+ public static String indentatedResponseBody(String responseBody) {
+ JsonParser jsonParser = new JsonParser();
+ if(responseBody!= null) {
+ JsonElement jsonElement = jsonParser.parse(responseBody);
+ String indentedResponseEntity = new GsonBuilder().setPrettyPrinting().create().toJson(jsonElement);
+ return indentedResponseEntity;
+ }
+ return responseBody;
+ }
+}
diff --git a/indexer-acceptance-test/src/test/java/org/opengroup/osdu/util/IndexerClientUtil.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/util/IndexerClientUtil.java
new file mode 100644
index 0000000000000000000000000000000000000000..adc4cc5b3983baa8ac436592b77d1062cd7049b0
--- /dev/null
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/util/IndexerClientUtil.java
@@ -0,0 +1,28 @@
+package org.opengroup.osdu.util;
+
+import com.sun.jersey.api.client.ClientResponse;
+import lombok.extern.java.Log;
+
+import javax.ws.rs.HttpMethod;
+import java.util.Map;
+
+import static org.opengroup.osdu.util.Config.getIndexerBaseURL;
+
+@Log
+public class IndexerClientUtil {
+
+ private final HTTPClient httpClient;
+ private Map<String, String> headers;
+
+ public IndexerClientUtil(HTTPClient httpClient) {
+ this.httpClient = httpClient;
+ headers = httpClient.getCommonHeader();
+ }
+
+ public void deleteIndex(String kind) {
+ String url = getIndexerBaseURL() + "index?kind=" + kind;
+ log.info("URL: " + url);
+ ClientResponse response = httpClient.send(HttpMethod.DELETE, url, "", headers, httpClient.getAccessToken());
+ log.info(response.toString());
+ }
+}
diff --git a/indexer-acceptance-test/src/test/java/org/opengroup/osdu/util/JsonPathMatcher.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/util/JsonPathMatcher.java
new file mode 100644
index 0000000000000000000000000000000000000000..26d24175e7a331d7af12b03dbe4dd9a192c7e8df
--- /dev/null
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/util/JsonPathMatcher.java
@@ -0,0 +1,141 @@
+package org.opengroup.osdu.util;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class JsonPathMatcher {
+
+ // Since there's no unit tests for tests, there's this:
+ public static void main(String[] args) {
+ String jsonGoodString = """
+{ "id": "tenant1:reference-data--IndexPropertyPathConfiguration:index-property--Wellbore:1.",
+ "data": {
+ "Name": "Wellbore-IndexPropertyPathConfiguration",
+ "Description": "The index property list for index-property--Wellbore:1., valid for all index-property--Wellbore kinds for major version 1.",
+ "Code": "test:indexer:index-property--Wellbore:1.",
+ "AttributionAuthority": "OSDU",
+ "Configurations": [{
+ "Name": "WellUWI",
+ "Policy": "ExtractFirstMatch",
+ "Paths": [{
+ "ValueExtraction.RelatedConditionMatches": [
+ "UniqueIdentifier:$",
+ "RegulatoryName:$",
+ "PreferredName:$",
+ "CommonName:$",
+ "ShortName:$"
+ ],
+ "ValueExtraction.RelatedConditionProperty": "data.NameAliases[].AliasNameTypeID",
+ "ValueExtraction.ValuePath": "data.NameAliases[].AliasName"
+
+ }
+ ],
+ "UseCase": "As a user I want to discover and match Wells by their UWI. I am aware that this is not globally reliable, however, I am able to specify a prioritized AliasNameType list to look up value in the NameAliases array."
+ }
+ ]
+ }
+ }
+ """;
+ String jsonBadString = """
+{ "id": "tenant1:reference-data--IndexPropertyPathConfiguration:index-property--Wellbore:1.",
+ "data": {
+ "Name": "Wellbore-IndexPropertyPathConfiguration",
+ "Description": "The index property list for index-property--Wellbore:1., valid for all index-property--Wellbore kinds for major version 1.",
+ "Code": "test:indexer:index-property--Wellbore:1.",
+ "AttributionAuthority": "OSDU",
+ "Configurations": [{
+ "Name": "WellUWI",
+ "Policy": "ExtractFirstMatch",
+ "Paths": [{
+ "ValueExtraction": {
+ "RelatedConditionMatches": "[UniqueIdentifier:$,RegulatoryName:$,PreferredName:$]",
+ "RelatedConditionProperty": "data.NameAliases[].AliasNameTypeID",
+ "ValuePath": "data.NameAliases[].AliasName"
+ }
+ }
+ ],
+ "UseCase": "As a user I want to discover and match Wells by their UWI. I am aware that this is not globally reliable, however, I am able to specify a prioritized AliasNameType list to look up value in the NameAliases array."
+ }
+ ]
+ }
+ }
+ """;
+
+ try {
+ ObjectMapper objectMapper = new ObjectMapper();
+ Map<String, Object> dataMap = objectMapper.readValue(jsonGoodString, Map.class);
+ List<String> stringList = java.util.Arrays.asList("data.Configurations.Paths.ValueExtraction.RelatedConditionMatches".split("\\."));
+ Object found = FindArrayInJson(dataMap, stringList);
+ System.out.println("in Good String Found? "+ objectMapper.writeValueAsString(found));
+ dataMap = objectMapper.readValue(jsonBadString, Map.class);
+ found = FindArrayInJson(dataMap, stringList);
+ System.out.println("in Bad String Found? "+ objectMapper.writeValueAsString(found));
+ stringList = java.util.Arrays.asList("data.Configurations.Paths.ValueExtraction.SomeDifferentKey".split("\\."));
+ found = FindArrayInJson(dataMap, stringList);
+ System.out.println("in Bad String Found? "+ objectMapper.writeValueAsString(found));
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ // Assure that field in the Json path (supporting flattened fields) is an array of strings
+ public static Object FindArrayInJson(Object data, List<String> stringList) {
+ if (data instanceof Map) {
+ // Handle Map
+ return handleMap((Map<String, Object>) data, stringList);
+ } else if (data instanceof ArrayList) {
+ // Handle ArrayList
+ return handleArrayList((ArrayList<?>) data, stringList);
+ } else {
+ // Handle other types
+ return null;
+ }
+ }
+
+ private static boolean isPrefix(List<String> potentialPrefix, List<String> target) {
+ boolean result = true;
+ for (Integer i = 0; i < potentialPrefix.size(); i++) {
+ if (!potentialPrefix.get(i).equals(target.get(i))) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static Object handleMap(Map<String, Object> map, List<String> stringList) {
+
+ for (Map.Entry<String, Object> entry : map.entrySet()) {
+ Object value = entry.getValue();
+ List<String> keyParts = java.util.Arrays.asList(entry.getKey().split("\\."));
+ if (keyParts.size() > stringList.size()) {
+ if (!isPrefix(keyParts.subList(0, stringList.size()), stringList)) {
+ continue;
+ }
+ Object result = FindArrayInJson(value, stringList.subList(0, 0));
+ if (result != null) { return result; }
+ } else {
+ if (!isPrefix(keyParts, stringList)) {
+ continue;
+ }
+ Object result = FindArrayInJson(value, stringList.subList(keyParts.size(), stringList.size()));
+ if (result != null) { return result; }
+ }
+ }
+ return null;
+ }
+
+ private static Object handleArrayList(ArrayList<?> arrayList, List<String> stringList) {
+ if (stringList.isEmpty()) return arrayList;
+ if (arrayList.isEmpty()) {
+ return null;
+ }
+ for (Object arrayElementValue : arrayList) {
+ Object result = FindArrayInJson(arrayElementValue, stringList);
+ if (result != null) { return result; }
+ }
+ return null;
+ }
+}
diff --git a/indexer-acceptance-test/src/test/java/org/opengroup/osdu/util/NotFoundIgnoringResponseErrorHandler.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/util/NotFoundIgnoringResponseErrorHandler.java
new file mode 100644
index 0000000000000000000000000000000000000000..2d77c0186aee6eb40e53d906e3b4272b944d4a76
--- /dev/null
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/util/NotFoundIgnoringResponseErrorHandler.java
@@ -0,0 +1,21 @@
+package org.opengroup.osdu.util;
+
+import org.springframework.http.HttpStatus;
+import org.springframework.http.HttpStatusCode;
+import org.springframework.http.client.ClientHttpResponse;
+import org.springframework.web.client.DefaultResponseErrorHandler;
+
+import java.io.IOException;
+
+public class NotFoundIgnoringResponseErrorHandler extends DefaultResponseErrorHandler {
+
+ @Override
+ public boolean hasError(ClientHttpResponse response) throws IOException {
+ return super.hasError(response) && response.getRawStatusCode() != HttpStatus.NOT_FOUND.value();
+ }
+
+ @Override
+ protected boolean hasError(HttpStatusCode statusCode) {
+ return super.hasError(statusCode) && statusCode != HttpStatus.NOT_FOUND;
+ }
+}
diff --git a/indexer-acceptance-test/src/test/java/org/opengroup/osdu/util/OpenIDTokenProvider.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/util/OpenIDTokenProvider.java
new file mode 100644
index 0000000000000000000000000000000000000000..10e88e7aaba3920c07dac3d67b70c52095a76faa
--- /dev/null
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/util/OpenIDTokenProvider.java
@@ -0,0 +1,75 @@
+/*
+ * Copyright 2020-2022 Google LLC
+ * Copyright 2020-2022 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.util;
+
+import com.nimbusds.oauth2.sdk.*;
+import com.nimbusds.oauth2.sdk.auth.ClientAuthentication;
+import com.nimbusds.oauth2.sdk.auth.ClientSecretBasic;
+import com.nimbusds.oauth2.sdk.auth.Secret;
+import com.nimbusds.oauth2.sdk.id.ClientID;
+import com.nimbusds.openid.connect.sdk.OIDCTokenResponseParser;
+import net.minidev.json.JSONObject;
+import org.opengroup.osdu.util.conf.OpenIDProviderConfig;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.Objects;
+
+public class OpenIDTokenProvider {
+
+ private static final OpenIDProviderConfig openIDProviderConfig = OpenIDProviderConfig.Instance();
+ private static final String ID_TOKEN = "id_token";
+ private final AuthorizationGrant clientGrant = new ClientCredentialsGrant();
+ private final URI tokenEndpointURI;
+ private final Scope scope;
+ private final ClientAuthentication clientAuthentication;
+
+ public OpenIDTokenProvider() {
+ this.tokenEndpointURI = openIDProviderConfig.getProviderMetadata().getTokenEndpointURI();
+ this.scope = new Scope(openIDProviderConfig.getScopes());
+ this.clientAuthentication =
+ new ClientSecretBasic(
+ new ClientID(openIDProviderConfig.getClientId()),
+ new Secret(openIDProviderConfig.getClientSecret())
+ );
+ }
+
+ public String getToken() {
+ try {
+ return requestToken();
+ } catch (ParseException | IOException e) {
+ throw new RuntimeException("Unable get credentials from INTEGRATION_TESTER variables", e);
+ }
+ }
+
+ private String requestToken() throws ParseException, IOException {
+ TokenRequest request = new TokenRequest(this.tokenEndpointURI, this.clientAuthentication, this.clientGrant, this.scope);
+ TokenResponse parse = OIDCTokenResponseParser.parse(request.toHTTPRequest().send());
+
+ if (!parse.indicatesSuccess()) {
+ throw new RuntimeException("Unable get credentials from INTEGRATION_TESTER variables");
+ }
+
+ JSONObject jsonObject = parse.toSuccessResponse().toJSONObject();
+ String idTokenValue = jsonObject.getAsString(ID_TOKEN);
+ if (Objects.isNull(idTokenValue) || idTokenValue.isEmpty()) {
+ throw new RuntimeException("Unable get credentials from INTEGRATION_TESTER variables");
+ }
+ return idTokenValue;
+ }
+}
diff --git a/indexer-acceptance-test/src/test/java/org/opengroup/osdu/util/SchemaServiceClient.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/util/SchemaServiceClient.java
new file mode 100644
index 0000000000000000000000000000000000000000..04d74f8b7eb0ab84182d61b342e802e343114e86
--- /dev/null
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/util/SchemaServiceClient.java
@@ -0,0 +1,72 @@
+package org.opengroup.osdu.util;
+
+import org.opengroup.osdu.models.schema.SchemaIdentity;
+import org.opengroup.osdu.models.schema.SchemaModel;
+import org.springframework.boot.web.client.RestTemplateBuilder;
+import org.springframework.http.*;
+import org.springframework.web.client.RestTemplate;
+import org.springframework.web.util.UriComponentsBuilder;
+
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import static java.util.Collections.singletonList;
+
+
+public class SchemaServiceClient {
+
+ private static final Logger LOGGER = Logger.getLogger(SchemaServiceClient.class.getName());
+
+ private final RestTemplate template;
+ private final String schemaBaseUrl;
+
+ public SchemaServiceClient(HTTPClient client) {
+ template = new RestTemplateBuilder()
+ .errorHandler(new NotFoundIgnoringResponseErrorHandler())
+ .additionalInterceptors((request, body, execution) -> {
+ request.getHeaders().add(HttpHeaders.AUTHORIZATION, client.getAccessToken());
+ request.getHeaders().put(HttpHeaders.ACCEPT, singletonList(MediaType.APPLICATION_JSON_VALUE));
+ request.getHeaders().add("data-partition-id", Config.getDataPartitionIdTenant1());
+ return execution.execute(request, body);
+ })
+ .build();
+ schemaBaseUrl = Config.getSchemaBaseURL();
+ }
+
+ public boolean exists(SchemaIdentity identity) {
+ String uri = buildSchemaUri(identity.getId());
+ LOGGER.log(Level.INFO, "Checking whether the schema exists having identity={0}", identity);
+ ResponseEntity<?> response = template.exchange(uri, HttpMethod.GET, null, Object.class);
+ LOGGER.log(Level.INFO, "Finished checking whether the schema exists having identity={0}, response={1}", new Object[]{identity, response});
+ return response.getStatusCode() == HttpStatus.OK;
+ }
+
+ public void create(SchemaModel schema) {
+ String uri = buildSchemaUri();
+ LOGGER.log(Level.INFO, "Creating the schema={0}", schema);
+ HttpHeaders headers = new HttpHeaders();
+ headers.put(HttpHeaders.CONTENT_TYPE, singletonList(MediaType.APPLICATION_JSON_VALUE));
+ HttpEntity<SchemaModel> httpEntity = new HttpEntity<>(schema, headers);
+ template.exchange(uri, HttpMethod.PUT, httpEntity, Object.class);
+ LOGGER.log(Level.INFO, "Finished creating the schema={0}", schema);
+ }
+
+ public void createIfNotExist(SchemaModel schema) {
+ if (!exists(schema.getSchemaInfo().getSchemaIdentity())) {
+ create(schema);
+ }
+ }
+
+ private String buildSchemaUri(String id) {
+ return UriComponentsBuilder.fromHttpUrl(schemaBaseUrl)
+ .path("/schema/{schema-id}")
+ .buildAndExpand(id).toUriString();
+ }
+
+ private String buildSchemaUri() {
+ return UriComponentsBuilder.fromHttpUrl(schemaBaseUrl)
+ .path("/schema")
+ .build().toUriString();
+ }
+
+}
diff --git a/indexer-acceptance-test/src/test/java/org/opengroup/osdu/util/conf/AnthosConfig.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/util/conf/AnthosConfig.java
new file mode 100644
index 0000000000000000000000000000000000000000..b7e332b5c7354f56d141f6880296c4cf55e88c7e
--- /dev/null
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/util/conf/AnthosConfig.java
@@ -0,0 +1,15 @@
+package org.opengroup.osdu.util.conf;
+
+import java.util.Optional;
+
+public class AnthosConfig {
+
+ private static final String GROUP_ID_VARIABLE = "GROUP_ID";
+ private static final String ENTITLEMENTS_DOMAIN_VARIABLE = "ENTITLEMENTS_DOMAIN";
+
+ public static void updateEntitlementsDomainVariable() {
+ String groupId = Optional.ofNullable(System.getProperty(GROUP_ID_VARIABLE, System.getenv(GROUP_ID_VARIABLE)))
+ .orElse("group");
+ System.setProperty(ENTITLEMENTS_DOMAIN_VARIABLE, groupId);
+ }
+}
diff --git a/indexer-acceptance-test/src/test/java/org/opengroup/osdu/util/conf/OpenIDProviderConfig.java b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/util/conf/OpenIDProviderConfig.java
new file mode 100644
index 0000000000000000000000000000000000000000..171a1d503758429550a1710e53504e9995822163
--- /dev/null
+++ b/indexer-acceptance-test/src/test/java/org/opengroup/osdu/util/conf/OpenIDProviderConfig.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2020-2022 Google LLC
+ * Copyright 2020-2022 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.util.conf;
+
+import com.nimbusds.oauth2.sdk.http.HTTPRequest;
+import com.nimbusds.oauth2.sdk.http.HTTPResponse;
+import com.nimbusds.oauth2.sdk.id.Issuer;
+import com.nimbusds.openid.connect.sdk.op.OIDCProviderConfigurationRequest;
+import com.nimbusds.openid.connect.sdk.op.OIDCProviderMetadata;
+
+public class OpenIDProviderConfig {
+
+ private String clientId;
+ private String url;
+ private String clientSecret;
+ private final String[] scopes = {"openid"};
+ private static final OpenIDProviderConfig openIDProviderConfig = new OpenIDProviderConfig();
+ private static OIDCProviderMetadata providerMetadata;
+
+ public static OpenIDProviderConfig Instance() {
+ try {
+ openIDProviderConfig.clientId = System.getProperty("PRIVILEGED_USER_OPENID_PROVIDER_CLIENT_ID", System.getenv("PRIVILEGED_USER_OPENID_PROVIDER_CLIENT_ID"));
+ openIDProviderConfig.url = System.getProperty("TEST_OPENID_PROVIDER_URL", System.getenv("TEST_OPENID_PROVIDER_URL"));
+ openIDProviderConfig.clientSecret = System.getProperty("PRIVILEGED_USER_OPENID_PROVIDER_CLIENT_SECRET", System.getenv("PRIVILEGED_USER_OPENID_PROVIDER_CLIENT_SECRET"));
+ Issuer issuer = new Issuer(openIDProviderConfig.url);
+ OIDCProviderConfigurationRequest request = new OIDCProviderConfigurationRequest(issuer);
+ HTTPRequest httpRequest = request.toHTTPRequest();
+ HTTPResponse httpResponse = httpRequest.send();
+ providerMetadata = OIDCProviderMetadata.parse(httpResponse.getContentAsJSONObject());
+ } catch (Exception e) {
+ throw new RuntimeException("Malformed token provider configuration", e);
+ }
+ return openIDProviderConfig;
+ }
+
+ public String getClientId() {
+ return clientId;
+ }
+
+ public String getClientSecret() {
+ return clientSecret;
+ }
+
+ public String[] getScopes() {
+ return scopes;
+ }
+
+ public OIDCProviderMetadata getProviderMetadata() {
+ return providerMetadata;
+ }
+}
diff --git a/indexer-acceptance-test/src/test/resources/cucumber.properties b/indexer-acceptance-test/src/test/resources/cucumber.properties
new file mode 100644
index 0000000000000000000000000000000000000000..a795aba76e7621a16077f570617fc501371d275e
--- /dev/null
+++ b/indexer-acceptance-test/src/test/resources/cucumber.properties
@@ -0,0 +1,20 @@
+#
+# Copyright 2020-2023 Google LLC
+# Copyright 2020-2023 EPAM Systems, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# tag indexer-extended disabled by default
+# tag as-ingested-coordinates disabled by default
+# tag keyword-lower disabled by default
+cucumber.options=--tags '@default'
diff --git a/indexer-acceptance-test/src/test/resources/features/indexrecord/indexRecord-schema-service.feature b/indexer-acceptance-test/src/test/resources/features/indexrecord/indexRecord-schema-service.feature
new file mode 100644
index 0000000000000000000000000000000000000000..aaac26d1399d814ab8adb8fcd14a404cc44916a3
--- /dev/null
+++ b/indexer-acceptance-test/src/test/resources/features/indexrecord/indexRecord-schema-service.feature
@@ -0,0 +1,196 @@
+Feature: Indexing of the documents
+ This feature deals with validation of the documents in Elastic Search ingested with different kinds and attributes.
+
+ Background:
+ Given the schema is created with the following kind
+ | kind | index | schemaFile |
+ | tenant1:indexer:test-data--Integration:1.1.0 | tenant1-indexer-test-data--integration-1.1.0 | index_records_1 |
+ | tenant1:indexer:test-data--Integration:2.0.1 | tenant1-indexer-test-data--integration-2.0.1 | index_records_2 |
+ | tenant1:indexer:test-data--Integration:3.0.1 | tenant1-indexer-test-data--integration-3.0.1 | index_records_3 |
+ | tenant1:wks:master-data--Wellbore:2.0.3 | tenant1-wks-master-data--wellbore-2.0.3 | r3-index_record_wks_master |
+ | tenant1:wks:ArraysOfObjectsTestCollection:4.0.0 | tenant1-wks-arraysofobjectstestcollection-4.0.0 | r3-index_record_arrayofobjects |
+ | tenant1:indexer:test-mapping--Sync:2.0.0 | tenant1-indexer-test-mapping--sync-2.0.0 | index_record_sync_mapping |
+ | tenant1:indexer:test-update-data--Integration:1.0.1 | tenant1-indexer-test-update-data--integration-1.0.1 | index_update_records_kind_v1 |
+ | tenant1:indexer:test-update-data--Integration:2.0.1 | tenant1-indexer-test-update-data--integration-2.0.1 | index_update_records_kind_v2 |
+ | tenant1:indexer:virtual-properties-Integration:1.0.0 | tenant1-indexer-virtual-properties-integration-1.0.0 | index_record_virtual_properties |
+ | tenant1:indexer:decimation-Integration:1.0.0 | tenant1-indexer-decimation-integration-1.0.0 | index_record_seismic_survey |
+ | osdu:wks:reference-data--IndexPropertyPathConfiguration:1.0.0 | osdu-wks-reference-data--indexpropertypathconfiguration-1.0.0 | osdu_wks_IndexPropertyPathConfiguration_v1 |
+ | test:indexer:index-property--Wellbore:1.0.0 | test-indexer-index-property--wellbore-1.0.0 | index-property-wellbore_v1 |
+ | test:indexer:index-property--WellLog:1.0.0 | test-indexer-index-property--welllog-1.0.0 | index-property-welllog_v1 |
+
+ @collaboration-test
+ Scenario Outline: X Ingest the record with no xcollab value and Index in the Elastic Search with tags
+ When I ingest records with xcollab value <xcollab> included with the <recordFile> with <acl> for a given <kind>
+ Then I should get the <number> documents with xcollab value <xcollab> included for the <index> in the Elastic Search
+
+ Examples:
+ | kind | recordFile | number | index | acl | field | value | xcollab |
+ | "test:indexer:index-property--Wellbore:1.0.0" | "index-property-wellbore_v2" | 1 | "test-indexer-index-property--wellbore-1.0.0" | "data.default.viewers@tenant1" | "data.FacilityName.keywordLower" | "facility_456" | "id=a99cef48-2ed6-4beb-8a43-002373431f92,application=pws" |
+
+ @indexer-extended
+ Scenario Outline: Prepare the index property configuration records and clean up index of the extended kinds in the Elastic Search
+ When I ingest records with the <recordFile> with <acl> for a given <kind>
+ Then I should get the <number> documents for the <index> in the Elastic Search
+ Then I set starting stateful scenarios
+
+ Examples:
+ | kind | recordFile | number | index | acl | extendedKinds |
+ | "osdu:wks:reference-data--IndexPropertyPathConfiguration:1.0.0" | "osdu_wks_IndexPropertyPathConfiguration_v1" | 2 | "osdu-wks-reference-data--indexpropertypathconfiguration-1.0.0" | "data.default.viewers@tenant1" | "test:indexer:index-property--Wellbore:1.0.0,test:indexer:index-property--WellLog:1.0.0" |
+
+ @indexer-extended
+ Scenario Outline: Ingest the records of the extended kinds, Index in the Elastic Search and Search string field
+ When I ingest records with the <recordFile> with <acl> for a given <kind>
+ Then I should be able to search <number> record with index <index> by extended data field <field> and value <value>
+
+ Examples:
+ | kind | recordFile | number | index | acl | field | value |
+ | "test:indexer:index-property--Wellbore:1.0.0" | "index-property-wellbore_v1" | 1 | "test-indexer-index-property--wellbore-1.0.0" | "data.default.viewers@tenant1" | "data.WellUWI" | "123454321" |
+ | "test:indexer:index-property--WellLog:1.0.0" | "index-property-welllog_v1" | 1 | "test-indexer-index-property--welllog-1.0.0" | "data.default.viewers@tenant1" | "data.WellboreName" | "Facility_123" |
+
+ @indexer-extended
+ Scenario Outline: Ingest the records of the extended kinds, Index in the Elastic Search and Search spatial field
+ When I ingest records with the <recordFile> with <acl> for a given <kind>
+ Then I should be able search <number> documents for the <index> by bounding box query with points (<top_left_latitude>, <top_left_longitude>) and (<bottom_right_latitude>, <bottom_right_longitude>) on field <field>
+
+ Examples:
+ | kind | recordFile | number | index | acl | field | top_left_latitude | top_left_longitude | bottom_right_latitude | bottom_right_longitude |
+ | "test:indexer:index-property--Wellbore:1.0.0" | "index-property-wellbore_v1" | 1 | "test-indexer-index-property--wellbore-1.0.0" | "data.default.viewers@tenant1" | "data.Location" | 30 | -96 | 29 | -95 |
+ | "test:indexer:index-property--WellLog:1.0.0" | "index-property-welllog_v1" | 1 | "test-indexer-index-property--welllog-1.0.0" | "data.default.viewers@tenant1" | "data.SpatialLocation" | 30 | -96 | 29 | -95 |
+
+ @indexer-extended
+ Scenario: End Stateful Scenarios
+ Then I set ending stateful scenarios
+
+ @default
+ Scenario Outline: Ingest the record and Index in the Elastic Search
+ When I ingest records with the <recordFile> with <acl> for a given <kind>
+ Then I should get the <number> documents for the <index> in the Elastic Search
+ Then I should get the elastic <mapping> for the <kind> and <index> in the Elastic Search
+ Then I can validate indexed meta attributes for the <index> and given <kind>
+
+ Examples:
+ | kind | recordFile | number | index | acl | mapping |
+ | "tenant1:indexer:test-data--Integration:1.1.0" | "index_records_schema_1" | 5 | "tenant1-indexer-test-data--integration-1.1.0" | "data.default.viewers@tenant1" | {"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Country":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"County":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"EmptyAttribute":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"RelatedRecordId":{"type":"keyword"},"ExternalIds":{"type":"keyword"},"Established":{"type":"date"},"Field":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellName":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellStatus":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellType":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"DblArray":{"type":"double"},"TextArray":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"authority":{"type":"constant_keyword","value":"<authority-id>"},"createTime":{"type":"date"},"createUser":{"type":"keyword"},"modifyTime":{"type":"date"},"modifyUser":{"type":"keyword"},"source":{"type":"constant_keyword","value":"<source-id>"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}} |
+ | "tenant1:indexer:test-data--Integration:3.0.1" | "index_records_schema_3" | 7 | "tenant1-indexer-test-data--integration-3.0.1" | "data.default.viewers@tenant1" | "{"mappings":{"well":{"dynamic":"false","properties":{"acl":{"properties":{"owners":{"type":"keyword"},"viewers":{"type":"keyword"}}},"ancestry":{"properties":{"parents":{"type":"keyword"}}},"data":{"properties":{"Basin":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Country":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"County":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"EmptyAttribute":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Established":{"type":"date"},"Field":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Location":{"type":"geo_point"},"OriginalOperator":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"Rank":{"type":"integer"},"Score":{"type":"integer"},"State":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellName":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellStatus":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"WellType":{"type":"text","fields":{"keyword":{"type":"keyword","null_value":"null","ignore_above":256}}},"DblArray":{"type":"double"}}},"id":{"type":"keyword"},"index":{"properties":{"lastUpdateTime":{"type":"date"},"statusCode":{"type":"integer"},"trace":{"type":"text"}}},"kind":{"type":"keyword"},"legal":{"properties":{"legaltags":{"type":"keyword"},"otherRelevantDataCountries":{"type":"keyword"},"status":{"type":"keyword"}}},"namespace":{"type":"keyword"},"type":{"type":"keyword"},"authority":{"type":"constant_keyword","value":"<authority-id>"},"createTime":{"type":"date"},"createUser":{"type":"keyword"},"modifyTime":{"type":"date"},"modifyUser":{"type":"keyword"},"source":{"type":"constant_keyword","value":"<source-id>"},"version":{"type":"long"},"x-acl":{"type":"keyword"}}}}}" |
+
+ @default
+ Scenario Outline: Ingest the record and Index in the Elastic Search with bad attribute
+ When I ingest records with the <recordFile> with <acl> for a given <kind>
+ Then I should get the <number> documents for the <index> in the Elastic Search with out <skippedAttribute>
+
+ Examples:
+ | kind | recordFile | number | index | skippedAttribute | acl |
+ | "tenant1:indexer:test-data--Integration:2.0.1" | "index_records_2" | 4 | "tenant1-indexer-test-data--integration-2.0.1" | "data.Location" | "data.default.viewers@tenant1" |
+ | "tenant1:indexer:test-data--Integration:2.0.1" | "index_records_2" | 1 | "tenant1-indexer-test-data--integration-2.0.1" | "data.InvalidInteger" | "data.default.viewers@tenant1" |
+
+ @default
+ Scenario Outline: Ingest the record and Index in the Elastic Search with tags
+ When I ingest records with the <recordFile> with <acl> for a given <kind>
+ Then I should be able to search <number> record with index <index> by tag <tagKey> and value <tagValue>
+
+ Examples:
+ | kind | recordFile | index | acl | tagKey | tagValue | number |
+ | "tenant1:indexer:test-data--Integration:1.1.0" | "index_records_schema_1" | "tenant1-indexer-test-data--integration-1.1.0" | "data.default.viewers@tenant1" | "testtag" | "testvalue" | 5 |
+
+ @default
+ Scenario Outline: Ingest the r3-record with geo-shape and Index in the Elastic Search
+ When I ingest records with the <recordFile> with <acl> for a given <kind>
+ Then I should be able search <number> documents for the <index> by bounding box query with points (<top_left_latitude>, <top_left_longitude>) and (<bottom_right_latitude>, <bottom_right_longitude>) on field <field>
+
+ Examples:
+ | kind | recordFile | number | index | acl | field | top_left_latitude | top_left_longitude | bottom_right_latitude | bottom_right_longitude |
+ | "tenant1:wks:master-data--Wellbore:2.0.3" | "r3-index_record_wks_master" | 1 | "tenant1-wks-master-data--wellbore-2.0.3" | "data.default.viewers@tenant1" | "data.SpatialLocation.Wgs84Coordinates" | 52 | -100 | 0 | 100 |
+
+ @keyword-lower
+ Scenario Outline: Ingest record and Index in the Elastic Search with keywordLower attribute for text fields
+ When I ingest records with the <recordFile> with <acl> for a given <kind>
+ Then I should be able to search <number> record with index <index> by extended data field <field> and value <value>
+
+ Examples:
+ | kind | recordFile | number | index | acl | field | value |
+ | "test:indexer:index-property--Wellbore:1.0.0" | "index-property-wellbore_v1" | 1 | "test-indexer-index-property--wellbore-1.0.0" | "data.default.viewers@tenant1" | "data.FacilityName.keywordLower" | "facility_123" |
+
+ @default
+ Scenario Outline: Ingest record and check for properly parsed array of Strings
+ When I ingest records with the <recordFile> with <acl> for a given <kind>
+ Then I should be able to search for record from <index> by <field> for value <value> and find String arrays in <arrayField> with <arrayValue>
+ Examples:
+ | kind | recordFile | index | field | value | acl | arrayField | arrayValue |
+ | "osdu:wks:reference-data--IndexPropertyPathConfiguration:1.0.0" | "well_augmenter_configuration" |"osdu-wks-reference-data--indexpropertypathconfiguration-1.0.0" | "data.AttributionAuthority.keyword" | "CustomAttributionAuthority" | "data.default.viewers@tenant1" | "data.Configurations.Paths.RelatedObjectsSpec.RelatedConditionMatches" | "opendes:reference-data--GeoPoliticalEntityType:Country:" |
+
+ @bag-of-words
+ Scenario Outline: Ingest record and Index bag of words as an attribute
+ When I ingest records with the <recordFile> with <acl> for a given <kind>
+ Then I should be able to search <number> record with index <index> by extended data field <field> and value <value>
+
+ Examples:
+ | kind | recordFile | number | index | acl | field | value |
+ | "test:indexer:index-property--Wellbore:1.0.0" | "index-property-wellbore_v1" | 1 | "test-indexer-index-property--wellbore-1.0.0" | "data.default.viewers@tenant1" | "bagOfWords" | "facility_123" |
+
+ @default
+ Scenario Outline: Ingest records with geo-shape and Index with virtual properties in the Elastic Search
+ When I ingest records with the <recordFile> with <acl> for a given <kind>
+ Then I should be able search <number> documents for the <index> by bounding box query with points (<top_left_latitude>, <top_left_longitude>) and (<bottom_right_latitude>, <bottom_right_longitude>) on field <field>
+
+ Examples:
+ | kind | recordFile | number | index | acl | field | top_left_latitude | top_left_longitude | bottom_right_latitude | bottom_right_longitude |
+ | "tenant1:indexer:virtual-properties-Integration:1.0.0" | "index_record_virtual_properties" | 3 | "tenant1-indexer-virtual-properties-integration-1.0.0" | "data.default.viewers@tenant1" | "data.VirtualProperties.DefaultLocation.Wgs84Coordinates" | 90 | -180 | -90 | 180 |
+ | "tenant1:indexer:decimation-Integration:1.0.0" | "index_record_seismic_survey" | 1 | "tenant1-indexer-decimation-integration-1.0.0" | "data.default.viewers@tenant1" | "data.VirtualProperties.DefaultLocation.Wgs84Coordinates" | 90 | -180 | -90 | 180 |
+
+ @default
+ Scenario Outline: Ingest the r3-record with arrays of objects and hints in schema and Index in the Elastic Search
+ When I ingest records with the <recordFile> with <acl> for a given <kind>
+ Then I should be able search <number> documents for the <index> by nested <path> and properties (<first_nested_field>, <first_nested_value>) and (<second_nested_field>, <second_nested_value>)
+ Then I should be able search <number> documents for the <index> by flattened inner properties (<flattened_inner_field>, <flattened_inner_value>)
+ Then I should get <object_inner_field> in response, without hints in schema for the <index> that present in the <recordFile> with <acl> for a given <kind>
+
+ Examples:
+ | kind | recordFile | number | index | acl | path | first_nested_field | first_nested_value | second_nested_field | second_nested_value | flattened_inner_field | flattened_inner_value | object_inner_field |
+ | "tenant1:wks:ArraysOfObjectsTestCollection:4.0.0" | "r3-index_record_arrayofobjects" | 1 | "tenant1-wks-arraysofobjectstestcollection-4.0.0" | "data.default.viewers@tenant1" | "data.NestedTest" | "data.NestedTest.NumberTest" | 12345 | "data.NestedTest.StringTest" | "test string" | "data.FlattenedTest.StringTest" | "test string" | "ObjectTest" |
+
+ @as-ingested-coordinates
+ Scenario Outline: Ingest the record and Index in the Elastic with AsIngestedCoordinates properties
+ When I ingest records with the <recordFile> with <acl> for a given <kind>
+ Then I should be able search <number> documents for the <index> by flattened inner properties (<flattened_inner_field>, <flattened_inner_value>)
+ Then I should get <inner_field> in search response for the <index>
+
+ Examples:
+ | kind | recordFile | number | index | acl | flattened_inner_field | flattened_inner_value | inner_field |
+ | "tenant1:wks:master-data--Wellbore:2.0.3" | "r3-index_record_wks_master" | 1 | "tenant1-wks-master-data--wellbore-2.0.3" | "data.default.viewers@tenant1" | "data.ProjectedBottomHoleLocation.AsIngestedCoordinates.CoordinateReferenceSystemID" | "osdu:reference-data--CoordinateReferenceSystem:ED_1950_UTM_Zone_31N:" | "ProjectedBottomHoleLocation.AsIngestedCoordinates.persistableReferenceCrs" |
+ | "tenant1:wks:master-data--Wellbore:2.0.3" | "r3-index_record_wks_master" | 1 | "tenant1-wks-master-data--wellbore-2.0.3" | "data.default.viewers@tenant1" | "data.SpatialLocation.AsIngestedCoordinates.CoordinateReferenceSystemID" | "osdu:reference-data--CoordinateReferenceSystem:ED_1950_UTM_Zone_31N:" | "SpatialLocation.AsIngestedCoordinates.persistableReferenceCrs" |
+ | "tenant1:indexer:virtual-properties-Integration:1.0.0" | "index_record_virtual_properties" | 1 | "tenant1-indexer-virtual-properties-integration-1.0.0" | "data.default.viewers@tenant1" | "data.ProjectedBottomHoleLocation.AsIngestedCoordinates.CoordinateReferenceSystemID" | "osdu:reference-data--CoordinateReferenceSystem:WGS_1984_World_Mercator:" | "ProjectedBottomHoleLocation.AsIngestedCoordinates.persistableReferenceCrs" |
+ | "tenant1:indexer:virtual-properties-Integration:1.0.0" | "index_record_virtual_properties" | 2 | "tenant1-indexer-virtual-properties-integration-1.0.0" | "data.default.viewers@tenant1" | "data.GeographicBottomHoleLocation.AsIngestedCoordinates.CoordinateReferenceSystemID" | "osdu:reference-data--CoordinateReferenceSystem:WGS_1984_World_Mercator:" | "GeographicBottomHoleLocation.AsIngestedCoordinates.persistableReferenceCrs" |
+ | "tenant1:indexer:virtual-properties-Integration:1.0.0" | "index_record_virtual_properties" | 3 | "tenant1-indexer-virtual-properties-integration-1.0.0" | "data.default.viewers@tenant1" | "data.SpatialLocation.AsIngestedCoordinates.CoordinateReferenceSystemID" | "osdu:reference-data--CoordinateReferenceSystem:WGS_1984_World_Mercator:" | "SpatialLocation.AsIngestedCoordinates.persistableReferenceCrs" |
+
+ @as-ingested-coordinates
+ Scenario Outline: Ingest the record and Index in the Elastic with AsIngestedCoordinates properties and search by coordinates
+ When I ingest records with the <recordFile> with <acl> for a given <kind>
+ Then I should be able search <number> documents for the <index> by bounding box query with points (<topPointX>, <bottomPointX>) on field <pointX> and points (<topPointY>, <bottomPointY>) on field <pointY>
+
+ Examples:
+ | kind | recordFile | number | index | acl | pointX | topPointX | bottomPointX | pointY | topPointY | bottomPointY |
+ | "tenant1:wks:master-data--Wellbore:2.0.3" | "r3-index_record_wks_master" | 1 | "tenant1-wks-master-data--wellbore-2.0.3" | "data.default.viewers@tenant1" | "data.ProjectedBottomHoleLocation.AsIngestedCoordinates.FirstPoint.X" | 710000 | 700000 | "data.ProjectedBottomHoleLocation.AsIngestedCoordinates.FirstPoint.Y" | 5800000 | 5700000 |
+ | "tenant1:indexer:virtual-properties-Integration:1.0.0" | "index_record_virtual_properties" | 1 | "tenant1-indexer-virtual-properties-integration-1.0.0" | "data.default.viewers@tenant1" | "data.ProjectedBottomHoleLocation.AsIngestedCoordinates.FirstPoint.X" | 2600000 | 2500000 | "data.ProjectedBottomHoleLocation.AsIngestedCoordinates.FirstPoint.Y" | -3500000 | -3600000 |
+ | "tenant1:indexer:decimation-Integration:1.0.0" | "index_record_seismic_survey" | 1 | "tenant1-indexer-decimation-integration-1.0.0" | "data.default.viewers@tenant1" | "data.SpatialLocation.AsIngestedCoordinates.FirstPoint.X" | 1151940 | 1151930 | "data.SpatialLocation.AsIngestedCoordinates.FirstPoint.Y" | 4843810 | 4843800 |
+
+ @default
+ Scenario Outline: Synchronize meta attribute mapping on existing indexed kind
+ When I create index with <mappingFile> for a given <index> and <kind>
+ And I ingest records with the <recordFile> with <acl> for a given <kind>
+ Then I can validate indexed meta attributes for the <index> and given <kind>
+
+ Examples:
+ | kind | index | recordFile | mappingFile | acl |
+ | "tenant1:indexer:test-mapping--Sync:2.0.0" | "tenant1-indexer-test-mapping--sync-2.0.0" | "index_record_sync_mapping" | "index_record_sync_mapping" | "data.default.viewers@tenant1" |
+
+ @default
+ Scenario Outline: Ingest the record and Index in the Elastic Search
+ When I ingest records with the <recordFile> with <acl> for a given <kind_v1>
+ Then I should get the 1 documents for the <index_v1> in the Elastic Search
+ Then I ingest records with the <recordFile> with <acl> for a given <kind_v2>
+ Then I should get the 1 documents for the <index_v2> in the Elastic Search
+ Then I should not get any documents for the <index_v1> in the Elastic Search
+
+ Examples:
+ | kind_v1 | index_v1 | recordFile | acl | kind_v2 | index_v2 |
+ | "tenant1:indexer:test-update-data--Integration:1.0.1" | "tenant1-indexer-test-update-data--integration-1.0.1" | "index_update_records_kind_v1" | "data.default.viewers@tenant1" | "tenant1:indexer:test-update-data--Integration:2.0.1" | "tenant1-indexer-test-update-data--integration-2.0.1" |
diff --git a/indexer-acceptance-test/src/test/resources/features/info/Info.feature b/indexer-acceptance-test/src/test/resources/features/info/Info.feature
new file mode 100644
index 0000000000000000000000000000000000000000..92a5e03726dd2f968d6445b5bd7e10fa183b4c40
--- /dev/null
+++ b/indexer-acceptance-test/src/test/resources/features/info/Info.feature
@@ -0,0 +1,11 @@
+Feature: Fetch info about maven build and git repository.
+
+ @default
+ Scenario: Verify version info endpoint content
+ When I send get request to version info endpoint
+ Then I should get version info in response
+
+ @default
+ Scenario: Verify version info endpoint content for request with trailing slash
+ When I send get request to version info endpoint with trailing slash
+ Then I should get version info in response
diff --git a/indexer-acceptance-test/src/test/resources/logback-test.xml b/indexer-acceptance-test/src/test/resources/logback-test.xml
new file mode 100644
index 0000000000000000000000000000000000000000..22c6175d45f368b69c4006deb9ebf3ff5084579a
--- /dev/null
+++ b/indexer-acceptance-test/src/test/resources/logback-test.xml
@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<configuration>
+ <include resource="org/springframework/boot/logging/logback/defaults.xml"/>
+ <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%yellow([%thread]) %highlight(| %-5level |) %green(%d) %cyan(| %logger{15} |) %highlight(%msg) %n</pattern>
+ <charset>utf8</charset>
+ </encoder>
+ </appender>
+ <root level="INFO">
+ <appender-ref ref="CONSOLE" />
+ </root>
+</configuration>
diff --git a/indexer-acceptance-test/src/test/resources/testData/index-property-wellbore_v1.json b/indexer-acceptance-test/src/test/resources/testData/index-property-wellbore_v1.json
new file mode 100644
index 0000000000000000000000000000000000000000..469a0d22c7637a8fd867bbf2ed9aad733b87b839
--- /dev/null
+++ b/indexer-acceptance-test/src/test/resources/testData/index-property-wellbore_v1.json
@@ -0,0 +1,16 @@
+[{
+ "id": "tenant1:index-property--Wellbore:testIngest1",
+ "data": {
+ "Location": {
+ "latitude": 29.170640,
+ "longitude": -95.002875
+ },
+ "FacilityName": "Facility_123",
+ "NameAliases": [{
+ "AliasName": "123454321",
+ "AliasNameTypeID": "tenant1:reference-data--AliasNameType:UniqueIdentifier:"
+ }
+ ]
+ }
+ }
+]
diff --git a/indexer-acceptance-test/src/test/resources/testData/index-property-wellbore_v1.schema.json b/indexer-acceptance-test/src/test/resources/testData/index-property-wellbore_v1.schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..93d616f44e393c27e176724c2a97c80028f613bb
--- /dev/null
+++ b/indexer-acceptance-test/src/test/resources/testData/index-property-wellbore_v1.schema.json
@@ -0,0 +1,118 @@
+{
+ "schemaInfo": {
+ "schemaIdentity": {
+ "authority": "test",
+ "source": "indexer",
+ "entityType": "index-property--Wellbore",
+ "schemaVersionMajor": 1,
+ "schemaVersionMinor": 0,
+ "schemaVersionPatch": 0,
+ "id": "test:indexer:index-property--Wellbore:1.0.0"
+ },
+ "status": "DEVELOPMENT"
+ },
+ "schema": {
+ "definitions": {
+ "opendes:wks:core_dl_geopoint:1.0.0": {
+ "description": "A 2D point location in latitude and longitude referenced to WGS 84 if not specified otherwise.",
+ "properties": {
+ "latitude": {
+ "description": "The latitude value in degrees of arc (dega). Value range [-90, 90].",
+ "maximum": 90,
+ "minimum": -90,
+ "title": "Latitude",
+ "type": "number"
+ },
+ "longitude": {
+ "description": "The longitude value in degrees of arc (dega). Value range [-180, 180]",
+ "maximum": 180,
+ "minimum": -180,
+ "title": "Longitude",
+ "type": "number"
+ }
+ },
+ "required": [
+ "latitude",
+ "longitude"
+ ],
+ "title": "2D Map Location",
+ "type": "object"
+ },
+ "opendes:wks:AbstractAliasNames:1.0.0": {
+ "title": "AbstractAliasNames",
+ "type": "object",
+ "properties": {
+ "AliasNameTypeID": {
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-AliasNameType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "A classification of alias names such as by role played or type of source, such as regulatory name, regulatory code, company code, international standard name, etc.",
+ "x-osdu-relationship": [{
+ "EntityType": "AliasNameType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "type": "string"
+ },
+ "EffectiveDateTime": {
+ "format": "date-time",
+ "type": "string",
+ "description": "The date and time when an alias name becomes effective."
+ },
+ "AliasName": {
+ "type": "string",
+ "description": "Alternative Name value of defined name type for an object."
+ },
+ "TerminationDateTime": {
+ "format": "date-time",
+ "type": "string",
+ "description": "The data and time when an alias name is no longer in effect."
+ },
+ "DefinitionOrganisationID": {
+ "pattern": "^[\\w\\-\\.]+:(reference-data\\-\\-StandardsOrganisation|master-data\\-\\-Organisation):[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The StandardsOrganisation (reference-data) or Organisation (master-data) that provided the name (the source).",
+ "x-osdu-relationship": [{
+ "EntityType": "StandardsOrganisation",
+ "GroupType": "reference-data"
+ }, {
+ "EntityType": "Organisation",
+ "GroupType": "master-data"
+ }
+ ],
+ "type": "string"
+ }
+ }
+ }
+ },
+ "properties": {
+ "data": {
+ "allOf": [
+ {
+ "type": "object",
+ "properties": {
+ "Location": {
+ "$ref": "#/definitions/opendes:wks:core_dl_geopoint:1.0.0",
+ "description": "The wellbore's position .",
+ "format": "core:dl:geopoint:1.0.0",
+ "title": "WGS 84 Position",
+ "type": "object"
+ },
+ "NameAliases": {
+ "x-osdu-indexing": {
+ "type": "nested"
+ },
+ "description": "Alternative names, including historical, by which this master data is/has been known (it should include all the identifiers).",
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/opendes:wks:AbstractAliasNames:1.0.0"
+ }
+ },
+ "FacilityName": {
+ "description": "Name of the Facility.",
+ "type": "string"
+ }
+ }
+ }
+ ]
+ }
+ }
+ }
+}
diff --git a/indexer-acceptance-test/src/test/resources/testData/index-property-wellbore_v2.json b/indexer-acceptance-test/src/test/resources/testData/index-property-wellbore_v2.json
new file mode 100644
index 0000000000000000000000000000000000000000..a8ac61482f4681f435e14960a07f70aa0d26320a
--- /dev/null
+++ b/indexer-acceptance-test/src/test/resources/testData/index-property-wellbore_v2.json
@@ -0,0 +1,16 @@
+[{
+ "id": "tenant1:index-property--Wellbore:testIngest1",
+ "data": {
+ "Location": {
+ "latitude": 29.170640,
+ "longitude": -95.002875
+ },
+ "FacilityName": "facility_456",
+ "NameAliases": [{
+ "AliasName": "123454321",
+ "AliasNameTypeID": "tenant1:reference-data--AliasNameType:UniqueIdentifier:"
+ }
+ ]
+ }
+ }
+]
diff --git a/indexer-acceptance-test/src/test/resources/testData/index-property-welllog_v1.json b/indexer-acceptance-test/src/test/resources/testData/index-property-welllog_v1.json
new file mode 100644
index 0000000000000000000000000000000000000000..b22a36722b78de757f1631809f9f4ea51fa02f25
--- /dev/null
+++ b/indexer-acceptance-test/src/test/resources/testData/index-property-welllog_v1.json
@@ -0,0 +1,8 @@
+[{
+ "id": "tenant1:index-property--WellLog:testIngest2",
+ "data": {
+ "Name": "Log_ABC",
+ "WellboreID": "tenant1:index-property--Wellbore:testIngest1"
+ }
+ }
+]
diff --git a/indexer-acceptance-test/src/test/resources/testData/index-property-welllog_v1.schema.json b/indexer-acceptance-test/src/test/resources/testData/index-property-welllog_v1.schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..1b9549b616e5cc889ed200f08decc0b08d4a481b
--- /dev/null
+++ b/indexer-acceptance-test/src/test/resources/testData/index-property-welllog_v1.schema.json
@@ -0,0 +1,36 @@
+{
+ "schemaInfo": {
+ "schemaIdentity": {
+ "authority": "test",
+ "source": "indexer",
+ "entityType": "index-property--WellLog",
+ "schemaVersionMajor": 1,
+ "schemaVersionMinor": 0,
+ "schemaVersionPatch": 0,
+ "id": "test:indexer:index-property--WellLog:1.0.0"
+ },
+ "status": "DEVELOPMENT"
+ },
+ "schema": {
+ "properties": {
+ "data": {
+ "allOf": [
+ {
+ "type": "object",
+ "properties": {
+ "WellboreID": {
+ "description": "Id of parent wellbore.",
+ "type": "string"
+ },
+ "Name": {
+ "description": "Name of welllog.",
+ "type": "string"
+ }
+ }
+ }
+ ]
+ }
+ }
+ }
+
+}
diff --git a/indexer-acceptance-test/src/test/resources/testData/index_record_seismic_survey.json b/indexer-acceptance-test/src/test/resources/testData/index_record_seismic_survey.json
new file mode 100644
index 0000000000000000000000000000000000000000..362a4468008c4f2bfad21ee37ec6d1077fadb33a
--- /dev/null
+++ b/indexer-acceptance-test/src/test/resources/testData/index_record_seismic_survey.json
@@ -0,0 +1,10276 @@
+[{
+ "id": "tenant1:<kindSubType>:testIngest1<timestamp>",
+ "data": {
+ "ProjectName": "NT94",
+ "SeismicGeometryTypeID": "opendes:reference-data--SeismicGeometryType:2D:",
+ "SpatialLocation": {
+ "AsIngestedCoordinates": {
+ "features": [{
+ "geometry": {
+ "coordinates": [
+ [
+ [
+ 1151937,
+ 4843801
+ ],
+ [
+ 1151932,
+ 4843790
+ ],
+ [
+ 1151925,
+ 4843780
+ ],
+ [
+ 1151919,
+ 4843769
+ ],
+ [
+ 1151912,
+ 4843759
+ ],
+ [
+ 1151906,
+ 4843748
+ ],
+ [
+ 1151899,
+ 4843738
+ ],
+ [
+ 1151893,
+ 4843727
+ ],
+ [
+ 1151886,
+ 4843717
+ ],
+ [
+ 1151880,
+ 4843706
+ ],
+ [
+ 1151873,
+ 4843696
+ ],
+ [
+ 1151867,
+ 4843685
+ ],
+ [
+ 1151860,
+ 4843675
+ ],
+ [
+ 1151854,
+ 4843664
+ ],
+ [
+ 1151847,
+ 4843654
+ ],
+ [
+ 1151841,
+ 4843643
+ ],
+ [
+ 1151834,
+ 4843633
+ ],
+ [
+ 1151828,
+ 4843622
+ ],
+ [
+ 1151821,
+ 4843611
+ ],
+ [
+ 1151815,
+ 4843600
+ ],
+ [
+ 1151808,
+ 4843590
+ ],
+ [
+ 1151802,
+ 4843579
+ ],
+ [
+ 1151795,
+ 4843569
+ ],
+ [
+ 1151789,
+ 4843558
+ ],
+ [
+ 1151782,
+ 4843548
+ ],
+ [
+ 1151776,
+ 4843537
+ ],
+ [
+ 1151769,
+ 4843527
+ ],
+ [
+ 1151763,
+ 4843516
+ ],
+ [
+ 1151756,
+ 4843506
+ ],
+ [
+ 1151750,
+ 4843495
+ ],
+ [
+ 1151743,
+ 4843485
+ ],
+ [
+ 1151737,
+ 4843474
+ ],
+ [
+ 1151730,
+ 4843464
+ ],
+ [
+ 1151724,
+ 4843453
+ ],
+ [
+ 1151717,
+ 4843443
+ ],
+ [
+ 1151711,
+ 4843432
+ ],
+ [
+ 1151704,
+ 4843422
+ ],
+ [
+ 1151698,
+ 4843411
+ ],
+ [
+ 1151691,
+ 4843401
+ ],
+ [
+ 1151685,
+ 4843390
+ ],
+ [
+ 1151678,
+ 4843380
+ ],
+ [
+ 1151672,
+ 4843369
+ ],
+ [
+ 1151665,
+ 4843359
+ ],
+ [
+ 1151659,
+ 4843348
+ ],
+ [
+ 1151652,
+ 4843337
+ ],
+ [
+ 1151646,
+ 4843326
+ ],
+ [
+ 1151639,
+ 4843316
+ ],
+ [
+ 1151633,
+ 4843305
+ ],
+ [
+ 1151626,
+ 4843295
+ ],
+ [
+ 1151620,
+ 4843284
+ ],
+ [
+ 1151613,
+ 4843274
+ ],
+ [
+ 1151607,
+ 4843263
+ ],
+ [
+ 1151600,
+ 4843253
+ ],
+ [
+ 1151594,
+ 4843242
+ ],
+ [
+ 1151587,
+ 4843232
+ ],
+ [
+ 1151581,
+ 4843221
+ ],
+ [
+ 1151574,
+ 4843211
+ ],
+ [
+ 1151568,
+ 4843200
+ ],
+ [
+ 1151561,
+ 4843190
+ ],
+ [
+ 1151555,
+ 4843179
+ ],
+ [
+ 1151548,
+ 4843169
+ ],
+ [
+ 1151542,
+ 4843158
+ ],
+ [
+ 1151535,
+ 4843148
+ ],
+ [
+ 1151529,
+ 4843137
+ ],
+ [
+ 1151522,
+ 4843127
+ ],
+ [
+ 1151516,
+ 4843116
+ ],
+ [
+ 1151509,
+ 4843106
+ ],
+ [
+ 1151503,
+ 4843095
+ ],
+ [
+ 1151496,
+ 4843085
+ ],
+ [
+ 1151490,
+ 4843074
+ ],
+ [
+ 1151483,
+ 4843064
+ ],
+ [
+ 1151477,
+ 4843052
+ ],
+ [
+ 1151470,
+ 4843042
+ ],
+ [
+ 1151464,
+ 4843031
+ ],
+ [
+ 1151457,
+ 4843021
+ ],
+ [
+ 1151451,
+ 4843010
+ ],
+ [
+ 1151444,
+ 4843000
+ ],
+ [
+ 1151438,
+ 4842989
+ ],
+ [
+ 1151431,
+ 4842979
+ ],
+ [
+ 1151425,
+ 4842968
+ ],
+ [
+ 1151418,
+ 4842958
+ ],
+ [
+ 1151412,
+ 4842947
+ ],
+ [
+ 1151405,
+ 4842937
+ ],
+ [
+ 1151399,
+ 4842926
+ ],
+ [
+ 1151392,
+ 4842916
+ ],
+ [
+ 1151386,
+ 4842905
+ ],
+ [
+ 1151379,
+ 4842895
+ ],
+ [
+ 1151374,
+ 4842884
+ ],
+ [
+ 1151367,
+ 4842874
+ ],
+ [
+ 1151361,
+ 4842863
+ ],
+ [
+ 1151354,
+ 4842853
+ ],
+ [
+ 1151348,
+ 4842842
+ ],
+ [
+ 1151341,
+ 4842832
+ ],
+ [
+ 1151335,
+ 4842821
+ ],
+ [
+ 1151328,
+ 4842811
+ ],
+ [
+ 1151322,
+ 4842800
+ ],
+ [
+ 1151315,
+ 4842790
+ ],
+ [
+ 1151309,
+ 4842778
+ ],
+ [
+ 1151302,
+ 4842768
+ ],
+ [
+ 1151296,
+ 4842757
+ ],
+ [
+ 1151289,
+ 4842747
+ ],
+ [
+ 1151283,
+ 4842736
+ ],
+ [
+ 1151276,
+ 4842726
+ ],
+ [
+ 1151270,
+ 4842715
+ ],
+ [
+ 1151263,
+ 4842705
+ ],
+ [
+ 1151257,
+ 4842694
+ ],
+ [
+ 1151250,
+ 4842684
+ ],
+ [
+ 1151244,
+ 4842673
+ ],
+ [
+ 1151237,
+ 4842663
+ ],
+ [
+ 1151231,
+ 4842652
+ ],
+ [
+ 1151224,
+ 4842642
+ ],
+ [
+ 1151218,
+ 4842631
+ ],
+ [
+ 1151211,
+ 4842621
+ ],
+ [
+ 1151205,
+ 4842610
+ ],
+ [
+ 1151198,
+ 4842600
+ ],
+ [
+ 1151192,
+ 4842589
+ ],
+ [
+ 1151185,
+ 4842579
+ ],
+ [
+ 1151179,
+ 4842568
+ ],
+ [
+ 1151172,
+ 4842558
+ ],
+ [
+ 1151166,
+ 4842547
+ ],
+ [
+ 1151159,
+ 4842537
+ ],
+ [
+ 1151153,
+ 4842526
+ ],
+ [
+ 1151146,
+ 4842516
+ ],
+ [
+ 1151140,
+ 4842505
+ ],
+ [
+ 1151133,
+ 4842494
+ ],
+ [
+ 1151127,
+ 4842483
+ ],
+ [
+ 1151120,
+ 4842473
+ ],
+ [
+ 1151114,
+ 4842462
+ ],
+ [
+ 1151107,
+ 4842452
+ ],
+ [
+ 1151101,
+ 4842441
+ ],
+ [
+ 1151094,
+ 4842431
+ ],
+ [
+ 1151088,
+ 4842420
+ ],
+ [
+ 1151081,
+ 4842410
+ ],
+ [
+ 1151075,
+ 4842399
+ ],
+ [
+ 1151068,
+ 4842389
+ ],
+ [
+ 1151062,
+ 4842378
+ ],
+ [
+ 1151055,
+ 4842368
+ ],
+ [
+ 1151049,
+ 4842357
+ ],
+ [
+ 1151042,
+ 4842347
+ ],
+ [
+ 1151036,
+ 4842336
+ ],
+ [
+ 1151029,
+ 4842326
+ ],
+ [
+ 1151023,
+ 4842315
+ ],
+ [
+ 1151016,
+ 4842305
+ ],
+ [
+ 1151010,
+ 4842294
+ ],
+ [
+ 1151003,
+ 4842284
+ ],
+ [
+ 1150997,
+ 4842273
+ ],
+ [
+ 1150990,
+ 4842263
+ ],
+ [
+ 1150984,
+ 4842252
+ ],
+ [
+ 1150977,
+ 4842242
+ ],
+ [
+ 1150971,
+ 4842231
+ ],
+ [
+ 1150964,
+ 4842221
+ ],
+ [
+ 1150958,
+ 4842209
+ ],
+ [
+ 1150951,
+ 4842199
+ ],
+ [
+ 1150945,
+ 4842188
+ ],
+ [
+ 1150938,
+ 4842178
+ ],
+ [
+ 1150932,
+ 4842167
+ ],
+ [
+ 1150925,
+ 4842157
+ ],
+ [
+ 1150919,
+ 4842146
+ ],
+ [
+ 1150912,
+ 4842136
+ ],
+ [
+ 1150906,
+ 4842125
+ ],
+ [
+ 1150899,
+ 4842115
+ ],
+ [
+ 1150893,
+ 4842104
+ ],
+ [
+ 1150886,
+ 4842094
+ ],
+ [
+ 1150880,
+ 4842083
+ ],
+ [
+ 1150873,
+ 4842073
+ ],
+ [
+ 1150867,
+ 4842062
+ ],
+ [
+ 1150860,
+ 4842052
+ ],
+ [
+ 1150853,
+ 4842042
+ ],
+ [
+ 1150845,
+ 4842032
+ ],
+ [
+ 1150838,
+ 4842021
+ ],
+ [
+ 1150831,
+ 4842011
+ ],
+ [
+ 1150824,
+ 4842000
+ ],
+ [
+ 1150817,
+ 4841990
+ ],
+ [
+ 1150811,
+ 4841979
+ ],
+ [
+ 1150803,
+ 4841969
+ ],
+ [
+ 1150796,
+ 4841959
+ ],
+ [
+ 1150788,
+ 4841949
+ ],
+ [
+ 1150781,
+ 4841938
+ ],
+ [
+ 1150774,
+ 4841928
+ ],
+ [
+ 1150767,
+ 4841917
+ ],
+ [
+ 1150759,
+ 4841907
+ ],
+ [
+ 1150752,
+ 4841897
+ ],
+ [
+ 1150745,
+ 4841887
+ ],
+ [
+ 1150738,
+ 4841876
+ ],
+ [
+ 1150730,
+ 4841866
+ ],
+ [
+ 1150723,
+ 4841856
+ ],
+ [
+ 1150715,
+ 4841846
+ ],
+ [
+ 1150708,
+ 4841836
+ ],
+ [
+ 1150700,
+ 4841826
+ ],
+ [
+ 1150693,
+ 4841816
+ ],
+ [
+ 1150685,
+ 4841806
+ ],
+ [
+ 1150678,
+ 4841796
+ ],
+ [
+ 1150670,
+ 4841786
+ ],
+ [
+ 1150663,
+ 4841776
+ ],
+ [
+ 1150655,
+ 4841767
+ ],
+ [
+ 1150648,
+ 4841756
+ ],
+ [
+ 1150640,
+ 4841746
+ ],
+ [
+ 1150633,
+ 4841736
+ ],
+ [
+ 1150625,
+ 4841727
+ ],
+ [
+ 1150618,
+ 4841717
+ ],
+ [
+ 1150610,
+ 4841707
+ ],
+ [
+ 1150603,
+ 4841697
+ ],
+ [
+ 1150596,
+ 4841687
+ ],
+ [
+ 1150589,
+ 4841675
+ ],
+ [
+ 1150582,
+ 4841665
+ ],
+ [
+ 1150575,
+ 4841654
+ ],
+ [
+ 1150568,
+ 4841644
+ ],
+ [
+ 1150561,
+ 4841633
+ ],
+ [
+ 1150554,
+ 4841623
+ ],
+ [
+ 1150547,
+ 4841613
+ ],
+ [
+ 1150540,
+ 4841603
+ ],
+ [
+ 1150534,
+ 4841592
+ ],
+ [
+ 1150527,
+ 4841582
+ ],
+ [
+ 1150520,
+ 4841571
+ ],
+ [
+ 1150513,
+ 4841561
+ ],
+ [
+ 1150506,
+ 4841550
+ ],
+ [
+ 1150498,
+ 4841540
+ ],
+ [
+ 1150491,
+ 4841529
+ ],
+ [
+ 1150484,
+ 4841519
+ ],
+ [
+ 1150477,
+ 4841509
+ ],
+ [
+ 1150469,
+ 4841499
+ ],
+ [
+ 1150462,
+ 4841488
+ ],
+ [
+ 1150454,
+ 4841478
+ ],
+ [
+ 1150447,
+ 4841468
+ ],
+ [
+ 1150440,
+ 4841458
+ ],
+ [
+ 1150433,
+ 4841447
+ ],
+ [
+ 1150425,
+ 4841437
+ ],
+ [
+ 1150418,
+ 4841427
+ ],
+ [
+ 1150411,
+ 4841416
+ ],
+ [
+ 1150404,
+ 4841406
+ ],
+ [
+ 1150396,
+ 4841396
+ ],
+ [
+ 1150389,
+ 4841386
+ ],
+ [
+ 1150381,
+ 4841376
+ ],
+ [
+ 1150374,
+ 4841366
+ ],
+ [
+ 1150366,
+ 4841356
+ ],
+ [
+ 1150359,
+ 4841346
+ ],
+ [
+ 1150351,
+ 4841336
+ ],
+ [
+ 1150344,
+ 4841326
+ ],
+ [
+ 1150336,
+ 4841316
+ ],
+ [
+ 1150329,
+ 4841306
+ ],
+ [
+ 1150321,
+ 4841296
+ ],
+ [
+ 1150314,
+ 4841286
+ ],
+ [
+ 1150306,
+ 4841276
+ ],
+ [
+ 1150299,
+ 4841265
+ ],
+ [
+ 1150291,
+ 4841255
+ ],
+ [
+ 1150284,
+ 4841245
+ ],
+ [
+ 1150277,
+ 4841235
+ ],
+ [
+ 1150270,
+ 4841224
+ ],
+ [
+ 1150262,
+ 4841214
+ ],
+ [
+ 1150255,
+ 4841204
+ ],
+ [
+ 1150248,
+ 4841194
+ ],
+ [
+ 1150241,
+ 4841183
+ ],
+ [
+ 1150233,
+ 4841173
+ ],
+ [
+ 1150226,
+ 4841163
+ ],
+ [
+ 1150219,
+ 4841152
+ ],
+ [
+ 1150212,
+ 4841142
+ ],
+ [
+ 1150204,
+ 4841132
+ ],
+ [
+ 1150197,
+ 4841122
+ ],
+ [
+ 1150189,
+ 4841112
+ ],
+ [
+ 1150182,
+ 4841102
+ ],
+ [
+ 1150174,
+ 4841092
+ ],
+ [
+ 1150167,
+ 4841082
+ ],
+ [
+ 1150160,
+ 4841072
+ ],
+ [
+ 1150153,
+ 4841062
+ ],
+ [
+ 1150145,
+ 4841052
+ ],
+ [
+ 1150138,
+ 4841042
+ ],
+ [
+ 1150130,
+ 4841032
+ ],
+ [
+ 1150123,
+ 4841021
+ ],
+ [
+ 1150115,
+ 4841011
+ ],
+ [
+ 1150107,
+ 4841001
+ ],
+ [
+ 1150099,
+ 4840991
+ ],
+ [
+ 1150091,
+ 4840981
+ ],
+ [
+ 1150083,
+ 4840971
+ ],
+ [
+ 1150076,
+ 4840961
+ ],
+ [
+ 1150068,
+ 4840951
+ ],
+ [
+ 1150061,
+ 4840941
+ ],
+ [
+ 1150053,
+ 4840931
+ ],
+ [
+ 1150046,
+ 4840921
+ ],
+ [
+ 1150038,
+ 4840911
+ ],
+ [
+ 1150031,
+ 4840900
+ ],
+ [
+ 1150023,
+ 4840890
+ ],
+ [
+ 1150016,
+ 4840880
+ ],
+ [
+ 1150008,
+ 4840871
+ ],
+ [
+ 1150001,
+ 4840860
+ ],
+ [
+ 1149995,
+ 4840850
+ ],
+ [
+ 1149988,
+ 4840840
+ ],
+ [
+ 1149980,
+ 4840831
+ ],
+ [
+ 1149973,
+ 4840821
+ ],
+ [
+ 1149965,
+ 4840811
+ ],
+ [
+ 1149957,
+ 4840801
+ ],
+ [
+ 1149949,
+ 4840791
+ ],
+ [
+ 1149942,
+ 4840781
+ ],
+ [
+ 1149934,
+ 4840771
+ ],
+ [
+ 1149927,
+ 4840761
+ ],
+ [
+ 1149919,
+ 4840751
+ ],
+ [
+ 1149912,
+ 4840741
+ ],
+ [
+ 1149904,
+ 4840732
+ ],
+ [
+ 1149897,
+ 4840722
+ ],
+ [
+ 1149889,
+ 4840712
+ ],
+ [
+ 1149882,
+ 4840702
+ ],
+ [
+ 1149874,
+ 4840692
+ ],
+ [
+ 1149867,
+ 4840682
+ ],
+ [
+ 1149859,
+ 4840672
+ ],
+ [
+ 1149852,
+ 4840662
+ ],
+ [
+ 1149844,
+ 4840652
+ ],
+ [
+ 1149837,
+ 4840640
+ ],
+ [
+ 1149830,
+ 4840630
+ ],
+ [
+ 1149823,
+ 4840620
+ ],
+ [
+ 1149815,
+ 4840610
+ ],
+ [
+ 1149808,
+ 4840600
+ ],
+ [
+ 1149800,
+ 4840590
+ ],
+ [
+ 1149793,
+ 4840579
+ ],
+ [
+ 1149785,
+ 4840569
+ ],
+ [
+ 1149778,
+ 4840559
+ ],
+ [
+ 1149771,
+ 4840549
+ ],
+ [
+ 1149764,
+ 4840539
+ ],
+ [
+ 1149756,
+ 4840529
+ ],
+ [
+ 1149749,
+ 4840518
+ ],
+ [
+ 1149742,
+ 4840508
+ ],
+ [
+ 1149735,
+ 4840498
+ ],
+ [
+ 1149727,
+ 4840488
+ ],
+ [
+ 1149720,
+ 4840477
+ ],
+ [
+ 1149712,
+ 4840467
+ ],
+ [
+ 1149705,
+ 4840457
+ ],
+ [
+ 1149697,
+ 4840447
+ ],
+ [
+ 1149690,
+ 4840437
+ ],
+ [
+ 1149682,
+ 4840427
+ ],
+ [
+ 1149675,
+ 4840417
+ ],
+ [
+ 1149668,
+ 4840407
+ ],
+ [
+ 1149660,
+ 4840397
+ ],
+ [
+ 1149652,
+ 4840386
+ ],
+ [
+ 1149645,
+ 4840376
+ ],
+ [
+ 1149637,
+ 4840366
+ ],
+ [
+ 1149630,
+ 4840356
+ ],
+ [
+ 1149622,
+ 4840346
+ ],
+ [
+ 1149615,
+ 4840336
+ ],
+ [
+ 1149607,
+ 4840326
+ ],
+ [
+ 1149600,
+ 4840316
+ ],
+ [
+ 1149592,
+ 4840306
+ ],
+ [
+ 1149585,
+ 4840296
+ ],
+ [
+ 1149578,
+ 4840286
+ ],
+ [
+ 1149571,
+ 4840276
+ ],
+ [
+ 1149563,
+ 4840266
+ ],
+ [
+ 1149556,
+ 4840256
+ ],
+ [
+ 1149548,
+ 4840246
+ ],
+ [
+ 1149541,
+ 4840235
+ ],
+ [
+ 1149534,
+ 4840225
+ ],
+ [
+ 1149527,
+ 4840215
+ ],
+ [
+ 1149519,
+ 4840205
+ ],
+ [
+ 1149512,
+ 4840195
+ ],
+ [
+ 1149504,
+ 4840185
+ ],
+ [
+ 1149497,
+ 4840174
+ ],
+ [
+ 1149489,
+ 4840164
+ ],
+ [
+ 1149482,
+ 4840154
+ ],
+ [
+ 1149474,
+ 4840144
+ ],
+ [
+ 1149467,
+ 4840134
+ ],
+ [
+ 1149459,
+ 4840123
+ ],
+ [
+ 1149452,
+ 4840113
+ ],
+ [
+ 1149444,
+ 4840103
+ ],
+ [
+ 1149437,
+ 4840093
+ ],
+ [
+ 1149429,
+ 4840083
+ ],
+ [
+ 1149422,
+ 4840073
+ ],
+ [
+ 1149414,
+ 4840063
+ ],
+ [
+ 1149407,
+ 4840053
+ ],
+ [
+ 1149399,
+ 4840043
+ ],
+ [
+ 1149392,
+ 4840033
+ ],
+ [
+ 1149384,
+ 4840024
+ ],
+ [
+ 1149377,
+ 4840014
+ ],
+ [
+ 1149369,
+ 4840004
+ ],
+ [
+ 1149362,
+ 4839993
+ ],
+ [
+ 1149354,
+ 4839983
+ ],
+ [
+ 1149347,
+ 4839973
+ ],
+ [
+ 1149340,
+ 4839963
+ ],
+ [
+ 1149333,
+ 4839953
+ ],
+ [
+ 1149325,
+ 4839943
+ ],
+ [
+ 1149318,
+ 4839932
+ ],
+ [
+ 1149310,
+ 4839922
+ ],
+ [
+ 1149303,
+ 4839912
+ ],
+ [
+ 1149296,
+ 4839902
+ ],
+ [
+ 1149289,
+ 4839891
+ ],
+ [
+ 1149281,
+ 4839881
+ ],
+ [
+ 1149274,
+ 4839870
+ ],
+ [
+ 1149266,
+ 4839860
+ ],
+ [
+ 1149259,
+ 4839850
+ ],
+ [
+ 1149251,
+ 4839840
+ ],
+ [
+ 1149244,
+ 4839830
+ ],
+ [
+ 1149236,
+ 4839820
+ ],
+ [
+ 1149229,
+ 4839810
+ ],
+ [
+ 1149221,
+ 4839800
+ ],
+ [
+ 1149214,
+ 4839790
+ ],
+ [
+ 1149206,
+ 4839780
+ ],
+ [
+ 1149199,
+ 4839769
+ ],
+ [
+ 1149191,
+ 4839759
+ ],
+ [
+ 1149184,
+ 4839749
+ ],
+ [
+ 1149176,
+ 4839739
+ ],
+ [
+ 1149169,
+ 4839729
+ ],
+ [
+ 1149161,
+ 4839719
+ ],
+ [
+ 1149154,
+ 4839709
+ ],
+ [
+ 1149146,
+ 4839699
+ ],
+ [
+ 1149139,
+ 4839689
+ ],
+ [
+ 1149131,
+ 4839680
+ ],
+ [
+ 1149124,
+ 4839670
+ ],
+ [
+ 1149116,
+ 4839660
+ ],
+ [
+ 1149109,
+ 4839650
+ ],
+ [
+ 1149101,
+ 4839640
+ ],
+ [
+ 1149094,
+ 4839630
+ ],
+ [
+ 1149086,
+ 4839621
+ ],
+ [
+ 1149079,
+ 4839610
+ ],
+ [
+ 1149071,
+ 4839600
+ ],
+ [
+ 1149064,
+ 4839590
+ ],
+ [
+ 1149056,
+ 4839580
+ ],
+ [
+ 1149049,
+ 4839570
+ ],
+ [
+ 1149041,
+ 4839560
+ ],
+ [
+ 1149034,
+ 4839550
+ ],
+ [
+ 1149026,
+ 4839540
+ ],
+ [
+ 1149019,
+ 4839529
+ ],
+ [
+ 1149011,
+ 4839519
+ ],
+ [
+ 1149004,
+ 4839508
+ ],
+ [
+ 1148996,
+ 4839498
+ ],
+ [
+ 1148989,
+ 4839488
+ ],
+ [
+ 1148981,
+ 4839478
+ ],
+ [
+ 1148974,
+ 4839468
+ ],
+ [
+ 1148966,
+ 4839458
+ ],
+ [
+ 1148959,
+ 4839448
+ ],
+ [
+ 1148951,
+ 4839438
+ ],
+ [
+ 1148944,
+ 4839428
+ ],
+ [
+ 1148936,
+ 4839418
+ ],
+ [
+ 1148929,
+ 4839408
+ ],
+ [
+ 1148921,
+ 4839398
+ ],
+ [
+ 1148913,
+ 4839388
+ ],
+ [
+ 1148905,
+ 4839378
+ ],
+ [
+ 1148897,
+ 4839368
+ ],
+ [
+ 1148889,
+ 4839357
+ ],
+ [
+ 1148882,
+ 4839347
+ ],
+ [
+ 1148874,
+ 4839338
+ ],
+ [
+ 1148866,
+ 4839328
+ ],
+ [
+ 1148858,
+ 4839318
+ ],
+ [
+ 1148851,
+ 4839308
+ ],
+ [
+ 1148843,
+ 4839299
+ ],
+ [
+ 1148836,
+ 4839289
+ ],
+ [
+ 1148828,
+ 4839279
+ ],
+ [
+ 1148820,
+ 4839269
+ ],
+ [
+ 1148812,
+ 4839260
+ ],
+ [
+ 1148804,
+ 4839250
+ ],
+ [
+ 1148796,
+ 4839240
+ ],
+ [
+ 1148789,
+ 4839230
+ ],
+ [
+ 1148781,
+ 4839220
+ ],
+ [
+ 1148774,
+ 4839210
+ ],
+ [
+ 1148766,
+ 4839201
+ ],
+ [
+ 1148758,
+ 4839191
+ ],
+ [
+ 1148750,
+ 4839181
+ ],
+ [
+ 1148743,
+ 4839171
+ ],
+ [
+ 1148735,
+ 4839161
+ ],
+ [
+ 1148728,
+ 4839151
+ ],
+ [
+ 1148720,
+ 4839141
+ ],
+ [
+ 1148713,
+ 4839131
+ ],
+ [
+ 1148705,
+ 4839121
+ ],
+ [
+ 1148698,
+ 4839110
+ ],
+ [
+ 1148690,
+ 4839101
+ ],
+ [
+ 1148683,
+ 4839091
+ ],
+ [
+ 1148675,
+ 4839081
+ ],
+ [
+ 1148667,
+ 4839071
+ ],
+ [
+ 1148659,
+ 4839061
+ ],
+ [
+ 1148652,
+ 4839051
+ ],
+ [
+ 1148644,
+ 4839041
+ ],
+ [
+ 1148637,
+ 4839031
+ ],
+ [
+ 1148629,
+ 4839021
+ ],
+ [
+ 1148622,
+ 4839011
+ ],
+ [
+ 1148614,
+ 4839002
+ ],
+ [
+ 1148607,
+ 4838992
+ ],
+ [
+ 1148599,
+ 4838982
+ ],
+ [
+ 1148592,
+ 4838972
+ ],
+ [
+ 1148584,
+ 4838962
+ ],
+ [
+ 1148577,
+ 4838952
+ ],
+ [
+ 1148569,
+ 4838942
+ ],
+ [
+ 1148562,
+ 4838932
+ ],
+ [
+ 1148554,
+ 4838922
+ ],
+ [
+ 1148547,
+ 4838912
+ ],
+ [
+ 1148539,
+ 4838902
+ ],
+ [
+ 1148532,
+ 4838892
+ ],
+ [
+ 1148525,
+ 4838882
+ ],
+ [
+ 1148518,
+ 4838871
+ ],
+ [
+ 1148510,
+ 4838861
+ ],
+ [
+ 1148503,
+ 4838850
+ ],
+ [
+ 1148495,
+ 4838840
+ ],
+ [
+ 1148488,
+ 4838829
+ ],
+ [
+ 1148480,
+ 4838819
+ ],
+ [
+ 1148473,
+ 4838809
+ ],
+ [
+ 1148465,
+ 4838799
+ ],
+ [
+ 1148458,
+ 4838789
+ ],
+ [
+ 1148451,
+ 4838779
+ ],
+ [
+ 1148444,
+ 4838769
+ ],
+ [
+ 1148436,
+ 4838759
+ ],
+ [
+ 1148429,
+ 4838749
+ ],
+ [
+ 1148421,
+ 4838739
+ ],
+ [
+ 1148414,
+ 4838728
+ ],
+ [
+ 1148406,
+ 4838718
+ ],
+ [
+ 1148399,
+ 4838708
+ ],
+ [
+ 1148392,
+ 4838698
+ ],
+ [
+ 1148385,
+ 4838688
+ ],
+ [
+ 1148377,
+ 4838678
+ ],
+ [
+ 1148370,
+ 4838668
+ ],
+ [
+ 1148362,
+ 4838658
+ ],
+ [
+ 1148355,
+ 4838647
+ ],
+ [
+ 1148347,
+ 4838637
+ ],
+ [
+ 1148340,
+ 4838627
+ ],
+ [
+ 1148332,
+ 4838617
+ ],
+ [
+ 1148325,
+ 4838607
+ ],
+ [
+ 1148317,
+ 4838597
+ ],
+ [
+ 1148310,
+ 4838587
+ ],
+ [
+ 1148302,
+ 4838577
+ ],
+ [
+ 1148295,
+ 4838567
+ ],
+ [
+ 1148287,
+ 4838557
+ ],
+ [
+ 1148280,
+ 4838547
+ ],
+ [
+ 1148272,
+ 4838537
+ ],
+ [
+ 1148264,
+ 4838527
+ ],
+ [
+ 1148257,
+ 4838517
+ ],
+ [
+ 1148250,
+ 4838507
+ ],
+ [
+ 1148242,
+ 4838497
+ ],
+ [
+ 1148235,
+ 4838487
+ ],
+ [
+ 1148227,
+ 4838478
+ ],
+ [
+ 1148220,
+ 4838468
+ ],
+ [
+ 1148212,
+ 4838458
+ ],
+ [
+ 1148205,
+ 4838448
+ ],
+ [
+ 1148197,
+ 4838438
+ ],
+ [
+ 1148190,
+ 4838428
+ ],
+ [
+ 1148183,
+ 4838418
+ ],
+ [
+ 1148176,
+ 4838407
+ ],
+ [
+ 1148168,
+ 4838397
+ ],
+ [
+ 1148161,
+ 4838386
+ ],
+ [
+ 1148153,
+ 4838376
+ ],
+ [
+ 1148146,
+ 4838365
+ ],
+ [
+ 1148138,
+ 4838355
+ ],
+ [
+ 1148130,
+ 4838344
+ ],
+ [
+ 1148122,
+ 4838334
+ ],
+ [
+ 1148115,
+ 4838324
+ ],
+ [
+ 1148107,
+ 4838314
+ ],
+ [
+ 1148099,
+ 4838304
+ ],
+ [
+ 1148091,
+ 4838294
+ ],
+ [
+ 1148083,
+ 4838284
+ ],
+ [
+ 1148075,
+ 4838275
+ ],
+ [
+ 1148067,
+ 4838265
+ ],
+ [
+ 1148059,
+ 4838255
+ ],
+ [
+ 1148052,
+ 4838245
+ ],
+ [
+ 1148044,
+ 4838236
+ ],
+ [
+ 1148036,
+ 4838226
+ ],
+ [
+ 1148028,
+ 4838217
+ ],
+ [
+ 1148020,
+ 4838207
+ ],
+ [
+ 1148012,
+ 4838197
+ ],
+ [
+ 1148004,
+ 4838187
+ ],
+ [
+ 1147996,
+ 4838178
+ ],
+ [
+ 1147988,
+ 4838168
+ ],
+ [
+ 1147980,
+ 4838159
+ ],
+ [
+ 1147973,
+ 4838149
+ ],
+ [
+ 1147965,
+ 4838140
+ ],
+ [
+ 1147958,
+ 4838130
+ ],
+ [
+ 1147950,
+ 4838121
+ ],
+ [
+ 1147942,
+ 4838111
+ ],
+ [
+ 1147934,
+ 4838100
+ ],
+ [
+ 1147927,
+ 4838090
+ ],
+ [
+ 1147919,
+ 4838080
+ ],
+ [
+ 1147912,
+ 4838070
+ ],
+ [
+ 1147904,
+ 4838060
+ ],
+ [
+ 1147897,
+ 4838050
+ ],
+ [
+ 1147889,
+ 4838040
+ ],
+ [
+ 1147882,
+ 4838030
+ ],
+ [
+ 1147874,
+ 4838021
+ ],
+ [
+ 1147867,
+ 4838011
+ ],
+ [
+ 1147859,
+ 4838001
+ ],
+ [
+ 1147852,
+ 4837991
+ ],
+ [
+ 1147844,
+ 4837981
+ ],
+ [
+ 1147837,
+ 4837971
+ ],
+ [
+ 1147830,
+ 4837961
+ ],
+ [
+ 1147823,
+ 4837950
+ ],
+ [
+ 1147815,
+ 4837940
+ ],
+ [
+ 1147808,
+ 4837930
+ ],
+ [
+ 1147800,
+ 4837920
+ ],
+ [
+ 1147793,
+ 4837909
+ ],
+ [
+ 1147785,
+ 4837899
+ ],
+ [
+ 1147778,
+ 4837889
+ ],
+ [
+ 1147771,
+ 4837879
+ ],
+ [
+ 1147764,
+ 4837869
+ ],
+ [
+ 1147756,
+ 4837859
+ ],
+ [
+ 1147749,
+ 4837848
+ ],
+ [
+ 1147741,
+ 4837837
+ ],
+ [
+ 1147734,
+ 4837827
+ ],
+ [
+ 1147726,
+ 4837817
+ ],
+ [
+ 1147719,
+ 4837807
+ ],
+ [
+ 1147711,
+ 4837797
+ ],
+ [
+ 1147704,
+ 4837787
+ ],
+ [
+ 1147696,
+ 4837777
+ ],
+ [
+ 1147689,
+ 4837767
+ ],
+ [
+ 1147681,
+ 4837757
+ ],
+ [
+ 1147674,
+ 4837747
+ ],
+ [
+ 1147666,
+ 4837737
+ ],
+ [
+ 1147659,
+ 4837727
+ ],
+ [
+ 1147651,
+ 4837717
+ ],
+ [
+ 1147644,
+ 4837707
+ ],
+ [
+ 1147636,
+ 4837697
+ ],
+ [
+ 1147629,
+ 4837687
+ ],
+ [
+ 1147621,
+ 4837677
+ ],
+ [
+ 1147614,
+ 4837667
+ ],
+ [
+ 1147606,
+ 4837657
+ ],
+ [
+ 1147599,
+ 4837647
+ ],
+ [
+ 1147591,
+ 4837637
+ ],
+ [
+ 1147584,
+ 4837627
+ ],
+ [
+ 1147576,
+ 4837617
+ ],
+ [
+ 1147569,
+ 4837607
+ ],
+ [
+ 1147561,
+ 4837598
+ ],
+ [
+ 1147554,
+ 4837586
+ ],
+ [
+ 1147546,
+ 4837576
+ ],
+ [
+ 1147539,
+ 4837566
+ ],
+ [
+ 1147531,
+ 4837556
+ ],
+ [
+ 1147524,
+ 4837546
+ ],
+ [
+ 1147516,
+ 4837536
+ ],
+ [
+ 1147509,
+ 4837526
+ ],
+ [
+ 1147502,
+ 4837516
+ ],
+ [
+ 1147495,
+ 4837506
+ ],
+ [
+ 1147487,
+ 4837496
+ ],
+ [
+ 1147480,
+ 4837485
+ ],
+ [
+ 1147472,
+ 4837475
+ ],
+ [
+ 1147465,
+ 4837465
+ ],
+ [
+ 1147457,
+ 4837455
+ ],
+ [
+ 1147450,
+ 4837444
+ ],
+ [
+ 1147442,
+ 4837434
+ ],
+ [
+ 1147435,
+ 4837424
+ ],
+ [
+ 1147427,
+ 4837414
+ ],
+ [
+ 1147420,
+ 4837404
+ ],
+ [
+ 1147412,
+ 4837394
+ ],
+ [
+ 1147405,
+ 4837384
+ ],
+ [
+ 1147397,
+ 4837374
+ ],
+ [
+ 1147389,
+ 4837364
+ ],
+ [
+ 1147381,
+ 4837354
+ ],
+ [
+ 1147375,
+ 4837344
+ ],
+ [
+ 1147367,
+ 4837334
+ ],
+ [
+ 1147360,
+ 4837324
+ ],
+ [
+ 1147352,
+ 4837314
+ ],
+ [
+ 1147345,
+ 4837304
+ ],
+ [
+ 1147337,
+ 4837294
+ ],
+ [
+ 1147330,
+ 4837284
+ ],
+ [
+ 1147322,
+ 4837274
+ ],
+ [
+ 1147315,
+ 4837264
+ ],
+ [
+ 1147307,
+ 4837255
+ ],
+ [
+ 1147300,
+ 4837245
+ ],
+ [
+ 1147292,
+ 4837235
+ ],
+ [
+ 1147285,
+ 4837225
+ ],
+ [
+ 1147277,
+ 4837215
+ ],
+ [
+ 1147270,
+ 4837205
+ ],
+ [
+ 1147262,
+ 4837195
+ ],
+ [
+ 1147255,
+ 4837185
+ ],
+ [
+ 1147247,
+ 4837175
+ ],
+ [
+ 1147240,
+ 4837165
+ ],
+ [
+ 1147232,
+ 4837155
+ ],
+ [
+ 1147225,
+ 4837144
+ ],
+ [
+ 1147217,
+ 4837134
+ ],
+ [
+ 1147210,
+ 4837124
+ ],
+ [
+ 1147202,
+ 4837114
+ ],
+ [
+ 1147194,
+ 4837103
+ ],
+ [
+ 1147186,
+ 4837093
+ ],
+ [
+ 1147179,
+ 4837082
+ ],
+ [
+ 1147171,
+ 4837072
+ ],
+ [
+ 1147164,
+ 4837062
+ ],
+ [
+ 1147156,
+ 4837052
+ ],
+ [
+ 1147149,
+ 4837042
+ ],
+ [
+ 1147141,
+ 4837033
+ ],
+ [
+ 1147134,
+ 4837023
+ ],
+ [
+ 1147127,
+ 4837014
+ ],
+ [
+ 1147120,
+ 4837004
+ ],
+ [
+ 1147112,
+ 4836994
+ ],
+ [
+ 1147104,
+ 4836983
+ ],
+ [
+ 1147096,
+ 4836973
+ ],
+ [
+ 1147089,
+ 4836963
+ ],
+ [
+ 1147082,
+ 4836953
+ ],
+ [
+ 1147075,
+ 4836943
+ ],
+ [
+ 1147067,
+ 4836933
+ ],
+ [
+ 1147060,
+ 4836923
+ ],
+ [
+ 1147053,
+ 4836913
+ ],
+ [
+ 1147046,
+ 4836903
+ ],
+ [
+ 1147038,
+ 4836893
+ ],
+ [
+ 1147031,
+ 4836883
+ ],
+ [
+ 1147023,
+ 4836873
+ ],
+ [
+ 1147016,
+ 4836862
+ ],
+ [
+ 1147009,
+ 4836852
+ ],
+ [
+ 1147002,
+ 4836841
+ ],
+ [
+ 1146994,
+ 4836830
+ ],
+ [
+ 1146987,
+ 4836820
+ ],
+ [
+ 1146979,
+ 4836810
+ ],
+ [
+ 1146972,
+ 4836800
+ ],
+ [
+ 1146965,
+ 4836790
+ ],
+ [
+ 1146958,
+ 4836780
+ ],
+ [
+ 1146950,
+ 4836770
+ ],
+ [
+ 1146943,
+ 4836759
+ ],
+ [
+ 1146935,
+ 4836749
+ ],
+ [
+ 1146928,
+ 4836739
+ ],
+ [
+ 1146920,
+ 4836729
+ ],
+ [
+ 1146913,
+ 4836719
+ ],
+ [
+ 1146905,
+ 4836709
+ ],
+ [
+ 1146898,
+ 4836699
+ ],
+ [
+ 1146890,
+ 4836689
+ ],
+ [
+ 1146883,
+ 4836679
+ ],
+ [
+ 1146875,
+ 4836669
+ ],
+ [
+ 1146868,
+ 4836659
+ ],
+ [
+ 1146860,
+ 4836649
+ ],
+ [
+ 1146852,
+ 4836639
+ ],
+ [
+ 1146844,
+ 4836629
+ ],
+ [
+ 1146837,
+ 4836619
+ ],
+ [
+ 1146829,
+ 4836609
+ ],
+ [
+ 1146821,
+ 4836599
+ ],
+ [
+ 1146813,
+ 4836589
+ ],
+ [
+ 1146806,
+ 4836578
+ ],
+ [
+ 1146798,
+ 4836569
+ ],
+ [
+ 1146790,
+ 4836559
+ ],
+ [
+ 1146782,
+ 4836549
+ ],
+ [
+ 1146775,
+ 4836539
+ ],
+ [
+ 1146767,
+ 4836530
+ ],
+ [
+ 1146760,
+ 4836520
+ ],
+ [
+ 1146752,
+ 4836510
+ ],
+ [
+ 1146744,
+ 4836500
+ ],
+ [
+ 1146736,
+ 4836491
+ ],
+ [
+ 1146729,
+ 4836481
+ ],
+ [
+ 1146721,
+ 4836471
+ ],
+ [
+ 1146714,
+ 4836461
+ ],
+ [
+ 1146706,
+ 4836451
+ ],
+ [
+ 1146698,
+ 4836441
+ ],
+ [
+ 1146690,
+ 4836432
+ ],
+ [
+ 1146683,
+ 4836422
+ ],
+ [
+ 1146675,
+ 4836412
+ ],
+ [
+ 1146668,
+ 4836402
+ ],
+ [
+ 1146660,
+ 4836392
+ ],
+ [
+ 1146653,
+ 4836382
+ ],
+ [
+ 1146645,
+ 4836372
+ ],
+ [
+ 1146638,
+ 4836362
+ ],
+ [
+ 1146630,
+ 4836352
+ ],
+ [
+ 1146622,
+ 4836342
+ ],
+ [
+ 1146614,
+ 4836332
+ ],
+ [
+ 1146607,
+ 4836321
+ ],
+ [
+ 1146599,
+ 4836311
+ ],
+ [
+ 1146591,
+ 4836301
+ ],
+ [
+ 1146583,
+ 4836292
+ ],
+ [
+ 1146575,
+ 4836282
+ ],
+ [
+ 1146567,
+ 4836273
+ ],
+ [
+ 1146559,
+ 4836263
+ ],
+ [
+ 1146550,
+ 4836253
+ ],
+ [
+ 1146543,
+ 4836244
+ ],
+ [
+ 1146535,
+ 4836235
+ ],
+ [
+ 1146527,
+ 4836225
+ ],
+ [
+ 1146519,
+ 4836215
+ ],
+ [
+ 1146511,
+ 4836205
+ ],
+ [
+ 1146503,
+ 4836196
+ ],
+ [
+ 1146495,
+ 4836186
+ ],
+ [
+ 1146487,
+ 4836177
+ ],
+ [
+ 1146479,
+ 4836167
+ ],
+ [
+ 1146471,
+ 4836158
+ ],
+ [
+ 1146464,
+ 4836148
+ ],
+ [
+ 1146456,
+ 4836139
+ ],
+ [
+ 1146449,
+ 4836129
+ ],
+ [
+ 1146441,
+ 4836120
+ ],
+ [
+ 1146433,
+ 4836110
+ ],
+ [
+ 1146425,
+ 4836101
+ ],
+ [
+ 1146418,
+ 4836091
+ ],
+ [
+ 1146410,
+ 4836080
+ ],
+ [
+ 1146403,
+ 4836070
+ ],
+ [
+ 1146395,
+ 4836061
+ ],
+ [
+ 1146388,
+ 4836051
+ ],
+ [
+ 1146380,
+ 4836041
+ ],
+ [
+ 1146373,
+ 4836031
+ ],
+ [
+ 1146365,
+ 4836021
+ ],
+ [
+ 1146358,
+ 4836011
+ ],
+ [
+ 1146350,
+ 4836001
+ ],
+ [
+ 1146343,
+ 4835991
+ ],
+ [
+ 1146335,
+ 4835981
+ ],
+ [
+ 1146328,
+ 4835971
+ ],
+ [
+ 1146320,
+ 4835961
+ ],
+ [
+ 1146313,
+ 4835951
+ ],
+ [
+ 1146305,
+ 4835941
+ ],
+ [
+ 1146298,
+ 4835930
+ ],
+ [
+ 1146290,
+ 4835920
+ ],
+ [
+ 1146283,
+ 4835910
+ ],
+ [
+ 1146275,
+ 4835900
+ ],
+ [
+ 1146268,
+ 4835890
+ ],
+ [
+ 1146260,
+ 4835880
+ ],
+ [
+ 1146252,
+ 4835870
+ ],
+ [
+ 1146244,
+ 4835860
+ ],
+ [
+ 1146237,
+ 4835850
+ ],
+ [
+ 1146229,
+ 4835841
+ ],
+ [
+ 1146222,
+ 4835830
+ ],
+ [
+ 1146214,
+ 4835820
+ ],
+ [
+ 1146207,
+ 4835810
+ ],
+ [
+ 1146199,
+ 4835800
+ ],
+ [
+ 1146192,
+ 4835790
+ ],
+ [
+ 1146184,
+ 4835780
+ ],
+ [
+ 1146177,
+ 4835770
+ ],
+ [
+ 1146169,
+ 4835761
+ ],
+ [
+ 1146162,
+ 4835751
+ ],
+ [
+ 1146154,
+ 4835741
+ ],
+ [
+ 1146147,
+ 4835731
+ ],
+ [
+ 1146139,
+ 4835721
+ ],
+ [
+ 1146131,
+ 4835711
+ ],
+ [
+ 1146123,
+ 4835701
+ ],
+ [
+ 1146116,
+ 4835691
+ ],
+ [
+ 1146108,
+ 4835681
+ ],
+ [
+ 1146101,
+ 4835671
+ ],
+ [
+ 1146093,
+ 4835661
+ ],
+ [
+ 1146086,
+ 4835651
+ ],
+ [
+ 1146078,
+ 4835641
+ ],
+ [
+ 1146071,
+ 4835631
+ ],
+ [
+ 1146063,
+ 4835621
+ ],
+ [
+ 1146056,
+ 4835611
+ ],
+ [
+ 1146048,
+ 4835601
+ ],
+ [
+ 1146041,
+ 4835591
+ ],
+ [
+ 1146033,
+ 4835580
+ ],
+ [
+ 1146026,
+ 4835570
+ ],
+ [
+ 1146018,
+ 4835560
+ ],
+ [
+ 1146011,
+ 4835550
+ ],
+ [
+ 1146003,
+ 4835541
+ ],
+ [
+ 1145996,
+ 4835530
+ ],
+ [
+ 1145989,
+ 4835520
+ ],
+ [
+ 1145982,
+ 4835510
+ ],
+ [
+ 1145974,
+ 4835500
+ ],
+ [
+ 1145967,
+ 4835489
+ ],
+ [
+ 1145959,
+ 4835479
+ ],
+ [
+ 1145952,
+ 4835469
+ ],
+ [
+ 1145944,
+ 4835459
+ ],
+ [
+ 1145937,
+ 4835448
+ ],
+ [
+ 1145929,
+ 4835438
+ ],
+ [
+ 1145922,
+ 4835428
+ ],
+ [
+ 1145914,
+ 4835418
+ ],
+ [
+ 1145907,
+ 4835408
+ ],
+ [
+ 1145899,
+ 4835399
+ ],
+ [
+ 1145892,
+ 4835389
+ ],
+ [
+ 1145884,
+ 4835379
+ ],
+ [
+ 1145877,
+ 4835369
+ ],
+ [
+ 1145869,
+ 4835359
+ ],
+ [
+ 1145862,
+ 4835349
+ ],
+ [
+ 1145854,
+ 4835339
+ ],
+ [
+ 1145847,
+ 4835328
+ ],
+ [
+ 1145839,
+ 4835318
+ ],
+ [
+ 1145832,
+ 4835308
+ ],
+ [
+ 1145824,
+ 4835298
+ ],
+ [
+ 1145817,
+ 4835288
+ ],
+ [
+ 1145809,
+ 4835278
+ ],
+ [
+ 1145802,
+ 4835268
+ ],
+ [
+ 1145794,
+ 4835258
+ ],
+ [
+ 1145787,
+ 4835248
+ ],
+ [
+ 1145779,
+ 4835239
+ ],
+ [
+ 1145772,
+ 4835228
+ ],
+ [
+ 1145764,
+ 4835218
+ ],
+ [
+ 1145757,
+ 4835207
+ ],
+ [
+ 1145749,
+ 4835197
+ ],
+ [
+ 1145742,
+ 4835187
+ ],
+ [
+ 1145734,
+ 4835177
+ ],
+ [
+ 1145727,
+ 4835167
+ ],
+ [
+ 1145719,
+ 4835157
+ ],
+ [
+ 1145712,
+ 4835147
+ ],
+ [
+ 1145704,
+ 4835138
+ ],
+ [
+ 1145696,
+ 4835128
+ ],
+ [
+ 1145688,
+ 4835118
+ ],
+ [
+ 1145681,
+ 4835108
+ ],
+ [
+ 1145673,
+ 4835099
+ ],
+ [
+ 1145666,
+ 4835089
+ ],
+ [
+ 1145658,
+ 4835078
+ ],
+ [
+ 1145651,
+ 4835068
+ ],
+ [
+ 1145644,
+ 4835058
+ ],
+ [
+ 1145636,
+ 4835048
+ ],
+ [
+ 1145628,
+ 4835038
+ ],
+ [
+ 1145621,
+ 4835028
+ ],
+ [
+ 1145613,
+ 4835018
+ ],
+ [
+ 1145606,
+ 4835007
+ ],
+ [
+ 1145599,
+ 4834997
+ ],
+ [
+ 1145592,
+ 4834987
+ ],
+ [
+ 1145584,
+ 4834977
+ ],
+ [
+ 1145577,
+ 4834967
+ ],
+ [
+ 1145569,
+ 4834957
+ ],
+ [
+ 1145562,
+ 4834947
+ ],
+ [
+ 1145554,
+ 4834937
+ ],
+ [
+ 1145547,
+ 4834927
+ ],
+ [
+ 1145539,
+ 4834917
+ ],
+ [
+ 1145532,
+ 4834907
+ ],
+ [
+ 1145524,
+ 4834897
+ ],
+ [
+ 1145516,
+ 4834887
+ ],
+ [
+ 1145508,
+ 4834878
+ ],
+ [
+ 1145501,
+ 4834868
+ ],
+ [
+ 1145493,
+ 4834858
+ ],
+ [
+ 1145486,
+ 4834848
+ ],
+ [
+ 1145478,
+ 4834839
+ ],
+ [
+ 1145471,
+ 4834827
+ ],
+ [
+ 1145463,
+ 4834817
+ ],
+ [
+ 1145455,
+ 4834807
+ ],
+ [
+ 1145447,
+ 4834797
+ ],
+ [
+ 1145440,
+ 4834787
+ ],
+ [
+ 1145432,
+ 4834777
+ ],
+ [
+ 1145425,
+ 4834767
+ ],
+ [
+ 1145417,
+ 4834757
+ ],
+ [
+ 1145410,
+ 4834747
+ ],
+ [
+ 1145402,
+ 4834737
+ ],
+ [
+ 1145394,
+ 4834727
+ ],
+ [
+ 1145386,
+ 4834717
+ ],
+ [
+ 1145378,
+ 4834707
+ ],
+ [
+ 1145370,
+ 4834698
+ ],
+ [
+ 1145362,
+ 4834688
+ ],
+ [
+ 1145354,
+ 4834679
+ ],
+ [
+ 1145347,
+ 4834669
+ ],
+ [
+ 1145339,
+ 4834660
+ ],
+ [
+ 1145331,
+ 4834650
+ ],
+ [
+ 1145323,
+ 4834640
+ ],
+ [
+ 1145316,
+ 4834630
+ ],
+ [
+ 1145308,
+ 4834621
+ ],
+ [
+ 1145300,
+ 4834611
+ ],
+ [
+ 1145292,
+ 4834602
+ ],
+ [
+ 1145285,
+ 4834592
+ ],
+ [
+ 1145277,
+ 4834581
+ ],
+ [
+ 1145269,
+ 4834571
+ ],
+ [
+ 1145261,
+ 4834561
+ ],
+ [
+ 1145254,
+ 4834551
+ ],
+ [
+ 1145246,
+ 4834542
+ ],
+ [
+ 1145239,
+ 4834532
+ ],
+ [
+ 1145231,
+ 4834522
+ ],
+ [
+ 1145224,
+ 4834512
+ ],
+ [
+ 1145216,
+ 4834503
+ ],
+ [
+ 1145208,
+ 4834493
+ ],
+ [
+ 1145200,
+ 4834483
+ ],
+ [
+ 1145193,
+ 4834473
+ ],
+ [
+ 1145185,
+ 4834463
+ ],
+ [
+ 1145178,
+ 4834453
+ ],
+ [
+ 1145170,
+ 4834443
+ ],
+ [
+ 1145163,
+ 4834433
+ ],
+ [
+ 1145155,
+ 4834423
+ ],
+ [
+ 1145148,
+ 4834413
+ ],
+ [
+ 1145140,
+ 4834403
+ ],
+ [
+ 1145133,
+ 4834393
+ ],
+ [
+ 1145125,
+ 4834383
+ ],
+ [
+ 1145118,
+ 4834373
+ ],
+ [
+ 1145110,
+ 4834363
+ ],
+ [
+ 1145103,
+ 4834353
+ ],
+ [
+ 1145095,
+ 4834342
+ ],
+ [
+ 1145088,
+ 4834332
+ ],
+ [
+ 1145080,
+ 4834322
+ ],
+ [
+ 1145073,
+ 4834312
+ ],
+ [
+ 1145065,
+ 4834302
+ ],
+ [
+ 1145057,
+ 4834292
+ ],
+ [
+ 1145049,
+ 4834282
+ ],
+ [
+ 1145042,
+ 4834272
+ ],
+ [
+ 1145034,
+ 4834262
+ ],
+ [
+ 1145027,
+ 4834252
+ ],
+ [
+ 1145019,
+ 4834243
+ ],
+ [
+ 1145012,
+ 4834233
+ ],
+ [
+ 1145004,
+ 4834223
+ ],
+ [
+ 1144997,
+ 4834213
+ ],
+ [
+ 1144989,
+ 4834203
+ ],
+ [
+ 1144981,
+ 4834193
+ ],
+ [
+ 1144973,
+ 4834183
+ ],
+ [
+ 1144966,
+ 4834173
+ ],
+ [
+ 1144958,
+ 4834163
+ ],
+ [
+ 1144951,
+ 4834153
+ ],
+ [
+ 1144943,
+ 4834143
+ ],
+ [
+ 1144936,
+ 4834133
+ ],
+ [
+ 1144928,
+ 4834123
+ ],
+ [
+ 1144921,
+ 4834113
+ ],
+ [
+ 1144913,
+ 4834103
+ ],
+ [
+ 1144905,
+ 4834092
+ ],
+ [
+ 1144897,
+ 4834082
+ ],
+ [
+ 1144890,
+ 4834072
+ ],
+ [
+ 1144882,
+ 4834063
+ ],
+ [
+ 1144875,
+ 4834053
+ ],
+ [
+ 1144867,
+ 4834043
+ ],
+ [
+ 1144860,
+ 4834033
+ ],
+ [
+ 1144852,
+ 4834024
+ ],
+ [
+ 1144845,
+ 4834014
+ ],
+ [
+ 1144837,
+ 4834004
+ ],
+ [
+ 1144829,
+ 4833994
+ ],
+ [
+ 1144821,
+ 4833985
+ ],
+ [
+ 1144813,
+ 4833975
+ ],
+ [
+ 1144805,
+ 4833965
+ ],
+ [
+ 1144798,
+ 4833954
+ ],
+ [
+ 1144790,
+ 4833944
+ ],
+ [
+ 1144783,
+ 4833934
+ ],
+ [
+ 1144775,
+ 4833924
+ ],
+ [
+ 1144768,
+ 4833914
+ ],
+ [
+ 1144760,
+ 4833904
+ ],
+ [
+ 1144753,
+ 4833894
+ ],
+ [
+ 1144745,
+ 4833885
+ ],
+ [
+ 1144738,
+ 4833875
+ ],
+ [
+ 1144730,
+ 4833865
+ ],
+ [
+ 1144723,
+ 4833855
+ ],
+ [
+ 1144715,
+ 4833844
+ ],
+ [
+ 1144707,
+ 4833834
+ ],
+ [
+ 1144699,
+ 4833825
+ ],
+ [
+ 1144692,
+ 4833814
+ ],
+ [
+ 1144685,
+ 4833804
+ ],
+ [
+ 1144678,
+ 4833794
+ ],
+ [
+ 1144670,
+ 4833784
+ ],
+ [
+ 1144663,
+ 4833774
+ ],
+ [
+ 1144655,
+ 4833764
+ ],
+ [
+ 1144648,
+ 4833754
+ ],
+ [
+ 1144640,
+ 4833745
+ ],
+ [
+ 1144633,
+ 4833735
+ ],
+ [
+ 1144625,
+ 4833725
+ ],
+ [
+ 1144618,
+ 4833715
+ ],
+ [
+ 1144611,
+ 4833705
+ ],
+ [
+ 1144604,
+ 4833694
+ ],
+ [
+ 1144596,
+ 4833684
+ ],
+ [
+ 1144589,
+ 4833674
+ ],
+ [
+ 1144581,
+ 4833664
+ ],
+ [
+ 1144574,
+ 4833654
+ ],
+ [
+ 1144566,
+ 4833644
+ ],
+ [
+ 1144559,
+ 4833634
+ ],
+ [
+ 1144551,
+ 4833624
+ ],
+ [
+ 1144544,
+ 4833614
+ ],
+ [
+ 1144537,
+ 4833604
+ ],
+ [
+ 1144530,
+ 4833593
+ ],
+ [
+ 1144522,
+ 4833583
+ ],
+ [
+ 1144514,
+ 4833572
+ ],
+ [
+ 1144506,
+ 4833562
+ ],
+ [
+ 1144499,
+ 4833552
+ ],
+ [
+ 1144492,
+ 4833543
+ ],
+ [
+ 1144484,
+ 4833533
+ ],
+ [
+ 1144476,
+ 4833523
+ ],
+ [
+ 1144469,
+ 4833513
+ ],
+ [
+ 1144461,
+ 4833503
+ ],
+ [
+ 1144454,
+ 4833493
+ ],
+ [
+ 1144446,
+ 4833483
+ ],
+ [
+ 1144439,
+ 4833473
+ ],
+ [
+ 1144431,
+ 4833464
+ ],
+ [
+ 1144424,
+ 4833454
+ ],
+ [
+ 1144417,
+ 4833444
+ ],
+ [
+ 1144409,
+ 4833433
+ ],
+ [
+ 1144401,
+ 4833423
+ ],
+ [
+ 1144394,
+ 4833413
+ ],
+ [
+ 1144386,
+ 4833403
+ ],
+ [
+ 1144379,
+ 4833393
+ ],
+ [
+ 1144372,
+ 4833383
+ ],
+ [
+ 1144365,
+ 4833373
+ ],
+ [
+ 1144357,
+ 4833363
+ ],
+ [
+ 1144350,
+ 4833352
+ ],
+ [
+ 1144342,
+ 4833341
+ ],
+ [
+ 1144335,
+ 4833331
+ ],
+ [
+ 1144327,
+ 4833321
+ ],
+ [
+ 1144320,
+ 4833311
+ ],
+ [
+ 1144312,
+ 4833301
+ ],
+ [
+ 1144305,
+ 4833291
+ ],
+ [
+ 1144297,
+ 4833281
+ ],
+ [
+ 1144290,
+ 4833271
+ ],
+ [
+ 1144282,
+ 4833261
+ ],
+ [
+ 1144275,
+ 4833251
+ ],
+ [
+ 1144267,
+ 4833241
+ ],
+ [
+ 1144259,
+ 4833231
+ ],
+ [
+ 1144251,
+ 4833221
+ ],
+ [
+ 1144244,
+ 4833211
+ ],
+ [
+ 1144236,
+ 4833201
+ ],
+ [
+ 1144229,
+ 4833191
+ ],
+ [
+ 1144221,
+ 4833181
+ ],
+ [
+ 1144214,
+ 4833171
+ ],
+ [
+ 1144206,
+ 4833162
+ ],
+ [
+ 1144199,
+ 4833152
+ ],
+ [
+ 1144191,
+ 4833142
+ ],
+ [
+ 1144183,
+ 4833132
+ ],
+ [
+ 1144175,
+ 4833122
+ ],
+ [
+ 1144168,
+ 4833112
+ ],
+ [
+ 1144160,
+ 4833103
+ ],
+ [
+ 1144152,
+ 4833092
+ ],
+ [
+ 1144144,
+ 4833082
+ ],
+ [
+ 1144136,
+ 4833072
+ ],
+ [
+ 1144128,
+ 4833063
+ ],
+ [
+ 1144121,
+ 4833053
+ ],
+ [
+ 1144113,
+ 4833043
+ ],
+ [
+ 1144105,
+ 4833033
+ ],
+ [
+ 1144097,
+ 4833024
+ ],
+ [
+ 1144089,
+ 4833014
+ ],
+ [
+ 1144081,
+ 4833004
+ ],
+ [
+ 1144073,
+ 4832994
+ ],
+ [
+ 1144065,
+ 4832985
+ ],
+ [
+ 1144057,
+ 4832975
+ ],
+ [
+ 1144049,
+ 4832966
+ ],
+ [
+ 1144041,
+ 4832957
+ ],
+ [
+ 1144032,
+ 4832948
+ ],
+ [
+ 1144024,
+ 4832939
+ ],
+ [
+ 1144015,
+ 4832930
+ ],
+ [
+ 1144007,
+ 4832921
+ ],
+ [
+ 1143998,
+ 4832912
+ ],
+ [
+ 1143990,
+ 4832903
+ ],
+ [
+ 1143981,
+ 4832894
+ ],
+ [
+ 1143973,
+ 4832884
+ ],
+ [
+ 1143964,
+ 4832875
+ ],
+ [
+ 1143956,
+ 4832865
+ ],
+ [
+ 1143948,
+ 4832856
+ ],
+ [
+ 1143940,
+ 4832846
+ ],
+ [
+ 1143932,
+ 4832837
+ ],
+ [
+ 1143924,
+ 4832828
+ ],
+ [
+ 1143916,
+ 4832819
+ ],
+ [
+ 1143908,
+ 4832809
+ ],
+ [
+ 1143900,
+ 4832799
+ ],
+ [
+ 1143892,
+ 4832789
+ ],
+ [
+ 1143884,
+ 4832780
+ ],
+ [
+ 1143877,
+ 4832770
+ ],
+ [
+ 1143869,
+ 4832761
+ ],
+ [
+ 1143862,
+ 4832751
+ ],
+ [
+ 1143854,
+ 4832742
+ ],
+ [
+ 1143846,
+ 4832732
+ ],
+ [
+ 1143838,
+ 4832722
+ ],
+ [
+ 1143831,
+ 4832712
+ ],
+ [
+ 1143823,
+ 4832702
+ ],
+ [
+ 1143816,
+ 4832692
+ ],
+ [
+ 1143809,
+ 4832682
+ ],
+ [
+ 1143802,
+ 4832672
+ ],
+ [
+ 1143794,
+ 4832662
+ ],
+ [
+ 1143787,
+ 4832651
+ ],
+ [
+ 1143779,
+ 4832641
+ ],
+ [
+ 1143772,
+ 4832631
+ ],
+ [
+ 1143765,
+ 4832620
+ ],
+ [
+ 1143758,
+ 4832610
+ ],
+ [
+ 1143750,
+ 4832600
+ ],
+ [
+ 1143744,
+ 4832590
+ ],
+ [
+ 1143737,
+ 4832580
+ ],
+ [
+ 1143730,
+ 4832569
+ ],
+ [
+ 1143722,
+ 4832559
+ ],
+ [
+ 1143715,
+ 4832548
+ ],
+ [
+ 1143708,
+ 4832538
+ ],
+ [
+ 1143701,
+ 4832528
+ ],
+ [
+ 1143693,
+ 4832518
+ ],
+ [
+ 1143686,
+ 4832508
+ ],
+ [
+ 1143678,
+ 4832498
+ ],
+ [
+ 1143671,
+ 4832488
+ ],
+ [
+ 1143663,
+ 4832478
+ ],
+ [
+ 1143656,
+ 4832468
+ ],
+ [
+ 1143648,
+ 4832458
+ ],
+ [
+ 1143641,
+ 4832448
+ ],
+ [
+ 1143634,
+ 4832438
+ ],
+ [
+ 1143627,
+ 4832428
+ ],
+ [
+ 1143619,
+ 4832418
+ ],
+ [
+ 1143612,
+ 4832408
+ ],
+ [
+ 1143604,
+ 4832398
+ ],
+ [
+ 1143597,
+ 4832387
+ ],
+ [
+ 1143590,
+ 4832377
+ ],
+ [
+ 1143583,
+ 4832366
+ ],
+ [
+ 1143575,
+ 4832356
+ ],
+ [
+ 1143568,
+ 4832345
+ ],
+ [
+ 1143560,
+ 4832335
+ ],
+ [
+ 1143552,
+ 4832325
+ ],
+ [
+ 1143544,
+ 4832315
+ ],
+ [
+ 1143537,
+ 4832305
+ ],
+ [
+ 1143529,
+ 4832295
+ ],
+ [
+ 1143522,
+ 4832285
+ ],
+ [
+ 1143514,
+ 4832276
+ ],
+ [
+ 1143507,
+ 4832265
+ ],
+ [
+ 1143500,
+ 4832255
+ ],
+ [
+ 1143493,
+ 4832245
+ ],
+ [
+ 1143485,
+ 4832236
+ ],
+ [
+ 1143478,
+ 4832225
+ ],
+ [
+ 1143470,
+ 4832215
+ ],
+ [
+ 1143463,
+ 4832205
+ ],
+ [
+ 1143455,
+ 4832195
+ ],
+ [
+ 1143448,
+ 4832185
+ ],
+ [
+ 1143440,
+ 4832175
+ ],
+ [
+ 1143433,
+ 4832165
+ ],
+ [
+ 1143425,
+ 4832155
+ ],
+ [
+ 1143418,
+ 4832145
+ ],
+ [
+ 1143410,
+ 4832135
+ ],
+ [
+ 1143402,
+ 4832125
+ ],
+ [
+ 1143394,
+ 4832115
+ ],
+ [
+ 1143387,
+ 4832105
+ ],
+ [
+ 1143379,
+ 4832095
+ ],
+ [
+ 1143372,
+ 4832085
+ ],
+ [
+ 1143364,
+ 4832075
+ ],
+ [
+ 1143357,
+ 4832065
+ ],
+ [
+ 1143350,
+ 4832055
+ ],
+ [
+ 1143343,
+ 4832045
+ ],
+ [
+ 1143335,
+ 4832035
+ ],
+ [
+ 1143328,
+ 4832024
+ ],
+ [
+ 1143320,
+ 4832014
+ ],
+ [
+ 1143313,
+ 4832004
+ ],
+ [
+ 1143305,
+ 4831994
+ ],
+ [
+ 1143298,
+ 4831983
+ ],
+ [
+ 1143291,
+ 4831973
+ ],
+ [
+ 1143284,
+ 4831963
+ ],
+ [
+ 1143277,
+ 4831953
+ ],
+ [
+ 1143270,
+ 4831943
+ ],
+ [
+ 1143262,
+ 4831933
+ ],
+ [
+ 1143255,
+ 4831923
+ ],
+ [
+ 1143248,
+ 4831913
+ ],
+ [
+ 1143241,
+ 4831903
+ ],
+ [
+ 1143233,
+ 4831893
+ ],
+ [
+ 1143225,
+ 4831882
+ ],
+ [
+ 1143217,
+ 4831872
+ ],
+ [
+ 1143210,
+ 4831861
+ ],
+ [
+ 1143202,
+ 4831851
+ ],
+ [
+ 1143195,
+ 4831841
+ ],
+ [
+ 1143187,
+ 4831831
+ ],
+ [
+ 1143180,
+ 4831821
+ ],
+ [
+ 1143172,
+ 4831811
+ ],
+ [
+ 1143165,
+ 4831801
+ ],
+ [
+ 1143157,
+ 4831791
+ ],
+ [
+ 1143150,
+ 4831781
+ ],
+ [
+ 1143142,
+ 4831771
+ ],
+ [
+ 1143135,
+ 4831761
+ ],
+ [
+ 1143128,
+ 4831751
+ ],
+ [
+ 1143121,
+ 4831741
+ ],
+ [
+ 1143113,
+ 4831731
+ ],
+ [
+ 1143106,
+ 4831721
+ ],
+ [
+ 1143098,
+ 4831711
+ ],
+ [
+ 1143091,
+ 4831700
+ ],
+ [
+ 1143083,
+ 4831690
+ ],
+ [
+ 1143076,
+ 4831680
+ ],
+ [
+ 1143068,
+ 4831671
+ ],
+ [
+ 1143061,
+ 4831661
+ ],
+ [
+ 1143053,
+ 4831651
+ ],
+ [
+ 1143046,
+ 4831641
+ ],
+ [
+ 1143038,
+ 4831631
+ ],
+ [
+ 1143031,
+ 4831620
+ ],
+ [
+ 1143023,
+ 4831610
+ ],
+ [
+ 1143016,
+ 4831599
+ ],
+ [
+ 1143008,
+ 4831589
+ ],
+ [
+ 1143001,
+ 4831579
+ ],
+ [
+ 1142993,
+ 4831569
+ ],
+ [
+ 1142986,
+ 4831558
+ ],
+ [
+ 1142978,
+ 4831548
+ ],
+ [
+ 1142971,
+ 4831538
+ ],
+ [
+ 1142963,
+ 4831528
+ ],
+ [
+ 1142955,
+ 4831518
+ ],
+ [
+ 1142948,
+ 4831508
+ ],
+ [
+ 1142940,
+ 4831498
+ ],
+ [
+ 1142932,
+ 4831489
+ ],
+ [
+ 1142924,
+ 4831479
+ ],
+ [
+ 1142916,
+ 4831470
+ ],
+ [
+ 1142908,
+ 4831460
+ ],
+ [
+ 1142900,
+ 4831451
+ ],
+ [
+ 1142893,
+ 4831441
+ ],
+ [
+ 1142885,
+ 4831432
+ ],
+ [
+ 1142877,
+ 4831422
+ ],
+ [
+ 1142869,
+ 4831413
+ ],
+ [
+ 1142861,
+ 4831403
+ ],
+ [
+ 1142853,
+ 4831393
+ ],
+ [
+ 1142845,
+ 4831383
+ ],
+ [
+ 1142837,
+ 4831373
+ ],
+ [
+ 1142830,
+ 4831363
+ ],
+ [
+ 1142822,
+ 4831354
+ ],
+ [
+ 1142815,
+ 4831344
+ ],
+ [
+ 1142807,
+ 4831334
+ ],
+ [
+ 1142800,
+ 4831324
+ ],
+ [
+ 1142792,
+ 4831314
+ ],
+ [
+ 1142785,
+ 4831304
+ ],
+ [
+ 1142777,
+ 4831294
+ ],
+ [
+ 1142770,
+ 4831284
+ ],
+ [
+ 1142762,
+ 4831275
+ ],
+ [
+ 1142755,
+ 4831265
+ ],
+ [
+ 1142748,
+ 4831255
+ ],
+ [
+ 1142741,
+ 4831244
+ ],
+ [
+ 1142733,
+ 4831234
+ ],
+ [
+ 1142726,
+ 4831224
+ ],
+ [
+ 1142718,
+ 4831214
+ ],
+ [
+ 1142711,
+ 4831204
+ ],
+ [
+ 1142703,
+ 4831194
+ ],
+ [
+ 1142696,
+ 4831184
+ ],
+ [
+ 1142688,
+ 4831174
+ ],
+ [
+ 1142681,
+ 4831163
+ ],
+ [
+ 1142673,
+ 4831153
+ ],
+ [
+ 1142666,
+ 4831143
+ ],
+ [
+ 1142658,
+ 4831133
+ ],
+ [
+ 1142651,
+ 4831122
+ ],
+ [
+ 1142643,
+ 4831112
+ ],
+ [
+ 1142636,
+ 4831102
+ ],
+ [
+ 1142628,
+ 4831092
+ ],
+ [
+ 1142621,
+ 4831082
+ ],
+ [
+ 1142614,
+ 4831072
+ ],
+ [
+ 1142607,
+ 4831062
+ ],
+ [
+ 1142599,
+ 4831052
+ ],
+ [
+ 1142592,
+ 4831042
+ ],
+ [
+ 1142584,
+ 4831032
+ ],
+ [
+ 1142577,
+ 4831022
+ ],
+ [
+ 1142569,
+ 4831012
+ ],
+ [
+ 1142562,
+ 4831002
+ ],
+ [
+ 1142555,
+ 4830992
+ ],
+ [
+ 1142548,
+ 4830982
+ ],
+ [
+ 1142541,
+ 4830972
+ ],
+ [
+ 1142534,
+ 4830961
+ ],
+ [
+ 1142526,
+ 4830951
+ ],
+ [
+ 1142519,
+ 4830940
+ ],
+ [
+ 1142512,
+ 4830930
+ ],
+ [
+ 1142505,
+ 4830920
+ ],
+ [
+ 1142497,
+ 4830910
+ ],
+ [
+ 1142490,
+ 4830899
+ ],
+ [
+ 1142482,
+ 4830889
+ ],
+ [
+ 1142475,
+ 4830877
+ ],
+ [
+ 1142467,
+ 4830867
+ ],
+ [
+ 1142460,
+ 4830857
+ ],
+ [
+ 1142452,
+ 4830847
+ ],
+ [
+ 1142445,
+ 4830837
+ ],
+ [
+ 1142437,
+ 4830827
+ ],
+ [
+ 1142429,
+ 4830817
+ ],
+ [
+ 1142421,
+ 4830807
+ ],
+ [
+ 1142414,
+ 4830797
+ ],
+ [
+ 1142406,
+ 4830787
+ ],
+ [
+ 1142399,
+ 4830777
+ ],
+ [
+ 1142392,
+ 4830768
+ ],
+ [
+ 1142385,
+ 4830757
+ ],
+ [
+ 1142377,
+ 4830747
+ ],
+ [
+ 1142370,
+ 4830737
+ ],
+ [
+ 1142362,
+ 4830727
+ ],
+ [
+ 1142355,
+ 4830717
+ ],
+ [
+ 1142347,
+ 4830708
+ ],
+ [
+ 1142340,
+ 4830698
+ ],
+ [
+ 1142332,
+ 4830688
+ ],
+ [
+ 1142325,
+ 4830677
+ ],
+ [
+ 1142317,
+ 4830667
+ ],
+ [
+ 1142310,
+ 4830657
+ ],
+ [
+ 1142303,
+ 4830647
+ ],
+ [
+ 1142295,
+ 4830637
+ ],
+ [
+ 1142287,
+ 4830626
+ ],
+ [
+ 1142280,
+ 4830616
+ ],
+ [
+ 1142272,
+ 4830606
+ ],
+ [
+ 1142265,
+ 4830596
+ ],
+ [
+ 1142258,
+ 4830586
+ ],
+ [
+ 1142251,
+ 4830576
+ ],
+ [
+ 1142243,
+ 4830567
+ ],
+ [
+ 1142235,
+ 4830557
+ ],
+ [
+ 1142227,
+ 4830547
+ ],
+ [
+ 1142220,
+ 4830536
+ ],
+ [
+ 1142212,
+ 4830526
+ ],
+ [
+ 1142205,
+ 4830516
+ ],
+ [
+ 1142197,
+ 4830506
+ ],
+ [
+ 1142190,
+ 4830495
+ ],
+ [
+ 1142182,
+ 4830485
+ ],
+ [
+ 1142175,
+ 4830475
+ ],
+ [
+ 1142168,
+ 4830465
+ ],
+ [
+ 1142160,
+ 4830455
+ ],
+ [
+ 1142152,
+ 4830445
+ ],
+ [
+ 1142145,
+ 4830435
+ ],
+ [
+ 1142137,
+ 4830425
+ ],
+ [
+ 1142130,
+ 4830415
+ ],
+ [
+ 1142122,
+ 4830405
+ ],
+ [
+ 1142116,
+ 4830395
+ ],
+ [
+ 1142108,
+ 4830386
+ ],
+ [
+ 1142101,
+ 4830375
+ ],
+ [
+ 1142093,
+ 4830365
+ ],
+ [
+ 1142086,
+ 4830355
+ ],
+ [
+ 1142078,
+ 4830346
+ ],
+ [
+ 1142071,
+ 4830336
+ ],
+ [
+ 1142063,
+ 4830326
+ ],
+ [
+ 1142055,
+ 4830316
+ ],
+ [
+ 1142047,
+ 4830306
+ ],
+ [
+ 1142040,
+ 4830296
+ ],
+ [
+ 1142032,
+ 4830286
+ ],
+ [
+ 1142025,
+ 4830276
+ ],
+ [
+ 1142018,
+ 4830267
+ ],
+ [
+ 1142011,
+ 4830256
+ ],
+ [
+ 1142003,
+ 4830246
+ ],
+ [
+ 1141995,
+ 4830236
+ ],
+ [
+ 1141987,
+ 4830226
+ ],
+ [
+ 1141980,
+ 4830215
+ ],
+ [
+ 1141972,
+ 4830205
+ ],
+ [
+ 1141964,
+ 4830195
+ ],
+ [
+ 1141956,
+ 4830185
+ ],
+ [
+ 1141949,
+ 4830175
+ ],
+ [
+ 1141941,
+ 4830165
+ ],
+ [
+ 1141934,
+ 4830155
+ ],
+ [
+ 1141926,
+ 4830146
+ ],
+ [
+ 1141919,
+ 4830135
+ ],
+ [
+ 1141911,
+ 4830126
+ ],
+ [
+ 1141903,
+ 4830116
+ ],
+ [
+ 1141895,
+ 4830106
+ ],
+ [
+ 1141888,
+ 4830096
+ ],
+ [
+ 1141880,
+ 4830086
+ ],
+ [
+ 1141873,
+ 4830076
+ ],
+ [
+ 1141865,
+ 4830066
+ ],
+ [
+ 1141858,
+ 4830056
+ ],
+ [
+ 1141850,
+ 4830047
+ ],
+ [
+ 1141843,
+ 4830037
+ ],
+ [
+ 1141835,
+ 4830027
+ ],
+ [
+ 1141828,
+ 4830017
+ ],
+ [
+ 1141820,
+ 4830007
+ ],
+ [
+ 1141813,
+ 4829997
+ ],
+ [
+ 1141805,
+ 4829987
+ ],
+ [
+ 1141798,
+ 4829977
+ ],
+ [
+ 1141790,
+ 4829967
+ ],
+ [
+ 1141783,
+ 4829957
+ ],
+ [
+ 1141775,
+ 4829947
+ ],
+ [
+ 1141767,
+ 4829937
+ ],
+ [
+ 1141759,
+ 4829927
+ ],
+ [
+ 1141752,
+ 4829916
+ ],
+ [
+ 1141745,
+ 4829906
+ ],
+ [
+ 1141738,
+ 4829896
+ ],
+ [
+ 1141730,
+ 4829886
+ ],
+ [
+ 1141723,
+ 4829876
+ ],
+ [
+ 1141715,
+ 4829866
+ ],
+ [
+ 1141708,
+ 4829856
+ ],
+ [
+ 1141700,
+ 4829846
+ ],
+ [
+ 1141693,
+ 4829836
+ ],
+ [
+ 1141685,
+ 4829826
+ ],
+ [
+ 1141678,
+ 4829816
+ ],
+ [
+ 1141670,
+ 4829806
+ ],
+ [
+ 1141663,
+ 4829796
+ ],
+ [
+ 1141655,
+ 4829786
+ ],
+ [
+ 1141648,
+ 4829776
+ ],
+ [
+ 1141640,
+ 4829766
+ ],
+ [
+ 1141633,
+ 4829756
+ ],
+ [
+ 1141625,
+ 4829746
+ ],
+ [
+ 1141618,
+ 4829735
+ ],
+ [
+ 1141610,
+ 4829725
+ ],
+ [
+ 1141603,
+ 4829715
+ ],
+ [
+ 1141596,
+ 4829705
+ ],
+ [
+ 1141589,
+ 4829695
+ ],
+ [
+ 1141581,
+ 4829685
+ ],
+ [
+ 1141574,
+ 4829675
+ ],
+ [
+ 1141566,
+ 4829665
+ ],
+ [
+ 1141559,
+ 4829655
+ ],
+ [
+ 1141551,
+ 4829644
+ ],
+ [
+ 1141544,
+ 4829634
+ ],
+ [
+ 1141536,
+ 4829624
+ ],
+ [
+ 1141529,
+ 4829614
+ ],
+ [
+ 1141521,
+ 4829604
+ ],
+ [
+ 1141514,
+ 4829594
+ ],
+ [
+ 1141506,
+ 4829584
+ ],
+ [
+ 1141499,
+ 4829574
+ ],
+ [
+ 1141491,
+ 4829564
+ ],
+ [
+ 1141484,
+ 4829554
+ ],
+ [
+ 1141476,
+ 4829544
+ ],
+ [
+ 1141469,
+ 4829534
+ ],
+ [
+ 1141461,
+ 4829524
+ ],
+ [
+ 1141454,
+ 4829514
+ ],
+ [
+ 1141446,
+ 4829504
+ ],
+ [
+ 1141438,
+ 4829494
+ ],
+ [
+ 1141430,
+ 4829484
+ ],
+ [
+ 1141423,
+ 4829474
+ ],
+ [
+ 1141415,
+ 4829464
+ ],
+ [
+ 1141407,
+ 4829454
+ ],
+ [
+ 1141399,
+ 4829444
+ ],
+ [
+ 1141392,
+ 4829434
+ ],
+ [
+ 1141384,
+ 4829425
+ ],
+ [
+ 1141376,
+ 4829415
+ ],
+ [
+ 1141368,
+ 4829405
+ ],
+ [
+ 1141360,
+ 4829394
+ ],
+ [
+ 1141352,
+ 4829385
+ ],
+ [
+ 1141345,
+ 4829375
+ ],
+ [
+ 1141337,
+ 4829365
+ ],
+ [
+ 1141329,
+ 4829355
+ ],
+ [
+ 1141321,
+ 4829346
+ ],
+ [
+ 1141314,
+ 4829336
+ ],
+ [
+ 1141306,
+ 4829326
+ ],
+ [
+ 1141298,
+ 4829316
+ ],
+ [
+ 1141290,
+ 4829307
+ ],
+ [
+ 1141283,
+ 4829297
+ ],
+ [
+ 1141276,
+ 4829287
+ ],
+ [
+ 1141268,
+ 4829277
+ ],
+ [
+ 1141260,
+ 4829268
+ ],
+ [
+ 1141253,
+ 4829258
+ ],
+ [
+ 1141245,
+ 4829248
+ ],
+ [
+ 1141237,
+ 4829238
+ ],
+ [
+ 1141229,
+ 4829229
+ ],
+ [
+ 1141222,
+ 4829219
+ ],
+ [
+ 1141214,
+ 4829209
+ ],
+ [
+ 1141207,
+ 4829199
+ ],
+ [
+ 1141199,
+ 4829190
+ ],
+ [
+ 1141191,
+ 4829180
+ ],
+ [
+ 1141183,
+ 4829170
+ ],
+ [
+ 1141176,
+ 4829159
+ ],
+ [
+ 1141168,
+ 4829149
+ ],
+ [
+ 1141161,
+ 4829139
+ ],
+ [
+ 1141153,
+ 4829130
+ ],
+ [
+ 1141146,
+ 4829120
+ ],
+ [
+ 1141138,
+ 4829110
+ ],
+ [
+ 1141131,
+ 4829100
+ ],
+ [
+ 1141123,
+ 4829090
+ ],
+ [
+ 1141115,
+ 4829080
+ ],
+ [
+ 1141107,
+ 4829070
+ ],
+ [
+ 1141100,
+ 4829060
+ ],
+ [
+ 1141092,
+ 4829050
+ ],
+ [
+ 1141085,
+ 4829040
+ ],
+ [
+ 1141077,
+ 4829030
+ ],
+ [
+ 1141070,
+ 4829020
+ ],
+ [
+ 1141062,
+ 4829010
+ ],
+ [
+ 1141055,
+ 4829000
+ ],
+ [
+ 1141047,
+ 4828991
+ ],
+ [
+ 1141040,
+ 4828981
+ ],
+ [
+ 1141032,
+ 4828971
+ ],
+ [
+ 1141025,
+ 4828961
+ ],
+ [
+ 1141017,
+ 4828951
+ ],
+ [
+ 1141009,
+ 4828941
+ ],
+ [
+ 1141001,
+ 4828931
+ ],
+ [
+ 1140994,
+ 4828921
+ ],
+ [
+ 1140986,
+ 4828911
+ ],
+ [
+ 1140979,
+ 4828901
+ ],
+ [
+ 1140972,
+ 4828891
+ ],
+ [
+ 1140965,
+ 4828880
+ ],
+ [
+ 1140957,
+ 4828870
+ ],
+ [
+ 1140950,
+ 4828860
+ ],
+ [
+ 1140942,
+ 4828851
+ ],
+ [
+ 1140934,
+ 4828840
+ ],
+ [
+ 1140926,
+ 4828830
+ ],
+ [
+ 1140919,
+ 4828820
+ ],
+ [
+ 1140911,
+ 4828810
+ ],
+ [
+ 1140903,
+ 4828800
+ ],
+ [
+ 1140895,
+ 4828791
+ ],
+ [
+ 1140888,
+ 4828781
+ ],
+ [
+ 1140880,
+ 4828771
+ ],
+ [
+ 1140873,
+ 4828761
+ ],
+ [
+ 1140865,
+ 4828752
+ ],
+ [
+ 1140857,
+ 4828742
+ ],
+ [
+ 1140849,
+ 4828732
+ ],
+ [
+ 1140842,
+ 4828722
+ ],
+ [
+ 1140834,
+ 4828712
+ ],
+ [
+ 1140826,
+ 4828702
+ ],
+ [
+ 1140818,
+ 4828693
+ ],
+ [
+ 1140811,
+ 4828683
+ ],
+ [
+ 1140803,
+ 4828673
+ ],
+ [
+ 1140795,
+ 4828663
+ ],
+ [
+ 1140787,
+ 4828653
+ ],
+ [
+ 1140779,
+ 4828643
+ ],
+ [
+ 1140771,
+ 4828633
+ ],
+ [
+ 1140763,
+ 4828623
+ ],
+ [
+ 1140755,
+ 4828613
+ ],
+ [
+ 1140748,
+ 4828603
+ ],
+ [
+ 1140740,
+ 4828594
+ ],
+ [
+ 1140733,
+ 4828584
+ ],
+ [
+ 1140725,
+ 4828575
+ ],
+ [
+ 1140717,
+ 4828565
+ ],
+ [
+ 1140709,
+ 4828555
+ ],
+ [
+ 1140702,
+ 4828545
+ ],
+ [
+ 1140694,
+ 4828535
+ ],
+ [
+ 1140687,
+ 4828525
+ ],
+ [
+ 1140679,
+ 4828515
+ ],
+ [
+ 1140672,
+ 4828505
+ ],
+ [
+ 1140664,
+ 4828496
+ ],
+ [
+ 1140657,
+ 4828486
+ ]
+ ],
+ [
+ [
+ 1120900,
+ 4834625
+ ],
+ [
+ 1120910,
+ 4834617
+ ],
+ [
+ 1120919,
+ 4834610
+ ],
+ [
+ 1120929,
+ 4834602
+ ],
+ [
+ 1120938,
+ 4834595
+ ],
+ [
+ 1120948,
+ 4834587
+ ],
+ [
+ 1120957,
+ 4834580
+ ],
+ [
+ 1120967,
+ 4834572
+ ],
+ [
+ 1120976,
+ 4834565
+ ],
+ [
+ 1120986,
+ 4834557
+ ],
+ [
+ 1120995,
+ 4834550
+ ],
+ [
+ 1121005,
+ 4834542
+ ],
+ [
+ 1121014,
+ 4834535
+ ],
+ [
+ 1121024,
+ 4834527
+ ],
+ [
+ 1121033,
+ 4834520
+ ],
+ [
+ 1121043,
+ 4834512
+ ],
+ [
+ 1121052,
+ 4834505
+ ],
+ [
+ 1121062,
+ 4834497
+ ],
+ [
+ 1121072,
+ 4834490
+ ],
+ [
+ 1121082,
+ 4834482
+ ],
+ [
+ 1121091,
+ 4834475
+ ],
+ [
+ 1121101,
+ 4834467
+ ],
+ [
+ 1121110,
+ 4834460
+ ],
+ [
+ 1121120,
+ 4834452
+ ],
+ [
+ 1121129,
+ 4834445
+ ],
+ [
+ 1121139,
+ 4834437
+ ],
+ [
+ 1121148,
+ 4834430
+ ],
+ [
+ 1121158,
+ 4834422
+ ],
+ [
+ 1121167,
+ 4834415
+ ],
+ [
+ 1121177,
+ 4834407
+ ],
+ [
+ 1121186,
+ 4834400
+ ],
+ [
+ 1121196,
+ 4834392
+ ],
+ [
+ 1121205,
+ 4834385
+ ],
+ [
+ 1121215,
+ 4834377
+ ],
+ [
+ 1121224,
+ 4834370
+ ],
+ [
+ 1121234,
+ 4834362
+ ],
+ [
+ 1121243,
+ 4834355
+ ],
+ [
+ 1121253,
+ 4834347
+ ],
+ [
+ 1121262,
+ 4834340
+ ],
+ [
+ 1121272,
+ 4834332
+ ],
+ [
+ 1121281,
+ 4834325
+ ],
+ [
+ 1121291,
+ 4834317
+ ],
+ [
+ 1121301,
+ 4834310
+ ],
+ [
+ 1121311,
+ 4834302
+ ],
+ [
+ 1121320,
+ 4834295
+ ],
+ [
+ 1121330,
+ 4834287
+ ],
+ [
+ 1121339,
+ 4834280
+ ],
+ [
+ 1121349,
+ 4834272
+ ],
+ [
+ 1121358,
+ 4834265
+ ],
+ [
+ 1121368,
+ 4834257
+ ],
+ [
+ 1121377,
+ 4834250
+ ],
+ [
+ 1121387,
+ 4834242
+ ],
+ [
+ 1121396,
+ 4834235
+ ],
+ [
+ 1121406,
+ 4834227
+ ],
+ [
+ 1121415,
+ 4834220
+ ],
+ [
+ 1121425,
+ 4834212
+ ],
+ [
+ 1121434,
+ 4834205
+ ],
+ [
+ 1121444,
+ 4834197
+ ],
+ [
+ 1121453,
+ 4834190
+ ],
+ [
+ 1121463,
+ 4834182
+ ],
+ [
+ 1121472,
+ 4834175
+ ],
+ [
+ 1121482,
+ 4834167
+ ],
+ [
+ 1121491,
+ 4834160
+ ],
+ [
+ 1121501,
+ 4834152
+ ],
+ [
+ 1121510,
+ 4834145
+ ],
+ [
+ 1121520,
+ 4834137
+ ],
+ [
+ 1121529,
+ 4834130
+ ],
+ [
+ 1121540,
+ 4834122
+ ],
+ [
+ 1121549,
+ 4834115
+ ],
+ [
+ 1121559,
+ 4834107
+ ],
+ [
+ 1121568,
+ 4834100
+ ],
+ [
+ 1121578,
+ 4834092
+ ],
+ [
+ 1121587,
+ 4834085
+ ],
+ [
+ 1121597,
+ 4834077
+ ],
+ [
+ 1121606,
+ 4834070
+ ],
+ [
+ 1121616,
+ 4834062
+ ],
+ [
+ 1121625,
+ 4834055
+ ],
+ [
+ 1121635,
+ 4834047
+ ],
+ [
+ 1121644,
+ 4834040
+ ],
+ [
+ 1121654,
+ 4834032
+ ],
+ [
+ 1121663,
+ 4834025
+ ],
+ [
+ 1121673,
+ 4834017
+ ],
+ [
+ 1121682,
+ 4834010
+ ],
+ [
+ 1121692,
+ 4834002
+ ],
+ [
+ 1121701,
+ 4833995
+ ],
+ [
+ 1121711,
+ 4833987
+ ],
+ [
+ 1121720,
+ 4833980
+ ],
+ [
+ 1121730,
+ 4833972
+ ],
+ [
+ 1121739,
+ 4833965
+ ],
+ [
+ 1121749,
+ 4833957
+ ],
+ [
+ 1121758,
+ 4833950
+ ],
+ [
+ 1121769,
+ 4833942
+ ],
+ [
+ 1121778,
+ 4833935
+ ],
+ [
+ 1121788,
+ 4833927
+ ],
+ [
+ 1121797,
+ 4833920
+ ],
+ [
+ 1121807,
+ 4833912
+ ],
+ [
+ 1121816,
+ 4833905
+ ],
+ [
+ 1121826,
+ 4833897
+ ],
+ [
+ 1121835,
+ 4833890
+ ],
+ [
+ 1121845,
+ 4833882
+ ],
+ [
+ 1121854,
+ 4833875
+ ],
+ [
+ 1121864,
+ 4833867
+ ],
+ [
+ 1121873,
+ 4833860
+ ],
+ [
+ 1121883,
+ 4833852
+ ],
+ [
+ 1121892,
+ 4833845
+ ],
+ [
+ 1121902,
+ 4833837
+ ],
+ [
+ 1121911,
+ 4833830
+ ],
+ [
+ 1121921,
+ 4833822
+ ],
+ [
+ 1121930,
+ 4833815
+ ],
+ [
+ 1121940,
+ 4833807
+ ],
+ [
+ 1121949,
+ 4833800
+ ],
+ [
+ 1121959,
+ 4833792
+ ],
+ [
+ 1121968,
+ 4833785
+ ],
+ [
+ 1121978,
+ 4833778
+ ],
+ [
+ 1121987,
+ 4833771
+ ],
+ [
+ 1121998,
+ 4833763
+ ],
+ [
+ 1122007,
+ 4833756
+ ],
+ [
+ 1122017,
+ 4833748
+ ],
+ [
+ 1122026,
+ 4833741
+ ],
+ [
+ 1122036,
+ 4833733
+ ],
+ [
+ 1122045,
+ 4833726
+ ],
+ [
+ 1122055,
+ 4833718
+ ],
+ [
+ 1122064,
+ 4833711
+ ],
+ [
+ 1122074,
+ 4833703
+ ],
+ [
+ 1122083,
+ 4833696
+ ],
+ [
+ 1122093,
+ 4833688
+ ],
+ [
+ 1122102,
+ 4833681
+ ],
+ [
+ 1122112,
+ 4833673
+ ],
+ [
+ 1122121,
+ 4833666
+ ],
+ [
+ 1122131,
+ 4833658
+ ],
+ [
+ 1122140,
+ 4833651
+ ],
+ [
+ 1122150,
+ 4833643
+ ],
+ [
+ 1122159,
+ 4833636
+ ],
+ [
+ 1122169,
+ 4833628
+ ],
+ [
+ 1122178,
+ 4833621
+ ],
+ [
+ 1122188,
+ 4833613
+ ],
+ [
+ 1122197,
+ 4833606
+ ],
+ [
+ 1122207,
+ 4833598
+ ],
+ [
+ 1122216,
+ 4833591
+ ],
+ [
+ 1122227,
+ 4833583
+ ],
+ [
+ 1122236,
+ 4833576
+ ],
+ [
+ 1122246,
+ 4833568
+ ],
+ [
+ 1122255,
+ 4833561
+ ],
+ [
+ 1122265,
+ 4833553
+ ],
+ [
+ 1122274,
+ 4833546
+ ],
+ [
+ 1122284,
+ 4833538
+ ],
+ [
+ 1122293,
+ 4833531
+ ],
+ [
+ 1122303,
+ 4833523
+ ],
+ [
+ 1122312,
+ 4833516
+ ],
+ [
+ 1122322,
+ 4833508
+ ],
+ [
+ 1122331,
+ 4833501
+ ],
+ [
+ 1122341,
+ 4833493
+ ],
+ [
+ 1122350,
+ 4833486
+ ],
+ [
+ 1122360,
+ 4833478
+ ],
+ [
+ 1122369,
+ 4833471
+ ],
+ [
+ 1122379,
+ 4833463
+ ],
+ [
+ 1122388,
+ 4833456
+ ],
+ [
+ 1122398,
+ 4833448
+ ],
+ [
+ 1122407,
+ 4833441
+ ],
+ [
+ 1122417,
+ 4833433
+ ],
+ [
+ 1122426,
+ 4833426
+ ],
+ [
+ 1122436,
+ 4833418
+ ],
+ [
+ 1122445,
+ 4833411
+ ],
+ [
+ 1122456,
+ 4833403
+ ],
+ [
+ 1122465,
+ 4833396
+ ],
+ [
+ 1122475,
+ 4833388
+ ],
+ [
+ 1122484,
+ 4833381
+ ],
+ [
+ 1122494,
+ 4833373
+ ],
+ [
+ 1122504,
+ 4833365
+ ],
+ [
+ 1122514,
+ 4833357
+ ],
+ [
+ 1122523,
+ 4833350
+ ],
+ [
+ 1122534,
+ 4833342
+ ],
+ [
+ 1122544,
+ 4833335
+ ],
+ [
+ 1122555,
+ 4833328
+ ],
+ [
+ 1122565,
+ 4833321
+ ],
+ [
+ 1122575,
+ 4833313
+ ],
+ [
+ 1122584,
+ 4833305
+ ],
+ [
+ 1122594,
+ 4833297
+ ],
+ [
+ 1122604,
+ 4833290
+ ],
+ [
+ 1122614,
+ 4833282
+ ],
+ [
+ 1122623,
+ 4833275
+ ],
+ [
+ 1122634,
+ 4833267
+ ],
+ [
+ 1122644,
+ 4833260
+ ],
+ [
+ 1122654,
+ 4833252
+ ],
+ [
+ 1122664,
+ 4833245
+ ],
+ [
+ 1122674,
+ 4833237
+ ],
+ [
+ 1122684,
+ 4833230
+ ],
+ [
+ 1122695,
+ 4833223
+ ],
+ [
+ 1122705,
+ 4833217
+ ],
+ [
+ 1122716,
+ 4833209
+ ],
+ [
+ 1122726,
+ 4833202
+ ],
+ [
+ 1122737,
+ 4833195
+ ],
+ [
+ 1122747,
+ 4833188
+ ],
+ [
+ 1122757,
+ 4833180
+ ],
+ [
+ 1122767,
+ 4833173
+ ],
+ [
+ 1122778,
+ 4833165
+ ],
+ [
+ 1122788,
+ 4833158
+ ],
+ [
+ 1122798,
+ 4833150
+ ],
+ [
+ 1122808,
+ 4833143
+ ],
+ [
+ 1122819,
+ 4833135
+ ],
+ [
+ 1122829,
+ 4833127
+ ],
+ [
+ 1122839,
+ 4833120
+ ],
+ [
+ 1122849,
+ 4833113
+ ],
+ [
+ 1122858,
+ 4833105
+ ],
+ [
+ 1122867,
+ 4833097
+ ],
+ [
+ 1122877,
+ 4833089
+ ],
+ [
+ 1122887,
+ 4833082
+ ],
+ [
+ 1122897,
+ 4833074
+ ],
+ [
+ 1122907,
+ 4833067
+ ],
+ [
+ 1122917,
+ 4833059
+ ],
+ [
+ 1122928,
+ 4833051
+ ],
+ [
+ 1122938,
+ 4833043
+ ],
+ [
+ 1122947,
+ 4833036
+ ],
+ [
+ 1122957,
+ 4833028
+ ],
+ [
+ 1122967,
+ 4833020
+ ],
+ [
+ 1122977,
+ 4833012
+ ],
+ [
+ 1122987,
+ 4833005
+ ],
+ [
+ 1122997,
+ 4832997
+ ],
+ [
+ 1123007,
+ 4832990
+ ],
+ [
+ 1123017,
+ 4832982
+ ],
+ [
+ 1123027,
+ 4832975
+ ],
+ [
+ 1123037,
+ 4832967
+ ],
+ [
+ 1123047,
+ 4832960
+ ],
+ [
+ 1123057,
+ 4832952
+ ],
+ [
+ 1123067,
+ 4832944
+ ],
+ [
+ 1123077,
+ 4832936
+ ],
+ [
+ 1123087,
+ 4832929
+ ],
+ [
+ 1123097,
+ 4832921
+ ],
+ [
+ 1123107,
+ 4832914
+ ],
+ [
+ 1123117,
+ 4832906
+ ],
+ [
+ 1123126,
+ 4832898
+ ],
+ [
+ 1123136,
+ 4832890
+ ],
+ [
+ 1123146,
+ 4832883
+ ],
+ [
+ 1123157,
+ 4832875
+ ],
+ [
+ 1123168,
+ 4832868
+ ],
+ [
+ 1123179,
+ 4832861
+ ],
+ [
+ 1123189,
+ 4832854
+ ],
+ [
+ 1123199,
+ 4832846
+ ],
+ [
+ 1123208,
+ 4832839
+ ],
+ [
+ 1123218,
+ 4832830
+ ],
+ [
+ 1123227,
+ 4832822
+ ],
+ [
+ 1123237,
+ 4832814
+ ],
+ [
+ 1123246,
+ 4832806
+ ],
+ [
+ 1123256,
+ 4832798
+ ],
+ [
+ 1123265,
+ 4832791
+ ],
+ [
+ 1123275,
+ 4832783
+ ],
+ [
+ 1123285,
+ 4832775
+ ],
+ [
+ 1123295,
+ 4832767
+ ],
+ [
+ 1123305,
+ 4832760
+ ],
+ [
+ 1123315,
+ 4832752
+ ],
+ [
+ 1123324,
+ 4832744
+ ],
+ [
+ 1123334,
+ 4832736
+ ],
+ [
+ 1123344,
+ 4832728
+ ],
+ [
+ 1123354,
+ 4832720
+ ],
+ [
+ 1123363,
+ 4832713
+ ],
+ [
+ 1123374,
+ 4832705
+ ],
+ [
+ 1123384,
+ 4832698
+ ],
+ [
+ 1123395,
+ 4832690
+ ],
+ [
+ 1123405,
+ 4832683
+ ],
+ [
+ 1123415,
+ 4832675
+ ],
+ [
+ 1123424,
+ 4832667
+ ],
+ [
+ 1123434,
+ 4832659
+ ],
+ [
+ 1123443,
+ 4832652
+ ],
+ [
+ 1123453,
+ 4832644
+ ],
+ [
+ 1123462,
+ 4832637
+ ],
+ [
+ 1123472,
+ 4832629
+ ],
+ [
+ 1123481,
+ 4832621
+ ],
+ [
+ 1123491,
+ 4832613
+ ],
+ [
+ 1123501,
+ 4832606
+ ],
+ [
+ 1123511,
+ 4832598
+ ],
+ [
+ 1123521,
+ 4832590
+ ],
+ [
+ 1123531,
+ 4832582
+ ],
+ [
+ 1123541,
+ 4832575
+ ],
+ [
+ 1123551,
+ 4832567
+ ],
+ [
+ 1123561,
+ 4832559
+ ],
+ [
+ 1123571,
+ 4832551
+ ],
+ [
+ 1123581,
+ 4832544
+ ],
+ [
+ 1123592,
+ 4832536
+ ],
+ [
+ 1123602,
+ 4832529
+ ],
+ [
+ 1123612,
+ 4832521
+ ],
+ [
+ 1123621,
+ 4832513
+ ],
+ [
+ 1123632,
+ 4832505
+ ],
+ [
+ 1123642,
+ 4832498
+ ],
+ [
+ 1123653,
+ 4832491
+ ],
+ [
+ 1123663,
+ 4832484
+ ],
+ [
+ 1123673,
+ 4832476
+ ],
+ [
+ 1123683,
+ 4832468
+ ],
+ [
+ 1123693,
+ 4832460
+ ],
+ [
+ 1123703,
+ 4832453
+ ],
+ [
+ 1123713,
+ 4832445
+ ],
+ [
+ 1123722,
+ 4832437
+ ],
+ [
+ 1123732,
+ 4832429
+ ],
+ [
+ 1123741,
+ 4832421
+ ],
+ [
+ 1123750,
+ 4832413
+ ],
+ [
+ 1123759,
+ 4832405
+ ],
+ [
+ 1123769,
+ 4832397
+ ],
+ [
+ 1123779,
+ 4832389
+ ],
+ [
+ 1123789,
+ 4832381
+ ],
+ [
+ 1123798,
+ 4832373
+ ],
+ [
+ 1123808,
+ 4832365
+ ],
+ [
+ 1123817,
+ 4832357
+ ],
+ [
+ 1123827,
+ 4832349
+ ],
+ [
+ 1123836,
+ 4832342
+ ],
+ [
+ 1123846,
+ 4832334
+ ],
+ [
+ 1123857,
+ 4832326
+ ],
+ [
+ 1123867,
+ 4832318
+ ],
+ [
+ 1123877,
+ 4832311
+ ],
+ [
+ 1123887,
+ 4832304
+ ],
+ [
+ 1123897,
+ 4832297
+ ],
+ [
+ 1123907,
+ 4832289
+ ],
+ [
+ 1123917,
+ 4832281
+ ],
+ [
+ 1123927,
+ 4832273
+ ],
+ [
+ 1123937,
+ 4832266
+ ],
+ [
+ 1123947,
+ 4832258
+ ],
+ [
+ 1123957,
+ 4832250
+ ],
+ [
+ 1123967,
+ 4832242
+ ],
+ [
+ 1123977,
+ 4832235
+ ],
+ [
+ 1123987,
+ 4832227
+ ],
+ [
+ 1123996,
+ 4832220
+ ],
+ [
+ 1124006,
+ 4832212
+ ],
+ [
+ 1124016,
+ 4832204
+ ],
+ [
+ 1124026,
+ 4832196
+ ],
+ [
+ 1124035,
+ 4832189
+ ],
+ [
+ 1124045,
+ 4832181
+ ],
+ [
+ 1124055,
+ 4832173
+ ],
+ [
+ 1124064,
+ 4832165
+ ],
+ [
+ 1124073,
+ 4832157
+ ],
+ [
+ 1124083,
+ 4832149
+ ],
+ [
+ 1124094,
+ 4832142
+ ],
+ [
+ 1124104,
+ 4832134
+ ],
+ [
+ 1124114,
+ 4832127
+ ],
+ [
+ 1124125,
+ 4832119
+ ],
+ [
+ 1124135,
+ 4832111
+ ],
+ [
+ 1124145,
+ 4832103
+ ],
+ [
+ 1124155,
+ 4832095
+ ],
+ [
+ 1124165,
+ 4832087
+ ],
+ [
+ 1124174,
+ 4832080
+ ],
+ [
+ 1124184,
+ 4832072
+ ],
+ [
+ 1124193,
+ 4832064
+ ],
+ [
+ 1124203,
+ 4832056
+ ],
+ [
+ 1124213,
+ 4832049
+ ],
+ [
+ 1124223,
+ 4832041
+ ],
+ [
+ 1124233,
+ 4832033
+ ],
+ [
+ 1124243,
+ 4832025
+ ],
+ [
+ 1124252,
+ 4832018
+ ],
+ [
+ 1124262,
+ 4832010
+ ],
+ [
+ 1124271,
+ 4832002
+ ],
+ [
+ 1124282,
+ 4831994
+ ],
+ [
+ 1124292,
+ 4831987
+ ],
+ [
+ 1124302,
+ 4831979
+ ],
+ [
+ 1124311,
+ 4831972
+ ],
+ [
+ 1124322,
+ 4831964
+ ],
+ [
+ 1124331,
+ 4831956
+ ],
+ [
+ 1124341,
+ 4831948
+ ],
+ [
+ 1124351,
+ 4831941
+ ],
+ [
+ 1124361,
+ 4831933
+ ],
+ [
+ 1124371,
+ 4831926
+ ],
+ [
+ 1124381,
+ 4831918
+ ],
+ [
+ 1124391,
+ 4831911
+ ],
+ [
+ 1124401,
+ 4831903
+ ],
+ [
+ 1124411,
+ 4831896
+ ],
+ [
+ 1124421,
+ 4831888
+ ],
+ [
+ 1124431,
+ 4831881
+ ],
+ [
+ 1124441,
+ 4831873
+ ],
+ [
+ 1124451,
+ 4831866
+ ],
+ [
+ 1124461,
+ 4831858
+ ],
+ [
+ 1124471,
+ 4831851
+ ],
+ [
+ 1124482,
+ 4831843
+ ],
+ [
+ 1124492,
+ 4831835
+ ],
+ [
+ 1124502,
+ 4831827
+ ],
+ [
+ 1124512,
+ 4831820
+ ],
+ [
+ 1124522,
+ 4831812
+ ],
+ [
+ 1124532,
+ 4831805
+ ],
+ [
+ 1124542,
+ 4831797
+ ],
+ [
+ 1124552,
+ 4831790
+ ],
+ [
+ 1124563,
+ 4831782
+ ],
+ [
+ 1124572,
+ 4831774
+ ],
+ [
+ 1124582,
+ 4831766
+ ],
+ [
+ 1124592,
+ 4831758
+ ],
+ [
+ 1124602,
+ 4831750
+ ],
+ [
+ 1124612,
+ 4831743
+ ],
+ [
+ 1124622,
+ 4831735
+ ],
+ [
+ 1124631,
+ 4831727
+ ],
+ [
+ 1124641,
+ 4831719
+ ],
+ [
+ 1124651,
+ 4831711
+ ],
+ [
+ 1124661,
+ 4831704
+ ],
+ [
+ 1124670,
+ 4831697
+ ],
+ [
+ 1124680,
+ 4831688
+ ],
+ [
+ 1124690,
+ 4831680
+ ],
+ [
+ 1124700,
+ 4831672
+ ],
+ [
+ 1124709,
+ 4831665
+ ],
+ [
+ 1124719,
+ 4831657
+ ],
+ [
+ 1124728,
+ 4831650
+ ],
+ [
+ 1124738,
+ 4831642
+ ],
+ [
+ 1124747,
+ 4831634
+ ],
+ [
+ 1124757,
+ 4831626
+ ],
+ [
+ 1124767,
+ 4831619
+ ],
+ [
+ 1124777,
+ 4831611
+ ],
+ [
+ 1124788,
+ 4831603
+ ],
+ [
+ 1124798,
+ 4831595
+ ],
+ [
+ 1124808,
+ 4831589
+ ],
+ [
+ 1124818,
+ 4831581
+ ],
+ [
+ 1124828,
+ 4831574
+ ],
+ [
+ 1124838,
+ 4831566
+ ],
+ [
+ 1124848,
+ 4831558
+ ],
+ [
+ 1124858,
+ 4831550
+ ],
+ [
+ 1124868,
+ 4831543
+ ],
+ [
+ 1124878,
+ 4831535
+ ],
+ [
+ 1124887,
+ 4831527
+ ],
+ [
+ 1124897,
+ 4831519
+ ],
+ [
+ 1124907,
+ 4831512
+ ],
+ [
+ 1124918,
+ 4831504
+ ],
+ [
+ 1124928,
+ 4831497
+ ],
+ [
+ 1124938,
+ 4831489
+ ],
+ [
+ 1124948,
+ 4831481
+ ],
+ [
+ 1124958,
+ 4831473
+ ],
+ [
+ 1124967,
+ 4831466
+ ],
+ [
+ 1124977,
+ 4831458
+ ],
+ [
+ 1124986,
+ 4831450
+ ],
+ [
+ 1124996,
+ 4831442
+ ],
+ [
+ 1125005,
+ 4831435
+ ],
+ [
+ 1125015,
+ 4831427
+ ],
+ [
+ 1125026,
+ 4831420
+ ],
+ [
+ 1125036,
+ 4831412
+ ],
+ [
+ 1125046,
+ 4831405
+ ],
+ [
+ 1125056,
+ 4831398
+ ],
+ [
+ 1125066,
+ 4831391
+ ],
+ [
+ 1125076,
+ 4831383
+ ],
+ [
+ 1125086,
+ 4831376
+ ],
+ [
+ 1125096,
+ 4831368
+ ],
+ [
+ 1125106,
+ 4831361
+ ],
+ [
+ 1125117,
+ 4831353
+ ],
+ [
+ 1125127,
+ 4831346
+ ],
+ [
+ 1125138,
+ 4831338
+ ],
+ [
+ 1125148,
+ 4831331
+ ],
+ [
+ 1125159,
+ 4831323
+ ],
+ [
+ 1125169,
+ 4831316
+ ],
+ [
+ 1125179,
+ 4831308
+ ],
+ [
+ 1125189,
+ 4831301
+ ],
+ [
+ 1125199,
+ 4831293
+ ],
+ [
+ 1125208,
+ 4831285
+ ],
+ [
+ 1125218,
+ 4831277
+ ],
+ [
+ 1125227,
+ 4831270
+ ],
+ [
+ 1125237,
+ 4831262
+ ],
+ [
+ 1125247,
+ 4831254
+ ],
+ [
+ 1125258,
+ 4831246
+ ],
+ [
+ 1125268,
+ 4831238
+ ],
+ [
+ 1125278,
+ 4831230
+ ],
+ [
+ 1125287,
+ 4831223
+ ],
+ [
+ 1125297,
+ 4831215
+ ],
+ [
+ 1125307,
+ 4831207
+ ],
+ [
+ 1125317,
+ 4831200
+ ],
+ [
+ 1125327,
+ 4831193
+ ],
+ [
+ 1125337,
+ 4831185
+ ],
+ [
+ 1125347,
+ 4831178
+ ],
+ [
+ 1125357,
+ 4831170
+ ],
+ [
+ 1125366,
+ 4831162
+ ],
+ [
+ 1125376,
+ 4831154
+ ],
+ [
+ 1125386,
+ 4831146
+ ],
+ [
+ 1125396,
+ 4831138
+ ],
+ [
+ 1125406,
+ 4831131
+ ],
+ [
+ 1125417,
+ 4831123
+ ],
+ [
+ 1125427,
+ 4831116
+ ],
+ [
+ 1125437,
+ 4831108
+ ],
+ [
+ 1125446,
+ 4831101
+ ],
+ [
+ 1125456,
+ 4831093
+ ],
+ [
+ 1125465,
+ 4831085
+ ],
+ [
+ 1125475,
+ 4831077
+ ],
+ [
+ 1125484,
+ 4831070
+ ],
+ [
+ 1125496,
+ 4831062
+ ],
+ [
+ 1125506,
+ 4831055
+ ],
+ [
+ 1125516,
+ 4831047
+ ],
+ [
+ 1125526,
+ 4831040
+ ],
+ [
+ 1125536,
+ 4831032
+ ],
+ [
+ 1125546,
+ 4831024
+ ],
+ [
+ 1125556,
+ 4831016
+ ],
+ [
+ 1125566,
+ 4831009
+ ],
+ [
+ 1125576,
+ 4831001
+ ],
+ [
+ 1125586,
+ 4830994
+ ],
+ [
+ 1125596,
+ 4830986
+ ],
+ [
+ 1125606,
+ 4830979
+ ],
+ [
+ 1125616,
+ 4830971
+ ],
+ [
+ 1125626,
+ 4830964
+ ],
+ [
+ 1125636,
+ 4830956
+ ],
+ [
+ 1125646,
+ 4830948
+ ],
+ [
+ 1125656,
+ 4830940
+ ],
+ [
+ 1125665,
+ 4830932
+ ],
+ [
+ 1125675,
+ 4830924
+ ],
+ [
+ 1125685,
+ 4830917
+ ],
+ [
+ 1125695,
+ 4830909
+ ],
+ [
+ 1125705,
+ 4830902
+ ],
+ [
+ 1125715,
+ 4830894
+ ],
+ [
+ 1125726,
+ 4830886
+ ],
+ [
+ 1125736,
+ 4830878
+ ],
+ [
+ 1125746,
+ 4830871
+ ],
+ [
+ 1125756,
+ 4830863
+ ],
+ [
+ 1125766,
+ 4830856
+ ],
+ [
+ 1125776,
+ 4830848
+ ],
+ [
+ 1125786,
+ 4830840
+ ],
+ [
+ 1125796,
+ 4830832
+ ],
+ [
+ 1125806,
+ 4830825
+ ],
+ [
+ 1125815,
+ 4830817
+ ],
+ [
+ 1125824,
+ 4830809
+ ],
+ [
+ 1125834,
+ 4830801
+ ],
+ [
+ 1125843,
+ 4830793
+ ],
+ [
+ 1125853,
+ 4830785
+ ],
+ [
+ 1125863,
+ 4830778
+ ],
+ [
+ 1125873,
+ 4830770
+ ],
+ [
+ 1125883,
+ 4830762
+ ],
+ [
+ 1125893,
+ 4830754
+ ],
+ [
+ 1125902,
+ 4830747
+ ],
+ [
+ 1125912,
+ 4830739
+ ],
+ [
+ 1125921,
+ 4830731
+ ],
+ [
+ 1125932,
+ 4830723
+ ],
+ [
+ 1125942,
+ 4830716
+ ],
+ [
+ 1125952,
+ 4830708
+ ],
+ [
+ 1125963,
+ 4830701
+ ],
+ [
+ 1125973,
+ 4830693
+ ],
+ [
+ 1125983,
+ 4830686
+ ],
+ [
+ 1125994,
+ 4830678
+ ],
+ [
+ 1126004,
+ 4830670
+ ],
+ [
+ 1126014,
+ 4830662
+ ],
+ [
+ 1126024,
+ 4830655
+ ],
+ [
+ 1126033,
+ 4830647
+ ],
+ [
+ 1126042,
+ 4830639
+ ],
+ [
+ 1126052,
+ 4830631
+ ],
+ [
+ 1126061,
+ 4830623
+ ],
+ [
+ 1126071,
+ 4830615
+ ],
+ [
+ 1126080,
+ 4830607
+ ],
+ [
+ 1126090,
+ 4830599
+ ],
+ [
+ 1126100,
+ 4830591
+ ],
+ [
+ 1126110,
+ 4830583
+ ],
+ [
+ 1126120,
+ 4830575
+ ],
+ [
+ 1126130,
+ 4830567
+ ],
+ [
+ 1126140,
+ 4830559
+ ],
+ [
+ 1126150,
+ 4830551
+ ],
+ [
+ 1126159,
+ 4830544
+ ],
+ [
+ 1126169,
+ 4830536
+ ],
+ [
+ 1126178,
+ 4830528
+ ],
+ [
+ 1126189,
+ 4830520
+ ],
+ [
+ 1126198,
+ 4830512
+ ],
+ [
+ 1126208,
+ 4830504
+ ],
+ [
+ 1126217,
+ 4830496
+ ],
+ [
+ 1126227,
+ 4830487
+ ],
+ [
+ 1126236,
+ 4830479
+ ],
+ [
+ 1126245,
+ 4830471
+ ],
+ [
+ 1126254,
+ 4830463
+ ],
+ [
+ 1126264,
+ 4830455
+ ],
+ [
+ 1126273,
+ 4830447
+ ],
+ [
+ 1126283,
+ 4830439
+ ],
+ [
+ 1126292,
+ 4830431
+ ],
+ [
+ 1126302,
+ 4830422
+ ],
+ [
+ 1126311,
+ 4830414
+ ],
+ [
+ 1126321,
+ 4830406
+ ],
+ [
+ 1126330,
+ 4830399
+ ],
+ [
+ 1126340,
+ 4830390
+ ],
+ [
+ 1126349,
+ 4830382
+ ],
+ [
+ 1126359,
+ 4830374
+ ],
+ [
+ 1126368,
+ 4830367
+ ],
+ [
+ 1126378,
+ 4830359
+ ],
+ [
+ 1126388,
+ 4830351
+ ],
+ [
+ 1126398,
+ 4830342
+ ],
+ [
+ 1126408,
+ 4830334
+ ],
+ [
+ 1126418,
+ 4830326
+ ],
+ [
+ 1126427,
+ 4830319
+ ],
+ [
+ 1126436,
+ 4830311
+ ],
+ [
+ 1126445,
+ 4830304
+ ],
+ [
+ 1126455,
+ 4830296
+ ],
+ [
+ 1126464,
+ 4830289
+ ],
+ [
+ 1126474,
+ 4830281
+ ],
+ [
+ 1126483,
+ 4830273
+ ],
+ [
+ 1126493,
+ 4830265
+ ],
+ [
+ 1126503,
+ 4830258
+ ],
+ [
+ 1126513,
+ 4830250
+ ],
+ [
+ 1126523,
+ 4830243
+ ],
+ [
+ 1126534,
+ 4830235
+ ],
+ [
+ 1126544,
+ 4830227
+ ],
+ [
+ 1126554,
+ 4830220
+ ],
+ [
+ 1126564,
+ 4830213
+ ],
+ [
+ 1126575,
+ 4830205
+ ],
+ [
+ 1126585,
+ 4830198
+ ],
+ [
+ 1126595,
+ 4830190
+ ],
+ [
+ 1126605,
+ 4830183
+ ],
+ [
+ 1126615,
+ 4830175
+ ],
+ [
+ 1126625,
+ 4830168
+ ],
+ [
+ 1126635,
+ 4830160
+ ],
+ [
+ 1126644,
+ 4830152
+ ],
+ [
+ 1126655,
+ 4830144
+ ],
+ [
+ 1126665,
+ 4830137
+ ],
+ [
+ 1126675,
+ 4830129
+ ],
+ [
+ 1126685,
+ 4830122
+ ],
+ [
+ 1126695,
+ 4830113
+ ],
+ [
+ 1126705,
+ 4830105
+ ],
+ [
+ 1126715,
+ 4830097
+ ],
+ [
+ 1126725,
+ 4830089
+ ],
+ [
+ 1126735,
+ 4830081
+ ],
+ [
+ 1126745,
+ 4830074
+ ],
+ [
+ 1126755,
+ 4830066
+ ],
+ [
+ 1126765,
+ 4830058
+ ],
+ [
+ 1126775,
+ 4830050
+ ],
+ [
+ 1126785,
+ 4830043
+ ],
+ [
+ 1126795,
+ 4830034
+ ],
+ [
+ 1126804,
+ 4830026
+ ],
+ [
+ 1126814,
+ 4830018
+ ],
+ [
+ 1126823,
+ 4830010
+ ],
+ [
+ 1126832,
+ 4830002
+ ],
+ [
+ 1126841,
+ 4829994
+ ],
+ [
+ 1126851,
+ 4829986
+ ],
+ [
+ 1126860,
+ 4829978
+ ],
+ [
+ 1126870,
+ 4829970
+ ],
+ [
+ 1126880,
+ 4829962
+ ],
+ [
+ 1126889,
+ 4829954
+ ],
+ [
+ 1126898,
+ 4829947
+ ],
+ [
+ 1126908,
+ 4829939
+ ],
+ [
+ 1126917,
+ 4829931
+ ],
+ [
+ 1126927,
+ 4829923
+ ],
+ [
+ 1126936,
+ 4829915
+ ],
+ [
+ 1126946,
+ 4829907
+ ],
+ [
+ 1126956,
+ 4829900
+ ],
+ [
+ 1126966,
+ 4829892
+ ],
+ [
+ 1126976,
+ 4829885
+ ],
+ [
+ 1126986,
+ 4829877
+ ],
+ [
+ 1126996,
+ 4829869
+ ],
+ [
+ 1127006,
+ 4829861
+ ],
+ [
+ 1127015,
+ 4829854
+ ],
+ [
+ 1127026,
+ 4829846
+ ],
+ [
+ 1127036,
+ 4829839
+ ],
+ [
+ 1127046,
+ 4829831
+ ],
+ [
+ 1127056,
+ 4829824
+ ],
+ [
+ 1127066,
+ 4829816
+ ],
+ [
+ 1127076,
+ 4829808
+ ],
+ [
+ 1127086,
+ 4829800
+ ],
+ [
+ 1127096,
+ 4829792
+ ],
+ [
+ 1127106,
+ 4829784
+ ],
+ [
+ 1127117,
+ 4829777
+ ],
+ [
+ 1127127,
+ 4829769
+ ],
+ [
+ 1127137,
+ 4829761
+ ],
+ [
+ 1127147,
+ 4829753
+ ],
+ [
+ 1127156,
+ 4829746
+ ],
+ [
+ 1127166,
+ 4829738
+ ],
+ [
+ 1127175,
+ 4829730
+ ],
+ [
+ 1127185,
+ 4829721
+ ],
+ [
+ 1127194,
+ 4829713
+ ],
+ [
+ 1127204,
+ 4829705
+ ],
+ [
+ 1127213,
+ 4829697
+ ],
+ [
+ 1127222,
+ 4829689
+ ],
+ [
+ 1127231,
+ 4829681
+ ],
+ [
+ 1127241,
+ 4829673
+ ],
+ [
+ 1127250,
+ 4829665
+ ],
+ [
+ 1127260,
+ 4829657
+ ],
+ [
+ 1127269,
+ 4829649
+ ],
+ [
+ 1127279,
+ 4829641
+ ],
+ [
+ 1127289,
+ 4829633
+ ],
+ [
+ 1127299,
+ 4829625
+ ],
+ [
+ 1127309,
+ 4829618
+ ],
+ [
+ 1127319,
+ 4829609
+ ],
+ [
+ 1127328,
+ 4829601
+ ],
+ [
+ 1127339,
+ 4829593
+ ],
+ [
+ 1127348,
+ 4829586
+ ],
+ [
+ 1127358,
+ 4829578
+ ],
+ [
+ 1127367,
+ 4829570
+ ],
+ [
+ 1127377,
+ 4829562
+ ],
+ [
+ 1127386,
+ 4829555
+ ],
+ [
+ 1127396,
+ 4829547
+ ],
+ [
+ 1127406,
+ 4829539
+ ],
+ [
+ 1127416,
+ 4829531
+ ],
+ [
+ 1127426,
+ 4829524
+ ],
+ [
+ 1127436,
+ 4829517
+ ],
+ [
+ 1127446,
+ 4829510
+ ],
+ [
+ 1127456,
+ 4829502
+ ],
+ [
+ 1127466,
+ 4829494
+ ],
+ [
+ 1127475,
+ 4829486
+ ],
+ [
+ 1127484,
+ 4829478
+ ],
+ [
+ 1127494,
+ 4829470
+ ],
+ [
+ 1127503,
+ 4829462
+ ],
+ [
+ 1127514,
+ 4829454
+ ],
+ [
+ 1127524,
+ 4829446
+ ],
+ [
+ 1127534,
+ 4829439
+ ],
+ [
+ 1127544,
+ 4829432
+ ],
+ [
+ 1127554,
+ 4829424
+ ],
+ [
+ 1127563,
+ 4829417
+ ],
+ [
+ 1127574,
+ 4829409
+ ],
+ [
+ 1127583,
+ 4829401
+ ],
+ [
+ 1127594,
+ 4829393
+ ],
+ [
+ 1127604,
+ 4829387
+ ],
+ [
+ 1127614,
+ 4829379
+ ],
+ [
+ 1127624,
+ 4829372
+ ],
+ [
+ 1127634,
+ 4829364
+ ],
+ [
+ 1127644,
+ 4829357
+ ],
+ [
+ 1127654,
+ 4829349
+ ],
+ [
+ 1127664,
+ 4829342
+ ],
+ [
+ 1127674,
+ 4829334
+ ],
+ [
+ 1127684,
+ 4829326
+ ],
+ [
+ 1127694,
+ 4829318
+ ],
+ [
+ 1127703,
+ 4829311
+ ],
+ [
+ 1127713,
+ 4829303
+ ],
+ [
+ 1127723,
+ 4829295
+ ],
+ [
+ 1127733,
+ 4829287
+ ],
+ [
+ 1127742,
+ 4829280
+ ],
+ [
+ 1127752,
+ 4829272
+ ],
+ [
+ 1127762,
+ 4829264
+ ],
+ [
+ 1127772,
+ 4829256
+ ],
+ [
+ 1127781,
+ 4829248
+ ],
+ [
+ 1127791,
+ 4829240
+ ],
+ [
+ 1127802,
+ 4829233
+ ],
+ [
+ 1127812,
+ 4829225
+ ],
+ [
+ 1127821,
+ 4829217
+ ],
+ [
+ 1127831,
+ 4829209
+ ],
+ [
+ 1127841,
+ 4829202
+ ],
+ [
+ 1127851,
+ 4829194
+ ],
+ [
+ 1127861,
+ 4829186
+ ],
+ [
+ 1127871,
+ 4829178
+ ],
+ [
+ 1127880,
+ 4829170
+ ],
+ [
+ 1127889,
+ 4829162
+ ],
+ [
+ 1127898,
+ 4829154
+ ],
+ [
+ 1127908,
+ 4829146
+ ],
+ [
+ 1127918,
+ 4829139
+ ],
+ [
+ 1127928,
+ 4829131
+ ],
+ [
+ 1127938,
+ 4829123
+ ],
+ [
+ 1127948,
+ 4829115
+ ],
+ [
+ 1127957,
+ 4829107
+ ],
+ [
+ 1127967,
+ 4829099
+ ],
+ [
+ 1127976,
+ 4829091
+ ],
+ [
+ 1127986,
+ 4829083
+ ],
+ [
+ 1127996,
+ 4829075
+ ],
+ [
+ 1128006,
+ 4829067
+ ],
+ [
+ 1128015,
+ 4829059
+ ],
+ [
+ 1128025,
+ 4829051
+ ],
+ [
+ 1128036,
+ 4829043
+ ],
+ [
+ 1128046,
+ 4829035
+ ],
+ [
+ 1128055,
+ 4829027
+ ],
+ [
+ 1128065,
+ 4829019
+ ],
+ [
+ 1128075,
+ 4829011
+ ],
+ [
+ 1128085,
+ 4829003
+ ],
+ [
+ 1128094,
+ 4828996
+ ],
+ [
+ 1128104,
+ 4828988
+ ],
+ [
+ 1128113,
+ 4828981
+ ],
+ [
+ 1128123,
+ 4828972
+ ],
+ [
+ 1128132,
+ 4828964
+ ],
+ [
+ 1128142,
+ 4828956
+ ],
+ [
+ 1128151,
+ 4828949
+ ],
+ [
+ 1128161,
+ 4828941
+ ],
+ [
+ 1128170,
+ 4828933
+ ],
+ [
+ 1128180,
+ 4828925
+ ],
+ [
+ 1128189,
+ 4828917
+ ],
+ [
+ 1128199,
+ 4828909
+ ],
+ [
+ 1128208,
+ 4828902
+ ],
+ [
+ 1128218,
+ 4828894
+ ],
+ [
+ 1128228,
+ 4828886
+ ],
+ [
+ 1128238,
+ 4828878
+ ],
+ [
+ 1128247,
+ 4828870
+ ],
+ [
+ 1128257,
+ 4828862
+ ],
+ [
+ 1128268,
+ 4828855
+ ],
+ [
+ 1128278,
+ 4828847
+ ],
+ [
+ 1128288,
+ 4828840
+ ],
+ [
+ 1128298,
+ 4828832
+ ],
+ [
+ 1128308,
+ 4828824
+ ],
+ [
+ 1128318,
+ 4828816
+ ],
+ [
+ 1128327,
+ 4828809
+ ],
+ [
+ 1128337,
+ 4828801
+ ],
+ [
+ 1128347,
+ 4828794
+ ],
+ [
+ 1128357,
+ 4828786
+ ],
+ [
+ 1128367,
+ 4828778
+ ],
+ [
+ 1128377,
+ 4828770
+ ],
+ [
+ 1128386,
+ 4828762
+ ],
+ [
+ 1128396,
+ 4828754
+ ],
+ [
+ 1128406,
+ 4828746
+ ],
+ [
+ 1128416,
+ 4828738
+ ],
+ [
+ 1128425,
+ 4828730
+ ],
+ [
+ 1128435,
+ 4828722
+ ],
+ [
+ 1128445,
+ 4828715
+ ],
+ [
+ 1128455,
+ 4828707
+ ],
+ [
+ 1128464,
+ 4828700
+ ],
+ [
+ 1128474,
+ 4828692
+ ],
+ [
+ 1128484,
+ 4828684
+ ],
+ [
+ 1128495,
+ 4828677
+ ],
+ [
+ 1128504,
+ 4828670
+ ],
+ [
+ 1128514,
+ 4828662
+ ],
+ [
+ 1128524,
+ 4828654
+ ],
+ [
+ 1128534,
+ 4828645
+ ],
+ [
+ 1128544,
+ 4828637
+ ],
+ [
+ 1128554,
+ 4828629
+ ],
+ [
+ 1128563,
+ 4828622
+ ],
+ [
+ 1128574,
+ 4828614
+ ],
+ [
+ 1128584,
+ 4828607
+ ],
+ [
+ 1128594,
+ 4828599
+ ],
+ [
+ 1128603,
+ 4828591
+ ],
+ [
+ 1128613,
+ 4828583
+ ],
+ [
+ 1128622,
+ 4828575
+ ],
+ [
+ 1128632,
+ 4828567
+ ],
+ [
+ 1128641,
+ 4828559
+ ],
+ [
+ 1128651,
+ 4828551
+ ],
+ [
+ 1128660,
+ 4828543
+ ],
+ [
+ 1128670,
+ 4828535
+ ],
+ [
+ 1128680,
+ 4828527
+ ],
+ [
+ 1128690,
+ 4828519
+ ],
+ [
+ 1128699,
+ 4828511
+ ],
+ [
+ 1128708,
+ 4828503
+ ],
+ [
+ 1128717,
+ 4828495
+ ],
+ [
+ 1128728,
+ 4828487
+ ],
+ [
+ 1128738,
+ 4828479
+ ],
+ [
+ 1128748,
+ 4828471
+ ],
+ [
+ 1128757,
+ 4828463
+ ],
+ [
+ 1128767,
+ 4828455
+ ],
+ [
+ 1128776,
+ 4828448
+ ],
+ [
+ 1128786,
+ 4828440
+ ],
+ [
+ 1128796,
+ 4828433
+ ],
+ [
+ 1128806,
+ 4828425
+ ],
+ [
+ 1128815,
+ 4828417
+ ],
+ [
+ 1128825,
+ 4828409
+ ],
+ [
+ 1128835,
+ 4828402
+ ],
+ [
+ 1128846,
+ 4828394
+ ],
+ [
+ 1128856,
+ 4828387
+ ],
+ [
+ 1128866,
+ 4828379
+ ],
+ [
+ 1128876,
+ 4828371
+ ],
+ [
+ 1128886,
+ 4828363
+ ],
+ [
+ 1128895,
+ 4828356
+ ],
+ [
+ 1128905,
+ 4828348
+ ],
+ [
+ 1128915,
+ 4828341
+ ],
+ [
+ 1128925,
+ 4828333
+ ],
+ [
+ 1128935,
+ 4828325
+ ],
+ [
+ 1128945,
+ 4828317
+ ],
+ [
+ 1128954,
+ 4828310
+ ],
+ [
+ 1128965,
+ 4828302
+ ],
+ [
+ 1128975,
+ 4828295
+ ],
+ [
+ 1128985,
+ 4828287
+ ],
+ [
+ 1128994,
+ 4828279
+ ],
+ [
+ 1129004,
+ 4828271
+ ],
+ [
+ 1129013,
+ 4828264
+ ],
+ [
+ 1129023,
+ 4828256
+ ],
+ [
+ 1129032,
+ 4828248
+ ],
+ [
+ 1129042,
+ 4828240
+ ],
+ [
+ 1129052,
+ 4828233
+ ],
+ [
+ 1129062,
+ 4828225
+ ],
+ [
+ 1129072,
+ 4828218
+ ],
+ [
+ 1129082,
+ 4828210
+ ],
+ [
+ 1129091,
+ 4828203
+ ],
+ [
+ 1129101,
+ 4828195
+ ],
+ [
+ 1129111,
+ 4828188
+ ],
+ [
+ 1129122,
+ 4828180
+ ],
+ [
+ 1129132,
+ 4828173
+ ],
+ [
+ 1129142,
+ 4828165
+ ],
+ [
+ 1129152,
+ 4828157
+ ],
+ [
+ 1129162,
+ 4828150
+ ],
+ [
+ 1129172,
+ 4828143
+ ],
+ [
+ 1129182,
+ 4828135
+ ],
+ [
+ 1129193,
+ 4828128
+ ],
+ [
+ 1129204,
+ 4828120
+ ],
+ [
+ 1129214,
+ 4828112
+ ],
+ [
+ 1129224,
+ 4828104
+ ],
+ [
+ 1129233,
+ 4828097
+ ],
+ [
+ 1129243,
+ 4828089
+ ],
+ [
+ 1129252,
+ 4828081
+ ],
+ [
+ 1129262,
+ 4828073
+ ],
+ [
+ 1129272,
+ 4828066
+ ],
+ [
+ 1129282,
+ 4828058
+ ],
+ [
+ 1129292,
+ 4828051
+ ],
+ [
+ 1129302,
+ 4828043
+ ],
+ [
+ 1129311,
+ 4828036
+ ],
+ [
+ 1129321,
+ 4828028
+ ],
+ [
+ 1129330,
+ 4828020
+ ],
+ [
+ 1129340,
+ 4828012
+ ],
+ [
+ 1129350,
+ 4828005
+ ],
+ [
+ 1129360,
+ 4827997
+ ],
+ [
+ 1129370,
+ 4827990
+ ],
+ [
+ 1129381,
+ 4827982
+ ],
+ [
+ 1129391,
+ 4827975
+ ],
+ [
+ 1129401,
+ 4827967
+ ],
+ [
+ 1129410,
+ 4827959
+ ],
+ [
+ 1129420,
+ 4827951
+ ],
+ [
+ 1129431,
+ 4827943
+ ],
+ [
+ 1129442,
+ 4827935
+ ],
+ [
+ 1129452,
+ 4827928
+ ],
+ [
+ 1129463,
+ 4827920
+ ],
+ [
+ 1129473,
+ 4827913
+ ],
+ [
+ 1129483,
+ 4827905
+ ],
+ [
+ 1129493,
+ 4827897
+ ],
+ [
+ 1129502,
+ 4827889
+ ],
+ [
+ 1129511,
+ 4827882
+ ],
+ [
+ 1129520,
+ 4827873
+ ],
+ [
+ 1129529,
+ 4827865
+ ],
+ [
+ 1129539,
+ 4827856
+ ],
+ [
+ 1129548,
+ 4827848
+ ],
+ [
+ 1129557,
+ 4827840
+ ],
+ [
+ 1129566,
+ 4827833
+ ],
+ [
+ 1129576,
+ 4827825
+ ],
+ [
+ 1129586,
+ 4827817
+ ],
+ [
+ 1129596,
+ 4827809
+ ],
+ [
+ 1129606,
+ 4827802
+ ],
+ [
+ 1129616,
+ 4827794
+ ],
+ [
+ 1129626,
+ 4827787
+ ],
+ [
+ 1129636,
+ 4827779
+ ],
+ [
+ 1129646,
+ 4827771
+ ],
+ [
+ 1129657,
+ 4827763
+ ],
+ [
+ 1129667,
+ 4827756
+ ],
+ [
+ 1129677,
+ 4827748
+ ],
+ [
+ 1129686,
+ 4827740
+ ],
+ [
+ 1129695,
+ 4827731
+ ],
+ [
+ 1129704,
+ 4827723
+ ],
+ [
+ 1129714,
+ 4827715
+ ],
+ [
+ 1129724,
+ 4827708
+ ],
+ [
+ 1129734,
+ 4827700
+ ],
+ [
+ 1129743,
+ 4827692
+ ],
+ [
+ 1129753,
+ 4827684
+ ],
+ [
+ 1129762,
+ 4827676
+ ],
+ [
+ 1129772,
+ 4827668
+ ],
+ [
+ 1129782,
+ 4827660
+ ],
+ [
+ 1129792,
+ 4827652
+ ],
+ [
+ 1129802,
+ 4827644
+ ],
+ [
+ 1129812,
+ 4827636
+ ],
+ [
+ 1129822,
+ 4827628
+ ],
+ [
+ 1129831,
+ 4827620
+ ],
+ [
+ 1129840,
+ 4827612
+ ],
+ [
+ 1129849,
+ 4827604
+ ],
+ [
+ 1129858,
+ 4827596
+ ],
+ [
+ 1129868,
+ 4827588
+ ],
+ [
+ 1129878,
+ 4827580
+ ],
+ [
+ 1129889,
+ 4827572
+ ],
+ [
+ 1129899,
+ 4827564
+ ],
+ [
+ 1129909,
+ 4827556
+ ],
+ [
+ 1129918,
+ 4827549
+ ],
+ [
+ 1129928,
+ 4827541
+ ],
+ [
+ 1129937,
+ 4827533
+ ],
+ [
+ 1129947,
+ 4827526
+ ],
+ [
+ 1129957,
+ 4827519
+ ],
+ [
+ 1129967,
+ 4827510
+ ],
+ [
+ 1129976,
+ 4827502
+ ],
+ [
+ 1129986,
+ 4827494
+ ],
+ [
+ 1129995,
+ 4827486
+ ],
+ [
+ 1130005,
+ 4827478
+ ],
+ [
+ 1130015,
+ 4827470
+ ],
+ [
+ 1130025,
+ 4827462
+ ],
+ [
+ 1130034,
+ 4827455
+ ],
+ [
+ 1130044,
+ 4827447
+ ],
+ [
+ 1130053,
+ 4827439
+ ],
+ [
+ 1130063,
+ 4827431
+ ],
+ [
+ 1130072,
+ 4827423
+ ],
+ [
+ 1130082,
+ 4827415
+ ],
+ [
+ 1130092,
+ 4827408
+ ],
+ [
+ 1130102,
+ 4827400
+ ],
+ [
+ 1130112,
+ 4827393
+ ],
+ [
+ 1130123,
+ 4827385
+ ],
+ [
+ 1130133,
+ 4827378
+ ],
+ [
+ 1130143,
+ 4827370
+ ],
+ [
+ 1130152,
+ 4827362
+ ],
+ [
+ 1130162,
+ 4827354
+ ],
+ [
+ 1130172,
+ 4827347
+ ],
+ [
+ 1130182,
+ 4827339
+ ],
+ [
+ 1130191,
+ 4827332
+ ],
+ [
+ 1130201,
+ 4827324
+ ],
+ [
+ 1130211,
+ 4827317
+ ],
+ [
+ 1130222,
+ 4827309
+ ],
+ [
+ 1130232,
+ 4827302
+ ],
+ [
+ 1130243,
+ 4827294
+ ],
+ [
+ 1130253,
+ 4827287
+ ],
+ [
+ 1130263,
+ 4827279
+ ],
+ [
+ 1130273,
+ 4827272
+ ],
+ [
+ 1130283,
+ 4827265
+ ],
+ [
+ 1130293,
+ 4827258
+ ],
+ [
+ 1130303,
+ 4827251
+ ],
+ [
+ 1130313,
+ 4827244
+ ],
+ [
+ 1130323,
+ 4827236
+ ],
+ [
+ 1130333,
+ 4827229
+ ],
+ [
+ 1130343,
+ 4827221
+ ],
+ [
+ 1130352,
+ 4827213
+ ],
+ [
+ 1130363,
+ 4827205
+ ],
+ [
+ 1130372,
+ 4827198
+ ],
+ [
+ 1130382,
+ 4827190
+ ],
+ [
+ 1130392,
+ 4827182
+ ],
+ [
+ 1130402,
+ 4827175
+ ],
+ [
+ 1130412,
+ 4827168
+ ],
+ [
+ 1130423,
+ 4827160
+ ],
+ [
+ 1130433,
+ 4827153
+ ],
+ [
+ 1130443,
+ 4827145
+ ],
+ [
+ 1130453,
+ 4827138
+ ],
+ [
+ 1130463,
+ 4827130
+ ],
+ [
+ 1130473,
+ 4827123
+ ],
+ [
+ 1130483,
+ 4827115
+ ],
+ [
+ 1130493,
+ 4827108
+ ],
+ [
+ 1130503,
+ 4827100
+ ],
+ [
+ 1130513,
+ 4827092
+ ],
+ [
+ 1130523,
+ 4827084
+ ],
+ [
+ 1130533,
+ 4827077
+ ],
+ [
+ 1130543,
+ 4827069
+ ],
+ [
+ 1130553,
+ 4827062
+ ],
+ [
+ 1130563,
+ 4827053
+ ],
+ [
+ 1130572,
+ 4827045
+ ],
+ [
+ 1130582,
+ 4827037
+ ],
+ [
+ 1130592,
+ 4827030
+ ],
+ [
+ 1130602,
+ 4827022
+ ],
+ [
+ 1130612,
+ 4827015
+ ],
+ [
+ 1130622,
+ 4827007
+ ],
+ [
+ 1130632,
+ 4826999
+ ],
+ [
+ 1130641,
+ 4826990
+ ],
+ [
+ 1130650,
+ 4826982
+ ],
+ [
+ 1130660,
+ 4826974
+ ],
+ [
+ 1130669,
+ 4826967
+ ],
+ [
+ 1130680,
+ 4826959
+ ],
+ [
+ 1130690,
+ 4826952
+ ],
+ [
+ 1130700,
+ 4826944
+ ],
+ [
+ 1130710,
+ 4826937
+ ],
+ [
+ 1130720,
+ 4826929
+ ],
+ [
+ 1130730,
+ 4826922
+ ],
+ [
+ 1130740,
+ 4826914
+ ],
+ [
+ 1130750,
+ 4826906
+ ],
+ [
+ 1130760,
+ 4826898
+ ],
+ [
+ 1130770,
+ 4826891
+ ],
+ [
+ 1130780,
+ 4826883
+ ],
+ [
+ 1130790,
+ 4826876
+ ],
+ [
+ 1130800,
+ 4826868
+ ],
+ [
+ 1130809,
+ 4826861
+ ],
+ [
+ 1130819,
+ 4826853
+ ],
+ [
+ 1130830,
+ 4826845
+ ],
+ [
+ 1130840,
+ 4826837
+ ],
+ [
+ 1130850,
+ 4826830
+ ],
+ [
+ 1130860,
+ 4826822
+ ],
+ [
+ 1130869,
+ 4826814
+ ],
+ [
+ 1130879,
+ 4826806
+ ],
+ [
+ 1130888,
+ 4826799
+ ],
+ [
+ 1130899,
+ 4826792
+ ],
+ [
+ 1130909,
+ 4826785
+ ],
+ [
+ 1130919,
+ 4826777
+ ],
+ [
+ 1130929,
+ 4826769
+ ],
+ [
+ 1130939,
+ 4826761
+ ],
+ [
+ 1130948,
+ 4826754
+ ],
+ [
+ 1130958,
+ 4826746
+ ],
+ [
+ 1130967,
+ 4826738
+ ],
+ [
+ 1130978,
+ 4826730
+ ],
+ [
+ 1130988,
+ 4826723
+ ],
+ [
+ 1130998,
+ 4826716
+ ],
+ [
+ 1131008,
+ 4826709
+ ],
+ [
+ 1131019,
+ 4826701
+ ],
+ [
+ 1131029,
+ 4826694
+ ],
+ [
+ 1131040,
+ 4826687
+ ],
+ [
+ 1131050,
+ 4826681
+ ],
+ [
+ 1131060,
+ 4826673
+ ],
+ [
+ 1131071,
+ 4826665
+ ],
+ [
+ 1131081,
+ 4826657
+ ],
+ [
+ 1131090,
+ 4826649
+ ],
+ [
+ 1131100,
+ 4826642
+ ]
+ ]
+ ],
+ "type": "AnyCrsMultiLineString"
+ },
+ "properties": {},
+ "type": "AnyCrsFeature"
+ }
+ ],
+ "persistableReferenceCrs": "{\"authCode\":{\"auth\":\"OSDU\",\"code\":\"2193001\"},\"lateBoundCRS\":{\"authCode\":{\"auth\":\"EPSG\",\"code\":\"2193\"},\"name\":\"NZGD_2000_New_Zealand_Transverse_Mercator\",\"type\":\"LBC\",\"ver\":\"PE_10_9_1\",\"wkt\":\"PROJCS[\\\"NZGD_2000_New_Zealand_Transverse_Mercator\\\",GEOGCS[\\\"GCS_NZGD_2000\\\",DATUM[\\\"D_NZGD_2000\\\",SPHEROID[\\\"GRS_1980\\\",6378137.0,298.257222101]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],PROJECTION[\\\"Transverse_Mercator\\\"],PARAMETER[\\\"False_Easting\\\",1600000.0],PARAMETER[\\\"False_Northing\\\",10000000.0],PARAMETER[\\\"Central_Meridian\\\",173.0],PARAMETER[\\\"Scale_Factor\\\",0.9996],PARAMETER[\\\"Latitude_Of_Origin\\\",0.0],UNIT[\\\"Meter\\\",1.0],AUTHORITY[\\\"EPSG\\\",2193]]\"},\"name\":\"NZGD2000 * OSG-Nzl / New Zealand Transverse Mercator 2000 [2193,1565]\",\"singleCT\":{\"authCode\":{\"auth\":\"EPSG\",\"code\":\"1565\"},\"name\":\"NZGD_2000_To_WGS_1984_1\",\"type\":\"ST\",\"ver\":\"PE_10_9_1\",\"wkt\":\"GEOGTRAN[\\\"NZGD_2000_To_WGS_1984_1\\\",GEOGCS[\\\"GCS_NZGD_2000\\\",DATUM[\\\"D_NZGD_2000\\\",SPHEROID[\\\"GRS_1980\\\",6378137.0,298.257222101]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],GEOGCS[\\\"GCS_WGS_1984\\\",DATUM[\\\"D_WGS_1984\\\",SPHEROID[\\\"WGS_1984\\\",6378137.0,298.257223563]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],METHOD[\\\"Geocentric_Translation\\\"],PARAMETER[\\\"X_Axis_Translation\\\",0.0],PARAMETER[\\\"Y_Axis_Translation\\\",0.0],PARAMETER[\\\"Z_Axis_Translation\\\",0.0],OPERATIONACCURACY[1.0],AUTHORITY[\\\"EPSG\\\",1565]]\"},\"type\":\"EBC\",\"ver\":\"PE_10_9_1\"}",
+ "type": "AnyCrsFeatureCollection"
+ }
+ }
+ }
+ }
+]
diff --git a/indexer-acceptance-test/src/test/resources/testData/index_record_seismic_survey.schema.json b/indexer-acceptance-test/src/test/resources/testData/index_record_seismic_survey.schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..dc7c5c6d4662dd5c8f32b620145cb5340eea5675
--- /dev/null
+++ b/indexer-acceptance-test/src/test/resources/testData/index_record_seismic_survey.schema.json
@@ -0,0 +1,2388 @@
+{
+ "schemaInfo": {
+ "schemaIdentity": {
+ "authority": "tenant1",
+ "source": "indexer",
+ "entityType": "decimation-Integration",
+ "schemaVersionMajor": "1",
+ "schemaVersionMinor": "0",
+ "schemaVersionPatch": "0"
+ },
+ "status": "DEVELOPMENT"
+ },
+ "schema": {
+ "x-osdu-inheriting-from-kind": [{
+ "kind": "osdu:wks:AbstractProject:1.0.0",
+ "name": "Project"
+ }
+ ],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:master-data--SeismicAcquisitionSurvey:1.0.0",
+ "description": "A seismic acquisition project is a type of business project that deploys resources to the field to record seismic data. It may be referred to as a field survey, acquisition survey, or field program. It is not the same as the geometry of the deployed equipment (nav), which is a work product component.",
+ "title": "SeismicAcquisitionSurvey",
+ "type": "object",
+ "required": [
+ "kind",
+ "acl",
+ "legal"
+ ],
+ "x-osdu-virtual-properties": {
+ "data.VirtualProperties.DefaultLocation": {
+ "type": "object",
+ "priority": [{
+ "path": "data.SpatialLocation"
+ }
+ ]
+ },
+ "data.VirtualProperties.DefaultName": {
+ "type": "string",
+ "priority": [{
+ "path": "data.ProjectName"
+ }
+ ]
+ }
+ },
+ "additionalProperties": false,
+ "definitions": {
+ "osdu:wks:AbstractCommonResources:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractCommonResources:1.0.0",
+ "description": "Common resources to be injected at root 'data' level for every entity, which is persistable in Storage. The insertion is performed by the OsduSchemaComposer script.",
+ "x-osdu-review-status": "Accepted",
+ "title": "OSDU Common Resources",
+ "type": "object",
+ "properties": {
+ "ResourceHomeRegionID": {
+ "x-osdu-relationship": [{
+ "EntityType": "OSDURegion",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-OSDURegion:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The name of the home [cloud environment] region for this OSDU resource object.",
+ "type": "string",
+ "title": "Resource Home Region ID"
+ },
+ "ResourceHostRegionIDs": {
+ "description": "The name of the host [cloud environment] region(s) for this OSDU resource object.",
+ "type": "array",
+ "title": "Resource Host Region ID",
+ "items": {
+ "x-osdu-relationship": [{
+ "EntityType": "OSDURegion",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-OSDURegion:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "type": "string"
+ }
+ },
+ "ResourceLifecycleStatus": {
+ "x-osdu-relationship": [{
+ "EntityType": "ResourceLifecycleStatus",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-ResourceLifecycleStatus:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Describes the current Resource Lifecycle status.",
+ "type": "string",
+ "title": "Resource Lifecycle Status"
+ },
+ "ResourceSecurityClassification": {
+ "x-osdu-relationship": [{
+ "EntityType": "ResourceSecurityClassification",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-ResourceSecurityClassification:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Classifies the security level of the resource.",
+ "type": "string",
+ "title": "Resource Security Classification"
+ },
+ "ResourceCurationStatus": {
+ "x-osdu-relationship": [{
+ "EntityType": "ResourceCurationStatus",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-ResourceCurationStatus:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Describes the current Curation status.",
+ "type": "string",
+ "title": "Resource Curation Status"
+ },
+ "ExistenceKind": {
+ "x-osdu-relationship": [{
+ "EntityType": "ExistenceKind",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-ExistenceKind:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Where does this data resource sit in the cradle-to-grave span of its existence?",
+ "type": "string",
+ "title": "Existence Kind"
+ },
+ "TechnicalAssuranceID": {
+ "x-osdu-relationship": [{
+ "EntityType": "TechnicalAssuranceType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-TechnicalAssuranceType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "DEPRECATED: Describes a record's overall suitability for general business consumption based on data quality. Clarifications: Since Certified is the highest classification of suitable quality, any further change or versioning of a Certified record should be carefully considered and justified. If a Technical Assurance value is not populated then one can assume the data has not been evaluated or its quality is unknown (=Unevaluated). Technical Assurance values are not intended to be used for the identification of a single \"preferred\" or \"definitive\" record by comparison with other records.",
+ "type": "string",
+ "title": "Technical Assurance ID"
+ },
+ "Source": {
+ "description": "The entity that produced the record, or from which it is received; could be an organization, agency, system, internal team, or individual. For informational purposes only, the list of sources is not governed.",
+ "type": "string",
+ "title": "Data Source"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractCommonResources.1.0.0.json"
+ },
+ "osdu:wks:AbstractMetaItem:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "oneOf": [{
+ "title": "FrameOfReferenceUOM",
+ "type": "object",
+ "properties": {
+ "persistableReference": {
+ "description": "The self-contained, persistable reference string uniquely identifying the Unit.",
+ "title": "UOM Persistable Reference",
+ "type": "string",
+ "example": "{\"abcd\":{\"a\":0.0,\"b\":1200.0,\"c\":3937.0,\"d\":0.0},\"symbol\":\"ft[US]\",\"baseMeasurement\":{\"ancestry\":\"L\",\"type\":\"UM\"},\"type\":\"UAD\"}"
+ },
+ "kind": {
+ "const": "Unit",
+ "description": "The kind of reference, 'Unit' for FrameOfReferenceUOM.",
+ "title": "UOM Reference Kind",
+ "type": "string"
+ },
+ "propertyNames": {
+ "description": "The list of property names, to which this meta data item provides Unit context to. A full path like \"StructureA.PropertyB\" is required to define a unique context; \"data\" is omitted since frame-of reference normalization only applies to the data block.",
+ "title": "UOM Property Names",
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "example": [
+ "HorizontalDeflection.EastWest",
+ "HorizontalDeflection.NorthSouth"
+ ]
+ },
+ "name": {
+ "description": "The unit symbol or name of the unit.",
+ "title": "UOM Unit Symbol",
+ "type": "string",
+ "example": "ft[US]"
+ },
+ "unitOfMeasureID": {
+ "x-osdu-relationship": [{
+ "EntityType": "UnitOfMeasure",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-UnitOfMeasure:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "SRN to unit of measure reference.",
+ "type": "string",
+ "example": "namespace:reference-data--UnitOfMeasure:ftUS:"
+ }
+ },
+ "required": [
+ "kind",
+ "persistableReference"
+ ]
+ }, {
+ "title": "FrameOfReferenceCRS",
+ "type": "object",
+ "properties": {
+ "coordinateReferenceSystemID": {
+ "x-osdu-relationship": [{
+ "EntityType": "CoordinateReferenceSystem",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-CoordinateReferenceSystem:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "SRN to CRS reference.",
+ "type": "string",
+ "example": "namespace:reference-data--CoordinateReferenceSystem:Projected:EPSG::32615:"
+ },
+ "persistableReference": {
+ "description": "The self-contained, persistable reference string uniquely identifying the CRS.",
+ "title": "CRS Persistable Reference",
+ "type": "string",
+ "example": "{\"authCode\":{\"auth\":\"EPSG\",\"code\":\"32615\"},\"name\":\"WGS_1984_UTM_Zone_15N\",\"type\":\"LBC\",\"ver\":\"PE_10_9_1\",\"wkt\":\"PROJCS[\\\"WGS_1984_UTM_Zone_15N\\\",GEOGCS[\\\"GCS_WGS_1984\\\",DATUM[\\\"D_WGS_1984\\\",SPHEROID[\\\"WGS_1984\\\",6378137.0,298.257223563]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],PROJECTION[\\\"Transverse_Mercator\\\"],PARAMETER[\\\"False_Easting\\\",500000.0],PARAMETER[\\\"False_Northing\\\",0.0],PARAMETER[\\\"Central_Meridian\\\",-93.0],PARAMETER[\\\"Scale_Factor\\\",0.9996],PARAMETER[\\\"Latitude_Of_Origin\\\",0.0],UNIT[\\\"Meter\\\",1.0],AUTHORITY[\\\"EPSG\\\",32615]]\"}"
+ },
+ "kind": {
+ "const": "CRS",
+ "description": "The kind of reference, constant 'CRS' for FrameOfReferenceCRS.",
+ "title": "CRS Reference Kind",
+ "type": "string"
+ },
+ "propertyNames": {
+ "description": "The list of property names, to which this meta data item provides CRS context to. A full path like \"StructureA.PropertyB\" is required to define a unique context; \"data\" is omitted since frame-of reference normalization only applies to the data block.",
+ "title": "CRS Property Names",
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "example": [
+ "KickOffPosition.X",
+ "KickOffPosition.Y"
+ ]
+ },
+ "name": {
+ "description": "The name of the CRS.",
+ "title": "CRS Name",
+ "type": "string",
+ "example": "WGS 84 / UTM zone 15N"
+ }
+ },
+ "required": [
+ "kind",
+ "persistableReference"
+ ]
+ }, {
+ "title": "FrameOfReferenceDateTime",
+ "type": "object",
+ "properties": {
+ "persistableReference": {
+ "description": "The self-contained, persistable reference string uniquely identifying DateTime reference.",
+ "title": "DateTime Persistable Reference",
+ "type": "string",
+ "example": "{\"format\":\"yyyy-MM-ddTHH:mm:ssZ\",\"timeZone\":\"UTC\",\"type\":\"DTM\"}"
+ },
+ "kind": {
+ "const": "DateTime",
+ "description": "The kind of reference, constant 'DateTime', for FrameOfReferenceDateTime.",
+ "title": "DateTime Reference Kind",
+ "type": "string"
+ },
+ "propertyNames": {
+ "description": "The list of property names, to which this meta data item provides DateTime context to. A full path like \"StructureA.PropertyB\" is required to define a unique context; \"data\" is omitted since frame-of reference normalization only applies to the data block.",
+ "title": "DateTime Property Names",
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "example": [
+ "Acquisition.StartTime",
+ "Acquisition.EndTime"
+ ]
+ },
+ "name": {
+ "description": "The name of the DateTime format and reference.",
+ "title": "DateTime Name",
+ "type": "string",
+ "example": "UTC"
+ }
+ },
+ "required": [
+ "kind",
+ "persistableReference"
+ ]
+ }, {
+ "title": "FrameOfReferenceAzimuthReference",
+ "type": "object",
+ "properties": {
+ "persistableReference": {
+ "description": "The self-contained, persistable reference string uniquely identifying AzimuthReference.",
+ "title": "AzimuthReference Persistable Reference",
+ "type": "string",
+ "example": "{\"code\":\"TrueNorth\",\"type\":\"AZR\"}"
+ },
+ "kind": {
+ "const": "AzimuthReference",
+ "description": "The kind of reference, constant 'AzimuthReference', for FrameOfReferenceAzimuthReference.",
+ "title": "AzimuthReference Reference Kind",
+ "type": "string"
+ },
+ "propertyNames": {
+ "description": "The list of property names, to which this meta data item provides AzimuthReference context to. A full path like \"StructureA.PropertyB\" is required to define a unique context; \"data\" is omitted since frame-of reference normalization only applies to the data block.",
+ "title": "AzimuthReference Property Names",
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "example": [
+ "Bearing"
+ ]
+ },
+ "name": {
+ "description": "The name of the CRS or the symbol/name of the unit.",
+ "title": "AzimuthReference Name",
+ "type": "string",
+ "example": "TrueNorth"
+ }
+ },
+ "required": [
+ "kind",
+ "persistableReference"
+ ]
+ }
+ ],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractMetaItem:1.0.0",
+ "description": "A meta data item, which allows the association of named properties or property values to a Unit/Measurement/CRS/Azimuth/Time context.",
+ "title": "Frame of Reference Meta Data Item",
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractMetaItem.1.0.0.json"
+ },
+ "osdu:wks:AbstractLegalParentList:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractLegalParentList:1.0.0",
+ "description": "A list of entity id:version references to record instances recorded in the data platform, from which the current record is derived and from which the legal tags must be derived. This structure is included by the SystemProperties \"ancestry\", which is part of all OSDU records. Not extensible.",
+ "additionalProperties": false,
+ "title": "Parent List",
+ "type": "object",
+ "properties": {
+ "parents": {
+ "description": "An array of none, one or many entity references of 'direct parents' in the data platform, which mark the current record as a derivative. In contrast to other relationships, the source record version is required. During record creation or update the ancestry.parents[] relationships are used to collect the legal tags from the sources and aggregate them in the legal.legaltags[] array. As a consequence, should e.g., one or more of the legal tags of the source data expire, the access to the derivatives is also terminated. For details, see ComplianceService tutorial, 'Creating derivative Records'.",
+ "title": "Parents",
+ "type": "array",
+ "items": {
+ "x-osdu-relationship": [],
+ "pattern": "^[\\w\\-\\.]+:[\\w\\-\\.]+:[\\w\\-\\.\\:\\%]+:[0-9]+$",
+ "type": "string"
+ },
+ "example": []
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractLegalParentList.1.0.0.json"
+ },
+ "osdu:wks:AbstractGeoContext:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "oneOf": [{
+ "$ref": "#/definitions/osdu:wks:AbstractGeoPoliticalContext:1.0.0"
+ }, {
+ "$ref": "#/definitions/osdu:wks:AbstractGeoBasinContext:1.0.0"
+ }, {
+ "$ref": "#/definitions/osdu:wks:AbstractGeoFieldContext:1.0.0"
+ }, {
+ "$ref": "#/definitions/osdu:wks:AbstractGeoPlayContext:1.0.0"
+ }, {
+ "$ref": "#/definitions/osdu:wks:AbstractGeoProspectContext:1.0.0"
+ }
+ ],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractGeoContext:1.0.0",
+ "description": "A geographic context to an entity. It can be either a reference to a GeoPoliticalEntity, Basin, Field, Play or Prospect.",
+ "title": "AbstractGeoContext",
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractGeoContext.1.0.0.json"
+ },
+ "osdu:wks:AbstractFeatureCollection:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractFeatureCollection:1.0.0",
+ "description": "GeoJSON feature collection as originally published in https://geojson.org/schema/FeatureCollection.json. Attention: the coordinate order is fixed: Longitude first, followed by Latitude, optionally height above MSL (EPSG:5714) as third coordinate.",
+ "title": "GeoJSON FeatureCollection",
+ "type": "object",
+ "required": [
+ "type",
+ "features"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "FeatureCollection"
+ ]
+ },
+ "features": {
+ "type": "array",
+ "items": {
+ "title": "GeoJSON Feature",
+ "type": "object",
+ "required": [
+ "type",
+ "properties",
+ "geometry"
+ ],
+ "properties": {
+ "geometry": {
+ "oneOf": [{
+ "type": "null"
+ }, {
+ "title": "GeoJSON Point",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "Point"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "GeoJSON LineString",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "LineString"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "GeoJSON Polygon",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "Polygon"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "GeoJSON MultiPoint",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "MultiPoint"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "GeoJSON MultiLineString",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "MultiLineString"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "GeoJSON MultiPolygon",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "items": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "MultiPolygon"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "GeoJSON GeometryCollection",
+ "type": "object",
+ "required": [
+ "type",
+ "geometries"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "GeometryCollection"
+ ]
+ },
+ "geometries": {
+ "type": "array",
+ "items": {
+ "oneOf": [{
+ "title": "GeoJSON Point",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "Point"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "GeoJSON LineString",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "LineString"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "GeoJSON Polygon",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "Polygon"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "GeoJSON MultiPoint",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "MultiPoint"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "GeoJSON MultiLineString",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "MultiLineString"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "GeoJSON MultiPolygon",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "items": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "MultiPolygon"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ ]
+ }
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ ]
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "Feature"
+ ]
+ },
+ "properties": {
+ "oneOf": [{
+ "type": "null"
+ }, {
+ "type": "object"
+ }
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractFeatureCollection.1.0.0.json"
+ },
+ "osdu:wks:AbstractAnyCrsFeatureCollection:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractAnyCrsFeatureCollection:1.0.0",
+ "description": "A schema like GeoJSON FeatureCollection with a non-WGS 84 CRS context; based on https://geojson.org/schema/FeatureCollection.json. Attention: the coordinate order is fixed: Longitude/Easting/Westing/X first, followed by Latitude/Northing/Southing/Y, optionally height as third coordinate.",
+ "title": "AbstractAnyCrsFeatureCollection",
+ "type": "object",
+ "required": [
+ "type",
+ "persistableReferenceCrs",
+ "features"
+ ],
+ "properties": {
+ "CoordinateReferenceSystemID": {
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-CoordinateReferenceSystem:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The CRS reference into the CoordinateReferenceSystem catalog.",
+ "x-osdu-relationship": [{
+ "EntityType": "CoordinateReferenceSystem",
+ "GroupType": "reference-data"
+ }
+ ],
+ "title": "Coordinate Reference System ID",
+ "type": "string",
+ "example": "namespace:reference-data--CoordinateReferenceSystem:BoundProjected:EPSG::32021_EPSG::15851:"
+ },
+ "persistableReferenceCrs": {
+ "description": "The CRS reference as persistableReference string. If populated, the CoordinateReferenceSystemID takes precedence.",
+ "type": "string",
+ "title": "CRS Reference",
+ "example": "{\"authCode\":{\"auth\":\"OSDU\",\"code\":\"32021079\"},\"lateBoundCRS\":{\"authCode\":{\"auth\":\"EPSG\",\"code\":\"32021\"},\"name\":\"NAD_1927_StatePlane_North_Dakota_South_FIPS_3302\",\"type\":\"LBC\",\"ver\":\"PE_10_9_1\",\"wkt\":\"PROJCS[\\\"NAD_1927_StatePlane_North_Dakota_South_FIPS_3302\\\",GEOGCS[\\\"GCS_North_American_1927\\\",DATUM[\\\"D_North_American_1927\\\",SPHEROID[\\\"Clarke_1866\\\",6378206.4,294.9786982]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],PROJECTION[\\\"Lambert_Conformal_Conic\\\"],PARAMETER[\\\"False_Easting\\\",2000000.0],PARAMETER[\\\"False_Northing\\\",0.0],PARAMETER[\\\"Central_Meridian\\\",-100.5],PARAMETER[\\\"Standard_Parallel_1\\\",46.18333333333333],PARAMETER[\\\"Standard_Parallel_2\\\",47.48333333333333],PARAMETER[\\\"Latitude_Of_Origin\\\",45.66666666666666],UNIT[\\\"Foot_US\\\",0.3048006096012192],AUTHORITY[\\\"EPSG\\\",32021]]\"},\"name\":\"NAD27 * OGP-Usa Conus / North Dakota CS27 South zone [32021,15851]\",\"singleCT\":{\"authCode\":{\"auth\":\"EPSG\",\"code\":\"15851\"},\"name\":\"NAD_1927_To_WGS_1984_79_CONUS\",\"type\":\"ST\",\"ver\":\"PE_10_9_1\",\"wkt\":\"GEOGTRAN[\\\"NAD_1927_To_WGS_1984_79_CONUS\\\",GEOGCS[\\\"GCS_North_American_1927\\\",DATUM[\\\"D_North_American_1927\\\",SPHEROID[\\\"Clarke_1866\\\",6378206.4,294.9786982]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],GEOGCS[\\\"GCS_WGS_1984\\\",DATUM[\\\"D_WGS_1984\\\",SPHEROID[\\\"WGS_1984\\\",6378137.0,298.257223563]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],METHOD[\\\"NADCON\\\"],PARAMETER[\\\"Dataset_conus\\\",0.0],OPERATIONACCURACY[5.0],AUTHORITY[\\\"EPSG\\\",15851]]\"},\"type\":\"EBC\",\"ver\":\"PE_10_9_1\"}"
+ },
+ "features": {
+ "type": "array",
+ "items": {
+ "title": "AnyCrsGeoJSON Feature",
+ "type": "object",
+ "required": [
+ "type",
+ "properties",
+ "geometry"
+ ],
+ "properties": {
+ "geometry": {
+ "oneOf": [{
+ "type": "null"
+ }, {
+ "title": "AnyCrsGeoJSON Point",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsPoint"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "AnyCrsGeoJSON LineString",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsLineString"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "AnyCrsGeoJSON Polygon",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsPolygon"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "AnyCrsGeoJSON MultiPoint",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsMultiPoint"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "AnyCrsGeoJSON MultiLineString",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsMultiLineString"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "AnyCrsGeoJSON MultiPolygon",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "items": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsMultiPolygon"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "AnyCrsGeoJSON GeometryCollection",
+ "type": "object",
+ "required": [
+ "type",
+ "geometries"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsGeometryCollection"
+ ]
+ },
+ "geometries": {
+ "type": "array",
+ "items": {
+ "oneOf": [{
+ "title": "AnyCrsGeoJSON Point",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsPoint"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "AnyCrsGeoJSON LineString",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsLineString"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "AnyCrsGeoJSON Polygon",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsPolygon"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "AnyCrsGeoJSON MultiPoint",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsMultiPoint"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "AnyCrsGeoJSON MultiLineString",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsMultiLineString"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "AnyCrsGeoJSON MultiPolygon",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "items": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsMultiPolygon"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ ]
+ }
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ ]
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsFeature"
+ ]
+ },
+ "properties": {
+ "oneOf": [{
+ "type": "null"
+ }, {
+ "type": "object"
+ }
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ },
+ "persistableReferenceUnitZ": {
+ "description": "The unit of measure for the Z-axis (only for 3-dimensional coordinates, where the CRS does not describe the vertical unit). Note that the direction is upwards positive, i.e. Z means height.",
+ "type": "string",
+ "title": "Z-Unit Reference",
+ "example": "{\"scaleOffset\":{\"scale\":1.0,\"offset\":0.0},\"symbol\":\"m\",\"baseMeasurement\":{\"ancestry\":\"Length\",\"type\":\"UM\"},\"type\":\"USO\"}"
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ },
+ "persistableReferenceVerticalCrs": {
+ "description": "The VerticalCRS reference as persistableReference string. If populated, the VerticalCoordinateReferenceSystemID takes precedence. The property is null or empty for 2D geometries. For 3D geometries and absent or null persistableReferenceVerticalCrs the vertical CRS is either provided via persistableReferenceCrs's CompoundCRS or it is implicitly defined as EPSG:5714 MSL height.",
+ "type": "string",
+ "title": "Vertical CRS Reference",
+ "example": "{\"authCode\":{\"auth\":\"EPSG\",\"code\":\"5714\"},\"name\":\"MSL_Height\",\"type\":\"LBC\",\"ver\":\"PE_10_9_1\",\"wkt\":\"VERTCS[\\\"MSL_Height\\\",VDATUM[\\\"Mean_Sea_Level\\\"],PARAMETER[\\\"Vertical_Shift\\\",0.0],PARAMETER[\\\"Direction\\\",1.0],UNIT[\\\"Meter\\\",1.0],AUTHORITY[\\\"EPSG\\\",5714]]\"}"
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsFeatureCollection"
+ ]
+ },
+ "VerticalCoordinateReferenceSystemID": {
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-CoordinateReferenceSystem:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The explicit VerticalCRS reference into the CoordinateReferenceSystem catalog. This property stays empty for 2D geometries. Absent or empty values for 3D geometries mean the context may be provided by a CompoundCRS in 'CoordinateReferenceSystemID' or implicitly EPSG:5714 MSL height",
+ "x-osdu-relationship": [{
+ "EntityType": "CoordinateReferenceSystem",
+ "GroupType": "reference-data"
+ }
+ ],
+ "title": "Vertical Coordinate Reference System ID",
+ "type": "string",
+ "example": "namespace:reference-data--CoordinateReferenceSystem:Vertical:EPSG::5714:"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractAnyCrsFeatureCollection.1.0.0.json"
+ },
+ "osdu:wks:AbstractProject:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractProject:1.0.0",
+ "description": "A Project is a business activity that consumes financial and human resources and produces (digital) work products.",
+ "x-osdu-review-status": "Accepted",
+ "title": "AbstractProject",
+ "type": "object",
+ "properties": {
+ "Operator": {
+ "x-osdu-relationship": [{
+ "EntityType": "Organisation",
+ "GroupType": "master-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:master-data\\-\\-Organisation:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The organisation which controlled the conduct of the project.",
+ "type": "string"
+ },
+ "ProjectBeginDate": {
+ "format": "date-time",
+ "description": "The date and time when the Project was initiated.",
+ "x-osdu-frame-of-reference": "DateTime",
+ "type": "string"
+ },
+ "ProjectStates": {
+ "x-osdu-indexing": {
+ "type": "flattened"
+ },
+ "description": "The history of life cycle states that the Project has been through..",
+ "type": "array",
+ "items": {
+ "description": "The history of life cycle states that the Project has been through..",
+ "type": "object",
+ "title": "ProjectStates",
+ "properties": {
+ "ProjectStateTypeID": {
+ "x-osdu-relationship": [{
+ "EntityType": "ProjectStateType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-ProjectStateType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The Project life cycle state from planning to completion.",
+ "type": "string"
+ },
+ "TerminationDateTime": {
+ "format": "date-time",
+ "description": "The date and time at which the state is no longer in effect.",
+ "x-osdu-frame-of-reference": "DateTime",
+ "type": "string"
+ },
+ "EffectiveDateTime": {
+ "format": "date-time",
+ "description": "The date and time at which the state becomes effective.",
+ "x-osdu-frame-of-reference": "DateTime",
+ "type": "string"
+ }
+ }
+ }
+ },
+ "ProjectName": {
+ "description": "The common or preferred name of a Project.",
+ "type": "string",
+ "x-osdu-natural-key": 0
+ },
+ "Contractors": {
+ "x-osdu-indexing": {
+ "type": "nested"
+ },
+ "description": "References to organisations which supplied services to the Project.",
+ "type": "array",
+ "items": {
+ "description": "References to organisations which supplied services to the Project.",
+ "type": "object",
+ "title": "Contractors",
+ "properties": {
+ "ContractorCrew": {
+ "description": "Name of the team, unit, crew, party, or other subdivision of the Contractor that provided services.",
+ "type": "string"
+ },
+ "ContractorOrganisationID": {
+ "x-osdu-relationship": [{
+ "EntityType": "Organisation",
+ "GroupType": "master-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:master-data\\-\\-Organisation:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Reference to a company that provided services.",
+ "type": "string"
+ },
+ "ContractorTypeID": {
+ "x-osdu-relationship": [{
+ "EntityType": "ContractorType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-ContractorType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The identifier of a reference value for the role of a contractor providing services, such as Recording, Line Clearing, Positioning, Data Processing.",
+ "type": "string"
+ }
+ }
+ }
+ },
+ "ProjectID": {
+ "description": "Native identifier from a Master Data Management System or other trusted source external to OSDU - stored here in order to allow for multi-system connection and synchronization. If used, the \"Source\" property should identify that source system.",
+ "type": "string",
+ "title": "External Project Identifier"
+ },
+ "Purpose": {
+ "description": "Description of the objectives of a Project.",
+ "type": "string"
+ },
+ "FundsAuthorizations": {
+ "x-osdu-indexing": {
+ "type": "nested"
+ },
+ "description": "The history of expenditure approvals.",
+ "type": "array",
+ "items": {
+ "description": "The history of expenditure approvals.",
+ "type": "object",
+ "title": "FundsAuthorizations",
+ "properties": {
+ "AuthorizationID": {
+ "description": "Internal Company control number which identifies the allocation of funds to the Project.",
+ "type": "string"
+ },
+ "FundsAmount": {
+ "description": "The level of expenditure approved.",
+ "type": "number"
+ },
+ "EffectiveDateTime": {
+ "format": "date-time",
+ "description": "The date and time when the funds were approved.",
+ "x-osdu-frame-of-reference": "DateTime",
+ "type": "string"
+ },
+ "CurrencyID": {
+ "x-osdu-relationship": [{
+ "EntityType": "Currency",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-Currency:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Type of currency for the authorized expenditure.",
+ "type": "string"
+ }
+ }
+ }
+ },
+ "Personnel": {
+ "x-osdu-indexing": {
+ "type": "flattened"
+ },
+ "description": "List of key individuals supporting the Project. This could be Abstracted for re-use, and could reference a separate Persons master data object.",
+ "type": "array",
+ "items": {
+ "description": "List of key individuals supporting the Project. This could be Abstracted for re-use, and could reference a separate Persons master data object.",
+ "type": "object",
+ "title": "Personnel",
+ "properties": {
+ "PersonName": {
+ "description": "Name of an individual supporting the Project.",
+ "type": "string"
+ },
+ "ProjectRoleID": {
+ "x-osdu-relationship": [{
+ "EntityType": "ProjectRole",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-ProjectRole:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The identifier of a reference value for the role of an individual supporting a Project, such as Project Manager, Party Chief, Client Representative, Senior Observer.",
+ "type": "string"
+ },
+ "CompanyOrganisationID": {
+ "x-osdu-relationship": [{
+ "EntityType": "Organisation",
+ "GroupType": "master-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:master-data\\-\\-Organisation:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Reference to the company which employs Personnel.",
+ "type": "string"
+ }
+ }
+ }
+ },
+ "ProjectSpecifications": {
+ "x-osdu-indexing": {
+ "type": "flattened"
+ },
+ "description": "General parameters defining the configuration of the Project. In the case of a seismic acquisition project it is like receiver interval, source depth, source type. In the case of a processing project, it is like replacement velocity, reference datum above mean sea level.",
+ "type": "array",
+ "items": {
+ "description": "General parameters defining the configuration of the Project. In the case of a seismic acquisition project it is like receiver interval, source depth, source type. In the case of a processing project, it is like replacement velocity, reference datum above mean sea level.",
+ "type": "object",
+ "title": "ProjectSpecifications",
+ "properties": {
+ "ProjectSpecificationQuantity": {
+ "description": "The value for the specified parameter type.",
+ "x-osdu-frame-of-reference": "UOM_via_property:UnitOfMeasureID",
+ "type": "number"
+ },
+ "ProjectSpecificationIndicator": {
+ "description": "The actual indicator value of the parameter.",
+ "type": "boolean"
+ },
+ "ProjectSpecificationText": {
+ "description": "The actual text value of the parameter.",
+ "type": "string"
+ },
+ "TerminationDateTime": {
+ "format": "date-time",
+ "description": "The date and time at which a ProjectSpecification is no longer in effect.",
+ "x-osdu-frame-of-reference": "DateTime",
+ "type": "string"
+ },
+ "EffectiveDateTime": {
+ "format": "date-time",
+ "description": "The date and time at which a ProjectSpecification becomes effective.",
+ "x-osdu-frame-of-reference": "DateTime",
+ "type": "string"
+ },
+ "UnitOfMeasureID": {
+ "x-osdu-relationship": [{
+ "EntityType": "UnitOfMeasure",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-UnitOfMeasure:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The unit for the quantity parameter if overriding the default for this ParameterType, like metre (m in SI units system) for quantity Length.",
+ "type": "string"
+ },
+ "ProjectSpecificationDateTime": {
+ "format": "date-time",
+ "description": "The actual date and time value of the parameter. ISO format permits specification of time or date only.",
+ "x-osdu-frame-of-reference": "DateTime",
+ "type": "string"
+ },
+ "ParameterTypeID": {
+ "x-osdu-relationship": [{
+ "EntityType": "ParameterType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-ParameterType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Parameter type of property or characteristic.",
+ "type": "string"
+ }
+ }
+ }
+ },
+ "ProjectEndDate": {
+ "format": "date-time",
+ "description": "The date and time when the Project was completed.",
+ "x-osdu-frame-of-reference": "DateTime",
+ "type": "string"
+ },
+ "ProjectNames": {
+ "description": "DEPRECATED: please use data.NameAliases. The history of Project names, codes, and other business identifiers.",
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/osdu:wks:AbstractAliasNames:1.0.0"
+ }
+ },
+ "ContractIDs": {
+ "description": "References to applicable agreements in external contract database system of record.",
+ "type": "array",
+ "items": {
+ "x-osdu-relationship": [{
+ "EntityType": "Agreement",
+ "GroupType": "master-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:master-data\\-\\-Agreement:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "type": "string"
+ }
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractProject.1.0.0.json"
+ },
+ "osdu:wks:AbstractGeoFieldContext:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractGeoFieldContext:1.0.0",
+ "description": "A single, typed field entity reference, which is 'abstracted' to AbstractGeoContext and then aggregated by GeoContexts properties.",
+ "title": "AbstractGeoFieldContext",
+ "type": "object",
+ "properties": {
+ "FieldID": {
+ "pattern": "^[\\w\\-\\.]+:master-data\\-\\-Field:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Reference to Field.",
+ "x-osdu-relationship": [{
+ "EntityType": "Field",
+ "GroupType": "master-data"
+ }
+ ],
+ "type": "string"
+ },
+ "GeoTypeID": {
+ "const": "Field",
+ "type": "string",
+ "description": "The fixed type 'Field' for this AbstractGeoFieldContext."
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractGeoFieldContext.1.0.0.json"
+ },
+ "osdu:wks:AbstractAliasNames:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractAliasNames:1.0.0",
+ "description": "A list of alternative names for an object. The preferred name is in a separate, scalar property. It may or may not be repeated in the alias list, though a best practice is to include it if the list is present, but to omit the list if there are no other names. Note that the abstract entity is an array so the $ref to it is a simple property reference.",
+ "x-osdu-review-status": "Accepted",
+ "title": "AbstractAliasNames",
+ "type": "object",
+ "properties": {
+ "AliasNameTypeID": {
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-AliasNameType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "A classification of alias names such as by role played or type of source, such as regulatory name, regulatory code, company code, international standard name, etc.",
+ "x-osdu-relationship": [{
+ "EntityType": "AliasNameType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "type": "string"
+ },
+ "EffectiveDateTime": {
+ "format": "date-time",
+ "type": "string",
+ "description": "The date and time when an alias name becomes effective."
+ },
+ "AliasName": {
+ "type": "string",
+ "description": "Alternative Name value of defined name type for an object."
+ },
+ "TerminationDateTime": {
+ "format": "date-time",
+ "type": "string",
+ "description": "The data and time when an alias name is no longer in effect."
+ },
+ "DefinitionOrganisationID": {
+ "pattern": "^[\\w\\-\\.]+:(reference-data\\-\\-StandardsOrganisation|master-data\\-\\-Organisation):[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The StandardsOrganisation (reference-data) or Organisation (master-data) that provided the name (the source).",
+ "x-osdu-relationship": [{
+ "EntityType": "StandardsOrganisation",
+ "GroupType": "reference-data"
+ }, {
+ "EntityType": "Organisation",
+ "GroupType": "master-data"
+ }
+ ],
+ "type": "string"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractAliasNames.1.0.0.json"
+ },
+ "osdu:wks:AbstractGeoProspectContext:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractGeoProspectContext:1.0.0",
+ "description": "A single, typed Prospect entity reference, which is 'abstracted' to AbstractGeoContext and then aggregated by GeoContexts properties.",
+ "x-osdu-review-status": "Accepted",
+ "title": "AbstractGeoProspectContext",
+ "type": "object",
+ "properties": {
+ "ProspectID": {
+ "pattern": "^[\\w\\-\\.]+:master-data\\-\\-Prospect:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Reference to the prospect.",
+ "x-osdu-relationship": [{
+ "EntityType": "Prospect",
+ "GroupType": "master-data"
+ }
+ ],
+ "type": "string"
+ },
+ "GeoTypeID": {
+ "x-osdu-is-derived": {
+ "RelationshipPropertyName": "ProspectID",
+ "TargetPropertyName": "ProspectTypeID"
+ },
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-ProspectType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The ProspectType reference of the Prospect (via ProspectID) for application convenience.",
+ "x-osdu-relationship": [{
+ "EntityType": "ProspectType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "type": "string"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractGeoProspectContext.1.0.0.json"
+ },
+ "osdu:wks:AbstractGeoBasinContext:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractGeoBasinContext:1.0.0",
+ "description": "A single, typed basin entity reference, which is 'abstracted' to AbstractGeoContext and then aggregated by GeoContexts properties.",
+ "x-osdu-review-status": "Accepted",
+ "title": "AbstractGeoBasinContext",
+ "type": "object",
+ "properties": {
+ "BasinID": {
+ "pattern": "^[\\w\\-\\.]+:master-data\\-\\-Basin:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Reference to Basin.",
+ "x-osdu-relationship": [{
+ "EntityType": "Basin",
+ "GroupType": "master-data"
+ }
+ ],
+ "type": "string"
+ },
+ "GeoTypeID": {
+ "x-osdu-is-derived": {
+ "RelationshipPropertyName": "BasinID",
+ "TargetPropertyName": "BasinTypeID"
+ },
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-BasinType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The BasinType reference of the Basin (via BasinID) for application convenience.",
+ "x-osdu-relationship": [{
+ "EntityType": "BasinType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "type": "string"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractGeoBasinContext.1.0.0.json"
+ },
+ "osdu:wks:AbstractSpatialLocation:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractSpatialLocation:1.0.0",
+ "description": "A geographic object which can be described by a set of points.",
+ "title": "AbstractSpatialLocation",
+ "type": "object",
+ "properties": {
+ "AsIngestedCoordinates": {
+ "description": "The original or 'as ingested' coordinates (Point, MultiPoint, LineString, MultiLineString, Polygon or MultiPolygon). The name 'AsIngestedCoordinates' was chosen to contrast it to 'OriginalCoordinates', which carries the uncertainty whether any coordinate operations took place before ingestion. In cases where the original CRS is different from the as-ingested CRS, the AppliedOperations can also contain the list of operations applied to the coordinate prior to ingestion. The data structure is similar to GeoJSON FeatureCollection, however in a CRS context explicitly defined within the AbstractAnyCrsFeatureCollection. The coordinate sequence follows GeoJSON standard, i.e. 'eastward/longitude', 'northward/latitude' {, 'upward/height' unless overridden by an explicit direction in the AsIngestedCoordinates.VerticalCoordinateReferenceSystemID}.",
+ "x-osdu-frame-of-reference": "CRS:",
+ "title": "As Ingested Coordinates",
+ "$ref": "#/definitions/osdu:wks:AbstractAnyCrsFeatureCollection:1.0.0"
+ },
+ "SpatialParameterTypeID": {
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-SpatialParameterType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "A type of spatial representation of an object, often general (e.g. an Outline, which could be applied to Field, Reservoir, Facility, etc.) or sometimes specific (e.g. Onshore Outline, State Offshore Outline, Federal Offshore Outline, 3 spatial representations that may be used by Countries).",
+ "x-osdu-relationship": [{
+ "EntityType": "SpatialParameterType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "type": "string"
+ },
+ "QuantitativeAccuracyBandID": {
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-QuantitativeAccuracyBand:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "An approximate quantitative assessment of the quality of a location (accurate to > 500 m (i.e. not very accurate)), to < 1 m, etc.",
+ "x-osdu-relationship": [{
+ "EntityType": "QuantitativeAccuracyBand",
+ "GroupType": "reference-data"
+ }
+ ],
+ "type": "string"
+ },
+ "CoordinateQualityCheckRemarks": {
+ "type": "array",
+ "description": "Freetext remarks on Quality Check.",
+ "items": {
+ "type": "string"
+ }
+ },
+ "AppliedOperations": {
+ "description": "The audit trail of operations applied to the coordinates from the original state to the current state. The list may contain operations applied prior to ingestion as well as the operations applied to produce the Wgs84Coordinates. The text elements refer to ESRI style CRS and Transformation names, which may have to be translated to EPSG standard names.",
+ "title": "Operations Applied",
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "example": [
+ "conversion from ED_1950_UTM_Zone_31N to GCS_European_1950; 1 points converted",
+ "transformation GCS_European_1950 to GCS_WGS_1984 using ED_1950_To_WGS_1984_24; 1 points successfully transformed"
+ ]
+ },
+ "QualitativeSpatialAccuracyTypeID": {
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-QualitativeSpatialAccuracyType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "A qualitative description of the quality of a spatial location, e.g. unverifiable, not verified, basic validation.",
+ "x-osdu-relationship": [{
+ "EntityType": "QualitativeSpatialAccuracyType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "type": "string"
+ },
+ "CoordinateQualityCheckPerformedBy": {
+ "type": "string",
+ "description": "The user who performed the Quality Check."
+ },
+ "SpatialLocationCoordinatesDate": {
+ "format": "date-time",
+ "description": "Date when coordinates were measured or retrieved.",
+ "x-osdu-frame-of-reference": "DateTime",
+ "type": "string"
+ },
+ "CoordinateQualityCheckDateTime": {
+ "format": "date-time",
+ "description": "The date of the Quality Check.",
+ "x-osdu-frame-of-reference": "DateTime",
+ "type": "string"
+ },
+ "Wgs84Coordinates": {
+ "title": "WGS 84 Coordinates",
+ "description": "The normalized coordinates (Point, MultiPoint, LineString, MultiLineString, Polygon or MultiPolygon) based on WGS 84 (EPSG:4326 for 2-dimensional coordinates, EPSG:4326 + EPSG:5714 (MSL) for 3-dimensional coordinates). This derived coordinate representation is intended for global discoverability only. The schema of this substructure is identical to the GeoJSON FeatureCollection https://geojson.org/schema/FeatureCollection.json. The coordinate sequence follows GeoJSON standard, i.e. longitude, latitude {, height}",
+ "$ref": "#/definitions/osdu:wks:AbstractFeatureCollection:1.0.0"
+ },
+ "SpatialGeometryTypeID": {
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-SpatialGeometryType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Indicates the expected look of the SpatialParameterType, e.g. Point, MultiPoint, LineString, MultiLineString, Polygon, MultiPolygon. The value constrains the type of geometries in the GeoJSON Wgs84Coordinates and AsIngestedCoordinates.",
+ "x-osdu-relationship": [{
+ "EntityType": "SpatialGeometryType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "type": "string"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractSpatialLocation.1.0.0.json"
+ },
+ "osdu:wks:AbstractLegalTags:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractLegalTags:1.0.0",
+ "description": "Legal meta data like legal tags, relevant other countries, legal status. This structure is included by the SystemProperties \"legal\", which is part of all OSDU records. Not extensible.",
+ "additionalProperties": false,
+ "title": "Legal Meta Data",
+ "type": "object",
+ "properties": {
+ "legaltags": {
+ "description": "The list of legal tags, which resolve to legal properties (like country of origin, export classification code, etc.) and rules with the help of the Compliance Service.",
+ "title": "Legal Tags",
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "otherRelevantDataCountries": {
+ "description": "The list of other relevant data countries as an array of two-letter country codes, see https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2.",
+ "title": "Other Relevant Data Countries",
+ "type": "array",
+ "items": {
+ "pattern": "^[A-Z]{2}$",
+ "type": "string"
+ }
+ },
+ "status": {
+ "pattern": "^(compliant|uncompliant)$",
+ "description": "The legal status. Set by the system after evaluation against the compliance rules associated with the \"legaltags\" using the Compliance Service.",
+ "title": "Legal Status",
+ "type": "string"
+ }
+ },
+ "required": [
+ "legaltags",
+ "otherRelevantDataCountries"
+ ],
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractLegalTags.1.0.0.json"
+ },
+ "osdu:wks:AbstractMaster:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractMaster:1.0.0",
+ "description": "Properties shared with all master-data schema instances.",
+ "x-osdu-review-status": "Accepted",
+ "title": "Abstract Master",
+ "type": "object",
+ "properties": {
+ "NameAliases": {
+ "x-osdu-indexing": {
+ "type": "nested"
+ },
+ "description": "Alternative names, including historical, by which this master data is/has been known (it should include all the identifiers).",
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/osdu:wks:AbstractAliasNames:1.0.0"
+ }
+ },
+ "SpatialLocation": {
+ "description": "The spatial location information such as coordinates, CRS information (left empty when not appropriate).",
+ "$ref": "#/definitions/osdu:wks:AbstractSpatialLocation:1.0.0"
+ },
+ "VersionCreationReason": {
+ "description": "This describes the reason that caused the creation of a new version of this master data.",
+ "type": "string"
+ },
+ "TechnicalAssuranceTypeID": {
+ "x-osdu-relationship": [{
+ "EntityType": "TechnicalAssuranceType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-TechnicalAssuranceType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Describes a master-data record's overall suitability for general business consumption based on data quality. Clarifications: Since Certified is the highest classification of suitable quality, any further change or versioning of a Certified record should be carefully considered and justified. If a Technical Assurance value is not populated then one can assume the data has not been evaluated or its quality is unknown (=Unevaluated). Technical Assurance values are not intended to be used for the identification of a single \"preferred\" or \"definitive\" record by comparison with other records.",
+ "type": "string",
+ "title": "Technical Assurance Type ID",
+ "example": "namespace:reference-data--TechnicalAssuranceType:Certified:"
+ },
+ "GeoContexts": {
+ "x-osdu-indexing": {
+ "type": "nested"
+ },
+ "description": "List of geographic entities which provide context to the master data. This may include multiple types or multiple values of the same type.",
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/osdu:wks:AbstractGeoContext:1.0.0"
+ }
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractMaster.1.0.0.json"
+ },
+ "osdu:wks:AbstractGeoPlayContext:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractGeoPlayContext:1.0.0",
+ "description": "A single, typed Play entity reference, which is 'abstracted' to AbstractGeoContext and then aggregated by GeoContexts properties.",
+ "x-osdu-review-status": "Accepted",
+ "title": "AbstractGeoPlayContext",
+ "type": "object",
+ "properties": {
+ "PlayID": {
+ "pattern": "^[\\w\\-\\.]+:master-data\\-\\-Play:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Reference to the play.",
+ "x-osdu-relationship": [{
+ "EntityType": "Play",
+ "GroupType": "master-data"
+ }
+ ],
+ "type": "string"
+ },
+ "GeoTypeID": {
+ "x-osdu-is-derived": {
+ "RelationshipPropertyName": "PlayID",
+ "TargetPropertyName": "PlayTypeID"
+ },
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-PlayType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The PlayType reference of the Play (via PlayID) for application convenience.",
+ "x-osdu-relationship": [{
+ "EntityType": "PlayType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "type": "string"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractGeoPlayContext.1.0.0.json"
+ },
+ "osdu:wks:AbstractGeoPoliticalContext:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractGeoPoliticalContext:1.0.0",
+ "description": "A single, typed geo-political entity reference, which is 'abstracted' to AbstractGeoContext and then aggregated by GeoContexts properties.",
+ "x-osdu-review-status": "Accepted",
+ "title": "AbstractGeoPoliticalContext",
+ "type": "object",
+ "properties": {
+ "GeoPoliticalEntityID": {
+ "pattern": "^[\\w\\-\\.]+:master-data\\-\\-GeoPoliticalEntity:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Reference to GeoPoliticalEntity.",
+ "x-osdu-relationship": [{
+ "EntityType": "GeoPoliticalEntity",
+ "GroupType": "master-data"
+ }
+ ],
+ "type": "string"
+ },
+ "GeoTypeID": {
+ "x-osdu-is-derived": {
+ "RelationshipPropertyName": "GeoPoliticalEntityID",
+ "TargetPropertyName": "GeoPoliticalEntityTypeID"
+ },
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-GeoPoliticalEntityType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The GeoPoliticalEntityType reference of the GeoPoliticalEntity (via GeoPoliticalEntityID) for application convenience.",
+ "x-osdu-relationship": [{
+ "EntityType": "GeoPoliticalEntityType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "type": "string"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractGeoPoliticalContext.1.0.0.json"
+ },
+ "osdu:wks:AbstractAccessControlList:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractAccessControlList:1.0.0",
+ "description": "The access control tags associated with this entity. This structure is included by the SystemProperties \"acl\", which is part of all OSDU records. Not extensible.",
+ "additionalProperties": false,
+ "title": "Access Control List",
+ "type": "object",
+ "properties": {
+ "viewers": {
+ "description": "The list of viewers to which this data record is accessible/visible/discoverable formatted as an email (core.common.model.storage.validation.ValidationDoc.EMAIL_REGEX).",
+ "title": "List of Viewers",
+ "type": "array",
+ "items": {
+ "pattern": "^[a-zA-Z0-9_+&*-]+(?:\\.[a-zA-Z0-9_+&*-]+)*@(?:[a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,7}$",
+ "type": "string"
+ }
+ },
+ "owners": {
+ "description": "The list of owners of this data record formatted as an email (core.common.model.storage.validation.ValidationDoc.EMAIL_REGEX).",
+ "title": "List of Owners",
+ "type": "array",
+ "items": {
+ "pattern": "^[a-zA-Z0-9_+&*-]+(?:\\.[a-zA-Z0-9_+&*-]+)*@(?:[a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,7}$",
+ "type": "string"
+ }
+ }
+ },
+ "required": [
+ "owners",
+ "viewers"
+ ],
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractAccessControlList.1.0.0.json"
+ }
+ },
+ "properties": {
+ "ancestry": {
+ "description": "The links to data, which constitute the inputs, from which this record instance is derived.",
+ "title": "Ancestry",
+ "$ref": "#/definitions/osdu:wks:AbstractLegalParentList:1.0.0"
+ },
+ "data": {
+ "allOf": [{
+ "$ref": "#/definitions/osdu:wks:AbstractCommonResources:1.0.0"
+ }, {
+ "$ref": "#/definitions/osdu:wks:AbstractMaster:1.0.0"
+ }, {
+ "$ref": "#/definitions/osdu:wks:AbstractProject:1.0.0"
+ }, {
+ "type": "object",
+ "title": "IndividualProperties",
+ "properties": {
+ "SourceArrayCount": {
+ "description": "Number of energy sources",
+ "type": "integer"
+ },
+ "ShootingAzimuthAngle": {
+ "description": "Orientation of plane between source and receivers.",
+ "x-osdu-frame-of-reference": "UOM:plane angle",
+ "type": "number"
+ },
+ "MaxOffsetDistance": {
+ "description": "Horizontal distance between source and last receiver",
+ "x-osdu-frame-of-reference": "UOM:length",
+ "type": "number"
+ },
+ "ShotpointIncrementDistance": {
+ "description": "Horizontal distance between shotpoint locations",
+ "x-osdu-frame-of-reference": "UOM:length",
+ "type": "number"
+ },
+ "CableCount": {
+ "description": "Number of receiver arrays (lines)",
+ "type": "integer"
+ },
+ "RecordLength": {
+ "description": "Length of record at time of acquisition",
+ "x-osdu-frame-of-reference": "UOM:time",
+ "type": "number"
+ },
+ "CableLength": {
+ "description": "Total length of receiver array",
+ "x-osdu-frame-of-reference": "UOM:length",
+ "type": "number"
+ },
+ "FoldCount": {
+ "description": "The number of times a point in the subsurface is sampled. It measures of the redundancy of common midpoint seismic data",
+ "type": "integer"
+ },
+ "OperatingEnvironmentID": {
+ "x-osdu-relationship": [{
+ "EntityType": "OperatingEnvironment",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-OperatingEnvironment:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Identifies the setting of acquisition (land, marine, transition zone).",
+ "type": "string"
+ },
+ "SourceArraySeparationDistance": {
+ "description": "Distance between energy Sources",
+ "x-osdu-frame-of-reference": "UOM:length",
+ "type": "number"
+ },
+ "SeismicGeometryTypeID": {
+ "x-osdu-relationship": [{
+ "EntityType": "SeismicGeometryType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-SeismicGeometryType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Reference to the standard values for the general layout of the acquisition. This is an hierarchical value. The top value is like 2D, 3D, 4D, Borehole, Passive. The second value is like NATS, WATS, Brick, Crosswell. Nodes are separated by forward slash.",
+ "type": "string"
+ },
+ "CableSpacingDistance": {
+ "description": "Horizontal distance between receiver arrays",
+ "x-osdu-frame-of-reference": "UOM:length",
+ "type": "number"
+ },
+ "AreaCalculated": {
+ "description": "The calculated are covered by the survey. This value is calculated during the loading of the survey.",
+ "x-osdu-frame-of-reference": "UOM:area",
+ "type": "number"
+ },
+ "AreaNominal": {
+ "description": "The nominal area covered by the survey. This value is usually entered by the end user.",
+ "x-osdu-frame-of-reference": "UOM:area",
+ "type": "number"
+ },
+ "SampleInterval": {
+ "description": "Vertical sampling interval of data at time of acquisition",
+ "x-osdu-frame-of-reference": "UOM:time",
+ "type": "number"
+ },
+ "EnergySourceTypeID": {
+ "x-osdu-relationship": [{
+ "EntityType": "SeismicEnergySourceType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-SeismicEnergySourceType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Seismic Source type. E.g.: Airgun, Vibroseis, Dynamite,Watergun",
+ "type": "string"
+ },
+ "VesselNames": {
+ "description": "List of names of the seismic acquisition (source and streamer) vessels used (marine environment only).",
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "MinOffsetDistance": {
+ "description": "Horizontal distance between source and first receiver",
+ "x-osdu-frame-of-reference": "UOM:length",
+ "type": "number"
+ }
+ }
+ }, {
+ "type": "object",
+ "title": "ExtensionProperties",
+ "properties": {
+ "ExtensionProperties": {
+ "type": "object"
+ }
+ }
+ }
+ ]
+ },
+ "kind": {
+ "pattern": "^[\\w\\-\\.]+:[\\w\\-\\.]+:[\\w\\-\\.]+:[0-9]+.[0-9]+.[0-9]+$",
+ "description": "The schema identification for the OSDU resource object following the pattern {Namespace}:{Source}:{Type}:{VersionMajor}.{VersionMinor}.{VersionPatch}. The versioning scheme follows the semantic versioning, https://semver.org/.",
+ "title": "Entity Kind",
+ "type": "string",
+ "example": "osdu:wks:master-data--SeismicAcquisitionSurvey:1.0.0"
+ },
+ "acl": {
+ "description": "The access control tags associated with this entity.",
+ "title": "Access Control List",
+ "$ref": "#/definitions/osdu:wks:AbstractAccessControlList:1.0.0"
+ },
+ "version": {
+ "format": "int64",
+ "description": "The version number of this OSDU resource; set by the framework.",
+ "title": "Version Number",
+ "type": "integer",
+ "example": 1562066009929332
+ },
+ "tags": {
+ "description": "A generic dictionary of string keys mapping to string value. Only strings are permitted as keys and values.",
+ "additionalProperties": {
+ "type": "string"
+ },
+ "title": "Tag Dictionary",
+ "type": "object",
+ "example": {
+ "NameOfKey": "String value"
+ }
+ },
+ "modifyUser": {
+ "description": "The user reference, which created this version of this resource object. Set by the System.",
+ "title": "Resource Object Version Creation User Reference",
+ "type": "string",
+ "example": "some-user@some-company-cloud.com"
+ },
+ "modifyTime": {
+ "format": "date-time",
+ "description": "Timestamp of the time at which this version of the OSDU resource object was created. Set by the System. The value is a combined date-time string in ISO-8601 given in UTC.",
+ "title": "Resource Object Version Creation DateTime",
+ "type": "string",
+ "example": "2020-12-16T11:52:24.477Z"
+ },
+ "createTime": {
+ "format": "date-time",
+ "description": "Timestamp of the time at which initial version of this OSDU resource object was created. Set by the System. The value is a combined date-time string in ISO-8601 given in UTC.",
+ "title": "Resource Object Creation DateTime",
+ "type": "string",
+ "example": "2020-12-16T11:46:20.163Z"
+ },
+ "meta": {
+ "description": "The Frame of Reference meta data section linking the named properties to self-contained definitions.",
+ "title": "Frame of Reference Meta Data",
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/osdu:wks:AbstractMetaItem:1.0.0"
+ }
+ },
+ "legal": {
+ "description": "The entity's legal tags and compliance status. The actual contents associated with the legal tags is managed by the Compliance Service.",
+ "title": "Legal Tags",
+ "$ref": "#/definitions/osdu:wks:AbstractLegalTags:1.0.0"
+ },
+ "createUser": {
+ "description": "The user reference, which created the first version of this resource object. Set by the System.",
+ "title": "Resource Object Creation User Reference",
+ "type": "string",
+ "example": "some-user@some-company-cloud.com"
+ },
+ "id": {
+ "pattern": "^[\\w\\-\\.]+:master-data\\-\\-SeismicAcquisitionSurvey:[\\w\\-\\.\\:\\%]+$",
+ "description": "Previously called ResourceID or SRN which identifies this OSDU resource object without version.",
+ "title": "Entity ID",
+ "type": "string",
+ "example": "namespace:master-data--SeismicAcquisitionSurvey:35218168-41d4-5e7a-a6d0-958ae8069385"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/master-data/SeismicAcquisitionSurvey.1.0.0.json"
+ }
+}
diff --git a/indexer-acceptance-test/src/test/resources/testData/index_record_sync_mapping.json b/indexer-acceptance-test/src/test/resources/testData/index_record_sync_mapping.json
new file mode 100644
index 0000000000000000000000000000000000000000..5fe3622f256c800f5bf10e55015049a167aba9cb
--- /dev/null
+++ b/indexer-acceptance-test/src/test/resources/testData/index_record_sync_mapping.json
@@ -0,0 +1,10 @@
+[
+ {
+ "id": "tenant1:<kindSubType>:testMappingSync<timestamp>",
+ "data": {
+ "Full Name": "The Open Group",
+ "Address": "Burlington, MA",
+ "Phone": "781-564-9200"
+ }
+ }
+]
\ No newline at end of file
diff --git a/indexer-acceptance-test/src/test/resources/testData/index_record_sync_mapping.mapping b/indexer-acceptance-test/src/test/resources/testData/index_record_sync_mapping.mapping
new file mode 100644
index 0000000000000000000000000000000000000000..59414dc5b119b3268824bf30c438358335018e34
--- /dev/null
+++ b/indexer-acceptance-test/src/test/resources/testData/index_record_sync_mapping.mapping
@@ -0,0 +1,103 @@
+{
+ "dynamic": "false",
+ "properties": {
+ "acl": {
+ "properties": {
+ "owners": {
+ "type": "keyword"
+ },
+ "viewers": {
+ "type": "keyword"
+ }
+ }
+ },
+ "ancestry": {
+ "properties": {
+ "parents": {
+ "type": "keyword"
+ }
+ }
+ },
+ "data": {
+ "properties": {
+ "Full Name": {
+ "type": "text",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "null_value": "null",
+ "ignore_above": 256
+ }
+ }
+ },
+ "Address": {
+ "type": "text",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "null_value": "null",
+ "ignore_above": 256
+ }
+ }
+ },
+ "Phone": {
+ "type": "text",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "null_value": "null",
+ "ignore_above": 256
+ }
+ }
+ }
+ }
+ },
+ "id": {
+ "type": "keyword"
+ },
+ "index": {
+ "properties": {
+ "lastUpdateTime": {
+ "type": "date"
+ },
+ "statusCode": {
+ "type": "integer"
+ },
+ "trace": {
+ "type": "text"
+ }
+ }
+ },
+ "kind": {
+ "type": "keyword"
+ },
+ "legal": {
+ "properties": {
+ "legaltags": {
+ "type": "keyword"
+ },
+ "otherRelevantDataCountries": {
+ "type": "keyword"
+ },
+ "status": {
+ "type": "keyword"
+ }
+ }
+ },
+ "namespace": {
+ "type": "keyword"
+ },
+ "tags": {
+ "type": "flattened"
+ },
+ "type": {
+ "type": "keyword"
+ },
+ "version": {
+ "type": "long"
+ },
+ "x-acl": {
+ "type": "keyword"
+ }
+ }
+}
\ No newline at end of file
diff --git a/indexer-acceptance-test/src/test/resources/testData/index_record_sync_mapping.schema.json b/indexer-acceptance-test/src/test/resources/testData/index_record_sync_mapping.schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..b86176627765584c455768d3acd47fcfdb4995cc
--- /dev/null
+++ b/indexer-acceptance-test/src/test/resources/testData/index_record_sync_mapping.schema.json
@@ -0,0 +1,35 @@
+{
+ "schemaInfo": {
+ "schemaIdentity": {
+ "authority": "tenant1",
+ "source": "indexer",
+ "entityType": "test-mapping--Sync",
+ "schemaVersionMajor": "2",
+ "schemaVersionMinor": "0",
+ "schemaVersionPatch": "0"
+ },
+ "status": "DEVELOPMENT"
+ },
+ "schema": {
+ "properties": {
+ "data": {
+ "allOf": [
+ {
+ "type": "object",
+ "properties": {
+ "FullName": {
+ "type": "string"
+ },
+ "Address": {
+ "type": "string"
+ },
+ "Phone": {
+ "type": "string"
+ }
+ }
+ }
+ ]
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/indexer-acceptance-test/src/test/resources/testData/index_record_virtual_properties.json b/indexer-acceptance-test/src/test/resources/testData/index_record_virtual_properties.json
new file mode 100644
index 0000000000000000000000000000000000000000..ec9d18fb56c16e46efac88f7bcaa8a559d908c95
--- /dev/null
+++ b/indexer-acceptance-test/src/test/resources/testData/index_record_virtual_properties.json
@@ -0,0 +1,240 @@
+[
+ {
+ "id": "tenant1:<kindSubType>:testIngest1<timestamp>",
+ "data": {
+ "FacilityID": "IntegrationTest-27869678942074",
+ "FacilityTypeID": "osdu:reference-data--FacilityType:Wellbore:",
+ "CurrentOperatorID": "osdu:master-data--Organisation:Renee%20Reyes:",
+ "FacilityName": "N17",
+ "FacilityEvents": [{
+ "FacilityEventTypeID": "osdu:reference-data--FacilityEventType:Spud:",
+ "EffectiveDateTime": "1976-10-21T00:00:00"
+ }
+ ],
+ "ProjectedBottomHoleLocation": {
+ "AsIngestedCoordinates": {
+ "type": "AnyCrsFeatureCollection",
+ "CoordinateReferenceSystemID": "osdu:reference-data--CoordinateReferenceSystem:WGS_1984_World_Mercator:",
+ "persistableReferenceCrs": "{\"authCode\":{\"auth\":\"EPSG\",\"code\":\"3395\"},\"type\":\"LBC\",\"ver\":\"PE_10_3_1\",\"name\":\"WGS_1984_World_Mercator\",\"wkt\":\"PROJCS[\\\"WGS_1984_World_Mercator\\\",GEOGCS[\\\"GCS_WGS_1984\\\",DATUM[\\\"D_WGS_1984\\\",SPHEROID[\\\"WGS_1984\\\",6378137.0,298.257223563]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],PROJECTION[\\\"Mercator\\\"],PARAMETER[\\\"False_Easting\\\",0.0],PARAMETER[\\\"False_Northing\\\",0.0],PARAMETER[\\\"Central_Meridian\\\",0.0],PARAMETER[\\\"Standard_Parallel_1\\\",0.0],UNIT[\\\"Meter\\\",1.0],AUTHORITY[\\\"EPSG\\\",3395]]\"}",
+ "persistableReferenceUnitZ": "ft",
+ "features": [{
+ "type": "AnyCrsFeature",
+ "geometry": {
+ "type": "AnyCrsPoint",
+ "coordinates": [
+ 2504888.13869565,
+ -3525752.63921785,
+ 13
+ ]
+ }
+ }
+ ]
+ },
+ "Wgs84Coordinates": {
+ "type": "FeatureCollection",
+ "features": [{
+ "type": "Feature",
+ "geometry": {
+ "type": "Point",
+ "coordinates": [
+ 22.501793,
+ -30.34003
+ ]
+ }
+ }
+ ]
+ },
+ "SpatialGeometryTypeID": "osdu:reference-data--SpatialGeometryType:Point:"
+ },
+ "GeographicBottomHoleLocation": {
+ "AsIngestedCoordinates": {
+ "type": "AnyCrsFeatureCollection",
+ "CoordinateReferenceSystemID": "osdu:reference-data--CoordinateReferenceSystem:WGS_1984_World_Mercator:",
+ "persistableReferenceCrs": "{\"authCode\":{\"auth\":\"EPSG\",\"code\":\"3395\"},\"type\":\"LBC\",\"ver\":\"PE_10_3_1\",\"name\":\"WGS_1984_World_Mercator\",\"wkt\":\"PROJCS[\\\"WGS_1984_World_Mercator\\\",GEOGCS[\\\"GCS_WGS_1984\\\",DATUM[\\\"D_WGS_1984\\\",SPHEROID[\\\"WGS_1984\\\",6378137.0,298.257223563]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],PROJECTION[\\\"Mercator\\\"],PARAMETER[\\\"False_Easting\\\",0.0],PARAMETER[\\\"False_Northing\\\",0.0],PARAMETER[\\\"Central_Meridian\\\",0.0],PARAMETER[\\\"Standard_Parallel_1\\\",0.0],UNIT[\\\"Meter\\\",1.0],AUTHORITY[\\\"EPSG\\\",3395]]\"}",
+ "persistableReferenceUnitZ": "ft",
+ "features": [{
+ "type": "AnyCrsFeature",
+ "geometry": {
+ "type": "AnyCrsPoint",
+ "coordinates": [
+ 2504888.13869565,
+ -3525752.63921785,
+ 13
+ ]
+ }
+ }
+ ]
+ },
+ "Wgs84Coordinates": {
+ "type": "FeatureCollection",
+ "features": [{
+ "type": "Feature",
+ "geometry": {
+ "type": "Point",
+ "coordinates": [
+ 22.501793,
+ -30.34003
+ ]
+ }
+ }
+ ]
+ },
+ "SpatialGeometryTypeID": "osdu:reference-data--SpatialGeometryType:Point:"
+ },
+ "SpatialLocation": {
+ "AsIngestedCoordinates": {
+ "type": "AnyCrsFeatureCollection",
+ "CoordinateReferenceSystemID": "osdu:reference-data--CoordinateReferenceSystem:WGS_1984_World_Mercator:",
+ "persistableReferenceCrs": "{\"authCode\":{\"auth\":\"EPSG\",\"code\":\"3395\"},\"type\":\"LBC\",\"ver\":\"PE_10_3_1\",\"name\":\"WGS_1984_World_Mercator\",\"wkt\":\"PROJCS[\\\"WGS_1984_World_Mercator\\\",GEOGCS[\\\"GCS_WGS_1984\\\",DATUM[\\\"D_WGS_1984\\\",SPHEROID[\\\"WGS_1984\\\",6378137.0,298.257223563]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],PROJECTION[\\\"Mercator\\\"],PARAMETER[\\\"False_Easting\\\",0.0],PARAMETER[\\\"False_Northing\\\",0.0],PARAMETER[\\\"Central_Meridian\\\",0.0],PARAMETER[\\\"Standard_Parallel_1\\\",0.0],UNIT[\\\"Meter\\\",1.0],AUTHORITY[\\\"EPSG\\\",3395]]\"}",
+ "persistableReferenceUnitZ": "ft",
+ "features": [{
+ "type": "AnyCrsFeature",
+ "geometry": {
+ "type": "AnyCrsPoint",
+ "coordinates": [
+ 2504888.13869565,
+ -3525752.63921785,
+ 13
+ ]
+ }
+ }
+ ]
+ },
+ "Wgs84Coordinates": {
+ "type": "FeatureCollection",
+ "features": [{
+ "type": "Feature",
+ "geometry": {
+ "type": "Point",
+ "coordinates": [
+ 22.501793,
+ -30.34003
+ ]
+ }
+ }
+ ]
+ },
+ "SpatialGeometryTypeID": "osdu:reference-data--SpatialGeometryType:Point:"
+ }
+ }
+ },
+ {
+ "id": "tenant1:<kindSubType>:testIngest2<timestamp>",
+ "data": {
+ "FacilityID": "IntegrationTest-27869678942075",
+ "FacilityTypeID": "osdu:reference-data--FacilityType:Wellbore:",
+ "CurrentOperatorID": "osdu:master-data--Organisation:Renee%20Reyes:",
+ "FacilityName": "N18",
+ "FacilityEvents": [{
+ "FacilityEventTypeID": "osdu:reference-data--FacilityEventType:Spud:",
+ "EffectiveDateTime": "1976-10-21T00:00:00"
+ }
+ ],
+ "GeographicBottomHoleLocation": {
+ "AsIngestedCoordinates": {
+ "type": "AnyCrsFeatureCollection",
+ "CoordinateReferenceSystemID": "osdu:reference-data--CoordinateReferenceSystem:WGS_1984_World_Mercator:",
+ "persistableReferenceCrs": "{\"authCode\":{\"auth\":\"EPSG\",\"code\":\"3395\"},\"type\":\"LBC\",\"ver\":\"PE_10_3_1\",\"name\":\"WGS_1984_World_Mercator\",\"wkt\":\"PROJCS[\\\"WGS_1984_World_Mercator\\\",GEOGCS[\\\"GCS_WGS_1984\\\",DATUM[\\\"D_WGS_1984\\\",SPHEROID[\\\"WGS_1984\\\",6378137.0,298.257223563]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],PROJECTION[\\\"Mercator\\\"],PARAMETER[\\\"False_Easting\\\",0.0],PARAMETER[\\\"False_Northing\\\",0.0],PARAMETER[\\\"Central_Meridian\\\",0.0],PARAMETER[\\\"Standard_Parallel_1\\\",0.0],UNIT[\\\"Meter\\\",1.0],AUTHORITY[\\\"EPSG\\\",3395]]\"}",
+ "persistableReferenceUnitZ": "ft",
+ "features": [{
+ "type": "AnyCrsFeature",
+ "geometry": {
+ "type": "AnyCrsPoint",
+ "coordinates": [
+ 2504888.13869565,
+ -3525752.63921785,
+ 13
+ ]
+ }
+ }
+ ]
+ },
+ "SpatialGeometryTypeID": "osdu:reference-data--SpatialGeometryType:Point:"
+ },
+ "SpatialLocation": {
+ "AsIngestedCoordinates": {
+ "type": "AnyCrsFeatureCollection",
+ "CoordinateReferenceSystemID": "osdu:reference-data--CoordinateReferenceSystem:WGS_1984_World_Mercator:",
+ "persistableReferenceCrs": "{\"authCode\":{\"auth\":\"EPSG\",\"code\":\"3395\"},\"type\":\"LBC\",\"ver\":\"PE_10_3_1\",\"name\":\"WGS_1984_World_Mercator\",\"wkt\":\"PROJCS[\\\"WGS_1984_World_Mercator\\\",GEOGCS[\\\"GCS_WGS_1984\\\",DATUM[\\\"D_WGS_1984\\\",SPHEROID[\\\"WGS_1984\\\",6378137.0,298.257223563]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],PROJECTION[\\\"Mercator\\\"],PARAMETER[\\\"False_Easting\\\",0.0],PARAMETER[\\\"False_Northing\\\",0.0],PARAMETER[\\\"Central_Meridian\\\",0.0],PARAMETER[\\\"Standard_Parallel_1\\\",0.0],UNIT[\\\"Meter\\\",1.0],AUTHORITY[\\\"EPSG\\\",3395]]\"}",
+ "persistableReferenceUnitZ": "ft",
+ "features": [{
+ "type": "AnyCrsFeature",
+ "geometry": {
+ "type": "AnyCrsPoint",
+ "coordinates": [
+ 2504888.13869565,
+ -3525752.63921785,
+ 13
+ ]
+ }
+ }
+ ]
+ },
+ "Wgs84Coordinates": {
+ "type": "FeatureCollection",
+ "features": [{
+ "type": "Feature",
+ "geometry": {
+ "type": "Point",
+ "coordinates": [
+ 22.501793,
+ -30.34003
+ ]
+ }
+ }
+ ]
+ },
+ "SpatialGeometryTypeID": "osdu:reference-data--SpatialGeometryType:Point:"
+ }
+ }
+ },
+ {
+ "id": "tenant1:<kindSubType>:testIngest3<timestamp>",
+ "data": {
+ "FacilityID": "IntegrationTest-27869678942076",
+ "FacilityTypeID": "osdu:reference-data--FacilityType:Wellbore:",
+ "CurrentOperatorID": "osdu:master-data--Organisation:Renee%20Reyes:",
+ "FacilityName": "N19",
+ "FacilityEvents": [{
+ "FacilityEventTypeID": "osdu:reference-data--FacilityEventType:Spud:",
+ "EffectiveDateTime": "1976-10-21T00:00:00"
+ }
+ ],
+ "SpatialLocation": {
+ "AsIngestedCoordinates": {
+ "type": "AnyCrsFeatureCollection",
+ "CoordinateReferenceSystemID": "osdu:reference-data--CoordinateReferenceSystem:WGS_1984_World_Mercator:",
+ "persistableReferenceCrs": "{\"authCode\":{\"auth\":\"EPSG\",\"code\":\"3395\"},\"type\":\"LBC\",\"ver\":\"PE_10_3_1\",\"name\":\"WGS_1984_World_Mercator\",\"wkt\":\"PROJCS[\\\"WGS_1984_World_Mercator\\\",GEOGCS[\\\"GCS_WGS_1984\\\",DATUM[\\\"D_WGS_1984\\\",SPHEROID[\\\"WGS_1984\\\",6378137.0,298.257223563]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],PROJECTION[\\\"Mercator\\\"],PARAMETER[\\\"False_Easting\\\",0.0],PARAMETER[\\\"False_Northing\\\",0.0],PARAMETER[\\\"Central_Meridian\\\",0.0],PARAMETER[\\\"Standard_Parallel_1\\\",0.0],UNIT[\\\"Meter\\\",1.0],AUTHORITY[\\\"EPSG\\\",3395]]\"}",
+ "persistableReferenceUnitZ": "ft",
+ "features": [{
+ "type": "AnyCrsFeature",
+ "geometry": {
+ "type": "AnyCrsPoint",
+ "coordinates": [
+ 2504888.13869565,
+ -3525752.63921785,
+ 13
+ ]
+ }
+ }
+ ]
+ },
+ "Wgs84Coordinates": {
+ "type": "FeatureCollection",
+ "features": [{
+ "type": "Feature",
+ "geometry": {
+ "type": "Point",
+ "coordinates": [
+ 22.501793,
+ -30.34003
+ ]
+ }
+ }
+ ]
+ },
+ "SpatialGeometryTypeID": "osdu:reference-data--SpatialGeometryType:Point:"
+ }
+ }
+ }
+]
diff --git a/indexer-acceptance-test/src/test/resources/testData/index_record_virtual_properties.schema.json b/indexer-acceptance-test/src/test/resources/testData/index_record_virtual_properties.schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..50983e17f6e414cb1642cbc96be4bcfcdec9c92c
--- /dev/null
+++ b/indexer-acceptance-test/src/test/resources/testData/index_record_virtual_properties.schema.json
@@ -0,0 +1,2509 @@
+{
+ "schemaInfo": {
+ "schemaIdentity": {
+ "authority": "tenant1",
+ "source": "indexer",
+ "entityType": "virtual-properties-Integration",
+ "schemaVersionMajor": "1",
+ "schemaVersionMinor": "0",
+ "schemaVersionPatch": "0"
+ },
+ "status": "DEVELOPMENT"
+ },
+ "schema": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:master-data--Test:1.0.0",
+ "description": "A hole in the ground extending from a point at the earth's surface to the maximum point of penetration.",
+ "title": "Wellbore",
+ "type": "object",
+ "required": [
+ "kind",
+ "acl",
+ "legal"
+ ],
+ "x-osdu-virtual-properties": {
+ "data.VirtualProperties.DefaultLocation": {
+ "type": "object",
+ "priority": [{
+ "path": "data.ProjectedBottomHoleLocation"
+ }, {
+ "path": "data.GeographicBottomHoleLocation"
+ }, {
+ "path": "data.SpatialLocation"
+ }
+ ]
+ },
+ "data.VirtualProperties.DefaultName": {
+ "type": "string",
+ "priority": [{
+ "path": "data.FacilityName"
+ }
+ ]
+ }
+ },
+ "additionalProperties": false,
+ "definitions": {
+ "osdu:wks:AbstractCommonResources:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractCommonResources:1.0.0",
+ "description": "Common resources to be injected at root 'data' level for every entity, which is persistable in Storage. The insertion is performed by the OsduSchemaComposer script.",
+ "x-osdu-review-status": "Accepted",
+ "title": "OSDU Common Resources",
+ "type": "object",
+ "properties": {
+ "ResourceHomeRegionID": {
+ "x-osdu-relationship": [{
+ "EntityType": "OSDURegion",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-OSDURegion:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The name of the home [cloud environment] region for this OSDU resource object.",
+ "type": "string",
+ "title": "Resource Home Region ID"
+ },
+ "ResourceHostRegionIDs": {
+ "description": "The name of the host [cloud environment] region(s) for this OSDU resource object.",
+ "type": "array",
+ "title": "Resource Host Region ID",
+ "items": {
+ "x-osdu-relationship": [{
+ "EntityType": "OSDURegion",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-OSDURegion:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "type": "string"
+ }
+ },
+ "ResourceLifecycleStatus": {
+ "x-osdu-relationship": [{
+ "EntityType": "ResourceLifecycleStatus",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-ResourceLifecycleStatus:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Describes the current Resource Lifecycle status.",
+ "type": "string",
+ "title": "Resource Lifecycle Status"
+ },
+ "ResourceSecurityClassification": {
+ "x-osdu-relationship": [{
+ "EntityType": "ResourceSecurityClassification",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-ResourceSecurityClassification:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Classifies the security level of the resource.",
+ "type": "string",
+ "title": "Resource Security Classification"
+ },
+ "ResourceCurationStatus": {
+ "x-osdu-relationship": [{
+ "EntityType": "ResourceCurationStatus",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-ResourceCurationStatus:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Describes the current Curation status.",
+ "type": "string",
+ "title": "Resource Curation Status"
+ },
+ "ExistenceKind": {
+ "x-osdu-relationship": [{
+ "EntityType": "ExistenceKind",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-ExistenceKind:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Where does this data resource sit in the cradle-to-grave span of its existence?",
+ "type": "string",
+ "title": "Existence Kind"
+ },
+ "TechnicalAssuranceID": {
+ "x-osdu-relationship": [{
+ "EntityType": "TechnicalAssuranceType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-TechnicalAssuranceType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Describes a record's overall suitability for general business consumption based on data quality. Clarifications: Since Certified is the highest classification of suitable quality, any further change or versioning of a Certified record should be carefully considered and justified. If a Technical Assurance value is not populated then one can assume the data has not been evaluated or its quality is unknown (=Unevaluated). Technical Assurance values are not intended to be used for the identification of a single \"preferred\" or \"definitive\" record by comparison with other records.",
+ "type": "string",
+ "title": "Technical Assurance ID"
+ },
+ "Source": {
+ "description": "The entity that produced the record, or from which it is received; could be an organization, agency, system, internal team, or individual. For informational purposes only, the list of sources is not governed.",
+ "type": "string",
+ "title": "Data Source"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractCommonResources.1.0.0.json"
+ },
+ "osdu:wks:AbstractMetaItem:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "oneOf": [{
+ "title": "FrameOfReferenceUOM",
+ "type": "object",
+ "properties": {
+ "persistableReference": {
+ "description": "The self-contained, persistable reference string uniquely identifying the Unit.",
+ "title": "UOM Persistable Reference",
+ "type": "string",
+ "example": "{\"abcd\":{\"a\":0.0,\"b\":1200.0,\"c\":3937.0,\"d\":0.0},\"symbol\":\"ft[US]\",\"baseMeasurement\":{\"ancestry\":\"L\",\"type\":\"UM\"},\"type\":\"UAD\"}"
+ },
+ "kind": {
+ "const": "Unit",
+ "description": "The kind of reference, 'Unit' for FrameOfReferenceUOM.",
+ "title": "UOM Reference Kind"
+ },
+ "propertyNames": {
+ "description": "The list of property names, to which this meta data item provides Unit context to. A full path like \"StructureA.PropertyB\" is required to define a unique context; \"data\" is omitted since frame-of reference normalization only applies to the data block.",
+ "title": "UOM Property Names",
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "example": [
+ "HorizontalDeflection.EastWest",
+ "HorizontalDeflection.NorthSouth"
+ ]
+ },
+ "name": {
+ "description": "The unit symbol or name of the unit.",
+ "title": "UOM Unit Symbol",
+ "type": "string",
+ "example": "ft[US]"
+ },
+ "unitOfMeasureID": {
+ "x-osdu-relationship": [{
+ "EntityType": "UnitOfMeasure",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-UnitOfMeasure:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "SRN to unit of measure reference.",
+ "type": "string",
+ "example": "namespace:reference-data--UnitOfMeasure:ftUS:"
+ }
+ },
+ "required": [
+ "kind",
+ "persistableReference"
+ ]
+ }, {
+ "title": "FrameOfReferenceCRS",
+ "type": "object",
+ "properties": {
+ "coordinateReferenceSystemID": {
+ "x-osdu-relationship": [{
+ "EntityType": "CoordinateReferenceSystem",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-CoordinateReferenceSystem:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "SRN to CRS reference.",
+ "type": "string",
+ "example": "namespace:reference-data--CoordinateReferenceSystem:Projected:EPSG::32615:"
+ },
+ "persistableReference": {
+ "description": "The self-contained, persistable reference string uniquely identifying the CRS.",
+ "title": "CRS Persistable Reference",
+ "type": "string",
+ "example": "{\"authCode\":{\"auth\":\"EPSG\",\"code\":\"32615\"},\"name\":\"WGS_1984_UTM_Zone_15N\",\"type\":\"LBC\",\"ver\":\"PE_10_9_1\",\"wkt\":\"PROJCS[\\\"WGS_1984_UTM_Zone_15N\\\",GEOGCS[\\\"GCS_WGS_1984\\\",DATUM[\\\"D_WGS_1984\\\",SPHEROID[\\\"WGS_1984\\\",6378137.0,298.257223563]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],PROJECTION[\\\"Transverse_Mercator\\\"],PARAMETER[\\\"False_Easting\\\",500000.0],PARAMETER[\\\"False_Northing\\\",0.0],PARAMETER[\\\"Central_Meridian\\\",-93.0],PARAMETER[\\\"Scale_Factor\\\",0.9996],PARAMETER[\\\"Latitude_Of_Origin\\\",0.0],UNIT[\\\"Meter\\\",1.0],AUTHORITY[\\\"EPSG\\\",32615]]\"}"
+ },
+ "kind": {
+ "const": "CRS",
+ "description": "The kind of reference, constant 'CRS' for FrameOfReferenceCRS.",
+ "title": "CRS Reference Kind"
+ },
+ "propertyNames": {
+ "description": "The list of property names, to which this meta data item provides CRS context to. A full path like \"StructureA.PropertyB\" is required to define a unique context; \"data\" is omitted since frame-of reference normalization only applies to the data block.",
+ "title": "CRS Property Names",
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "example": [
+ "KickOffPosition.X",
+ "KickOffPosition.Y"
+ ]
+ },
+ "name": {
+ "description": "The name of the CRS.",
+ "title": "CRS Name",
+ "type": "string",
+ "example": "WGS 84 / UTM zone 15N"
+ }
+ },
+ "required": [
+ "kind",
+ "persistableReference"
+ ]
+ }, {
+ "title": "FrameOfReferenceDateTime",
+ "type": "object",
+ "properties": {
+ "persistableReference": {
+ "description": "The self-contained, persistable reference string uniquely identifying DateTime reference.",
+ "title": "DateTime Persistable Reference",
+ "type": "string",
+ "example": "{\"format\":\"yyyy-MM-ddTHH:mm:ssZ\",\"timeZone\":\"UTC\",\"type\":\"DTM\"}"
+ },
+ "kind": {
+ "const": "DateTime",
+ "description": "The kind of reference, constant 'DateTime', for FrameOfReferenceDateTime.",
+ "title": "DateTime Reference Kind"
+ },
+ "propertyNames": {
+ "description": "The list of property names, to which this meta data item provides DateTime context to. A full path like \"StructureA.PropertyB\" is required to define a unique context; \"data\" is omitted since frame-of reference normalization only applies to the data block.",
+ "title": "DateTime Property Names",
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "example": [
+ "Acquisition.StartTime",
+ "Acquisition.EndTime"
+ ]
+ },
+ "name": {
+ "description": "The name of the DateTime format and reference.",
+ "title": "DateTime Name",
+ "type": "string",
+ "example": "UTC"
+ }
+ },
+ "required": [
+ "kind",
+ "persistableReference"
+ ]
+ }, {
+ "title": "FrameOfReferenceAzimuthReference",
+ "type": "object",
+ "properties": {
+ "persistableReference": {
+ "description": "The self-contained, persistable reference string uniquely identifying AzimuthReference.",
+ "title": "AzimuthReference Persistable Reference",
+ "type": "string",
+ "example": "{\"code\":\"TrueNorth\",\"type\":\"AZR\"}"
+ },
+ "kind": {
+ "const": "AzimuthReference",
+ "description": "The kind of reference, constant 'AzimuthReference', for FrameOfReferenceAzimuthReference.",
+ "title": "AzimuthReference Reference Kind"
+ },
+ "propertyNames": {
+ "description": "The list of property names, to which this meta data item provides AzimuthReference context to. A full path like \"StructureA.PropertyB\" is required to define a unique context; \"data\" is omitted since frame-of reference normalization only applies to the data block.",
+ "title": "AzimuthReference Property Names",
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "example": [
+ "Bearing"
+ ]
+ },
+ "name": {
+ "description": "The name of the CRS or the symbol/name of the unit.",
+ "title": "AzimuthReference Name",
+ "type": "string",
+ "example": "TrueNorth"
+ }
+ },
+ "required": [
+ "kind",
+ "persistableReference"
+ ]
+ }
+ ],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractMetaItem:1.0.0",
+ "description": "A meta data item, which allows the association of named properties or property values to a Unit/Measurement/CRS/Azimuth/Time context.",
+ "title": "Frame of Reference Meta Data Item",
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractMetaItem.1.0.0.json"
+ },
+ "osdu:wks:AbstractLegalParentList:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractLegalParentList:1.0.0",
+ "description": "A list of entity id:version references to record instances recorded in the data platform, from which the current record is derived and from which the legal tags must be derived. This structure is included by the SystemProperties \"ancestry\", which is part of all OSDU records. Not extensible.",
+ "additionalProperties": false,
+ "title": "Parent List",
+ "type": "object",
+ "properties": {
+ "parents": {
+ "description": "An array of none, one or many entity references of 'direct parents' in the data platform, which mark the current record as a derivative. In contrast to other relationships, the source record version is required. During record creation or update the ancestry.parents[] relationships are used to collect the legal tags from the sources and aggregate them in the legal.legaltags[] array. As a consequence, should e.g., one or more of the legal tags of the source data expire, the access to the derivatives is also terminated. For details, see ComplianceService tutorial, 'Creating derivative Records'.",
+ "title": "Parents",
+ "type": "array",
+ "items": {
+ "x-osdu-relationship": [],
+ "pattern": "^[\\w\\-\\.]+:[\\w\\-\\.]+:[\\w\\-\\.\\:\\%]+:[0-9]+$",
+ "type": "string"
+ },
+ "example": []
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractLegalParentList.1.0.0.json"
+ },
+ "osdu:wks:AbstractGeoContext:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "oneOf": [{
+ "$ref": "#/definitions/osdu:wks:AbstractGeoPoliticalContext:1.0.0"
+ }, {
+ "$ref": "#/definitions/osdu:wks:AbstractGeoBasinContext:1.0.0"
+ }, {
+ "$ref": "#/definitions/osdu:wks:AbstractGeoFieldContext:1.0.0"
+ }, {
+ "$ref": "#/definitions/osdu:wks:AbstractGeoPlayContext:1.0.0"
+ }, {
+ "$ref": "#/definitions/osdu:wks:AbstractGeoProspectContext:1.0.0"
+ }
+ ],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractGeoContext:1.0.0",
+ "description": "A geographic context to an entity. It can be either a reference to a GeoPoliticalEntity, Basin, Field, Play or Prospect.",
+ "title": "AbstractGeoContext",
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractGeoContext.1.0.0.json"
+ },
+ "osdu:wks:AbstractFeatureCollection:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractFeatureCollection:1.0.0",
+ "description": "GeoJSON feature collection as originally published in https://geojson.org/schema/FeatureCollection.json. Attention: the coordinate order is fixed: Longitude first, followed by Latitude, optionally height above MSL (EPSG:5714) as third coordinate.",
+ "title": "GeoJSON FeatureCollection",
+ "type": "object",
+ "required": [
+ "type",
+ "features"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "FeatureCollection"
+ ]
+ },
+ "features": {
+ "type": "array",
+ "items": {
+ "title": "GeoJSON Feature",
+ "type": "object",
+ "required": [
+ "type",
+ "properties",
+ "geometry"
+ ],
+ "properties": {
+ "geometry": {
+ "oneOf": [{
+ "type": "null"
+ }, {
+ "title": "GeoJSON Point",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "Point"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "GeoJSON LineString",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "LineString"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "GeoJSON Polygon",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "Polygon"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "GeoJSON MultiPoint",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "MultiPoint"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "GeoJSON MultiLineString",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "MultiLineString"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "GeoJSON MultiPolygon",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "items": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "MultiPolygon"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "GeoJSON GeometryCollection",
+ "type": "object",
+ "required": [
+ "type",
+ "geometries"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "GeometryCollection"
+ ]
+ },
+ "geometries": {
+ "type": "array",
+ "items": {
+ "oneOf": [{
+ "title": "GeoJSON Point",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "Point"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "GeoJSON LineString",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "LineString"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "GeoJSON Polygon",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "Polygon"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "GeoJSON MultiPoint",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "MultiPoint"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "GeoJSON MultiLineString",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "MultiLineString"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "GeoJSON MultiPolygon",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "items": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "MultiPolygon"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ ]
+ }
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ ]
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "Feature"
+ ]
+ },
+ "properties": {
+ "oneOf": [{
+ "type": "null"
+ }, {
+ "type": "object"
+ }
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractFeatureCollection.1.0.0.json"
+ },
+ "osdu:wks:AbstractFacility:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractFacility:1.0.0",
+ "description": "The schema fragment included by facilities. A facility is a grouping of equipment that is located within a specific geographic boundary or site and that is used in the context of energy-related activities such as exploration, extraction, generation, storage, processing, disposal, supply, or transfer. Clarifications: (1) A facility may be surface or subsurface located. (2) Usually facility equipment is commonly owned or operated. (3) Industry definitions may vary and differ from this one. This schema fragment is included by Well, Wellbore, Rig, as well as Tank Batteries, Compression Stations, Storage Facilities, Wind Farms, Wind Turbines, Mining Facilities, etc., once these types are included in to the OSDU.",
+ "x-osdu-review-status": "Accepted",
+ "title": "AbstractFacility",
+ "type": "object",
+ "properties": {
+ "FacilityStates": {
+ "x-osdu-indexing": {
+ "type": "nested"
+ },
+ "description": "The history of life cycle states the facility has been through.",
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/osdu:wks:AbstractFacilityState:1.0.0"
+ }
+ },
+ "FacilityID": {
+ "description": "Native identifier from a Master Data Management System or other trusted source external to OSDU - stored here in order to allow for multi-system connection and synchronization. If used, the \"Source\" property should identify that source system.",
+ "type": "string",
+ "title": "External Facility Identifier"
+ },
+ "OperatingEnvironmentID": {
+ "x-osdu-relationship": [{
+ "EntityType": "OperatingEnvironment",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-OperatingEnvironment:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Identifies the Facility's general location as being onshore vs. offshore.",
+ "type": "string"
+ },
+ "FacilityNameAliases": {
+ "description": "DEPRECATED: please use data.NameAliases. Alternative names, including historical, by which this facility is/has been known.",
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/osdu:wks:AbstractAliasNames:1.0.0"
+ }
+ },
+ "FacilityEvents": {
+ "x-osdu-indexing": {
+ "type": "nested"
+ },
+ "description": "A list of key facility events.",
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/osdu:wks:AbstractFacilityEvent:1.0.0"
+ }
+ },
+ "FacilitySpecifications": {
+ "x-osdu-indexing": {
+ "type": "flattened"
+ },
+ "description": "facilitySpecification maintains the specification like slot name, wellbore drilling permit number, rig name etc.",
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/osdu:wks:AbstractFacilitySpecification:1.0.0"
+ }
+ },
+ "DataSourceOrganisationID": {
+ "x-osdu-relationship": [{
+ "EntityType": "Organisation",
+ "GroupType": "master-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:master-data\\-\\-Organisation:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The main source of the header information.",
+ "type": "string"
+ },
+ "InitialOperatorID": {
+ "x-osdu-relationship": [{
+ "EntityType": "Organisation",
+ "GroupType": "master-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:master-data\\-\\-Organisation:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "A initial operator organization ID; the organization ID may also be found in the FacilityOperatorOrganisationID of the FacilityOperator array providing the actual dates.",
+ "type": "string",
+ "title": "Initial Operator ID"
+ },
+ "CurrentOperatorID": {
+ "x-osdu-relationship": [{
+ "EntityType": "Organisation",
+ "GroupType": "master-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:master-data\\-\\-Organisation:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The current operator organization ID; the organization ID may also be found in the FacilityOperatorOrganisationID of the FacilityOperator array providing the actual dates.",
+ "type": "string",
+ "title": "Current Operator ID"
+ },
+ "FacilityOperators": {
+ "x-osdu-indexing": {
+ "type": "nested"
+ },
+ "description": "The history of operator organizations of the facility.",
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/osdu:wks:AbstractFacilityOperator:1.0.0"
+ }
+ },
+ "FacilityName": {
+ "description": "Name of the Facility.",
+ "type": "string"
+ },
+ "FacilityTypeID": {
+ "x-osdu-relationship": [{
+ "EntityType": "FacilityType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-FacilityType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The definition of a kind of capability to perform a business function or a service.",
+ "type": "string"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractFacility.1.0.0.json"
+ },
+ "osdu:wks:AbstractFacilityEvent:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractFacilityEvent:1.0.0",
+ "description": "A significant occurrence in the life of a facility, which often changes its state, or the state of one of its components. It can describe a point-in-time (event) or a time interval of a specific type (FacilityEventType).",
+ "title": "AbstractFacilityEvent",
+ "type": "object",
+ "properties": {
+ "EffectiveDateTime": {
+ "format": "date-time",
+ "description": "The date and time at which the event took place or takes effect.",
+ "x-osdu-frame-of-reference": "DateTime",
+ "type": "string"
+ },
+ "TerminationDateTime": {
+ "format": "date-time",
+ "description": "The date and time at which the event is no longer in effect. For point-in-time events the 'TerminationDateTime' must be set equal to 'EffectiveDateTime'. Open time intervals have an absent 'TerminationDateTime'.",
+ "x-osdu-frame-of-reference": "DateTime",
+ "type": "string"
+ },
+ "FacilityEventTypeID": {
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-FacilityEventType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The facility event type is a picklist. Examples: 'Permit', 'Spud', 'Abandon', etc.",
+ "x-osdu-relationship": [{
+ "EntityType": "FacilityEventType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "type": "string"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractFacilityEvent.1.0.0.json"
+ },
+ "osdu:wks:AbstractAnyCrsFeatureCollection:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractAnyCrsFeatureCollection:1.0.0",
+ "description": "A schema like GeoJSON FeatureCollection with a non-WGS 84 CRS context; based on https://geojson.org/schema/FeatureCollection.json. Attention: the coordinate order is fixed: Longitude/Easting/Westing/X first, followed by Latitude/Northing/Southing/Y, optionally height as third coordinate.",
+ "title": "AbstractAnyCrsFeatureCollection",
+ "type": "object",
+ "required": [
+ "type",
+ "persistableReferenceCrs",
+ "features"
+ ],
+ "properties": {
+ "CoordinateReferenceSystemID": {
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-CoordinateReferenceSystem:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The CRS reference into the CoordinateReferenceSystem catalog.",
+ "x-osdu-relationship": [{
+ "EntityType": "CoordinateReferenceSystem",
+ "GroupType": "reference-data"
+ }
+ ],
+ "title": "Coordinate Reference System ID",
+ "type": "string",
+ "example": "namespace:reference-data--CoordinateReferenceSystem:BoundProjected:EPSG::32021_EPSG::15851:"
+ },
+ "persistableReferenceCrs": {
+ "description": "The CRS reference as persistableReference string. If populated, the CoordinateReferenceSystemID takes precedence.",
+ "type": "string",
+ "title": "CRS Reference",
+ "example": "{\"authCode\":{\"auth\":\"OSDU\",\"code\":\"32021079\"},\"lateBoundCRS\":{\"authCode\":{\"auth\":\"EPSG\",\"code\":\"32021\"},\"name\":\"NAD_1927_StatePlane_North_Dakota_South_FIPS_3302\",\"type\":\"LBC\",\"ver\":\"PE_10_9_1\",\"wkt\":\"PROJCS[\\\"NAD_1927_StatePlane_North_Dakota_South_FIPS_3302\\\",GEOGCS[\\\"GCS_North_American_1927\\\",DATUM[\\\"D_North_American_1927\\\",SPHEROID[\\\"Clarke_1866\\\",6378206.4,294.9786982]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],PROJECTION[\\\"Lambert_Conformal_Conic\\\"],PARAMETER[\\\"False_Easting\\\",2000000.0],PARAMETER[\\\"False_Northing\\\",0.0],PARAMETER[\\\"Central_Meridian\\\",-100.5],PARAMETER[\\\"Standard_Parallel_1\\\",46.18333333333333],PARAMETER[\\\"Standard_Parallel_2\\\",47.48333333333333],PARAMETER[\\\"Latitude_Of_Origin\\\",45.66666666666666],UNIT[\\\"Foot_US\\\",0.3048006096012192],AUTHORITY[\\\"EPSG\\\",32021]]\"},\"name\":\"NAD27 * OGP-Usa Conus / North Dakota CS27 South zone [32021,15851]\",\"singleCT\":{\"authCode\":{\"auth\":\"EPSG\",\"code\":\"15851\"},\"name\":\"NAD_1927_To_WGS_1984_79_CONUS\",\"type\":\"ST\",\"ver\":\"PE_10_9_1\",\"wkt\":\"GEOGTRAN[\\\"NAD_1927_To_WGS_1984_79_CONUS\\\",GEOGCS[\\\"GCS_North_American_1927\\\",DATUM[\\\"D_North_American_1927\\\",SPHEROID[\\\"Clarke_1866\\\",6378206.4,294.9786982]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],GEOGCS[\\\"GCS_WGS_1984\\\",DATUM[\\\"D_WGS_1984\\\",SPHEROID[\\\"WGS_1984\\\",6378137.0,298.257223563]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],METHOD[\\\"NADCON\\\"],PARAMETER[\\\"Dataset_conus\\\",0.0],OPERATIONACCURACY[5.0],AUTHORITY[\\\"EPSG\\\",15851]]\"},\"type\":\"EBC\",\"ver\":\"PE_10_9_1\"}"
+ },
+ "features": {
+ "type": "array",
+ "items": {
+ "title": "AnyCrsGeoJSON Feature",
+ "type": "object",
+ "required": [
+ "type",
+ "properties",
+ "geometry"
+ ],
+ "properties": {
+ "geometry": {
+ "oneOf": [{
+ "type": "null"
+ }, {
+ "title": "AnyCrsGeoJSON Point",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsPoint"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "AnyCrsGeoJSON LineString",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsLineString"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "AnyCrsGeoJSON Polygon",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsPolygon"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "AnyCrsGeoJSON MultiPoint",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsMultiPoint"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "AnyCrsGeoJSON MultiLineString",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsMultiLineString"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "AnyCrsGeoJSON MultiPolygon",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "items": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsMultiPolygon"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "AnyCrsGeoJSON GeometryCollection",
+ "type": "object",
+ "required": [
+ "type",
+ "geometries"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsGeometryCollection"
+ ]
+ },
+ "geometries": {
+ "type": "array",
+ "items": {
+ "oneOf": [{
+ "title": "AnyCrsGeoJSON Point",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsPoint"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "AnyCrsGeoJSON LineString",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsLineString"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "AnyCrsGeoJSON Polygon",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsPolygon"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "AnyCrsGeoJSON MultiPoint",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsMultiPoint"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "AnyCrsGeoJSON MultiLineString",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsMultiLineString"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }, {
+ "title": "AnyCrsGeoJSON MultiPolygon",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "items": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsMultiPolygon"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ ]
+ }
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ ]
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsFeature"
+ ]
+ },
+ "properties": {
+ "oneOf": [{
+ "type": "null"
+ }, {
+ "type": "object"
+ }
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ },
+ "persistableReferenceUnitZ": {
+ "description": "The unit of measure for the Z-axis (only for 3-dimensional coordinates, where the CRS does not describe the vertical unit). Note that the direction is upwards positive, i.e. Z means height.",
+ "type": "string",
+ "title": "Z-Unit Reference",
+ "example": "{\"scaleOffset\":{\"scale\":1.0,\"offset\":0.0},\"symbol\":\"m\",\"baseMeasurement\":{\"ancestry\":\"Length\",\"type\":\"UM\"},\"type\":\"USO\"}"
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ },
+ "persistableReferenceVerticalCrs": {
+ "description": "The VerticalCRS reference as persistableReference string. If populated, the VerticalCoordinateReferenceSystemID takes precedence. The property is null or empty for 2D geometries. For 3D geometries and absent or null persistableReferenceVerticalCrs the vertical CRS is either provided via persistableReferenceCrs's CompoundCRS or it is implicitly defined as EPSG:5714 MSL height.",
+ "type": "string",
+ "title": "Vertical CRS Reference",
+ "example": "{\"authCode\":{\"auth\":\"EPSG\",\"code\":\"5714\"},\"name\":\"MSL_Height\",\"type\":\"LBC\",\"ver\":\"PE_10_9_1\",\"wkt\":\"VERTCS[\\\"MSL_Height\\\",VDATUM[\\\"Mean_Sea_Level\\\"],PARAMETER[\\\"Vertical_Shift\\\",0.0],PARAMETER[\\\"Direction\\\",1.0],UNIT[\\\"Meter\\\",1.0],AUTHORITY[\\\"EPSG\\\",5714]]\"}"
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsFeatureCollection"
+ ]
+ },
+ "VerticalCoordinateReferenceSystemID": {
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-CoordinateReferenceSystem:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The explicit VerticalCRS reference into the CoordinateReferenceSystem catalog. This property stays empty for 2D geometries. Absent or empty values for 3D geometries mean the context may be provided by a CompoundCRS in 'CoordinateReferenceSystemID' or implicitly EPSG:5714 MSL height",
+ "x-osdu-relationship": [{
+ "EntityType": "CoordinateReferenceSystem",
+ "GroupType": "reference-data"
+ }
+ ],
+ "title": "Vertical Coordinate Reference System ID",
+ "type": "string",
+ "example": "namespace:reference-data--CoordinateReferenceSystem:Vertical:EPSG::5714:"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractAnyCrsFeatureCollection.1.0.0.json"
+ },
+ "osdu:wks:AbstractGeoFieldContext:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractGeoFieldContext:1.0.0",
+ "description": "A single, typed field entity reference, which is 'abstracted' to AbstractGeoContext and then aggregated by GeoContexts properties.",
+ "title": "AbstractGeoFieldContext",
+ "type": "object",
+ "properties": {
+ "FieldID": {
+ "pattern": "^[\\w\\-\\.]+:master-data\\-\\-Field:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Reference to Field.",
+ "x-osdu-relationship": [{
+ "EntityType": "Field",
+ "GroupType": "master-data"
+ }
+ ],
+ "type": "string"
+ },
+ "GeoTypeID": {
+ "const": "Field",
+ "description": "The fixed type 'Field' for this AbstractGeoFieldContext."
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractGeoFieldContext.1.0.0.json"
+ },
+ "osdu:wks:AbstractAliasNames:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractAliasNames:1.0.0",
+ "description": "A list of alternative names for an object. The preferred name is in a separate, scalar property. It may or may not be repeated in the alias list, though a best practice is to include it if the list is present, but to omit the list if there are no other names. Note that the abstract entity is an array so the $ref to it is a simple property reference.",
+ "x-osdu-review-status": "Accepted",
+ "title": "AbstractAliasNames",
+ "type": "object",
+ "properties": {
+ "AliasNameTypeID": {
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-AliasNameType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "A classification of alias names such as by role played or type of source, such as regulatory name, regulatory code, company code, international standard name, etc.",
+ "x-osdu-relationship": [{
+ "EntityType": "AliasNameType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "type": "string"
+ },
+ "EffectiveDateTime": {
+ "format": "date-time",
+ "type": "string",
+ "description": "The date and time when an alias name becomes effective."
+ },
+ "AliasName": {
+ "type": "string",
+ "description": "Alternative Name value of defined name type for an object."
+ },
+ "TerminationDateTime": {
+ "format": "date-time",
+ "type": "string",
+ "description": "The data and time when an alias name is no longer in effect."
+ },
+ "DefinitionOrganisationID": {
+ "pattern": "^[\\w\\-\\.]+:(reference-data\\-\\-StandardsOrganisation|master-data\\-\\-Organisation):[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The StandardsOrganisation (reference-data) or Organisation (master-data) that provided the name (the source).",
+ "x-osdu-relationship": [{
+ "EntityType": "StandardsOrganisation",
+ "GroupType": "reference-data"
+ }, {
+ "EntityType": "Organisation",
+ "GroupType": "master-data"
+ }
+ ],
+ "type": "string"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractAliasNames.1.0.0.json"
+ },
+ "osdu:wks:AbstractGeoProspectContext:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractGeoProspectContext:1.0.0",
+ "description": "A single, typed Prospect entity reference, which is 'abstracted' to AbstractGeoContext and then aggregated by GeoContexts properties.",
+ "x-osdu-review-status": "Accepted",
+ "title": "AbstractGeoProspectContext",
+ "type": "object",
+ "properties": {
+ "ProspectID": {
+ "pattern": "^[\\w\\-\\.]+:master-data\\-\\-Prospect:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Reference to the prospect.",
+ "x-osdu-relationship": [{
+ "EntityType": "Prospect",
+ "GroupType": "master-data"
+ }
+ ],
+ "type": "string"
+ },
+ "GeoTypeID": {
+ "x-osdu-is-derived": {
+ "RelationshipPropertyName": "ProspectID",
+ "TargetPropertyName": "ProspectTypeID"
+ },
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-ProspectType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The ProspectType reference of the Prospect (via ProspectID) for application convenience.",
+ "x-osdu-relationship": [{
+ "EntityType": "ProspectType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "type": "string"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractGeoProspectContext.1.0.0.json"
+ },
+ "osdu:wks:AbstractGeoBasinContext:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractGeoBasinContext:1.0.0",
+ "description": "A single, typed basin entity reference, which is 'abstracted' to AbstractGeoContext and then aggregated by GeoContexts properties.",
+ "x-osdu-review-status": "Accepted",
+ "title": "AbstractGeoBasinContext",
+ "type": "object",
+ "properties": {
+ "BasinID": {
+ "pattern": "^[\\w\\-\\.]+:master-data\\-\\-Basin:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Reference to Basin.",
+ "x-osdu-relationship": [{
+ "EntityType": "Basin",
+ "GroupType": "master-data"
+ }
+ ],
+ "type": "string"
+ },
+ "GeoTypeID": {
+ "x-osdu-is-derived": {
+ "RelationshipPropertyName": "BasinID",
+ "TargetPropertyName": "BasinTypeID"
+ },
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-BasinType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The BasinType reference of the Basin (via BasinID) for application convenience.",
+ "x-osdu-relationship": [{
+ "EntityType": "BasinType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "type": "string"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractGeoBasinContext.1.0.0.json"
+ },
+ "osdu:wks:AbstractSpatialLocation:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractSpatialLocation:1.0.0",
+ "description": "A geographic object which can be described by a set of points.",
+ "title": "AbstractSpatialLocation",
+ "type": "object",
+ "properties": {
+ "AsIngestedCoordinates": {
+ "description": "The original or 'as ingested' coordinates (Point, MultiPoint, LineString, MultiLineString, Polygon or MultiPolygon). The name 'AsIngestedCoordinates' was chosen to contrast it to 'OriginalCoordinates', which carries the uncertainty whether any coordinate operations took place before ingestion. In cases where the original CRS is different from the as-ingested CRS, the AppliedOperations can also contain the list of operations applied to the coordinate prior to ingestion. The data structure is similar to GeoJSON FeatureCollection, however in a CRS context explicitly defined within the AbstractAnyCrsFeatureCollection. The coordinate sequence follows GeoJSON standard, i.e. 'eastward/longitude', 'northward/latitude' {, 'upward/height' unless overridden by an explicit direction in the AsIngestedCoordinates.VerticalCoordinateReferenceSystemID}.",
+ "x-osdu-frame-of-reference": "CRS:",
+ "title": "As Ingested Coordinates",
+ "$ref": "#/definitions/osdu:wks:AbstractAnyCrsFeatureCollection:1.0.0"
+ },
+ "SpatialParameterTypeID": {
+ "x-osdu-relationship": [{
+ "EntityType": "SpatialParameterType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-SpatialParameterType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "A type of spatial representation of an object, often general (e.g. an Outline, which could be applied to Field, Reservoir, Facility, etc.) or sometimes specific (e.g. Onshore Outline, State Offshore Outline, Federal Offshore Outline, 3 spatial representations that may be used by Countries).",
+ "type": "string"
+ },
+ "QuantitativeAccuracyBandID": {
+ "x-osdu-relationship": [{
+ "EntityType": "QuantitativeAccuracyBand",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-QuantitativeAccuracyBand:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "An approximate quantitative assessment of the quality of a location (accurate to > 500 m (i.e. not very accurate)), to < 1 m, etc.",
+ "type": "string"
+ },
+ "CoordinateQualityCheckRemarks": {
+ "description": "Freetext remarks on Quality Check.",
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "AppliedOperations": {
+ "description": "The audit trail of operations applied to the coordinates from the original state to the current state. The list may contain operations applied prior to ingestion as well as the operations applied to produce the Wgs84Coordinates. The text elements refer to ESRI style CRS and Transformation names, which may have to be translated to EPSG standard names.",
+ "title": "Operations Applied",
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "example": [
+ "conversion from ED_1950_UTM_Zone_31N to GCS_European_1950; 1 points converted",
+ "transformation GCS_European_1950 to GCS_WGS_1984 using ED_1950_To_WGS_1984_24; 1 points successfully transformed"
+ ]
+ },
+ "QualitativeSpatialAccuracyTypeID": {
+ "x-osdu-relationship": [{
+ "EntityType": "QualitativeSpatialAccuracyType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-QualitativeSpatialAccuracyType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "A qualitative description of the quality of a spatial location, e.g. unverifiable, not verified, basic validation.",
+ "type": "string"
+ },
+ "CoordinateQualityCheckPerformedBy": {
+ "description": "The user who performed the Quality Check.",
+ "type": "string"
+ },
+ "SpatialLocationCoordinatesDate": {
+ "format": "date-time",
+ "description": "Date when coordinates were measured or retrieved.",
+ "x-osdu-frame-of-reference": "DateTime",
+ "type": "string"
+ },
+ "CoordinateQualityCheckDateTime": {
+ "format": "date-time",
+ "description": "The date of the Quality Check.",
+ "x-osdu-frame-of-reference": "DateTime",
+ "type": "string"
+ },
+ "Wgs84Coordinates": {
+ "description": "The normalized coordinates (Point, MultiPoint, LineString, MultiLineString, Polygon or MultiPolygon) based on WGS 84 (EPSG:4326 for 2-dimensional coordinates, EPSG:4326 + EPSG:5714 (MSL) for 3-dimensional coordinates). This derived coordinate representation is intended for global discoverability only. The schema of this substructure is identical to the GeoJSON FeatureCollection https://geojson.org/schema/FeatureCollection.json. The coordinate sequence follows GeoJSON standard, i.e. longitude, latitude {, height}",
+ "title": "WGS 84 Coordinates",
+ "$ref": "#/definitions/osdu:wks:AbstractFeatureCollection:1.0.0"
+ },
+ "SpatialGeometryTypeID": {
+ "x-osdu-relationship": [{
+ "EntityType": "SpatialGeometryType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-SpatialGeometryType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Indicates the expected look of the SpatialParameterType, e.g. Point, MultiPoint, LineString, MultiLineString, Polygon, MultiPolygon. The value constrains the type of geometries in the GeoJSON Wgs84Coordinates and AsIngestedCoordinates.",
+ "type": "string"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractSpatialLocation.1.0.0.json"
+ },
+ "osdu:wks:AbstractLegalTags:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractLegalTags:1.0.0",
+ "description": "Legal meta data like legal tags, relevant other countries, legal status. This structure is included by the SystemProperties \"legal\", which is part of all OSDU records. Not extensible.",
+ "additionalProperties": false,
+ "title": "Legal Meta Data",
+ "type": "object",
+ "properties": {
+ "legaltags": {
+ "description": "The list of legal tags, which resolve to legal properties (like country of origin, export classification code, etc.) and rules with the help of the Compliance Service.",
+ "title": "Legal Tags",
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "otherRelevantDataCountries": {
+ "description": "The list of other relevant data countries as an array of two-letter country codes, see https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2.",
+ "title": "Other Relevant Data Countries",
+ "type": "array",
+ "items": {
+ "pattern": "^[A-Z]{2}$",
+ "type": "string"
+ }
+ },
+ "status": {
+ "pattern": "^(compliant|uncompliant)$",
+ "description": "The legal status. Set by the system after evaluation against the compliance rules associated with the \"legaltags\" using the Compliance Service.",
+ "title": "Legal Status",
+ "type": "string"
+ }
+ },
+ "required": [
+ "legaltags",
+ "otherRelevantDataCountries"
+ ],
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractLegalTags.1.0.0.json"
+ },
+ "osdu:wks:AbstractMaster:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractMaster:1.0.0",
+ "description": "Properties shared with all master-data schema instances.",
+ "x-osdu-review-status": "Accepted",
+ "title": "Abstract Master",
+ "type": "object",
+ "properties": {
+ "NameAliases": {
+ "x-osdu-indexing": {
+ "type": "nested"
+ },
+ "description": "Alternative names, including historical, by which this master data is/has been known (it should include all the identifiers).",
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/osdu:wks:AbstractAliasNames:1.0.0"
+ }
+ },
+ "SpatialLocation": {
+ "description": "The spatial location information such as coordinates, CRS information (left empty when not appropriate).",
+ "$ref": "#/definitions/osdu:wks:AbstractSpatialLocation:1.0.0"
+ },
+ "VersionCreationReason": {
+ "description": "This describes the reason that caused the creation of a new version of this master data.",
+ "type": "string"
+ },
+ "GeoContexts": {
+ "x-osdu-indexing": {
+ "type": "nested"
+ },
+ "description": "List of geographic entities which provide context to the master data. This may include multiple types or multiple values of the same type.",
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/osdu:wks:AbstractGeoContext:1.0.0"
+ }
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractMaster.1.0.0.json"
+ },
+ "osdu:wks:AbstractFacilitySpecification:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractFacilitySpecification:1.0.0",
+ "description": "A property, characteristic, or attribute about a facility that is not described explicitly elsewhere.",
+ "title": "AbstractFacilitySpecification",
+ "type": "object",
+ "properties": {
+ "FacilitySpecificationText": {
+ "type": "string",
+ "description": "The actual text value of the parameter."
+ },
+ "FacilitySpecificationDateTime": {
+ "format": "date-time",
+ "description": "The actual date and time value of the parameter.",
+ "x-osdu-frame-of-reference": "DateTime",
+ "type": "string"
+ },
+ "FacilitySpecificationIndicator": {
+ "type": "boolean",
+ "description": "The actual indicator value of the parameter."
+ },
+ "TerminationDateTime": {
+ "format": "date-time",
+ "description": "The date and time at which the facility specification instance is no longer in effect.",
+ "x-osdu-frame-of-reference": "DateTime",
+ "type": "string"
+ },
+ "EffectiveDateTime": {
+ "format": "date-time",
+ "description": "The date and time at which the facility specification instance becomes effective.",
+ "x-osdu-frame-of-reference": "DateTime",
+ "type": "string"
+ },
+ "UnitOfMeasureID": {
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-UnitOfMeasure:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The unit for the quantity parameter, like metre (m in SI units system) for quantity Length.",
+ "x-osdu-relationship": [{
+ "EntityType": "UnitOfMeasure",
+ "GroupType": "reference-data"
+ }
+ ],
+ "type": "string"
+ },
+ "FacilitySpecificationQuantity": {
+ "type": "number",
+ "description": "The value for the specified parameter type.",
+ "x-osdu-frame-of-reference": "UOM_via_property:UnitOfMeasureID"
+ },
+ "ParameterTypeID": {
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-ParameterType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Parameter type of property or characteristic.",
+ "x-osdu-relationship": [{
+ "EntityType": "ParameterType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "type": "string"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractFacilitySpecification.1.0.0.json"
+ },
+ "osdu:wks:AbstractGeoPlayContext:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractGeoPlayContext:1.0.0",
+ "description": "A single, typed Play entity reference, which is 'abstracted' to AbstractGeoContext and then aggregated by GeoContexts properties.",
+ "x-osdu-review-status": "Accepted",
+ "title": "AbstractGeoPlayContext",
+ "type": "object",
+ "properties": {
+ "PlayID": {
+ "pattern": "^[\\w\\-\\.]+:master-data\\-\\-Play:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Reference to the play.",
+ "x-osdu-relationship": [{
+ "EntityType": "Play",
+ "GroupType": "master-data"
+ }
+ ],
+ "type": "string"
+ },
+ "GeoTypeID": {
+ "x-osdu-is-derived": {
+ "RelationshipPropertyName": "PlayID",
+ "TargetPropertyName": "PlayTypeID"
+ },
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-PlayType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The PlayType reference of the Play (via PlayID) for application convenience.",
+ "x-osdu-relationship": [{
+ "EntityType": "PlayType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "type": "string"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractGeoPlayContext.1.0.0.json"
+ },
+ "osdu:wks:AbstractFacilityState:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractFacilityState:1.0.0",
+ "description": "The life cycle status of a facility at some point in time.",
+ "title": "AbstractFacilityState",
+ "type": "object",
+ "properties": {
+ "EffectiveDateTime": {
+ "format": "date-time",
+ "description": "The date and time at which the facility state becomes effective.",
+ "x-osdu-frame-of-reference": "DateTime",
+ "type": "string"
+ },
+ "FacilityStateTypeID": {
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-FacilityStateType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The facility life cycle state from planning to abandonment.",
+ "x-osdu-relationship": [{
+ "EntityType": "FacilityStateType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "type": "string"
+ },
+ "TerminationDateTime": {
+ "format": "date-time",
+ "description": "The date and time at which the facility state is no longer in effect.",
+ "x-osdu-frame-of-reference": "DateTime",
+ "type": "string"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractFacilityState.1.0.0.json"
+ },
+ "osdu:wks:AbstractFacilityOperator:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractFacilityOperator:1.0.0",
+ "description": "The organisation that was responsible for a facility at some point in time.",
+ "title": "AbstractFacilityOperator",
+ "type": "object",
+ "properties": {
+ "FacilityOperatorID": {
+ "type": "string",
+ "title": "Facility Operator ID",
+ "description": "Internal, unique identifier for an item 'AbstractFacilityOperator'. This identifier is used by 'AbstractFacility.CurrentOperatorID' and 'AbstractFacility.InitialOperatorID'."
+ },
+ "EffectiveDateTime": {
+ "format": "date-time",
+ "description": "The date and time at which the facility operator becomes effective.",
+ "x-osdu-frame-of-reference": "DateTime",
+ "type": "string"
+ },
+ "FacilityOperatorOrganisationID": {
+ "pattern": "^[\\w\\-\\.]+:master-data\\-\\-Organisation:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The company that currently operates, or previously operated the facility",
+ "x-osdu-relationship": [{
+ "EntityType": "Organisation",
+ "GroupType": "master-data"
+ }
+ ],
+ "type": "string"
+ },
+ "TerminationDateTime": {
+ "format": "date-time",
+ "description": "The date and time at which the facility operator is no longer in effect. If the operator is still effective, the 'TerminationDateTime' is left absent.",
+ "x-osdu-frame-of-reference": "DateTime",
+ "type": "string"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractFacilityOperator.1.0.0.json"
+ },
+ "osdu:wks:AbstractFacilityVerticalMeasurement:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractFacilityVerticalMeasurement:1.0.0",
+ "description": "A location along a wellbore, _usually_ associated with some aspect of the drilling of the wellbore, but not with any intersecting _subsurface_ natural surfaces.",
+ "title": "AbstractFacilityVerticalMeasurement",
+ "type": "object",
+ "properties": {
+ "WellboreTVDTrajectoryID": {
+ "x-osdu-relationship": [{
+ "EntityType": "WellboreTrajectory",
+ "GroupType": "work-product-component"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:work-product-component\\-\\-WellboreTrajectory:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Specifies what directional survey or wellpath was used to calculate the TVD.",
+ "type": "string"
+ },
+ "VerticalCRSID": {
+ "x-osdu-relationship": [{
+ "EntityType": "CoordinateReferenceSystem",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-CoordinateReferenceSystem:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "A vertical coordinate reference system defines the origin for height or depth values. It is expected that either VerticalCRSID or VerticalReferenceID reference is provided in a given vertical measurement array object, but not both.",
+ "type": "string"
+ },
+ "VerticalMeasurementSourceID": {
+ "x-osdu-relationship": [{
+ "EntityType": "VerticalMeasurementSource",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-VerticalMeasurementSource:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Specifies Driller vs Logger.",
+ "type": "string"
+ },
+ "VerticalReferenceID": {
+ "description": "The reference point from which the relative vertical measurement is made. This is only populated if the measurement has no VerticalCRSID specified. The value entered must be the VerticalMeasurementID for another vertical measurement array element in this resource or its parent facility, and as a chain of measurements, they must resolve ultimately to a Vertical CRS. It is expected that a VerticalCRSID or a VerticalReferenceID is provided in a given vertical measurement array object, but not both.",
+ "type": "string"
+ },
+ "TerminationDateTime": {
+ "format": "date-time",
+ "description": "The date and time at which a vertical measurement instance is no longer in effect.",
+ "x-osdu-frame-of-reference": "DateTime",
+ "type": "string"
+ },
+ "VerticalMeasurementPathID": {
+ "x-osdu-relationship": [{
+ "EntityType": "VerticalMeasurementPath",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-VerticalMeasurementPath:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Specifies Measured Depth, True Vertical Depth, or Elevation.",
+ "type": "string"
+ },
+ "EffectiveDateTime": {
+ "format": "date-time",
+ "description": "The date and time at which a vertical measurement instance becomes effective.",
+ "x-osdu-frame-of-reference": "DateTime",
+ "type": "string"
+ },
+ "VerticalMeasurement": {
+ "description": "The value of the elevation or depth. Depth is positive downwards from a vertical reference or geodetic datum along a path, which can be vertical; elevation is positive upwards from a geodetic datum along a vertical path. Either can be negative.",
+ "x-osdu-frame-of-reference": "UOM_via_property:VerticalMeasurementUnitOfMeasureID",
+ "type": "number"
+ },
+ "VerticalMeasurementTypeID": {
+ "x-osdu-relationship": [{
+ "EntityType": "VerticalMeasurementType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-VerticalMeasurementType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Specifies the type of vertical measurement (TD, Plugback, Kickoff, Drill Floor, Rotary Table...).",
+ "type": "string"
+ },
+ "VerticalMeasurementDescription": {
+ "description": "Text which describes a vertical measurement in detail.",
+ "type": "string"
+ },
+ "VerticalMeasurementUnitOfMeasureID": {
+ "x-osdu-relationship": [{
+ "EntityType": "UnitOfMeasure",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-UnitOfMeasure:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The unit of measure for the vertical measurement. If a unit of measure and a vertical CRS are provided, the unit of measure provided is taken over the unit of measure from the CRS.",
+ "type": "string"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractFacilityVerticalMeasurement.1.0.0.json"
+ },
+ "osdu:wks:AbstractGeoPoliticalContext:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractGeoPoliticalContext:1.0.0",
+ "description": "A single, typed geo-political entity reference, which is 'abstracted' to AbstractGeoContext and then aggregated by GeoContexts properties.",
+ "x-osdu-review-status": "Accepted",
+ "title": "AbstractGeoPoliticalContext",
+ "type": "object",
+ "properties": {
+ "GeoPoliticalEntityID": {
+ "pattern": "^[\\w\\-\\.]+:master-data\\-\\-GeoPoliticalEntity:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Reference to GeoPoliticalEntity.",
+ "x-osdu-relationship": [{
+ "EntityType": "GeoPoliticalEntity",
+ "GroupType": "master-data"
+ }
+ ],
+ "type": "string"
+ },
+ "GeoTypeID": {
+ "x-osdu-is-derived": {
+ "RelationshipPropertyName": "GeoPoliticalEntityID",
+ "TargetPropertyName": "GeoPoliticalEntityTypeID"
+ },
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-GeoPoliticalEntityType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The GeoPoliticalEntityType reference of the GeoPoliticalEntity (via GeoPoliticalEntityID) for application convenience.",
+ "x-osdu-relationship": [{
+ "EntityType": "GeoPoliticalEntityType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "type": "string"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractGeoPoliticalContext.1.0.0.json"
+ },
+ "osdu:wks:AbstractWellboreDrillingReason:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractWellboreDrillingReason:1.0.0",
+ "description": "Purpose for drilling a wellbore, which often is an indication of the level of risk.",
+ "title": "AbstractWellboreDrillingReason",
+ "type": "object",
+ "properties": {
+ "DrillingReasonTypeID": {
+ "x-osdu-relationship": [{
+ "EntityType": "DrillingReasonType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-DrillingReasonType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Identifier of the drilling reason type for the corresponding time period.",
+ "type": "string"
+ },
+ "TerminationDateTime": {
+ "format": "date-time",
+ "description": "The date and time at which the event is no longer in effect.",
+ "x-osdu-frame-of-reference": "DateTime",
+ "type": "string"
+ },
+ "EffectiveDateTime": {
+ "format": "date-time",
+ "description": "The date and time at which the event becomes effective.",
+ "x-osdu-frame-of-reference": "DateTime",
+ "type": "string"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractWellboreDrillingReason.1.0.0.json"
+ },
+ "osdu:wks:AbstractAccessControlList:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractAccessControlList:1.0.0",
+ "description": "The access control tags associated with this entity. This structure is included by the SystemProperties \"acl\", which is part of all OSDU records. Not extensible.",
+ "additionalProperties": false,
+ "title": "Access Control List",
+ "type": "object",
+ "properties": {
+ "viewers": {
+ "description": "The list of viewers to which this data record is accessible/visible/discoverable formatted as an email (core.common.model.storage.validation.ValidationDoc.EMAIL_REGEX).",
+ "title": "List of Viewers",
+ "type": "array",
+ "items": {
+ "pattern": "^[a-zA-Z0-9_+&*-]+(?:\\.[a-zA-Z0-9_+&*-]+)*@(?:[a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,7}$",
+ "type": "string"
+ }
+ },
+ "owners": {
+ "description": "The list of owners of this data record formatted as an email (core.common.model.storage.validation.ValidationDoc.EMAIL_REGEX).",
+ "title": "List of Owners",
+ "type": "array",
+ "items": {
+ "pattern": "^[a-zA-Z0-9_+&*-]+(?:\\.[a-zA-Z0-9_+&*-]+)*@(?:[a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,7}$",
+ "type": "string"
+ }
+ }
+ },
+ "required": [
+ "owners",
+ "viewers"
+ ],
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractAccessControlList.1.0.0.json"
+ }
+ },
+ "properties": {
+ "ancestry": {
+ "description": "The links to data, which constitute the inputs, from which this record instance is derived.",
+ "title": "Ancestry",
+ "$ref": "#/definitions/osdu:wks:AbstractLegalParentList:1.0.0"
+ },
+ "data": {
+ "allOf": [{
+ "$ref": "#/definitions/osdu:wks:AbstractCommonResources:1.0.0"
+ }, {
+ "$ref": "#/definitions/osdu:wks:AbstractMaster:1.0.0"
+ }, {
+ "$ref": "#/definitions/osdu:wks:AbstractFacility:1.0.0"
+ }, {
+ "type": "object",
+ "title": "IndividualProperties",
+ "properties": {
+ "GeographicBottomHoleLocation": {
+ "description": "The bottom hole location of the wellbore denoted by a specified geographic horizontal coordinate reference system (Horizontal CRS), such as WGS84, NAD27, or ED50. If both GeographicBottomHoleLocation and ProjectedBottomHoleLocation properties are populated on this wellbore, they must identify the same point, just in different CRSs.",
+ "$ref": "#/definitions/osdu:wks:AbstractSpatialLocation:1.0.0"
+ },
+ "DrillingReasons": {
+ "description": "The history of drilling reasons of the wellbore.",
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/osdu:wks:AbstractWellboreDrillingReason:1.0.0"
+ }
+ },
+ "VerticalMeasurements": {
+ "x-osdu-indexing": {
+ "type": "nested"
+ },
+ "description": "List of all depths and elevations pertaining to the wellbore, like, plug back measured depth, total measured depth, KB elevation",
+ "type": "array",
+ "items": {
+ "allOf": [{
+ "type": "object",
+ "title": "Vertical Measurement ID",
+ "properties": {
+ "VerticalMeasurementID": {
+ "description": "The ID for a distinct vertical measurement within the Wellbore VerticalMeasurements array so that it may be referenced by other vertical measurements if necessary.",
+ "type": "string"
+ }
+ }
+ }, {
+ "$ref": "#/definitions/osdu:wks:AbstractFacilityVerticalMeasurement:1.0.0"
+ }
+ ]
+ }
+ },
+ "PrimaryMaterialID": {
+ "x-osdu-relationship": [{
+ "EntityType": "MaterialType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-MaterialType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The primary material injected/produced from the wellbore.",
+ "type": "string"
+ },
+ "SequenceNumber": {
+ "description": "A number that indicates the order in which wellbores were drilled.",
+ "type": "integer"
+ },
+ "TargetFormation": {
+ "x-osdu-relationship": [{
+ "EntityType": "GeologicalFormation",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-GeologicalFormation:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The Formation of interest for which the Wellbore is drilled to interact with. The Wellbore may terminate in a lower formation if the requirement is to drill through the entirety of the target formation, therefore this is not necessarily the Formation at TD.",
+ "type": "string"
+ },
+ "KickOffWellbore": {
+ "x-osdu-relationship": [{
+ "EntityType": "Wellbore",
+ "GroupType": "master-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:master-data\\-\\-Wellbore:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "This is a pointer to the parent wellbore. The wellbore that starts from top has no parent.",
+ "type": "string"
+ },
+ "DefaultVerticalMeasurementID": {
+ "description": "The default datum reference point, or zero depth point, used to determine other points vertically in a wellbore. References an entry in the Vertical Measurements array of this wellbore.",
+ "type": "string"
+ },
+ "ProjectedBottomHoleLocation": {
+ "description": "The bottom hole location of the wellbore denoted by a projected horizontal coordinate reference system (Horizontal CRS), such a UTM zone. 'Projected' in this property does not mean 'planned' or 'projected-to-bit'. If both GeographicBottomHoleLocation and ProjectedBottomHoleLocation properties are populated on this wellbore, they must identify the same point, just in different CRSs.",
+ "$ref": "#/definitions/osdu:wks:AbstractSpatialLocation:1.0.0"
+ },
+ "WellID": {
+ "x-osdu-relationship": [{
+ "EntityType": "Well",
+ "GroupType": "master-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:master-data\\-\\-Well:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "type": "string"
+ },
+ "TrajectoryTypeID": {
+ "x-osdu-relationship": [{
+ "EntityType": "WellboreTrajectoryType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-WellboreTrajectoryType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Describes the predominant shapes the wellbore path can follow if deviated from vertical. Sample Values: Horizontal, Vertical, Directional.",
+ "type": "string"
+ },
+ "DefinitiveTrajectoryID": {
+ "x-osdu-relationship": [{
+ "EntityType": "WellboreTrajectory",
+ "GroupType": "work-product-component"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:work-product-component\\-\\-WellboreTrajectory:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "SRN of Wellbore Trajectory which is considered the authoritative or preferred version.",
+ "type": "string"
+ }
+ }
+ }, {
+ "type": "object",
+ "title": "ExtensionProperties",
+ "properties": {
+ "ExtensionProperties": {
+ "type": "object"
+ }
+ }
+ }
+ ]
+ },
+ "kind": {
+ "pattern": "^[\\w\\-\\.]+:[\\w\\-\\.]+:[\\w\\-\\.]+:[0-9]+.[0-9]+.[0-9]+$",
+ "description": "The schema identification for the OSDU resource object following the pattern {Namespace}:{Source}:{Type}:{VersionMajor}.{VersionMinor}.{VersionPatch}. The versioning scheme follows the semantic versioning, https://semver.org/.",
+ "title": "Entity Kind",
+ "type": "string",
+ "example": "osdu:wks:master-data--Wellbore:1.0.0"
+ },
+ "acl": {
+ "description": "The access control tags associated with this entity.",
+ "title": "Access Control List",
+ "$ref": "#/definitions/osdu:wks:AbstractAccessControlList:1.0.0"
+ },
+ "version": {
+ "format": "int64",
+ "description": "The version number of this OSDU resource; set by the framework.",
+ "title": "Version Number",
+ "type": "integer",
+ "example": 1562066009929332
+ },
+ "tags": {
+ "description": "A generic dictionary of string keys mapping to string value. Only strings are permitted as keys and values.",
+ "additionalProperties": {
+ "type": "string"
+ },
+ "title": "Tag Dictionary",
+ "type": "object",
+ "example": {
+ "NameOfKey": "String value"
+ }
+ },
+ "modifyUser": {
+ "description": "The user reference, which created this version of this resource object. Set by the System.",
+ "title": "Resource Object Version Creation User Reference",
+ "type": "string",
+ "example": "some-user@some-company-cloud.com"
+ },
+ "modifyTime": {
+ "format": "date-time",
+ "description": "Timestamp of the time at which this version of the OSDU resource object was created. Set by the System. The value is a combined date-time string in ISO-8601 given in UTC.",
+ "title": "Resource Object Version Creation DateTime",
+ "type": "string",
+ "example": "2020-12-16T11:52:24.477Z"
+ },
+ "createTime": {
+ "format": "date-time",
+ "description": "Timestamp of the time at which initial version of this OSDU resource object was created. Set by the System. The value is a combined date-time string in ISO-8601 given in UTC.",
+ "title": "Resource Object Creation DateTime",
+ "type": "string",
+ "example": "2020-12-16T11:46:20.163Z"
+ },
+ "meta": {
+ "description": "The Frame of Reference meta data section linking the named properties to self-contained definitions.",
+ "title": "Frame of Reference Meta Data",
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/osdu:wks:AbstractMetaItem:1.0.0"
+ }
+ },
+ "legal": {
+ "description": "The entity's legal tags and compliance status. The actual contents associated with the legal tags is managed by the Compliance Service.",
+ "title": "Legal Tags",
+ "$ref": "#/definitions/osdu:wks:AbstractLegalTags:1.0.0"
+ },
+ "createUser": {
+ "description": "The user reference, which created the first version of this resource object. Set by the System.",
+ "title": "Resource Object Creation User Reference",
+ "type": "string",
+ "example": "some-user@some-company-cloud.com"
+ },
+ "id": {
+ "pattern": "^[\\w\\-\\.]+:master-data\\-\\-Test:[\\w\\-\\.\\:\\%]+$",
+ "description": "Previously called ResourceID or SRN which identifies this OSDU resource object without version.",
+ "title": "Entity ID",
+ "type": "string",
+ "example": "namespace:master-data--Test:c7c421a7-f496-5aef-8093-298c32bfdea9"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/master-data/Test.1.0.0.json"
+ }
+}
diff --git a/indexer-acceptance-test/src/test/resources/testData/index_records_1.json b/indexer-acceptance-test/src/test/resources/testData/index_records_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..43727af8ddc1eae6914d09d8c296c75c052e7604
--- /dev/null
+++ b/indexer-acceptance-test/src/test/resources/testData/index_records_1.json
@@ -0,0 +1,170 @@
+[
+ {
+ "id": "tenant1:<kindSubType>:testIngest2<timestamp>",
+ "tags": {
+ "testtag": "testvalue"
+ },
+ "data": {
+ "Field": "OSDU OFFICE - 2",
+ "Location": {
+ "lat": 32.406402588,
+ "lon": -86.565592762
+ },
+ "Basin": "Houston",
+ "County": "Harris",
+ "State": "TX",
+ "Country": "USA",
+ "WellStatus": "Under development",
+ "OriginalOperator": "OFFICE - 2",
+ "WellName": "Data Platform Services",
+ "WellType": "Data Lake Cloud",
+ "EmptyAttribute": "",
+ "RelatedRecordId": "id1",
+ "ExternalIds": [
+ "ExternalId1"
+ ],
+ "Rank": 1,
+ "Score": 10,
+ "Established": "2000-03-27T23:38:48Z",
+ "DblArray": [
+ 32.40,
+ 36.45,
+ 40.0
+ ],
+ "TextArray": [
+ "Elevation (m)",
+ "SpatialLocation",
+ "Well Type"
+ ]
+ }
+ },
+ {
+ "id": "tenant1:<kindSubType>:testIngest3<timestamp>",
+ "tags": {
+ "testtag": "testvalue"
+ },
+ "data": {
+ "Field": "OSDU OFFICE - 2",
+ "Location": {
+ "lat": 32.406402588,
+ "lon": -86.565592762
+ },
+ "Basin": "Houston",
+ "County": "Harris",
+ "State": "TX",
+ "Country": "USA",
+ "WellStatus": "Under development",
+ "OriginalOperator": "OFFICE2",
+ "WellName": "Data Platform Services",
+ "WellType": "Data Lake Cloud",
+ "EmptyAttribute": "",
+ "RelatedRecordId": "id2",
+ "ExternalIds": [
+ "ExternalId2"
+ ],
+ "Rank": 1,
+ "Score": 10,
+ "Established": "2000-03-27T23:38:48Z",
+ "DblArray": [
+ 62.40,
+ 60.0
+ ],
+ "TextArray": [
+ "WCR",
+ "License",
+ "Completion Date"
+ ]
+ }
+ },
+ {
+ "id": "tenant1:<kindSubType>:testIngest4<timestamp>",
+ "tags": {
+ "testtag": "testvalue"
+ },
+ "data": {
+ "Field": "OSDU OFFICE - 2",
+ "Location": {
+ "lat": 32.406402588,
+ "lon": -86.565592762
+ },
+ "Basin": "Houston",
+ "County": "Harris",
+ "State": "TX",
+ "Country": "USA",
+ "WellStatus": "Under development",
+ "OriginalOperator": "OFFICE2",
+ "WellName": "Data Platform Services",
+ "WellType": "Data Lake Cloud",
+ "EmptyAttribute": "",
+ "RelatedRecordId": "id3",
+ "ExternalIds": [
+ "ExternalId3"
+ ],
+ "Rank": 1,
+ "Score": 10,
+ "Established": "2000-03-27T23:38:48Z",
+ "TextArray": [
+ "Wgs84Coordinates",
+ "Petroleum Field"
+ ]
+ }
+ },
+ {
+ "id": "tenant1:<kindSubType>:testIngest5<timestamp>",
+ "tags": {
+ "testtag": "testvalue"
+ },
+ "data": {
+ "Field": "OSDU OFFICE - 2",
+ "Location": {
+ "lat": 32.406402588,
+ "lon": -86.565592762
+ },
+ "Basin": "Houston",
+ "County": "Harris",
+ "State": "TX",
+ "Country": "USA",
+ "WellStatus": "Under development",
+ "OriginalOperator": "OFFICE2",
+ "WellName": "Data Platform Services",
+ "WellType": "Data Lake Cloud",
+ "EmptyAttribute": "",
+ "RelatedRecordId": "id4",
+ "ExternalIds": [
+ "ExternalId4"
+ ],
+ "Rank": 1,
+ "Score": 10,
+ "Established": "2000-03-27T23:38:48Z"
+ }
+ },
+ {
+ "id": "tenant1:<kindSubType>:testIngest6<timestamp>",
+ "tags": {
+ "testtag": "testvalue"
+ },
+ "data": {
+ "Field": "OSDU OFFICE - 2",
+ "Location": {
+ "lat": 32.406402588,
+ "lon": -86.565592762
+ },
+ "Basin": "Houston",
+ "County": "Harris",
+ "State": "TX",
+ "Country": "USA",
+ "WellStatus": "Under development",
+ "OriginalOperator": "OFFICE2",
+ "WellName": "Data Platform Services",
+ "WellType": "Data Lake Cloud",
+ "EmptyAttribute": "",
+ "ExternalIds": [
+ "ExternalId5",
+ "ExternalId6"
+ ],
+ "Rank": 1,
+ "Score": 10,
+ "Established": "2000-03-27T23:38:48Z"
+ }
+ }
+]
diff --git a/indexer-acceptance-test/src/test/resources/testData/index_records_1.schema b/indexer-acceptance-test/src/test/resources/testData/index_records_1.schema
new file mode 100644
index 0000000000000000000000000000000000000000..2c589fd3d7eefb194158d1dc33e940d557c6153d
--- /dev/null
+++ b/indexer-acceptance-test/src/test/resources/testData/index_records_1.schema
@@ -0,0 +1,73 @@
+{
+ "kind": "KIND_VAL",
+ "schema": [
+ {
+ "path": "Field",
+ "kind": "string"
+ },
+ {
+ "path": "Location",
+ "kind": "core:dl:geopoint:1.0.0"
+ },
+ {
+ "path": "Basin",
+ "kind": "string"
+ },
+ {
+ "path": "County",
+ "kind": "string"
+ },
+ {
+ "path": "State",
+ "kind": "string"
+ },
+ {
+ "path": "Country",
+ "kind": "string"
+ },
+ {
+ "path": "WellStatus",
+ "kind": "string"
+ },
+ {
+ "path": "OriginalOperator",
+ "kind": "string"
+ },
+ {
+ "path": "WellName",
+ "kind": "string"
+ },
+ {
+ "path": "WellType",
+ "kind": "string"
+ },
+ {
+ "path": "EmptyAttribute",
+ "kind": "string"
+ },
+ {
+ "path": "RelatedRecordId",
+ "kind": "link"
+ },
+ {
+ "path": "ExternalIds",
+ "kind": "[]link"
+ },
+ {
+ "path": "Rank",
+ "kind": "int"
+ },
+ {
+ "path": "Score",
+ "kind": "int"
+ },
+ {
+ "path": "Established",
+ "kind": "datetime"
+ },
+ {
+ "path": "DblArray",
+ "kind": "[]double"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/indexer-acceptance-test/src/test/resources/testData/index_records_1.schema.json b/indexer-acceptance-test/src/test/resources/testData/index_records_1.schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..3109cce84b16614000bec7b4afa1c8a3843857c1
--- /dev/null
+++ b/indexer-acceptance-test/src/test/resources/testData/index_records_1.schema.json
@@ -0,0 +1,134 @@
+{
+ "schemaInfo": {
+ "schemaIdentity": {
+ "authority": "tenant1",
+ "source": "indexer",
+ "entityType": "test-data--Integration",
+ "schemaVersionMajor": "1",
+ "schemaVersionMinor": "1",
+ "schemaVersionPatch": "0"
+ },
+ "status": "DEVELOPMENT"
+ },
+ "schema": {
+ "properties": {
+ "data": {
+ "allOf": [
+ {
+ "type": "object",
+ "properties": {
+ "Field": {
+ "type": "string"
+ },
+ "Location": {
+ "$ref": "#/definitions/opendes:wks:core_dl_geopoint:1.0.0",
+ "description": "The wellbore's position .",
+ "format": "core:dl:geopoint:1.0.0",
+ "title": "WGS 84 Position",
+ "type": "object"
+ },
+ "Basin": {
+ "type": "string"
+ },
+ "County": {
+ "type": "string"
+ }
+ }
+ }
+ ],
+ "anyOf": [
+ {
+ "type": "object",
+ "properties": {
+ "State": {
+ "type": "string"
+ },
+ "Country": {
+ "type": "string"
+ },
+ "WellStatus": {
+ "type": "string"
+ },
+ "OriginalOperator": {
+ "type": "string"
+ },
+ "WellName": {
+ "type": "string"
+ },
+ "WellType": {
+ "type": "string"
+ },
+ "EmptyAttribute": {
+ "type": "string"
+ },
+ "RelatedRecordId": {
+ "format": "link",
+ "type": "string"
+ },
+ "ExternalIds": {
+ "type": "[]link"
+ }
+ }
+ }
+ ],
+ "oneOf": [
+ {
+ "type": "object",
+ "properties": {
+ "Rank": {
+ "type": "integer"
+ },
+ "Score": {
+ "type": "integer"
+ },
+ "Established": {
+ "type": "date-time"
+ },
+ "DblArray": {
+ "description": "The name of the host [cloud environment] region(s) for this OSDU resource object.",
+ "title": "Resource Host Region ID",
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ },
+ "TextArray": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ }
+ }
+ }
+ ]
+ }
+ },
+ "definitions": {
+ "opendes:wks:core_dl_geopoint:1.0.0": {
+ "description": "A 2D point location in latitude and longitude referenced to WGS 84 if not specified otherwise.",
+ "properties": {
+ "latitude": {
+ "description": "The latitude value in degrees of arc (dega). Value range [-90, 90].",
+ "maximum": 90,
+ "minimum": -90,
+ "title": "Latitude",
+ "type": "number"
+ },
+ "longitude": {
+ "description": "The longitude value in degrees of arc (dega). Value range [-180, 180]",
+ "maximum": 180,
+ "minimum": -180,
+ "title": "Longitude",
+ "type": "number"
+ }
+ },
+ "required": [
+ "latitude",
+ "longitude"
+ ],
+ "title": "2D Map Location",
+ "type": "object"
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/indexer-acceptance-test/src/test/resources/testData/index_records_2.json b/indexer-acceptance-test/src/test/resources/testData/index_records_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..ff23f4c2c970d9488d7d0842294cebf58a0e8d40
--- /dev/null
+++ b/indexer-acceptance-test/src/test/resources/testData/index_records_2.json
@@ -0,0 +1,117 @@
+[
+ {
+ "id": "tenant1:<kindSubType>:testIngest7<timestamp>",
+ "data": {
+ "Field": "OSDU OFFICE - 2",
+ "Location": {
+ "lat":32.406402588,
+ "lon":-86.565592762
+ },
+ "Basin": "Houston",
+ "County": "Harris",
+ "State": "TX",
+ "Country": "USA",
+ "WellStatus": "Under development",
+ "OriginalOperator": "OFFICE2",
+ "WellName": "Data Platform Services",
+ "WellType": "Data Lake Cloud",
+ "EmptyAttribute": "",
+ "Rank": 1,
+ "Score" : 10,
+ "Established": "2000-03-27T23:38:48Z",
+ "InvalidInteger": 123456789123456789
+ }
+ },
+ {
+ "id": "tenant1:<kindSubType>:testIngest8<timestamp>",
+ "data": {
+ "Field": "OSDU OFFICE - 2",
+ "Location": {
+ "lat":32.406402588,
+ "lon":-86.565592762
+ },
+ "Basin": "Houston",
+ "County": "Harris",
+ "State": "TX",
+ "Country": "USA",
+ "WellStatus": "Under development",
+ "OriginalOperator": "OFFICE2",
+ "WellName": "Data Platform Services",
+ "WellType": "Data Lake Cloud",
+ "EmptyAttribute": "",
+ "Rank": 1,
+ "Score" : 10,
+ "Established": "2000-03-27T23:38:48Z",
+ "InvalidInteger": 123456789123456789
+ }
+ },
+ {
+ "id": "tenant1:<kindSubType>:testIngest9<timestamp>",
+ "data": {
+ "Field": "OSDU OFFICE - 2",
+ "Location": {
+ "lat":32.406402588,
+ "lon":-86.565592762
+ },
+ "Basin": "Houston",
+ "County": "Harris",
+ "State": "TX",
+ "Country": "USA",
+ "WellStatus": "Under development",
+ "OriginalOperator": "OFFICE2",
+ "WellName": "Data Platform Services",
+ "WellType": "Data Lake Cloud",
+ "EmptyAttribute": "",
+ "Rank": 1,
+ "Score" : 10,
+ "Established": "2000-03-27T23:38:48Z",
+ "InvalidInteger": 123456789123456789
+ }
+ },
+ {
+ "id": "tenant1:<kindSubType>:testIngest10<timestamp>",
+ "data": {
+ "Field": "OSDU OFFICE - 2",
+ "Location": {
+ "lat":32.406402588,
+ "lon":-86.565592762
+ },
+ "Basin": "Houston",
+ "County": "Harris",
+ "State": "TX",
+ "Country": "USA",
+ "WellStatus": "Under development",
+ "OriginalOperator": "OFFICE2",
+ "WellName": "Data Platform Services",
+ "WellType": "Data Lake Cloud",
+ "EmptyAttribute": "",
+ "Rank": 1,
+ "Score" : 10,
+ "Established": "2000-03-27T23:38:48Z",
+ "InvalidInteger": 123456789123456789
+ }
+ },
+ {
+ "id": "tenant1:<kindSubType>:testIngest11<timestamp>",
+ "data": {
+ "Field": 1234,
+ "Location": {
+ "lat":"BA1",
+ "lon":-86.565592762
+ },
+ "Basin": 789,
+ "County": 0.99,
+ "State": 0.56,
+ "Country": 1234,
+ "WellStatus": 528693,
+ "OriginalOperator": 564,
+ "WellName": 0.98,
+ "WellType": 454476578,
+ "EmptyAttribute": 1234,
+ "Rank": "Test",
+ "Score" : 10,
+ "Established": 123456,
+ "InvalidInteger": 8888
+ }
+ }
+]
\ No newline at end of file
diff --git a/indexer-acceptance-test/src/test/resources/testData/index_records_2.schema b/indexer-acceptance-test/src/test/resources/testData/index_records_2.schema
new file mode 100644
index 0000000000000000000000000000000000000000..2d272b735099a5d36d2e50a56b6da8e303f36e50
--- /dev/null
+++ b/indexer-acceptance-test/src/test/resources/testData/index_records_2.schema
@@ -0,0 +1,65 @@
+{
+ "kind": "KIND_VAL",
+ "schema": [
+ {
+ "path": "Field",
+ "kind": "string"
+ },
+ {
+ "path": "Location",
+ "kind": "core:dl:geopoint:1.0.0"
+ },
+ {
+ "path": "Basin",
+ "kind": "string"
+ },
+ {
+ "path": "County",
+ "kind": "string"
+ },
+ {
+ "path": "State",
+ "kind": "string"
+ },
+ {
+ "path": "Country",
+ "kind": "string"
+ },
+ {
+ "path": "WellStatus",
+ "kind": "string"
+ },
+ {
+ "path": "OriginalOperator",
+ "kind": "string"
+ },
+ {
+ "path": "WellName",
+ "kind": "string"
+ },
+ {
+ "path": "WellType",
+ "kind": "string"
+ },
+ {
+ "path": "EmptyAttribute",
+ "kind": "string"
+ },
+ {
+ "path": "Rank",
+ "kind": "int"
+ },
+ {
+ "path": "Score",
+ "kind": "int"
+ },
+ {
+ "path": "Established",
+ "kind": "datetime"
+ },
+ {
+ "path": "InvalidInteger",
+ "kind": "int"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/indexer-acceptance-test/src/test/resources/testData/index_records_2.schema.json b/indexer-acceptance-test/src/test/resources/testData/index_records_2.schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..e5ee1cb04eedeafed586fa3baa9664ba1d57d2cb
--- /dev/null
+++ b/indexer-acceptance-test/src/test/resources/testData/index_records_2.schema.json
@@ -0,0 +1,102 @@
+{
+ "schemaInfo": {
+ "schemaIdentity": {
+ "authority": "tenant1",
+ "source": "indexer",
+ "entityType": "test-data--Integration",
+ "schemaVersionMajor": "2",
+ "schemaVersionMinor": "0",
+ "schemaVersionPatch": "1"
+ },
+ "status": "DEVELOPMENT"
+ },
+ "schema": {
+ "properties": {
+ "data": {
+ "allOf": [
+ {
+ "type": "object",
+ "properties": {
+ "Field": {
+ "type": "string"
+ },
+ "Location": {
+ "$ref": "#/definitions/opendes:wks:core_dl_geopoint:1.0.0",
+ "description": "The wellbore's position .",
+ "format": "core:dl:geopoint:1.0.0",
+ "title": "WGS 84 Position",
+ "type": "object"
+ },
+ "Basin": {
+ "type": "string"
+ },
+ "County": {
+ "type": "string"
+ },
+ "State": {
+ "type": "string"
+ },
+ "Country": {
+ "type": "string"
+ },
+ "WellStatus": {
+ "type": "string"
+ },
+ "OriginalOperator": {
+ "type": "string"
+ },
+ "WellName": {
+ "type": "string"
+ },
+ "WellType": {
+ "type": "string"
+ },
+ "EmptyAttribute": {
+ "type": "string"
+ },
+ "Rank": {
+ "type": "integer"
+ },
+ "Score": {
+ "type": "integer"
+ },
+ "Established": {
+ "type": "date-time"
+ },
+ "InvalidInteger": {
+ "type": "integer"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "definitions": {
+ "opendes:wks:core_dl_geopoint:1.0.0": {
+ "description": "A 2D point location in latitude and longitude referenced to WGS 84 if not specified otherwise.",
+ "properties": {
+ "latitude": {
+ "description": "The latitude value in degrees of arc (dega). Value range [-90, 90].",
+ "maximum": 90,
+ "minimum": -90,
+ "title": "Latitude",
+ "type": "number"
+ },
+ "longitude": {
+ "description": "The longitude value in degrees of arc (dega). Value range [-180, 180]",
+ "maximum": 180,
+ "minimum": -180,
+ "title": "Longitude",
+ "type": "number"
+ }
+ },
+ "required": [
+ "latitude",
+ "longitude"
+ ],
+ "title": "2D Map Location",
+ "type": "object"
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/indexer-acceptance-test/src/test/resources/testData/index_records_3.json b/indexer-acceptance-test/src/test/resources/testData/index_records_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..58a7e0deefb054eee8fcf7c6ad1719ecd7ff879f
--- /dev/null
+++ b/indexer-acceptance-test/src/test/resources/testData/index_records_3.json
@@ -0,0 +1,589 @@
+[
+ {
+ "id": "tenant1:<kindSubType>:testIngest51<timestamp>",
+ "data": {
+ "WellName": "Data Platform Services - 51",
+ "GeoShape": {
+ "type": "FeatureCollection",
+ "features": [
+ {
+ "type": "Feature",
+ "properties": {},
+ "geometry": {
+ "type": "Point",
+ "coordinates": [
+ -105.01621,
+ 39.57422
+ ]
+ }
+ }
+ ]
+ }
+ }
+ },
+ {
+ "id": "tenant1:<kindSubType>:testIngest52<timestamp>",
+ "data": {
+ "WellName": "Data Platform Services - 52",
+ "GeoShape": {
+ "type": "FeatureCollection",
+ "features": [
+ {
+ "type": "Feature",
+ "properties": {},
+ "geometry": {
+ "type": "MultiPoint",
+ "coordinates": [
+ [
+ -105.01621,
+ 39.57422
+ ],
+ [
+ -80.666513,
+ 35.053994
+ ]
+ ]
+ }
+ }
+ ]
+ }
+ }
+ },
+ {
+ "id": "tenant1:<kindSubType>:testIngest53<timestamp>",
+ "data": {
+ "WellName": "Data Platform Services - 53",
+ "GeoShape": {
+ "type": "FeatureCollection",
+ "features": [
+ {
+ "type": "Feature",
+ "properties": {},
+ "geometry": {
+ "type": "LineString",
+ "coordinates": [
+ [
+ -101.744384,
+ 39.32155
+ ],
+ [
+ -101.552124,
+ 39.330048
+ ],
+ [
+ -101.403808,
+ 39.330048
+ ],
+ [
+ -101.332397,
+ 39.364032
+ ],
+ [
+ -101.041259,
+ 39.368279
+ ],
+ [
+ -100.975341,
+ 39.304549
+ ],
+ [
+ -100.914916,
+ 39.245016
+ ],
+ [
+ -100.843505,
+ 39.164141
+ ],
+ [
+ -100.805053,
+ 39.104488
+ ],
+ [
+ -100.491943,
+ 39.100226
+ ],
+ [
+ -100.437011,
+ 39.095962
+ ],
+ [
+ -100.338134,
+ 39.095962
+ ],
+ [
+ -100.195312,
+ 39.027718
+ ],
+ [
+ -100.008544,
+ 39.010647
+ ],
+ [
+ -99.865722,
+ 39.00211
+ ],
+ [
+ -99.684448,
+ 38.972221
+ ],
+ [
+ -99.51416,
+ 38.929502
+ ],
+ [
+ -99.382324,
+ 38.920955
+ ],
+ [
+ -99.321899,
+ 38.895308
+ ],
+ [
+ -99.113159,
+ 38.869651
+ ],
+ [
+ -99.0802,
+ 38.85682
+ ],
+ [
+ -98.822021,
+ 38.85682
+ ],
+ [
+ -98.448486,
+ 38.848264
+ ],
+ [
+ -98.206787,
+ 38.848264
+ ],
+ [
+ -98.020019,
+ 38.878204
+ ],
+ [
+ -97.635498,
+ 38.873928
+ ]
+ ]
+ }
+ }
+ ]
+ }
+ }
+ },
+ {
+ "id": "tenant1:<kindSubType>:testIngest54<timestamp>",
+ "data": {
+ "WellName": "Data Platform Services - 54",
+ "GeoShape": {
+ "type": "FeatureCollection",
+ "features": [
+ {
+ "type": "Feature",
+ "properties": {},
+ "geometry": {
+ "type": "MultiLineString",
+ "coordinates": [
+ [
+ [
+ -105.021443,
+ 39.578057
+ ],
+ [
+ -105.021507,
+ 39.577809
+ ],
+ [
+ -105.021572,
+ 39.577495
+ ],
+ [
+ -105.021572,
+ 39.577164
+ ],
+ [
+ -105.021572,
+ 39.577032
+ ],
+ [
+ -105.021529,
+ 39.576784
+ ]
+ ],
+ [
+ [
+ -105.019898,
+ 39.574997
+ ],
+ [
+ -105.019598,
+ 39.574898
+ ],
+ [
+ -105.019061,
+ 39.574782
+ ]
+ ],
+ [
+ [
+ -105.017173,
+ 39.574402
+ ],
+ [
+ -105.01698,
+ 39.574385
+ ],
+ [
+ -105.016636,
+ 39.574385
+ ],
+ [
+ -105.016508,
+ 39.574402
+ ],
+ [
+ -105.01595,
+ 39.57427
+ ]
+ ],
+ [
+ [
+ -105.014276,
+ 39.573972
+ ],
+ [
+ -105.014126,
+ 39.574038
+ ],
+ [
+ -105.013825,
+ 39.57417
+ ],
+ [
+ -105.01331,
+ 39.574452
+ ]
+ ]
+ ]
+ }
+ }
+ ]
+ }
+ }
+ },
+ {
+ "id": "tenant1:<kindSubType>:testIngest55<timestamp>",
+ "data": {
+ "WellName": "Data Platform Services - 55",
+ "GeoShape": {
+ "type": "FeatureCollection",
+ "features": [
+ {
+ "type": "Feature",
+ "properties": {},
+ "geometry": {
+ "type": "Polygon",
+ "coordinates": [
+ [
+ [
+ 100,
+ 0
+ ],
+ [
+ 101,
+ 0
+ ],
+ [
+ 101,
+ 1
+ ],
+ [
+ 100,
+ 1
+ ],
+ [
+ 100,
+ 0
+ ]
+ ]
+ ]
+ }
+ }
+ ]
+ }
+ }
+ },
+ {
+ "id": "tenant1:<kindSubType>:testIngest56<timestamp>",
+ "data": {
+ "WellName": "Data Platform Services - 56",
+ "GeoShape": {
+ "type": "FeatureCollection",
+ "features": [
+ {
+ "type": "Feature",
+ "properties": {},
+ "geometry": {
+ "type": "MultiPolygon",
+ "coordinates": [
+ [
+ [
+ [
+ 107,
+ 7
+ ],
+ [
+ 108,
+ 7
+ ],
+ [
+ 108,
+ 8
+ ],
+ [
+ 107,
+ 8
+ ],
+ [
+ 107,
+ 7
+ ]
+ ]
+ ],
+ [
+ [
+ [
+ 100,
+ 0
+ ],
+ [
+ 101,
+ 0
+ ],
+ [
+ 101,
+ 1
+ ],
+ [
+ 100,
+ 1
+ ],
+ [
+ 100,
+ 0
+ ]
+ ]
+ ]
+ ]
+ }
+ }
+ ]
+ }
+ }
+ },
+ {
+ "id": "tenant1:<kindSubType>:testIngest57<timestamp>",
+ "data": {
+ "WellName": "Data Platform Services - 57",
+ "GeoShape": {
+ "type": "FeatureCollection",
+ "features": [
+ {
+ "type": "Feature",
+ "properties": {},
+ "geometry": {
+ "type": "GeometryCollection",
+ "geometries": [
+ {
+ "type": "Point",
+ "coordinates": [
+ -80.660805,
+ 35.049392
+ ]
+ },
+ {
+ "type": "Polygon",
+ "coordinates": [
+ [
+ [
+ -80.664582,
+ 35.044965
+ ],
+ [
+ -80.663874,
+ 35.04428
+ ],
+ [
+ -80.662586,
+ 35.04558
+ ],
+ [
+ -80.663444,
+ 35.046036
+ ],
+ [
+ -80.664582,
+ 35.044965
+ ]
+ ]
+ ]
+ },
+ {
+ "type": "LineString",
+ "coordinates": [
+ [
+ -80.662372,
+ 35.059509
+ ],
+ [
+ -80.662693,
+ 35.059263
+ ],
+ [
+ -80.662844,
+ 35.05893
+ ],
+ [
+ -80.66308,
+ 35.058332
+ ],
+ [
+ -80.663595,
+ 35.057753
+ ],
+ [
+ -80.663874,
+ 35.057401
+ ],
+ [
+ -80.66441,
+ 35.057033
+ ],
+ [
+ -80.664861,
+ 35.056787
+ ],
+ [
+ -80.665419,
+ 35.056506
+ ],
+ [
+ -80.665633,
+ 35.056312
+ ],
+ [
+ -80.666019,
+ 35.055891
+ ],
+ [
+ -80.666191,
+ 35.055452
+ ],
+ [
+ -80.666191,
+ 35.055171
+ ],
+ [
+ -80.666255,
+ 35.05489
+ ],
+ [
+ -80.666213,
+ 35.054222
+ ],
+ [
+ -80.666213,
+ 35.053924
+ ],
+ [
+ -80.665955,
+ 35.052905
+ ],
+ [
+ -80.665698,
+ 35.052044
+ ],
+ [
+ -80.665504,
+ 35.051482
+ ],
+ [
+ -80.665762,
+ 35.050481
+ ],
+ [
+ -80.66617,
+ 35.049725
+ ],
+ [
+ -80.666513,
+ 35.049286
+ ],
+ [
+ -80.666921,
+ 35.048531
+ ],
+ [
+ -80.667006,
+ 35.048215
+ ],
+ [
+ -80.667071,
+ 35.047775
+ ],
+ [
+ -80.667049,
+ 35.047389
+ ],
+ [
+ -80.666964,
+ 35.046985
+ ],
+ [
+ -80.666813,
+ 35.046353
+ ],
+ [
+ -80.666599,
+ 35.045966
+ ],
+ [
+ -80.666406,
+ 35.045615
+ ],
+ [
+ -80.665998,
+ 35.045193
+ ],
+ [
+ -80.665526,
+ 35.044877
+ ],
+ [
+ -80.664989,
+ 35.044543
+ ],
+ [
+ -80.664496,
+ 35.044174
+ ],
+ [
+ -80.663852,
+ 35.043876
+ ],
+ [
+ -80.663037,
+ 35.043717
+ ]
+ ]
+ }
+ ]
+ }
+ }
+ ]
+ }
+ }
+ }
+]
\ No newline at end of file
diff --git a/indexer-acceptance-test/src/test/resources/testData/index_records_3.schema b/indexer-acceptance-test/src/test/resources/testData/index_records_3.schema
new file mode 100644
index 0000000000000000000000000000000000000000..bc347c7d037542ca59243e5162bb7ec08e7af2da
--- /dev/null
+++ b/indexer-acceptance-test/src/test/resources/testData/index_records_3.schema
@@ -0,0 +1,13 @@
+{
+ "kind": "KIND_VAL",
+ "schema": [
+ {
+ "path": "GeoShape",
+ "kind": "core:dl:geoshape:1.0.0"
+ },
+ {
+ "path": "WellName",
+ "kind": "string"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/indexer-acceptance-test/src/test/resources/testData/index_records_3.schema.json b/indexer-acceptance-test/src/test/resources/testData/index_records_3.schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..cb41544608a800f73b61de0ea449cd5a1b12e537
--- /dev/null
+++ b/indexer-acceptance-test/src/test/resources/testData/index_records_3.schema.json
@@ -0,0 +1,563 @@
+{
+ "schemaInfo": {
+ "schemaIdentity": {
+ "authority": "tenant1",
+ "source": "indexer",
+ "entityType": "test-data--Integration",
+ "schemaVersionMajor": "3",
+ "schemaVersionMinor": "0",
+ "schemaVersionPatch": "1"
+ },
+ "status": "DEVELOPMENT"
+ },
+ "schema": {
+ "properties": {
+ "data": {
+ "allOf": [
+ {
+ "type": "object",
+ "properties": {
+ "GeoShape": {
+ "$ref": "#/definitions/opendes:wks:AbstractFeatureCollection:1.0.0",
+ "description": "The wellbore's position .",
+ "format": "core:dl:geopoint:1.0.0",
+ "title": "WGS 84 Position",
+ "type": "object"
+ },
+ "WellName": {
+ "type": "string"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "definitions": {
+ "opendes:wks:AbstractFeatureCollection:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2021, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractFeatureCollection:1.0.0",
+ "description": "GeoJSON feature collection as originally published in https://geojson.org/schema/FeatureCollection.json. Attention: the coordinate order is fixed: Longitude first, followed by Latitude, optionally height above MSL (EPSG:5714) as third coordinate.",
+ "title": "GeoJSON FeatureCollection",
+ "type": "object",
+ "required": [
+ "type",
+ "features"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "FeatureCollection"
+ ]
+ },
+ "features": {
+ "type": "array",
+ "items": {
+ "title": "GeoJSON Feature",
+ "type": "object",
+ "required": [
+ "type",
+ "properties",
+ "geometry"
+ ],
+ "properties": {
+ "geometry": {
+ "oneOf": [
+ {
+ "type": "null"
+ },
+ {
+ "title": "GeoJSON Point",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "Point"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "GeoJSON LineString",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "LineString"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "GeoJSON Polygon",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "Polygon"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "GeoJSON MultiPoint",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "MultiPoint"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "GeoJSON MultiLineString",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "MultiLineString"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "GeoJSON MultiPolygon",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "items": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "MultiPolygon"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "GeoJSON GeometryCollection",
+ "type": "object",
+ "required": [
+ "type",
+ "geometries"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "GeometryCollection"
+ ]
+ },
+ "geometries": {
+ "type": "array",
+ "items": {
+ "oneOf": [
+ {
+ "title": "GeoJSON Point",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "Point"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "GeoJSON LineString",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "LineString"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "GeoJSON Polygon",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "Polygon"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "GeoJSON MultiPoint",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "MultiPoint"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "GeoJSON MultiLineString",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "MultiLineString"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "GeoJSON MultiPolygon",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "items": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "MultiPolygon"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ ]
+ }
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ ]
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "Feature"
+ ]
+ },
+ "properties": {
+ "oneOf": [
+ {
+ "type": "null"
+ },
+ {
+ "type": "object"
+ }
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractFeatureCollection.1.0.0.json"
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/indexer-acceptance-test/src/test/resources/testData/index_update_records_kind_v1.json b/indexer-acceptance-test/src/test/resources/testData/index_update_records_kind_v1.json
new file mode 100644
index 0000000000000000000000000000000000000000..b54dfca20dafd30d13be1cdb684faa49f8756b1f
--- /dev/null
+++ b/indexer-acceptance-test/src/test/resources/testData/index_update_records_kind_v1.json
@@ -0,0 +1,8 @@
+[
+ {
+ "id": "tenant1:test:testSchemaUpdate<timestamp>",
+ "data": {
+ "message": "test record1"
+ }
+ }
+]
\ No newline at end of file
diff --git a/indexer-acceptance-test/src/test/resources/testData/index_update_records_kind_v1.schema.json b/indexer-acceptance-test/src/test/resources/testData/index_update_records_kind_v1.schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..8af4b78a7f2c148061813ba2630b215d2f9e6aef
--- /dev/null
+++ b/indexer-acceptance-test/src/test/resources/testData/index_update_records_kind_v1.schema.json
@@ -0,0 +1,29 @@
+{
+ "schemaInfo": {
+ "schemaIdentity": {
+ "authority": "tenant1",
+ "source": "indexer",
+ "entityType": "test-update-data--Integration",
+ "schemaVersionMajor": "1",
+ "schemaVersionMinor": "0",
+ "schemaVersionPatch": "1"
+ },
+ "status": "DEVELOPMENT"
+ },
+ "schema": {
+ "properties": {
+ "data": {
+ "allOf": [
+ {
+ "type": "object",
+ "properties": {
+ "message": {
+ "type": "string"
+ }
+ }
+ }
+ ]
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/indexer-acceptance-test/src/test/resources/testData/index_update_records_kind_v2.json b/indexer-acceptance-test/src/test/resources/testData/index_update_records_kind_v2.json
new file mode 100644
index 0000000000000000000000000000000000000000..52c53de1058735b04702a0b60e821c9319cd9e6a
--- /dev/null
+++ b/indexer-acceptance-test/src/test/resources/testData/index_update_records_kind_v2.json
@@ -0,0 +1,8 @@
+[
+ {
+ "id": "tenant1:test:testSchemaUpdate<timestamp>",
+ "data": {
+ "message": "test record2"
+ }
+ }
+]
\ No newline at end of file
diff --git a/indexer-acceptance-test/src/test/resources/testData/index_update_records_kind_v2.schema.json b/indexer-acceptance-test/src/test/resources/testData/index_update_records_kind_v2.schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..161ff0658601905ef02bdbc5b5d94c0e34b95a33
--- /dev/null
+++ b/indexer-acceptance-test/src/test/resources/testData/index_update_records_kind_v2.schema.json
@@ -0,0 +1,29 @@
+{
+ "schemaInfo": {
+ "schemaIdentity": {
+ "authority": "tenant1",
+ "source": "indexer",
+ "entityType": "test-update-data--Integration",
+ "schemaVersionMajor": "2",
+ "schemaVersionMinor": "0",
+ "schemaVersionPatch": "1"
+ },
+ "status": "DEVELOPMENT"
+ },
+ "schema": {
+ "properties": {
+ "data": {
+ "allOf": [
+ {
+ "type": "object",
+ "properties": {
+ "message": {
+ "type": "string"
+ }
+ }
+ }
+ ]
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/indexer-acceptance-test/src/test/resources/testData/osdu_wks_IndexPropertyPathConfiguration_v1.json b/indexer-acceptance-test/src/test/resources/testData/osdu_wks_IndexPropertyPathConfiguration_v1.json
new file mode 100644
index 0000000000000000000000000000000000000000..6b31efc09f9e73926e8feb83660fc2a57be38c14
--- /dev/null
+++ b/indexer-acceptance-test/src/test/resources/testData/osdu_wks_IndexPropertyPathConfiguration_v1.json
@@ -0,0 +1,70 @@
+[{
+ "id": "tenant1:reference-data--IndexPropertyPathConfiguration:index-property--WellLog:1",
+ "data": {
+ "Name": "WellLogIndex-PropertyPathConfiguration",
+ "Description": "The index property list for WellLog:1., valid for all WellLog kinds for major version 1.",
+ "Code": "test:indexer:index-property--WellLog:1.",
+ "AttributionAuthority": "OSDU",
+ "Configurations": [{
+ "Name": "WellboreName",
+ "Policy": "ExtractFirstMatch",
+ "Paths": [{
+ "RelatedObjectsSpec": {
+ "RelationshipDirection": "ChildToParent",
+ "RelatedObjectKind": "test:indexer:index-property--Wellbore:1.",
+ "RelatedObjectID": "data.WellboreID"
+ },
+ "ValueExtraction": {
+ "ValuePath": "data.FacilityName"
+ }
+ }
+ ],
+ "UseCase": "As a user I want to discover WellLog instances by the wellbore's name value."
+ }, {
+ "Name": "SpatialLocation",
+ "Policy": "ExtractFirstMatch",
+ "Paths": [{
+ "RelatedObjectsSpec": {
+ "RelationshipDirection": "ChildToParent",
+ "RelatedObjectKind": "test:indexer:index-property--Wellbore:1.",
+ "RelatedObjectID": "data.WellboreID"
+ },
+ "ValueExtraction": {
+ "ValuePath": "data.Location"
+ }
+ }
+ ],
+ "UseCase": "As a user I want to discover WellLog instances by spatial location."
+ }
+ ]
+ }
+ }, {
+ "id": "tenant1:reference-data--IndexPropertyPathConfiguration:index-property--Wellbore:1",
+ "data": {
+ "Name": "Wellbore-IndexPropertyPathConfiguration",
+ "Description": "The index property list for index-property--Wellbore:1., valid for all index-property--Wellbore kinds for major version 1.",
+ "Code": "test:indexer:index-property--Wellbore:1.",
+ "AttributionAuthority": "OSDU",
+ "Configurations": [{
+ "Name": "WellUWI",
+ "Policy": "ExtractFirstMatch",
+ "Paths": [{
+ "ValueExtraction": {
+ "RelatedConditionMatches": [
+ "^[\\w\\-\\.]+:reference-data\\-\\-AliasNameType:UniqueIdentifier:$",
+ "^[\\w\\-\\.]+:reference-data\\-\\-AliasNameType:RegulatoryName:$",
+ "^[\\w\\-\\.]+:reference-data\\-\\-AliasNameType:PreferredName:$",
+ "^[\\w\\-\\.]+:reference-data\\-\\-AliasNameType:CommonName:$",
+ "^[\\w\\-\\.]+:reference-data\\-\\-AliasNameType:ShortName:$"
+ ],
+ "RelatedConditionProperty": "data.NameAliases[].AliasNameTypeID",
+ "ValuePath": "data.NameAliases[].AliasName"
+ }
+ }
+ ],
+ "UseCase": "As a user I want to discover and match Wells by their UWI. I am aware that this is not globally reliable, however, I am able to specify a prioritized AliasNameType list to look up value in the NameAliases array."
+ }
+ ]
+ }
+ }
+]
diff --git a/indexer-acceptance-test/src/test/resources/testData/osdu_wks_IndexPropertyPathConfiguration_v1.schema.json b/indexer-acceptance-test/src/test/resources/testData/osdu_wks_IndexPropertyPathConfiguration_v1.schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..1f1b3249c7d803c6d41b92bd95aacbe67fc17461
--- /dev/null
+++ b/indexer-acceptance-test/src/test/resources/testData/osdu_wks_IndexPropertyPathConfiguration_v1.schema.json
@@ -0,0 +1,780 @@
+{
+ "schemaInfo": {
+ "schemaIdentity": {
+ "authority": "osdu",
+ "source": "wks",
+ "entityType": "reference-data--IndexPropertyPathConfiguration",
+ "schemaVersionMajor": 1,
+ "schemaVersionMinor": 0,
+ "schemaVersionPatch": 0,
+ "id": "osdu:wks:reference-data--IndexPropertyPathConfiguration:1.0.0"
+ },
+ "status": "PUBLISHED"
+ },
+ "schema": {
+ "x-osdu-governance-model": "OPEN",
+ "x-osdu-inheriting-from-kind": [{
+ "kind": "osdu:wks:AbstractReferenceType:1.0.0",
+ "name": "ReferenceType"
+ }
+ ],
+ "x-osdu-license": "Copyright 2023, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:reference-data--IndexPropertyPathConfiguration:1.0.0",
+ "description": "IndexPropertyPathConfiguration contains the de-normalization configuration settings for the Search index. The record id contains the kind so that there is a unique relationship between the kind and the kind's index extensions.",
+ "title": "IndexPropertyPathConfiguration",
+ "type": "object",
+ "x-osdu-review-status": "Accepted",
+ "required": [
+ "kind",
+ "acl",
+ "legal"
+ ],
+ "x-osdu-governance-authorities": [
+ "OSDU"
+ ],
+ "x-osdu-virtual-properties": {
+ "data.VirtualProperties.DefaultName": {
+ "type": "string",
+ "priority": [{
+ "path": "data.Name"
+ }
+ ]
+ }
+ },
+ "additionalProperties": false,
+ "definitions": {
+ "osdu:wks:AbstractCommonResources:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractCommonResources:1.0.0",
+ "description": "Common resources to be injected at root 'data' level for every entity, which is persistable in Storage. The insertion is performed by the OsduSchemaComposer script.",
+ "x-osdu-review-status": "Accepted",
+ "title": "OSDU Common Resources",
+ "type": "object",
+ "properties": {
+ "ResourceHomeRegionID": {
+ "x-osdu-relationship": [{
+ "EntityType": "OSDURegion",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-OSDURegion:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The name of the home [cloud environment] region for this OSDU resource object.",
+ "type": "string",
+ "title": "Resource Home Region ID"
+ },
+ "ResourceHostRegionIDs": {
+ "description": "The name of the host [cloud environment] region(s) for this OSDU resource object.",
+ "type": "array",
+ "title": "Resource Host Region ID",
+ "items": {
+ "x-osdu-relationship": [{
+ "EntityType": "OSDURegion",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-OSDURegion:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "type": "string"
+ }
+ },
+ "ResourceLifecycleStatus": {
+ "x-osdu-relationship": [{
+ "EntityType": "ResourceLifecycleStatus",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-ResourceLifecycleStatus:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Describes the current Resource Lifecycle status.",
+ "type": "string",
+ "title": "Resource Lifecycle Status"
+ },
+ "ResourceSecurityClassification": {
+ "x-osdu-relationship": [{
+ "EntityType": "ResourceSecurityClassification",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-ResourceSecurityClassification:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Classifies the security level of the resource.",
+ "type": "string",
+ "title": "Resource Security Classification"
+ },
+ "ResourceCurationStatus": {
+ "x-osdu-relationship": [{
+ "EntityType": "ResourceCurationStatus",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-ResourceCurationStatus:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Describes the current Curation status.",
+ "type": "string",
+ "title": "Resource Curation Status"
+ },
+ "ExistenceKind": {
+ "x-osdu-relationship": [{
+ "EntityType": "ExistenceKind",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-ExistenceKind:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Where does this data resource sit in the cradle-to-grave span of its existence?",
+ "type": "string",
+ "title": "Existence Kind"
+ },
+ "TechnicalAssuranceID": {
+ "x-osdu-relationship": [{
+ "EntityType": "TechnicalAssuranceType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-TechnicalAssuranceType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "DEPRECATED: Describes a record's overall suitability for general business consumption based on data quality. Clarifications: Since Certified is the highest classification of suitable quality, any further change or versioning of a Certified record should be carefully considered and justified. If a Technical Assurance value is not populated then one can assume the data has not been evaluated or its quality is unknown (=Unevaluated). Technical Assurance values are not intended to be used for the identification of a single \"preferred\" or \"definitive\" record by comparison with other records.",
+ "type": "string",
+ "title": "Technical Assurance ID"
+ },
+ "Source": {
+ "description": "The entity that produced the record, or from which it is received; could be an organization, agency, system, internal team, or individual. For informational purposes only, the list of sources is not governed.",
+ "type": "string",
+ "title": "Data Source"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractCommonResources.1.0.0.json"
+ },
+ "osdu:wks:AbstractMetaItem:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "oneOf": [{
+ "title": "FrameOfReferenceUOM",
+ "type": "object",
+ "properties": {
+ "persistableReference": {
+ "description": "The self-contained, persistable reference string uniquely identifying the Unit.",
+ "title": "UOM Persistable Reference",
+ "type": "string",
+ "example": "{\"abcd\":{\"a\":0.0,\"b\":1200.0,\"c\":3937.0,\"d\":0.0},\"symbol\":\"ft[US]\",\"baseMeasurement\":{\"ancestry\":\"L\",\"type\":\"UM\"},\"type\":\"UAD\"}"
+ },
+ "kind": {
+ "const": "Unit",
+ "description": "The kind of reference, 'Unit' for FrameOfReferenceUOM.",
+ "title": "UOM Reference Kind",
+ "type": "string"
+ },
+ "propertyNames": {
+ "description": "The list of property names, to which this meta data item provides Unit context to. A full path like \"StructureA.PropertyB\" is required to define a unique context; \"data\" is omitted since frame-of reference normalization only applies to the data block.",
+ "title": "UOM Property Names",
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "example": [
+ "HorizontalDeflection.EastWest",
+ "HorizontalDeflection.NorthSouth"
+ ]
+ },
+ "name": {
+ "description": "The unit symbol or name of the unit.",
+ "title": "UOM Unit Symbol",
+ "type": "string",
+ "example": "ft[US]"
+ },
+ "unitOfMeasureID": {
+ "x-osdu-relationship": [{
+ "EntityType": "UnitOfMeasure",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-UnitOfMeasure:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "SRN to unit of measure reference.",
+ "type": "string",
+ "example": "namespace:reference-data--UnitOfMeasure:ftUS:"
+ }
+ },
+ "required": [
+ "kind",
+ "persistableReference"
+ ]
+ }, {
+ "title": "FrameOfReferenceCRS",
+ "type": "object",
+ "properties": {
+ "coordinateReferenceSystemID": {
+ "x-osdu-relationship": [{
+ "EntityType": "CoordinateReferenceSystem",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-CoordinateReferenceSystem:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "SRN to CRS reference.",
+ "type": "string",
+ "example": "namespace:reference-data--CoordinateReferenceSystem:Projected:EPSG::32615:"
+ },
+ "persistableReference": {
+ "description": "The self-contained, persistable reference string uniquely identifying the CRS.",
+ "title": "CRS Persistable Reference",
+ "type": "string",
+ "example": "{\"authCode\":{\"auth\":\"EPSG\",\"code\":\"32615\"},\"name\":\"WGS_1984_UTM_Zone_15N\",\"type\":\"LBC\",\"ver\":\"PE_10_9_1\",\"wkt\":\"PROJCS[\\\"WGS_1984_UTM_Zone_15N\\\",GEOGCS[\\\"GCS_WGS_1984\\\",DATUM[\\\"D_WGS_1984\\\",SPHEROID[\\\"WGS_1984\\\",6378137.0,298.257223563]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],PROJECTION[\\\"Transverse_Mercator\\\"],PARAMETER[\\\"False_Easting\\\",500000.0],PARAMETER[\\\"False_Northing\\\",0.0],PARAMETER[\\\"Central_Meridian\\\",-93.0],PARAMETER[\\\"Scale_Factor\\\",0.9996],PARAMETER[\\\"Latitude_Of_Origin\\\",0.0],UNIT[\\\"Meter\\\",1.0],AUTHORITY[\\\"EPSG\\\",32615]]\"}"
+ },
+ "kind": {
+ "const": "CRS",
+ "description": "The kind of reference, constant 'CRS' for FrameOfReferenceCRS.",
+ "title": "CRS Reference Kind",
+ "type": "string"
+ },
+ "propertyNames": {
+ "description": "The list of property names, to which this meta data item provides CRS context to. A full path like \"StructureA.PropertyB\" is required to define a unique context; \"data\" is omitted since frame-of reference normalization only applies to the data block.",
+ "title": "CRS Property Names",
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "example": [
+ "KickOffPosition.X",
+ "KickOffPosition.Y"
+ ]
+ },
+ "name": {
+ "description": "The name of the CRS.",
+ "title": "CRS Name",
+ "type": "string",
+ "example": "WGS 84 / UTM zone 15N"
+ }
+ },
+ "required": [
+ "kind",
+ "persistableReference"
+ ]
+ }, {
+ "title": "FrameOfReferenceDateTime",
+ "type": "object",
+ "properties": {
+ "persistableReference": {
+ "description": "The self-contained, persistable reference string uniquely identifying DateTime reference.",
+ "title": "DateTime Persistable Reference",
+ "type": "string",
+ "example": "{\"format\":\"yyyy-MM-ddTHH:mm:ssZ\",\"timeZone\":\"UTC\",\"type\":\"DTM\"}"
+ },
+ "kind": {
+ "const": "DateTime",
+ "description": "The kind of reference, constant 'DateTime', for FrameOfReferenceDateTime.",
+ "title": "DateTime Reference Kind",
+ "type": "string"
+ },
+ "propertyNames": {
+ "description": "The list of property names, to which this meta data item provides DateTime context to. A full path like \"StructureA.PropertyB\" is required to define a unique context; \"data\" is omitted since frame-of reference normalization only applies to the data block.",
+ "title": "DateTime Property Names",
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "example": [
+ "Acquisition.StartTime",
+ "Acquisition.EndTime"
+ ]
+ },
+ "name": {
+ "description": "The name of the DateTime format and reference.",
+ "title": "DateTime Name",
+ "type": "string",
+ "example": "UTC"
+ }
+ },
+ "required": [
+ "kind",
+ "persistableReference"
+ ]
+ }, {
+ "title": "FrameOfReferenceAzimuthReference",
+ "type": "object",
+ "properties": {
+ "persistableReference": {
+ "description": "The self-contained, persistable reference string uniquely identifying AzimuthReference.",
+ "title": "AzimuthReference Persistable Reference",
+ "type": "string",
+ "example": "{\"code\":\"TrueNorth\",\"type\":\"AZR\"}"
+ },
+ "kind": {
+ "const": "AzimuthReference",
+ "description": "The kind of reference, constant 'AzimuthReference', for FrameOfReferenceAzimuthReference.",
+ "title": "AzimuthReference Reference Kind",
+ "type": "string"
+ },
+ "propertyNames": {
+ "description": "The list of property names, to which this meta data item provides AzimuthReference context to. A full path like \"StructureA.PropertyB\" is required to define a unique context; \"data\" is omitted since frame-of reference normalization only applies to the data block.",
+ "title": "AzimuthReference Property Names",
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "example": [
+ "Bearing"
+ ]
+ },
+ "name": {
+ "description": "The name of the CRS or the symbol/name of the unit.",
+ "title": "AzimuthReference Name",
+ "type": "string",
+ "example": "TrueNorth"
+ }
+ },
+ "required": [
+ "kind",
+ "persistableReference"
+ ]
+ }
+ ],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractMetaItem:1.0.0",
+ "description": "A meta data item, which allows the association of named properties or property values to a Unit/Measurement/CRS/Azimuth/Time context.",
+ "title": "Frame of Reference Meta Data Item",
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractMetaItem.1.0.0.json"
+ },
+ "osdu:wks:AbstractLegalTags:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractLegalTags:1.0.0",
+ "description": "Legal meta data like legal tags, relevant other countries, legal status. This structure is included by the SystemProperties \"legal\", which is part of all OSDU records. Not extensible.",
+ "additionalProperties": false,
+ "title": "Legal Meta Data",
+ "type": "object",
+ "properties": {
+ "legaltags": {
+ "description": "The list of legal tags, which resolve to legal properties (like country of origin, export classification code, etc.) and rules with the help of the Compliance Service.",
+ "title": "Legal Tags",
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "otherRelevantDataCountries": {
+ "description": "The list of other relevant data countries as an array of two-letter country codes, see https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2.",
+ "title": "Other Relevant Data Countries",
+ "type": "array",
+ "items": {
+ "pattern": "^[A-Z]{2}$",
+ "type": "string"
+ }
+ },
+ "status": {
+ "pattern": "^(compliant|uncompliant)$",
+ "description": "The legal status. Set by the system after evaluation against the compliance rules associated with the \"legaltags\" using the Compliance Service.",
+ "title": "Legal Status",
+ "type": "string"
+ }
+ },
+ "required": [
+ "legaltags",
+ "otherRelevantDataCountries"
+ ],
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractLegalTags.1.0.0.json"
+ },
+ "osdu:wks:AbstractReferenceType:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractReferenceType:1.0.0",
+ "description": "Generic reference object containing the universal properties of reference data, especially the ones commonly thought of as Types",
+ "x-osdu-review-status": "Accepted",
+ "title": "AbstractReferenceType",
+ "type": "object",
+ "properties": {
+ "AttributionAuthority": {
+ "description": "Name of the authority, or organisation, which governs the entity value and from which it is sourced.",
+ "type": "string"
+ },
+ "AttributionPublication": {
+ "description": "Name, URL, or other identifier of the publication, or repository, of the attribution source organisation from which the entity value is sourced.",
+ "type": "string"
+ },
+ "InactiveIndicator": {
+ "description": "By default reference values are considered as 'active'. An absent 'InactiveIndicator' property value means the reference value is in active use. When 'InactiveIndicator' is set true the reverence value is no longer in use and should no longer be offered as a choice.",
+ "type": "boolean",
+ "title": "Inactive Indicator"
+ },
+ "Description": {
+ "description": "The text which describes a NAME TYPE in detail.",
+ "type": "string"
+ },
+ "NameAlias": {
+ "x-osdu-indexing": {
+ "type": "nested"
+ },
+ "description": "Alternative names, including historical, by which this entity instance is/has been known.",
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/osdu:wks:AbstractAliasNames:1.0.0"
+ }
+ },
+ "AttributionRevision": {
+ "description": "The distinct instance of the attribution publication, by version number, sequence number, date of publication, etc., that was used for the entity value.",
+ "type": "string"
+ },
+ "ID": {
+ "description": "Native identifier from a Master Data Management System or other trusted source external to OSDU - stored here in order to allow for multi-system connection and synchronization. If used, the \"Source\" property should identify that source system.",
+ "type": "string",
+ "title": "External Identifier"
+ },
+ "Code": {
+ "description": "The abbreviation or mnemonic for a reference type if defined. Example: WELL and WLBR.",
+ "type": "string",
+ "x-osdu-natural-key": 0
+ },
+ "CommitDate": {
+ "format": "date-time",
+ "description": "For reference values published and governed by OSDU: The date and time the record was committed into the OSDU member GitLab reference-values repository. The sole purpose of this date is to optimise the OSDU milestone upgrades. It allows the upgrade code to figure out whether or not the record must be PUT into reference value storage.",
+ "type": "string",
+ "title": "Commit Date",
+ "example": "2021-02-08T21:36:40-05:00"
+ },
+ "Name": {
+ "description": "The name of the entity instance.",
+ "type": "string",
+ "x-osdu-natural-key": 1
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractReferenceType.1.0.0.json"
+ },
+ "osdu:wks:AbstractLegalParentList:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractLegalParentList:1.0.0",
+ "description": "A list of entity id:version references to record instances recorded in the data platform, from which the current record is derived and from which the legal tags must be derived. This structure is included by the SystemProperties \"ancestry\", which is part of all OSDU records. Not extensible.",
+ "additionalProperties": false,
+ "title": "Parent List",
+ "type": "object",
+ "properties": {
+ "parents": {
+ "description": "An array of none, one or many entity references of 'direct parents' in the data platform, which mark the current record as a derivative. In contrast to other relationships, the source record version is required. During record creation or update the ancestry.parents[] relationships are used to collect the legal tags from the sources and aggregate them in the legal.legaltags[] array. As a consequence, should e.g., one or more of the legal tags of the source data expire, the access to the derivatives is also terminated. For details, see ComplianceService tutorial, 'Creating derivative Records'.",
+ "title": "Parents",
+ "type": "array",
+ "items": {
+ "x-osdu-relationship": [],
+ "pattern": "^[\\w\\-\\.]+:[\\w\\-\\.]+:[\\w\\-\\.\\:\\%]+:[0-9]+$",
+ "type": "string"
+ },
+ "example": []
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractLegalParentList.1.0.0.json"
+ },
+ "osdu:wks:AbstractAliasNames:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractAliasNames:1.0.0",
+ "description": "A list of alternative names for an object. The preferred name is in a separate, scalar property. It may or may not be repeated in the alias list, though a best practice is to include it if the list is present, but to omit the list if there are no other names. Note that the abstract entity is an array so the $ref to it is a simple property reference.",
+ "x-osdu-review-status": "Accepted",
+ "title": "AbstractAliasNames",
+ "type": "object",
+ "properties": {
+ "AliasNameTypeID": {
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-AliasNameType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "A classification of alias names such as by role played or type of source, such as regulatory name, regulatory code, company code, international standard name, etc.",
+ "x-osdu-relationship": [{
+ "EntityType": "AliasNameType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "type": "string"
+ },
+ "EffectiveDateTime": {
+ "format": "date-time",
+ "type": "string",
+ "description": "The date and time when an alias name becomes effective."
+ },
+ "AliasName": {
+ "type": "string",
+ "description": "Alternative Name value of defined name type for an object."
+ },
+ "TerminationDateTime": {
+ "format": "date-time",
+ "type": "string",
+ "description": "The data and time when an alias name is no longer in effect."
+ },
+ "DefinitionOrganisationID": {
+ "pattern": "^[\\w\\-\\.]+:(reference-data\\-\\-StandardsOrganisation|master-data\\-\\-Organisation):[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The StandardsOrganisation (reference-data) or Organisation (master-data) that provided the name (the source).",
+ "x-osdu-relationship": [{
+ "EntityType": "StandardsOrganisation",
+ "GroupType": "reference-data"
+ }, {
+ "EntityType": "Organisation",
+ "GroupType": "master-data"
+ }
+ ],
+ "type": "string"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractAliasNames.1.0.0.json"
+ },
+ "osdu:wks:AbstractAccessControlList:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractAccessControlList:1.0.0",
+ "description": "The access control tags associated with this entity. This structure is included by the SystemProperties \"acl\", which is part of all OSDU records. Not extensible.",
+ "additionalProperties": false,
+ "title": "Access Control List",
+ "type": "object",
+ "properties": {
+ "viewers": {
+ "description": "The list of viewers to which this data record is accessible/visible/discoverable formatted as an email (core.common.model.storage.validation.ValidationDoc.EMAIL_REGEX).",
+ "title": "List of Viewers",
+ "type": "array",
+ "items": {
+ "pattern": "^[a-zA-Z0-9_+&*-]+(?:\\.[a-zA-Z0-9_+&*-]+)*@(?:[a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,7}$",
+ "type": "string"
+ }
+ },
+ "owners": {
+ "description": "The list of owners of this data record formatted as an email (core.common.model.storage.validation.ValidationDoc.EMAIL_REGEX).",
+ "title": "List of Owners",
+ "type": "array",
+ "items": {
+ "pattern": "^[a-zA-Z0-9_+&*-]+(?:\\.[a-zA-Z0-9_+&*-]+)*@(?:[a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,7}$",
+ "type": "string"
+ }
+ }
+ },
+ "required": [
+ "owners",
+ "viewers"
+ ],
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractAccessControlList.1.0.0.json"
+ }
+ },
+ "properties": {
+ "ancestry": {
+ "description": "The links to data, which constitute the inputs, from which this record instance is derived.",
+ "title": "Ancestry",
+ "$ref": "#/definitions/osdu:wks:AbstractLegalParentList:1.0.0"
+ },
+ "data": {
+ "allOf": [{
+ "$ref": "#/definitions/osdu:wks:AbstractCommonResources:1.0.0"
+ }, {
+ "$ref": "#/definitions/osdu:wks:AbstractReferenceType:1.0.0"
+ }, {
+ "type": "object",
+ "title": "IndividualProperties",
+ "properties": {
+ "Configurations": {
+ "x-osdu-indexing": {
+ "type": "nested"
+ },
+ "description": "The list of index property configurations for the specific kind.",
+ "type": "array",
+ "title": "Configurations",
+ "items": {
+ "description": "One single configuration to derive an Search index property value and assign it to the index 'column' with Name.",
+ "type": "object",
+ "title": "Configuration",
+ "properties": {
+ "Policy": {
+ "pattern": "^(ExtractFirstMatch|ExtractAllMatches)$",
+ "description": "Current supported policies are 'ExtractAllMatches' resulting in an array of values or 'ExtractFirstMatch' single value. The policy applies only to the Paths[].ValueExtraction.",
+ "type": "string",
+ "title": "Extraction Policy",
+ "example": "ExtractAllMatches"
+ },
+ "UseCase": {
+ "description": "The use case description this configuration satisfies.",
+ "type": "string",
+ "title": "Use Case",
+ "example": "As a user I want to find objects by a country name, with the understanding that an object may extend over country boundaries."
+ },
+ "Paths": {
+ "x-osdu-indexing": {
+ "type": "nested"
+ },
+ "description": "The list of path definitions to derive the property value from.",
+ "type": "array",
+ "title": "Paths",
+ "items": {
+ "description": "A single path definition to derive a property value from.",
+ "type": "object",
+ "title": "Path",
+ "properties": {
+ "RelatedObjectsSpec": {
+ "description": "The specification to extract related objects, from which to derive the ValueExtraction. If this property is empty or absent, the ValueExtraction is done on the current object to be indexed.",
+ "type": "object",
+ "title": "Related Objects Specification",
+ "properties": {
+ "RelatedObjectID": {
+ "description": "The path to the property containing the ID of the target record to chase. This property is only populated if the property is extracted from a related object, which must be chased. If the property is derived from 'within' the same record, which triggered the indexing, the RelatedObjectID is left absent.",
+ "type": "string",
+ "title": "Related Object ID",
+ "example": "GeoContexts[].GeoPoliticalEntityID"
+ },
+ "RelatedObjectKind": {
+ "pattern": "^[\\w\\-\\.]+:[\\w\\-\\.]+:[\\w\\-\\.]+:[0-9]+.$",
+ "description": "The kind or schema id expected as the target object type. This property is only populated if the property is extracted from a related object, which must be chased. If the property is derived from 'within' the same record, which triggered the indexing, the RelatedObjectKind is left absent.",
+ "type": "string",
+ "title": "Related Object Kind",
+ "example": "osdu:wks:master-data--GeoPoliticalEntity:1."
+ },
+ "RelationshipDirection": {
+ "pattern": "^(ChildToParent|ParentToChildren)$",
+ "description": "The direction of the relationship definition seen from the object being indexed. 'ChildToParent' assumes an outgoing relationship with the target record defined in the object being indexed. 'ParentToChildren' assumes that the related objects have a relationship by RelatedObjectID to the id of the record being indexed.",
+ "type": "string",
+ "title": "Relationship Direction",
+ "example": "ChildToParent"
+ },
+ "RelatedConditionMatches": {
+ "description": "The RelatedConditionProperty values, which need to match in order to be accepted as de-normalized value(s). If the Policy is ExtractFirstMatch, the list is prioritized and the first match is accepted as final value. Policy ExtractAllMatches collects all matching values as array.",
+ "type": "array",
+ "title": "Related Condition Matches",
+ "items": {
+ "type": "string"
+ },
+ "example": [
+ "namespace:reference-data--GeoPoliticalEntityType:Country:"
+ ]
+ },
+ "RelatedConditionProperty": {
+ "description": "The property path of the target record data block, which needs subjected to the conditional matching. The data prefix is not required.",
+ "type": "string",
+ "title": "Related Condition Property",
+ "example": "GeoContexts[].GeoTypeID"
+ }
+ }
+ },
+ "ValueExtraction": {
+ "description": "The instructions from where to derive the value.",
+ "type": "object",
+ "title": "Value Extraction",
+ "properties": {
+ "RelatedConditionMatches": {
+ "description": "The RelatedConditionProperty values, which need to match in order to be accepted as de-normalized value(s). If the Policy is ExtractFirstMatch, the list is prioritized and the first match is accepted as final value. Policy ExtractAllMatches collects all matching values as array.",
+ "type": "array",
+ "title": "Related Condition Matches",
+ "items": {
+ "type": "string"
+ }
+ },
+ "ValuePath": {
+ "description": "The path to the property from where to extract the de-normalized value. The data prefix is not required in the path.",
+ "type": "string",
+ "title": "Value Path",
+ "example": "GeoPoliticalEntityName"
+ },
+ "RelatedConditionProperty": {
+ "description": "The property path of the target record data block, which needs to be subjected to the conditional matching. The data prefix is not required in the path.",
+ "type": "string",
+ "title": "Related Condition Property"
+ }
+ },
+ "required": [
+ "ValuePath"
+ ]
+ }
+ },
+ "required": [
+ "ValueExtraction"
+ ]
+ }
+ },
+ "Name": {
+ "description": "The name of the indexed property, i.e., this is the property name used in Search.",
+ "type": "string",
+ "title": "Name",
+ "example": "CountryNames"
+ }
+ }
+ }
+ }
+ }
+ }, {
+ "type": "object",
+ "title": "ExtensionProperties",
+ "properties": {
+ "ExtensionProperties": {
+ "type": "object"
+ }
+ }
+ }
+ ]
+ },
+ "kind": {
+ "pattern": "^[\\w\\-\\.]+:[\\w\\-\\.]+:[\\w\\-\\.]+:[0-9]+.[0-9]+.[0-9]+$",
+ "description": "The schema identification for the OSDU resource object following the pattern {Namespace}:{Source}:{Type}:{VersionMajor}.{VersionMinor}.{VersionPatch}. The versioning scheme follows the semantic versioning, https://semver.org/.",
+ "title": "Entity Kind",
+ "type": "string",
+ "example": "osdu:wks:reference-data--IndexPropertyPathConfiguration:1.0.0"
+ },
+ "acl": {
+ "description": "The access control tags associated with this entity.",
+ "title": "Access Control List",
+ "$ref": "#/definitions/osdu:wks:AbstractAccessControlList:1.0.0"
+ },
+ "version": {
+ "format": "int64",
+ "description": "The version number of this OSDU resource; set by the framework.",
+ "title": "Version Number",
+ "type": "integer",
+ "example": 1562066009929332
+ },
+ "tags": {
+ "description": "A generic dictionary of string keys mapping to string value. Only strings are permitted as keys and values.",
+ "additionalProperties": {
+ "type": "string"
+ },
+ "title": "Tag Dictionary",
+ "type": "object",
+ "example": {
+ "NameOfKey": "String value"
+ }
+ },
+ "modifyUser": {
+ "description": "The user reference, which created this version of this resource object. Set by the System.",
+ "title": "Resource Object Version Creation User Reference",
+ "type": "string",
+ "example": "some-user@some-company-cloud.com"
+ },
+ "modifyTime": {
+ "format": "date-time",
+ "description": "Timestamp of the time at which this version of the OSDU resource object was created. Set by the System. The value is a combined date-time string in ISO-8601 given in UTC.",
+ "title": "Resource Object Version Creation DateTime",
+ "type": "string",
+ "example": "2020-12-16T11:52:24.477Z"
+ },
+ "createTime": {
+ "format": "date-time",
+ "description": "Timestamp of the time at which initial version of this OSDU resource object was created. Set by the System. The value is a combined date-time string in ISO-8601 given in UTC.",
+ "title": "Resource Object Creation DateTime",
+ "type": "string",
+ "example": "2020-12-16T11:46:20.163Z"
+ },
+ "meta": {
+ "description": "The Frame of Reference meta data section linking the named properties to self-contained definitions.",
+ "title": "Frame of Reference Meta Data",
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/osdu:wks:AbstractMetaItem:1.0.0"
+ }
+ },
+ "legal": {
+ "description": "The entity's legal tags and compliance status. The actual contents associated with the legal tags is managed by the Compliance Service.",
+ "title": "Legal Tags",
+ "$ref": "#/definitions/osdu:wks:AbstractLegalTags:1.0.0"
+ },
+ "createUser": {
+ "description": "The user reference, which created the first version of this resource object. Set by the System.",
+ "title": "Resource Object Creation User Reference",
+ "type": "string",
+ "example": "some-user@some-company-cloud.com"
+ },
+ "id": {
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-IndexPropertyPathConfiguration:[\\w\\-\\.\\:\\%]+$",
+ "description": "Previously called ResourceID or SRN which identifies this OSDU resource object without version.",
+ "title": "Entity ID",
+ "type": "string",
+ "example": "namespace:reference-data--IndexPropertyPathConfiguration:c9d84708-2b1b-5e0b-954e-9621132f7154"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/reference-data/IndexPropertyPathConfiguration.1.0.0.json"
+ }
+}
diff --git a/indexer-acceptance-test/src/test/resources/testData/r3-index_record_arrayofobjects.json b/indexer-acceptance-test/src/test/resources/testData/r3-index_record_arrayofobjects.json
new file mode 100644
index 0000000000000000000000000000000000000000..f37563c946a8b75e530dd1cd96017233ab40db8f
--- /dev/null
+++ b/indexer-acceptance-test/src/test/resources/testData/r3-index_record_arrayofobjects.json
@@ -0,0 +1,43 @@
+[
+ {
+ "id": "tenant1:<kindSubType>:testIngest70",
+ "data": {
+ "NestedTest": [
+ {
+ "DateTimeTest": "2020-02-13T09:13:15.55Z",
+ "NumberTest": "12345",
+ "StringTest": "test string"
+ },
+ {
+ "DateTimeTest": "2020-02-13T09:13:15.55Z",
+ "NumberTest": "567890",
+ "StringTest": "test string"
+ }
+ ],
+ "FlattenedTest": [
+ {
+ "DateTimeTest": "2020-02-13T09:13:15.55Z",
+ "NumberTest": "12345",
+ "StringTest": "test string"
+ },
+ {
+ "DateTimeTest": "2020-02-13T09:13:15.55Z",
+ "NumberTest": "567890",
+ "StringTest": "test string"
+ }
+ ],
+ "ObjectTest": [
+ {
+ "DateTimeTest": "2020-02-13T09:13:15.55Z",
+ "NumberTest": "12345",
+ "StringTest": "test string"
+ },
+ {
+ "DateTimeTest": "2020-02-13T09:13:15.55Z",
+ "NumberTest": "567890",
+ "StringTest": "test string"
+ }
+ ]
+ }
+ }
+]
\ No newline at end of file
diff --git a/indexer-acceptance-test/src/test/resources/testData/r3-index_record_arrayofobjects.schema.json b/indexer-acceptance-test/src/test/resources/testData/r3-index_record_arrayofobjects.schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..c68efaceb42e6b11cb8ea857a85459e1dedc89a1
--- /dev/null
+++ b/indexer-acceptance-test/src/test/resources/testData/r3-index_record_arrayofobjects.schema.json
@@ -0,0 +1,112 @@
+{
+ "schemaInfo": {
+ "schemaIdentity": {
+ "authority": "tenant1",
+ "source": "wks",
+ "entityType": "ArraysOfObjectsTestCollection",
+ "schemaVersionMajor": 4,
+ "schemaVersionMinor": 0,
+ "schemaVersionPatch": 0
+ },
+ "status": "DEVELOPMENT"
+ },
+ "schema": {
+ "x-osdu-license": "Copyright 2021, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "description": "Array of objects testing schema",
+ "title": "Test",
+ "type": "object",
+ "required": [
+ "kind",
+ "acl",
+ "legal"
+ ],
+ "properties": {
+ "data": {
+ "allOf": [
+ {
+ "type": "object",
+ "properties": {
+ "NestedTest": {
+ "description": "nested type test",
+ "type": "array",
+ "x-osdu-indexing": {
+ "type": "nested"
+ },
+ "items": {
+ "type": "object",
+ "properties": {
+ "DateTimeTest": {
+ "description": "date and time test",
+ "type": "string",
+ "format": "date-time",
+ "x-osdu-frame-of-reference": "DateTime"
+ },
+ "NumberTest": {
+ "description": "number test",
+ "type": "number"
+ },
+ "StringTest": {
+ "description": "string test",
+ "type": "string"
+ }
+ }
+ }
+ },
+ "FlattenedTest": {
+ "description": "flattened type test",
+ "type": "array",
+ "x-osdu-indexing": {
+ "type": "flattened"
+ },
+ "items": {
+ "type": "object",
+ "properties": {
+ "DateTimeTest": {
+ "description": "date and time test",
+ "type": "string",
+ "format": "date-time",
+ "x-osdu-frame-of-reference": "DateTime"
+ },
+ "NumberTest": {
+ "description": "number test",
+ "type": "number"
+ },
+ "StringTest": {
+ "description": "string test",
+ "type": "string"
+ }
+ }
+ }
+ },
+ "ObjectTest": {
+ "description": "default object type test",
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "DateTimeTest": {
+ "description": "date and time test",
+ "type": "string",
+ "format": "date-time",
+ "x-osdu-frame-of-reference": "DateTime"
+ },
+ "NumberTest": {
+ "description": "number test",
+ "type": "number"
+ },
+ "StringTest": {
+ "description": "string test",
+ "type": "string"
+ }
+ }
+ }
+ }
+ }
+ }
+ ]
+ }
+ },
+ "x-osdu-inheriting-from-kind": []
+ }
+}
\ No newline at end of file
diff --git a/indexer-acceptance-test/src/test/resources/testData/r3-index_record_wks_master.json b/indexer-acceptance-test/src/test/resources/testData/r3-index_record_wks_master.json
new file mode 100644
index 0000000000000000000000000000000000000000..de6e08dab7f12b7dc2fdc4a8108b87692694a9cb
--- /dev/null
+++ b/indexer-acceptance-test/src/test/resources/testData/r3-index_record_wks_master.json
@@ -0,0 +1,143 @@
+[
+ {
+ "id": "tenant1:<kindSubType>:testIngest61<timestamp>",
+ "data": {
+ "NameAliases": [
+ {
+ "AliasName": "1001",
+ "AliasNameTypeID": "osdu:reference-data--AliasNameType:UWI:"
+ },
+ {
+ "AliasName": "AHM-01",
+ "AliasNameTypeID": "osdu:reference-data--AliasNameType:Borehole%20Code:"
+ },
+ {
+ "AliasName": "ARNHEM-01",
+ "AliasNameTypeID": "osdu:reference-data--AliasNameType:Borehole:"
+ }
+ ],
+ "GeoContexts": [
+ {
+ "GeoPoliticalEntityID": "osdu:master-data--GeoPoliticalEntity:Gelderland:",
+ "GeoTypeID": "osdu:reference-data--GeoPoliticalEntityType:Province:"
+ },
+ {
+ "GeoPoliticalEntityID": "osdu:master-data--GeoPoliticalEntity:Lingewaard:",
+ "GeoTypeID": "osdu:reference-data--GeoPoliticalEntityType:Municipality:"
+ }
+ ],
+ "SpatialLocation": {
+ "AsIngestedCoordinates": {
+ "type": "AnyCrsFeatureCollection",
+ "CoordinateReferenceSystemID": "osdu:reference-data--CoordinateReferenceSystem:ED_1950_UTM_Zone_31N:",
+ "features": [
+ {
+ "type": "AnyCrsFeature",
+ "geometry": {
+ "type": "AnyCrsPoint",
+ "coordinates": [
+ 700113.0,
+ 5757315.0
+ ]
+ },
+ "properties": {}
+ }
+ ],
+ "persistableReferenceCrs": ""
+ },
+ "Wgs84Coordinates": {
+ "type": "FeatureCollection",
+ "features": [
+ {
+ "type": "Feature",
+ "geometry": {
+ "type": "Point",
+ "coordinates": [
+ 5.90929597,
+ 51.92868061
+ ]
+ },
+ "properties": {}
+ }
+ ]
+ }
+ },
+ "InitialOperatorID": "osdu:master-data--Organisation:Bataafse%20Petroleum%20Maatschappij:",
+ "CurrentOperatorID": "osdu:master-data--Organisation:Nederlandse%20Aardolie%20Maatschappij%20B.V.:",
+ "OperatingEnvironmentID": "osdu:reference-data--OperatingEnvironment:ON:",
+ "FacilityStates": [
+ {
+ "FacilityStateTypeID": "osdu:reference-data--FacilityStateType:Abandoned:"
+ }
+ ],
+ "FacilityEvents": [
+ {
+ "FacilityEventTypeID": "osdu:reference-data--FacilityEventType:Drilling%20Start:",
+ "EffectiveDateTime": "1944-03-01T00:00:00"
+ },
+ {
+ "FacilityEventTypeID": "osdu:reference-data--FacilityEventType:Drilling%20Finish:",
+ "EffectiveDateTime": "1944-10-02T00:00:00"
+ }
+ ],
+ "WellID": "osdu:master-data--Well:1001:",
+ "SequenceNumber": 1,
+ "VerticalMeasurements": [
+ {
+ "VerticalMeasurementID": "TD-Original",
+ "VerticalMeasurement": 662.0,
+ "VerticalMeasurementTypeID": "osdu:reference-data--VerticalMeasurementType:Total%20Depth:",
+ "VerticalMeasurementPathID": "osdu:reference-data--VerticalMeasurementPath:Measured%20Depth:",
+ "VerticalMeasurementUnitOfMeasureID": "osdu:reference-data--UnitOfMeasure:M:",
+ "VerticalReferenceID": "Measured_From"
+ },
+ {
+ "VerticalMeasurementID": "TVD",
+ "VerticalMeasurement": 662.0,
+ "VerticalMeasurementTypeID": "osdu:reference-data--VerticalMeasurementType:TD:",
+ "VerticalMeasurementPathID": "osdu:reference-data--VerticalMeasurementPath:True%20Vertical%20Depth:",
+ "VerticalMeasurementUnitOfMeasureID": "osdu:reference-data--UnitOfMeasure:M:",
+ "VerticalReferenceID": "Measured_From"
+ },
+ {
+ "VerticalMeasurementID": "Measured_From",
+ "VerticalMeasurement": 11.01,
+ "VerticalMeasurementTypeID": "osdu:reference-data--VerticalMeasurementType:Rotary%20Table:",
+ "VerticalMeasurementPathID": "osdu:reference-data--VerticalMeasurementPath:Elevation:",
+ "VerticalMeasurementUnitOfMeasureID": "osdu:reference-data--UnitOfMeasure:M:",
+ "VerticalCRSID": "osdu:reference-data--CoordinateReferenceSystem:NAP:"
+ }
+ ],
+ "DrillingReasons": [
+ {
+ "DrillingReasonTypeID": "osdu:reference-data--DrillingReasonType:EXP-HC:"
+ },
+ {
+ "DrillingReasonTypeID": "osdu:reference-data--DrillingReasonType:Exploratie%20koolwaterstof:"
+ }
+ ],
+ "TrajectoryTypeID": "osdu:reference-data--WellboreTrajectoryType:Vertikaal:",
+ "PrimaryMaterialID": "osdu:reference-data--MaterialType:OIL:",
+ "ProjectedBottomHoleLocation": {
+ "AsIngestedCoordinates": {
+ "type": "AnyCrsFeatureCollection",
+ "CoordinateReferenceSystemID": "osdu:reference-data--CoordinateReferenceSystem:ED_1950_UTM_Zone_31N:",
+ "features": [
+ {
+ "type": "AnyCrsFeature",
+ "geometry": {
+ "type": "AnyCrsPoint",
+ "coordinates": [
+ 700113.0,
+ 5757315.0
+ ]
+ },
+ "properties": {}
+ }
+ ],
+ "persistableReferenceCrs": ""
+ }
+ }
+ }
+ }
+]
\ No newline at end of file
diff --git a/indexer-acceptance-test/src/test/resources/testData/r3-index_record_wks_master.schema.json b/indexer-acceptance-test/src/test/resources/testData/r3-index_record_wks_master.schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..0d69ae534698474cac255148334f09da4c08551e
--- /dev/null
+++ b/indexer-acceptance-test/src/test/resources/testData/r3-index_record_wks_master.schema.json
@@ -0,0 +1,2012 @@
+{
+ "schemaInfo": {
+ "schemaIdentity": {
+ "authority": "tenant1",
+ "source": "wks",
+ "entityType": "master-data--Wellbore",
+ "schemaVersionMajor": "2",
+ "schemaVersionMinor": "0",
+ "schemaVersionPatch": "3"
+ },
+ "status": "DEVELOPMENT"
+ },
+ "schema": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2021, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:master-data--Test:1.0.0",
+ "description": "Enter a meaningful description for Test.",
+ "title": "Test",
+ "type": "object",
+ "x-osdu-review-status": "Pending",
+ "required": [
+ "kind",
+ "acl",
+ "legal"
+ ],
+ "additionalProperties": false,
+ "definitions": {
+ "opendes:wks:AbstractGeoPoliticalContext:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2021, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractGeoPoliticalContext:1.0.0",
+ "description": "A single, typed geo-political entity reference, which is 'abstracted' to AbstractGeoContext and then aggregated by GeoContexts properties.",
+ "x-osdu-review-status": "Accepted",
+ "title": "AbstractGeoPoliticalContext",
+ "type": "object",
+ "properties": {
+ "GeoPoliticalEntityID": {
+ "pattern": "^[\\w\\-\\.]+:master-data\\-\\-GeoPoliticalEntity:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Reference to GeoPoliticalEntity.",
+ "x-osdu-relationship": [
+ {
+ "EntityType": "GeoPoliticalEntity",
+ "GroupType": "master-data"
+ }
+ ],
+ "type": "string"
+ },
+ "GeoTypeID": {
+ "x-osdu-is-derived": {
+ "RelationshipPropertyName": "GeoPoliticalEntityID",
+ "TargetPropertyName": "GeoPoliticalEntityTypeID"
+ },
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-GeoPoliticalEntityType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The GeoPoliticalEntityType reference of the GeoPoliticalEntity (via GeoPoliticalEntityID) for application convenience.",
+ "x-osdu-relationship": [
+ {
+ "EntityType": "GeoPoliticalEntityType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "type": "string"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractGeoPoliticalContext.1.0.0.json"
+ },
+ "opendes:wks:AbstractCommonResources:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2021, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractCommonResources:1.0.0",
+ "description": "Common resources to be injected at root 'data' level for every entity, which is persistable in Storage. The insertion is performed by the OsduSchemaComposer script.",
+ "title": "OSDU Common Resources",
+ "type": "object",
+ "properties": {
+ "ResourceHomeRegionID": {
+ "x-osdu-relationship": [
+ {
+ "EntityType": "OSDURegion",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-OSDURegion:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The name of the home [cloud environment] region for this OSDU resource object.",
+ "title": "Resource Home Region ID",
+ "type": "string"
+ },
+ "ResourceHostRegionIDs": {
+ "description": "The name of the host [cloud environment] region(s) for this OSDU resource object.",
+ "title": "Resource Host Region ID",
+ "type": "array",
+ "items": {
+ "x-osdu-relationship": [
+ {
+ "EntityType": "OSDURegion",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-OSDURegion:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "type": "string"
+ }
+ },
+ "ResourceLifecycleStatus": {
+ "x-osdu-relationship": [
+ {
+ "EntityType": "ResourceLifecycleStatus",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-ResourceLifecycleStatus:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Describes the current Resource Lifecycle status.",
+ "title": "Resource Lifecycle Status",
+ "type": "string"
+ },
+ "ResourceSecurityClassification": {
+ "x-osdu-relationship": [
+ {
+ "EntityType": "ResourceSecurityClassification",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-ResourceSecurityClassification:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Classifies the security level of the resource.",
+ "title": "Resource Security Classification",
+ "type": "string"
+ },
+ "ResourceCurationStatus": {
+ "x-osdu-relationship": [
+ {
+ "EntityType": "ResourceCurationStatus",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-ResourceCurationStatus:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Describes the current Curation status.",
+ "title": "Resource Curation Status",
+ "type": "string"
+ },
+ "ExistenceKind": {
+ "x-osdu-relationship": [
+ {
+ "EntityType": "ExistenceKind",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-ExistenceKind:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Where does this data resource sit in the cradle-to-grave span of its existence?",
+ "title": "Existence Kind",
+ "type": "string"
+ },
+ "Source": {
+ "description": "The entity that produced the record, or from which it is received; could be an organization, agency, system, internal team, or individual. For informational purposes only, the list of sources is not governed.",
+ "title": "Data Source",
+ "type": "string"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractCommonResources.1.0.0.json"
+ },
+ "opendes:wks:AbstractAliasNames:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2021, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractAliasNames:1.0.0",
+ "description": "A list of alternative names for an object. The preferred name is in a separate, scalar property. It may or may not be repeated in the alias list, though a best practice is to include it if the list is present, but to omit the list if there are no other names. Note that the abstract entity is an array so the $ref to it is a simple property reference.",
+ "x-osdu-review-status": "Accepted",
+ "title": "AbstractAliasNames",
+ "type": "object",
+ "properties": {
+ "AliasNameTypeID": {
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-AliasNameType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "A classification of alias names such as by role played or type of source, such as regulatory name, regulatory code, company code, international standard name, etc.",
+ "x-osdu-relationship": [
+ {
+ "EntityType": "AliasNameType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "type": "string"
+ },
+ "EffectiveDateTime": {
+ "format": "date-time",
+ "type": "string",
+ "description": "The date and time when an alias name becomes effective."
+ },
+ "AliasName": {
+ "type": "string",
+ "description": "Alternative Name value of defined name type for an object."
+ },
+ "TerminationDateTime": {
+ "format": "date-time",
+ "type": "string",
+ "description": "The data and time when an alias name is no longer in effect."
+ },
+ "DefinitionOrganisationID": {
+ "pattern": "^[\\w\\-\\.]+:(reference-data\\-\\-StandardsOrganisation|master-data\\-\\-Organisation):[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The StandardsOrganisation (reference-data) or Organisation (master-data) that provided the name (the source).",
+ "x-osdu-relationship": [
+ {
+ "EntityType": "StandardsOrganisation",
+ "GroupType": "reference-data"
+ },
+ {
+ "EntityType": "Organisation",
+ "GroupType": "master-data"
+ }
+ ],
+ "type": "string"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractAliasNames.1.0.0.json"
+ },
+ "opendes:wks:AbstractAnyCrsFeatureCollection:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2021, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractAnyCrsFeatureCollection:1.0.0",
+ "description": "A schema like GeoJSON FeatureCollection with a non-WGS 84 CRS context; based on https://geojson.org/schema/FeatureCollection.json. Attention: the coordinate order is fixed: Longitude/Easting/Westing/X first, followed by Latitude/Northing/Southing/Y, optionally height as third coordinate.",
+ "title": "AbstractAnyCrsFeatureCollection",
+ "type": "object",
+ "required": [
+ "type",
+ "persistableReferenceCrs",
+ "features"
+ ],
+ "properties": {
+ "CoordinateReferenceSystemID": {
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-CoordinateReferenceSystem:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The CRS reference into the CoordinateReferenceSystem catalog.",
+ "x-osdu-relationship": [
+ {
+ "EntityType": "CoordinateReferenceSystem",
+ "GroupType": "reference-data"
+ }
+ ],
+ "title": "Coordinate Reference System ID",
+ "type": "string",
+ "example": "namespace:reference-data--CoordinateReferenceSystem:BoundCRS.SLB.32021.15851:"
+ },
+ "persistableReferenceCrs": {
+ "description": "The CRS reference as persistableReference string. If populated, the CoordinateReferenceSystemID takes precedence.",
+ "type": "string",
+ "title": "CRS Reference",
+ "example": "{\"lateBoundCRS\":{\"wkt\":\"PROJCS[\\\"NAD_1927_StatePlane_North_Dakota_South_FIPS_3302\\\",GEOGCS[\\\"GCS_North_American_1927\\\",DATUM[\\\"D_North_American_1927\\\",SPHEROID[\\\"Clarke_1866\\\",6378206.4,294.9786982]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],PROJECTION[\\\"Lambert_Conformal_Conic\\\"],PARAMETER[\\\"False_Easting\\\",2000000.0],PARAMETER[\\\"False_Northing\\\",0.0],PARAMETER[\\\"Central_Meridian\\\",-100.5],PARAMETER[\\\"Standard_Parallel_1\\\",46.1833333333333],PARAMETER[\\\"Standard_Parallel_2\\\",47.4833333333333],PARAMETER[\\\"Latitude_Of_Origin\\\",45.6666666666667],UNIT[\\\"Foot_US\\\",0.304800609601219],AUTHORITY[\\\"EPSG\\\",32021]]\",\"ver\":\"PE_10_3_1\",\"name\":\"NAD_1927_StatePlane_North_Dakota_South_FIPS_3302\",\"authCode\":{\"auth\":\"EPSG\",\"code\":\"32021\"},\"type\":\"LBC\"},\"singleCT\":{\"wkt\":\"GEOGTRAN[\\\"NAD_1927_To_WGS_1984_79_CONUS\\\",GEOGCS[\\\"GCS_North_American_1927\\\",DATUM[\\\"D_North_American_1927\\\",SPHEROID[\\\"Clarke_1866\\\",6378206.4,294.9786982]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],GEOGCS[\\\"GCS_WGS_1984\\\",DATUM[\\\"D_WGS_1984\\\",SPHEROID[\\\"WGS_1984\\\",6378137.0,298.257223563]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],METHOD[\\\"NADCON\\\"],PARAMETER[\\\"Dataset_conus\\\",0.0],AUTHORITY[\\\"EPSG\\\",15851]]\",\"ver\":\"PE_10_3_1\",\"name\":\"NAD_1927_To_WGS_1984_79_CONUS\",\"authCode\":{\"auth\":\"EPSG\",\"code\":\"15851\"},\"type\":\"ST\"},\"ver\":\"PE_10_3_1\",\"name\":\"NAD27 * OGP-Usa Conus / North Dakota South [32021,15851]\",\"authCode\":{\"auth\":\"SLB\",\"code\":\"32021079\"},\"type\":\"EBC\"}"
+ },
+ "features": {
+ "type": "array",
+ "items": {
+ "title": "AnyCrsGeoJSON Feature",
+ "type": "object",
+ "required": [
+ "type",
+ "properties",
+ "geometry"
+ ],
+ "properties": {
+ "geometry": {
+ "oneOf": [
+ {
+ "type": "null"
+ },
+ {
+ "title": "AnyCrsGeoJSON Point",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsPoint"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "AnyCrsGeoJSON LineString",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsLineString"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "AnyCrsGeoJSON Polygon",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsPolygon"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "AnyCrsGeoJSON MultiPoint",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsMultiPoint"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "AnyCrsGeoJSON MultiLineString",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsMultiLineString"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "AnyCrsGeoJSON MultiPolygon",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "items": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsMultiPolygon"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "AnyCrsGeoJSON GeometryCollection",
+ "type": "object",
+ "required": [
+ "type",
+ "geometries"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsGeometryCollection"
+ ]
+ },
+ "geometries": {
+ "type": "array",
+ "items": {
+ "oneOf": [
+ {
+ "title": "AnyCrsGeoJSON Point",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsPoint"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "AnyCrsGeoJSON LineString",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsLineString"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "AnyCrsGeoJSON Polygon",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsPolygon"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "AnyCrsGeoJSON MultiPoint",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsMultiPoint"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "AnyCrsGeoJSON MultiLineString",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsMultiLineString"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "AnyCrsGeoJSON MultiPolygon",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "items": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsMultiPolygon"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ ]
+ }
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ ]
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsFeature"
+ ]
+ },
+ "properties": {
+ "oneOf": [
+ {
+ "type": "null"
+ },
+ {
+ "type": "object"
+ }
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ },
+ "persistableReferenceUnitZ": {
+ "description": "The unit of measure for the Z-axis (only for 3-dimensional coordinates, where the CRS does not describe the vertical unit). Note that the direction is upwards positive, i.e. Z means height.",
+ "type": "string",
+ "title": "Z-Unit Reference",
+ "example": "{\"scaleOffset\":{\"scale\":1.0,\"offset\":0.0},\"symbol\":\"m\",\"baseMeasurement\":{\"ancestry\":\"Length\",\"type\":\"UM\"},\"type\":\"USO\"}"
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ },
+ "persistableReferenceVerticalCrs": {
+ "description": "The VerticalCRS reference as persistableReference string. If populated, the VerticalCoordinateReferenceSystemID takes precedence. The property is null or empty for 2D geometries. For 3D geometries and absent or null persistableReferenceVerticalCrs the vertical CRS is either provided via persistableReferenceCrs's CompoundCRS or it is implicitly defined as EPSG:5714 MSL height.",
+ "type": "string",
+ "title": "Vertical CRS Reference",
+ "example": "{\"authCode\":{\"auth\":\"EPSG\",\"code\":\"5773\"},\"type\":\"LBC\",\"ver\":\"PE_10_3_1\",\"name\":\"EGM96_Geoid\",\"wkt\":\"VERTCS[\\\"EGM96_Geoid\\\",VDATUM[\\\"EGM96_Geoid\\\"],PARAMETER[\\\"Vertical_Shift\\\",0.0],PARAMETER[\\\"Direction\\\",1.0],UNIT[\\\"Meter\\\",1.0],AUTHORITY[\\\"EPSG\\\",5773]]\"}"
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "AnyCrsFeatureCollection"
+ ]
+ },
+ "VerticalCoordinateReferenceSystemID": {
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-CoordinateReferenceSystem:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The explicit VerticalCRS reference into the CoordinateReferenceSystem catalog. This property stays empty for 2D geometries. Absent or empty values for 3D geometries mean the context may be provided by a CompoundCRS in 'CoordinateReferenceSystemID' or implicitly EPSG:5714 MSL height",
+ "x-osdu-relationship": [
+ {
+ "EntityType": "CoordinateReferenceSystem",
+ "GroupType": "reference-data"
+ }
+ ],
+ "title": "Vertical Coordinate Reference System ID",
+ "type": "string",
+ "example": "namespace:reference-data--CoordinateReferenceSystem:VerticalCRS.EPSG.5773:"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractAnyCrsFeatureCollection.1.0.0.json"
+ },
+ "opendes:wks:AbstractMetaItem:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "oneOf": [
+ {
+ "title": "FrameOfReferenceUOM",
+ "type": "object",
+ "properties": {
+ "persistableReference": {
+ "description": "The self-contained, persistable reference string uniquely identifying the Unit.",
+ "title": "UOM Persistable Reference",
+ "type": "string",
+ "example": "{\"abcd\":{\"a\":0.0,\"b\":1200.0,\"c\":3937.0,\"d\":0.0},\"symbol\":\"ft[US]\",\"baseMeasurement\":{\"ancestry\":\"L\",\"type\":\"UM\"},\"type\":\"UAD\"}"
+ },
+ "kind": {
+ "const": "Unit",
+ "description": "The kind of reference, 'Unit' for FrameOfReferenceUOM.",
+ "title": "UOM Reference Kind"
+ },
+ "propertyNames": {
+ "description": "The list of property names, to which this meta data item provides Unit context to. Data structures, which come in a single frame of reference, can register the property name, others require a full path like \"Data.StructureA.PropertyB\" to define a unique context.",
+ "title": "UOM Property Names",
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "example": [
+ "HorizontalDeflection.EastWest",
+ "HorizontalDeflection.NorthSouth"
+ ]
+ },
+ "name": {
+ "description": "The unit symbol or name of the unit.",
+ "title": "UOM Unit Symbol",
+ "type": "string",
+ "example": "ft[US]"
+ },
+ "unitOfMeasureID": {
+ "x-osdu-relationship": [
+ {
+ "EntityType": "UnitOfMeasure",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-UnitOfMeasure:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "SRN to unit of measure reference.",
+ "type": "string",
+ "example": "namespace:reference-data--UnitOfMeasure:ftUS:"
+ }
+ },
+ "required": [
+ "kind",
+ "persistableReference"
+ ]
+ },
+ {
+ "title": "FrameOfReferenceCRS",
+ "type": "object",
+ "properties": {
+ "coordinateReferenceSystemID": {
+ "x-osdu-relationship": [
+ {
+ "EntityType": "CoordinateReferenceSystem",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-CoordinateReferenceSystem:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "SRN to CRS reference.",
+ "type": "string",
+ "example": "namespace:reference-data--CoordinateReferenceSystem:EPSG.32615:"
+ },
+ "persistableReference": {
+ "description": "The self-contained, persistable reference string uniquely identifying the CRS.",
+ "title": "CRS Persistable Reference",
+ "type": "string",
+ "example": "{\"authCode\":{\"auth\":\"EPSG\",\"code\":\"32615\"},\"type\":\"LBC\",\"ver\":\"PE_10_3_1\",\"name\":\"WGS_1984_UTM_Zone_15N\",\"wkt\":\"PROJCS[\\\"WGS_1984_UTM_Zone_15N\\\",GEOGCS[\\\"GCS_WGS_1984\\\",DATUM[\\\"D_WGS_1984\\\",SPHEROID[\\\"WGS_1984\\\",6378137.0,298.257223563]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],PROJECTION[\\\"Transverse_Mercator\\\"],PARAMETER[\\\"False_Easting\\\",500000.0],PARAMETER[\\\"False_Northing\\\",0.0],PARAMETER[\\\"Central_Meridian\\\",-93.0],PARAMETER[\\\"Scale_Factor\\\",0.9996],PARAMETER[\\\"Latitude_Of_Origin\\\",0.0],UNIT[\\\"Meter\\\",1.0],AUTHORITY[\\\"EPSG\\\",32615]]\"}"
+ },
+ "kind": {
+ "const": "CRS",
+ "description": "The kind of reference, constant 'CRS' for FrameOfReferenceCRS.",
+ "title": "CRS Reference Kind"
+ },
+ "propertyNames": {
+ "description": "The list of property names, to which this meta data item provides CRS context to. Data structures, which come in a single frame of reference, can register the property name, others require a full path like \"Data.StructureA.PropertyB\" to define a unique context.",
+ "title": "CRS Property Names",
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "example": [
+ "KickOffPosition.X",
+ "KickOffPosition.Y"
+ ]
+ },
+ "name": {
+ "description": "The name of the CRS.",
+ "title": "CRS Name",
+ "type": "string",
+ "example": "NAD27 * OGP-Usa Conus / North Dakota South [32021,15851]"
+ }
+ },
+ "required": [
+ "kind",
+ "persistableReference"
+ ]
+ },
+ {
+ "title": "FrameOfReferenceDateTime",
+ "type": "object",
+ "properties": {
+ "persistableReference": {
+ "description": "The self-contained, persistable reference string uniquely identifying DateTime reference.",
+ "title": "DateTime Persistable Reference",
+ "type": "string",
+ "example": "{\"format\":\"yyyy-MM-ddTHH:mm:ssZ\",\"timeZone\":\"UTC\",\"type\":\"DTM\"}"
+ },
+ "kind": {
+ "const": "DateTime",
+ "description": "The kind of reference, constant 'DateTime', for FrameOfReferenceDateTime.",
+ "title": "DateTime Reference Kind"
+ },
+ "propertyNames": {
+ "description": "The list of property names, to which this meta data item provides DateTime context to. Data structures, which come in a single frame of reference, can register the property name, others require a full path like \"Data.StructureA.PropertyB\" to define a unique context.",
+ "title": "DateTime Property Names",
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "example": [
+ "Acquisition.StartTime",
+ "Acquisition.EndTime"
+ ]
+ },
+ "name": {
+ "description": "The name of the DateTime format and reference.",
+ "title": "DateTime Name",
+ "type": "string",
+ "example": "UTC"
+ }
+ },
+ "required": [
+ "kind",
+ "persistableReference"
+ ]
+ },
+ {
+ "title": "FrameOfReferenceAzimuthReference",
+ "type": "object",
+ "properties": {
+ "persistableReference": {
+ "description": "The self-contained, persistable reference string uniquely identifying AzimuthReference.",
+ "title": "AzimuthReference Persistable Reference",
+ "type": "string",
+ "example": "{\"code\":\"TrueNorth\",\"type\":\"AZR\"}"
+ },
+ "kind": {
+ "const": "AzimuthReference",
+ "description": "The kind of reference, constant 'AzimuthReference', for FrameOfReferenceAzimuthReference.",
+ "title": "AzimuthReference Reference Kind"
+ },
+ "propertyNames": {
+ "description": "The list of property names, to which this meta data item provides AzimuthReference context to. Data structures, which come in a single frame of reference, can register the property name, others require a full path like \"Data.StructureA.PropertyB\" to define a unique context.",
+ "title": "AzimuthReference Property Names",
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "example": [
+ "Bearing"
+ ]
+ },
+ "name": {
+ "description": "The name of the CRS or the symbol/name of the unit.",
+ "title": "AzimuthReference Name",
+ "type": "string",
+ "example": "TrueNorth"
+ }
+ },
+ "required": [
+ "kind",
+ "persistableReference"
+ ]
+ }
+ ],
+ "x-osdu-license": "Copyright 2021, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractMetaItem:1.0.0",
+ "description": "A meta data item, which allows the association of named properties or property values to a Unit/Measurement/CRS/Azimuth/Time context.",
+ "title": "Frame of Reference Meta Data Item",
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractMetaItem.1.0.0.json"
+ },
+ "opendes:wks:AbstractGeoContext:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "oneOf": [
+ {
+ "$ref": "#/definitions/opendes:wks:AbstractGeoPoliticalContext:1.0.0"
+ },
+ {
+ "$ref": "#/definitions/opendes:wks:AbstractGeoBasinContext:1.0.0"
+ },
+ {
+ "$ref": "#/definitions/opendes:wks:AbstractGeoFieldContext:1.0.0"
+ },
+ {
+ "$ref": "#/definitions/opendes:wks:AbstractGeoPlayContext:1.0.0"
+ },
+ {
+ "$ref": "#/definitions/opendes:wks:AbstractGeoProspectContext:1.0.0"
+ }
+ ],
+ "x-osdu-license": "Copyright 2021, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractGeoContext:1.0.0",
+ "description": "A geographic context to an entity. It can be either a reference to a GeoPoliticalEntity, Basin, Field, Play or Prospect.",
+ "title": "AbstractGeoContext",
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractGeoContext.1.0.0.json"
+ },
+ "opendes:wks:AbstractLegalTags:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2021, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractLegalTags:1.0.0",
+ "description": "Legal meta data like legal tags, relevant other countries, legal status. This structure is included by the SystemProperties \"legal\", which is part of all OSDU records. Not extensible.",
+ "additionalProperties": false,
+ "title": "Legal Meta Data",
+ "type": "object",
+ "properties": {
+ "legaltags": {
+ "description": "The list of legal tags, which resolve to legal properties (like country of origin, export classification code, etc.) and rules with the help of the Compliance Service.",
+ "title": "Legal Tags",
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "otherRelevantDataCountries": {
+ "description": "The list of other relevant data countries as an array of two-letter country codes, see https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2.",
+ "title": "Other Relevant Data Countries",
+ "type": "array",
+ "items": {
+ "pattern": "^[A-Z]{2}$",
+ "type": "string"
+ }
+ },
+ "status": {
+ "pattern": "^(compliant|uncompliant)$",
+ "description": "The legal status. Set by the system after evaluation against the compliance rules associated with the \"legaltags\" using the Compliance Service.",
+ "title": "Legal Status",
+ "type": "string"
+ }
+ },
+ "required": [
+ "legaltags",
+ "otherRelevantDataCountries"
+ ],
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractLegalTags.1.0.0.json"
+ },
+ "opendes:wks:AbstractGeoBasinContext:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2021, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractGeoBasinContext:1.0.0",
+ "description": "A single, typed basin entity reference, which is 'abstracted' to AbstractGeoContext and then aggregated by GeoContexts properties.",
+ "x-osdu-review-status": "Accepted",
+ "title": "AbstractGeoBasinContext",
+ "type": "object",
+ "properties": {
+ "BasinID": {
+ "pattern": "^[\\w\\-\\.]+:master-data\\-\\-Basin:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Reference to Basin.",
+ "x-osdu-relationship": [
+ {
+ "EntityType": "Basin",
+ "GroupType": "master-data"
+ }
+ ],
+ "type": "string"
+ },
+ "GeoTypeID": {
+ "x-osdu-is-derived": {
+ "RelationshipPropertyName": "BasinID",
+ "TargetPropertyName": "BasinTypeID"
+ },
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-BasinType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The BasinType reference of the Basin (via BasinID) for application convenience.",
+ "x-osdu-relationship": [
+ {
+ "EntityType": "BasinType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "type": "string"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractGeoBasinContext.1.0.0.json"
+ },
+ "opendes:wks:AbstractMaster:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2021, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractMaster:1.0.0",
+ "description": "Properties shared with all master-data schema instances.",
+ "x-osdu-review-status": "Accepted",
+ "title": "Abstract Master",
+ "type": "object",
+ "properties": {
+ "NameAliases": {
+ "description": "Alternative names, including historical, by which this master data is/has been known (it should include all the identifiers).",
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/opendes:wks:AbstractAliasNames:1.0.0"
+ }
+ },
+ "SpatialLocation": {
+ "description": "The spatial location information such as coordinates, CRS information (left empty when not appropriate).",
+ "$ref": "#/definitions/opendes:wks:AbstractSpatialLocation:1.0.0"
+ },
+ "VersionCreationReason": {
+ "description": "This describes the reason that caused the creation of a new version of this master data.",
+ "type": "string"
+ },
+ "GeoContexts": {
+ "description": "List of geographic entities which provide context to the master data. This may include multiple types or multiple values of the same type.",
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/opendes:wks:AbstractGeoContext:1.0.0"
+ }
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractMaster.1.0.0.json"
+ },
+ "opendes:wks:AbstractGeoProspectContext:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2021, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractGeoProspectContext:1.0.0",
+ "description": "A single, typed Prospect entity reference, which is 'abstracted' to AbstractGeoContext and then aggregated by GeoContexts properties.",
+ "x-osdu-review-status": "Accepted",
+ "title": "AbstractGeoProspectContext",
+ "type": "object",
+ "properties": {
+ "ProspectID": {
+ "pattern": "^[\\w\\-\\.]+:master-data\\-\\-Prospect:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Reference to the prospect.",
+ "x-osdu-relationship": [
+ {
+ "EntityType": "Prospect",
+ "GroupType": "master-data"
+ }
+ ],
+ "type": "string"
+ },
+ "GeoTypeID": {
+ "x-osdu-is-derived": {
+ "RelationshipPropertyName": "ProspectID",
+ "TargetPropertyName": "ProspectTypeID"
+ },
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-ProspectType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The ProspectType reference of the Prospect (via ProspectID) for application convenience.",
+ "x-osdu-relationship": [
+ {
+ "EntityType": "ProspectType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "type": "string"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractGeoProspectContext.1.0.0.json"
+ },
+ "opendes:wks:AbstractSpatialLocation:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2021, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractSpatialLocation:1.0.0",
+ "description": "A geographic object which can be described by a set of points.",
+ "title": "AbstractSpatialLocation",
+ "type": "object",
+ "properties": {
+ "AsIngestedCoordinates": {
+ "description": "The original or 'as ingested' coordinates (Point, MultiPoint, LineString, MultiLineString, Polygon or MultiPolygon). The name 'AsIngestedCoordinates' was chosen to contrast it to 'OriginalCoordinates', which carries the uncertainty whether any coordinate operations took place before ingestion. In cases where the original CRS is different from the as-ingested CRS, the OperationsApplied can also contain the list of operations applied to the coordinate prior to ingestion. The data structure is similar to GeoJSON FeatureCollection, however in a CRS context explicitly defined within the AbstractAnyCrsFeatureCollection. The coordinate sequence follows GeoJSON standard, i.e. 'eastward/longitude', 'northward/latitude' {, 'upward/height' unless overridden by an explicit direction in the AsIngestedCoordinates.VerticalCoordinateReferenceSystemID}.",
+ "x-osdu-frame-of-reference": "CRS:",
+ "title": "As Ingested Coordinates",
+ "$ref": "#/definitions/opendes:wks:AbstractAnyCrsFeatureCollection:1.0.0"
+ },
+ "SpatialParameterTypeID": {
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-SpatialParameterType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "A type of spatial representation of an object, often general (e.g. an Outline, which could be applied to Field, Reservoir, Facility, etc.) or sometimes specific (e.g. Onshore Outline, State Offshore Outline, Federal Offshore Outline, 3 spatial representations that may be used by Countries).",
+ "x-osdu-relationship": [
+ {
+ "EntityType": "SpatialParameterType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "type": "string"
+ },
+ "QuantitativeAccuracyBandID": {
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-QuantitativeAccuracyBand:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "An approximate quantitative assessment of the quality of a location (accurate to > 500 m (i.e. not very accurate)), to < 1 m, etc.",
+ "x-osdu-relationship": [
+ {
+ "EntityType": "QuantitativeAccuracyBand",
+ "GroupType": "reference-data"
+ }
+ ],
+ "type": "string"
+ },
+ "CoordinateQualityCheckRemarks": {
+ "type": "array",
+ "description": "Freetext remarks on Quality Check.",
+ "items": {
+ "type": "string"
+ }
+ },
+ "AppliedOperations": {
+ "description": "The audit trail of operations applied to the coordinates from the original state to the current state. The list may contain operations applied prior to ingestion as well as the operations applied to produce the Wgs84Coordinates. The text elements refer to ESRI style CRS and Transformation names, which may have to be translated to EPSG standard names.",
+ "title": "Operations Applied",
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "example": [
+ "conversion from ED_1950_UTM_Zone_31N to GCS_European_1950; 1 points converted",
+ "transformation GCS_European_1950 to GCS_WGS_1984 using ED_1950_To_WGS_1984_24; 1 points successfully transformed"
+ ]
+ },
+ "QualitativeSpatialAccuracyTypeID": {
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-QualitativeSpatialAccuracyType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "A qualitative description of the quality of a spatial location, e.g. unverifiable, not verified, basic validation.",
+ "x-osdu-relationship": [
+ {
+ "EntityType": "QualitativeSpatialAccuracyType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "type": "string"
+ },
+ "CoordinateQualityCheckPerformedBy": {
+ "type": "string",
+ "description": "The user who performed the Quality Check."
+ },
+ "SpatialLocationCoordinatesDate": {
+ "format": "date-time",
+ "description": "Date when coordinates were measured or retrieved.",
+ "x-osdu-frame-of-reference": "DateTime",
+ "type": "string"
+ },
+ "CoordinateQualityCheckDateTime": {
+ "format": "date-time",
+ "description": "The date of the Quality Check.",
+ "x-osdu-frame-of-reference": "DateTime",
+ "type": "string"
+ },
+ "Wgs84Coordinates": {
+ "title": "WGS 84 Coordinates",
+ "description": "The normalized coordinates (Point, MultiPoint, LineString, MultiLineString, Polygon or MultiPolygon) based on WGS 84 (EPSG:4326 for 2-dimensional coordinates, EPSG:4326 + EPSG:5714 (MSL) for 3-dimensional coordinates). This derived coordinate representation is intended for global discoverability only. The schema of this substructure is identical to the GeoJSON FeatureCollection https://geojson.org/schema/FeatureCollection.json. The coordinate sequence follows GeoJSON standard, i.e. longitude, latitude {, height}",
+ "$ref": "#/definitions/opendes:wks:AbstractFeatureCollection:1.0.0"
+ },
+ "SpatialGeometryTypeID": {
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-SpatialGeometryType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Indicates the expected look of the SpatialParameterType, e.g. Point, MultiPoint, LineString, MultiLineString, Polygon, MultiPolygon. The value constrains the type of geometries in the GeoJSON Wgs84Coordinates and AsIngestedCoordinates.",
+ "x-osdu-relationship": [
+ {
+ "EntityType": "SpatialGeometryType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "type": "string"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractSpatialLocation.1.0.0.json"
+ },
+ "opendes:wks:AbstractGeoFieldContext:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2021, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractGeoFieldContext:1.0.0",
+ "description": "A single, typed field entity reference, which is 'abstracted' to AbstractGeoContext and then aggregated by GeoContexts properties.",
+ "title": "AbstractGeoFieldContext",
+ "type": "object",
+ "properties": {
+ "FieldID": {
+ "pattern": "^[\\w\\-\\.]+:master-data\\-\\-Field:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Reference to Field.",
+ "x-osdu-relationship": [
+ {
+ "EntityType": "Field",
+ "GroupType": "master-data"
+ }
+ ],
+ "type": "string"
+ },
+ "GeoTypeID": {
+ "const": "Field",
+ "description": "The fixed type 'Field' for this AbstractGeoFieldContext."
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractGeoFieldContext.1.0.0.json"
+ },
+ "opendes:wks:AbstractFeatureCollection:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2021, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractFeatureCollection:1.0.0",
+ "description": "GeoJSON feature collection as originally published in https://geojson.org/schema/FeatureCollection.json. Attention: the coordinate order is fixed: Longitude first, followed by Latitude, optionally height above MSL (EPSG:5714) as third coordinate.",
+ "title": "GeoJSON FeatureCollection",
+ "type": "object",
+ "required": [
+ "type",
+ "features"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "FeatureCollection"
+ ]
+ },
+ "features": {
+ "type": "array",
+ "items": {
+ "title": "GeoJSON Feature",
+ "type": "object",
+ "required": [
+ "type",
+ "properties",
+ "geometry"
+ ],
+ "properties": {
+ "geometry": {
+ "oneOf": [
+ {
+ "type": "null"
+ },
+ {
+ "title": "GeoJSON Point",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "Point"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "GeoJSON LineString",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "LineString"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "GeoJSON Polygon",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "Polygon"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "GeoJSON MultiPoint",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "MultiPoint"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "GeoJSON MultiLineString",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "MultiLineString"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "GeoJSON MultiPolygon",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "items": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "MultiPolygon"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "GeoJSON GeometryCollection",
+ "type": "object",
+ "required": [
+ "type",
+ "geometries"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": [
+ "GeometryCollection"
+ ]
+ },
+ "geometries": {
+ "type": "array",
+ "items": {
+ "oneOf": [
+ {
+ "title": "GeoJSON Point",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "Point"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "GeoJSON LineString",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "LineString"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "GeoJSON Polygon",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "Polygon"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "GeoJSON MultiPoint",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "MultiPoint"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "GeoJSON MultiLineString",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "MultiLineString"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ },
+ {
+ "title": "GeoJSON MultiPolygon",
+ "type": "object",
+ "required": [
+ "type",
+ "coordinates"
+ ],
+ "properties": {
+ "coordinates": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "items": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "minItems": 2,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "MultiPolygon"
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ ]
+ }
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ ]
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "Feature"
+ ]
+ },
+ "properties": {
+ "oneOf": [
+ {
+ "type": "null"
+ },
+ {
+ "type": "object"
+ }
+ ]
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ }
+ }
+ },
+ "bbox": {
+ "minItems": 4,
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractFeatureCollection.1.0.0.json"
+ },
+ "opendes:wks:AbstractAccessControlList:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2021, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractAccessControlList:1.0.0",
+ "description": "The access control tags associated with this entity. This structure is included by the SystemProperties \"acl\", which is part of all OSDU records. Not extensible.",
+ "additionalProperties": false,
+ "title": "Access Control List",
+ "type": "object",
+ "properties": {
+ "viewers": {
+ "description": "The list of viewers to which this data record is accessible/visible/discoverable formatted as an email (core.common.model.storage.validation.ValidationDoc.EMAIL_REGEX).",
+ "title": "List of Viewers",
+ "type": "array",
+ "items": {
+ "pattern": "^[a-zA-Z0-9_+&*-]+(?:\\.[a-zA-Z0-9_+&*-]+)*@(?:[a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,7}$",
+ "type": "string"
+ }
+ },
+ "owners": {
+ "description": "The list of owners of this data record formatted as an email (core.common.model.storage.validation.ValidationDoc.EMAIL_REGEX).",
+ "title": "List of Owners",
+ "type": "array",
+ "items": {
+ "pattern": "^[a-zA-Z0-9_+&*-]+(?:\\.[a-zA-Z0-9_+&*-]+)*@(?:[a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,7}$",
+ "type": "string"
+ }
+ }
+ },
+ "required": [
+ "owners",
+ "viewers"
+ ],
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractAccessControlList.1.0.0.json"
+ },
+ "opendes:wks:AbstractGeoPlayContext:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2021, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractGeoPlayContext:1.0.0",
+ "description": "A single, typed Play entity reference, which is 'abstracted' to AbstractGeoContext and then aggregated by GeoContexts properties.",
+ "x-osdu-review-status": "Accepted",
+ "title": "AbstractGeoPlayContext",
+ "type": "object",
+ "properties": {
+ "PlayID": {
+ "pattern": "^[\\w\\-\\.]+:master-data\\-\\-Play:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Reference to the play.",
+ "x-osdu-relationship": [
+ {
+ "EntityType": "Play",
+ "GroupType": "master-data"
+ }
+ ],
+ "type": "string"
+ },
+ "GeoTypeID": {
+ "x-osdu-is-derived": {
+ "RelationshipPropertyName": "PlayID",
+ "TargetPropertyName": "PlayTypeID"
+ },
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-PlayType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The PlayType reference of the Play (via PlayID) for application convenience.",
+ "x-osdu-relationship": [
+ {
+ "EntityType": "PlayType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "type": "string"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractGeoPlayContext.1.0.0.json"
+ },
+ "opendes:wks:AbstractLegalParentList:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2021, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractLegalParentList:1.0.0",
+ "description": "A list of entity IDs in the data ecosystem, which act as legal parents to the current entity. This structure is included by the SystemProperties \"ancestry\", which is part of all OSDU records. Not extensible.",
+ "additionalProperties": false,
+ "title": "Parent List",
+ "type": "object",
+ "properties": {
+ "parents": {
+ "description": "An array of none, one or many entity references in the data ecosystem, which identify the source of data in the legal sense. In contract to other relationships, the source record version is required. Example: the 'parents' will be queried when e.g. the subscription of source data services is terminated; access to the derivatives is also terminated.",
+ "title": "Parents",
+ "type": "array",
+ "items": {
+ "x-osdu-relationship": [],
+ "pattern": "^[\\w\\-\\.]+:[\\w\\-\\.]+:[\\w\\-\\.\\:\\%]+:[0-9]+$",
+ "type": "string"
+ },
+ "example": []
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractLegalParentList.1.0.0.json"
+ }
+ },
+ "properties": {
+ "ancestry": {
+ "description": "The links to data, which constitute the inputs.",
+ "title": "Ancestry",
+ "$ref": "#/definitions/opendes:wks:AbstractLegalParentList:1.0.0"
+ },
+ "data": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/opendes:wks:AbstractCommonResources:1.0.0"
+ },
+ {
+ "$ref": "#/definitions/opendes:wks:AbstractMaster:1.0.0"
+ },
+ {
+ "type": "object",
+ "properties": {}
+ },
+ {
+ "type": "object",
+ "properties": {
+ "ExtensionProperties": {
+ "type": "object"
+ }
+ }
+ }
+ ]
+ },
+ "kind": {
+ "pattern": "^[\\w\\-\\.]+:[\\w\\-\\.]+:[\\w\\-\\.]+:[0-9]+.[0-9]+.[0-9]+$",
+ "description": "The schema identification for the OSDU resource object following the pattern {Namespace}:{Source}:{Type}:{VersionMajor}.{VersionMinor}.{VersionPatch}. The versioning scheme follows the semantic versioning, https://semver.org/.",
+ "title": "Entity Kind",
+ "type": "string",
+ "example": "osdu:wks:master-data--Test:1.0.0"
+ },
+ "acl": {
+ "description": "The access control tags associated with this entity.",
+ "title": "Access Control List",
+ "$ref": "#/definitions/opendes:wks:AbstractAccessControlList:1.0.0"
+ },
+ "version": {
+ "format": "int64",
+ "description": "The version number of this OSDU resource; set by the framework.",
+ "title": "Version Number",
+ "type": "integer",
+ "example": 1562066009929332
+ },
+ "tags": {
+ "description": "A generic dictionary of string keys mapping to string value. Only strings are permitted as keys and values.",
+ "additionalProperties": {
+ "type": "string"
+ },
+ "title": "Tag Dictionary",
+ "type": "object",
+ "example": {
+ "NameOfKey": "String value"
+ }
+ },
+ "modifyUser": {
+ "description": "The user reference, which created this version of this resource object. Set by the System.",
+ "title": "Resource Object Version Creation User Reference",
+ "type": "string",
+ "example": "some-user@some-company-cloud.com"
+ },
+ "modifyTime": {
+ "format": "date-time",
+ "description": "Timestamp of the time at which this version of the OSDU resource object was created. Set by the System. The value is a combined date-time string in ISO-8601 given in UTC.",
+ "title": "Resource Object Version Creation DateTime",
+ "type": "string",
+ "example": "2020-12-16T11:52:24.477Z"
+ },
+ "createTime": {
+ "format": "date-time",
+ "description": "Timestamp of the time at which initial version of this OSDU resource object was created. Set by the System. The value is a combined date-time string in ISO-8601 given in UTC.",
+ "title": "Resource Object Creation DateTime",
+ "type": "string",
+ "example": "2020-12-16T11:46:20.163Z"
+ },
+ "meta": {
+ "description": "The Frame of Reference meta data section linking the named properties to self-contained definitions.",
+ "title": "Frame of Reference Meta Data",
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/opendes:wks:AbstractMetaItem:1.0.0"
+ }
+ },
+ "legal": {
+ "description": "The entity's legal tags and compliance status. The actual contents associated with the legal tags is managed by the Compliance Service.",
+ "title": "Legal Tags",
+ "$ref": "#/definitions/opendes:wks:AbstractLegalTags:1.0.0"
+ },
+ "createUser": {
+ "description": "The user reference, which created the first version of this resource object. Set by the System.",
+ "title": "Resource Object Creation User Reference",
+ "type": "string",
+ "example": "some-user@some-company-cloud.com"
+ },
+ "id": {
+ "pattern": "^[\\w\\-\\.]+:master-data\\-\\-Test:[\\w\\-\\.\\:\\%]+$",
+ "description": "Previously called ResourceID or SRN which identifies this OSDU resource object without version.",
+ "title": "Entity ID",
+ "type": "string",
+ "example": "namespace:master-data--Test:ded98862-efe4-5517-a509-d99f4b941b20"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/master-data/Test.1.0.0.json"
+ }
+}
\ No newline at end of file
diff --git a/indexer-acceptance-test/src/test/resources/testData/well_augmenter_configuration.json b/indexer-acceptance-test/src/test/resources/testData/well_augmenter_configuration.json
new file mode 100644
index 0000000000000000000000000000000000000000..21e34e88e76eaffe5e37048790022e424cda35d4
--- /dev/null
+++ b/indexer-acceptance-test/src/test/resources/testData/well_augmenter_configuration.json
@@ -0,0 +1,52 @@
+[{
+ "id": "tenant1:reference-data--IndexPropertyPathConfiguration:index-property--Well:1",
+ "data": {
+ "Name": "Well-IndexPropertyPathConfiguration",
+ "Description": "The index property list for master-data--Well:1., valid for all master-data--Well kinds for major version 1.",
+ "Code": "osdu:wks:master-data--Well:1.",
+ "AttributionAuthority": "CustomAttributionAuthority",
+ "Configurations": [
+ {
+ "Name": "CountryNames",
+ "Policy": "ExtractAllMatches",
+ "Paths": [
+ {
+ "RelatedObjectsSpec": {
+ "RelationshipDirection": "ChildToParent",
+ "RelatedObjectID": "data.GeoContexts[].GeoPoliticalEntityID",
+ "RelatedObjectKind": "osdu:wks:master-data--GeoPoliticalEntity:1.",
+ "RelatedConditionMatches": [
+ "opendes:reference-data--GeoPoliticalEntityType:Country:"
+ ],
+ "RelatedConditionProperty": "data.GeoContexts[].GeoTypeID"
+ },
+ "ValueExtraction": {
+ "ValuePath": "data.GeoPoliticalEntityName"
+ }
+ }
+ ],
+ "UseCase": "As a user I want to find objects by a country name, with the understanding that an object may extend over country boundaries."
+ },
+ {
+ "Name": "WellUWI",
+ "Policy": "ExtractFirstMatch",
+ "Paths": [
+ {
+ "ValueExtraction": {
+ "RelatedConditionMatches": [
+ "opendes:reference-data--AliasNameType:UniqueIdentifier:",
+ "opendes:reference-data--AliasNameType:RegulatoryName:",
+ "opendes:reference-data--AliasNameType:PreferredName:",
+ "opendes:reference-data--AliasNameType:CommonName:",
+ "opendes:reference-data--AliasNameType:ShortName:"
+ ],
+ "RelatedConditionProperty": "data.NameAliases[].AliasNameTypeID",
+ "ValuePath": "data.NameAliases[].AliasName"
+ }
+ }
+ ],
+ "UseCase": "As a user I want to discover and match Wells by their UWI. I am aware that this is not globally reliable, however, I am able to specify a prioritized AliasNameType list to look up value in the NameAliases array."
+ }
+ ]
+ }
+}]
diff --git a/indexer-core-plus/docs/baremetal/README.md b/indexer-core-plus/docs/baremetal/README.md
index 27a7dccdde5b3f012d6058a31dab40027bfc7c59..c591cbfa7d9175ccfcb141346bbcbb12f8628940 100644
--- a/indexer-core-plus/docs/baremetal/README.md
+++ b/indexer-core-plus/docs/baremetal/README.md
@@ -22,7 +22,6 @@ Must have:
| name | value | description | sensitive? | source |
|----------------------------------------------|--------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------|--------|
-| `SPRING_PROFILES_ACTIVE` | ex `anthos` | Spring profile that activate default configuration for Google Cloud environment | false | - |
| `OPENID_PROVIDER_CLIENT_ID` | `*****` | Client id that represents this service and serves to request tokens, example `workload-identity-legal` | yes | - |
| `OPENID_PROVIDER_CLIENT_SECRET` | `*****` | This client secret that serves to request tokens | yes | - |
| `OPENID_PROVIDER_URL` | `https://keycloack.com/auth/realms/master` | URL of OpenID Connect provider, it will be used as `<OpenID URL> + /.well-known/openid-configuration` to auto configure endpoint for token request | no | - |
@@ -67,11 +66,11 @@ This variable should be present in environment of service that need that variabl
Example:
```json
"elasticsearch.port": {
- "sensitive": false, <- value not sensitive
+ "sensitive": false, <- value not sensitive
"value": "9243" <- will be used as is.
},
"elasticsearch.password": {
- "sensitive": true, <- value is sensitive
+ "sensitive": true, <- value is sensitive
"value": "ELASTIC_SEARCH_PASSWORD_OSDU" <- service consumer should have env variable ELASTIC_SEARCH_PASSWORD_OSDU with elastic search password
}
```
@@ -265,7 +264,6 @@ You will need to have the following environment variables defined.
| `LEGAL_TAG` | ex `opendes-demo-legaltag` | valid legal tag with a other relevant data countries from `DEFAULT_OTHER_RELEVANT_DATA_COUNTRIES` | no | - |
| `DEFAULT_DATA_PARTITION_ID_TENANT1` | ex `opendes` | HTTP Header 'Data-Partition-ID' | no | - |
| `DEFAULT_DATA_PARTITION_ID_TENANT2` | ex `opendes` | HTTP Header 'Data-Partition-ID' | no | - |
-| `SEARCH_HOST` | ex `http://localhost:8080/api/search/v2/` | Endpoint of search service | no | - |
| `STORAGE_HOST` | ex `http://os-storage-dot-opendes.appspot.com/api/storage/v2/` | Storage API endpoint | no | output of infrastructure deployment |
| `SECURITY_HTTPS_CERTIFICATE_TRUST` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment |
| `TEST_OPENID_PROVIDER_CLIENT_ID` | `********` | Client Id for `$INTEGRATION_TESTER` | yes | -- |
@@ -275,7 +273,7 @@ You will need to have the following environment variables defined.
**Entitlements configuration for integration accounts**
-| INTEGRATION_TESTER | NO_DATA_ACCESS_TESTER |
+| INTEGRATION_TESTER | NO_DATA_ACCESS_TESTER |
|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------|
| users<br/>users.datalake.ops<br/>service.storage.creator<br/>service.entitlements.user<br/>service.search.user<br/>service.search.admin<br/>data.test1<br/>data.integration.test<br/>users@{tenant1}@{groupId}.com | |
diff --git a/indexer-core-plus/pom.xml b/indexer-core-plus/pom.xml
index f2ab06f3f8414012b35b0d9b2b8368a7ce81dd4e..ad989d639fa66e7d1ae4e1b6fe10401e3288f2a2 100644
--- a/indexer-core-plus/pom.xml
+++ b/indexer-core-plus/pom.xml
@@ -18,13 +18,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-plus-common-lib</artifactId>
- <version>0.24.0-rc4</version>
- <exclusions>
- <exclusion>
- <artifactId>os-core-common</artifactId>
- <groupId>org.opengroup.osdu</groupId>
- </exclusion>
- </exclusions>
+ <version>0.27.2</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
@@ -66,12 +60,6 @@
<version>3.12.0</version>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.projectlombok</groupId>
- <artifactId>lombok</artifactId>
- <version>1.18.26</version>
- <scope>provided</scope>
- </dependency>
<dependency>
<groupId>org.codehaus.plexus</groupId>
<artifactId>plexus-utils</artifactId>
@@ -111,24 +99,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-oqm-core</artifactId>
- <version>0.25.0-rc1</version>
- <exclusions>
- <exclusion>
- <artifactId>os-core-common</artifactId>
- <groupId>org.opengroup.osdu</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.opengroup.osdu</groupId>
- <artifactId>apd-core</artifactId>
- <version>0.27.0-rc1</version>
- <exclusions>
- <exclusion>
- <artifactId>os-core-common</artifactId>
- <groupId>org.opengroup.osdu</groupId>
- </exclusion>
- </exclusions>
+ <version>0.27.2</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
diff --git a/indexer-core-plus/src/main/java/org/opengroup/osdu/indexer/IndexerCorePlusApplication.java b/indexer-core-plus/src/main/java/org/opengroup/osdu/indexer/IndexerCorePlusApplication.java
index 6d87d6c696e2c217256133505394485d7d4de2cc..7440110cf8097b8ad968e81c4112b95299c60628 100644
--- a/indexer-core-plus/src/main/java/org/opengroup/osdu/indexer/IndexerCorePlusApplication.java
+++ b/indexer-core-plus/src/main/java/org/opengroup/osdu/indexer/IndexerCorePlusApplication.java
@@ -1,6 +1,6 @@
/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
+ * Copyright 2020-2024 Google LLC
+ * Copyright 2020-2024 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -22,15 +22,15 @@ import org.opengroup.osdu.indexer.web.config.WebAppMainContextConfiguration;
import org.springframework.boot.SpringBootConfiguration;
import org.springframework.boot.WebApplicationType;
import org.springframework.boot.builder.SpringApplicationBuilder;
-import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.context.annotation.PropertySource;
@SpringBootConfiguration
@PropertySource("classpath:swagger.properties")
public class IndexerCorePlusApplication {
- public static void main(String[] args) {
- SpringApplicationBuilder springApplicationBuilder = new SpringApplicationBuilder(IndexerCorePlusApplication.class)
+ public static void main(String[] args) {
+ SpringApplicationBuilder springApplicationBuilder =
+ new SpringApplicationBuilder(IndexerCorePlusApplication.class)
.sources(IndexerCorePlusApplication.class)
.web(WebApplicationType.NONE)
.child(CustomContextConfiguration.class)
@@ -38,6 +38,6 @@ public class IndexerCorePlusApplication {
.child(WebAppMainContextConfiguration.class)
.web(WebApplicationType.SERVLET);
- ConfigurableApplicationContext context = springApplicationBuilder.run(args);
- }
+ springApplicationBuilder.run(args);
+ }
}
diff --git a/indexer-core-plus/src/main/java/org/opengroup/osdu/indexer/common/cache/AttributesCache.java b/indexer-core-plus/src/main/java/org/opengroup/osdu/indexer/common/cache/AttributesCache.java
deleted file mode 100644
index fcc13c16f85fa0f7aad417c64efe0b28b14a2963..0000000000000000000000000000000000000000
--- a/indexer-core-plus/src/main/java/org/opengroup/osdu/indexer/common/cache/AttributesCache.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.common.cache;
-
-import java.util.Set;
-import jakarta.inject.Inject;
-import org.opengroup.osdu.core.common.cache.RedisCache;
-import org.opengroup.osdu.core.common.provider.interfaces.IAttributesCache;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.springframework.stereotype.Component;
-
-@Component
-public class AttributesCache implements IAttributesCache<String, Set>, AutoCloseable {
-
- private RedisCache<String, Set> cache;
-
- @Inject
- public AttributesCache(final IndexerConfigurationProperties properties) {
- cache = new RedisCache(properties.getRedisSearchHost(), Integer.parseInt(properties.getRedisSearchPort()),
- properties.getElasticCacheExpiration() * 60, String.class, Boolean.class);
- }
-
- @Override
- public void put(String key, Set value) {
- this.cache.put(key, value);
- }
-
- @Override
- public Set get(String key) {
- return this.cache.get(key);
- }
-
- @Override
- public void delete(String key) {
- this.cache.delete(key);
- }
-
- @Override
- public void clearAll() {
- this.cache.clearAll();
- }
-
- @Override
- public void close() {
- this.cache.close();
- }
-}
diff --git a/indexer-core-plus/src/main/java/org/opengroup/osdu/indexer/common/cache/ElasticCredentialsCache.java b/indexer-core-plus/src/main/java/org/opengroup/osdu/indexer/common/cache/ElasticCredentialsCache.java
index 2c7a3f2cc8540fef90db21a0ca47e4af54489717..b582b5568c87c89b7809ca06d8f5ffaab76a6760 100644
--- a/indexer-core-plus/src/main/java/org/opengroup/osdu/indexer/common/cache/ElasticCredentialsCache.java
+++ b/indexer-core-plus/src/main/java/org/opengroup/osdu/indexer/common/cache/ElasticCredentialsCache.java
@@ -17,7 +17,7 @@
package org.opengroup.osdu.indexer.common.cache;
-import com.lambdaworks.redis.RedisException;
+import io.lettuce.core.RedisException;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.opengroup.osdu.core.common.cache.RedisCache;
diff --git a/indexer-core-plus/src/main/resources/application.properties b/indexer-core-plus/src/main/resources/application.properties
index 75995196e9fa3b69cf38f364be24395369c73401..2f1c6c98b8bf64380cb22325a88753578b3571eb 100644
--- a/indexer-core-plus/src/main/resources/application.properties
+++ b/indexer-core-plus/src/main/resources/application.properties
@@ -51,7 +51,6 @@ STORAGE_SCHEMA_HOST=${STORAGE_API}/schemas
SCHEMA_BASE_HOST=http://schema
SCHEMA_PATH=/api/schema-service/v1/schema
SCHEMA_HOST=${SCHEMA_BASE_HOST}${SCHEMA_PATH}
-SEARCH_HOST=${SEARCH_BASE_HOST}/api/search/v2
# OQM config
records-changed-topic-name=records-changed
@@ -61,8 +60,9 @@ reprocess-topic-name=reprocess
reindex-topic-name=reindex
rabbitmq-retry-delay=20000
rabbitmq-retry-limit=5
+
# Feature flag settings
-featureFlag.strategy=appProperty
+featureFlag.strategy=dataPartition
featureFlag.asIngestedCoordinates.enabled=false
featureFlag.keywordLower.enabled=false
featureFlag.bagOfWords.enabled=false
@@ -73,3 +73,6 @@ management.server.port=${MANAGEMENT_SERVER_PORT:8081}
management.endpoints.web.base-path=${MANAGEMENT_ENDPOINTS_WEB_BASE:/}
management.endpoints.web.exposure.include=health
management.health.probes.enabled=true
+
+# Elasticsearch prefix
+elasticsearch-properties-prefix=elasticsearch.8
diff --git a/indexer-core/pom.xml b/indexer-core/pom.xml
index e23927f604faffc672a7918042ab2b50d628438c..7adc918ee24505efab2c0c70cb845d8e8ac49288 100644
--- a/indexer-core/pom.xml
+++ b/indexer-core/pom.xml
@@ -32,8 +32,8 @@
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
- <artifactId>os-core-common-spring6</artifactId>
- <version>${os-core-common-spring6.version}</version>
+ <artifactId>os-core-common</artifactId>
+ <version>${os-core-common.version}</version>
</dependency>
<!-- spring boot dependencies -->
@@ -150,9 +150,9 @@
<version>3.8.1</version>
</dependency>
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.13.2</version>
+ <groupId>org.junit.vintage</groupId>
+ <artifactId>junit-vintage-engine</artifactId>
+ <version>5.10.2</version>
<scope>test</scope>
</dependency>
<dependency>
@@ -192,9 +192,12 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
- <version>2.22.2</version>
+ <version>3.3.1</version>
<configuration>
<argLine>@{argLine} --add-opens java.base/java.lang=ALL_UNNAMED</argLine>
+ <includes>
+ <include>**/*Test.java</include>
+ </includes>
</configuration>
</plugin>
</plugins>
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java
index 3af68014e571e1badc230f03bb2521b77f2c1ccc..980b75ced75a10442a9c00178a8096809cb63342 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java
@@ -79,7 +79,8 @@ public class ReindexApi {
})
@PreAuthorize("@authorizationFilter.hasPermission('" + SearchServiceRole.ADMIN + "')")
@PostMapping(path = "/records", consumes = "application/json")
- public ResponseEntity<?> reindexRecords(@NotNull @Valid @RequestBody ReindexRecordsRequest reindexRecordsRequest) {
+ public ResponseEntity<?> reindexRecords(
+ @NotNull @Valid @RequestBody ReindexRecordsRequest reindexRecordsRequest) {
Records records = this.reIndexService.reindexRecords(reindexRecordsRequest.getRecordIds());
List<String> reindexedRecords = records.getRecords().stream().map(Records.Entity::getId).collect(Collectors.toList());
if (!reindexedRecords.isEmpty()) {
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/ElasticsearchClientCache.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/ElasticsearchClientCache.java
index 2f5009360706b1996fd9b377bc727e99a17ffcdc..69375147496eda9a5bdcb10525a728927b24d445 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/ElasticsearchClientCache.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/cache/ElasticsearchClientCache.java
@@ -18,11 +18,11 @@
package org.opengroup.osdu.indexer.cache;
import static org.opengroup.osdu.indexer.model.Constants.CLIENT_CACHE_EXPIRATION;
-import static org.opengroup.osdu.indexer.model.Constants.CLIENT_CACHE_SIZE;
import co.elastic.clients.elasticsearch.ElasticsearchClient;
import org.opengroup.osdu.core.common.cache.ICache;
import org.opengroup.osdu.core.common.cache.VmCache;
+import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@Component
@@ -30,8 +30,11 @@ public class ElasticsearchClientCache implements ICache<String, ElasticsearchCli
private final VmCache<String, ElasticsearchClient> vmCache;
+ @Value("${elasticsearch.client.cache.size:50}")
+ private int clientCacheSize;
+
public ElasticsearchClientCache() {
- this.vmCache = new VmCache<>(CLIENT_CACHE_EXPIRATION, CLIENT_CACHE_SIZE);
+ this.vmCache = new VmCache<>(CLIENT_CACHE_EXPIRATION, clientCacheSize);
}
@Override
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/config/WebConfiguration.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/config/WebConfiguration.java
new file mode 100644
index 0000000000000000000000000000000000000000..1894f2b00ce19627ef400d92034a29c8ae7891b4
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/config/WebConfiguration.java
@@ -0,0 +1,14 @@
+package org.opengroup.osdu.indexer.config;
+
+import org.springframework.context.annotation.Configuration;
+import org.springframework.web.servlet.config.annotation.PathMatchConfigurer;
+import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
+
+@Configuration
+public class WebConfiguration implements WebMvcConfigurer {
+ @Override
+ public void configurePathMatch(PathMatchConfigurer configurer) {
+ configurer.setUseTrailingSlashMatch(true);
+ }
+
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/error/GlobalExceptionMapperCore.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/error/GlobalExceptionMapperCore.java
index 684254cdbfd930509da8f32d286d991402d8f1be..2b9a2a183381e5f627f1e1dff949b01ae5dc700f 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/error/GlobalExceptionMapperCore.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/error/GlobalExceptionMapperCore.java
@@ -105,6 +105,10 @@ public class GlobalExceptionMapperCore extends ResponseEntityExceptionHandler {
this.jaxRsDpsLogger.warning(exceptionMsg, e);
}
+ if (e.getError().getDebuggingInfo() != null) {
+ this.jaxRsDpsLogger.warning(e.getError().getDebuggingInfo());
+ }
+
// Support for non standard HttpStatus Codes
HttpStatus httpStatus = HttpStatus.resolve(e.getError().getCode());
if (httpStatus == null) {
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/Constants.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/Constants.java
index d8b400caac66da682dfb8d163e3708d5b8f3b630..163a405ebe5599b66d146b1db69edb37cca17c94 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/Constants.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/Constants.java
@@ -25,7 +25,6 @@ public class Constants {
public static final int SPEC_MAX_CACHE_SIZE = 20000;
public static final int CLIENT_CACHE_EXPIRATION = 600;
- public static final int CLIENT_CACHE_SIZE = 50;
// Data id itself is partition safe
public static final int DATA_CACHE_EXPIRATION = 120;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SearchRequest.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SearchRequest.java
deleted file mode 100644
index 3055519c7d22942b204c39bc45daeee800ea09e7..0000000000000000000000000000000000000000
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SearchRequest.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Copyright © Schlumberger
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0 *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.model;
-
-import lombok.Data;
-import lombok.ToString;
-import org.opengroup.osdu.core.common.model.search.SortQuery;
-
-import jakarta.validation.constraints.NotNull;
-import java.util.List;
-
-@Data
-@ToString
-public class SearchRequest {
- @NotNull(message = "Kind is missing")
- private Object kind;
- private String query;
- private int limit;
- private int offset;
- private String cursor;
- private List<String> returnedFields;
- private SortQuery sort;
- private boolean trackTotalCount;
-}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/XcollaborationHolder.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/XcollaborationHolder.java
index 5310ac5f35ac2ef626bf511c5c91e107bca8012b..dbf1a6aa7820fcadaf4c8a0c6ec7b0f320e95cf9 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/XcollaborationHolder.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/XcollaborationHolder.java
@@ -75,7 +75,7 @@ import org.springframework.web.context.annotation.RequestScope;
public class XcollaborationHolder {
public static final String X_COLLABORATION = RecordMetaAttribute.COLLABORATION_ID.getValue();
- private Optional<CollaborationContext> collaborationContext;
+ private Optional<CollaborationContext> collaborationContext = Optional.<CollaborationContext>empty();
@Autowired
private IFeatureFlag featureFlagChecker;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AugmenterConfigurationServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AugmenterConfigurationServiceImpl.java
index 57d665e371dcd83fa5e6a01d5d9937a9b7a7f1f0..c13ad177f4151e7f0aa288c097d47223b45a7eea 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AugmenterConfigurationServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/AugmenterConfigurationServiceImpl.java
@@ -15,18 +15,22 @@
package org.opengroup.osdu.indexer.service;
+import co.elastic.clients.elasticsearch._types.SortOptions;
+import co.elastic.clients.elasticsearch._types.SortOrder;
+import co.elastic.clients.elasticsearch._types.query_dsl.Query;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Strings;
import com.google.gson.Gson;
import org.apache.commons.collections.CollectionUtils;
+import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.indexer.IndexingStatus;
+import org.opengroup.osdu.core.common.model.indexer.JobStatus;
import org.opengroup.osdu.core.common.model.indexer.OperationType;
import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
-import org.opengroup.osdu.core.common.model.search.SortOrder;
-import org.opengroup.osdu.core.common.model.search.SortQuery;
import org.opengroup.osdu.core.common.model.storage.RecordData;
import org.opengroup.osdu.core.common.model.storage.Schema;
import org.opengroup.osdu.core.common.model.storage.SchemaItem;
@@ -35,12 +39,14 @@ import org.opengroup.osdu.indexer.cache.partitionsafe.*;
import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
import org.opengroup.osdu.indexer.model.*;
import org.opengroup.osdu.indexer.model.indexproperty.*;
+import org.opengroup.osdu.indexer.util.QueryUtil;
import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
import org.opengroup.osdu.indexer.util.PropertyUtil;
+import org.opengroup.osdu.indexer.util.SearchClient;
+import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import jakarta.inject.Inject;
-import java.net.URISyntaxException;
import java.util.*;
import java.util.stream.Collectors;
@@ -52,14 +58,6 @@ public class AugmenterConfigurationServiceImpl implements AugmenterConfiguration
private static final String ASSOCIATED_IDENTITIES_PROPERTY_STORAGE_FORMAT_TYPE = "[]string";
private static final String INDEX_PROPERTY_PATH_CONFIGURATION_KIND = "osdu:wks:reference-data--IndexPropertyPathConfiguration:*";
private static final String ANCESTRY_KINDS_DELIMITER = ",";
- private static final String PARENT_CHILDREN_CONFIGURATION_QUERY_FORMAT =
- "nested(data.Configurations, nested(data.Configurations.Paths, (RelatedObjectsSpec.RelationshipDirection: ParentToChildren AND RelatedObjectsSpec.RelatedObjectKind:\"%s\")))";
- private static final String CHILDREN_PARENT_CONFIGURATION_QUERY_FORMAT =
- "nested(data.Configurations, nested(data.Configurations.Paths, (RelatedObjectsSpec.RelationshipDirection: ChildToParent AND RelatedObjectsSpec.RelatedObjectKind:\"%s\")))";
- private static final String HAS_CONFIGURATIONS_QUERY_FORMAT = "data.Code: \"%s\" OR nested(data.Configurations, nested(data.Configurations.Paths, (RelatedObjectsSpec.RelatedObjectKind:\"%s\")))";
- private static final int MAX_SEARCH_LIMIT = 1000;
- private static final int MAX_TOTAL_COUNT = 10000;
-
private static final String PROPERTY_DELIMITER = ".";
private static final String NESTED_OBJECT_DELIMITER = "[].";
private static final String ARRAY_SYMBOL = "[]";
@@ -70,7 +68,9 @@ public class AugmenterConfigurationServiceImpl implements AugmenterConfiguration
private static final String STRING_ARRAY_KIND = "[]string";
private static final AugmenterConfiguration EMPTY_AUGMENTER_CONFIGURATION = new AugmenterConfiguration();
- private static final String SEARCH_GENERAL_ERROR = "Failed to call search service.";
+ private static final String SEARCH_GENERAL_ERROR = "Augmenter: Failed to search.";
+
+ private static final int NO_LIMIT = -1;
private final Gson gson = new Gson();
private final ObjectMapper objectMapper = new ObjectMapper();
@@ -92,8 +92,6 @@ public class AugmenterConfigurationServiceImpl implements AugmenterConfiguration
@Inject
private RecordChangeInfoCache recordChangeInfoCache;
@Inject
- private SearchService searchService;
- @Inject
private SchemaService schemaService;
@Inject
private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
@@ -101,6 +99,13 @@ public class AugmenterConfigurationServiceImpl implements AugmenterConfiguration
private IRequestInfo requestInfo;
@Inject
private JaxRsDpsLog jaxRsDpsLog;
+ @Inject
+ private JobStatus jobStatus;
+ @Inject
+ private SearchClient searchClient;
+
+ @Value("${augmenter.extended_list_value.max_size:2000}")
+ int maxSizeOfExtendedListValue;
@Override
public boolean isConfigurationEnabled(String kind) {
@@ -110,11 +115,8 @@ public class AugmenterConfigurationServiceImpl implements AugmenterConfiguration
Boolean enabled = augmenterConfigurationEnabledCache.get(kind);
if(enabled == null) {
- SearchRequest searchRequest = new SearchRequest();
- searchRequest.setKind(INDEX_PROPERTY_PATH_CONFIGURATION_KIND);
- String query = String.format(HAS_CONFIGURATIONS_QUERY_FORMAT, kind, kind);
- searchRequest.setQuery(query);
- if(searchFirstRecord(searchRequest) != null) {
+ Query query = QueryUtil.createQueryForConfigurations(kind);
+ if(searchFirstRecord(INDEX_PROPERTY_PATH_CONFIGURATION_KIND, query) != null) {
enabled = true;
}
else {
@@ -196,13 +198,9 @@ public class AugmenterConfigurationServiceImpl implements AugmenterConfiguration
RelatedObjectsSpec relatedObjectsSpec = path.getRelatedObjectsSpec();
if (relatedObjectsSpec.isChildToParent()) {
List<String> relatedObjectIds = getRelatedObjectIds(originalDataMap, relatedObjectsSpec);
- for (String relatedObjectId : relatedObjectIds) {
- // Store all ids
- associatedIdentities.add(PropertyUtil.removeIdPostfix(relatedObjectId));
- }
-
for (String relatedObjectId : relatedObjectIds) {
String id = PropertyUtil.removeIdPostfix(relatedObjectId);
+ associatedIdentities.add(id);
Map<String, Object> relatedObject = idObjectDataMap.getOrDefault(id, new HashMap<>());
Map<String, Object> propertyValues = getExtendedPropertyValues(extendedPropertyName, relatedObject, path.getValueExtraction(), configuration.isExtractFirstMatch());
if (allPropertyValues.isEmpty() && configuration.isExtractFirstMatch()) {
@@ -243,6 +241,10 @@ public class AugmenterConfigurationServiceImpl implements AugmenterConfiguration
extendedDataMap.putAll(allPropertyValues);
}
+
+ // Remove oversized extended properties with array type
+ removeOversizedExtendedProperties(objectId, extendedDataMap);
+
if (!associatedIdentities.isEmpty()) {
extendedDataMap.put(ASSOCIATED_IDENTITIES_PROPERTY, Arrays.asList(associatedIdentities.toArray()));
}
@@ -451,20 +453,7 @@ public class AugmenterConfigurationServiceImpl implements AugmenterConfiguration
}
private String createIdsQuery(List<String> ids) {
- return String.format("id: (%s)", createIdsFilter(ids));
- }
-
- private String createIdsFilter(List<String> ids) {
- StringBuilder idsBuilder = new StringBuilder();
- for (String id : ids) {
- if (idsBuilder.length() > 0) {
- idsBuilder.append(" OR ");
- }
- idsBuilder.append("\"");
- idsBuilder.append(PropertyUtil.removeIdPostfix(id));
- idsBuilder.append("\"");
- }
- return idsBuilder.toString();
+ return String.format("id: (%s)", QueryUtil.createIdsFilter(ids));
}
private void createWorkerTask(String ancestors, List<RecordInfo> recordInfos) {
@@ -486,12 +475,9 @@ public class AugmenterConfigurationServiceImpl implements AugmenterConfiguration
RecordData recordData = relatedObjectCache.get(id);
Map<String, Object> data = (recordData != null)? recordData.getData() : null;
if (data == null) {
- SearchRequest searchRequest = new SearchRequest();
- searchRequest.setKind(kind);
- String query = String.format("id: \"%s\"", id);
- searchRequest.setQuery(query);
- SearchRecord searchRecord = searchFirstRecord(searchRequest);
-
+ String queryString = String.format("id: \"%s\"", id);
+ Query query = QueryUtil.createSimpleTextQuery(queryString);
+ SearchRecord searchRecord = searchFirstRecord(kind, query);
if (searchRecord != null) {
data = searchRecord.getData();
recordData = new RecordData();
@@ -678,6 +664,21 @@ public class AugmenterConfigurationServiceImpl implements AugmenterConfiguration
return relatedObjectIds;
}
+ private void removeOversizedExtendedProperties(String objectId, Map<String, Object> extendedPropertyValues) {
+ List<String> propertyNames = new ArrayList<>(extendedPropertyValues.keySet());
+ for (String propertyName : propertyNames) {
+ Object value = extendedPropertyValues.get(propertyName);
+ if( value instanceof List<? extends Object>) {
+ List<?> values = (List<?>) value;
+ if(values != null && values.size() > maxSizeOfExtendedListValue) {
+ String message = String.format("Extended property '%s' has an over-sized array with %d items. Removed", propertyName, values.size());
+ this.jobStatus.addOrUpdateRecordStatus(objectId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, message);
+ extendedPropertyValues.remove(propertyName);
+ }
+ }
+ }
+ }
+
private Map<String, Object> getExtendedPropertyValues(String extendedPropertyName, Map<String, Object> dataMap, ValueExtraction valueExtraction, boolean isExtractFirstMatch) {
if (dataMap == null || dataMap.isEmpty() || valueExtraction == null || !valueExtraction.isValid())
return new HashMap<>();
@@ -741,7 +742,7 @@ public class AugmenterConfigurationServiceImpl implements AugmenterConfiguration
propertyValues.put(key, values);
}
}
- if (isExtractFirstMatch)
+ if (!subPropertyValues.isEmpty() && isExtractFirstMatch)
break;
}
}
@@ -946,9 +947,6 @@ public class AugmenterConfigurationServiceImpl implements AugmenterConfiguration
}
private void updateAssociatedChildrenRecords(String ancestors, String parentKind, List<RecordChangeInfo> recordChangeInfos) {
- List<String> processedIds = recordChangeInfos.stream().map(recordChangeInfo -> recordChangeInfo.getRecordInfo().getId()).toList();
- String query = String.format("data.%s:(%s)", ASSOCIATED_IDENTITIES_PROPERTY, createIdsFilter(processedIds));
-
List<String> childrenKinds = getChildrenKinds(parentKind);
for (String ancestryKind : ancestors.split(ANCESTRY_KINDS_DELIMITER)) {
// Exclude the kinds in the ancestryKinds to prevent circular chasing
@@ -963,12 +961,14 @@ public class AugmenterConfigurationServiceImpl implements AugmenterConfiguration
String kindWithMajor = PropertyUtil.getKindWithMajor(kind) + "*.*";
multiKinds.add(kindWithMajor);
}
- SearchRequest searchRequest = new SearchRequest();
- searchRequest.setKind(multiKinds);
- searchRequest.setQuery(query);
- searchRequest.setReturnedFields(Arrays.asList("kind", "id", "data." + ASSOCIATED_IDENTITIES_PROPERTY));
+ List<String> processedIds = recordChangeInfos.stream().map(recordChangeInfo -> recordChangeInfo.getRecordInfo().getId()).toList();
+ String queryString = String.format("data.%s:(%s)", ASSOCIATED_IDENTITIES_PROPERTY, QueryUtil.createIdsFilter(processedIds));
+ Query query = QueryUtil.createSimpleTextQuery(queryString);
+ List<String> returnedFields = List.of("kind", "id", "data." + ASSOCIATED_IDENTITIES_PROPERTY);
+ List<SearchRecord> records = this.search(multiKinds, query, null, returnedFields, NO_LIMIT);
+
List<RecordInfo> recordInfos = new ArrayList<>();
- for (SearchRecord searchRecord : searchRecords(searchRequest)) {
+ for (SearchRecord searchRecord : records) {
Map<String, Object> data = searchRecord.getData();
if (!data.containsKey(ASSOCIATED_IDENTITIES_PROPERTY) || data.get(ASSOCIATED_IDENTITIES_PROPERTY) == null)
continue;
@@ -1044,24 +1044,20 @@ public class AugmenterConfigurationServiceImpl implements AugmenterConfiguration
}
/****************************** search methods that use search service to get the data **************************************/
- private SearchRequest createSearchRequest(String kind, String query) {
- SearchRequest searchRequest = new SearchRequest();
- searchRequest.setKind(kind);
- searchRequest.setQuery(query);
- return searchRequest;
- }
-
private AugmenterConfiguration searchConfiguration(String kind) {
- String query = String.format("data.Code: \"%s\"", kind);
- SearchRequest searchRequest = createSearchRequest(INDEX_PROPERTY_PATH_CONFIGURATION_KIND, query);
+ String queryString = String.format("data.Code: \"%s\"", kind);
+ Query query = QueryUtil.createSimpleTextQuery(queryString);
// If there is more than PropertyConfigurations, pick the one that was last modified.
// Given the property "modifyTime" is not set for new created record, we use property "version"
// to sort the search result in descending order
- SortQuery sort = new SortQuery();
- sort.setField(Arrays.asList(VERSION_PROPERTY));
- sort.setOrder(Arrays.asList(SortOrder.DESC));
- searchRequest.setSort(sort);
- List<AugmenterConfiguration> augmenterConfigurations = searchConfigurations(searchRequest);
+ List<SortOptions> sortOptionsList = null;
+ try {
+ sortOptionsList = QueryUtil.createSortOptionsList(List.of(VERSION_PROPERTY), List.of(SortOrder.Desc));
+ } catch(Exception ex) {
+ // Should not reach here. Ignore
+ }
+
+ List<AugmenterConfiguration> augmenterConfigurations = searchConfigurations(query, sortOptionsList);
if(!augmenterConfigurations.isEmpty()) {
if(augmenterConfigurations.size() > 1) {
jaxRsDpsLog.warning(String.format("There is more than one PropertyConfigurations for kind: %s", kind));
@@ -1072,20 +1068,19 @@ public class AugmenterConfigurationServiceImpl implements AugmenterConfiguration
}
private List<AugmenterConfiguration> searchParentKindConfigurations(String childKind) {
- String query = String.format(PARENT_CHILDREN_CONFIGURATION_QUERY_FORMAT, childKind);
- SearchRequest searchRequest = createSearchRequest(INDEX_PROPERTY_PATH_CONFIGURATION_KIND, query);
- return searchConfigurations(searchRequest);
+ Query query = QueryUtil.createQueryForParentConfigs(childKind);
+ return searchConfigurations(query, null);
}
private List<AugmenterConfiguration> searchChildrenKindConfigurations(String parentKind) {
- String query = String.format(CHILDREN_PARENT_CONFIGURATION_QUERY_FORMAT, parentKind);
- SearchRequest searchRequest = createSearchRequest(INDEX_PROPERTY_PATH_CONFIGURATION_KIND, query);
- return searchConfigurations(searchRequest);
+ Query query = QueryUtil.createQueryForChildrenConfigs(parentKind);
+ return searchConfigurations(query,null);
}
- private List<AugmenterConfiguration> searchConfigurations(SearchRequest searchRequest) {
+ private List<AugmenterConfiguration> searchConfigurations(Query query, List<SortOptions> sortOptions) {
List<AugmenterConfiguration> augmenterConfigurations = new ArrayList<>();
- for (SearchRecord searchRecord : searchRecords(searchRequest)) {
+ List<SearchRecord> records = this.search(INDEX_PROPERTY_PATH_CONFIGURATION_KIND, query, sortOptions, null, NO_LIMIT);
+ for (SearchRecord searchRecord : records) {
try {
String data = objectMapper.writeValueAsString(searchRecord.getData());
AugmenterConfiguration configurations = objectMapper.readValue(data, AugmenterConfiguration.class);
@@ -1093,35 +1088,33 @@ public class AugmenterConfigurationServiceImpl implements AugmenterConfiguration
augmenterConfigurationCache.put(kind, configurations);
augmenterConfigurations.add(configurations);
} catch (JsonProcessingException e) {
- jaxRsDpsLog.error("failed to deserialize PropertyConfigurations object", e);
+ jaxRsDpsLog.error("Augmenter(searchConfigurations): failed to deserialize PropertyConfigurations object", e);
}
}
return augmenterConfigurations;
}
private List<SearchRecord> searchRelatedRecords(List<String> relatedObjectKinds, List<String> relatedObjectIds) {
- SearchRequest searchRequest = new SearchRequest();
List<String> kinds = new ArrayList<>();
for(String kind : relatedObjectKinds) {
if(!PropertyUtil.isConcreteKind(kind))
kind += "*";
kinds.add(kind);
}
- searchRequest.setKind(kinds);
- String query = createIdsQuery(relatedObjectIds);
- searchRequest.setQuery(query);
- return searchRecords(searchRequest);
+ String queryString = createIdsQuery(relatedObjectIds);
+ Query query = QueryUtil.createSimpleTextQuery(queryString);
+ return this.search(kinds, query, null, null, NO_LIMIT);
}
private Map<String, List<String>> searchKindIds(String majorKind, List<String> ids) {
- Map<String, List<String>> kindIds = new HashMap<>();
- SearchRequest searchRequest = new SearchRequest();
String kind = PropertyUtil.isConcreteKind(majorKind) ? majorKind : majorKind + "*";
- searchRequest.setKind(kind);
- String query = createIdsQuery(ids);
- searchRequest.setReturnedFields(Arrays.asList("kind", "id"));
- searchRequest.setQuery(query);
- for (SearchRecord searchRecord : searchRecords(searchRequest)) {
+ String queryString = createIdsQuery(ids);
+ Query query = QueryUtil.createSimpleTextQuery(queryString);
+ List<String> returnedFields = List.of("kind", "id");
+
+ Map<String, List<String>> kindIds = new HashMap<>();
+ List<SearchRecord> records = this.search(kind, query, null, returnedFields, NO_LIMIT);
+ for (SearchRecord searchRecord : records) {
if (kindIds.containsKey(searchRecord.getKind())) {
kindIds.get(searchRecord.getKind()).add(searchRecord.getId());
} else {
@@ -1135,13 +1128,12 @@ public class AugmenterConfigurationServiceImpl implements AugmenterConfiguration
private List<String> searchUniqueParentIds(String childKind, List<String> childRecordIds, String parentObjectIdPath) {
Set<String> parentIds = new HashSet<>();
- SearchRequest searchRequest = new SearchRequest();
- searchRequest.setKind(childKind);
- String query = createIdsQuery(childRecordIds);
- searchRequest.setReturnedFields(Arrays.asList(parentObjectIdPath));
- searchRequest.setQuery(query);
+ String queryString = createIdsQuery(childRecordIds);
+ Query query = QueryUtil.createSimpleTextQuery(queryString);
+ List<String> returnedFields = Arrays.asList(parentObjectIdPath);
+
+ List<SearchRecord> searchRecords = this.search(childKind, query, null, returnedFields, NO_LIMIT);
parentObjectIdPath = PropertyUtil.removeDataPrefix(parentObjectIdPath);
- List<SearchRecord> searchRecords = searchRecords(searchRequest);
Map<String, SearchRecord> idRecords = searchRecords.stream().collect(Collectors.toMap(SearchRecord::getId, record -> record));
for(String childRecordId : childRecordIds) {
RecordData recordData = this.relatedObjectCache.get(childRecordId);
@@ -1155,7 +1147,6 @@ public class AugmenterConfigurationServiceImpl implements AugmenterConfiguration
data = searchRecord.getData();
}
}
-
if(data != null && data.containsKey(parentObjectIdPath)) {
Object id = data.get(parentObjectIdPath);
if (id != null && !parentIds.contains(id)) {
@@ -1163,79 +1154,41 @@ public class AugmenterConfigurationServiceImpl implements AugmenterConfiguration
}
}
}
-
return new ArrayList<>(parentIds);
}
private List<SearchRecord> searchChildrenRecords(String childrenObjectKind, String childrenObjectField, String parentId) {
String kind = PropertyUtil.isConcreteKind(childrenObjectKind) ? childrenObjectKind : childrenObjectKind + "*";
- SearchRequest searchRequest = new SearchRequest();
- searchRequest.setKind(kind);
- String query = String.format("%s: \"%s\"", childrenObjectField, parentId);
- searchRequest.setQuery(query);
- return searchRecords(searchRequest);
+ String queryString = String.format("%s: \"%s\"", childrenObjectField, parentId);
+ Query query = QueryUtil.createSimpleTextQuery(queryString);
+ return this.search(kind, query, null, null, NO_LIMIT);
}
- private List<SearchRecord> searchRecordsWithCursor(SearchRequest searchRequest) {
- searchRequest.setLimit(MAX_SEARCH_LIMIT);
- List<SearchRecord> allRecords = new ArrayList<>();
- try {
- List<SearchRecord> results = null;
- String cursor = null;
- do {
- SearchResponse searchResponse = searchService.queryWithCursor(searchRequest);
- results = searchResponse.getResults();
- cursor = searchResponse.getCursor();
- if (!CollectionUtils.isEmpty(results)) {
- allRecords.addAll(results);
- if (!Strings.isNullOrEmpty(cursor)) {
- searchRequest.setCursor(cursor);
- }
- }
- } while(results != null && cursor != null);
- } catch (URISyntaxException e) {
- jaxRsDpsLog.error(SEARCH_GENERAL_ERROR, e);
+ private SearchRecord searchFirstRecord(String kind, Query query) {
+ List<SearchRecord> results = this.search(kind, query, null, null, 1);
+ if (!results.isEmpty()) {
+ return results.get(0);
}
- return allRecords;
+ return null;
}
- private List<SearchRecord> searchRecords(SearchRequest searchRequest) {
- searchRequest.setLimit(MAX_SEARCH_LIMIT);
- int offset = 0;
- List<SearchRecord> allRecords = new ArrayList<>();
+ private List<SearchRecord> search(String kind, Query query, List<SortOptions> sortOptions, List<String> returnedFields, int limit) {
try {
- List<SearchRecord> results = null;
- do {
- SearchResponse searchResponse = searchService.query(searchRequest);
- if(searchResponse.getTotalCount() >= MAX_TOTAL_COUNT) {
- // The search without cursor can return max. 10,000 records
- // If the totalCount reaches 10,1000 records, we should switch to using query with cursor
- return searchRecordsWithCursor(searchRequest);
- }
- results = searchResponse.getResults();
- if (!CollectionUtils.isEmpty(results)) {
- allRecords.addAll(results);
- offset += results.size();
- searchRequest.setOffset(offset);
- }
- } while(results != null && results.size() == MAX_SEARCH_LIMIT);
- } catch (URISyntaxException e) {
- jaxRsDpsLog.error(SEARCH_GENERAL_ERROR, e);
+ return searchClient.search(kind, query, sortOptions, returnedFields, limit);
+ }
+ catch (Exception ex) {
+ this.jaxRsDpsLog.error(SEARCH_GENERAL_ERROR, ex);
}
- return allRecords;
+ return new ArrayList<>();
}
- private SearchRecord searchFirstRecord(SearchRequest searchRequest) {
- searchRequest.setLimit(1);
+ private List<SearchRecord> search(List<String> kinds, Query query, List<SortOptions> sortOptions, List<String> returnedFields, int limit) {
try {
- SearchResponse searchResponse = searchService.query(searchRequest);
- List<SearchRecord> results = searchResponse.getResults();
- if (results != null && !results.isEmpty()) {
- return results.get(0);
- }
- } catch (URISyntaxException e) {
- jaxRsDpsLog.error(SEARCH_GENERAL_ERROR, e);
+ return searchClient.search(kinds, query, sortOptions, returnedFields, limit);
}
- return null;
+ catch (Exception ex) {
+ this.jaxRsDpsLog.error(SEARCH_GENERAL_ERROR, ex);
+ }
+ return new ArrayList<>();
}
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
index 40361be4aa7ee8f51a2458736926e993e3adc0f4..549c05baea6c39dbfae71ad77c2e293719d88a95 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
@@ -44,6 +44,7 @@ import org.opengroup.osdu.indexer.cache.partitionsafe.VirtualPropertiesSchemaCac
import org.opengroup.osdu.indexer.model.Kind;
import org.opengroup.osdu.indexer.model.indexproperty.AugmenterConfiguration;
import org.opengroup.osdu.indexer.schema.converter.exeption.SchemaProcessingException;
+import org.opengroup.osdu.indexer.service.exception.ElasticsearchMappingException;
import org.opengroup.osdu.indexer.util.AugmenterSetting;
import org.opengroup.osdu.indexer.util.ElasticClientHandler;
import org.opengroup.osdu.indexer.util.TypeMapper;
@@ -134,10 +135,10 @@ public class IndexSchemaServiceImpl implements IndexSchemaService {
try {
// merge the mapping
this.mappingService.createMapping(restClient, schemaObj, index, true);
- } catch (AppException e) {
+ } catch (ElasticsearchMappingException e) {
// acknowledge for TaskQueue and not retry
- if (e.getError().getCode() == HttpStatus.SC_BAD_REQUEST) {
- throw new AppException(RequestStatus.SCHEMA_CONFLICT, e.getError().getReason(), "error creating or merging index mapping");
+ if (e.getStatus() == HttpStatus.SC_BAD_REQUEST) {
+ throw new AppException(RequestStatus.SCHEMA_CONFLICT, e.getMessage(), "error creating or merging index mapping");
}
throw e;
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
index fae8237a144e4abdc23ecf44916d544f85322d38..5c4feac66d54605222fd85b33ab900f61a774eee 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
@@ -14,11 +14,6 @@
package org.opengroup.osdu.indexer.service;
-import static org.opengroup.osdu.core.common.model.search.RecordMetaAttribute.BAG_OF_WORDS;
-import static org.opengroup.osdu.indexer.config.IndexerConfigurationProperties.BAG_OF_WORDS_FEATURE_NAME;
-import static org.opengroup.osdu.indexer.config.IndexerConfigurationProperties.KEYWORD_LOWER_FEATURE_NAME;
-import static org.opengroup.osdu.indexer.model.XcollaborationHolder.X_COLLABORATION;
-
import co.elastic.clients.elasticsearch.ElasticsearchClient;
import co.elastic.clients.elasticsearch._types.ElasticsearchException;
import co.elastic.clients.elasticsearch._types.Time;
@@ -29,17 +24,8 @@ import co.elastic.clients.elasticsearch.indices.PutMappingResponse;
import co.elastic.clients.elasticsearch.indices.get_mapping.IndexMappingRecord;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
-import com.lambdaworks.redis.RedisException;
-import java.util.stream.Collectors;
-import java.util.stream.Collectors;
+import io.lettuce.core.RedisException;
import jakarta.inject.Inject;
-import java.io.IOException;
-import java.io.StringReader;
-import java.lang.reflect.Type;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
import org.opengroup.osdu.core.common.Constants;
import org.opengroup.osdu.core.common.feature.IFeatureFlag;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
@@ -51,20 +37,22 @@ import org.opengroup.osdu.indexer.cache.partitionsafe.IndexCache;
import org.opengroup.osdu.indexer.model.Kind;
import org.opengroup.osdu.indexer.model.XcollaborationHolder;
import org.opengroup.osdu.indexer.service.exception.ElasticsearchMappingException;
+import org.opengroup.osdu.indexer.util.CustomIndexAnalyzerSetting;
import org.opengroup.osdu.indexer.util.TypeMapper;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
-import jakarta.inject.Inject;
import java.io.IOException;
+import java.io.StringReader;
import java.lang.reflect.Type;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.stream.Collectors;
import static org.opengroup.osdu.core.common.model.search.RecordMetaAttribute.BAG_OF_WORDS;
-import static org.opengroup.osdu.indexer.config.IndexerConfigurationProperties.KEYWORD_LOWER_FEATURE_NAME;
import static org.opengroup.osdu.indexer.config.IndexerConfigurationProperties.BAG_OF_WORDS_FEATURE_NAME;
+import static org.opengroup.osdu.indexer.config.IndexerConfigurationProperties.KEYWORD_LOWER_FEATURE_NAME;
import static org.opengroup.osdu.indexer.model.XcollaborationHolder.X_COLLABORATION;
@Service
@@ -81,6 +69,8 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements IMa
private IFeatureFlag featureFlagChecker;
@Autowired
private XcollaborationHolder xcollaborationHolder;
+ @Autowired
+ private CustomIndexAnalyzerSetting customIndexAnalyzerSetting;
/**
@@ -179,11 +169,12 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements IMa
Map<String, Object> dataMapping = new HashMap<>();
boolean keywordLowerEnabled = this.featureFlagChecker.isFeatureEnabled(KEYWORD_LOWER_FEATURE_NAME);
boolean bagOfWordsEnabled = this.featureFlagChecker.isFeatureEnabled(BAG_OF_WORDS_FEATURE_NAME);
+ boolean customIndexAnalyzerEnabled = this.customIndexAnalyzerSetting.isEnabled();
if (schema.getDataSchema() == null || schema.getDataSchema().isEmpty()) return dataMapping;
for (Map.Entry<String, Object> entry : schema.getDataSchema().entrySet()) {
- dataMapping.put(entry.getKey(), TypeMapper.getDataAttributeIndexerMapping(entry.getValue(), keywordLowerEnabled, bagOfWordsEnabled));
+ dataMapping.put(entry.getKey(), TypeMapper.getDataAttributeIndexerMapping(entry.getValue(), keywordLowerEnabled, bagOfWordsEnabled, customIndexAnalyzerEnabled));
}
return dataMapping;
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
index de2582309181b064be2fce32eed0c501cbf5ffc5..e01b4a5f5a539cf82a515a9a8e8251fee7691c2c 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
@@ -202,9 +202,11 @@ public class IndexerServiceImpl implements IndexerService {
}
if (this.augmenterSetting.isEnabled()) {
+ Map<String, List<String>> upsertKindIds = null;
+ Map<String, List<String>> deleteKindIds = null;
try {
- Map<String, List<String>> upsertKindIds = getUpsertRecordIdsForConfigurationsEnabledKinds(upsertRecordMap, retryRecordIds);
- Map<String, List<String>> deleteKindIds = getDeleteRecordIdsForConfigurationsEnabledKinds(deleteRecordMap, retryRecordIds);
+ upsertKindIds = getUpsertRecordIdsForConfigurationsEnabledKinds(upsertRecordMap, retryRecordIds);
+ deleteKindIds = getDeleteRecordIdsForConfigurationsEnabledKinds(deleteRecordMap, retryRecordIds);
if (!upsertKindIds.isEmpty() || !deleteKindIds.isEmpty()) {
augmenterConfigurationService.updateAssociatedRecords(message, upsertKindIds, deleteKindIds, deletedRecordsWithParentReferred);
}
@@ -215,7 +217,14 @@ public class IndexerServiceImpl implements IndexerService {
}
}
catch(Exception ex) {
- jaxRsDpsLog.error("Augmenter: Failed to update associated records", ex);
+ List<String> ids = new ArrayList<>();
+ if(upsertKindIds != null) {
+ upsertKindIds.values().forEach(ids::addAll);
+ }
+ if(deleteKindIds != null) {
+ deleteKindIds.values().forEach(ids::addAll);
+ }
+ jaxRsDpsLog.error(String.format("Augmenter: Failed to update associated records of the records:[%s]", String.join(",", ids)), ex);
}
}
} catch (IOException e) {
@@ -483,6 +492,8 @@ public class IndexerServiceImpl implements IndexerService {
}
}
catch(Exception ex) {
+ String message = String.format("Augmenter error: %s", ex.getMessage());
+ this.jobStatus.addOrUpdateRecordStatus(storageRecord.getId(), IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, message);
jaxRsDpsLog.error(String.format("Augmenter: Failed to merge extended properties of the record with id: '%s' and kind: '%s'", storageRecord.getId(), storageRecord.getKind()), ex);
}
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java
index 5f810f30216f125bf8287158ae7c99c3a520361a..35d92b5f6168f3516f2e858a1e69d0bf5f5d2207 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndicesServiceImpl.java
@@ -16,10 +16,13 @@ package org.opengroup.osdu.indexer.service;
import co.elastic.clients.elasticsearch.ElasticsearchClient;
import co.elastic.clients.elasticsearch._types.ElasticsearchException;
-import co.elastic.clients.elasticsearch._types.HealthStatus;
import co.elastic.clients.elasticsearch._types.Time;
-import co.elastic.clients.elasticsearch.cluster.HealthRequest;
-import co.elastic.clients.elasticsearch.cluster.HealthResponse;
+import co.elastic.clients.elasticsearch._types.analysis.Analyzer;
+import co.elastic.clients.elasticsearch._types.analysis.CharFilter;
+import co.elastic.clients.elasticsearch._types.analysis.CharFilterDefinition;
+import co.elastic.clients.elasticsearch._types.analysis.CustomAnalyzer;
+import co.elastic.clients.elasticsearch._types.analysis.MappingCharFilter;
+import co.elastic.clients.elasticsearch._types.analysis.PatternReplaceCharFilter;
import co.elastic.clients.elasticsearch.indices.CreateIndexRequest;
import co.elastic.clients.elasticsearch.indices.CreateIndexRequest.Builder;
import co.elastic.clients.elasticsearch.indices.CreateIndexResponse;
@@ -29,21 +32,15 @@ import co.elastic.clients.elasticsearch.indices.ExistsRequest;
import co.elastic.clients.elasticsearch.indices.GetIndexRequest;
import co.elastic.clients.elasticsearch.indices.GetIndexResponse;
import co.elastic.clients.elasticsearch.indices.IndexSettings;
+import co.elastic.clients.elasticsearch.indices.IndexSettingsAnalysis;
import co.elastic.clients.transport.endpoints.BooleanResponse;
import co.elastic.clients.transport.rest_client.RestClientTransport;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Strings;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
-import com.lambdaworks.redis.RedisException;
+import io.lettuce.core.RedisException;
import jakarta.inject.Inject;
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.lang.reflect.Type;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
import org.apache.http.HttpStatus;
import org.apache.http.util.EntityUtils;
import org.elasticsearch.client.Request;
@@ -54,11 +51,26 @@ import org.opengroup.osdu.core.common.model.search.IndexInfo;
import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
import org.opengroup.osdu.core.common.search.Preconditions;
import org.opengroup.osdu.indexer.cache.partitionsafe.IndexCache;
+import org.opengroup.osdu.indexer.util.CustomIndexAnalyzerSetting;
import org.opengroup.osdu.indexer.util.ElasticClientHandler;
+import org.opengroup.osdu.indexer.util.TypeMapper;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.web.context.annotation.RequestScope;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.lang.reflect.Type;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+
+import static co.elastic.clients.elasticsearch._types.HealthStatus.*;
+
@Service
@RequestScope
public class IndicesServiceImpl implements IndicesService {
@@ -68,7 +80,6 @@ public class IndicesServiceImpl implements IndicesService {
private static final String INDEX_CANNOT_BE_NULL = "index cannot be null";
private static final Time REQUEST_TIMEOUT = Time.of(builder -> builder.time("1m"));
- private static final IndexSettings DEFAULT_INDEX_SETTINGS;
@Autowired
private ElasticClientHandler elasticClientHandler;
@@ -82,14 +93,12 @@ public class IndicesServiceImpl implements IndicesService {
private JaxRsDpsLog log;
@Autowired
private ObjectMapper objectMapper;
-
- static {
- IndexSettings.Builder builder = new IndexSettings.Builder();
- builder.refreshInterval(Time.of(timeBuilder -> timeBuilder.time("30s")));
- builder.numberOfShards("1");
- builder.numberOfReplicas("1");
- DEFAULT_INDEX_SETTINGS = builder.build();
- }
+ @Autowired
+ private CustomIndexAnalyzerSetting customIndexAnalyzerSetting;
+ @Value("${index.health.retry.threshold:5}")
+ private int healthRetryThreshold;
+ @Value("${index.health.retry.sleepPeriodInMilliseconds:5000}")
+ private int healthRetrySleepPeriodInMilliseconds;
/**
* Create a new index in Elasticsearch
@@ -106,7 +115,7 @@ public class IndicesServiceImpl implements IndicesService {
try {
CreateIndexRequest.Builder createIndexBuilder = new Builder();
createIndexBuilder.index(index);
- createIndexBuilder.settings(settings != null ? settings : DEFAULT_INDEX_SETTINGS);
+ createIndexBuilder.settings(settings != null ? settings : getDefaultIndexSettings());
if (mapping != null) {
Map<String, Map<String, Object>> mappings = Map.of("mappings", mapping);
@@ -115,13 +124,14 @@ public class IndicesServiceImpl implements IndicesService {
}
createIndexBuilder.timeout(REQUEST_TIMEOUT);
long startTime = System.currentTimeMillis();
+
CreateIndexResponse createIndexResponse = client.indices().create(createIndexBuilder.build());
long stopTime = System.currentTimeMillis();
// cache the index status
boolean indexStatus = createIndexResponse.acknowledged() && createIndexResponse.shardsAcknowledged();
if (indexStatus) {
this.indexCache.put(index, true);
- this.log.info(String.format("Time taken to successfully create new index %s : %d milliseconds", index, stopTime-startTime));
+ this.log.info(String.format("Time taken to successfully create new index %s : %d milliseconds", index, stopTime - startTime));
// Create alias for index
indexAliasService.createIndexAlias(client, elasticIndexNameResolver.getKindFromIndexName(index));
@@ -162,10 +172,10 @@ public class IndicesServiceImpl implements IndicesService {
return false;
}
throw new AppException(
- exception.status(),
- exception.getMessage(),
- String.format("Error getting index: %s status", index),
- exception);
+ exception.status(),
+ exception.getMessage(),
+ String.format("Error getting index: %s status", index),
+ exception);
}
}
@@ -178,32 +188,91 @@ public class IndicesServiceImpl implements IndicesService {
*/
public boolean isIndexReady(ElasticsearchClient client, String index) throws IOException {
try {
- if (this.indexExistInCache(index)){
+ if (this.indexExistInCache(index)) {
return true;
}
- ExistsRequest existsRequest = ExistsRequest.of(builder -> builder.index(index));
- BooleanResponse exists = client.indices().exists(existsRequest);
- if (!exists.value()){
- return false;
- }
- HealthRequest healthRequest = HealthRequest.of(builder -> builder.index(index)
- .timeout(REQUEST_TIMEOUT)
- .waitForStatus(HealthStatus.Yellow)
- );
- HealthResponse health = client.cluster().health(healthRequest);
- if (health.status() == HealthStatus.Green || health.status() == HealthStatus.Yellow) {
+ boolean isHealthy = isIndexHealthy(client, index);
+ if (isHealthy) {
this.indexCache.put(index, true);
- return true;
}
- return false;
+ return isHealthy;
} catch (ElasticsearchException exception) {
if (exception.status() == HttpStatus.SC_NOT_FOUND) return false;
throw new AppException(
- exception.status(),
- exception.getMessage(),
- String.format("Error getting index: %s status", index),
- exception);
+ exception.status(),
+ exception.getMessage(),
+ String.format("Error getting index: %s status", index),
+ exception);
+ }
+ }
+
+ private boolean isIndexHealthy(ElasticsearchClient client, String index) throws IOException {
+ ExistsRequest existsRequest = ExistsRequest.of(builder -> builder.index(index));
+ BooleanResponse exists = client.indices().exists(existsRequest);
+ if (!exists.value()) {
+ return false;
+ }
+ String actualHealthStatus = null;
+ for(int retryCount = 0; retryCount <= healthRetryThreshold; retryCount++) {
+ List<IndexInfo> indexHealthInfos = this.getIndexInfo(client, index);
+ if (!indexHealthInfos.isEmpty()) {
+ actualHealthStatus = indexHealthInfos.get(0).getHealth();
+ if (Green.jsonValue().equalsIgnoreCase(actualHealthStatus) || Yellow.jsonValue().equalsIgnoreCase(actualHealthStatus)) {
+ return true;
+ }
+ }
+ sleepInMilliSeconds(healthRetrySleepPeriodInMilliseconds);
+ }
+ if (Red.jsonValue().equalsIgnoreCase(actualHealthStatus)) {
+ throw new AppException(HttpStatus.SC_SERVICE_UNAVAILABLE,
+ "Index not available for indexing",
+ String.format("Index: %s primary shards are unassigned", index));
}
+ return false;
+ }
+
+ private void sleepInMilliSeconds(int milliseconds) {
+ try {
+ TimeUnit.MILLISECONDS.sleep(milliseconds);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private IndexSettings getDefaultIndexSettings() {
+ IndexSettings.Builder builder = new IndexSettings.Builder();
+ builder.refreshInterval(Time.of(timeBuilder -> timeBuilder.time("30s")));
+ builder.numberOfShards("1");
+ builder.numberOfReplicas("1");
+ if (customIndexAnalyzerSetting.isEnabled()) {
+ IndexSettingsAnalysis analysis = getCustomAnalyzer();
+ builder.analysis(analysis);
+ }
+ return builder.build();
+ }
+
+ private IndexSettingsAnalysis getCustomAnalyzer() {
+ MappingCharFilter mappingCharFilter = new MappingCharFilter.Builder().mappings("_=>\\u0020").build();
+ PatternReplaceCharFilter patternReplaceCharFilter = new PatternReplaceCharFilter.Builder().pattern("(\\D)\\.|\\.(?=\\D)").replacement("$1 ").build();
+
+ Map<String, CharFilter> charFilterMap = new HashMap<>();
+ charFilterMap.put("osdu_mapping_charFilter_for_underscore", new CharFilter.Builder()
+ .definition(new CharFilterDefinition.Builder().mapping(mappingCharFilter).build())
+ .build());
+ charFilterMap.put("osdu_patternReplace_charFilter_for_dot", new CharFilter.Builder()
+ .definition(new CharFilterDefinition.Builder().patternReplace(patternReplaceCharFilter).build())
+ .build());
+ Analyzer analyzer = new CustomAnalyzer.Builder()
+ .tokenizer("standard")
+ .filter("lowercase")
+ .charFilter(charFilterMap.keySet().stream().toList())
+ .build()
+ ._toAnalyzer();
+ IndexSettingsAnalysis analysis = new IndexSettingsAnalysis.Builder()
+ .charFilter(charFilterMap)
+ .analyzer(TypeMapper.OSDU_CUSTOM_ANALYZER, analyzer)
+ .build();
+ return analysis;
}
private boolean indexExistInCache(String index) {
@@ -226,7 +295,7 @@ public class IndicesServiceImpl implements IndicesService {
public boolean deleteIndex(ElasticsearchClient client, String index) throws ElasticsearchException, IOException, AppException {
List<String> indices = this.resolveIndex(client, index);
boolean responseStatus = true;
- for (String idx: indices) {
+ for (String idx : indices) {
responseStatus &= removeIndexInElasticsearch(client, idx);
}
if (responseStatus) {
@@ -282,25 +351,27 @@ public class IndicesServiceImpl implements IndicesService {
}
/**
- * Remove index in Elasticsearch
+ * Get index information from Elasticsearch
*
* @param client Elasticsearch client
* @param indexPattern Index pattern
+ * @return List of indices matching indexPattern
* @throws IOException Throws {@link IOException} if elastic cannot complete the request
*/
public List<IndexInfo> getIndexInfo(ElasticsearchClient client, String indexPattern) throws IOException {
Objects.requireNonNull(client, CLIENT_CANNOT_BE_NULL);
String requestUrl = (indexPattern == null || indexPattern.isEmpty())
- ? "/_cat/indices/*,-.*?h=index,docs.count,creation.date&s=docs.count:asc&format=json"
- : String.format("/_cat/indices/%s?h=index,docs.count,creation.date&format=json", indexPattern);
+ ? "/_cat/indices/*,-.*?h=index,health,docs.count,creation.date&s=docs.count:asc&format=json"
+ : String.format("/_cat/indices/%s?h=index,health,docs.count,creation.date&format=json", indexPattern);
- try (RestClientTransport clientTransport = (RestClientTransport)client._transport()){
+ try (RestClientTransport clientTransport = (RestClientTransport) client._transport()) {
Request request = new Request("GET", requestUrl);
Response response = clientTransport.restClient().performRequest(request);
String responseBody = EntityUtils.toString(response.getEntity());
- Type typeOf = new TypeToken<List<IndexInfo>>() {}.getType();
+ Type typeOf = new TypeToken<List<IndexInfo>>() {
+ }.getType();
return new Gson().fromJson(responseBody, typeOf);
}
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java
index f2c101c7c5a48744ac63033474ff8c719da812d7..ec1c5b1b55ebf279617f86feaf13073cec8e5c23 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/ReindexServiceImpl.java
@@ -14,6 +14,7 @@
package org.opengroup.osdu.indexer.service;
+import co.elastic.clients.elasticsearch._types.query_dsl.Query;
import com.google.common.base.Strings;
import com.google.gson.Gson;
@@ -26,10 +27,13 @@ import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.indexer.*;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.opengroup.osdu.indexer.model.SearchRecord;
import org.opengroup.osdu.indexer.model.XcollaborationHolder;
+import org.opengroup.osdu.indexer.util.QueryUtil;
import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
+import org.opengroup.osdu.indexer.util.SearchClient;
import org.springframework.stereotype.Component;
import jakarta.inject.Inject;
@@ -52,6 +56,8 @@ public class ReindexServiceImpl implements ReindexService {
private JaxRsDpsLog jaxRsDpsLog;
@Inject
private XcollaborationHolder xCollaborationHolder;
+ @Inject
+ private SearchClient searchClient;
@SneakyThrows
@Override
@@ -98,11 +104,26 @@ public class ReindexServiceImpl implements ReindexService {
@Override
public Records reindexRecords(List<String> recordIds) {
Records records = this.storageService.getStorageRecords(recordIds);
- if (records.getRecords().size() > 0) {
+ if (!records.getRecords().isEmpty()) {
List<RecordInfo> msgs = records.getRecords().stream()
.map(record -> RecordInfo.builder().id(record.getId()).kind(record.getKind()).op(OperationType.create.name()).build()).collect(Collectors.toList());
this.replayReindexMsg(msgs, 0L, null);
}
+ if (!records.getNotFound().isEmpty()) {
+ String queryString = String.format("id: (%s)", QueryUtil.createIdsFilter(records.getNotFound()));
+ Query query = QueryUtil.createSimpleTextQuery(queryString);
+ List<String> returnedFields = List.of("kind", "id");
+ try {
+ List<SearchRecord> results = searchClient.search("*:*:*:*", query, null, returnedFields, records.getNotFound().size());
+ List<RecordInfo> msgs = results.stream()
+ .map(record -> RecordInfo.builder().id(record.getId()).kind(record.getKind()).op(OperationType.delete.name()).build()).collect(Collectors.toList());
+ this.replayReindexMsg(msgs, 0L, null);
+ }
+ catch(Exception ex) {
+ jaxRsDpsLog.error("reindexRecords: Failed to search", ex);
+ }
+
+ }
return records;
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SearchService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SearchService.java
deleted file mode 100644
index 6617e4b90bb8fccce1f0d0e2cc8ae6de941a5dcc..0000000000000000000000000000000000000000
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SearchService.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright © Schlumberger
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0 *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.service;
-
-import org.opengroup.osdu.indexer.model.SearchRequest;
-import org.opengroup.osdu.indexer.model.SearchResponse;
-
-import java.net.URISyntaxException;
-
-/**
- * Interface to consume schemas from the Schema Service
- */
-public interface SearchService {
- /**
- *
- * @param searchRequest
- * @return
- * @throws URISyntaxException
- */
- SearchResponse query(SearchRequest searchRequest) throws URISyntaxException;
-
- /**
- *
- * @param searchRequest
- * @return
- * @throws URISyntaxException
- */
- SearchResponse queryWithCursor(SearchRequest searchRequest) throws URISyntaxException;
-}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SearchServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SearchServiceImpl.java
deleted file mode 100644
index d79b6f8bc1ecd1892eb4489a8b81564c2870719f..0000000000000000000000000000000000000000
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SearchServiceImpl.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Copyright © Schlumberger
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0 *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.service;
-
-import com.google.api.client.http.HttpMethods;
-import com.google.common.base.Strings;
-import com.google.gson.Gson;
-import org.opengroup.osdu.core.common.http.FetchServiceHttpRequest;
-import org.opengroup.osdu.core.common.http.IUrlFetchService;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.http.HttpResponse;
-import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.opengroup.osdu.indexer.model.SearchRequest;
-import org.opengroup.osdu.indexer.model.SearchResponse;
-import org.springframework.stereotype.Component;
-
-import jakarta.inject.Inject;
-import java.net.URISyntaxException;
-
-@Component
-public class SearchServiceImpl implements SearchService {
- private static final String ERROR_MESSAGE = "Search service: failed to call the search service.";
- private static final String QUERY_PATH = "query";
- private static final String QUERY_WITH_CURSOR_PATH = "query_with_cursor";
- private final Gson gson = new Gson();
- private static final int OK_CODE = 200;
-
- @Inject
- private IUrlFetchService urlFetchService;
- @Inject
- private IRequestInfo requestInfo;
- @Inject
- private IndexerConfigurationProperties configurationProperties;
- @Inject
- private JaxRsDpsLog jaxRsDpsLog;
-
- @Override
- public SearchResponse query(SearchRequest searchRequest) throws URISyntaxException {
- return searchRecords(searchRequest, QUERY_PATH);
- }
-
- @Override
- public SearchResponse queryWithCursor(SearchRequest searchRequest) throws URISyntaxException {
- return searchRecords(searchRequest, QUERY_WITH_CURSOR_PATH);
- }
-
- private SearchResponse searchRecords(SearchRequest searchRequest, String path) throws URISyntaxException {
- if(Strings.isNullOrEmpty(configurationProperties.getSearchHost())) {
- jaxRsDpsLog.error("SEARCH_HOST", "The environment variable SEARCH_HOST is not setup");
- return new SearchResponse();
- }
-
- try {
- String body = this.gson.toJson(searchRequest);
- String url = String.format("%s/%s", configurationProperties.getSearchHost(), path);
- FetchServiceHttpRequest request = FetchServiceHttpRequest.builder()
- .httpMethod(HttpMethods.POST)
- .url(url)
- .headers(this.requestInfo.getHeaders())
- .body(body)
- .build();
- HttpResponse response = this.urlFetchService.sendRequest(request);
-
- if (response != null && response.getResponseCode() == OK_CODE) {
- return gson.fromJson(response.getBody(), SearchResponse.class);
- } else {
- if (response != null)
- jaxRsDpsLog.error(ERROR_MESSAGE + String.format(" responseCode = %d", response.getResponseCode()));
- else
- jaxRsDpsLog.error(String.format(ERROR_MESSAGE + " The response is null."));
- return new SearchResponse();
- }
- }
- catch(URISyntaxException ex) {
- throw ex;
- }
- catch(Exception ex) {
- jaxRsDpsLog.error(ERROR_MESSAGE, ex);
- throw new URISyntaxException(ex.getMessage(), "Unexpected exception type: " + ex.getClass().getName());
- }
- }
-}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/exception/ElasticsearchMappingException.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/exception/ElasticsearchMappingException.java
index e3309d913f24f84ac114d51b2de4dc8e6bb0c6a4..da862793812c5bc9c6bfa42a1a504f87b6255d14 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/exception/ElasticsearchMappingException.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/exception/ElasticsearchMappingException.java
@@ -22,7 +22,7 @@ public class ElasticsearchMappingException extends RuntimeException {
private Integer status;
public ElasticsearchMappingException(final String message, Integer status) {
super(message);
- this.status = status;
+ this.status = status != null ? status : 500;
}
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/AugmenterSetting.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/AugmenterSetting.java
index 9e4a7820bc07953914e700ac5c4986805eea29ae..5ac7b0e1851218ef1ecb1475b9f14062064ced84 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/AugmenterSetting.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/AugmenterSetting.java
@@ -26,6 +26,6 @@ public class AugmenterSetting {
private BooleanFeatureFlagClient booleanFeatureFlagClient;
public boolean isEnabled() {
- return booleanFeatureFlagClient.isEnabled(PROPERTY_NAME, false);
+ return booleanFeatureFlagClient.isEnabled(PROPERTY_NAME, true);
}
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/BooleanFeatureFlagClient.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/BooleanFeatureFlagClient.java
index 156c9f39edede7eabf8a9de2f3b2ecc30dc7f590..26cfef881bfac9ccd455dba9d607493ef6f0f15d 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/BooleanFeatureFlagClient.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/BooleanFeatureFlagClient.java
@@ -43,16 +43,18 @@ public class BooleanFeatureFlagClient {
private IServiceAccountJwtClient tokenService;
public boolean isEnabled(String featureName, boolean defaultValue) {
- String dataPartitionId = headers.getPartitionId();
- if (cache != null && cache.get(featureName) != null)
- return cache.get(featureName);
+ Boolean isEnabled = this.cache.get(featureName);
+ if (isEnabled != null)
+ return isEnabled;
- boolean isEnabled = defaultValue;
+ String dataPartitionId = headers.getPartitionId();
try {
PartitionInfo partitionInfo = getPartitionInfo(dataPartitionId);
isEnabled = getFeatureValue(partitionInfo, featureName, defaultValue);
+ this.logger.info(String.format("BooleanFeatureFlagClient: The feature flag '%s' in data partition '%s' is set to %s", featureName, dataPartitionId, isEnabled));
} catch (Exception e) {
- this.logger.error(String.format("PartitionService: Error getting %s for dataPartition with Id: %s. Turn on the feature flag by default.", featureName, dataPartitionId), e);
+ isEnabled = defaultValue;
+ this.logger.error(String.format("BooleanFeatureFlagClient: Error on getting the feature flag '%s' for data partition '%s'. Using default value %s.", featureName, dataPartitionId, isEnabled), e);
}
this.cache.put(featureName, isEnabled);
return isEnabled;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SearchResponse.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/CustomIndexAnalyzerSetting.java
similarity index 55%
rename from indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SearchResponse.java
rename to indexer-core/src/main/java/org/opengroup/osdu/indexer/util/CustomIndexAnalyzerSetting.java
index 61136de58ce26dc958780dc1574dd6718cde4df0..d0ab0b682dc61a7e224e4bd7018b3c51c090b2d0 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/SearchResponse.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/CustomIndexAnalyzerSetting.java
@@ -13,17 +13,19 @@
* limitations under the License.
*/
-package org.opengroup.osdu.indexer.model;
+package org.opengroup.osdu.indexer.util;
-import lombok.Data;
-import lombok.ToString;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
-import java.util.List;
+@Component
+public class CustomIndexAnalyzerSetting {
+ private static final String PROPERTY_NAME = "custom-index-analyzer-enabled";
-@Data
-@ToString
-public class SearchResponse {
- private String cursor;
- private List<SearchRecord> results;
- private int totalCount;
+ @Autowired
+ private BooleanFeatureFlagClient booleanFeatureFlagClient;
+
+ public boolean isEnabled() {
+ return booleanFeatureFlagClient.isEnabled(PROPERTY_NAME, false);
+ }
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/ElasticClientHandler.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/ElasticClientHandler.java
index 9c4d2bf11993c2ba54a07361cf5de1bb9b08782c..6a577eaae9f5e5ad3534e857f4fd1e046ddede02 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/ElasticClientHandler.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/ElasticClientHandler.java
@@ -8,13 +8,18 @@ import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.util.Base64;
import java.util.Objects;
+import java.util.concurrent.TimeUnit;
import javax.net.ssl.SSLContext;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
import lombok.extern.java.Log;
import org.apache.http.Header;
import org.apache.http.HttpHost;
import org.apache.http.HttpStatus;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
+import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
import org.apache.http.message.BasicHeader;
import org.apache.http.ssl.SSLContextBuilder;
import org.elasticsearch.client.RestClient;
@@ -37,6 +42,7 @@ public class ElasticClientHandler {
private static final int REST_CLIENT_CONNECT_TIMEOUT = 60000;
private static final int REST_CLIENT_SOCKET_TIMEOUT = 60000;
private static final int REST_CLIENT_RETRY_TIMEOUT = 60000;
+ private static final int REST_CLIENT_CONNECTION_TTL_SECONDS = 60;
@Value("#{new Boolean('${security.https.certificate.trust:false}')}")
private Boolean isSecurityHttpsCertificateTrust;
@@ -89,7 +95,9 @@ public class ElasticClientHandler {
RestClientBuilder builder = createClientBuilder(host, basicAuthenticationHeaderVal, port,
protocolScheme, tls);
- RestClientTransport transport = new RestClientTransport(builder.build(), new JacksonJsonpMapper());
+ ObjectMapper objectMapper = new ObjectMapper();
+ objectMapper.configure(SerializationFeature.INDENT_OUTPUT, false);
+ RestClientTransport transport = new RestClientTransport(builder.build(), new JacksonJsonpMapper(objectMapper));
return new ElasticsearchClient(transport);
} catch (AppException e) {
@@ -126,13 +134,16 @@ public class ElasticClientHandler {
"Elastic client connection uses protocolScheme = %s with a flag "
+ "'security.https.certificate.trust' = %s",
protocolScheme, isSecurityHttpsCertificateTrust));
+ HttpAsyncClientBuilder httpAsyncClientBuilder = HttpAsyncClientBuilder.create();
+ httpAsyncClientBuilder.setConnectionTimeToLive(REST_CLIENT_CONNECTION_TTL_SECONDS, TimeUnit.SECONDS);
if ("https".equals(protocolScheme) && isSecurityHttpsCertificateTrust) {
log.fine( "Elastic client connection uses TrustSelfSignedStrategy()");
SSLContext sslContext = createSSLContext();
- builder.setHttpClientConfigCallback(httpClientBuilder ->
- httpClientBuilder.setSSLContext(sslContext)
- .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE));
+ httpAsyncClientBuilder
+ .setSSLContext(sslContext)
+ .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE);
}
+ builder.setHttpClientConfigCallback(httpClientBuilder -> httpAsyncClientBuilder);
builder.setDefaultHeaders(defaultHeaders);
return builder;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/PropertyUtil.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/PropertyUtil.java
index 5b7b10c4f33d74678a469ff3635ce1a54d35f382..0e40891cecdae63a38f4298f5730e341662f4e87 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/PropertyUtil.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/PropertyUtil.java
@@ -216,6 +216,13 @@ public class PropertyUtil {
if(rightMap == null) {
rightMap = new HashMap<>();
}
+ // If leftMap does not have property A and rightMap has a property A with null value
+ // Maps.difference will consider that leftMap and rightMap are different.
+ // In our case, they are the same. The properties with null value should be removed
+ // in the cloned map objects
+ leftMap = removeNullValues(leftMap);
+ rightMap = removeNullValues(rightMap);
+
MapDifference<String, Object> difference = Maps.difference(leftMap, rightMap);
if(difference.areEqual()) {
return new ArrayList<>();
@@ -235,7 +242,7 @@ public class PropertyUtil {
Object left = valueDifference.leftValue();
Object right = valueDifference.rightValue();
if(left == null && right == null) {
- continue;
+ //Same
}
else if(left == null || right == null) {
changedProperties.add(entry.getKey());
@@ -289,4 +296,11 @@ public class PropertyUtil {
return new ArrayList<>(changedProperties);
}
+
+ private static Map<String, Object> removeNullValues(Map<String, Object> dataMap) {
+ // The original object should not be changed
+ dataMap = new HashMap<>(dataMap);
+ dataMap.entrySet().removeIf(entry -> entry.getValue() == null);
+ return dataMap;
+ }
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/QueryUtil.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/QueryUtil.java
new file mode 100644
index 0000000000000000000000000000000000000000..ce7bbdeae86ee2722e848958ee2f8713a24a4945
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/QueryUtil.java
@@ -0,0 +1,140 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.util;
+
+import co.elastic.clients.elasticsearch._types.SortOptions;
+import co.elastic.clients.elasticsearch._types.SortOrder;
+import co.elastic.clients.elasticsearch._types.mapping.FieldType;
+import co.elastic.clients.elasticsearch._types.query_dsl.*;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class QueryUtil {
+ private static final String CONFIGURATIONS_PATH = "data.Configurations";
+ private static final String CONFIGURATIONS_PATHS_PATH = "data.Configurations.Paths";
+ private static final String RELATIONSHIP_DIRECTION_FIELD = "data.Configurations.Paths.RelatedObjectsSpec.RelationshipDirection";
+ private static final String RELATED_OBJECT_KIND_FIELD = "data.Configurations.Paths.RelatedObjectsSpec.RelatedObjectKind";
+
+ public static Query createQueryForParentConfigs(String childKind) {
+ String queryString =
+ String.format("%s:ParentToChildren AND %s:\"%s\"", RELATIONSHIP_DIRECTION_FIELD, RELATED_OBJECT_KIND_FIELD, childKind);
+ return createQueryForRelatedConfig(queryString);
+ }
+
+ public static Query createQueryForChildrenConfigs(String parentKind) {
+ String queryString =
+ String.format("%s:ChildToParent AND %s:\"%s\"", RELATIONSHIP_DIRECTION_FIELD, RELATED_OBJECT_KIND_FIELD, parentKind);
+ return createQueryForRelatedConfig(queryString);
+ }
+
+ public static Query createQueryForConfigurations(String kind) {
+ Query singleQuery = createSimpleTextQuery(String.format("data.Code:\"%s\"", kind));
+ String nestedQueryString = String.format("%s:\"%s\"", RELATED_OBJECT_KIND_FIELD, kind);
+ Query nestedQuery = createQueryForRelatedConfig(nestedQueryString);
+
+ BoolQuery.Builder boolQueryBuilder = QueryBuilders.bool();
+ boolQueryBuilder.boost(1.0F);
+ boolQueryBuilder.should(singleQuery);
+ boolQueryBuilder.should(nestedQuery);
+ return new Query.Builder().bool(boolQueryBuilder.build()).build();
+ }
+
+ public static Query createSimpleTextQuery(String queryString) {
+ QueryStringQuery.Builder queryStringQueryBuilder = getQueryStringQueryBuilder(queryString);
+ Query.Builder queryBuilder = (Query.Builder) new Query.Builder().queryString(queryStringQueryBuilder.build());
+ return queryBuilder.build();
+ }
+
+ public static String createIdsFilter(List<String> ids) {
+ StringBuilder idsBuilder = new StringBuilder();
+ for (String id : ids) {
+ if (!idsBuilder.isEmpty()) {
+ idsBuilder.append(" OR ");
+ }
+ idsBuilder.append("\"");
+ idsBuilder.append(PropertyUtil.removeIdPostfix(id));
+ idsBuilder.append("\"");
+ }
+ return idsBuilder.toString();
+ }
+
+ public static List<SortOptions> createSortOptionsList(List<String> fields, List<SortOrder> orders) throws Exception {
+ if(fields == null || fields.isEmpty() || orders == null || orders.isEmpty()) {
+ return null;
+ }
+ if(fields.size() != orders.size()) {
+ throw new Exception("The size of the fields is not matched to the size of the orders");
+ }
+
+ List<SortOptions> sortOptionsList = new ArrayList<>();
+ for(int i = 0; i < fields.size(); i++) {
+ // If field is a text property under data block, make sure that the field is ended with ".keyword"
+ // e.g. "data.Code.keyword instead of "data.Code"
+ String field = fields.get(i);
+ SortOrder order = orders.get(i);
+ SortOptions sortOptions = new SortOptions.Builder()
+ .field(f ->f.field(field)
+ .order(order)
+ .missing("_last")
+ .unmappedType(FieldType.Keyword))
+ .build();
+ sortOptionsList.add(sortOptions);
+ }
+
+ return sortOptionsList;
+ }
+
+ private static QueryStringQuery.Builder getQueryStringQueryBuilder(String queryString) {
+ if(StringUtils.isBlank(queryString)) {
+ queryString = "*";
+ }
+ return new QueryStringQuery.Builder()
+ .query(queryString)
+ .fields(new ArrayList<>())
+ .type(TextQueryType.BestFields)
+ .defaultOperator(co.elastic.clients.elasticsearch._types.query_dsl.Operator.Or)
+ .maxDeterminizedStates(10000)
+ .allowLeadingWildcard(false)
+ .enablePositionIncrements(true)
+ .fuzziness("AUTO")
+ .fuzzyPrefixLength(0)
+ .fuzzyMaxExpansions(50)
+ .phraseSlop(0.0)
+ .escape(false)
+ .autoGenerateSynonymsPhraseQuery(true)
+ .fuzzyTranspositions(true)
+ .boost(1.0f);
+ }
+
+ private static Query createQueryForRelatedConfig(String queryString) {
+ Query simpleTextQuery = createSimpleTextQuery(queryString);
+ Query.Builder innerNestedBuilder = (Query.Builder) new Query.Builder().nested(
+ n ->n.path(CONFIGURATIONS_PATHS_PATH)
+ .query(simpleTextQuery)
+ .boost(1.0f)
+ .ignoreUnmapped(true)
+ .scoreMode(ChildScoreMode.Avg));
+ Query.Builder nestedBuilder = (Query.Builder) new Query.Builder().nested(
+ n ->n.path(CONFIGURATIONS_PATH)
+ .query(innerNestedBuilder.build())
+ .boost(1.0f)
+ .ignoreUnmapped(true)
+ .scoreMode(ChildScoreMode.Avg));
+ return nestedBuilder.build();
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/SearchClient.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/SearchClient.java
new file mode 100644
index 0000000000000000000000000000000000000000..09e3b6205e1aabd6c6afe179b27f3037de7ef85e
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/SearchClient.java
@@ -0,0 +1,250 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.util;
+
+import co.elastic.clients.elasticsearch.ElasticsearchClient;
+import co.elastic.clients.elasticsearch._types.ExpandWildcard;
+import co.elastic.clients.elasticsearch._types.FieldValue;
+import co.elastic.clients.elasticsearch._types.SearchType;
+import co.elastic.clients.elasticsearch._types.SortOptions;
+import co.elastic.clients.elasticsearch._types.SortOrder;
+import co.elastic.clients.elasticsearch._types.Time;
+import co.elastic.clients.elasticsearch._types.query_dsl.Query;
+import co.elastic.clients.elasticsearch.core.ClosePointInTimeRequest;
+import co.elastic.clients.elasticsearch.core.OpenPointInTimeRequest;
+import co.elastic.clients.elasticsearch.core.OpenPointInTimeResponse;
+import co.elastic.clients.elasticsearch.core.SearchRequest;
+import co.elastic.clients.elasticsearch.core.SearchResponse;
+import co.elastic.clients.elasticsearch.core.search.Hit;
+import co.elastic.clients.elasticsearch.core.search.HitsMetadata;
+import co.elastic.clients.elasticsearch.core.search.ResponseBody;
+import co.elastic.clients.elasticsearch.core.search.SourceConfig;
+import com.google.gson.Gson;
+import jakarta.inject.Inject;
+import org.apache.commons.lang3.StringUtils;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
+import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
+import org.opengroup.osdu.indexer.model.SearchRecord;
+import org.springframework.stereotype.Component;
+import java.io.IOException;
+import java.lang.reflect.Type;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+@Component
+public class SearchClient {
+
+ @Inject
+ private ElasticClientHandler elasticClientHandler;
+
+ @Inject
+ private ElasticIndexNameResolver elasticIndexNameResolver;
+
+ @Inject
+ private JaxRsDpsLog logger;
+
+ private final static int MAX_PAGE_SIZE = 5000; //5k
+ private final static int MAX_RECORDS_COUNT = Integer.MAX_VALUE;
+ private final Time SEARCH_TIMEOUT = Time.of(t -> t.time("120s"));
+
+ // if returnedField contains property matching from excludes than query result will NOT include that property
+ private final Set<String> excludes =
+ new HashSet<>(Collections.singletonList(RecordMetaAttribute.X_ACL.getValue()));
+
+ // queryableExcludes properties can be returned by query results
+ private final Set<String> queryableExcludes =
+ new HashSet<>(Collections.singletonList(RecordMetaAttribute.INDEX_STATUS.getValue()));
+
+ private final Gson gson = new Gson();
+
+ public List<SearchRecord> search(String kind, Query query, List<SortOptions> sortOptions, List<String> returnedFields, int limit) throws Exception {
+ if(StringUtils.isEmpty(kind)) {
+ throw new Exception("kind can't be null or empty");
+ }
+ return search(List.of(kind), query, sortOptions, returnedFields, limit);
+ }
+
+ public List<SearchRecord> search(List<String> kinds, Query query, List<SortOptions> sortOptions, List<String> returnedFields, int limit) throws Exception {
+ if(kinds == null || kinds.isEmpty()) {
+ throw new Exception("kinds can't be null or empty");
+ }
+ if(query == null) {
+ throw new Exception("query can't be null");
+ }
+
+ ElasticsearchClient client = this.elasticClientHandler.getOrCreateRestClient();
+ String index = this.getIndex(kinds);
+ limit = (limit <= 0)? MAX_RECORDS_COUNT : limit;
+ int pageSize = (limit > MAX_PAGE_SIZE)? MAX_PAGE_SIZE: limit;
+
+ // Use normal query without pagination
+ List<SearchRecord> records = query(client, index, query, sortOptions, returnedFields, pageSize);
+ if(records.size() < MAX_PAGE_SIZE || records.size() >= limit) {
+ return records;
+ }
+
+ // Use search_after and PIT to do pagination if the number of records is larger than one page size
+ String pitId = null;
+ try {
+ pitId = openPointInTime(client, index);
+ return queryWithPit(client, pitId, query, sortOptions, returnedFields, pageSize, limit);
+ }
+ finally {
+ if(pitId != null) {
+ closePointInTime(client, pitId);
+ }
+ }
+ }
+
+ private List<SearchRecord> query(ElasticsearchClient client, String index, Query query, List<SortOptions> sortOptions, List<String> returnedFields, int pageSize) throws Exception {
+ // Build the SearchRequest
+ SearchRequest.Builder queryBuilder = createSearchBuilder(query, sortOptions, returnedFields, pageSize);
+ SearchRequest elasticSearchRequest = queryBuilder
+ .index(index)
+ .from(0)
+ .allowNoIndices(true)
+ .expandWildcards(ExpandWildcard.Open, ExpandWildcard.Closed)
+ .ignoreUnavailable(true)
+ .ignoreThrottled(true)
+ .ccsMinimizeRoundtrips(true)
+ .build();
+
+ // Execute
+ SearchResponse<Map<String, Object>> searchResponse = client.search(elasticSearchRequest, (Type) Map.class);
+
+ // Convert SearchResponse
+ List<SearchRecord> results = getSearchRecords(searchResponse);
+
+ return results;
+ }
+
+ private List<SearchRecord> queryWithPit(ElasticsearchClient client, String pitId, Query query, List<SortOptions> sortOptions, List<String> returnedFields, int pageSize, int limit) throws Exception {
+ List<SearchRecord> results = new ArrayList<>();
+ List<FieldValue> fieldValues = null;
+ // SortOptions can't be null or empty in search with search_after and PIT
+ // Otherwise, the returned fieldValues will be empty and cause infinite loop
+ if(sortOptions == null || sortOptions.isEmpty()) {
+ sortOptions = new ArrayList<>();
+ sortOptions.add(SortOptions.of(so -> so.score(s -> s.order(SortOrder.Desc))));
+ }
+ while(true) {
+ // Build the SearchRequest
+ SearchRequest.Builder queryBuilder = createSearchBuilder(query, sortOptions, returnedFields, pageSize);
+ if(fieldValues != null) {
+ queryBuilder.searchAfter(fieldValues);
+ }
+ SearchRequest elasticSearchRequest = queryBuilder
+ .pit(pit -> pit.id(pitId).keepAlive(SEARCH_TIMEOUT))
+ .build();
+
+ // Execute
+ SearchResponse<Map<String, Object>> searchResponse = client.search(elasticSearchRequest, (Type) Map.class);
+
+ // Convert SearchResponse
+ List<SearchRecord> batch = getSearchRecords(searchResponse);
+ results.addAll(batch);
+
+ if(batch.size() < pageSize || results.size() >= limit) {
+ break; // Done
+ }
+ else {
+ // Prepare for next page
+ HitsMetadata<Map<String, Object>> searchHits = searchResponse.hits();
+ int length = searchHits.hits().size();
+ fieldValues = searchHits.hits().get(length -1).sort();
+ if(results.size() + pageSize > limit) {
+ pageSize = limit - results.size();
+ }
+ }
+ }
+
+ return results;
+ }
+
+ private SearchRequest.Builder createSearchBuilder(Query query, List<SortOptions> sortOptions, List<String> returnedFields, int pageSize) {
+ SearchRequest.Builder searchBuilder = new SearchRequest.Builder()
+ .query(query)
+ .size(pageSize)
+ .searchType(SearchType.QueryThenFetch)
+ .batchedReduceSize(512L);
+ if(sortOptions != null) {
+ searchBuilder.sort(sortOptions);
+ }
+
+ // set the return fields
+ if (returnedFields == null) {
+ returnedFields = new ArrayList<>();
+ }
+ Set<String> returnedFieldsSet = new HashSet<>(returnedFields);
+ // remove all matching returnedField and queryable from excludes
+ Set<String> requestQueryableExcludes = new HashSet<>(queryableExcludes);
+ Set<String> requestExcludes = new HashSet<>(excludes);
+ requestQueryableExcludes.removeAll(returnedFields);
+ requestExcludes.addAll(requestQueryableExcludes);
+ searchBuilder.source(SourceConfig.of(
+ sc ->sc.filter(
+ f ->f.includes(returnedFieldsSet.stream().toList())
+ .excludes(requestExcludes.stream().toList()))));
+ return searchBuilder;
+ }
+
+ private List<SearchRecord> getSearchRecords(
+ ResponseBody<Map<String, Object>> searchResponse) {
+ List<SearchRecord> results = new ArrayList<>();
+ HitsMetadata<Map<String, Object>> searchHits = searchResponse.hits();
+ if (searchHits.hits() != null) {
+ for (Hit<Map<String, Object>> hit : searchHits.hits()) {
+ Map<String, Object> hitFields = hit.source();
+ results.add(gson.fromJson(gson.toJson(hitFields), SearchRecord.class));
+ }
+ }
+ return results;
+ }
+
+ private String getIndex(List<String> kinds) {
+ List<String> indices = new ArrayList<>();
+ for(String kind : kinds) {
+ String index = this.elasticIndexNameResolver.getIndexNameFromKind(kind);
+ indices.add(index);
+ }
+ return String.join(",", indices);
+ }
+
+ private String openPointInTime(ElasticsearchClient client, String index) throws IOException {
+ OpenPointInTimeRequest openRequest = OpenPointInTimeRequest.of(builder ->
+ builder.index(index)
+ .ignoreUnavailable(true)
+ .keepAlive(SEARCH_TIMEOUT));
+ OpenPointInTimeResponse openResponse = client.openPointInTime(openRequest);
+ return openResponse.id();
+ }
+
+ private void closePointInTime(ElasticsearchClient client, String pitId) {
+ ClosePointInTimeRequest closeRequest = ClosePointInTimeRequest.of(builder -> builder.id(pitId));
+ try {
+ client.closePointInTime(closeRequest);
+ }
+ catch(Exception ex) {
+ logger.warning("Failed to close point in time", ex);
+ }
+ }
+
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java
index 7ece8707021eea6743bc1a54445fa1366175b8d6..9bc32afdd89ecc144fa4b9be1fc7876d2556ae31 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java
@@ -14,8 +14,6 @@
package org.opengroup.osdu.indexer.util;
-import static org.opengroup.osdu.indexer.model.XcollaborationHolder.X_COLLABORATION;
-
import com.google.gson.Gson;
import com.google.gson.JsonElement;
import com.google.gson.JsonParser;
@@ -46,6 +44,8 @@ public class TypeMapper {
private static final String BAG_OF_WORDS = "bagOfWords";
+ public static final String OSDU_CUSTOM_ANALYZER = "osdu_custom_analyzer";
+
static {
metaAttributeIndexerType.put(RecordMetaAttribute.KIND.getValue(), ElasticType.KEYWORD.getValue());
@@ -122,9 +122,9 @@ public class TypeMapper {
return Records.Type.builder().type(metaAttributeIndexerType.get(key).toString()).build();
}
- public static Object getDataAttributeIndexerMapping(Object indexerType, Boolean keywordLowerEnabled, Boolean bagOfWordsEnabled) {
+ public static Object getDataAttributeIndexerMapping(Object indexerType, Boolean keywordLowerEnabled, Boolean bagOfWordsEnabled, Boolean customIndexAnalyzerEnabled) {
if (ElasticType.TEXT.getValue().equalsIgnoreCase(indexerType.toString())) {
- return getTextIndexerMapping(keywordLowerEnabled, bagOfWordsEnabled);
+ return getTextIndexerMapping(keywordLowerEnabled, bagOfWordsEnabled, customIndexAnalyzerEnabled);
}
if (ElasticType.KEYWORD.getValue().equalsIgnoreCase(indexerType.toString())) {
@@ -134,7 +134,7 @@ public class TypeMapper {
if (isArray(indexerType.toString())) {
String memberType = getArrayMemberType(indexerType.toString());
if (ElasticType.TEXT.getValue().equalsIgnoreCase(memberType)) {
- return getTextIndexerMapping(keywordLowerEnabled, bagOfWordsEnabled);
+ return getTextIndexerMapping(keywordLowerEnabled, bagOfWordsEnabled, customIndexAnalyzerEnabled);
}
if (ElasticType.KEYWORD.getValue().equalsIgnoreCase(memberType)) {
return getKeywordIndexerMapping(bagOfWordsEnabled);
@@ -147,9 +147,9 @@ public class TypeMapper {
Map<String, Object> propertiesMap = (Map<String, Object>) type.get(Constants.PROPERTIES);
for (Map.Entry<String, Object> entry : propertiesMap.entrySet()) {
if (isMap(entry.getValue())) {
- entry.setValue(getDataAttributeIndexerMapping(entry.getValue(), keywordLowerEnabled, bagOfWordsEnabled));
+ entry.setValue(getDataAttributeIndexerMapping(entry.getValue(), keywordLowerEnabled, bagOfWordsEnabled, customIndexAnalyzerEnabled));
} else if (ElasticType.TEXT.getValue().equalsIgnoreCase(String.valueOf(entry.getValue()))) {
- entry.setValue(getTextIndexerMapping(keywordLowerEnabled, bagOfWordsEnabled));
+ entry.setValue(getTextIndexerMapping(keywordLowerEnabled, bagOfWordsEnabled, customIndexAnalyzerEnabled));
} else if (ElasticType.KEYWORD.getValue().equalsIgnoreCase(String.valueOf(entry.getValue()))) {
entry.setValue(getKeywordIndexerMapping(bagOfWordsEnabled));
} else if (isArray(String.valueOf(entry.getValue()))) {
@@ -228,7 +228,7 @@ public class TypeMapper {
return indexStatusProperties;
}
- private static Object getTextIndexerMapping(Boolean keywordLowerEnabled, Boolean bagOfWordsEnabled) {
+ private static Object getTextIndexerMapping(Boolean keywordLowerEnabled, Boolean bagOfWordsEnabled, Boolean customIndexAnalyzerEnabled) {
Map<String, Object> keywordMap = getKeywordMap();
Map<String, Object> keywordLowerMap = getKeywordLowerMap();
Map<String, Object> fieldIndexTypeMap = new HashMap<>();
@@ -242,6 +242,9 @@ public class TypeMapper {
if (bagOfWordsEnabled) {
textMap.put("copy_to", BAG_OF_WORDS);
}
+ if(customIndexAnalyzerEnabled) {
+ textMap.put("analyzer", OSDU_CUSTOM_ANALYZER);
+ }
return textMap;
}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/HealthCheckApiTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/HealthCheckApiTest.java
index 1e13f0ed51854a9cfb5b88d2875079f020851ddf..556997b96eb112489222bd4e032861eb4caa8748 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/HealthCheckApiTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/HealthCheckApiTest.java
@@ -43,4 +43,4 @@ public class HealthCheckApiTest {
public void should_returnHttp200_when_checkReadiness() {
assertEquals(HttpStatus.OK.value(), this.sut.readinessCheck().getStatusCodeValue());
}
-}
\ No newline at end of file
+}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/RecordIndexerApiTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/RecordIndexerApiTest.java
index 81908d1fa2c63d7ccfdec4dda3de52b75df9ae21..3eb36589185fd074d24d04faf1557ac83b0ad234 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/RecordIndexerApiTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/RecordIndexerApiTest.java
@@ -16,8 +16,9 @@ package org.opengroup.osdu.indexer.api;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
-import static org.mockito.MockitoAnnotations.initMocks;
import static org.mockito.Mockito.when;
+import static org.mockito.MockitoAnnotations.initMocks;
+
import com.google.gson.Gson;
import org.junit.Before;
import org.junit.Test;
@@ -25,7 +26,7 @@ import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
-import org.opengroup.osdu.core.common.http.HeadersUtil;
+import org.mockito.junit.MockitoJUnitRunner;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
@@ -34,9 +35,6 @@ import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
import org.opengroup.osdu.indexer.service.IndexerService;
import org.opengroup.osdu.indexer.service.SchemaEventsProcessor;
-import org.opengroup.osdu.indexer.service.SchemaService;
-import org.mockito.junit.MockitoJUnitRunner;
-import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/AugmenterConfigurationServiceImplTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/AugmenterConfigurationServiceImplTest.java
index ec3bb8c8996e715faee4fca8a9b6266e7d37d17f..2bb95bf016ad1be799569342d790400ea120d5c7 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/AugmenterConfigurationServiceImplTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/AugmenterConfigurationServiceImplTest.java
@@ -16,13 +16,34 @@
package org.opengroup.osdu.indexer.service;
+import static org.mockito.ArgumentMatchers.*;
+import static org.mockito.Mockito.anyString;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import co.elastic.clients.elasticsearch._types.query_dsl.Query;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Strings;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.UnsupportedEncodingException;
+import java.lang.reflect.Type;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
import lombok.SneakyThrows;
+import org.apache.http.HttpStatus;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.ArgumentCaptor;
@@ -30,6 +51,8 @@ import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.indexer.IndexingStatus;
+import org.opengroup.osdu.core.common.model.indexer.JobStatus;
import org.opengroup.osdu.core.common.model.indexer.OperationType;
import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
@@ -37,25 +60,25 @@ import org.opengroup.osdu.core.common.model.storage.RecordData;
import org.opengroup.osdu.core.common.model.storage.Schema;
import org.opengroup.osdu.core.common.model.storage.SchemaItem;
import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
-import org.opengroup.osdu.indexer.cache.partitionsafe.*;
+import org.opengroup.osdu.indexer.cache.partitionsafe.AugmenterConfigurationCache;
+import org.opengroup.osdu.indexer.cache.partitionsafe.AugmenterConfigurationEnabledCache;
+import org.opengroup.osdu.indexer.cache.partitionsafe.ChildRelationshipSpecsCache;
+import org.opengroup.osdu.indexer.cache.partitionsafe.ChildrenKindsCache;
+import org.opengroup.osdu.indexer.cache.partitionsafe.KindCache;
+import org.opengroup.osdu.indexer.cache.partitionsafe.RecordChangeInfoCache;
+import org.opengroup.osdu.indexer.cache.partitionsafe.RelatedObjectCache;
import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.opengroup.osdu.indexer.model.*;
+import org.opengroup.osdu.indexer.model.Constants;
+import org.opengroup.osdu.indexer.model.RecordChangeInfo;
+import org.opengroup.osdu.indexer.model.SchemaIdentity;
+import org.opengroup.osdu.indexer.model.SchemaInfo;
+import org.opengroup.osdu.indexer.model.SchemaInfoResponse;
+import org.opengroup.osdu.indexer.model.SearchRecord;
import org.opengroup.osdu.indexer.model.indexproperty.AugmenterConfiguration;
import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
+import org.opengroup.osdu.indexer.util.SearchClient;
import org.springframework.test.context.junit4.SpringRunner;
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.UnsupportedEncodingException;
-import java.lang.reflect.Type;
-import java.net.URISyntaxException;
-import java.util.*;
-
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.Mockito.*;
-
@RunWith(SpringRunner.class)
public class AugmenterConfigurationServiceImplTest {
private final Gson gson = new Gson();
@@ -80,7 +103,7 @@ public class AugmenterConfigurationServiceImplTest {
@Mock
private RecordChangeInfoCache recordChangeInfoCache;
@Mock
- private SearchService searchService;
+ private SearchClient searchClient;
@Mock
private SchemaService schemaService;
@Mock
@@ -89,6 +112,8 @@ public class AugmenterConfigurationServiceImplTest {
private IRequestInfo requestInfo;
@Mock
private JaxRsDpsLog jaxRsDpsLog;
+ @Mock
+ private JobStatus jobStatus;
private final String augmenterConfigurationKind = "osdu:wks:reference-data--IndexPropertyPathConfiguration:*";
private String childKind;
@@ -96,6 +121,11 @@ public class AugmenterConfigurationServiceImplTest {
private String parentKind;
private String parentId;
+ @Before
+ public void setup() throws Exception {
+ this.sut.maxSizeOfExtendedListValue = 1000;
+ }
+
@Test
public void isAugmenterConfigurationEnabled_invalid_kind() {
Assert.assertFalse(sut.isConfigurationEnabled(null));
@@ -120,22 +150,19 @@ public class AugmenterConfigurationServiceImplTest {
}
@Test
- public void isAugmenterConfigurationEnabled_with_result_from_search() throws URISyntaxException {
+ public void isAugmenterConfigurationEnabled_with_result_from_search() throws Exception {
String kind = "anyAuth:anySource:anyEntity:1.";
- SearchResponse response = new SearchResponse();
- response.setResults(Arrays.asList(new SearchRecord()));
when(this.augmenterConfigurationEnabledCache.get(any())).thenReturn(null);
- when(this.searchService.query(any())).thenReturn(response);
+ when(this.searchClient.search(anyString(), any(), any(), any(), anyInt())).thenReturn(List.of(new SearchRecord()));
Assert.assertTrue(sut.isConfigurationEnabled(kind));
verify(this.augmenterConfigurationEnabledCache, times(1)).put(any(), any());
}
@Test
- public void isAugmenterConfigurationEnabled_without_result_from_search() throws URISyntaxException {
+ public void isAugmenterConfigurationEnabled_without_result_from_search() throws Exception {
String kind = "anyAuth:anySource:anyEntity:1.";
- SearchResponse response = new SearchResponse();
when(this.augmenterConfigurationEnabledCache.get(any())).thenReturn(null);
- when(this.searchService.query(any())).thenReturn(response);
+ when(this.searchClient.search(anyString(), any(), any(), any(), anyInt())).thenReturn(new ArrayList<>());
Assert.assertFalse(sut.isConfigurationEnabled(kind));
verify(this.augmenterConfigurationEnabledCache, times(1)).put(any(), any());
}
@@ -172,15 +199,12 @@ public class AugmenterConfigurationServiceImplTest {
}
@Test
- public void getAugmenterConfiguration_with_result_from_search() throws URISyntaxException {
+ public void getAugmenterConfiguration_with_result_from_search() throws Exception {
Map<String, Object> data = this.getDataMap("well_configuration_record.json");
SearchRecord searchRecord = new SearchRecord();
searchRecord.setData(data);
List<SearchRecord> results = Arrays.asList(searchRecord);
- SearchResponse searchResponse = new SearchResponse();
- searchResponse.setResults(results);
- searchResponse.setTotalCount(results.size());
- when(this.searchService.query(any())).thenReturn(searchResponse);
+ when(this.searchClient.search(anyString(), any(), any(), any(), anyInt())).thenReturn(results);
String kind = "osdu:wks:master-data--Well:1.0.0";
String code = "osdu:wks:master-data--Well:1.";
AugmenterConfiguration configuration = sut.getConfiguration(kind);
@@ -194,8 +218,8 @@ public class AugmenterConfigurationServiceImplTest {
}
@Test
- public void getAugmenterConfiguration_without_result_from_search() throws URISyntaxException {
- when(this.searchService.query(any())).thenReturn(new SearchResponse());
+ public void getAugmenterConfiguration_without_result_from_search() throws Exception {
+ when(this.searchClient.search(anyString(), any(), any(), any(), anyInt())).thenReturn(new ArrayList<>());
String kind = "osdu:wks:master-data--Well:1.0.0";
AugmenterConfiguration configuration = sut.getConfiguration(kind);
@@ -207,30 +231,47 @@ public class AugmenterConfigurationServiceImplTest {
}
@Test
- public void getExtendedProperties_from_children_objects() throws JsonProcessingException, URISyntaxException {
+ public void getExtendedProperties_from_children_objects() throws Exception {
AugmenterConfiguration propertyConfigurations = getConfiguration("wellbore_configuration_record.json");
Map<String, Object> originalDataMap = getDataMap("wellbore_data.json");
String jsonText = getJsonFromFile("welllog_search_records.json");
Type type = new TypeToken<List<SearchRecord>>() {}.getType();
List<SearchRecord> childrenRecords = gson.fromJson(jsonText, type);
- SearchResponse response = new SearchResponse();
- response.setResults(childrenRecords);
- when(this.searchService.query(any())).thenReturn(response);
+ when(this.searchClient.search(anyString(), any(), any(), any(), anyInt())).thenReturn(childrenRecords);
Map<String, Object> extendedProperties = this.sut.getExtendedProperties("anyId", originalDataMap, propertyConfigurations);
Map<String, Object> expectedExtendedProperties = getDataMap("wellbore_extended_data.json");
verifyMap(expectedExtendedProperties, extendedProperties);
+ List<String> wellLogs = (List<String>)extendedProperties.getOrDefault("WellLogs", null);
+ Assert.assertNotNull(wellLogs);
+ Assert.assertEquals(88, wellLogs.size());
+ verify(jobStatus, times(0)).addOrUpdateRecordStatus(anyString(), eq(IndexingStatus.WARN), eq(HttpStatus.SC_BAD_REQUEST), anyString());
}
@Test
- public void getExtendedProperties_from_self_and_parent_objects() throws JsonProcessingException, URISyntaxException {
+ public void getExtendedProperties_from_children_objects_with_small_list_size() throws Exception {
+ // Set small threshold on the size of extended list value
+ this.sut.maxSizeOfExtendedListValue = 50;
+ AugmenterConfiguration propertyConfigurations = getConfiguration("wellbore_configuration_record.json");
+ Map<String, Object> originalDataMap = getDataMap("wellbore_data.json");
+ String jsonText = getJsonFromFile("welllog_search_records.json");
+ Type type = new TypeToken<List<SearchRecord>>() {}.getType();
+ List<SearchRecord> childrenRecords = gson.fromJson(jsonText, type);
+ when(this.searchClient.search(anyString(), any(), any(), any(), anyInt())).thenReturn(childrenRecords);
+
+ Map<String, Object> extendedProperties = this.sut.getExtendedProperties("anyId", originalDataMap, propertyConfigurations);
+ List<String> wellLogs = (List<String>)extendedProperties.getOrDefault("WellLogs", null);
+ // WellLogs property should be removed because of oversize
+ Assert.assertNull(wellLogs);
+ verify(jobStatus, times(1)).addOrUpdateRecordStatus(anyString(), eq(IndexingStatus.WARN), eq(HttpStatus.SC_BAD_REQUEST), anyString());
+ }
+
+ @Test
+ public void getExtendedProperties_from_self_and_parent_objects() throws Exception {
AugmenterConfiguration propertyConfigurations = getConfiguration("welllog_configuration_record.json");
Map<String, Object> originalDataMap = getDataMap("welllog_original_data.json");
- SearchResponse searchResponse = new SearchResponse();
List<SearchRecord> records = new ArrayList<>();
- searchResponse.setResults(records);
-
Map<String, Object> relatedObjectData = getDataMap("wellbore_data.json");
SearchRecord record = new SearchRecord();
record.setId("opendes:master-data--Wellbore:nz-100000113552");
@@ -249,13 +290,35 @@ public class AugmenterConfigurationServiceImplTest {
record.setData(relatedObjectData);
records.add(record);
- when(this.searchService.query(any())).thenReturn(searchResponse);
+ when(this.searchClient.search(any(List.class), any(), any(), any(), anyInt())).thenReturn(records);
Map<String, Object> extendedProperties = this.sut.getExtendedProperties("anyId", originalDataMap, propertyConfigurations);
Map<String, Object> expectedExtendedProperties = getDataMap("welllog_extended_data.json");
verifyMap(expectedExtendedProperties, extendedProperties);
}
+ @Test
+ public void getExtendedProperties_value_extract_match() throws Exception {
+ AugmenterConfiguration propertyConfigurations = getConfiguration("value_extraction_match_configuration_record.json");
+ Map<String, Object> originalDataMap = getDataMap("value_extraction_match_data_record.json");
+
+ List<SearchRecord> records = new ArrayList<>();
+ SearchRecord record = new SearchRecord();
+ record.setData(originalDataMap);
+ records.add(record);
+
+ when(this.searchClient.search(any(List.class), any(), any(), any(), anyInt())).thenReturn(records);
+
+ Map<String, Object> extendedProperties = this.sut.getExtendedProperties("anyId", originalDataMap, propertyConfigurations);
+ Double groundLevel = (Double)extendedProperties.getOrDefault("DCGroundlevel", null);
+ Assert.assertNotNull(groundLevel);
+ Assert.assertEquals(1143, groundLevel, 0.00001);
+
+ Double totalDepth = (Double)extendedProperties.getOrDefault("DCTotalDepth", null);
+ Assert.assertNotNull(totalDepth);
+ Assert.assertEquals(4977, totalDepth, 0.00001);
+ }
+
private void verifyMap(Map<String, Object> expectedExtendedProperties, Map<String, Object> extendedProperties) {
Assert.assertEquals(expectedExtendedProperties.size(), extendedProperties.size());
@@ -389,7 +452,7 @@ public class AugmenterConfigurationServiceImplTest {
}
@Test
- public void cacheDataRecord_create_record() throws URISyntaxException {
+ public void cacheDataRecord_create_record() throws Exception {
ArgumentCaptor<RecordChangeInfo> recordInfoArgumentCaptor = ArgumentCaptor.forClass(RecordChangeInfo.class);
ArgumentCaptor<RecordData> dataMapArgumentCaptor = ArgumentCaptor.forClass(RecordData.class);
String recordId = "anyId";
@@ -397,7 +460,7 @@ public class AugmenterConfigurationServiceImplTest {
Map<String, Object> dataMap = new HashMap<>();
dataMap.put("p1", "v1");
- when(this.searchService.query(any())).thenReturn(new SearchResponse());
+ when(this.searchClient.search(anyString(), any(), any(), any(), anyInt())).thenReturn(new ArrayList<>());
this.sut.cacheDataRecord(recordId, kind, dataMap);
@@ -409,7 +472,7 @@ public class AugmenterConfigurationServiceImplTest {
}
@Test
- public void cacheDataRecord_update_record() throws URISyntaxException {
+ public void cacheDataRecord_update_record() throws Exception {
ArgumentCaptor<RecordChangeInfo> recordInfoArgumentCaptor = ArgumentCaptor.forClass(RecordChangeInfo.class);
ArgumentCaptor<RecordData> dataMapArgumentCaptor = ArgumentCaptor.forClass(RecordData.class);
String recordId = "anyId";
@@ -424,10 +487,7 @@ public class AugmenterConfigurationServiceImplTest {
searchRecord.setKind(kind);
searchRecord.setId(recordId);
searchRecord.setData(previousDataMap);
- SearchResponse searchResponse = new SearchResponse();
- searchResponse.setResults(Arrays.asList(searchRecord));
- searchResponse.setTotalCount(1);
- when(this.searchService.query(any())).thenReturn(searchResponse);
+ when(this.searchClient.search(anyString(), any(), any(), any(), anyInt())).thenReturn(List.of(searchRecord));
this.sut.cacheDataRecord(recordId, kind, dataMap);
@@ -444,7 +504,7 @@ public class AugmenterConfigurationServiceImplTest {
}
@Test
- public void cacheDataRecord_update_record_merge_previous_UpdateChangedInfo() throws URISyntaxException {
+ public void cacheDataRecord_update_record_merge_previous_UpdateChangedInfo() throws Exception {
ArgumentCaptor<RecordChangeInfo> recordInfoArgumentCaptor = ArgumentCaptor.forClass(RecordChangeInfo.class);
ArgumentCaptor<RecordData> dataMapArgumentCaptor = ArgumentCaptor.forClass(RecordData.class);
String recordId = "anyId";
@@ -467,11 +527,8 @@ public class AugmenterConfigurationServiceImplTest {
searchRecord.setKind(kind);
searchRecord.setId(recordId);
searchRecord.setData(previousDataMap);
- SearchResponse searchResponse = new SearchResponse();
- searchResponse.setResults(Arrays.asList(searchRecord));
- searchResponse.setTotalCount(1);
- when(this.searchService.query(any())).thenReturn(searchResponse);
+ when(this.searchClient.search(anyString(), any(), any(), any(), anyInt())).thenReturn(List.of(searchRecord));
when(this.recordChangeInfoCache.get(any())).thenReturn(previousChangedInfo);
this.sut.cacheDataRecord(recordId, kind, dataMap);
@@ -489,7 +546,7 @@ public class AugmenterConfigurationServiceImplTest {
}
@Test
- public void cacheDataRecord_update_record_merge_previous_CreateChangedInfo() throws URISyntaxException {
+ public void cacheDataRecord_update_record_merge_previous_CreateChangedInfo() throws Exception {
ArgumentCaptor<RecordChangeInfo> recordInfoArgumentCaptor = ArgumentCaptor.forClass(RecordChangeInfo.class);
ArgumentCaptor<RecordData> dataMapArgumentCaptor = ArgumentCaptor.forClass(RecordData.class);
String recordId = "anyId";
@@ -511,11 +568,8 @@ public class AugmenterConfigurationServiceImplTest {
searchRecord.setKind(kind);
searchRecord.setId(recordId);
searchRecord.setData(previousDataMap);
- SearchResponse searchResponse = new SearchResponse();
- searchResponse.setResults(Arrays.asList(searchRecord));
- searchResponse.setTotalCount(1);
- when(this.searchService.query(any())).thenReturn(searchResponse);
+ when(this.searchClient.search(anyString(), any(), any(), any(), anyInt())).thenReturn(List.of(searchRecord));
when(this.recordChangeInfoCache.get(any())).thenReturn(previousChangedInfo);
this.sut.cacheDataRecord(recordId, kind, dataMap);
@@ -531,16 +585,16 @@ public class AugmenterConfigurationServiceImplTest {
}
@Test
- public void updateAssociatedRecords_updateAssociatedParentRecords_for_created_childRecord() throws URISyntaxException {
+ public void updateAssociatedRecords_updateAssociatedParentRecords_for_created_childRecord() throws Exception {
updateAssociatedRecords_updateAssociatedParentRecords_for_created_delete(OperationType.create);
}
@Test
- public void updateAssociatedRecords_updateAssociatedParentRecords_for_deleted_childRecord() throws URISyntaxException {
+ public void updateAssociatedRecords_updateAssociatedParentRecords_for_deleted_childRecord() throws Exception {
updateAssociatedRecords_updateAssociatedParentRecords_for_created_delete(OperationType.delete);
}
- private void updateAssociatedRecords_updateAssociatedParentRecords_for_created_delete(OperationType operationType) throws URISyntaxException {
+ private void updateAssociatedRecords_updateAssociatedParentRecords_for_created_delete(OperationType operationType) throws Exception {
updateAssociatedRecords_updateAssociatedParentRecords_baseSetup();
// Test
@@ -568,7 +622,7 @@ public class AugmenterConfigurationServiceImplTest {
}
@Test
- public void updateAssociatedRecords_updateAssociatedParentRecords_for_updated_childRecord_with_extendedPropertyChanged() throws URISyntaxException {
+ public void updateAssociatedRecords_updateAssociatedParentRecords_for_updated_childRecord_with_extendedPropertyChanged() throws Exception {
updateAssociatedRecords_updateAssociatedParentRecords_baseSetup();
RecordChangeInfo recordChangeInfo = new RecordChangeInfo();
@@ -602,7 +656,7 @@ public class AugmenterConfigurationServiceImplTest {
}
@Test
- public void updateAssociatedRecords_updateAssociatedParentRecords_for_updated_childRecord_without_extendedPropertyChanged() throws URISyntaxException {
+ public void updateAssociatedRecords_updateAssociatedParentRecords_for_updated_childRecord_without_extendedPropertyChanged() throws Exception {
updateAssociatedRecords_updateAssociatedParentRecords_baseSetup();
RecordChangeInfo recordChangeInfo = new RecordChangeInfo();
@@ -627,7 +681,7 @@ public class AugmenterConfigurationServiceImplTest {
}
@Test
- public void updateAssociatedRecords_updateAssociatedParentRecords_circularIndexing() throws URISyntaxException {
+ public void updateAssociatedRecords_updateAssociatedParentRecords_circularIndexing() throws Exception {
updateAssociatedRecords_updateAssociatedParentRecords_baseSetup();
// Test
@@ -644,46 +698,52 @@ public class AugmenterConfigurationServiceImplTest {
verify(this.indexerQueueTaskBuilder,times(0)).createWorkerTask(any(), any(), any());
}
- private void updateAssociatedRecords_updateAssociatedParentRecords_baseSetup() throws URISyntaxException {
+ private void updateAssociatedRecords_updateAssociatedParentRecords_baseSetup() throws Exception {
childKind = "osdu:wks:work-product-component--WellLog:1.0.0";
childId = "anyChildId";
parentKind = "osdu:wks:master-data--Wellbore:1.0.0";
parentId = "anyParentId";
- // Setup search response for searchService.query(...)
- when(this.searchService.query(any())).thenAnswer(invocation -> {
- SearchRequest searchRequest = invocation.getArgument(0);
- SearchResponse searchResponse = new SearchResponse();
- if (searchRequest.getKind().toString().equals(augmenterConfigurationKind)) {
- if (searchRequest.getQuery().contains("ParentToChildren")) {
+ // Setup search response for searchClient.search(...)
+ when(this.searchClient.search(anyString(), any(), any(), any(), anyInt())).thenAnswer(invocation -> {
+ String kind = invocation.getArgument(0);
+ return this.searchClient.search(List.of(kind), invocation.getArgument(1), invocation.getArgument(2), invocation.getArgument(3), invocation.getArgument(4));
+ });
+ when(this.searchClient.search(any(List.class), any(), any(), any(), anyInt())).thenAnswer(invocation -> {
+ String kind = ((List<String>)invocation.getArgument(0)).get(0);
+ List<SearchRecord> records = new ArrayList<>();
+ if (kind.toString().equals(augmenterConfigurationKind)) {
+ String queryString = invocation.getArgument(1).toString();
+ if (queryString.contains("ParentToChildren")) {
// Return of getParentChildRelatedObjectsSpecs(...)
Map<String, Object> dataMap = getDataMap("wellbore_configuration_record.json");
SearchRecord searchRecord = new SearchRecord();
searchRecord.setData(dataMap);
- searchResponse.setResults(Arrays.asList(searchRecord));
+ records.add(searchRecord);
} else {
// search ChildToParent.
// NO result
}
} else {
- if(searchRequest.getKind().toString().equals(childKind)) {
+ if(kind.toString().equals(childKind)) {
// Return of searchUniqueParentIds(...)
SearchRecord searchRecord = new SearchRecord();
Map<String, Object> childDataMap = new HashMap<>();
childDataMap.put("WellboreID", parentId);
searchRecord.setKind(childKind);
+ searchRecord.setId(childId);
searchRecord.setData(childDataMap);
- searchResponse.setResults(Arrays.asList(searchRecord));
+ records.add(searchRecord);
}
- else if(searchRequest.getKind().toString().equals("osdu:wks:master-data--Wellbore:1.*")) {
+ else if(kind.toString().equals("osdu:wks:master-data--Wellbore:1.*")) {
// Return of searchKindIds(...)
SearchRecord searchRecord = new SearchRecord();
searchRecord.setKind(parentKind);
searchRecord.setId(parentId);
- searchResponse.setResults(Arrays.asList(searchRecord));
+ records.add(searchRecord);
}
}
- return searchResponse;
+ return records;
});
// setup headers
@@ -695,16 +755,16 @@ public class AugmenterConfigurationServiceImplTest {
}
@Test
- public void updateAssociatedRecords_updateAssociatedChildrenRecords_for_created_parentRecord() throws URISyntaxException {
+ public void updateAssociatedRecords_updateAssociatedChildrenRecords_for_created_parentRecord() throws Exception {
updateAssociatedRecords_updateAssociatedChildrenRecords_for_created_delete(OperationType.create);
}
@Test
- public void updateAssociatedRecords_updateAssociatedChildrenRecords_for_deleted_parentRecord() throws URISyntaxException {
+ public void updateAssociatedRecords_updateAssociatedChildrenRecords_for_deleted_parentRecord() throws Exception {
updateAssociatedRecords_updateAssociatedChildrenRecords_for_created_delete(OperationType.delete);
}
- private void updateAssociatedRecords_updateAssociatedChildrenRecords_for_created_delete(OperationType op) throws URISyntaxException {
+ private void updateAssociatedRecords_updateAssociatedChildrenRecords_for_created_delete(OperationType op) throws Exception {
updateAssociatedRecords_updateAssociatedChildrenRecords_baseSetup();
// Test
@@ -732,7 +792,7 @@ public class AugmenterConfigurationServiceImplTest {
}
@Test
- public void updateAssociatedRecords_updateAssociatedChildrenRecords_for_updated_parentRecord_with_extendedPropertyChanged() throws URISyntaxException {
+ public void updateAssociatedRecords_updateAssociatedChildrenRecords_for_updated_parentRecord_with_extendedPropertyChanged() throws Exception {
updateAssociatedRecords_updateAssociatedChildrenRecords_baseSetup();
RecordChangeInfo recordChangeInfo = new RecordChangeInfo();
@@ -763,11 +823,11 @@ public class AugmenterConfigurationServiceImplTest {
Assert.assertEquals(1, infoList.size());
Assert.assertEquals(childKind, infoList.get(0).getKind());
Assert.assertEquals(childId, infoList.get(0).getId());
- verify(this.searchService,times(0)).queryWithCursor(any());
+ verify(this.searchClient,times(4)).search(any(List.class), any(), any(), any(), anyInt());
}
@Test
- public void updateAssociatedRecords_updateAssociatedChildrenRecords_for_updated_parentRecord_without_extendedPropertyChanged() throws URISyntaxException {
+ public void updateAssociatedRecords_updateAssociatedChildrenRecords_for_updated_parentRecord_without_extendedPropertyChanged() throws Exception {
updateAssociatedRecords_updateAssociatedChildrenRecords_baseSetup();
RecordChangeInfo recordChangeInfo = new RecordChangeInfo();
@@ -789,11 +849,11 @@ public class AugmenterConfigurationServiceImplTest {
// Verify
verify(this.indexerQueueTaskBuilder,times(0)).createWorkerTask(any(), any(), any());
- verify(this.searchService,times(0)).queryWithCursor(any());
+ verify(this.searchClient,times(4)).search(any(List.class), any(), any(), any(), anyInt());
}
@Test
- public void updateAssociatedRecords_updateAssociatedChildrenRecords_circularIndexing() throws URISyntaxException {
+ public void updateAssociatedRecords_updateAssociatedChildrenRecords_circularIndexing() throws Exception {
updateAssociatedRecords_updateAssociatedChildrenRecords_baseSetup();
// Test
@@ -808,166 +868,39 @@ public class AugmenterConfigurationServiceImplTest {
// Verify
verify(this.indexerQueueTaskBuilder,times(0)).createWorkerTask(any(), any(), any());
- verify(this.searchService,times(0)).queryWithCursor(any());
+ verify(this.searchClient,times(2)).search(any(List.class), any(), any(), any(), anyInt());
}
- private void updateAssociatedRecords_updateAssociatedChildrenRecords_baseSetup() throws URISyntaxException {
+ private void updateAssociatedRecords_updateAssociatedChildrenRecords_baseSetup() throws Exception {
childKind = "osdu:wks:master-data--Well:1.0.0";
childId = "anyChildId";
parentKind = "osdu:wks:master-data--GeoPoliticalEntity:1.0.0";
parentId = "anyParentId";
- // Setup search response for searchService.query(...)
- when(this.searchService.query(any())).thenAnswer(invocation -> {
- SearchRequest searchRequest = invocation.getArgument(0);
- SearchResponse searchResponse = new SearchResponse();
- if (searchRequest.getKind().toString().equals(augmenterConfigurationKind)) {
- if (searchRequest.getQuery().contains("ChildToParent") || searchRequest.getQuery().contains("data.Code:")) {
- // Return of getParentChildRelatedObjectsSpecs(...) or
- // getPropertyConfigurations(...)
- Map<String, Object> dataMap = getDataMap("well_configuration_record.json");
- SearchRecord searchRecord = new SearchRecord();
- searchRecord.setData(dataMap);
- searchResponse.setResults(Arrays.asList(searchRecord));
- } else {
- // Search ParentToChildren
- // No result
- }
- }
- else if(searchRequest.getKind().toString().contains("osdu:wks:master-data--Well:1.")) {
- // Return of searchUniqueParentIds(...)
- SearchRecord searchRecord = new SearchRecord();
- Map<String, Object> childDataMap = new HashMap<>();
- childDataMap.put("AssociatedIdentities", Arrays.asList(parentId));
- searchRecord.setKind(childKind);
- searchRecord.setId(childId);
- searchRecord.setData(childDataMap);
- searchResponse.setResults(Arrays.asList(searchRecord));
- }
- else {
- // This branch is a setup for test case:
- // updateAssociatedRecords_updateAssociatedChildrenRecords_circularIndexing
- throw new Exception("Unexpected search");
- }
- return searchResponse;
+ // Setup search response for searchClient.search(...)
+ when(this.searchClient.search(anyString(), any(), any(), any(), anyInt())).thenAnswer(invocation -> {
+ String kind = invocation.getArgument(0);
+ return this.searchClient.search(List.of(kind), invocation.getArgument(1), invocation.getArgument(2), invocation.getArgument(3), invocation.getArgument(4));
});
-
- // setup headers
- DpsHeaders dpsHeaders = new DpsHeaders();
- dpsHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
- dpsHeaders.put(DpsHeaders.DATA_PARTITION_ID, "opendes");
- dpsHeaders.put(DpsHeaders.CORRELATION_ID, "123");
- when(this.requestInfo.getHeadersWithDwdAuthZ()).thenReturn(dpsHeaders);
- }
-
- @Test
- public void updateAssociatedRecords_updateAssociatedChildrenRecords_with_cursor_query_for_updated_parentRecord_with_extendedPropertyChanged() throws URISyntaxException {
- updateAssociatedRecords_updateAssociatedChildrenRecords_with_query_by_cursor_baseSetup();
-
- RecordChangeInfo recordChangeInfo = new RecordChangeInfo();
- RecordInfo recordInfo = new RecordInfo();
- recordInfo.setKind(parentKind);
- recordInfo.setId(parentId);
- recordInfo.setOp(OperationType.update.getValue());
- recordChangeInfo.setRecordInfo(recordInfo);
- recordChangeInfo.setUpdatedProperties(Arrays.asList("GeoPoliticalEntityName"));
- when(this.recordChangeInfoCache.get(any())).thenReturn(recordChangeInfo);
-
- // Test
- RecordChangedMessages recordChangedMessages = new RecordChangedMessages();
- recordChangedMessages.setAttributes(new HashMap<>());
- Map<String, List<String>> upsertKindIds = new HashMap<>();
- Map<String, List<String>> deleteKindIds = new HashMap<>();
- upsertKindIds.put(parentKind, Arrays.asList(parentId));
- this.sut.updateAssociatedRecords(recordChangedMessages, upsertKindIds, deleteKindIds, new ArrayList<>());
-
- // Verify
- ArgumentCaptor<String> payloadArgumentCaptor = ArgumentCaptor.forClass(String.class);
- verify(this.indexerQueueTaskBuilder,times(1)).createWorkerTask(payloadArgumentCaptor.capture(), any(), any());
-
- RecordChangedMessages newMessages = gson.fromJson(payloadArgumentCaptor.getValue(), RecordChangedMessages.class);
- Type type = new TypeToken<List<RecordInfo>>() {}.getType();
- List<RecordInfo> infoList = gson.fromJson(newMessages.getData(), type);
- Assert.assertEquals(parentKind, newMessages.getAttributes().get(Constants.ANCESTRY_KINDS));
- Assert.assertEquals(1, infoList.size());
- Assert.assertEquals(childKind, infoList.get(0).getKind());
- Assert.assertEquals(childId, infoList.get(0).getId());
-
- verify(this.searchService,times(1)).queryWithCursor(any());
- }
-
- @Test
- public void updateAssociatedRecords_updateAssociatedChildrenRecords_with_cursor_query_for_updated_parentRecord_without_extendedPropertyChanged() throws URISyntaxException {
- updateAssociatedRecords_updateAssociatedChildrenRecords_with_query_by_cursor_baseSetup();
-
- RecordChangeInfo recordChangeInfo = new RecordChangeInfo();
- RecordInfo recordInfo = new RecordInfo();
- recordInfo.setKind(parentKind);
- recordInfo.setId(parentId);
- recordInfo.setOp(OperationType.update.getValue());
- recordChangeInfo.setRecordInfo(recordInfo);
- recordChangeInfo.setUpdatedProperties(Arrays.asList("abc"));
- when(this.recordChangeInfoCache.get(any())).thenReturn(recordChangeInfo);
-
- // Test
- RecordChangedMessages recordChangedMessages = new RecordChangedMessages();
- recordChangedMessages.setAttributes(new HashMap<>());
- Map<String, List<String>> upsertKindIds = new HashMap<>();
- Map<String, List<String>> deleteKindIds = new HashMap<>();
- upsertKindIds.put(parentKind, Arrays.asList(parentId));
- this.sut.updateAssociatedRecords(recordChangedMessages, upsertKindIds, deleteKindIds, new ArrayList<>());
-
- // Verify
- verify(this.indexerQueueTaskBuilder,times(0)).createWorkerTask(any(), any(), any());
- verify(this.searchService,times(1)).queryWithCursor(any());
- }
-
- private void updateAssociatedRecords_updateAssociatedChildrenRecords_with_query_by_cursor_baseSetup() throws URISyntaxException {
- childKind = "osdu:wks:master-data--Well:1.0.0";
- childId = "anyChildId";
- parentKind = "osdu:wks:master-data--GeoPoliticalEntity:1.0.0";
- parentId = "anyParentId";
-
- // Setup search response for searchService.query(...)
- when(this.searchService.query(any())).thenAnswer(invocation -> {
- SearchRequest searchRequest = invocation.getArgument(0);
- SearchResponse searchResponse = new SearchResponse();
- if (searchRequest.getKind().toString().equals(augmenterConfigurationKind)) {
- if (searchRequest.getQuery().contains("ChildToParent") || searchRequest.getQuery().contains("data.Code:")) {
+ when(this.searchClient.search(any(List.class), any(), any(), any(), anyInt())).thenAnswer(invocation -> {
+ String kind = ((List<String>)invocation.getArgument(0)).get(0);
+ Query query = invocation.getArgument(1);
+ List<SearchRecord> records = new ArrayList<>();
+ if (kind.toString().equals(augmenterConfigurationKind)) {
+ String queryString = query.toString();
+ if (queryString.contains("ChildToParent") || queryString.contains("data.Code:")) {
// Return of getParentChildRelatedObjectsSpecs(...) or
// getPropertyConfigurations(...)
Map<String, Object> dataMap = getDataMap("well_configuration_record.json");
SearchRecord searchRecord = new SearchRecord();
searchRecord.setData(dataMap);
- searchResponse.setResults(Arrays.asList(searchRecord));
+ records.add(searchRecord);
} else {
// Search ParentToChildren
// No result
}
}
- else if(searchRequest.getKind().toString().contains("osdu:wks:master-data--Well:1.")) {
- // Return of searchUniqueParentIds(...)
- SearchRecord searchRecord = new SearchRecord();
- Map<String, Object> childDataMap = new HashMap<>();
- childDataMap.put("AssociatedIdentities", Arrays.asList(parentId));
- searchRecord.setKind(childKind);
- searchRecord.setId(childId);
- searchRecord.setData(childDataMap);
- searchResponse.setResults(Arrays.asList(searchRecord));
- searchResponse.setTotalCount(10000);
- }
- else {
- // This branch is a setup for test case:
- // updateAssociatedRecords_updateAssociatedChildrenRecords_circularIndexing
- throw new Exception("Unexpected search");
- }
- return searchResponse;
- });
-
- when(this.searchService.queryWithCursor(any())).thenAnswer(invocation -> {
- SearchRequest searchRequest = invocation.getArgument(0);
- SearchResponse searchResponse = new SearchResponse();
- if(searchRequest.getKind().toString().contains("osdu:wks:master-data--Well:1.")) {
+ else if(kind.toString().contains("osdu:wks:master-data--Well:1.")) {
// Return of searchUniqueParentIds(...)
SearchRecord searchRecord = new SearchRecord();
Map<String, Object> childDataMap = new HashMap<>();
@@ -975,14 +908,14 @@ public class AugmenterConfigurationServiceImplTest {
searchRecord.setKind(childKind);
searchRecord.setId(childId);
searchRecord.setData(childDataMap);
- searchResponse.setResults(Arrays.asList(searchRecord));
+ records.add(searchRecord);
}
else {
// This branch is a setup for test case:
// updateAssociatedRecords_updateAssociatedChildrenRecords_circularIndexing
throw new Exception("Unexpected search");
}
- return searchResponse;
+ return records;
});
// setup headers
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java
index ff973b13ed938d93ac499ebd201b1f258c69ce59..62b57418f9124827107fb1b54c283c2e68da114b 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java
@@ -31,7 +31,6 @@ import static org.opengroup.osdu.indexer.config.IndexerConfigurationProperties.B
import static org.opengroup.osdu.indexer.config.IndexerConfigurationProperties.KEYWORD_LOWER_FEATURE_NAME;
import co.elastic.clients.elasticsearch.ElasticsearchClient;
-import co.elastic.clients.elasticsearch._types.AcknowledgedResponse;
import co.elastic.clients.elasticsearch._types.ElasticsearchException;
import co.elastic.clients.elasticsearch._types.ErrorCause;
import co.elastic.clients.elasticsearch._types.ErrorResponse;
@@ -40,6 +39,7 @@ import co.elastic.clients.elasticsearch.indices.ElasticsearchIndicesClient;
import co.elastic.clients.elasticsearch.indices.GetMappingRequest;
import co.elastic.clients.elasticsearch.indices.GetMappingResponse;
import co.elastic.clients.elasticsearch.indices.PutMappingRequest;
+import co.elastic.clients.elasticsearch.indices.PutMappingResponse;
import co.elastic.clients.elasticsearch.indices.get_mapping.IndexMappingRecord;
import com.google.gson.Gson;
import java.io.IOException;
@@ -47,6 +47,7 @@ import java.io.StringReader;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
+import org.apache.http.HttpStatus;
import org.apache.http.StatusLine;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestClient;
@@ -65,7 +66,9 @@ import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
import org.opengroup.osdu.indexer.cache.partitionsafe.IndexCache;
+import org.opengroup.osdu.indexer.model.XcollaborationHolder;
import org.opengroup.osdu.indexer.service.exception.ElasticsearchMappingException;
+import org.opengroup.osdu.indexer.util.CustomIndexAnalyzerSetting;
import org.opengroup.osdu.indexer.util.ElasticClientHandler;
import org.opengroup.osdu.indexer.util.TypeMapper;
import org.springframework.boot.test.context.SpringBootTest;
@@ -99,8 +102,12 @@ public class IndexerMappingServiceTest {
private IndicesService indicesService;
@Mock
private ElasticIndexNameResolver elasticIndexNameResolver;
+ @Mock
+ private XcollaborationHolder xcollaborationHolder;
@MockBean
private IFeatureFlag featureFlag;
+ @Mock
+ private CustomIndexAnalyzerSetting customIndexAnalyzerSetting;
@Spy
@InjectMocks
@@ -110,7 +117,7 @@ public class IndexerMappingServiceTest {
private IndexSchema noDataIndexSchema;
private ElasticsearchIndicesClient indicesClient;
- private ElasticsearchClient restHighLevelClient;
+ private ElasticsearchClient elasticsearchClient;
@Before
public void setup() throws IOException {
@@ -120,13 +127,19 @@ public class IndexerMappingServiceTest {
this.noDataIndexSchema = IndexSchema.builder().kind(kind).type(type).dataSchema(null).metaSchema(getMetaAttributeMapping()).build();
this.indicesClient = mock(ElasticsearchIndicesClient.class);
- this.restHighLevelClient = mock(ElasticsearchClient.class);
+ this.elasticsearchClient = mock(ElasticsearchClient.class);
when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(index);
// when(this.restHighLevelClient.getLowLevelClient()).thenReturn(restClient);
when(this.restClient.performRequest(any())).thenReturn(response);
when(this.response.getStatusLine()).thenReturn(statusLine);
when(this.statusLine.getStatusCode()).thenReturn(200);
+ when(xcollaborationHolder.isFeatureEnabledAndHeaderExists()).thenReturn(false);
+
+ when(this.elasticsearchClient.indices()).thenReturn(this.indicesClient);
+ GetMappingResponse mappingResponse = mock(GetMappingResponse.class);
+ when(this.indicesClient.getMapping(any(GetMappingRequest.class))).thenReturn(mappingResponse);
+ when(this.customIndexAnalyzerSetting.isEnabled()).thenReturn(false);
}
private Map<String, Object> getMetaAttributeMapping() {
@@ -144,7 +157,7 @@ public class IndexerMappingServiceTest {
dataMapping.put("Msg", "text");
Map<String, Object> intervalNestedAttribute = new HashMap<>();
Map<String, Object> intervalProperties = new HashMap<>();
- intervalProperties.put("StopMarkerID", "link");
+ intervalProperties.put("StopMarkerID", "keyword");
intervalProperties.put("GeologicUnitInterpretationIDs", "text");
intervalProperties.put("StopMeasuredDepth", "double");
intervalNestedAttribute.put("properties", intervalProperties);
@@ -153,10 +166,14 @@ public class IndexerMappingServiceTest {
}
@Test
- public void should_returnValidMapping_givenFalseMerge_keywordLowerDisabled_createMappingTest() {
+ public void should_returnValidMapping_givenFalseMerge_keywordLowerDisabled_createMappingTest() throws IOException {
when(this.featureFlag.isFeatureEnabled(KEYWORD_LOWER_FEATURE_NAME)).thenReturn(false);
+ PutMappingResponse putMappingResponse = mock(PutMappingResponse.class);
+ when(putMappingResponse.acknowledged()).thenReturn(true);
+ when(indicesClient.putMapping(any(PutMappingRequest.class))).thenReturn(putMappingResponse);
+
try {
- String mapping = this.sut.createMapping(restHighLevelClient, indexSchema, index, false);
+ String mapping = this.sut.createMapping(elasticsearchClient, indexSchema, index, false);
assertEquals(validMapping, mapping);
} catch (Exception e) {
fail("Should not throw this exception" + e.getMessage());
@@ -164,10 +181,15 @@ public class IndexerMappingServiceTest {
}
@Test
- public void should_returnValidMapping_givenFalseMerge_keywordLowerEnabled_createMappingTest() {
+ public void should_returnValidMapping_givenFalseMerge_keywordLowerEnabled_createMappingTest() throws IOException {
when(this.featureFlag.isFeatureEnabled(KEYWORD_LOWER_FEATURE_NAME)).thenReturn(true);
+
+ PutMappingResponse putMappingResponse = mock(PutMappingResponse.class);
+ when(putMappingResponse.acknowledged()).thenReturn(true);
+ when(indicesClient.putMapping(any(PutMappingRequest.class))).thenReturn(putMappingResponse);
+
try {
- String mapping = this.sut.createMapping(restHighLevelClient, indexSchema, index, false);
+ String mapping = this.sut.createMapping(elasticsearchClient, indexSchema, index, false);
assertEquals(validKeywordLowerMapping, mapping);
} catch (Exception e) {
fail("Should not throw this exception" + e.getMessage());
@@ -178,12 +200,12 @@ public class IndexerMappingServiceTest {
@Test
public void should_returnValidMapping_givenTrueMerge_createMappingTest() {
try {
- AcknowledgedResponse mappingResponse = mock(AcknowledgedResponse.class);
+ PutMappingResponse mappingResponse = mock(PutMappingResponse.class);
doReturn(true).when(mappingResponse).acknowledged();
- doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ doReturn(this.indicesClient).when(this.elasticsearchClient).indices();
doReturn(mappingResponse).when(this.indicesClient).putMapping(any(PutMappingRequest.class));
- String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true);
+ String mapping = this.sut.createMapping(this.elasticsearchClient, this.indexSchema, this.index, true);
assertEquals(this.validMapping, mapping);
} catch (Exception e) {
fail("Should not throw this exception" + e.getMessage());
@@ -194,12 +216,12 @@ public class IndexerMappingServiceTest {
public void should_returnValidMapping_givenTrueMerge_keywordLowerEnabled_createMappingTest() {
when(this.featureFlag.isFeatureEnabled(KEYWORD_LOWER_FEATURE_NAME)).thenReturn(true);
try {
- AcknowledgedResponse mappingResponse = mock(AcknowledgedResponse.class);
+ PutMappingResponse mappingResponse = mock(PutMappingResponse.class);
doReturn(true).when(mappingResponse).acknowledged();
- doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ doReturn(this.indicesClient).when(this.elasticsearchClient).indices();
doReturn(mappingResponse).when(this.indicesClient).putMapping(any(PutMappingRequest.class));
- String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true);
+ String mapping = this.sut.createMapping(this.elasticsearchClient, this.indexSchema, this.index, true);
assertEquals(validKeywordLowerMapping, mapping);
} catch (Exception e) {
fail("Should not throw this exception" + e.getMessage());
@@ -209,13 +231,13 @@ public class IndexerMappingServiceTest {
@Test
public void should_returnValidMapping_givenExistType_createMappingTest() {
try {
- AcknowledgedResponse mappingResponse = mock(AcknowledgedResponse.class);
+ PutMappingResponse mappingResponse = mock(PutMappingResponse.class);
doReturn(true).when(mappingResponse).acknowledged();
- doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ doReturn(this.indicesClient).when(this.elasticsearchClient).indices();
doReturn(mappingResponse).when(this.indicesClient).putMapping(any(PutMappingRequest.class));
IndexerMappingServiceImpl indexerMappingServiceLocal = spy(new IndexerMappingServiceImpl());
- String mapping = this.sut.createMapping(this.restHighLevelClient, this.indexSchema, this.index, true);
+ String mapping = this.sut.createMapping(this.elasticsearchClient, this.indexSchema, this.index, true);
assertEquals(validMapping, mapping);
} catch (Exception e) {
fail("Should not throw this exception" + e.getMessage());
@@ -250,7 +272,7 @@ public class IndexerMappingServiceTest {
final String cacheKey = String.format("metaAttributeMappingSynced-%s", index);
when(this.indexCache.get(cacheKey)).thenReturn(true);
- this.sut.syncMetaAttributeIndexMappingIfRequired(restHighLevelClient, indexSchema);
+ this.sut.syncMetaAttributeIndexMappingIfRequired(elasticsearchClient, indexSchema);
Collection<Invocation> invocations = mockingDetails(this.sut).getInvocations();
assertEquals(1, invocations.size());
}
@@ -259,8 +281,13 @@ public class IndexerMappingServiceTest {
public void should_applyNoUpdate_givenUpdateIndex() throws Exception {
final String cacheKey = String.format("metaAttributeMappingSynced-%s", index);
final String mapping = "{\"dynamic\":\"false\",\"properties\":{\"acl\":{\"properties\":{\"owners\":{\"type\":\"keyword\"},\"viewers\":{\"type\":\"keyword\"}}},\"ancestry\":{\"properties\":{\"parents\":{\"type\":\"keyword\"}}},\"authority\":{\"type\":\"constant_keyword\",\"value\":\"opendes\"},\"createTime\":{\"type\":\"date\"},\"createUser\":{\"type\":\"keyword\"},\"data\":{\"properties\":{\"message\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"null_value\":\"null\",\"ignore_above\":256}}}}},\"id\":{\"type\":\"keyword\"},\"index\":{\"properties\":{\"lastUpdateTime\":{\"type\":\"date\"},\"statusCode\":{\"type\":\"integer\"},\"trace\":{\"type\":\"text\"}}},\"kind\":{\"type\":\"keyword\"},\"legal\":{\"properties\":{\"legaltags\":{\"type\":\"keyword\"},\"otherRelevantDataCountries\":{\"type\":\"keyword\"},\"status\":{\"type\":\"keyword\"}}},\"modifyTime\":{\"type\":\"date\"},\"modifyUser\":{\"type\":\"keyword\"},\"namespace\":{\"type\":\"keyword\"},\"source\":{\"type\":\"constant_keyword\",\"value\":\"test\"},\"tags\":{\"type\":\"flattened\"},\"type\":{\"type\":\"keyword\"},\"version\":{\"type\":\"long\"},\"x-acl\":{\"type\":\"keyword\"},\"bagOfWords\":{\"search_analyzer\":\"whitespace\",\"analyzer\":\"detailExtractor\",\"store\":true,\"type\":\"text\",\"fields\":{\"autocomplete\":{\"type\":\"completion\",\"analyzer\":\"detailExtractor\",\"search_analyzer\":\"whitespace\",\"max_input_length\":256}}}}}";
- doReturn(mapping).when(this.sut).getIndexMapping(restHighLevelClient, index);
- this.sut.syncMetaAttributeIndexMappingIfRequired(restHighLevelClient, noDataIndexSchema);
+ doReturn(mapping).when(this.sut).getIndexMapping(elasticsearchClient, index);
+
+ PutMappingResponse putMappingResponse = mock(PutMappingResponse.class);
+ when(putMappingResponse.acknowledged()).thenReturn(true);
+ when(indicesClient.putMapping(any(PutMappingRequest.class))).thenReturn(putMappingResponse);
+
+ this.sut.syncMetaAttributeIndexMappingIfRequired(elasticsearchClient, noDataIndexSchema);
verify(this.indexCache, times(1)).get(cacheKey);
verify(this.indexCache, times(1)).put(cacheKey, true);
@@ -270,14 +297,14 @@ public class IndexerMappingServiceTest {
public void should_applyUpdate_givenExistingIndex() throws Exception {
final String cacheKey = String.format("metaAttributeMappingSynced-%s", index);
final String mapping = "{\"dynamic\":\"false\",\"properties\":{\"acl\":{\"properties\":{\"owners\":{\"type\":\"keyword\"},\"viewers\":{\"type\":\"keyword\"}}},\"ancestry\":{\"properties\":{\"parents\":{\"type\":\"keyword\"}}},\"data\":{\"properties\":{\"message\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"null_value\":\"null\",\"ignore_above\":256}}}}},\"id\":{\"type\":\"keyword\"},\"index\":{\"properties\":{\"lastUpdateTime\":{\"type\":\"date\"},\"statusCode\":{\"type\":\"integer\"},\"trace\":{\"type\":\"text\"}}},\"kind\":{\"type\":\"keyword\"},\"legal\":{\"properties\":{\"legaltags\":{\"type\":\"keyword\"},\"otherRelevantDataCountries\":{\"type\":\"keyword\"},\"status\":{\"type\":\"keyword\"}}},\"namespace\":{\"type\":\"keyword\"},\"tags\":{\"type\":\"flattened\"},\"type\":{\"type\":\"keyword\"},\"version\":{\"type\":\"long\"},\"x-acl\":{\"type\":\"keyword\"}}}";
- doReturn(mapping).when(this.sut).getIndexMapping(restHighLevelClient, index);
+ doReturn(mapping).when(this.sut).getIndexMapping(elasticsearchClient, index);
- AcknowledgedResponse mappingResponse = mock(AcknowledgedResponse.class);
+ PutMappingResponse mappingResponse = mock(PutMappingResponse.class);
doReturn(true).when(mappingResponse).acknowledged();
- doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ doReturn(this.indicesClient).when(this.elasticsearchClient).indices();
doReturn(mappingResponse).when(this.indicesClient).putMapping(any(PutMappingRequest.class));
- this.sut.syncMetaAttributeIndexMappingIfRequired(restHighLevelClient, indexSchema);
+ this.sut.syncMetaAttributeIndexMappingIfRequired(elasticsearchClient, indexSchema);
verify(this.indexCache, times(1)).get(cacheKey);
verify(this.indexCache, times(1)).put(cacheKey, true);
@@ -288,9 +315,18 @@ public class IndexerMappingServiceTest {
public void should_applyNoUpdate_onDataChange_givenMetaUpdate_onIndex() throws Exception {
final String cacheKey = String.format("metaAttributeMappingSynced-%s", index);
final String mapping = "{\"dynamic\":\"false\",\"properties\":{\"acl\":{\"properties\":{\"owners\":{\"type\":\"keyword\"},\"viewers\":{\"type\":\"keyword\"}}},\"ancestry\":{\"properties\":{\"parents\":{\"type\":\"keyword\"}}},\"authority\":{\"type\":\"constant_keyword\",\"value\":\"opendes\"},\"createTime\":{\"type\":\"date\"},\"createUser\":{\"type\":\"keyword\"},\"data\":{\"properties\":{\"message\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"null_value\":\"null\",\"ignore_above\":256}}}}},\"id\":{\"type\":\"keyword\"},\"index\":{\"properties\":{\"lastUpdateTime\":{\"type\":\"date\"},\"statusCode\":{\"type\":\"integer\"},\"trace\":{\"type\":\"text\"}}},\"kind\":{\"type\":\"keyword\"},\"legal\":{\"properties\":{\"legaltags\":{\"type\":\"keyword\"},\"otherRelevantDataCountries\":{\"type\":\"keyword\"},\"status\":{\"type\":\"keyword\"}}},\"modifyTime\":{\"type\":\"date\"},\"modifyUser\":{\"type\":\"keyword\"},\"namespace\":{\"type\":\"keyword\"},\"source\":{\"type\":\"constant_keyword\",\"value\":\"test\"},\"tags\":{\"type\":\"flattened\"},\"type\":{\"type\":\"keyword\"},\"version\":{\"type\":\"long\"},\"x-acl\":{\"type\":\"keyword\"},\"bagOfWords\":{\"search_analyzer\":\"whitespace\",\"analyzer\":\"detailExtractor\",\"store\":true,\"type\":\"text\",\"fields\":{\"autocomplete\":{\"type\":\"completion\",\"analyzer\":\"detailExtractor\",\"search_analyzer\":\"whitespace\",\"max_input_length\":256}}}}}";
- doReturn(mapping).when(this.sut).getIndexMapping(restHighLevelClient, index);
+ doReturn(mapping).when(this.sut).getIndexMapping(elasticsearchClient, index);
when(this.featureFlag.isFeatureEnabled(BAG_OF_WORDS_FEATURE_NAME)).thenReturn(false);
- this.sut.syncMetaAttributeIndexMappingIfRequired(restHighLevelClient, noDataIndexSchema);
+
+ when(this.elasticsearchClient.indices()).thenReturn(this.indicesClient);
+ GetMappingResponse mappingResponse = mock(GetMappingResponse.class);
+ when(this.indicesClient.getMapping(any(GetMappingRequest.class))).thenReturn(mappingResponse);
+
+ PutMappingResponse putMappingResponse = mock(PutMappingResponse.class);
+ when(putMappingResponse.acknowledged()).thenReturn(true);
+ when(indicesClient.putMapping(any(PutMappingRequest.class))).thenReturn(putMappingResponse);
+
+ this.sut.syncMetaAttributeIndexMappingIfRequired(elasticsearchClient, noDataIndexSchema);
verify(this.indexCache, times(1)).get(cacheKey);
verify(this.indexCache, times(1)).put(cacheKey, true);
@@ -305,32 +341,39 @@ public class IndexerMappingServiceTest {
String mapping = "{\"dynamic\":\"false\",\"properties\":{\"acl\":{\"properties\":{\"owners\":{\"type\":\"keyword\"},\"viewers\":{\"type\":\"keyword\"}}},\"ancestry\":{\"properties\":{\"parents\":{\"type\":\"keyword\"}}},\"data\":{\"properties\":{\"message\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"null_value\":\"null\",\"ignore_above\":256}}}}},\"id\":{\"type\":\"keyword\"},\"index\":{\"properties\":{\"lastUpdateTime\":{\"type\":\"date\"},\"statusCode\":{\"type\":\"integer\"},\"trace\":{\"type\":\"text\"}}},\"kind\":{\"type\":\"keyword\"},\"legal\":{\"properties\":{\"legaltags\":{\"type\":\"keyword\"},\"otherRelevantDataCountries\":{\"type\":\"keyword\"},\"status\":{\"type\":\"keyword\"}}},\"namespace\":{\"type\":\"keyword\"},\"tags\":{\"type\":\"flattened\"},\"type\":{\"type\":\"keyword\"},\"version\":{\"type\":\"long\"},\"x-acl\":{\"type\":\"keyword\"}}}";
String message = "testExceptionMessage";
- ErrorResponse errorResponse = ErrorResponse.of(errorRespBuilder -> errorRespBuilder.error(
- ErrorCause.of(errorCauseBuilder -> errorCauseBuilder.reason(message))));
+ ErrorResponse errorResponse = ErrorResponse.of(errorRespBuilder -> errorRespBuilder
+ .error(ErrorCause.of(errorCauseBuilder -> errorCauseBuilder.reason(message)))
+ .status(HttpStatus.SC_BAD_REQUEST)
+ );
- doReturn(mapping).when(this.sut).getIndexMapping(restHighLevelClient, index);
- doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
+ doReturn(mapping).when(this.sut).getIndexMapping(elasticsearchClient, index);
+ doReturn(this.indicesClient).when(this.elasticsearchClient).indices();
doThrow(new ElasticsearchException(message, errorResponse)).when(this.indicesClient).putMapping(any(PutMappingRequest.class));
+ GetMappingResponse mappingResponse = mock(GetMappingResponse.class);
+ when(this.indicesClient.getMapping(any(GetMappingRequest.class))).thenReturn(mappingResponse);
+
ElasticsearchMappingException exception = assertThrows(
ElasticsearchMappingException.class,
- () -> this.sut.syncMetaAttributeIndexMappingIfRequired(restHighLevelClient, indexSchema));
+ () -> this.sut.syncMetaAttributeIndexMappingIfRequired(elasticsearchClient, indexSchema));
- assertEquals(message, exception.getMessage());
+ String expectedMessage = "Failed to create mapping: [%s] failed: [null] %s".formatted(message, message);
+ assertEquals(expectedMessage, exception.getMessage());
}
@Test
public void testGetIndexMapping() throws Exception {
when(indicesService.isIndexExist(any(), any())).thenReturn(true);
- when(restHighLevelClient.indices()).thenReturn(indicesClient);
+ when(elasticsearchClient.indices()).thenReturn(indicesClient);
TypeMapping typeMapping = TypeMapping.of(builder -> builder.withJson(new StringReader(mapping)));
+
GetMappingResponse getMappingResponse = GetMappingResponse.of(
- responseBuilder -> responseBuilder.putResult(
+ responseBuilder -> responseBuilder.result(
"index", IndexMappingRecord.of(mappingRecordBuilder -> mappingRecordBuilder.mappings(typeMapping))
)
);
when(indicesClient.getMapping(any(GetMappingRequest.class))).thenReturn(getMappingResponse);
- String actualMapping = this.sut.getIndexMapping(restHighLevelClient, "index");
+ String actualMapping = this.sut.getIndexMapping(elasticsearchClient, "index");
assertEquals(mapping, actualMapping);
}
}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java
index 6f1e3864636658a92b9cc282e4b2af522c935102..f74d225ca9e588015bf2d59f41363e54a37720bb 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerSchemaServiceTest.java
@@ -72,6 +72,7 @@ import org.opengroup.osdu.indexer.cache.partitionsafe.SchemaCache;
import org.opengroup.osdu.indexer.cache.partitionsafe.VirtualPropertiesSchemaCache;
import org.opengroup.osdu.indexer.model.indexproperty.AugmenterConfiguration;
import org.opengroup.osdu.indexer.schema.converter.exeption.SchemaProcessingException;
+import org.opengroup.osdu.indexer.service.exception.ElasticsearchMappingException;
import org.opengroup.osdu.indexer.util.AugmenterSetting;
import org.opengroup.osdu.indexer.util.ElasticClientHandler;
import org.springframework.test.context.junit4.SpringRunner;
@@ -422,7 +423,7 @@ public class IndexerSchemaServiceTest {
when(this.schemaCache.get(kind)).thenReturn(null);
when(this.indicesService.isIndexExist(any(), any())).thenReturn(true);
when(this.schemaService.getSchema(kind)).thenReturn(storageSchema);
- when(this.mappingService.createMapping(any(), any(), any(), anyBoolean())).thenThrow(new AppException(HttpStatus.SC_BAD_REQUEST, reason, ""));
+ when(this.mappingService.createMapping(any(), any(), any(), anyBoolean())).thenThrow(new ElasticsearchMappingException(reason, HttpStatus.SC_BAD_REQUEST));
try {
this.sut.processSchemaMessages(schemaMessages);
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceImplTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceImplTest.java
index a5708a411b9d0c2b0ee14c1588f9f5de347ea8b8..42e7e3e96822fc176ffd5d2ac181b7f08fcc17f5 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceImplTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerServiceImplTest.java
@@ -71,6 +71,7 @@ import org.opengroup.osdu.core.common.model.storage.ConversionStatus;
import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
import org.opengroup.osdu.indexer.logging.AuditLogger;
+import org.opengroup.osdu.indexer.model.XcollaborationHolder;
import org.opengroup.osdu.indexer.model.indexproperty.AugmenterConfiguration;
import org.opengroup.osdu.indexer.provider.interfaces.IPublisher;
import org.opengroup.osdu.indexer.service.exception.ElasticsearchMappingException;
@@ -121,6 +122,8 @@ public class IndexerServiceImplTest {
private AugmenterSetting augmenterSetting;
@Mock
private IFeatureFlag asIngestedCoordinatesFeatureFlag;
+ @Mock
+ private XcollaborationHolder xcollaborationHolder;
private List<RecordInfo> recordInfos = new ArrayList<>();
@@ -160,6 +163,7 @@ public class IndexerServiceImplTest {
mockedAcls = mockStatic(Acl.class);
initMocks(this);
when(augmenterSetting.isEnabled()).thenReturn(true);
+ when(xcollaborationHolder.isFeatureEnabledAndHeaderExists()).thenReturn(false);
}
@After
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndicesServiceTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndicesServiceTest.java
index 193d5e94eadb06ab641916cf88ce7d2a45029fd1..a70e1ad6f5b07a083213729423d51875a1622cf0 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndicesServiceTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndicesServiceTest.java
@@ -14,49 +14,18 @@
package org.opengroup.osdu.indexer.service;
-import static junit.framework.TestCase.assertTrue;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.fail;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-import static org.mockito.MockitoAnnotations.initMocks;
-
import co.elastic.clients.elasticsearch.ElasticsearchClient;
import co.elastic.clients.elasticsearch._types.ElasticsearchException;
import co.elastic.clients.elasticsearch._types.ErrorCause;
import co.elastic.clients.elasticsearch._types.ErrorResponse;
-import co.elastic.clients.elasticsearch._types.HealthStatus;
-import co.elastic.clients.elasticsearch.cluster.ElasticsearchClusterClient;
-import co.elastic.clients.elasticsearch.cluster.HealthRequest;
-import co.elastic.clients.elasticsearch.cluster.HealthResponse;
-import co.elastic.clients.elasticsearch.indices.CreateIndexRequest;
-import co.elastic.clients.elasticsearch.indices.CreateIndexResponse;
-import co.elastic.clients.elasticsearch.indices.DeleteIndexRequest;
-import co.elastic.clients.elasticsearch.indices.DeleteIndexResponse;
-import co.elastic.clients.elasticsearch.indices.ElasticsearchIndicesClient;
-import co.elastic.clients.elasticsearch.indices.ExistsRequest;
-import co.elastic.clients.elasticsearch.indices.GetIndexRequest;
-import co.elastic.clients.elasticsearch.indices.GetIndexResponse;
-import co.elastic.clients.elasticsearch.indices.IndexState;
-import co.elastic.clients.elasticsearch.indices.PutAliasRequest;
-import co.elastic.clients.elasticsearch.indices.PutAliasResponse;
+import co.elastic.clients.elasticsearch._types.analysis.Analyzer;
+import co.elastic.clients.elasticsearch._types.analysis.CharFilter;
+import co.elastic.clients.elasticsearch.indices.*;
import co.elastic.clients.transport.endpoints.BooleanResponse;
import co.elastic.clients.transport.rest_client.RestClientTransport;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.reflect.TypeToken;
import com.google.gson.Gson;
-import java.io.IOException;
-import java.lang.reflect.Type;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
import org.apache.http.HttpEntity;
import org.apache.http.HttpStatus;
import org.apache.http.entity.ContentType;
@@ -64,9 +33,11 @@ import org.apache.http.entity.StringEntity;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestClient;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Spy;
@@ -75,10 +46,24 @@ import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.search.IndexInfo;
import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
import org.opengroup.osdu.indexer.cache.partitionsafe.IndexCache;
+import org.opengroup.osdu.indexer.util.CustomIndexAnalyzerSetting;
import org.opengroup.osdu.indexer.util.ElasticClientHandler;
import org.springframework.context.annotation.Lazy;
import org.springframework.test.context.junit4.SpringRunner;
+import java.io.IOException;
+import java.lang.reflect.Type;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static junit.framework.TestCase.assertTrue;
+import static org.junit.Assert.*;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.*;
+import static org.mockito.MockitoAnnotations.initMocks;
+import static org.opengroup.osdu.indexer.testutils.ReflectionTestUtil.setFieldValueForClass;
+
@RunWith(SpringRunner.class)
public class IndicesServiceTest {
@Mock
@@ -98,6 +83,8 @@ public class IndicesServiceTest {
private HttpEntity httpEntity;
@Mock
private IndexAliasService indexAliasService;
+ @Mock
+ private CustomIndexAnalyzerSetting customIndexAnalyzerSetting;
@Spy
private ObjectMapper objectMapper = new ObjectMapper();
@InjectMocks
@@ -105,16 +92,16 @@ public class IndicesServiceTest {
private ElasticsearchClient restHighLevelClient;
private ElasticsearchIndicesClient indicesClient;
- private ElasticsearchClusterClient clusterClient;
private RestClientTransport restClientTransport;
@Before
public void setup() {
initMocks(this);
indicesClient = mock(ElasticsearchIndicesClient.class);
- clusterClient = mock(ElasticsearchClusterClient.class);
restHighLevelClient = mock(ElasticsearchClient.class);
restClientTransport = mock(RestClientTransport.class);
+ setFieldValueForClass(sut, "healthRetryThreshold", 1);
+ setFieldValueForClass(sut, "healthRetrySleepPeriodInMilliseconds", 1);
}
@Test
@@ -122,16 +109,115 @@ public class IndicesServiceTest {
String index = "common-welldb-wellbore-1.2.0";
CreateIndexResponse indexResponse = CreateIndexResponse.of(builder -> builder.index(index).acknowledged(true).shardsAcknowledged(true));
PutAliasResponse putAliasResponse = PutAliasResponse.of(builder -> builder.acknowledged(true));
+ ArgumentCaptor<CreateIndexRequest> createIndexRequestArgumentCaptor = ArgumentCaptor.forClass(CreateIndexRequest.class);
+
+ when(elasticIndexNameResolver.getIndexNameFromKind(any())).thenReturn(index);
+ when(restHighLevelClient.indices()).thenReturn(indicesClient);
+ when(indicesClient.create(any(CreateIndexRequest.class))).thenReturn(indexResponse);
+ when(indicesClient.putAlias(any(PutAliasRequest.class))).thenReturn(putAliasResponse);
+ when(customIndexAnalyzerSetting.isEnabled()).thenReturn(false);
+
+ boolean response = this.sut.createIndex(restHighLevelClient, index, null, new HashMap<>());
+ assertTrue(response);
+ when(this.indicesExistCache.get(index)).thenReturn(true);
+ verify(this.indexAliasService, times(1)).createIndexAlias(any(), any());
+ verify(indicesClient).create(createIndexRequestArgumentCaptor.capture());
+
+ CreateIndexRequest request = createIndexRequestArgumentCaptor.getValue();
+ IndexSettingsAnalysis analysis =request.settings().analysis();
+ Assert.assertNull(analysis);
+ }
+
+ @Test
+ public void create_elasticIndex_with_custom_analyzer() throws Exception {
+ String index = "common-welldb-wellbore-1.2.0";
+ CreateIndexResponse indexResponse = CreateIndexResponse.of(builder -> builder.index(index).acknowledged(true).shardsAcknowledged(true));
+ PutAliasResponse putAliasResponse = PutAliasResponse.of(builder -> builder.acknowledged(true));
+ ArgumentCaptor<CreateIndexRequest> createIndexRequestArgumentCaptor = ArgumentCaptor.forClass(CreateIndexRequest.class);
when(elasticIndexNameResolver.getIndexNameFromKind(any())).thenReturn(index);
when(restHighLevelClient.indices()).thenReturn(indicesClient);
when(indicesClient.create(any(CreateIndexRequest.class))).thenReturn(indexResponse);
when(indicesClient.putAlias(any(PutAliasRequest.class))).thenReturn(putAliasResponse);
+ when(customIndexAnalyzerSetting.isEnabled()).thenReturn(true);
boolean response = this.sut.createIndex(restHighLevelClient, index, null, new HashMap<>());
assertTrue(response);
when(this.indicesExistCache.get(index)).thenReturn(true);
verify(this.indexAliasService, times(1)).createIndexAlias(any(), any());
+ verify(indicesClient).create(createIndexRequestArgumentCaptor.capture());
+
+ CreateIndexRequest request = createIndexRequestArgumentCaptor.getValue();
+ IndexSettingsAnalysis analysis =request.settings().analysis();
+ Assert.assertNotNull(analysis);
+ /*
+ {
+ "analyzer": {
+ "osdu_custom_analyzer": {
+ "type": "custom",
+ "char_filter": ["osdu_mapping_charFilter_for_underscore", "osdu_patternReplace_charFilter_for_dot"],
+ "filter": ["lowercase"],
+ "tokenizer": "standard"
+ }
+ },
+ "char_filter": {
+ "osdu_mapping_charFilter_for_underscore": {
+ "type": "mapping",
+ "mappings": ["_=>\\\\u0020"]
+ },
+ "osdu_patternReplace_charFilter_for_dot": {
+ "type": "pattern_replace",
+ "pattern": "(\\\\D)\\\\.|\\\\.(?=\\\\D)",
+ "replacement": "$1 "
+ }
+ }
+ }
+ */
+ Assert.assertTrue(analysis.analyzer().containsKey("osdu_custom_analyzer"));
+ Analyzer analyzer = analysis.analyzer().get("osdu_custom_analyzer");
+ Assert.assertTrue(analyzer.isCustom());
+ Assert.assertEquals("standard", analyzer.custom().tokenizer());
+ Assert.assertEquals(List.of("lowercase"), analyzer.custom().filter());
+ Assert.assertEquals(List.of("osdu_mapping_charFilter_for_underscore", "osdu_patternReplace_charFilter_for_dot"),
+ analyzer.custom().charFilter().stream().sorted().toList());
+
+ Assert.assertTrue(analysis.charFilter().containsKey("osdu_mapping_charFilter_for_underscore"));
+ CharFilter mappingCharFilter = analysis.charFilter().get("osdu_mapping_charFilter_for_underscore");
+ Assert.assertTrue(mappingCharFilter.definition().isMapping());
+ Assert.assertEquals(List.of("_=>\\u0020"), mappingCharFilter.definition().mapping().mappings());
+
+ Assert.assertTrue(analysis.charFilter().containsKey("osdu_patternReplace_charFilter_for_dot"));
+ CharFilter patternReplaceCharFilter = analysis.charFilter().get("osdu_patternReplace_charFilter_for_dot");
+ Assert.assertTrue(patternReplaceCharFilter.definition().isPatternReplace());
+ Assert.assertEquals("(\\D)\\.|\\.(?=\\D)", patternReplaceCharFilter.definition().patternReplace().pattern());
+ Assert.assertEquals("$1 ", patternReplaceCharFilter.definition().patternReplace().replacement());
+ }
+
+ @Test
+ public void custom_analyzer_comprehensive_char_filters() {
+ String matchRegex = "(\\D)\\.|\\.(?=\\D)";
+ String replacement = "$1 ";
+ String input = "dev_tools_console.pdf";
+ String output = input.replaceAll("_", " ").replaceAll(matchRegex, replacement);
+ Assert.assertEquals("dev tools console pdf", output);
+
+ // Replace dot with whitespace for non-number string
+ input = "AAA.BBB";
+ output = input.replaceAll("_", " ").replaceAll(matchRegex, replacement);
+ Assert.assertEquals("AAA BBB", output);
+
+ input = "AAA.12";
+ output = input.replaceAll("_", " ").replaceAll(matchRegex, replacement);
+ Assert.assertEquals("AAA 12", output);
+
+ input = "100.BBB";
+ output = input.replaceAll("_", " ").replaceAll(matchRegex, replacement);
+ Assert.assertEquals("100 BBB", output);
+
+ // No replacement for number string
+ input = "100.12";
+ output = input.replaceAll("_", " ").replaceAll(matchRegex, replacement);
+ Assert.assertEquals(input, output);
}
@Test
@@ -141,6 +227,7 @@ public class IndicesServiceTest {
when(restHighLevelClient.indices()).thenReturn(indicesClient);
when(indicesClient.create(any(CreateIndexRequest.class))).thenReturn(indexResponse);
+ when(customIndexAnalyzerSetting.isEnabled()).thenReturn(false);
boolean response = this.sut.createIndex(restHighLevelClient, index, null, new HashMap<>());
assertFalse(response);
verify(this.indicesExistCache, times(0)).put(any(), any());
@@ -160,6 +247,7 @@ public class IndicesServiceTest {
when(restHighLevelClient.indices()).thenReturn(indicesClient);
when(indicesClient.create(any(CreateIndexRequest.class))).thenThrow(existsException);
+ when(customIndexAnalyzerSetting.isEnabled()).thenReturn(false);
boolean response = this.sut.createIndex(restHighLevelClient, index, null, new HashMap<>());
assertTrue(response);
verify(this.indicesExistCache, times(1)).put(any(), any());
@@ -285,7 +373,7 @@ public class IndicesServiceTest {
" \"creation.date\": \"1545912860994\"" +
" }" +
"]";
- Request request = new Request("GET", "/_cat/indices/*,-.*?h=index,docs.count,creation.date&s=docs.count:asc&format=json");
+ Request request = new Request("GET", "/_cat/indices/*,-.*?h=index,health,docs.count,creation.date&s=docs.count:asc&format=json");
StringEntity entity = new StringEntity(responseJson, ContentType.APPLICATION_JSON);
when(this.restHighLevelClient._transport()).thenReturn(this.restClientTransport);
when(this.restClientTransport.restClient()).thenReturn(this.restClient);
@@ -311,7 +399,7 @@ public class IndicesServiceTest {
" \"creation.date\": \"1545912868416\"" +
" }" +
"]";
- Request request = new Request("GET", "/_cat/indices/tenant1-aapg-*?h=index,docs.count,creation.date&format=json");
+ Request request = new Request("GET", "/_cat/indices/tenant1-aapg-*?h=index,health,docs.count,creation.date&format=json");
StringEntity entity = new StringEntity(responseJson, ContentType.APPLICATION_JSON);
when(this.restHighLevelClient._transport()).thenReturn(this.restClientTransport);
when(this.restClientTransport.restClient()).thenReturn(this.restClient);
@@ -377,10 +465,20 @@ public class IndicesServiceTest {
BooleanResponse booleanResponse = new BooleanResponse(true);
doReturn(booleanResponse).when(indicesClient).exists(any(ExistsRequest.class));
- HealthResponse healthResponse = mock(HealthResponse.class);
- when(healthResponse.status()).thenReturn(HealthStatus.Green);
- doReturn(clusterClient).when(restHighLevelClient).cluster();
- doReturn(healthResponse).when(clusterClient).health(any(HealthRequest.class));
+ String responseJson = "[" +
+ " {" +
+ " \"index\": \"anyIndex\"," +
+ " \"health\": \"yellow\"," +
+ " \"docs.count\": \"92\"," +
+ " \"creation.date\": \"1545912860994\"" +
+ " }" +
+ "]";
+ Request request = new Request("GET", "/_cat/indices/anyIndex?h=index,health,docs.count,creation.date&format=json");
+ StringEntity entity = new StringEntity(responseJson, ContentType.APPLICATION_JSON);
+ when(this.restHighLevelClient._transport()).thenReturn(this.restClientTransport);
+ when(this.restClientTransport.restClient()).thenReturn(this.restClient);
+ when(this.restClient.performRequest(request)).thenReturn(response);
+ when(this.response.getEntity()).thenReturn(entity);
boolean result = this.sut.isIndexReady(restHighLevelClient, "anyIndex");
@@ -388,4 +486,31 @@ public class IndicesServiceTest {
verify(this.indicesExistCache, times(1)).get("anyIndex");
verify(this.indicesExistCache, times(1)).put("anyIndex", true);
}
+
+ @Test(expected = AppException.class)
+ public void should_throwIndexUnavailableException_whenIndexNotReady() throws IOException {
+ when(this.indicesExistCache.get("anyIndex")).thenReturn(false);
+ doReturn(indicesClient).when(restHighLevelClient).indices();
+
+ BooleanResponse booleanResponse = new BooleanResponse(true);
+ doReturn(booleanResponse).when(indicesClient).exists(any(ExistsRequest.class));
+
+ String responseJson = "[" +
+ " {" +
+ " \"index\": \"anyIndex\"," +
+ " \"health\": \"red\"," +
+ " \"docs.count\": \"92\"," +
+ " \"creation.date\": \"1545912860994\"" +
+ " }" +
+ "]";
+ Request request = new Request("GET", "/_cat/indices/anyIndex?h=index,health,docs.count,creation.date&format=json");
+ StringEntity entity = new StringEntity(responseJson, ContentType.APPLICATION_JSON);
+ when(this.restHighLevelClient._transport()).thenReturn(this.restClientTransport);
+ when(this.restClientTransport.restClient()).thenReturn(this.restClient);
+ when(this.restClient.performRequest(request)).thenReturn(response);
+ when(this.response.getEntity()).thenReturn(entity);
+
+ this.sut.isIndexReady(restHighLevelClient, "anyIndex");
+ }
+
}
diff --git a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/ReindexServiceTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java
similarity index 78%
rename from provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/ReindexServiceTest.java
rename to indexer-core/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java
index 6cdeb05c1148a68fc681bc1d8da424565e5a8945..947edc2fe790f88628535a213e46db427c69023e 100644
--- a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/ReindexServiceTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java
@@ -12,41 +12,41 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package org.opengroup.osdu.indexer.azure.service;
-
+package org.opengroup.osdu.indexer.service;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
-import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
-import org.mockito.ArgumentMatchers;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.MockedStatic;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.indexer.Records;
+import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.opengroup.osdu.indexer.service.ReindexServiceImpl;
-import org.opengroup.osdu.indexer.service.StorageService;
+import org.opengroup.osdu.indexer.model.SearchRecord;
import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
-import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
import org.mockito.junit.MockitoJUnitRunner;
+import org.opengroup.osdu.indexer.util.QueryUtil;
+import org.opengroup.osdu.indexer.util.SearchClient;
-import java.net.URISyntaxException;
import java.util.*;
import static org.junit.Assert.fail;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mockStatic;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.mockito.MockitoAnnotations.initMocks;
-@Ignore
+
@RunWith(MockitoJUnitRunner.class)
public class ReindexServiceTest {
@@ -56,14 +56,14 @@ public class ReindexServiceTest {
private final String correlationId = UUID.randomUUID().toString();
+ @Mock
+ private IndexerConfigurationProperties configurationProperties;
@Mock
private StorageService storageService;
-
@Mock
- private IndexerConfigurationProperties configurationProperties;
-
+ private SearchClient searchClient;
@Mock
- private Map<String, String> httpHeaders;
+ private QueryUtil queryUtil;
@Mock
private IRequestInfo requestInfo;
@Mock
@@ -76,6 +76,8 @@ public class ReindexServiceTest {
private RecordReindexRequest recordReindexRequest;
private RecordQueryResponse recordQueryResponse;
+ private Map<String, String> httpHeaders;
+
@Before
public void setup() {
initMocks(this);
@@ -89,8 +91,7 @@ public class ReindexServiceTest {
httpHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
httpHeaders.put(DpsHeaders.CORRELATION_ID, correlationId);
DpsHeaders standardHeaders = DpsHeaders.createFromMap(httpHeaders);
- when(requestInfo.getHeaders()).thenReturn(standardHeaders);
- when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(httpHeaders);
+ when(requestInfo.getHeadersWithDwdAuthZ()).thenReturn(standardHeaders);
}
@After
@@ -102,7 +103,7 @@ public class ReindexServiceTest {
public void should_returnNull_givenNullResponseResult_reIndexRecordsTest() {
try {
recordQueryResponse.setResults(null);
- when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
+ when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
String response = sut.reindexKind(recordReindexRequest, false, false);
@@ -116,7 +117,7 @@ public class ReindexServiceTest {
public void should_returnNull_givenEmptyResponseResult_reIndexRecordsTest() {
try {
recordQueryResponse.setResults(new ArrayList<>());
- when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
+ when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
String response = sut.reindexKind(recordReindexRequest, false, false);
@@ -134,9 +135,9 @@ public class ReindexServiceTest {
results.add("test1");
recordQueryResponse.setResults(results);
- when(configurationProperties.getStorageRecordsBatchSize()).thenReturn(1);
+ when(configurationProperties.getStorageRecordsByKindBatchSize()).thenReturn(1);
- when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
+ when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
String taskQueuePayload = sut.reindexKind(recordReindexRequest, false, false);
@@ -152,28 +153,33 @@ public class ReindexServiceTest {
List<String> results = new ArrayList<>();
results.add("test1");
recordQueryResponse.setResults(results);
- when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
+ when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
String taskQueuePayload = sut.reindexKind(recordReindexRequest, false, false);
- Assert.assertEquals(String.format("{\"data\":\"[{\\\"id\\\":\\\"test1\\\",\\\"kind\\\":\\\"tenant:test:test:1.0.0\\\",\\\"op\\\":\\\"create\\\"}]\",\"attributes\":{\"slb-correlation-id\":\"%s\"}}", correlationId), taskQueuePayload);
+ Assert.assertEquals(String.format("{\"data\":\"[{\\\"id\\\":\\\"test1\\\",\\\"kind\\\":\\\"tenant:test:test:1.0.0\\\",\\\"op\\\":\\\"create\\\"}]\",\"attributes\":{\"correlation-id\":\"%s\"}}", correlationId), taskQueuePayload);
} catch (Exception e) {
fail("Should not throw exception" + e.getMessage());
}
}
@Test
- public void should_createReindexTaskForValidRecords_givenValidRecordIds_reIndexRecordsTest() throws URISyntaxException {
+ public void should_createReindexTaskForValidRecords_givenValidRecordIds_reIndexRecordsTest() throws Exception {
DpsHeaders headers = new DpsHeaders();
when(requestInfo.getHeadersWithDwdAuthZ()).thenReturn(headers);
- when(configurationProperties.getStorageRecordsBatchSize()).thenReturn(2);
List<String> recordIds = Arrays.asList("id1", "id2");
when(storageService.getStorageRecords(recordIds)).thenReturn(
Records.builder().records(Collections.singletonList(Records.Entity.builder().id("id1").kind("kind1").build())).notFound(Collections.singletonList("id2")).build()
);
+ SearchRecord deletedRecord = new SearchRecord();
+ deletedRecord.setKind("kind2");
+ deletedRecord.setId("id2");
+ List<SearchRecord> mockNotFoundRecordsSearchResponse = List.of(deletedRecord);
+ when(searchClient.search(anyString(), any(), any(), any(), anyInt())).thenReturn(mockNotFoundRecordsSearchResponse);
Records records = sut.reindexRecords(recordIds);
Assert.assertEquals(1, records.getRecords().size());
Assert.assertEquals(1, records.getNotFound().size());
verify(indexerQueueTaskBuilder).createWorkerTask("{\"data\":\"[{\\\"id\\\":\\\"id1\\\",\\\"kind\\\":\\\"kind1\\\",\\\"op\\\":\\\"create\\\"}]\",\"attributes\":{}}", 0L, headers);
+ verify(indexerQueueTaskBuilder).createWorkerTask("{\"data\":\"[{\\\"id\\\":\\\"id2\\\",\\\"kind\\\":\\\"kind2\\\",\\\"op\\\":\\\"delete\\\"}]\",\"attributes\":{}}", 0L, headers);
}
}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/SearchServiceImplTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/SearchServiceImplTest.java
deleted file mode 100644
index a299529d394569bebf2c66a8e4963229535876cb..0000000000000000000000000000000000000000
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/SearchServiceImplTest.java
+++ /dev/null
@@ -1,155 +0,0 @@
-package org.opengroup.osdu.indexer.service;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.opengroup.osdu.core.common.http.IUrlFetchService;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.http.HttpResponse;
-import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.opengroup.osdu.indexer.model.SearchRequest;
-import org.opengroup.osdu.indexer.model.SearchResponse;
-import org.mockito.junit.MockitoJUnitRunner;
-
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.when;
-
-import java.net.URISyntaxException;
-
-@RunWith(MockitoJUnitRunner.class)
-public class SearchServiceImplTest {
- @InjectMocks
- private SearchServiceImpl sut;
-
- @Mock
- private IUrlFetchService urlFetchService;
- @Mock
- private IRequestInfo requestInfo;
- @Mock
- private IndexerConfigurationProperties configurationProperties;
- @Mock
- private JaxRsDpsLog jaxRsDpsLog;
-
- private String searchHost = "http://localhost";
-
- @Test
- public void query_without_searchHostSetting() throws URISyntaxException {
- when(this.configurationProperties.getSearchHost()).thenReturn(null);
- SearchResponse response = sut.query(new SearchRequest());
- Assert.assertNotNull(response);
- Assert.assertNull(response.getResults());
- }
-
- @Test
- public void query_with_responseCode_OK() throws URISyntaxException {
- String bodyJson = "{\n" +
- " \"results\": [\n" +
- " {\n" +
- " \"data\": {\n" +
- " \"FacilityName\": \"A123\"\n" +
- " },\n" +
- " \"kind\": \"osdu:wks:master-data--Wellbore:1.0.0\"\n" +
- " },\n" +
- " {\n" +
- " \"data\": {\n" +
- " \"FacilityName\": \"B123\"\n" +
- " },\n" +
- " \"kind\": \"osdu:wks:master-data--Wellbore:1.0.0\"\n" +
- " }\n" +
- " ],\n" +
- " \"aggregations\": null,\n" +
- " \"totalCount\": 10000\n" +
- "}";
- HttpResponse response = new HttpResponse();
- response.setResponseCode(200);
- response.setBody(bodyJson);
- when(this.configurationProperties.getSearchHost()).thenReturn(searchHost);
- when(this.urlFetchService.sendRequest(any())).thenReturn(response);
- SearchResponse searchResponse = sut.query(new SearchRequest());
- Assert.assertNotNull(searchResponse);
- Assert.assertEquals(10000,searchResponse.getTotalCount());
- Assert.assertEquals(2,searchResponse.getResults().size());
- }
-
- @Test
- public void query_with_cursor_with_responseCode_OK() throws URISyntaxException {
- String bodyJson = "{\n" +
- " \"cursor\": \"509E144E7F9B81F8148327D6CB73BB6F\",\n" +
- " \"results\": [\n" +
- " {\n" +
- " \"kind\": \"osdu:wks:master-data--Wellbore:1.0.0\"\n" +
- " },\n" +
- " {\n" +
- " \"kind\": \"osdu:wks:master-data--Wellbore:1.0.1\"\n" +
- " }\n" +
- " ],\n" +
- " \"totalCount\": 1000\n" +
- "}";
- HttpResponse response = new HttpResponse();
- response.setResponseCode(200);
- response.setBody(bodyJson);
- when(this.configurationProperties.getSearchHost()).thenReturn(searchHost);
- when(this.urlFetchService.sendRequest(any())).thenReturn(response);
- SearchResponse searchResponse = sut.query(new SearchRequest());
- Assert.assertNotNull(searchResponse);
- Assert.assertNotNull(searchResponse.getCursor());
- Assert.assertEquals(1000,searchResponse.getTotalCount());
- Assert.assertEquals(2,searchResponse.getResults().size());
- }
-
- @Test
- public void query_with_responseCode_OK_EmptyResult() throws URISyntaxException {
- String bodyJson = "{\n" +
- " \"results\": [],\n" +
- " \"aggregations\": [],\n" +
- " \"totalCount\": 0\n" +
- "}";
- HttpResponse response = new HttpResponse();
- response.setResponseCode(200);
- response.setBody(bodyJson);
- when(this.configurationProperties.getSearchHost()).thenReturn(searchHost);
- when(this.urlFetchService.sendRequest(any())).thenReturn(response);
- SearchResponse searchResponse = sut.query(new SearchRequest());
- Assert.assertNotNull(searchResponse);
- Assert.assertEquals(0,searchResponse.getTotalCount());
- Assert.assertEquals(0,searchResponse.getResults().size());
- }
-
- @Test
- public void query_with_responseCode_BadRequest() throws URISyntaxException {
- String bodyJson = "{\n" +
- " \"code\": 400,\n" +
- " \"reason\": \"Bad Request\",\n" +
- " \"message\": \"Invalid parameters were given on search request\"\n" +
- "}";
- HttpResponse response = new HttpResponse();
- response.setResponseCode(200);
- response.setBody(bodyJson);
- when(this.configurationProperties.getSearchHost()).thenReturn(searchHost);
- when(this.urlFetchService.sendRequest(any())).thenReturn(response);
- SearchResponse searchResponse = sut.query(new SearchRequest());
- Assert.assertNotNull(searchResponse);
- Assert.assertNull(searchResponse.getResults());
- }
-
- @Test
- public void query_with_null_response() throws URISyntaxException {
- when(this.configurationProperties.getSearchHost()).thenReturn(searchHost);
- when(this.urlFetchService.sendRequest(any())).thenReturn(null);
- SearchResponse searchResponse = sut.query(new SearchRequest());
- Assert.assertNotNull(searchResponse);
- Assert.assertNull(searchResponse.getResults());
- }
-
- @Test
- public void query_with_exception() throws URISyntaxException {
- when(this.configurationProperties.getSearchHost()).thenReturn(searchHost);
- when(this.urlFetchService.sendRequest(any())).thenThrow(new AppException(415, "upstream server responded with unsupported media type: text/plain", "Unsupported media type" ));
- assertThrows(URISyntaxException.class, () -> sut.query(new SearchRequest()));
- }
-}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceImplTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceImplTest.java
index d3f20cfb137ff78f17dee699a708b3a2eb66a4d1..25385fb6e9d63f002f8cf9f740138c413cea6583 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceImplTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/StorageServiceImplTest.java
@@ -14,9 +14,27 @@
package org.opengroup.osdu.indexer.service;
+import static java.util.Collections.singletonList;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.fail;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
+import java.lang.reflect.Type;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -39,23 +57,10 @@ import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
import org.opengroup.osdu.core.common.model.indexer.Records;
import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.opengroup.osdu.indexer.model.XcollaborationHolder;
import org.springframework.http.HttpStatus;
import org.springframework.test.context.junit4.SpringRunner;
-import java.lang.reflect.Type;
-import java.net.URISyntaxException;
-import java.util.*;
-
-import static java.util.Collections.singletonList;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.fail;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
@RunWith(SpringRunner.class)
public class StorageServiceImplTest {
@Mock
@@ -70,6 +75,8 @@ public class StorageServiceImplTest {
private IRequestInfo requestInfo;
@Mock
private IndexerConfigurationProperties configurationProperties;
+ @Mock
+ private XcollaborationHolder xcollaborationHolder;
@Spy
private ObjectMapper objectMapper = new ObjectMapper();
@InjectMocks
@@ -98,6 +105,7 @@ public class StorageServiceImplTest {
ids = Arrays.asList(RECORD_ID1, RECORDS_ID2);
when(configurationProperties.getStorageRecordsBatchSize()).thenReturn(20);
+ when(xcollaborationHolder.isFeatureEnabledAndHeaderExists()).thenReturn(false);
}
@Test
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/testutils/ReflectionTestUtil.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/testutils/ReflectionTestUtil.java
new file mode 100644
index 0000000000000000000000000000000000000000..51ae2218aeb15b468f62f160f57b84cb92b866f3
--- /dev/null
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/testutils/ReflectionTestUtil.java
@@ -0,0 +1,14 @@
+package org.opengroup.osdu.indexer.testutils;
+
+import org.springframework.util.ReflectionUtils;
+
+import java.lang.reflect.Field;
+
+public class ReflectionTestUtil {
+
+ public static <T> void setFieldValueForClass(T targetObject, String fieldName, Object fieldValue) {
+ Field field = ReflectionUtils.findField(targetObject.getClass(), fieldName);
+ field.setAccessible(true);
+ ReflectionUtils.setField(field, targetObject, fieldValue);
+ }
+}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/CustomIndexAnalyzerSettingTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/CustomIndexAnalyzerSettingTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..a73d5aa165b3c2b33d38417e889c9e0ef0ff33cb
--- /dev/null
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/CustomIndexAnalyzerSettingTest.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.util;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.when;
+
+@RunWith(SpringRunner.class)
+public class CustomIndexAnalyzerSettingTest {
+ @Mock
+ private BooleanFeatureFlagClient booleanFeatureFlagClient;
+
+ @InjectMocks
+ private CustomIndexAnalyzerSetting customIndexAnalyzerSetting;
+
+ @Test
+ public void isEnabled_is_true() {
+ when(booleanFeatureFlagClient.isEnabled(eq("custom-index-analyzer-enabled"), anyBoolean())).thenReturn(true);
+ Assert.assertTrue(customIndexAnalyzerSetting.isEnabled());
+ }
+
+ @Test
+ public void isEnabled_is_false() {
+ when(booleanFeatureFlagClient.isEnabled(eq("custom-index-analyzer-enabled"), anyBoolean())).thenReturn(false);
+ Assert.assertFalse(customIndexAnalyzerSetting.isEnabled());
+ }
+}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/ElasticClientHandlerTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/ElasticClientHandlerTest.java
index 30ce3133c9b3ca13f824aea01e966194a106f90e..0080bd4a1333b427426256159bb91c18d5c79f2d 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/ElasticClientHandlerTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/ElasticClientHandlerTest.java
@@ -33,10 +33,12 @@ import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.MockedStatic;
import org.mockito.junit.MockitoJUnitRunner;
+import org.opengroup.osdu.core.common.cache.ICache;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.indexer.IElasticSettingService;
import org.opengroup.osdu.core.common.model.search.ClusterSettings;
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
@RunWith(MockitoJUnitRunner.class)
@@ -49,14 +51,16 @@ public class ElasticClientHandlerTest {
private IndexerConfigurationProperties configurationProperties;
@Mock
private IElasticSettingService elasticSettingService;
-
@Mock
private RestClientBuilder builder;
-
@Mock
private RestClient restClient;
@Mock
private JaxRsDpsLog log;
+ @Mock
+ private ICache<String, ElasticsearchClient> clientCache;
+ @Mock
+ private TenantInfo tenantInfo;
@InjectMocks
private ElasticClientHandler elasticClientHandler;
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/PropertyUtilTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/PropertyUtilTest.java
index cb5e2a75da807db7f042f64856d96f5a264fbece..256da5397b397fd70f6c5942a662d852c9c02e96 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/PropertyUtilTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/PropertyUtilTest.java
@@ -18,18 +18,21 @@ package org.opengroup.osdu.indexer.util;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.reflect.TypeToken;
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.lang.reflect.Type;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import lombok.SneakyThrows;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.test.context.junit4.SpringRunner;
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.lang.reflect.Type;
-import java.util.*;
-
@RunWith(SpringRunner.class)
public class PropertyUtilTest {
private static final Gson gson = new Gson();
@@ -71,9 +74,9 @@ public class PropertyUtilTest {
Assert.assertTrue(value.containsKey(propertyName));
Assert.assertNotNull(value.get(propertyName));
}
+ // The null value is ignored
for(String propertyName : propertyNamesWithNullValues) {
- Assert.assertTrue(value.containsKey(propertyName));
- Assert.assertNull(value.get(propertyName));
+ Assert.assertFalse(value.containsKey(propertyName));
}
value = PropertyUtil.getValueOfNoneNestedProperty("Curves[].CurveID", data);
@@ -335,6 +338,10 @@ public class PropertyUtilTest {
List<String> expectedChangedWellProperties = Arrays.asList("VirtualProperties.DefaultName", "VerticalMeasurements[].VerticalMeasurementID", "FacilityName");
changedProperties.forEach(p -> Assert.assertTrue(expectedChangedWellProperties.contains(p)));
+ dataMapRight = getDataMap("well_no_null.json");
+ changedProperties = PropertyUtil.getChangedProperties(dataMapLeft, dataMapRight);
+ Assert.assertTrue(changedProperties.isEmpty());
+
dataMapLeft = getDataMap("wellLog.json");
dataMapRight = getDataMap("wellLog2.json");
changedProperties = PropertyUtil.getChangedProperties(dataMapLeft, dataMapRight);
@@ -342,6 +349,19 @@ public class PropertyUtilTest {
changedProperties.forEach(p -> Assert.assertTrue(expectedChangedWellLogProperties.contains(p)));
}
+ @Test
+ public void getChangedProperties_should_not_modify_originalObjects() {
+ Map<String, Object> dataMapLeft = getDataMap("well.json");
+ Map<String, Object> dataMapRight = getDataMap("well2.json");
+ String dataMapLeftText = dataMapLeft.toString();
+ String dataMapRightText = dataMapRight.toString();
+
+ PropertyUtil.getChangedProperties(dataMapLeft, dataMapRight);
+
+ Assert.assertEquals(dataMapLeftText, dataMapLeft.toString());
+ Assert.assertEquals(dataMapRightText, dataMapRightText.toString());
+ }
+
private Map<String, Object> getDataMap(String file) {
String jsonText = getJsonFromFile(file);
Type type = new TypeToken<Map<String, Object>>() {}.getType();
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/QueryUtilTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/QueryUtilTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..e72e823ee19d7c4f1671245b9fad373a45aeacce
--- /dev/null
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/QueryUtilTest.java
@@ -0,0 +1,112 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.util;
+
+import co.elastic.clients.elasticsearch._types.SortOptions;
+import co.elastic.clients.elasticsearch._types.SortOrder;
+import co.elastic.clients.elasticsearch._types.query_dsl.Query;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.List;
+
+@RunWith(MockitoJUnitRunner.class)
+public class QueryUtilTest {
+ @Test
+ public void createQueryForParentConfigs() {
+ String childKind = "osdu:wks:work-product-component--WellLog:1.";
+ Query actualQuery = QueryUtil.createQueryForParentConfigs(childKind);
+ Query expectedQuery = getExpectedQuery("parents_configuration_query");
+ Assert.assertEquals(expectedQuery.toString(), actualQuery.toString());
+ }
+
+ @Test
+ public void createQueryForChildrenConfigs() {
+ String parentKind = "osdu:wks:master-data--Wellbore:1.";
+ Query actualQuery = QueryUtil.createQueryForChildrenConfigs(parentKind);
+ Query expectedQuery = getExpectedQuery("children_configuration_query");
+ Assert.assertEquals(expectedQuery.toString(), actualQuery.toString());
+ }
+
+ @Test
+ public void createQueryForConfigurations() {
+ String kind = "osdu:wks:work-product-component--WellLog:1.";
+ Query actualQuery = QueryUtil.createQueryForConfigurations(kind);
+ Query expectedQuery = getExpectedQuery("configuration_query");
+ Assert.assertEquals(expectedQuery.toString(), actualQuery.toString());
+ }
+
+ @Test
+ public void createSimpleTextQuery() {
+ String queryString = "id:(\"id1\" OR \"id2\")";
+ Query actualQuery = QueryUtil.createSimpleTextQuery(queryString);
+ Query expectedQuery = getExpectedQuery("simple_query");
+ Assert.assertEquals(expectedQuery.toString(), actualQuery.toString());
+ }
+
+ @Test
+ public void createIdsFilter_with_emptyIdList() {
+ Assert.assertEquals("", QueryUtil.createIdsFilter(new ArrayList<String>()));
+ }
+
+ @Test
+ public void createIdsFilter_with_idList() {
+ Assert.assertEquals("\"id1\" OR \"id2\"", QueryUtil.createIdsFilter(List.of("id1", "id2:")));
+ }
+
+ @Test
+ public void createSortOptionsList() throws Exception {
+ List<SortOptions> sortOptionsList =
+ QueryUtil.createSortOptionsList(List.of("id", "data.text.keyword"), List.of(SortOrder.Asc, SortOrder.Desc));
+ Assert.assertEquals(2, sortOptionsList.size());
+ Assert.assertEquals("id", sortOptionsList.get(0).field().field());
+ Assert.assertEquals(SortOrder.Asc, sortOptionsList.get(0).field().order());
+ Assert.assertEquals("data.text.keyword", sortOptionsList.get(1).field().field());
+ Assert.assertEquals(SortOrder.Desc, sortOptionsList.get(1).field().order());
+ }
+
+ @Test
+ public void createSortOptionsList_with_empty_fields() throws Exception {
+ List<SortOptions> sortOptionsList =
+ QueryUtil.createSortOptionsList(new ArrayList<>(), new ArrayList<>());
+ Assert.assertNull(sortOptionsList);
+ }
+
+ @Test
+ public void createSortOptionsList_with_null_fields() throws Exception {
+ List<SortOptions> sortOptionsList =
+ QueryUtil.createSortOptionsList(null, null);
+ Assert.assertNull(sortOptionsList);
+ }
+
+ @Test
+ public void createSortOptionsList_throws_exception_whenUnmatched_field_order_size() {
+ Assert.assertThrows(Exception.class, () -> QueryUtil.createSortOptionsList(List.of("id", "data.text.keyword"), List.of(SortOrder.Asc)));
+ }
+
+ private Query getExpectedQuery(String queryFile) {
+ InputStream inStream = this.getClass().getResourceAsStream("/queryutil/" + queryFile + ".json");
+ BufferedReader br = new BufferedReader(new InputStreamReader(inStream));
+ Query.Builder builder = new Query.Builder();
+ return builder.withJson(br).build();
+ }
+}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/SearchClientTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/SearchClientTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..4365b951c20bd1ed2867e4895f7652d11eff1748
--- /dev/null
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/SearchClientTest.java
@@ -0,0 +1,260 @@
+/*
+ * Copyright © Schlumberger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.util;
+
+import co.elastic.clients.elasticsearch.ElasticsearchClient;
+import co.elastic.clients.elasticsearch._types.FieldValue;
+import co.elastic.clients.elasticsearch._types.SortOptions;
+import co.elastic.clients.elasticsearch._types.SortOrder;
+import co.elastic.clients.elasticsearch._types.query_dsl.Query;
+import co.elastic.clients.elasticsearch.core.*;
+import co.elastic.clients.elasticsearch.core.search.Hit;
+import co.elastic.clients.elasticsearch.core.search.HitsMetadata;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
+import org.opengroup.osdu.indexer.model.SearchRecord;
+import java.lang.reflect.Type;
+import java.util.*;
+import static org.junit.Assert.*;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.*;
+
+@RunWith(MockitoJUnitRunner.class)
+public class SearchClientTest {
+
+ @InjectMocks
+ SearchClient sut;
+
+ @Mock
+ private ElasticClientHandler elasticClientHandler;
+
+ @Mock
+ private ElasticIndexNameResolver elasticIndexNameResolver;
+
+ @Mock
+ private JaxRsDpsLog logger;
+
+ @Mock
+ private ElasticsearchClient client;
+
+ @Mock
+ private Query query;
+
+ @Mock
+ private HitsMetadata<Map<String, Object>> searchHits;
+
+ @Mock
+ private Hit<Map<String, Object>> searchHit;
+
+ private static final String kind = "a:b:c:1.0.0";
+
+ private static final String pitId = "pitId";
+
+ @Before
+ public void init() {
+ doReturn(client).when(elasticClientHandler).getOrCreateRestClient();
+ when(elasticIndexNameResolver.getIndexNameFromKind(anyString())).thenAnswer(invocation -> {
+ String kind = invocation.getArgument(0);
+ return kind.replace(":", "-").toLowerCase();
+ });
+ }
+
+ @Test
+ public void search_with_normalQuery_whenSearchHitsIsNotEmpty() throws Exception {
+ List<Hit<Map<String, Object>>> hits = new ArrayList<>();
+ hits.add(searchHit);
+ Map<String, Object> hitFields = new HashMap<>();
+
+ SearchResponse searchResponse = mock(SearchResponse.class);
+ doReturn(searchResponse).when(client).search(any(SearchRequest.class), eq((Type)Map.class));
+ doReturn(searchHits).when(searchResponse).hits();
+ doReturn(hits).when(searchHits).hits();
+ doReturn(hitFields).when(searchHit).source();
+
+ // act
+ List<SearchRecord> records = sut.search(kind, query, null, null, -1);
+
+ // assert
+ ArgumentCaptor<SearchRequest> searchRequestArgumentCaptor = ArgumentCaptor.forClass(SearchRequest.class);
+ verify(client, times(1)).search(searchRequestArgumentCaptor.capture(), eq((Type)Map.class));
+ verify(client, times(0)).openPointInTime(any(OpenPointInTimeRequest.class));
+ SearchRequest searchRequest = searchRequestArgumentCaptor.getValue();
+ assertNull(searchRequest.pit());
+ assertEquals(searchRequest.sort().size(), 0);
+ assertEquals(searchRequest.source().filter().includes().size(), 0);
+ assertEquals(records.size(), 1);
+ }
+
+ @Test
+ public void search_with_whenSortOptionsAndReturnedFieldsAreNotEmpty() throws Exception {
+ List<Hit<Map<String, Object>>> hits = new ArrayList<>();
+ hits.add(searchHit);
+ Map<String, Object> hitFields = new HashMap<>();
+
+ SearchResponse searchResponse = mock(SearchResponse.class);
+ doReturn(searchResponse).when(client).search(any(SearchRequest.class), eq((Type)Map.class));
+ doReturn(searchHits).when(searchResponse).hits();
+ doReturn(hits).when(searchHits).hits();
+ doReturn(hitFields).when(searchHit).source();
+ List<SortOptions> sortOptions = List.of(SortOptions.of(so -> so.score(s -> s.order(SortOrder.Desc))));
+ List<String> returnedFields = List.of("id", "kind");
+
+ // act
+ List<SearchRecord> records = sut.search(kind, query, sortOptions, returnedFields, -1);
+
+ // assert
+ ArgumentCaptor<SearchRequest> searchRequestArgumentCaptor = ArgumentCaptor.forClass(SearchRequest.class);
+ verify(client, times(1)).search(searchRequestArgumentCaptor.capture(), eq((Type)Map.class));
+ verify(client, times(0)).openPointInTime(any(OpenPointInTimeRequest.class));
+ SearchRequest searchRequest = searchRequestArgumentCaptor.getValue();
+ assertEquals(searchRequest.sort(), sortOptions);
+ assertEquals(searchRequest.source().filter().includes().size(), 2);
+ assertTrue(searchRequest.source().filter().includes().contains("id"));
+ assertTrue(searchRequest.source().filter().includes().contains("kind"));
+ assertNull(searchRequest.pit());
+ assertEquals(records.size(), 1);
+ }
+
+ @Test
+ public void search_with_queryWithPIT_whenSortOptionsIsNotEmpty() throws Exception {
+ List<SortOptions> sortOptions = List.of(SortOptions.of(so -> so.score(s -> s.order(SortOrder.Desc))));
+ List<String> returnedFields = List.of("id", "kind");
+
+ int totalRecordCount = 5100;
+ prepare_search_with_queryWithPIT(totalRecordCount);
+
+ // act
+ List<SearchRecord> records = sut.search(kind, query, sortOptions, returnedFields, -1);
+
+ // assert
+ ArgumentCaptor<SearchRequest> searchRequestArgumentCaptor = ArgumentCaptor.forClass(SearchRequest.class);
+ verify(client, times(3)).search(searchRequestArgumentCaptor.capture(), eq((Type)Map.class));
+ verify(client, times(1)).openPointInTime(any(OpenPointInTimeRequest.class));
+ verify(client, times(1)).closePointInTime(any(ClosePointInTimeRequest.class));
+ SearchRequest searchRequest = searchRequestArgumentCaptor.getValue();
+ assertEquals(searchRequest.sort(), sortOptions);
+ assertEquals(searchRequest.source().filter().includes().size(), 2);
+ assertTrue(searchRequest.source().filter().includes().contains("id"));
+ assertTrue(searchRequest.source().filter().includes().contains("kind"));
+ assertEquals(searchRequest.pit().id(), pitId);
+ assertEquals(records.size(), totalRecordCount);
+ }
+
+ @Test
+ public void search_with_queryWithPIT_whenSortOptionsIsEmpty() throws Exception {
+ List<String> returnedFields = List.of("id", "kind");
+
+ int totalRecordCount = 15000;
+ prepare_search_with_queryWithPIT(totalRecordCount);
+
+ // act
+ List<SearchRecord> records = sut.search(kind, query, null, returnedFields, -1);
+
+ // assert
+ ArgumentCaptor<SearchRequest> searchRequestArgumentCaptor = ArgumentCaptor.forClass(SearchRequest.class);
+ verify(client, times(5)).search(searchRequestArgumentCaptor.capture(), eq((Type)Map.class));
+ verify(client, times(1)).openPointInTime(any(OpenPointInTimeRequest.class));
+ verify(client, times(1)).closePointInTime(any(ClosePointInTimeRequest.class));
+ SearchRequest searchRequest = searchRequestArgumentCaptor.getValue();
+ List<SortOptions> sortOptions = searchRequest.sort();
+ assertEquals(sortOptions.size(), 1);
+ assertEquals(searchRequest.source().filter().includes().size(), 2);
+ assertTrue(searchRequest.source().filter().includes().contains("id"));
+ assertTrue(searchRequest.source().filter().includes().contains("kind"));
+ assertEquals(searchRequest.pit().id(), pitId);
+ assertEquals(records.size(), totalRecordCount);
+ }
+
+ private void prepare_search_with_queryWithPIT(int totalRecordCount) throws Exception {
+ List<List<Hit<Map<String, Object>>>> batches = new ArrayList<>();
+ List<Hit<Map<String, Object>>> hits = new ArrayList<>();
+ for(int i = 0; i < totalRecordCount; i++) {
+ if(i % 5000 == 0 && i > 0) {
+ batches.add(hits);
+ hits = new ArrayList<>();
+ }
+ hits.add(searchHit);
+ }
+ if(!hits.isEmpty())
+ batches.add(hits);
+ int batchCount = batches.size();
+
+ Map<String, Object> hitFields = new HashMap<>();
+ Map<String, Integer> searchCallsCount = new HashMap<>();
+ searchCallsCount.put("Count", 0);
+ List<FieldValue> fieldValues = new ArrayList<>();
+
+ OpenPointInTimeResponse openResponse = mock(OpenPointInTimeResponse.class);
+ SearchResponse searchResponse = mock(SearchResponse.class);
+ doReturn(openResponse).when(client).openPointInTime(any(OpenPointInTimeRequest.class));
+ when(client.search(any(SearchRequest.class), eq((Type)Map.class))).thenAnswer(
+ invocationOnMock -> {
+ searchCallsCount.put("Count", searchCallsCount.get("Count") + 1);
+ return searchResponse;
+ });
+ doReturn(pitId).when(openResponse).id();
+ doReturn(searchHits).when(searchResponse).hits();
+ when(searchHits.hits()).thenAnswer(invocation -> {
+ // First call is normal query
+ // The second and the rest of the calls are queries with search_after and PIT
+ int searchCalls = searchCallsCount.get("Count");
+ if(searchCalls < 3) {
+ return batches.get(0);
+ }
+ else {
+ if(batches.size() > 0 && batches.size() + searchCalls >= batchCount + 3) {
+ batches.remove(0); // Remove the top batch from the last search
+ }
+ if(batches.size() > 0) {
+ return batches.get(0);
+ }
+ else {
+ return new ArrayList<>();
+ }
+ }
+
+ });
+ doReturn(hitFields).when(searchHit).source();
+ doReturn(fieldValues).when(searchHit).sort();
+ }
+
+ @Test
+ public void search_throws_exception_whenKindIsNullOrEmpty() {
+ Assert.assertThrows(Exception.class, () -> sut.search((String)null, query, null, null, -1));
+ Assert.assertThrows(Exception.class, () -> sut.search("", query, null, null, -1));
+ }
+
+ @Test
+ public void search_throws_exception_whenKindsAreNullOrEmpty() {
+ Assert.assertThrows(Exception.class, () -> sut.search((List)null, query, null, null, -1));
+ Assert.assertThrows(Exception.class, () -> sut.search(new ArrayList<>(), query, null, null, -1));
+ }
+
+ @Test
+ public void search_throws_exception_whenQueryIsNull() {
+ Assert.assertThrows(Exception.class, () -> sut.search(kind, null, null, null, -1));
+ Assert.assertThrows(Exception.class, () -> sut.search(List.of(kind), null, null, null, -1));
+ }
+}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/TypeMapperTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/TypeMapperTest.java
index 518804bd6f29658a196e297efca0477b182deed6..05326ef3c7acef13fe72f4e5285b053495fabb0f 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/TypeMapperTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/TypeMapperTest.java
@@ -13,8 +13,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
public class TypeMapperTest {
@@ -135,14 +134,21 @@ public class TypeMapperTest {
@Test
public void validate_dataAttribute_ofTextType_indexerMapping() {
- Object textMapping = TypeMapper.getDataAttributeIndexerMapping(ElasticType.TEXT.getValue(), false, false);
+ Object textMapping = TypeMapper.getDataAttributeIndexerMapping(ElasticType.TEXT.getValue(), false, false, false);
Map<String, Object> textAttributeIndexMapping = (Map<String, Object>) textMapping;
- verifyTextIndexMapping(textAttributeIndexMapping);
+ verifyTextIndexMapping_without_customAnalyzer_enabled(textAttributeIndexMapping);
+ }
+
+ @Test
+ public void validate_dataAttribute_ofTextType_indexerMapping_with_customAnalyzer_enabled() {
+ Object textMapping = TypeMapper.getDataAttributeIndexerMapping(ElasticType.TEXT.getValue(), false, false, true);
+ Map<String, Object> textAttributeIndexMapping = (Map<String, Object>) textMapping;
+ verifyTextIndexMapping_with_customAnalyzer_enabled(textAttributeIndexMapping);
}
@Test
public void validate_dataAttribute_ofKeywordType_indexerMapping() {
- Object keywordMapping = TypeMapper.getDataAttributeIndexerMapping(ElasticType.KEYWORD.getValue(), false, false);
+ Object keywordMapping = TypeMapper.getDataAttributeIndexerMapping(ElasticType.KEYWORD.getValue(), false, false, false);
Map<String, Object> keywordIndexMapping = (Map<String, Object>) keywordMapping;
assertEquals("keyword", keywordIndexMapping.get("type"));
}
@@ -157,7 +163,7 @@ public class TypeMapperTest {
facilityStateDataAttribute.put("type", "nested");
facilityStateDataAttribute.put(Constants.PROPERTIES, nestedAttribute);
- Object nestedDataAttributeMapping = TypeMapper.getDataAttributeIndexerMapping(facilityStateDataAttribute, false, false);
+ Object nestedDataAttributeMapping = TypeMapper.getDataAttributeIndexerMapping(facilityStateDataAttribute, false, false, false);
Map<String, Object> nestedDataAttributeIndexMapping = (Map<String, Object>) nestedDataAttributeMapping;
assertEquals("nested", nestedDataAttributeIndexMapping.get("type"));
Map<String, Object> propertiesIndexMapping = (Map<String, Object>) nestedDataAttributeIndexMapping.get("properties");
@@ -166,7 +172,7 @@ public class TypeMapperTest {
Records.Type effectiveDateTimeIndexType = (Records.Type) propertiesIndexMapping.get("EffectiveDateTime");
assertEquals("date", effectiveDateTimeIndexType.getType());
Map<String, Object> facilityStateIndexMapping = (Map<String, Object>) propertiesIndexMapping.get("FacilityStateTypeID");
- verifyTextIndexMapping(facilityStateIndexMapping);
+ verifyTextIndexMapping_without_customAnalyzer_enabled(facilityStateIndexMapping);
}
@Test
@@ -184,7 +190,7 @@ public class TypeMapperTest {
ElasticType.DOUBLE,
ElasticType.BOOLEAN);
for (ElasticType type : types) {
- Object indexerMapping = TypeMapper.getDataAttributeIndexerMapping(type.getValue(), false, false);
+ Object indexerMapping = TypeMapper.getDataAttributeIndexerMapping(type.getValue(), false, false, false);
Records.Type indexType = (Records.Type) indexerMapping;
assertEquals(type.getValue(), indexType.getType());
}
@@ -192,17 +198,17 @@ public class TypeMapperTest {
@Test
public void validate_dataAttribute_ofArrayType_indexerMapping() {
- Object textArrayMapping = TypeMapper.getDataAttributeIndexerMapping(ElasticType.TEXT_ARRAY.getValue(), false, false);
+ Object textArrayMapping = TypeMapper.getDataAttributeIndexerMapping(ElasticType.TEXT_ARRAY.getValue(), false, false, false);
Map<String, Object> textArrayIndexMapping = (Map<String, Object>) textArrayMapping;
- verifyTextIndexMapping(textArrayIndexMapping);
+ verifyTextIndexMapping_without_customAnalyzer_enabled(textArrayIndexMapping);
- Object keywordArrayMapping = TypeMapper.getDataAttributeIndexerMapping(ElasticType.KEYWORD_ARRAY.getValue(), false, false);
+ Object keywordArrayMapping = TypeMapper.getDataAttributeIndexerMapping(ElasticType.KEYWORD_ARRAY.getValue(), false, false, false);
Map<String, Object> keywordArrayIndexMapping = (Map<String, Object>) keywordArrayMapping;
assertEquals("keyword", keywordArrayIndexMapping.get("type"));
List<ElasticType> types = Arrays.asList(ElasticType.INTEGER_ARRAY, ElasticType.LONG_ARRAY, ElasticType.FLOAT_ARRAY, ElasticType.DOUBLE_ARRAY, ElasticType.BOOLEAN_ARRAY, ElasticType.DATE_ARRAY);
for (ElasticType type : types) {
- Object indexerMapping = TypeMapper.getDataAttributeIndexerMapping(type.getValue(), false, false);
+ Object indexerMapping = TypeMapper.getDataAttributeIndexerMapping(type.getValue(), false, false, false);
Records.Type indexType = (Records.Type) indexerMapping;
assertEquals(StringUtils.substringBefore(type.getValue(), "_"), indexType.getType());
}
@@ -215,16 +221,16 @@ public class TypeMapperTest {
textAttribute.put("name", "text");
Map<String, Object> nestedTextDataAttribute = new HashMap<>();
nestedTextDataAttribute.put(Constants.PROPERTIES, textAttribute);
- Object textMapping = TypeMapper.getDataAttributeIndexerMapping(nestedTextDataAttribute, false, false);
+ Object textMapping = TypeMapper.getDataAttributeIndexerMapping(nestedTextDataAttribute, false, false, false);
Map<String, Object> textPropertiesIndexMapping = (Map<String, Object>) ((Map<String, Object>) textMapping).get("properties");
- verifyTextIndexMapping((Map<String, Object>) textPropertiesIndexMapping.get("name"));
+ verifyTextIndexMapping_without_customAnalyzer_enabled((Map<String, Object>) textPropertiesIndexMapping.get("name"));
// keyword
Map<String, Object> keywordAttribute = new HashMap<>();
keywordAttribute.put("name", "keyword");
Map<String, Object> nestedKeywordDataAttribute = new HashMap<>();
nestedKeywordDataAttribute.put(Constants.PROPERTIES, keywordAttribute);
- Object keywordMapping = TypeMapper.getDataAttributeIndexerMapping(nestedKeywordDataAttribute, false, false);
+ Object keywordMapping = TypeMapper.getDataAttributeIndexerMapping(nestedKeywordDataAttribute, false, false, false);
Map<String, Object> keywordPropertiesIndexMapping = (Map<String, Object>) ((Map<String, Object>) keywordMapping).get("properties");
Map<String, Object> keywordIndexMapping = (Map<String, Object>) keywordPropertiesIndexMapping.get("name");
assertEquals("keyword", keywordIndexMapping.get("type"));
@@ -244,7 +250,7 @@ public class TypeMapperTest {
objectAttribute.put("HistoricalInterests", "object");
Map<String, Object> nestedObjectDataAttribute = new HashMap<>();
nestedObjectDataAttribute.put(Constants.PROPERTIES, objectAttribute);
- Object objectIndexMapping = TypeMapper.getDataAttributeIndexerMapping(nestedObjectDataAttribute, false, false);
+ Object objectIndexMapping = TypeMapper.getDataAttributeIndexerMapping(nestedObjectDataAttribute, false, false, false);
Map<String, Object> objectPropertiesIndexMapping = (Map<String, Object>) ((Map<String, Object>) objectIndexMapping).get("properties");
Records.Type historicalInterestsIndexMapping = (Records.Type) objectPropertiesIndexMapping.get("HistoricalInterests");
assertEquals("object", historicalInterestsIndexMapping.getType());
@@ -254,7 +260,7 @@ public class TypeMapperTest {
flattenedAttribute.put("FacilitySpecifications", "flattened");
Map<String, Object> nestedFlattenedDataAttribute = new HashMap<>();
nestedFlattenedDataAttribute.put(Constants.PROPERTIES, flattenedAttribute);
- Object flattenedIndexMapping = TypeMapper.getDataAttributeIndexerMapping(nestedFlattenedDataAttribute, false, false);
+ Object flattenedIndexMapping = TypeMapper.getDataAttributeIndexerMapping(nestedFlattenedDataAttribute, false, false, false);
Map<String, Object> flattenedPropertiesIndexMapping = (Map<String, Object>) ((Map<String, Object>) flattenedIndexMapping).get("properties");
Records.Type facilitySpecificationsIndexMapping = (Records.Type) flattenedPropertiesIndexMapping.get("FacilitySpecifications");
assertEquals("flattened", facilitySpecificationsIndexMapping.getType());
@@ -265,12 +271,22 @@ public class TypeMapperTest {
textArrayAttribute.put("name", arrayType);
Map<String, Object> nestedTextArrayDataAttribute = new HashMap<>();
nestedTextArrayDataAttribute.put(Constants.PROPERTIES, textArrayAttribute);
- Object textArrayMapping = TypeMapper.getDataAttributeIndexerMapping(nestedTextArrayDataAttribute, false, false);
+ Object textArrayMapping = TypeMapper.getDataAttributeIndexerMapping(nestedTextArrayDataAttribute, false, false, false);
Map<String, Object> textArrayPropertiesIndexMapping = (Map<String, Object>) ((Map<String, Object>) textArrayMapping).get("properties");
Records.Type textArrayType = (Records.Type) textArrayPropertiesIndexMapping.get("name");
assertEquals(memberType, textArrayType.getType());
}
+ private static void verifyTextIndexMapping_without_customAnalyzer_enabled(Map<String, Object> textAttributeIndexMapping) {
+ verifyTextIndexMapping(textAttributeIndexMapping);
+ assertNull(textAttributeIndexMapping.getOrDefault("analyzer", null));
+ }
+
+ private static void verifyTextIndexMapping_with_customAnalyzer_enabled(Map<String, Object> textAttributeIndexMapping) {
+ verifyTextIndexMapping(textAttributeIndexMapping);
+ assertEquals("osdu_custom_analyzer", textAttributeIndexMapping.getOrDefault("analyzer", null));
+ }
+
private static void verifyTextIndexMapping(Map<String, Object> textAttributeIndexMapping) {
assertEquals("text", textAttributeIndexMapping.get("type"));
Map<String, Object> fieldsMap = (Map<String, Object>) textAttributeIndexMapping.get("fields");
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/parser/NumberParserTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/parser/NumberParserTest.java
index dbc1fe08a84f4392242504ac74e221825e4edddb..ffb71f43634fa47a35e0d0dc53891f7725dc135c 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/parser/NumberParserTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/parser/NumberParserTest.java
@@ -14,19 +14,20 @@
package org.opengroup.osdu.indexer.util.parser;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.springframework.test.context.junit4.SpringRunner;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.fail;
+import static org.junit.matchers.JUnitMatchers.containsString;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.Arrays;
import java.util.List;
import java.util.function.BiConsumer;
-
-import static org.junit.matchers.JUnitMatchers.*;
-import static org.junit.Assert.*;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.springframework.test.context.junit4.SpringRunner;
@RunWith(SpringRunner.class)
public class NumberParserTest {
@@ -96,8 +97,13 @@ public class NumberParserTest {
new BigInteger("9223372036854775808"),
new BigInteger("-9223372036854775809")
);
- this.validateInput(this.sut::parseLong, outOfRangeInputs, "long", " is out of range for a long");
-
+ for (Object val : outOfRangeInputs) {
+ try {
+ this.sut.parseLong("dummyAttribute", val);
+ } catch (IllegalArgumentException e) {
+ assertEquals(NumberFormatException.class, e.getClass());
+ }
+ }
final List<Object> invalidInputs = Arrays.asList(
"garbage",
"135.5ga"
diff --git a/indexer-core/src/test/resources/indexproperty/value_extraction_match_configuration_record.json b/indexer-core/src/test/resources/indexproperty/value_extraction_match_configuration_record.json
new file mode 100644
index 0000000000000000000000000000000000000000..4f906bf1430196f7a8a0fcf6d46da39270c83c9f
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/value_extraction_match_configuration_record.json
@@ -0,0 +1,52 @@
+{
+ "AttributionPublication": null,
+ "InactiveIndicator": null,
+ "Description": "valid for all master-data--Wellbore kinds for major version 1.",
+ "ResourceLifecycleStatus": null,
+ "ResourceCurationStatus": null,
+ "TechnicalAssuranceID": null,
+ "Code": "osdu:wks:master-data--Wellbore:1.",
+ "Source": null,
+ "Name": "Wellbore-IndexPropertyPathConfiguration",
+ "AttributionAuthority": "OSDU",
+ "ResourceHomeRegionID": null,
+ "VirtualProperties.DefaultName": "Wellbore-IndexPropertyPathConfiguration",
+ "AttributionRevision": null,
+ "ResourceSecurityClassification": null,
+ "ID": null,
+ "ExistenceKind": null,
+ "Configurations": [{
+ "Policy": "ExtractFirstMatch",
+ "UseCase": "As a user I want to discover and match Wellbore records by their custom field value.",
+ "Paths": [{
+ "RelatedObjectsSpec.RelatedObjectID": null,
+ "ValueExtraction.ValuePath": "data.VerticalMeasurements[].VerticalMeasurement",
+ "RelatedObjectsSpec.RelatedObjectKind": null,
+ "RelatedObjectsSpec.RelatedConditionProperty": null,
+ "RelatedObjectsSpec.RelationshipDirection": null,
+ "ValueExtraction.RelatedConditionMatches": [
+ "^[\\w\\-\\.]+:reference-data--VerticalMeasurementType:GroundLevel:"
+ ],
+ "ValueExtraction.RelatedConditionProperty": "data.VerticalMeasurements[].VerticalMeasurementTypeID"
+ }
+ ],
+ "Name": "DCGroundlevel"
+ }, {
+ "Policy": "ExtractFirstMatch",
+ "UseCase": "As a user I want to discover and match Wellbore records by their custom field value.",
+ "Paths": [{
+ "RelatedObjectsSpec.RelatedObjectID": null,
+ "ValueExtraction.ValuePath": "data.VerticalMeasurements[].VerticalMeasurement",
+ "RelatedObjectsSpec.RelatedObjectKind": null,
+ "RelatedObjectsSpec.RelatedConditionProperty": null,
+ "RelatedObjectsSpec.RelationshipDirection": null,
+ "ValueExtraction.RelatedConditionMatches": [
+ "^[\\w\\-\\.]+:reference-data--VerticalMeasurementType:TotalDepth:"
+ ],
+ "ValueExtraction.RelatedConditionProperty": "data.VerticalMeasurements[].VerticalMeasurementTypeID"
+ }
+ ],
+ "Name": "DCTotalDepth"
+ }
+ ]
+}
diff --git a/indexer-core/src/test/resources/indexproperty/value_extraction_match_data_record.json b/indexer-core/src/test/resources/indexproperty/value_extraction_match_data_record.json
new file mode 100644
index 0000000000000000000000000000000000000000..d3809bbb95c9eccbcfdc2be735d7af4ee05110ab
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/value_extraction_match_data_record.json
@@ -0,0 +1,15 @@
+{
+ "VerticalMeasurements": [{
+ "VerticalCRSID": "opendes:reference-data--CoordinateReferenceSystem:Vertical:EPSG::5714:",
+ "VerticalMeasurementPathID": "opendes:reference-data--VerticalMeasurementPath:TrueVerticalHeight:",
+ "VerticalMeasurement": 1143,
+ "VerticalMeasurementTypeID": "opendes:reference-data--VerticalMeasurementType:GroundLevel:"
+ }, {
+ "VerticalMeasurementPathID": "opendes:reference-data--VerticalMeasurementPath:MeasuredDepth:",
+ "VerticalMeasurement": 4977,
+ "VerticalMeasurementTypeID": "opendes:reference-data--VerticalMeasurementType:TotalDepth:"
+ }
+ ],
+ "FacilityName": "OCLT2209D",
+ "FacilityTypeID": "opendes:reference-data--FacilityType:Wellbore:"
+}
diff --git a/indexer-core/src/test/resources/indexproperty/well_no_null.json b/indexer-core/src/test/resources/indexproperty/well_no_null.json
new file mode 100644
index 0000000000000000000000000000000000000000..5ae2192f787091798973a66b63b659c2bfb8dc73
--- /dev/null
+++ b/indexer-core/src/test/resources/indexproperty/well_no_null.json
@@ -0,0 +1,72 @@
+{
+ "DefaultVerticalMeasurementID": "KB",
+ "SpatialLocation.Wgs84Coordinates": {
+ "type": "geometrycollection",
+ "geometries": [{
+ "type": "point",
+ "coordinates": [171.8745361, -41.2456122]
+ }
+ ]
+ },
+ "TechnicalAssuranceID": "opendes:reference-data--TechnicalAssuranceType:Unsuitable:",
+ "VirtualProperties.DefaultLocation.SpatialGeometryTypeID": "opendes:reference-data--SpatialGeometryType:Point:",
+ "FacilityName": "Kongahu-1",
+ "VirtualProperties.DefaultName": "Kongahu-1",
+ "VerticalMeasurements": [{
+ "VerticalMeasurementID": "KB",
+ "VerticalMeasurementPathID": "opendes:reference-data--VerticalMeasurementPath:ELEV:",
+ "VerticalMeasurement": 26.0,
+ "VerticalMeasurementTypeID": "opendes:reference-data--VerticalMeasurementType:KB:",
+ "VerticalMeasurementUnitOfMeasureID": "opendes:reference-data--UnitOfMeasure:m:"
+ }, {
+ "VerticalMeasurementID": "Well Head Elevation",
+ "VerticalMeasurementPathID": "opendes:reference-data--VerticalMeasurementPath:ELEV:",
+ "VerticalMeasurement": 26.0,
+ "VerticalMeasurementUnitOfMeasureID": "opendes:reference-data--UnitOfMeasure:m:"
+ }, {
+ "VerticalMeasurementID": "Water Depth",
+ "VerticalMeasurementPathID": "opendes:reference-data--VerticalMeasurementPath:ELEV:",
+ "VerticalMeasurement": 94.0,
+ "VerticalMeasurementUnitOfMeasureID": "opendes:reference-data--UnitOfMeasure:m:"
+ }
+ ],
+ "FacilityOperators": [{
+ "FacilityOperatorID": "Operator",
+ "FacilityOperatorOrganisationID": "opendes:master-data--Organisation:HOME%20ENERGY%20NZ%20LTD:"
+ }
+ ],
+ "FacilityTypeID": "opendes:reference-data--FacilityType:Well:",
+ "NameAliases": [{
+ "AliasName": "100000113552",
+ "AliasNameTypeID": "opendes:reference-data--AliasNameType:UniqueIdentifier:"
+ }, {
+ "AliasName": "Well1",
+ "AliasNameTypeID": "opendes:reference-data--AliasNameType:CommonName:"
+ }
+ ],
+ "FacilityEvents": [{
+ "EffectiveDateTime": "1984-06-21T00:00:00+0000",
+ "FacilityEventTypeID": "opendes:reference-data--FacilityEventType:Spud:"
+ }
+ ],
+ "VirtualProperties.DefaultLocation.IsDecimated": false,
+ "GeoContexts": [{
+ "GeoPoliticalEntityID": "opendes:master-data--GeoPoliticalEntity:New%20Zealand:",
+ "GeoTypeID": "opendes:reference-data--GeoPoliticalEntityType:Country:"
+ }, {
+ "GeoPoliticalEntityID": "opendes:master-data--GeoPoliticalEntity:38058:",
+ "GeoTypeID": "opendes:reference-data--GeoPoliticalEntityType:LicenseBlock:"
+ }
+ ],
+ "FakeGeoContexts": ["opendes:reference-data--GeoPoliticalEntityType:Country:", "opendes:reference-data--GeoPoliticalEntityType:LicenseBlock:", "opendes:reference-data--GeoPoliticalEntityType:Country:"],
+ "SpatialLocation.SpatialGeometryTypeID": "opendes:reference-data--SpatialGeometryType:Point:",
+ "OperatingEnvironmentID": "opendes:reference-data--OperatingEnvironment:Offshore:",
+ "VirtualProperties.DefaultLocation.Wgs84Coordinates": {
+ "type": "geometrycollection",
+ "geometries": [{
+ "type": "point",
+ "coordinates": [171.8745361, -41.2456122]
+ }
+ ]
+ }
+}
diff --git a/indexer-core/src/test/resources/queryutil/children_configuration_query.json b/indexer-core/src/test/resources/queryutil/children_configuration_query.json
new file mode 100644
index 0000000000000000000000000000000000000000..90eb4d280369a54ed6aae0e4e80e182a8d025f56
--- /dev/null
+++ b/indexer-core/src/test/resources/queryutil/children_configuration_query.json
@@ -0,0 +1,35 @@
+{
+ "nested": {
+ "boost": 1.0,
+ "ignore_unmapped": true,
+ "path": "data.Configurations",
+ "query": {
+ "nested": {
+ "boost": 1.0,
+ "ignore_unmapped": true,
+ "path": "data.Configurations.Paths",
+ "query": {
+ "query_string": {
+ "boost": 1.0,
+ "allow_leading_wildcard": false,
+ "auto_generate_synonyms_phrase_query": true,
+ "default_operator": "or",
+ "enable_position_increments": true,
+ "escape": false,
+ "fields": [],
+ "fuzziness": "AUTO",
+ "fuzzy_max_expansions": 50,
+ "fuzzy_prefix_length": 0,
+ "fuzzy_transpositions": true,
+ "max_determinized_states": 10000,
+ "phrase_slop": 0.0,
+ "query": "data.Configurations.Paths.RelatedObjectsSpec.RelationshipDirection:ChildToParent AND data.Configurations.Paths.RelatedObjectsSpec.RelatedObjectKind:\"osdu:wks:master-data--Wellbore:1.\"",
+ "type": "best_fields"
+ }
+ },
+ "score_mode": "avg"
+ }
+ },
+ "score_mode": "avg"
+ }
+}
diff --git a/indexer-core/src/test/resources/queryutil/configuration_query.json b/indexer-core/src/test/resources/queryutil/configuration_query.json
new file mode 100644
index 0000000000000000000000000000000000000000..53fc650f8cbac64af1e7c542306c519e571b242b
--- /dev/null
+++ b/indexer-core/src/test/resources/queryutil/configuration_query.json
@@ -0,0 +1,59 @@
+{
+ "bool": {
+ "boost": 1.0,
+ "should": [{
+ "query_string": {
+ "boost": 1.0,
+ "allow_leading_wildcard": false,
+ "auto_generate_synonyms_phrase_query": true,
+ "default_operator": "or",
+ "enable_position_increments": true,
+ "escape": false,
+ "fields": [],
+ "fuzziness": "AUTO",
+ "fuzzy_max_expansions": 50,
+ "fuzzy_prefix_length": 0,
+ "fuzzy_transpositions": true,
+ "max_determinized_states": 10000,
+ "phrase_slop": 0.0,
+ "query": "data.Code:\"osdu:wks:work-product-component--WellLog:1.\"",
+ "type": "best_fields"
+ }
+ }, {
+ "nested": {
+ "boost": 1.0,
+ "ignore_unmapped": true,
+ "path": "data.Configurations",
+ "query": {
+ "nested": {
+ "boost": 1.0,
+ "ignore_unmapped": true,
+ "path": "data.Configurations.Paths",
+ "query": {
+ "query_string": {
+ "boost": 1.0,
+ "allow_leading_wildcard": false,
+ "auto_generate_synonyms_phrase_query": true,
+ "default_operator": "or",
+ "enable_position_increments": true,
+ "escape": false,
+ "fields": [],
+ "fuzziness": "AUTO",
+ "fuzzy_max_expansions": 50,
+ "fuzzy_prefix_length": 0,
+ "fuzzy_transpositions": true,
+ "max_determinized_states": 10000,
+ "phrase_slop": 0.0,
+ "query": "data.Configurations.Paths.RelatedObjectsSpec.RelatedObjectKind:\"osdu:wks:work-product-component--WellLog:1.\"",
+ "type": "best_fields"
+ }
+ },
+ "score_mode": "avg"
+ }
+ },
+ "score_mode": "avg"
+ }
+ }
+ ]
+ }
+}
diff --git a/indexer-core/src/test/resources/queryutil/parents_configuration_query.json b/indexer-core/src/test/resources/queryutil/parents_configuration_query.json
new file mode 100644
index 0000000000000000000000000000000000000000..97bb992145639244c70f8d32a01c3a5907d0b4e3
--- /dev/null
+++ b/indexer-core/src/test/resources/queryutil/parents_configuration_query.json
@@ -0,0 +1,35 @@
+{
+ "nested": {
+ "boost": 1.0,
+ "ignore_unmapped": true,
+ "path": "data.Configurations",
+ "query": {
+ "nested": {
+ "boost": 1.0,
+ "ignore_unmapped": true,
+ "path": "data.Configurations.Paths",
+ "query": {
+ "query_string": {
+ "boost": 1.0,
+ "allow_leading_wildcard": false,
+ "auto_generate_synonyms_phrase_query": true,
+ "default_operator": "or",
+ "enable_position_increments": true,
+ "escape": false,
+ "fields": [],
+ "fuzziness": "AUTO",
+ "fuzzy_max_expansions": 50,
+ "fuzzy_prefix_length": 0,
+ "fuzzy_transpositions": true,
+ "max_determinized_states": 10000,
+ "phrase_slop": 0.0,
+ "query": "data.Configurations.Paths.RelatedObjectsSpec.RelationshipDirection:ParentToChildren AND data.Configurations.Paths.RelatedObjectsSpec.RelatedObjectKind:\"osdu:wks:work-product-component--WellLog:1.\"",
+ "type": "best_fields"
+ }
+ },
+ "score_mode": "avg"
+ }
+ },
+ "score_mode": "avg"
+ }
+}
diff --git a/indexer-core/src/test/resources/queryutil/simple_query.json b/indexer-core/src/test/resources/queryutil/simple_query.json
new file mode 100644
index 0000000000000000000000000000000000000000..d0892b4118c28164d81005b963dd175d9c0aa2dd
--- /dev/null
+++ b/indexer-core/src/test/resources/queryutil/simple_query.json
@@ -0,0 +1,19 @@
+{
+ "query_string": {
+ "boost": 1.0,
+ "allow_leading_wildcard": false,
+ "auto_generate_synonyms_phrase_query": true,
+ "default_operator": "or",
+ "enable_position_increments": true,
+ "escape": false,
+ "fields": [],
+ "fuzziness": "AUTO",
+ "fuzzy_max_expansions": 50,
+ "fuzzy_prefix_length": 0,
+ "fuzzy_transpositions": true,
+ "max_determinized_states": 10000,
+ "phrase_slop": 0.0,
+ "query": "id:(\"id1\" OR \"id2\")",
+ "type": "best_fields"
+ }
+}
diff --git a/pom.xml b/pom.xml
index b0c4a87aff45a3a9b145cc93e69d2d40024ae446..357f128fe075acf5a7d721e6b1ec8cf0df82404c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -12,19 +12,19 @@
<java.version>17</java.version>
<maven.compiler.target>17</maven.compiler.target>
<maven.compiler.source>17</maven.compiler.source>
- <os-core-common-spring6.version>0.27.0-rc3</os-core-common-spring6.version>
+ <os-core-common.version>3.1.0</os-core-common.version>
<hibernate-validator.version>8.0.0.Final</hibernate-validator.version>
<common-codec.version>1.14</common-codec.version>
<netty.version>4.1.51.Final</netty.version>
<reactor-netty.version>0.8.20.RELEASE</reactor-netty.version>
<woodstox-core.version>6.4.0</woodstox-core.version>
<log4j2.version>2.21.1</log4j2.version>
- <spring-boot-maven-plugin.version>3.2.5</spring-boot-maven-plugin.version>
+ <spring-boot-maven-plugin.version>3.3.4</spring-boot-maven-plugin.version>
<json-smart.version>2.5.0</json-smart.version>
- <openapi.version>2.3.0</openapi.version>
- <spring.boot.version>3.3.1</spring.boot.version>
- <spring.framework.version>6.1.10</spring.framework.version>
- <spring.security.version>6.3.1</spring.security.version>
+ <openapi.version>2.6.0</openapi.version>
+ <spring.boot.version>3.3.7</spring.boot.version>
+ <spring.framework.version>6.1.16</spring.framework.version>
+ <spring.security.version>6.3.4</spring.security.version>
</properties>
<licenses>
@@ -84,23 +84,23 @@
<dependencies>
<dependency>
<groupId>org.opengroup.osdu</groupId>
- <artifactId>os-core-common-spring6</artifactId>
- <version>${os-core-common-spring6.version}</version>
+ <artifactId>os-core-common</artifactId>
+ <version>${os-core-common.version}</version>
</dependency>
<dependency>
<groupId>co.elastic.clients</groupId>
<artifactId>elasticsearch-java</artifactId>
- <version>7.17.22</version>
+ <version>8.15.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
- <version>7.17.22</version>
+ <version>8.15.2</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
- <version>1.18.16</version>
+ <version>1.18.34</version>
<scope>provided</scope>
</dependency>
<!--OpenApi -->
diff --git a/provider/indexer-aws/SECURITY.md b/provider/indexer-aws/SECURITY.md
new file mode 100644
index 0000000000000000000000000000000000000000..d59287b678296d1d3150182417cf0b0165082b1e
--- /dev/null
+++ b/provider/indexer-aws/SECURITY.md
@@ -0,0 +1,6 @@
+Reporting Security Issues
+-------------------------------------------------------------------------------------------------------------------------------------------------
+We take all security reports seriously. When we receive such reports, we will investigate and
+subsequently address any potential vulnerabilities as quickly as possible. If you discover a potential
+security issue in this project, please notify AWS/Amazon Security via our [vulnerability reporting page](https://aws.amazon.com/security/vulnerability-reporting/) or
+directly via email to [AWS Security](aws-security@amazon.com). Please do NOT create a public GitLab issue in this project.
\ No newline at end of file
diff --git a/provider/indexer-aws/build-aws/Dockerfile b/provider/indexer-aws/build-aws/Dockerfile
index da923aed030bbe68063a6d19a709e2d603f439dd..5edd356bdf2a11c6017a1e0d62468e3a4db0edea 100644
--- a/provider/indexer-aws/build-aws/Dockerfile
+++ b/provider/indexer-aws/build-aws/Dockerfile
@@ -13,9 +13,16 @@
# limitations under the License.
# https://docs.spring.io/spring-boot/docs/current/reference/html/deployment.html
-FROM public.ecr.aws/amazoncorretto/amazoncorretto:17
+FROM registry.repo.osdu.aws/amazoncorretto:17-alpine
-RUN yum update -y
+RUN apk update && \
+ apk add --no-cache \
+ bash \
+ curl \
+ shadow \
+ && \
+ /usr/sbin/groupadd -g 10001 -r appgroup && \
+ /usr/sbin/useradd -M -g 10001 -r -u 10001 appuser
ARG JAR_FILE=provider/indexer-aws/target/*spring-boot.jar
@@ -25,4 +32,5 @@ COPY ${JAR_FILE} app.jar
COPY /provider/indexer-aws/build-aws/entrypoint.sh /entrypoint.sh
EXPOSE 8080
+USER 10001:10001
ENTRYPOINT ["/bin/sh", "-c", ". /entrypoint.sh"]
diff --git a/provider/indexer-aws/build-aws/build-info.py b/provider/indexer-aws/build-aws/build-info.py
index 5112d1231b073facd7ee2471986844e5e1701e03..33c0f0a73c22cfe2beebc6c3630b9bcb136c5b0c 100644
--- a/provider/indexer-aws/build-aws/build-info.py
+++ b/provider/indexer-aws/build-aws/build-info.py
@@ -1,17 +1,4 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import boto3
import json
import os
@@ -19,12 +6,11 @@ import argparse
# Create the build-info.json
parser = argparse.ArgumentParser(description="")
-
-# env - CODEBUILD_SOURCE_VERSION
parser.add_argument("--branch", type=str, help="")
-
-# env - CODEBUILD_RESOLVED_SOURCE_VERSION
-parser.add_argument("--commit", type=str, help="")
+parser.add_argument("--commitId", type=str, help="")
+parser.add_argument("--commitMessage", type=str, help="")
+parser.add_argument("--commitAuthor", type=str, help="")
+parser.add_argument("--commitDate", type=str, help="")
# env - CODEBUILD_BUILD_ID
parser.add_argument("--buildid", type=str, help="")
@@ -32,7 +18,6 @@ parser.add_argument("--buildid", type=str, help="")
# env - CODEBUILD_BUILD_NUMBER
parser.add_argument("--buildnumber", type=str, help="")
-# Get from directory name
parser.add_argument("--reponame", type=str, help="")
# env OUTPUT_DIR
@@ -46,7 +31,10 @@ parser.add_argument("--artifact", type=str, action="append", help="")
args = parser.parse_args()
branch = args.branch
-commitId = args.commit
+commitId = args.commitId
+commitMessage = args.commitMessage
+commitAuthor = args.commitAuthor
+commitDate = args.commitDate
buildId = args.buildid
buildNumber = args.buildnumber
repoName = args.reponame
@@ -57,32 +45,20 @@ buildInfoFilePath = os.path.join(".", outputDir, "build-info.json")
print(buildInfoFilePath)
-commitArgs = {
- "repositoryName": repoName,
- "commitId": commitId
-}
-
-commitDetail = {
- "commit": ""
-}
-
-# get the commit detail
-try:
- codecommit = boto3.client("codecommit")
- commitDetail = codecommit.get_commit(**commitArgs)
-except Exception as e:
- print("Getting commit information from codecommit failed")
-
buildInfo = {
"branch": branch,
"build-id": buildId,
"build-number": buildNumber,
"repo": repoName,
"artifacts": artifacts,
- "commit": commitDetail["commit"]
+ "commit-id": commitId,
+ "commit-message": commitMessage,
+ "commit-author": commitAuthor,
+ "commit-date":commitDate
}
-print(json.dumps(buildInfo, sort_keys=True, indent=4))
+
+print(json.dumps(buildInfo, sort_keys=True, indent=4))
# write the build.json file to dist
f = open(buildInfoFilePath, "w")
f.write(json.dumps(buildInfo, sort_keys=True, indent=4))
diff --git a/provider/indexer-aws/build-aws/buildspec.yaml b/provider/indexer-aws/build-aws/buildspec.yaml
index 6af561f95ede2da98f981630930b3a8f98add6ba..88011f261a204f76424b6aa02af4856d40416347 100644
--- a/provider/indexer-aws/build-aws/buildspec.yaml
+++ b/provider/indexer-aws/build-aws/buildspec.yaml
@@ -32,7 +32,6 @@ phases:
runtime-versions:
java: corretto17
commands:
- - if [ $(echo $CODEBUILD_SOURCE_VERSION | grep -c ^refs/heads.*) -eq 1 ]; then echo "Branch name found"; else echo "This build only supports branch builds" && exit 1; fi
- apt-get update -y
- apt-get install -y maven
- java -version
@@ -52,10 +51,9 @@ phases:
- docker login -u ${DOCKER_USERNAME} -p ${DOCKER_PASSWORD}
build:
commands:
- - export REPO_NAME=${PWD##*/}
- export OUTPUT_DIR="dist"
- - export BRANCH_NAME=`echo ${CODEBUILD_SOURCE_VERSION} | awk '{gsub("refs/heads/","");gsub("\\.","-");gsub("[[:space:]]","-")}1' | sed 's/\//-/g' | awk '{print tolower($0)}'`
- - export ECR_TAG=`echo build.${BRANCH_NAME}.${CODEBUILD_BUILD_NUMBER}.${CODEBUILD_RESOLVED_SOURCE_VERSION} | cut -c 1-120`
+ - export BRANCH_NAME=`echo ${BRANCH_NAME} | awk '{gsub("refs/heads/","");gsub("\\.","-");gsub("[[:space:]]","-")}1' | sed 's/\//-/g' | awk '{print tolower($0)}'`
+ - export ECR_TAG=`echo build.${BRANCH_NAME}.${CODEBUILD_BUILD_NUMBER}.${COMMIT_ID} | cut -c 1-120`
- export ECR_IMAGE=${ECR_REGISTRY}:${ECR_TAG}
- export ECR_IMAGE_BRANCH_LATEST=${ECR_REGISTRY}:${BRANCH_NAME}
- export INTEGRATION_TEST_OUTPUT=${OUTPUT_DIR}/testing/integration
@@ -77,20 +75,15 @@ phases:
- echo "Building integration testing assemblies and gathering artifacts..."
- ./testing/indexer-test-aws/build-aws/prepare-dist.sh
- - sed -i "s|__CONTAINER__|$ECR_IMAGE|" devops/aws/chart/values.yaml
- - echo "--Copying Helm Charts to ${OUTPUT_DIR:-dist}--"
- - rsync -r devops/aws/* "${OUTPUT_DIR:-dist}"
-
- - echo "Building docker image..."
- - docker build -f provider/indexer-aws/build-aws/Dockerfile -t ${ECR_IMAGE} .
- - docker tag ${ECR_IMAGE} ${ECR_IMAGE_BRANCH_LATEST}
- - echo "Pushing docker image..."
- - docker push ${ECR_IMAGE}
- - docker push ${ECR_IMAGE_BRANCH_LATEST}
+ - echo "Build and push docker image..."
+ - docker buildx create --driver docker-container --use
+ - docker buildx inspect --bootstrap
+ - docker buildx build --platform linux/arm64/v8,linux/amd64 --pull --push -f provider/indexer-aws/build-aws/Dockerfile -t ${ECR_IMAGE} -t ${ECR_IMAGE_BRANCH_LATEST} .
- echo "Generate build-info.json"
- |
- python provider/indexer-aws/build-aws/build-info.py --branch ${CODEBUILD_SOURCE_VERSION} --commit ${CODEBUILD_RESOLVED_SOURCE_VERSION} \
+ python provider/indexer-aws/build-aws/build-info.py --branch ${BRANCH_NAME} --commitId ${COMMIT_ID} \
+ --commitMessage "\"${COMMIT_MESSAGE}\"" --commitAuthor "\"${COMMIT_AUTHOR}\"" --commitDate "\"${COMMIT_DATE}\"" \
--buildid ${CODEBUILD_BUILD_ID} --buildnumber ${CODEBUILD_BUILD_NUMBER} --reponame ${REPO_NAME} --outdir ${OUTPUT_DIR} \
--artifact ${ECR_IMAGE}
@@ -104,7 +97,7 @@ artifacts:
files:
- "**/*"
base-directory: "dist"
- name: ${REPO_NAME}_${BRANCH_NAME}_$(date +%F)_${CODEBUILD_BUILD_NUMBER}.zip
+ name: ${BRANCH_NAME}/${PACKAGE_NAME}.zip
cache:
paths:
- "/root/.m2/**/*"
diff --git a/provider/indexer-aws/pom.xml b/provider/indexer-aws/pom.xml
index e2c4cbba84b18405b87d9a22c8b4788f5af7398c..071073661b2e872bad1b092a0981f96d566254c9 100644
--- a/provider/indexer-aws/pom.xml
+++ b/provider/indexer-aws/pom.xml
@@ -30,7 +30,7 @@
<properties>
<deployment.environment>dev</deployment.environment>
- <netty.version>4.1.51.Final</netty.version>
+ <netty.version>4.1.115.Final</netty.version>
<mockito.version>3.11.2</mockito.version>
</properties>
@@ -50,8 +50,8 @@
<!-- Internal packages -->
<dependency>
<groupId>org.opengroup.osdu</groupId>
- <artifactId>os-core-common-spring6</artifactId>
- <version>${os-core-common-spring6.version}</version>
+ <artifactId>os-core-common</artifactId>
+ <version>${os-core-common.version}</version>
<exclusions>
<exclusion>
<groupId>org.springframework</groupId>
@@ -67,21 +67,19 @@
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
<artifactId>os-core-lib-aws</artifactId>
- <version>0.26.0-spring6-rc2</version>
+ <version>3.0.2</version>
</dependency>
- <!-- AWS managed packages -->
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-codec</artifactId>
- <version>4.1.104.Final</version>
- </dependency>
-
<!-- Third party Apache 2.0 license packages -->
<dependency>
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-config</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-web</artifactId>
+ <version>6.3.4</version>
+ </dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-autoconfigure</artifactId>
@@ -196,11 +194,6 @@
<artifactId>spring-data-commons</artifactId>
<scope>compile</scope>
</dependency>
-<!-- <dependency>-->
-<!-- <groupId>org.springframework.security</groupId>-->
-<!-- <artifactId>spring-security-core</artifactId>-->
-<!-- <version>${spring-security-core.version}</version>-->
-<!-- </dependency>-->
</dependencies>
<build>
<plugins>
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/CacheBuilder.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/CacheBuilder.java
index a0a5a901914e3c3f6aad422560056c2f68bc3e2a..7b76d8cc6513bb7c68706b003079e0d134095ada 100644
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/CacheBuilder.java
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/CacheBuilder.java
@@ -16,15 +16,10 @@
package org.opengroup.osdu.indexer.aws.cache;
-import lombok.RequiredArgsConstructor;
-import org.opengroup.osdu.core.aws.cache.DummyCache;
-import org.opengroup.osdu.core.common.cache.ICache;
-import org.opengroup.osdu.core.common.cache.RedisCache;
-import org.opengroup.osdu.core.common.cache.VmCache;
+import org.opengroup.osdu.core.aws.cache.CacheParameters;
+import org.opengroup.osdu.core.aws.cache.NameSpacedCache;
import org.opengroup.osdu.indexer.aws.di.AWSCacheConfiguration;
-import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Conditional;
import org.springframework.stereotype.Component;
@@ -35,61 +30,32 @@ For this, there should exist a bean with the `RedisCache` type. But of course, w
on local mode without a connection to the Elasticache server. That's why conditional bean injection is required.
*/
@Component
-@RequiredArgsConstructor
public class CacheBuilder {
-
+ private static final int DEFAULT_MAX_SIZE = 10;
+
+ private <V> NameSpacedCache<V> createCacheFromDefaultCacheParams(String namespace, Class<V> clazzOfV, AWSCacheConfiguration cacheConfiguration) {
+ CacheParameters<String, V> cacheParams = CacheParameters.<String, V>builder()
+ .defaultHost(cacheConfiguration.getRedisSearchHost())
+ .defaultPassword(cacheConfiguration.getRedisSearchKey())
+ .defaultPort(cacheConfiguration.getRedisSearchPort())
+ .expTimeSeconds(cacheConfiguration.getCacheExpireTimeInSeconds())
+ .maxSize(DEFAULT_MAX_SIZE)
+ .keyNamespace(namespace)
+ .build()
+ .initFromLocalParameters(String.class, clazzOfV);
+ return new NameSpacedCache<>(cacheParams);
+ }
+ static final String INDEX_CACHE_NAMESPACE = "indexerIndex";
+ static final String SCHEMA_CACHE_NAMESPACE = "indexerSchema";
@Bean
- public IndexCacheImpl indexInitCache(@Qualifier("awsIndexCache") ICache<String, Boolean> awsIndexCache, AWSCacheConfiguration awsAppServiceConfig) {
- return new IndexCacheImpl(awsIndexCache, awsAppServiceConfig.isLocalMode());
+ public IndexCacheImpl indexInitCache(AWSCacheConfiguration awsAppServiceConfig) {
+ NameSpacedCache<Boolean> awsIndexCache = this.createCacheFromDefaultCacheParams(INDEX_CACHE_NAMESPACE, Boolean.class, awsAppServiceConfig);
+ return new IndexCacheImpl(awsIndexCache);
}
@Bean
- public SchemaCacheImpl schemaInitCache(@Qualifier("awsSchemaCache") ICache<String, String> awsSchemaCache, AWSCacheConfiguration awsAppServiceConfig) {
- return new SchemaCacheImpl(awsSchemaCache, awsAppServiceConfig.isLocalMode());
- }
-
- @Bean("awsIndexCache")
- @Conditional(ShouldUseDummyCacheCondition.class)
- public DummyCache<String, Boolean> createIndexDummyCache() {
- return new DummyCache<>();
- }
-
- @Bean("awsSchemaCache")
- @Conditional(ShouldUseDummyCacheCondition.class)
- public DummyCache<String, String> createSchemaDummyCache() {
- return new DummyCache<>();
- }
-
- @Bean("awsIndexCache")
- @Conditional(ShouldUseVMCacheCondition.class)
- public VmCache<String, Boolean> createIndexVMCache(AWSCacheConfiguration awsAppServiceConfig) {
- return new VmCache<>(awsAppServiceConfig.getCacheExpireTimeInSeconds(), 10);
- }
-
- @Bean("awsSchemaCache")
- @Conditional(ShouldUseVMCacheCondition.class)
- public VmCache<String, String> createSchemaVMCache(AWSCacheConfiguration awsAppServiceConfig) {
- return new VmCache<>(awsAppServiceConfig.getCacheExpireTimeInSeconds(), 10);
- }
-
- @Bean("awsIndexCache")
- @Conditional(ShouldUseElasticacheCondition.class)
- public RedisCache<String, Boolean> createIndexElasticCache(AWSCacheConfiguration awsAppServiceConfig) {
- return new RedisCache<>(awsAppServiceConfig.getCacheClusterHost(),
- awsAppServiceConfig.getCacheClusterPort(),
- awsAppServiceConfig.getCacheClusterKey(),
- awsAppServiceConfig.getCacheExpireTimeInSeconds(),
- String.class,
- Boolean.class);
- }
- @Bean("awsSchemaCache")
- @Conditional(ShouldUseElasticacheCondition.class)
- public RedisCache<String, String> createSchemaElasticCache(AWSCacheConfiguration awsAppServiceConfig) {
- return new RedisCache<>(awsAppServiceConfig.getCacheClusterHost(),
- awsAppServiceConfig.getCacheClusterPort(),
- awsAppServiceConfig.getCacheClusterKey(),
- awsAppServiceConfig.getCacheExpireTimeInSeconds(),
- String.class,
- String.class);
+ public SchemaCacheImpl schemaInitCache(AWSCacheConfiguration awsAppServiceConfig) {
+ NameSpacedCache<String> awsSchemaCache = this.createCacheFromDefaultCacheParams(SCHEMA_CACHE_NAMESPACE, String.class, awsAppServiceConfig);
+ return new SchemaCacheImpl(awsSchemaCache);
}
}
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/IndexCacheImpl.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/IndexCacheImpl.java
index 45fe98f419437b9c815bf2d53b578e4dd54963ca..73842186be1e70c9513d63338ee6a6f0c7a0d75f 100644
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/IndexCacheImpl.java
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/IndexCacheImpl.java
@@ -16,28 +16,21 @@
package org.opengroup.osdu.indexer.aws.cache;
-import lombok.RequiredArgsConstructor;
import org.opengroup.osdu.core.common.cache.ICache;
import org.opengroup.osdu.core.common.provider.interfaces.IIndexCache;
-@RequiredArgsConstructor
public class IndexCacheImpl implements IIndexCache<String, Boolean>, AutoCloseable {
- private ICache<String, Boolean> cache;
- private Boolean local;
+ private final ICache<String, Boolean> cache;
- public IndexCacheImpl(ICache<String, Boolean> cache, boolean local) {
+ public IndexCacheImpl(ICache<String, Boolean> cache) {
this.cache = cache;
- this.local = local;
}
@Override
public void close() throws Exception {
- if (Boolean.TRUE.equals(this.local)) {
- // do nothing, this is using local dummy cache or vm cache
- } else {
- // cast to redis cache so it can be closed
- ((AutoCloseable) this.cache).close();
+ if (this.cache instanceof AutoCloseable autoCloseable) {
+ autoCloseable.close();
}
}
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/SchemaCacheImpl.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/SchemaCacheImpl.java
index 3e29a0c5cf31e60f7a3500066eb059503641393c..7b9ba2ee865a211df33cc082e1b8473197b9ebfc 100644
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/SchemaCacheImpl.java
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/SchemaCacheImpl.java
@@ -16,27 +16,20 @@
package org.opengroup.osdu.indexer.aws.cache;
-import lombok.RequiredArgsConstructor;
import org.opengroup.osdu.core.common.cache.ICache;
import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache;
-@RequiredArgsConstructor
public class SchemaCacheImpl implements ISchemaCache<String, String>, AutoCloseable {
- private ICache<String, String> cache;
- private Boolean local = false;
+ private final ICache<String, String> cache;
- public SchemaCacheImpl(ICache<String, String> cache, boolean local) {
+ public SchemaCacheImpl(ICache<String, String> cache) {
this.cache = cache;
- this.local = local;
}
@Override
public void close() throws Exception {
- if (Boolean.TRUE.equals(this.local)) {
- // do nothing, this is using local dummy cache or vm cache
- } else {
- // cast to redis cache so it can be closed
- ((AutoCloseable) this.cache).close();
+ if (this.cache instanceof AutoCloseable autoCloseable) {
+ autoCloseable.close();
}
}
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/ShouldUseDummyCacheCondition.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/ShouldUseDummyCacheCondition.java
deleted file mode 100644
index a2a3b14583bf0042c56894a848c8192622a8ca72..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/ShouldUseDummyCacheCondition.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.aws.cache;
-
-import org.opengroup.osdu.core.aws.ssm.K8sLocalParameterProvider;
-import org.springframework.context.annotation.Condition;
-import org.springframework.context.annotation.ConditionContext;
-import org.springframework.core.type.AnnotatedTypeMetadata;
-
-public class ShouldUseDummyCacheCondition implements Condition {
- K8sLocalParameterProvider provider;
- boolean disableCache;
-
- public ShouldUseDummyCacheCondition() {
- this(new K8sLocalParameterProvider(), Boolean.parseBoolean(System.getenv("DISABLE_CACHE")));
- }
-
- public ShouldUseDummyCacheCondition(K8sLocalParameterProvider provider, boolean disableCache) {
- this.provider = provider;
- this.disableCache = disableCache;
- }
-
- @Override
- public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) {
- return provider.getLocalMode() && disableCache;
- }
-}
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/ShouldUseElasticacheCondition.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/ShouldUseElasticacheCondition.java
deleted file mode 100644
index ea4da5080bf2c151c20ff111bb167a07d91e59b6..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/ShouldUseElasticacheCondition.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.aws.cache;
-
-import org.opengroup.osdu.core.aws.ssm.K8sLocalParameterProvider;
-import org.springframework.context.annotation.Condition;
-import org.springframework.context.annotation.ConditionContext;
-import org.springframework.core.type.AnnotatedTypeMetadata;
-
-public class ShouldUseElasticacheCondition implements Condition {
-
- K8sLocalParameterProvider provider;
-
- public ShouldUseElasticacheCondition() {
- this(new K8sLocalParameterProvider());
- }
-
- public ShouldUseElasticacheCondition(K8sLocalParameterProvider provider) {
- this.provider = provider;
- }
- @Override
- public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) {
- return !provider.getLocalMode();
- }
-}
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/ShouldUseVMCacheCondition.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/ShouldUseVMCacheCondition.java
deleted file mode 100644
index 28e77752d17593c131c944b0c3d13e05e852699f..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/cache/ShouldUseVMCacheCondition.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.aws.cache;
-
-import org.opengroup.osdu.core.aws.ssm.K8sLocalParameterProvider;
-import org.springframework.context.annotation.Condition;
-import org.springframework.context.annotation.ConditionContext;
-import org.springframework.core.type.AnnotatedTypeMetadata;
-
-public class ShouldUseVMCacheCondition implements Condition {
-
- K8sLocalParameterProvider provider;
- boolean disableCache;
-
- public ShouldUseVMCacheCondition() {
- this(new K8sLocalParameterProvider(), Boolean.parseBoolean(System.getenv("DISABLE_CACHE")));
- }
-
- public ShouldUseVMCacheCondition(K8sLocalParameterProvider provider, boolean disableCache) {
- this.provider = provider;
- this.disableCache = disableCache;
- }
-
- @Override
- public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) {
- return provider.getLocalMode() && !disableCache;
- }
-}
diff --git a/provider/indexer-aws/src/main/resources/application.properties b/provider/indexer-aws/src/main/resources/application.properties
index fd1321af815dfd96193ba0c57cf43b84fca4ca7c..d6180a3d81ad42b702d5136f20baee7f5e0b1df1 100644
--- a/provider/indexer-aws/src/main/resources/application.properties
+++ b/provider/indexer-aws/src/main/resources/application.properties
@@ -88,4 +88,4 @@ featureFlag.strategy=appProperty
featureFlag.asIngestedCoordinates.enabled=true
featureFlag.keywordLower.enabled=true
featureFlag.bagOfWords.enabled=true
-collaborations-enabled=true
+collaborations-enabled=${COLLABORATION_FF_ENABLED:true}
diff --git a/provider/indexer-aws/src/test/java/org/opengroup/osdu/indexer/aws/cache/BaseCacheImplTest.java b/provider/indexer-aws/src/test/java/org/opengroup/osdu/indexer/aws/cache/BaseCacheImplTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..879889896aa1f2146fd3c900e4eb696b7c1a23f0
--- /dev/null
+++ b/provider/indexer-aws/src/test/java/org/opengroup/osdu/indexer/aws/cache/BaseCacheImplTest.java
@@ -0,0 +1,100 @@
+/* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.aws.cache;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+import org.opengroup.osdu.core.common.cache.ICache;
+import org.opengroup.osdu.core.common.cache.RedisCache;
+
+@RunWith(MockitoJUnitRunner.class)
+public abstract class BaseCacheImplTest<K, V> {
+ @Mock
+ private ICache<K, V> normalCache;
+ @Mock
+ private RedisCache<K, V> redisCache;
+ abstract ICache<K, V> getCache(ICache<K, V> supplied);
+ abstract AutoCloseable getCloseable(ICache<K, V> supplied);
+ abstract K getKey();
+ abstract V getVal();
+
+ @Test
+ public void should_autoCloseRedis_when_redisCacheIsUsed() throws Exception {
+ AutoCloseable closeable = getCloseable(redisCache);
+ closeable.close();
+ verify(redisCache, times(1)).close();
+ }
+
+ @Test
+ public void should_notError_when_nonRedisCacheIsUsed() throws Exception {
+ AutoCloseable closeable = getCloseable(normalCache);
+ assertDoesNotThrow(closeable::close);
+ }
+
+ @Test
+ public void should_passThrough_get() {
+ when(normalCache.get(getKey())).thenReturn(getVal());
+ ICache<K, V> cache = getCache(normalCache);
+ V val = cache.get(getKey());
+ assertEquals(getVal(), val);
+ verify(normalCache, times(1)).get(any());
+ verify(normalCache, never()).put(any(),any());
+ verify(normalCache, never()).delete(any());
+ verify(normalCache, never()).clearAll();
+ }
+
+ @Test
+ public void should_passThrough_put() {
+ ICache<K, V> cache = getCache(normalCache);
+ cache.put(getKey(), getVal());
+ verify(normalCache, never()).get(any());
+ verify(normalCache, times(1)).put(any(),any());
+ verify(normalCache, times(1)).put(getKey(),getVal());
+ verify(normalCache, never()).delete(any());
+ verify(normalCache, never()).clearAll();
+ }
+
+ @Test
+ public void should_passThrough_delete() {
+ ICache<K, V> cache = getCache(normalCache);
+ cache.delete(getKey());
+ verify(normalCache, never()).get(any());
+ verify(normalCache, never()).put(any(),any());
+ verify(normalCache, times(1)).delete(any());
+ verify(normalCache, times(1)).delete(getKey());
+ verify(normalCache, never()).clearAll();
+ }
+
+ @Test
+ public void should_passThrough_clearAll() {
+ ICache<K, V> cache = getCache(normalCache);
+ cache.clearAll();
+ verify(normalCache, never()).get(any());
+ verify(normalCache, never()).put(any(),any());
+ verify(normalCache, never()).delete(any());
+ verify(normalCache, times(1)).clearAll();
+ }
+}
diff --git a/provider/indexer-aws/src/test/java/org/opengroup/osdu/indexer/aws/cache/CacheBuilderTest.java b/provider/indexer-aws/src/test/java/org/opengroup/osdu/indexer/aws/cache/CacheBuilderTest.java
index 7f0bf8e18c7b8c219ad303104272a1001a82817c..e0ec8275c7c38ed1275f8371196030597df2dfed 100644
--- a/provider/indexer-aws/src/test/java/org/opengroup/osdu/indexer/aws/cache/CacheBuilderTest.java
+++ b/provider/indexer-aws/src/test/java/org/opengroup/osdu/indexer/aws/cache/CacheBuilderTest.java
@@ -16,29 +16,28 @@
package org.opengroup.osdu.indexer.aws.cache;
+import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Captor;
import org.mockito.Mock;
import org.mockito.MockedConstruction;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
-import org.opengroup.osdu.core.aws.cache.DummyCache;
-import org.opengroup.osdu.core.aws.ssm.K8sLocalParameterProvider;
-import org.opengroup.osdu.core.common.cache.RedisCache;
-import org.opengroup.osdu.core.common.cache.VmCache;
-import org.opengroup.osdu.core.common.provider.interfaces.IIndexCache;
+import org.opengroup.osdu.core.aws.cache.CacheFactory;
+import org.opengroup.osdu.core.aws.cache.CacheParameters;
+import org.opengroup.osdu.core.aws.cache.NameSpacedCache;
+import org.opengroup.osdu.core.common.cache.ICache;
import org.opengroup.osdu.indexer.aws.di.AWSCacheConfiguration;
-import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache;
-import org.springframework.test.context.junit4.SpringRunner;
-
-import java.util.HashMap;
-import java.util.Map;
+import org.springframework.test.util.ReflectionTestUtils;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertNotNull;
import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class)
@@ -46,67 +45,55 @@ public class CacheBuilderTest {
@Mock
AWSCacheConfiguration config;
+ @Mock
+ ICache cacheMock;
+ @Captor
+ ArgumentCaptor<CacheParameters<?, ?>> cacheParameterCaptor;
private final String s = "s";
private final String so = "o";
private final boolean o = true;
+ private final int port = 1234;
+ private final String defaultEndpoint = "some-endpoint";
+ private final String defaultPassword = "some-password";
+ private final int expireTimeSeconds = 4321;
+ MockedConstruction<NameSpacedCache> nameSpacedCache;
- @Test
- public void test_indexDummyCache() throws Exception {
- CacheBuilder cb = new CacheBuilder();
- when(config.isLocalMode()).thenReturn(true);
- DummyCache<String, Boolean> vmCache = cb.createIndexDummyCache();
- IndexCacheImpl cacheImpl = cb.indexInitCache(vmCache, config);
- cacheImpl.put(s, o);
- // Dummy cache doesn't really cache, so should return null
- assertNull(cacheImpl.get(s));
- cacheImpl.delete(s);
- assertNull(cacheImpl.get(s));
- cacheImpl.clearAll();
- cacheImpl.close();
+ @Before
+ public void setup() {
+ nameSpacedCache = Mockito.mockConstruction(NameSpacedCache.class, (mock, context) -> {
+ validateCacheParams((CacheParameters<String, Object>)context.arguments().get(0));
+ });
+ when(config.getCacheExpireTimeInSeconds()).thenReturn(expireTimeSeconds);
+ when(config.getRedisSearchKey()).thenReturn(defaultPassword);
+ when(config.getRedisSearchHost()).thenReturn(defaultEndpoint);
+ when(config.getRedisSearchPort()).thenReturn(Integer.toString(port));
}
- @Test
- public void test_schemaDummyCache() throws Exception {
- CacheBuilder cb = new CacheBuilder();
- when(config.isLocalMode()).thenReturn(true);
- DummyCache<String, String> vmCache = cb.createSchemaDummyCache();
- SchemaCacheImpl cacheImpl = cb.schemaInitCache(vmCache, config);
- cacheImpl.put(s, so);
- // Dummy cache doesn't really cache, so should return null
- assertNull(cacheImpl.get(s));
- cacheImpl.delete(s);
- assertNull(cacheImpl.get(s));
- cacheImpl.clearAll();
- cacheImpl.close();
+ @After
+ public void teardown() {
+ nameSpacedCache.close();
}
+ private <V> void validateCacheParams(CacheParameters<String, V> cacheParameters) {
+ assertEquals(defaultEndpoint, cacheParameters.getDefaultHost());
+ assertEquals(port, Integer.parseInt(cacheParameters.getDefaultPort()));
+ assertEquals(defaultPassword, cacheParameters.getDefaultPassword());
+ assertEquals(expireTimeSeconds, cacheParameters.getExpTimeSeconds());
+ }
@Test
- public void test_indexVMCache() throws Exception {
- CacheBuilder cb = new CacheBuilder();
- when(config.isLocalMode()).thenReturn(true);
- when(config.getCacheExpireTimeInSeconds()).thenReturn(3600);
- VmCache<String, Boolean> vmCache = cb.createIndexVMCache(config);
- IndexCacheImpl cacheImpl = cb.indexInitCache(vmCache, config);
- cacheImpl.put(s, o);
- assertEquals(o, cacheImpl.get(s));
- cacheImpl.delete(s);
- assertNull(cacheImpl.get(s));
- cacheImpl.clearAll();
- cacheImpl.close();
+ public void shouldCreateIndexCache_with_ExpectedParameters() {
+ CacheBuilder cacheBuilder = new CacheBuilder();
+ IndexCacheImpl implCache = cacheBuilder.indexInitCache(config);
+ ICache<String, Boolean> cache = (ICache<String, Boolean>) ReflectionTestUtils.getField(implCache, "cache");
+ assertEquals(1, nameSpacedCache.constructed().size());
+ assertEquals(nameSpacedCache.constructed().get(0), cache);
}
-
@Test
- public void test_schemaVMCache() throws Exception {
- CacheBuilder cb = new CacheBuilder();
- when(config.isLocalMode()).thenReturn(true);
- when(config.getCacheExpireTimeInSeconds()).thenReturn(3600);
- VmCache<String, String> vmCache = cb.createSchemaVMCache(config);
- SchemaCacheImpl cacheImpl = cb.schemaInitCache(vmCache, config);
- cacheImpl.put(s, so);
- assertEquals(so, cacheImpl.get(s));
- cacheImpl.delete(s);
- assertNull(cacheImpl.get(s));
- cacheImpl.clearAll();
- cacheImpl.close();
+ public void shouldCreateSchemaCache_with_ExpectedParameters() {
+ CacheBuilder cacheBuilder = new CacheBuilder();
+ SchemaCacheImpl implCache = cacheBuilder.schemaInitCache(config);
+ ICache<String, String> cache = (ICache<String, String>) ReflectionTestUtils.getField(implCache, "cache");
+ assertEquals(1, nameSpacedCache.constructed().size());
+ assertEquals(nameSpacedCache.constructed().get(0), cache);
}
}
diff --git a/provider/indexer-aws/src/test/java/org/opengroup/osdu/indexer/aws/cache/IndexCacheImplTest.java b/provider/indexer-aws/src/test/java/org/opengroup/osdu/indexer/aws/cache/IndexCacheImplTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..323bbb1036e25f92a98e754d866bef1475350368
--- /dev/null
+++ b/provider/indexer-aws/src/test/java/org/opengroup/osdu/indexer/aws/cache/IndexCacheImplTest.java
@@ -0,0 +1,41 @@
+/* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.aws.cache;
+
+import org.opengroup.osdu.core.common.cache.ICache;
+
+public class IndexCacheImplTest extends BaseCacheImplTest<String, Boolean>{
+
+ @Override
+ ICache<String, Boolean> getCache(ICache<String, Boolean> supplied) {
+ return new IndexCacheImpl(supplied);
+ }
+
+ @Override
+ AutoCloseable getCloseable(ICache<String, Boolean> supplied) {
+ return new IndexCacheImpl(supplied);
+ }
+
+ @Override
+ String getKey() {
+ return "some-key";
+ }
+
+ @Override
+ Boolean getVal() {
+ return true;
+ }
+}
diff --git a/provider/indexer-aws/src/test/java/org/opengroup/osdu/indexer/aws/cache/SchemaCacheImplTest.java b/provider/indexer-aws/src/test/java/org/opengroup/osdu/indexer/aws/cache/SchemaCacheImplTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..0680a427519f06fb7a413c075a8fb29a1658c781
--- /dev/null
+++ b/provider/indexer-aws/src/test/java/org/opengroup/osdu/indexer/aws/cache/SchemaCacheImplTest.java
@@ -0,0 +1,42 @@
+/* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.indexer.aws.cache;
+
+
+import org.opengroup.osdu.core.common.cache.ICache;
+
+public class SchemaCacheImplTest extends BaseCacheImplTest<String, String> {
+
+ @Override
+ ICache<String, String> getCache(ICache<String, String> supplied) {
+ return new SchemaCacheImpl(supplied);
+ }
+
+ @Override
+ AutoCloseable getCloseable(ICache<String, String> supplied) {
+ return new SchemaCacheImpl(supplied);
+ }
+
+ @Override
+ String getKey() {
+ return "some-key";
+ }
+
+ @Override
+ String getVal() {
+ return "some-val";
+ }
+}
diff --git a/provider/indexer-aws/src/test/java/org/opengroup/osdu/indexer/aws/cache/ShouldUseDummyCacheConditionTest.java b/provider/indexer-aws/src/test/java/org/opengroup/osdu/indexer/aws/cache/ShouldUseDummyCacheConditionTest.java
deleted file mode 100644
index 1b7a8ae9849f1fe01d0177a3677e4e24ca0f55dd..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/src/test/java/org/opengroup/osdu/indexer/aws/cache/ShouldUseDummyCacheConditionTest.java
+++ /dev/null
@@ -1,43 +0,0 @@
-package org.opengroup.osdu.indexer.aws.cache;
-
-import org.junit.runner.RunWith;
-import org.junit.Test;
-import org.mockito.Mock;
-import org.mockito.junit.MockitoJUnitRunner;
-import org.opengroup.osdu.core.aws.ssm.K8sLocalParameterProvider;
-import org.springframework.context.annotation.ConditionContext;
-import org.springframework.core.type.AnnotatedTypeMetadata;
-
-import static junit.framework.TestCase.assertTrue;
-import static junit.framework.TestCase.assertFalse;
-import static org.mockito.Mockito.when;
-
-@RunWith(MockitoJUnitRunner.class)
-public class ShouldUseDummyCacheConditionTest {
-
- @Mock
- K8sLocalParameterProvider provider;
- @Mock
- ConditionContext context;
- @Mock
- AnnotatedTypeMetadata metadata;
-
- @Test
- public void test_match() {
- when(provider.getLocalMode()).thenReturn(true);
- ShouldUseDummyCacheCondition condition = new ShouldUseDummyCacheCondition(provider, true);
- assertTrue(condition.matches(context, metadata));
- }
-
- @Test
- public void test_nomatch() {
- when(provider.getLocalMode()).thenReturn(true);
- ShouldUseDummyCacheCondition condition = new ShouldUseDummyCacheCondition(provider, false);
- assertFalse(condition.matches(context, metadata));
-
- when(provider.getLocalMode()).thenReturn(false);
- // Second parameter doesn't really matter here
- condition = new ShouldUseDummyCacheCondition(provider, false);
- assertFalse(condition.matches(context, metadata));
- }
-}
diff --git a/provider/indexer-aws/src/test/java/org/opengroup/osdu/indexer/aws/cache/ShouldUseElasticacheConditionTest.java b/provider/indexer-aws/src/test/java/org/opengroup/osdu/indexer/aws/cache/ShouldUseElasticacheConditionTest.java
deleted file mode 100644
index e74846c46a3819bbe60ea94ae09cd9be1c7e853a..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/src/test/java/org/opengroup/osdu/indexer/aws/cache/ShouldUseElasticacheConditionTest.java
+++ /dev/null
@@ -1,38 +0,0 @@
-package org.opengroup.osdu.indexer.aws.cache;
-
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.Mock;
-import org.mockito.junit.MockitoJUnitRunner;
-import org.opengroup.osdu.core.aws.ssm.K8sLocalParameterProvider;
-import org.springframework.context.annotation.ConditionContext;
-import org.springframework.core.type.AnnotatedTypeMetadata;
-
-import static junit.framework.TestCase.assertFalse;
-import static junit.framework.TestCase.assertTrue;
-import static org.mockito.Mockito.when;
-
-@RunWith(MockitoJUnitRunner.class)
-public class ShouldUseElasticacheConditionTest {
-
- @Mock
- K8sLocalParameterProvider provider;
- @Mock
- ConditionContext context;
- @Mock
- AnnotatedTypeMetadata metadata;
-
- @Test
- public void test_match() {
- when(provider.getLocalMode()).thenReturn(false);
- ShouldUseElasticacheCondition condition = new ShouldUseElasticacheCondition(provider);
- assertTrue(condition.matches(context, metadata));
- }
-
- @Test
- public void test_nomatch() {
- when(provider.getLocalMode()).thenReturn(true);
- ShouldUseElasticacheCondition condition = new ShouldUseElasticacheCondition(provider);
- assertFalse(condition.matches(context, metadata));
- }
-}
diff --git a/provider/indexer-aws/src/test/java/org/opengroup/osdu/indexer/aws/cache/ShouldUseVMCacheConditionTest.java b/provider/indexer-aws/src/test/java/org/opengroup/osdu/indexer/aws/cache/ShouldUseVMCacheConditionTest.java
deleted file mode 100644
index 99517d88504087d57ec8760b6ae561def3204df1..0000000000000000000000000000000000000000
--- a/provider/indexer-aws/src/test/java/org/opengroup/osdu/indexer/aws/cache/ShouldUseVMCacheConditionTest.java
+++ /dev/null
@@ -1,43 +0,0 @@
-package org.opengroup.osdu.indexer.aws.cache;
-
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.Mock;
-import org.mockito.junit.MockitoJUnitRunner;
-import org.opengroup.osdu.core.aws.ssm.K8sLocalParameterProvider;
-import org.springframework.context.annotation.ConditionContext;
-import org.springframework.core.type.AnnotatedTypeMetadata;
-
-import static junit.framework.TestCase.assertFalse;
-import static junit.framework.TestCase.assertTrue;
-import static org.mockito.Mockito.when;
-
-@RunWith(MockitoJUnitRunner.class)
-public class ShouldUseVMCacheConditionTest {
-
- @Mock
- K8sLocalParameterProvider provider;
- @Mock
- ConditionContext context;
- @Mock
- AnnotatedTypeMetadata metadata;
-
- @Test
- public void test_match() {
- when(provider.getLocalMode()).thenReturn(true);
- ShouldUseVMCacheCondition condition = new ShouldUseVMCacheCondition(provider, false);
- assertTrue(condition.matches(context, metadata));
- }
-
- @Test
- public void test_nomatch() {
- when(provider.getLocalMode()).thenReturn(true);
- ShouldUseVMCacheCondition condition = new ShouldUseVMCacheCondition(provider, true);
- assertFalse(condition.matches(context, metadata));
-
- when(provider.getLocalMode()).thenReturn(false);
- // Second parameter doesn't really matter here
- condition = new ShouldUseVMCacheCondition(provider, false);
- assertFalse(condition.matches(context, metadata));
- }
-}
diff --git a/provider/indexer-azure/pom.xml b/provider/indexer-azure/pom.xml
index 926a3aae360d837033ff7acb0fdf17c7cc46a2a1..1333f55c27beee66e66c068b7630714b999601e6 100644
--- a/provider/indexer-azure/pom.xml
+++ b/provider/indexer-azure/pom.xml
@@ -1,418 +1,163 @@
-<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright © Microsoft Corporation
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.opengroup.osdu.indexer</groupId>
- <artifactId>indexer-service</artifactId>
- <version>0.28.0-SNAPSHOT</version>
- <relativePath>../../pom.xml</relativePath>
- </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>indexer-azure</artifactId>
+ <description>Indexer Service Azure</description>
+ <packaging>jar</packaging>
- <artifactId>indexer-azure</artifactId>
+ <parent>
+ <groupId>org.opengroup.osdu.indexer</groupId>
+ <artifactId>indexer-service</artifactId>
<version>0.28.0-SNAPSHOT</version>
- <name>indexer-azure</name>
- <description>Indexer Service Azure</description>
- <packaging>jar</packaging>
-
- <properties>
- <azure.version>2.1.7</azure.version>
- <azure.appservice.resourcegroup />
- <azure.appservice.plan />
- <azure.appservice.appname />
- <azure.appservice.subscription />
- <nimbus-jose-jwt.version>8.20.2</nimbus-jose-jwt.version>
- <indexer-core.version>0.28.0-SNAPSHOT</indexer-core.version>
- <spring-security-jwt.version>1.1.1.RELEASE</spring-security-jwt.version>
- <core-lib-azure-spring6.version>0.27.0-rc3</core-lib-azure-spring6.version>
- <reactor-netty.version>0.9.12.RELEASE</reactor-netty.version>
- <java-jwt.version>3.8.1</java-jwt.version>
- <mockito.version>3.11.2</mockito.version>
- <reactor-netty.version>1.1.14</reactor-netty.version>
- <reactor-core.version>3.4.24</reactor-core.version>
- <oauth2-oidc-sdk.version>6.5</oauth2-oidc-sdk.version>
- <azure-core.version>1.46.0</azure-core.version>
- <azure-security-keyvault-keys.version>4.7.3</azure-security-keyvault-keys.version>
- <azure-security-keyvault-secrets.version>4.7.3</azure-security-keyvault-secrets.version>
- <jackson-bom.version>2.14.1</jackson-bom.version>
- </properties>
-
- <dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-dependencies</artifactId>
- <version>${spring.boot.version}</version>
- <type>pom</type>
- <scope>import</scope>
- </dependency>
- <dependency>
- <groupId>org.opengroup.osdu</groupId>
- <artifactId>core-lib-azure-spring6</artifactId>
- <version>${core-lib-azure-spring6.version}</version>
- <type>pom</type>
- <scope>import</scope>
- </dependency>
- </dependencies>
- </dependencyManagement>
-
- <dependencies>
- <dependency>
- <groupId>org.yaml</groupId>
- <artifactId>snakeyaml</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.hibernate.validator</groupId>
- <artifactId>hibernate-validator</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.nimbusds</groupId>
- <artifactId>nimbus-jose-jwt</artifactId>
- <version>${nimbus-jose-jwt.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.opengroup.osdu.indexer</groupId>
- <artifactId>indexer-core</artifactId>
- <version>${indexer-core.version}</version>
- </dependency>
-
- <dependency>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.azure.spring</groupId>
- <artifactId>spring-cloud-azure-starter-active-directory</artifactId>
- <version>5.14.0</version>
- </dependency>
- <dependency>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-starter-security</artifactId>
- <exclusions>
- <exclusion>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-starter-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.springframework.security</groupId>
- <artifactId>spring-security-jwt</artifactId>
- <version>${spring-security-jwt.version}</version>
- </dependency>
- <dependency>
- <groupId>org.springframework.security</groupId>
- <artifactId>spring-security-oauth2-client</artifactId>
- </dependency>
- <dependency>
- <groupId>org.springframework.security</groupId>
- <artifactId>spring-security-oauth2-jose</artifactId>
- </dependency>
- <dependency>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-configuration-processor</artifactId>
- <optional>true</optional>
- </dependency>
- <dependency>
- <groupId>org.opengroup.osdu</groupId>
- <artifactId>core-lib-azure-spring6</artifactId>
- <version>${core-lib-azure-spring6.version}</version>
- <exclusions>
- <exclusion>
- <artifactId>azure-security-keyvault-keys</artifactId>
- <groupId>com.azure</groupId>
- </exclusion>
- <exclusion>
- <artifactId>azure-security-keyvault-secrets</artifactId>
- <groupId>com.azure</groupId>
- </exclusion>
- <exclusion>
- <artifactId>azure-core</artifactId>
- <groupId>com.azure</groupId>
- </exclusion>
- <exclusion>
- <artifactId>azure-identity</artifactId>
- <groupId>com.azure</groupId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>com.microsoft.azure</groupId>
- <artifactId>msal4j</artifactId>
- <version>1.15.1</version>
- </dependency>
-
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.dataformat</groupId>
- <artifactId>jackson-dataformat-cbor</artifactId>
- </dependency>
- <!-- Resilience4j Dependencies-->
- <dependency>
- <groupId>io.github.resilience4j</groupId>
- <artifactId>resilience4j-retry</artifactId>
- <version>2.0.0</version>
- </dependency>
- <dependency>
- <groupId>io.github.resilience4j</groupId>
- <artifactId>resilience4j-core</artifactId>
- <version>2.0.0</version>
- </dependency>
-
- <!-- Prometheus Dependency -->
- <dependency>
- <groupId>io.micrometer</groupId>
- <artifactId>micrometer-registry-prometheus</artifactId>
- <scope>runtime</scope>
- </dependency>
-
- <!-- Test Dependencies -->
- <dependency>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-starter-test</artifactId>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.junit</groupId>
- <artifactId>junit</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.springframework.security</groupId>
- <artifactId>spring-security-test</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>com.auth0</groupId>
- <artifactId>java-jwt</artifactId>
- <version>${java-jwt.version}</version>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-core</artifactId>
- <version>${mockito.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-inline</artifactId>
- <version>2.13.0</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.projectlombok</groupId>
- <artifactId>lombok</artifactId>
- <version>1.18.26</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>xerces</groupId>
- <artifactId>xercesImpl</artifactId>
- <version>2.12.2</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.codehaus.plexus</groupId>
- <artifactId>plexus-utils</artifactId>
- <version>4.0.1</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>io.projectreactor.netty</groupId>
- <artifactId>reactor-netty</artifactId>
- <version>${reactor-netty.version}</version>
- </dependency>
- <dependency>
- <groupId>io.projectreactor.netty</groupId>
- <artifactId>reactor-netty-http</artifactId>
- <version>${reactor-netty.version}</version>
- </dependency>
- <dependency>
- <groupId>io.projectreactor</groupId>
- <artifactId>reactor-core</artifactId>
- <version>${reactor-core.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.dataformat</groupId>
- <artifactId>jackson-dataformat-xml</artifactId>
- </dependency>
- <dependency>
- <groupId>com.nimbusds</groupId>
- <artifactId>oauth2-oidc-sdk</artifactId>
- <version>${oauth2-oidc-sdk.version}</version>
- </dependency>
-
- <dependency>
- <groupId>com.azure</groupId>
- <artifactId>azure-core</artifactId>
- <version>${azure-core.version}</version>
- </dependency>
- <dependency>
- <groupId>com.azure</groupId>
- <artifactId>azure-security-keyvault-keys</artifactId>
- <version>${azure-security-keyvault-keys.version}</version>
- </dependency>
- <dependency>
- <groupId>com.azure</groupId>
- <artifactId>azure-security-keyvault-secrets</artifactId>
- <version>${azure-security-keyvault-secrets.version}</version>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- <version>1.7.36</version>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-maven-plugin</artifactId>
- <version>${spring-boot-maven-plugin.version}</version>
- <executions>
- <execution>
- <goals>
- <goal>repackage</goal>
- </goals>
- <configuration>
- <classifier>spring-boot</classifier>
- <mainClass>
- org.opengroup.osdu.indexer.azure.IndexerAzureApplication
- </mainClass>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.jacoco</groupId>
- <artifactId>jacoco-maven-plugin</artifactId>
- <version>0.8.10</version>
- <configuration>
- <excludes>
- <exclude>**/cache/*</exclude>
- <exclude>**/security/*</exclude>
- <exclude>**/config/*</exclude>
- <exclude>**/IndexerAzureApplication.class</exclude>
- </excludes>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>prepare-agent</goal>
- </goals>
- </execution>
- <execution>
- <id>report</id>
- <phase>prepare-package</phase>
- <goals>
- <goal>report</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>com.microsoft.azure</groupId>
- <artifactId>azure-webapp-maven-plugin</artifactId>
- <version>1.7.0</version>
- <configuration>
- <schemaVersion>V2</schemaVersion>
- <authentication>
- <serverId>azure-auth</serverId>
- </authentication>
- <subscriptionId>${azure.appservice.subscription}</subscriptionId>
- <resourceGroup>${azure.appservice.resourcegroup}</resourceGroup>
- <appServicePlanName>${azure.appservice.plan}</appServicePlanName>
- <appName>${azure.appservice.appname}</appName>
- <appSettings>
- <property>
- <name>JAVA_OPTS</name>
- <value>-Dserver.port=80</value>
- </property>
- </appSettings>
- <deployment>
- <resources>
- <resource>
- <directory>${project.basedir}/target</directory>
- <includes>
- <include>*spring-boot.jar</include>
- </includes>
- </resource>
- </resources>
- </deployment>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <version>2.21.0</version>
- <configuration>
- <argLine>@{argLine} --add-opens java.base/java.lang=ALL_UNNAMED</argLine>
- </configuration>
- </plugin>
- </plugins>
- </build>
- <reporting>
- <plugins>
- <plugin>
- <groupId>org.jacoco</groupId>
- <artifactId>jacoco-maven-plugin</artifactId>
- <reportSets>
- <reportSet>
- <reports>
- <!-- select non-aggregate reports -->
- <report>report</report>
- </reports>
- </reportSet>
- </reportSets>
- </plugin>
- </plugins>
- </reporting>
-</project>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+
+ <properties>
+ <!-- Core Dependencies -->
+ <core-lib-azure.version>2.0.4</core-lib-azure.version>
+ <azure-core.version>1.46.0</azure-core.version>
+
+ <!-- Plugin Versions -->
+ <jacoco-plugin.version>0.8.12</jacoco-plugin.version>
+ <surefire-plugin.version>3.2.2</surefire-plugin.version>
+ </properties>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.opengroup.osdu</groupId>
+ <artifactId>core-lib-azure</artifactId>
+ <version>${core-lib-azure.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
+ <dependencies>
+ <!-- OSDU Dependencies -->
+ <dependency>
+ <groupId>org.opengroup.osdu.indexer</groupId>
+ <artifactId>indexer-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.opengroup.osdu</groupId>
+ <artifactId>core-lib-azure</artifactId>
+ <version>${core-lib-azure.version}</version>
+ </dependency>
+
+ <!-- Spring Dependencies -->
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-security</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-to-slf4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-actuator</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springdoc</groupId>
+ <artifactId>springdoc-openapi-starter-webmvc-ui</artifactId>
+ </dependency>
+
+ <!-- Azure Dependencies -->
+ <dependency>
+ <groupId>com.azure.spring</groupId>
+ <artifactId>spring-cloud-azure-starter-active-directory</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.azure</groupId>
+ <artifactId>azure-core</artifactId>
+ </dependency>
+
+ <!-- Test Dependencies -->
+ <dependency>
+ <groupId>org.junit.vintage</groupId>
+ <artifactId>junit-vintage-engine</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <!-- Spring Boot Maven Plugin -->
+ <plugin>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>repackage</goal>
+ </goals>
+ <configuration>
+ <classifier>spring-boot</classifier>
+ <mainClass>
+ org.opengroup.osdu.indexer.azure.IndexerAzureApplication
+ </mainClass>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!-- JaCoCo Plugin -->
+ <plugin>
+ <groupId>org.jacoco</groupId>
+ <artifactId>jacoco-maven-plugin</artifactId>
+ <version>${jacoco-plugin.version}</version>
+ <configuration>
+ <excludes>
+ <exclude>**/cache/*</exclude>
+ <exclude>**/security/*</exclude>
+ <exclude>**/config/*</exclude>
+ <exclude>**/IndexerAzureApplication.class</exclude>
+ </excludes>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>prepare-agent</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>report</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>report</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/config/RetryPolicyConfig.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/config/RetryPolicyConfig.java
new file mode 100644
index 0000000000000000000000000000000000000000..6cec10c9267fa33d913db5a00efbe260ffa71e95
--- /dev/null
+++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/config/RetryPolicyConfig.java
@@ -0,0 +1,29 @@
+// Copyright © SLB
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.opengroup.osdu.indexer.azure.config;
+
+import lombok.Data;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+@Data
+@Configuration
+@ConfigurationProperties(prefix = "azure.storage.client.retry")
+public class RetryPolicyConfig {
+
+ private int MAX_ATTEMPTS;
+ private int INITIAL_DELAY;
+
+}
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/service/RetryPolicy.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/service/RetryPolicy.java
index 2d7bfae5c0e324a18f97685d9ad29004f3ca3be7..f8cb971dfb7e07de719f9eead7179456ce400606 100644
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/service/RetryPolicy.java
+++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/service/RetryPolicy.java
@@ -24,8 +24,7 @@ import lombok.Data;
import lombok.extern.java.Log;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.HttpResponse;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.opengroup.osdu.indexer.azure.config.RetryPolicyConfig;
import org.springframework.stereotype.Component;
import java.util.function.Predicate;
@@ -38,23 +37,27 @@ import java.util.function.Predicate;
@Log
@Component
@Data
-@ConfigurationProperties(prefix = "azure.storage.client.retry")
public class RetryPolicy {
- @Autowired
private JaxRsDpsLog logger;
- private static int MAX_ATTEMPTS = 5;
- private static int INITIAL_DELAY = 1000;
+ private RetryPolicyConfig retryPolicyConfig;
+
private final String RECORD_NOT_FOUND = "notFound";
+ public RetryPolicy(JaxRsDpsLog logger, RetryPolicyConfig retryPolicyConfig){
+ this.logger = logger;
+ this.retryPolicyConfig = retryPolicyConfig;
+ }
+
+
/**
* @return RetryConfig with 3 attempts and 1 sec wait time
*/
public RetryConfig retryConfig(Predicate<HttpResponse> predicate) {
return RetryConfig.<HttpResponse>custom()
- .maxAttempts(MAX_ATTEMPTS)
- .intervalFunction(IntervalFunction.ofExponentialBackoff(INITIAL_DELAY, 2))
+ .maxAttempts(retryPolicyConfig.getMAX_ATTEMPTS())
+ .intervalFunction(IntervalFunction.ofExponentialBackoff(retryPolicyConfig.getINITIAL_DELAY(), 2))
.retryOnResult(predicate)
.build();
}
@@ -70,6 +73,11 @@ public class RetryPolicy {
if (defaultResponseRetry(response)) return true;
+ if (response.getResponseCode() == 429) {
+ logger.debug("Storage batch API 429 retry");
+ return true;
+ }
+
JsonObject jsonObject = new JsonParser().parse(response.getBody()).getAsJsonObject();
JsonElement notFoundElement = (JsonArray) jsonObject.get(RECORD_NOT_FOUND);
if (notFoundElement == null ||
diff --git a/provider/indexer-azure/src/main/resources/application.properties b/provider/indexer-azure/src/main/resources/application.properties
index 59f75c1565751b2e1dd915e2c6529fd1c0eb8de2..08b7c659d222276589ff08c0fff05960a32363b9 100644
--- a/provider/indexer-azure/src/main/resources/application.properties
+++ b/provider/indexer-azure/src/main/resources/application.properties
@@ -112,3 +112,9 @@ featureFlag.xCollaboration.enabled=${featureFlag_xCollaboration_enabled:false}
# To enable the full server path url in OpenAPI Swagger
api.server.fullUrl.enabled=${swaggerFullUrlEnabled:true}
+
+#Retry settings
+azure.storage.client.retry.MAX_ATTEMPTS=3
+azure.storage.client.retry.INITIAL_DELAY=1000
+
+elasticsearch.client.cache.size=100
\ No newline at end of file
diff --git a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/RetryPolicyTest.java b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/RetryPolicyTest.java
index e130e5fa5f1f68326f4c9c37c9f3a5b6640da3db..fb1a85951a1afc64766bc2b4c3256f640470d73f 100644
--- a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/RetryPolicyTest.java
+++ b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/RetryPolicyTest.java
@@ -24,9 +24,12 @@ import org.opengroup.osdu.core.common.http.FetchServiceHttpRequest;
import org.opengroup.osdu.core.common.http.UrlFetchServiceImpl;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.HttpResponse;
+import org.opengroup.osdu.indexer.azure.config.RetryPolicyConfig;
+
import java.util.function.Predicate;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class)
public class RetryPolicyTest {
@@ -124,9 +127,13 @@ public class RetryPolicyTest {
private RetryPolicy retryPolicy;
@Mock
private JaxRsDpsLog logger;
+ @Mock
+ private RetryPolicyConfig retryPolicyConfig;
@Test
public void retry_should_be_true_for_jsonResponseWithNotFound() {
+ when(retryPolicyConfig.getMAX_ATTEMPTS()).thenReturn(3);
+ when(retryPolicyConfig.getINITIAL_DELAY()).thenReturn(1000);
RetryConfig config = this.retryPolicy.retryConfig(response -> this.retryPolicy.batchRetryPolicy(response));
Predicate<HttpResponse> retry = config.getResultPredicate();
response.setBody(JSON_RESPONSE_WITH_NOT_FOUND);
@@ -138,6 +145,8 @@ public class RetryPolicyTest {
@Test
public void retry_should_be_false_for_jsonResponse1WithOut_NotFound() {
+ when(retryPolicyConfig.getMAX_ATTEMPTS()).thenReturn(3);
+ when(retryPolicyConfig.getINITIAL_DELAY()).thenReturn(1000);
RetryConfig config = this.retryPolicy.retryConfig(response -> this.retryPolicy.batchRetryPolicy(response));
Predicate<HttpResponse> retry = config.getResultPredicate();
response.setBody(JSON_RESPONSE1_WITHOUT_NOT_FOUND);
@@ -148,6 +157,8 @@ public class RetryPolicyTest {
@Test
public void retry_should_be_false_for_jsonResponse2WithOut_NotFound() {
+ when(retryPolicyConfig.getMAX_ATTEMPTS()).thenReturn(3);
+ when(retryPolicyConfig.getINITIAL_DELAY()).thenReturn(1000);
RetryConfig config = this.retryPolicy.retryConfig(response -> this.retryPolicy.batchRetryPolicy(response));
Predicate<HttpResponse> retry = config.getResultPredicate();
response.setBody(JSON_RESPONSE2_WITHOUT_NOT_FOUND);
@@ -223,4 +234,15 @@ public class RetryPolicyTest {
assertTrue(value);
}
+
+ @Test
+ public void shouldReturnTrue_when_429ResponseCode_for_batchRetryPolicy() {
+ HttpResponse response=new HttpResponse();
+ response.setBody(JSON_RESPONSE1_WITHOUT_NOT_FOUND);
+ response.setResponseCode(429);
+
+ boolean value=retryPolicy.batchRetryPolicy(response);
+
+ assertTrue(value);
+ }
}
diff --git a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/UrlFetchServiceAzureImplTest.java b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/UrlFetchServiceAzureImplTest.java
index 255619afc0c45e1c8793604d765bb216bdc261fb..dfb8ae83692d32b9591a5c54ef4e12ac463ee363 100644
--- a/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/UrlFetchServiceAzureImplTest.java
+++ b/provider/indexer-azure/src/test/java/org/opengroup/osdu/indexer/azure/service/UrlFetchServiceAzureImplTest.java
@@ -14,6 +14,7 @@
package org.opengroup.osdu.indexer.azure.service;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
@@ -23,9 +24,12 @@ import org.opengroup.osdu.core.common.http.FetchServiceHttpRequest;
import org.opengroup.osdu.core.common.http.UrlFetchServiceImpl;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.HttpResponse;
+import org.opengroup.osdu.indexer.azure.config.RetryPolicyConfig;
import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.*;
+import static org.mockito.Mockito.atMost;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class)
@@ -44,6 +48,8 @@ public class UrlFetchServiceAzureImplTest {
@InjectMocks
private UrlFetchServiceAzureImpl urlFetchServiceAzure;
+ private static RetryPolicyConfig retryPolicyConfig;
+
private static final String JSON1 = "{\n" +
" \"records\": [\n" +
" {\n" +
@@ -93,11 +99,18 @@ public class UrlFetchServiceAzureImplTest {
private static final String STORAGE_API_URL = "https://demo/api/storage/v2/schemas";
private static final String SCHEMA_API_URL = "https://demo/api/schema-service/v1/schema/osdu:file:gom:1.0.0";
+ @BeforeClass
+ public static void setup() {
+ retryPolicyConfig = new RetryPolicyConfig();
+ retryPolicyConfig.setINITIAL_DELAY(1000);
+ retryPolicyConfig.setMAX_ATTEMPTS(3);
+ }
+
@Test
public void shouldRetry_ForJSON1_when_storageQueryRecordCallIsMade() throws Exception {
response.setBody(JSON1);
httpRequest.setUrl(BATCH_API_URL);
- when(this.retryPolicy.retryConfig(any())).thenReturn(new RetryPolicy().retryConfig(response -> this.retryPolicy.batchRetryPolicy(response)));
+ when(this.retryPolicy.retryConfig(any())).thenReturn(new RetryPolicy(this.logger, this.retryPolicyConfig).retryConfig(response -> this.retryPolicy.batchRetryPolicy(response)));
when(urlFetchService.sendRequest(httpRequest)).thenReturn(response);
urlFetchServiceAzure.sendRequest(httpRequest);
@@ -109,7 +122,7 @@ public class UrlFetchServiceAzureImplTest {
public void shouldRetry_ForJSON1_when_schemaRecordCallIsMade() throws Exception {
response.setBody(JSON1);
httpRequest.setUrl(SCHEMA_API_URL);
- when(this.retryPolicy.retryConfig(any())).thenReturn(new RetryPolicy().retryConfig(response -> this.retryPolicy.schemaRetryPolicy(response)));
+ when(this.retryPolicy.retryConfig(any())).thenReturn(new RetryPolicy(this.logger, this.retryPolicyConfig).retryConfig(response -> this.retryPolicy.schemaRetryPolicy(response)));
when(urlFetchService.sendRequest(httpRequest)).thenReturn(response);
urlFetchServiceAzure.sendRequest(httpRequest);
@@ -121,7 +134,7 @@ public class UrlFetchServiceAzureImplTest {
public void shouldRetry_when_anyOtherCallIsMade() throws Exception {
response.setBody(JSON2);
httpRequest.setUrl(STORAGE_API_URL);
- when(this.retryPolicy.retryConfig(any())).thenReturn(new RetryPolicy().retryConfig(response -> this.retryPolicy.defaultRetryPolicy(response)));
+ when(this.retryPolicy.retryConfig(any())).thenReturn(new RetryPolicy(this.logger, this.retryPolicyConfig).retryConfig(response -> this.retryPolicy.defaultRetryPolicy(response)));
when(urlFetchService.sendRequest(httpRequest)).thenReturn(response);
urlFetchServiceAzure.sendRequest(httpRequest);
diff --git a/provider/indexer-gc/README.md b/provider/indexer-gc/README.md
index 22e77c33bbd094573845b7d5ad974792a0812ab0..52db26dda73501f6cbea02de90226baec0b8c3eb 100644
--- a/provider/indexer-gc/README.md
+++ b/provider/indexer-gc/README.md
@@ -1,6 +1,6 @@
# Indexer Service
-Indexer-gc is a [Spring Boot](https://spring.io/projects/spring-boot) service that is responsible for indexing Records that enable the `search` service to execute OSDU R2 domain searches against Elasticsearch.
+Indexer GC is a [Spring Boot](https://spring.io/projects/spring-boot) service that is responsible for indexing Records that enable the `search` service to execute OSDU R3 domain searches against Elasticsearch.
## Table of Contents <a name="TOC"></a>
@@ -21,22 +21,10 @@ These instructions will get you a copy of the project up and running on your loc
## Service Configuration
-### Baremetal
-
-[Baremetal service configuration](docs/baremetal/README.md)
-
### Google Cloud
[Google Cloud service configuration](docs/gc/README.md)
-## Mappers
-
-This is a universal solution created using EPAM OQM mappers technology. It allows you to work with various
-implementations of message brokers.
-
-For more information about mappers:
-* [OQM Readme](https://community.opengroup.org/osdu/platform/system/lib/cloud/gcp/oqm/-/blob/master/README.md)
-
### Limitations of the current version
In the current version, the mappers are equipped with several drivers to the stores and the message broker:
@@ -57,10 +45,6 @@ In the current version, the mappers are equipped with several drivers to the sto
* GCloud SDK with java (latest version)
-### Baremetal Service Configuration
-
-[Baremetal service configuration](docs/baremetal/README.md)
-
### Google Cloud Service Configuration
[Google Cloud service configuration](docs/gc/README.md)
@@ -124,11 +108,6 @@ gcloud auth application-default login
```
* Navigate to indexer service's root folder and run:
-
-```bash
-mvn jetty:run
-## Testing
-* Navigate to indexer service's root folder and run:
```bash
mvn clean install
@@ -145,9 +124,15 @@ mvn clean install -DskipTests
After configuring your environment as specified above, you can follow these steps to build and run the application. These steps should be invoked from the *repository root.*
```bash
-cd provider/indexer-gc && mvn spring-boot:run
+cd provider/indexer-gc
```
+```bash
+java --add-opens java.base/java.lang=ALL-UNNAMED \
+ --add-opens java.base/java.lang.reflect=ALL-UNNAMED \
+ -Dloader.main=org.opengroup.osdu.indexer.provider.gcp.IndexerGcpApplication \
+ -jar /target/indexer-${VERSION}-spring-boot.jar
+```
## Testing
Navigate to indexer service's root folder and run all the tests:
@@ -161,54 +146,28 @@ $ (cd testing/indexer-test-core/ && mvn clean install)
This section describes how to run cloud OSDU E2E tests.
-### Baremetal test configuration
-
-[Baremetal service configuration](docs/baremetal/README.md)
-
### Google Cloud test configuration
[Google Cloud service configuration](docs/gc/README.md)
## Deployment
-* Data-Lake Indexer Google Cloud Endpoints on App Engine Flex environment
- * Edit the app.yaml
- * Open the [app.yaml](indexer/src/main/appengine/app.yaml) file in editor, and replace the YOUR-PROJECT-ID `GOOGLE_CLOUD_PROJECT` line with Google Cloud Platform project Id. Also update `AUTHORIZE_API`, `CRON_JOB_IP`, `LEGAL_HOSTNAME`, `REGION` and `SECURITY_HTTPS_CERTIFICATE_TRUST` based on your deployment
-
- * Deploy
-
- ```sh
- mvn appengine:deploy -pl org.opengroup.osdu.indexer:indexer -amd
- ```
-
- * If you wish to deploy the search service without running tests
-
- ```sh
- mvn appengine:deploy -pl org.opengroup.osdu.indexer:indexer -amd -DskipTests
- ```
-
-or
-
-* Google Documentation: <https://cloud.google.com/cloud-build/docs/deploying-builds/deploy-appengine>
+Indexer Service is compatible with Cloud Run.
-#### Memory Store (Redis Instance) Setup
-
-Create a new Standard tier Redis instance on the ***service project***
-
-The Redis instance must be created under the same region with the App Engine application which needs to access it.
-
-```bash
- gcloud beta redis instances create redis-cache-search --size=10 --region=<service-deployment-region> --zone=<service-deployment-zone> --tier=STANDARD
-```
+- To deploy into Cloud run, please, use this documentation:
+ <https://cloud.google.com/run/docs/quickstarts/build-and-deploy>
+- To deploy into GKE, please, use this documentation:
+ <https://cloud.google.com/cloud-build/docs/deploying-builds/deploy-gke>
## Entitlements groups
-
-Storage service account should have entitlements groups listed below:
-* service.entitlements.user
-* users
-* service.schema-service.viewers
-* service.storage.admin
-* service.search.admin
+Entitlements groups of Indexer Service Account:
+
+| Required Roles |
+|----------------------------------|
+| service.entitlements.user |
+| service.schema-service.viewers |
+| service.storage.admin |
+| service.search.admin |
## Licence
diff --git a/provider/indexer-gc/cloudbuild/Dockerfile.cloudbuild b/provider/indexer-gc/cloudbuild/Dockerfile.cloudbuild
index 101e50f3f6618144c5a19017a9c5e08429737c1e..3badd41853fccca60ba6d834c6fd81c9e68865dd 100644
--- a/provider/indexer-gc/cloudbuild/Dockerfile.cloudbuild
+++ b/provider/indexer-gc/cloudbuild/Dockerfile.cloudbuild
@@ -8,6 +8,11 @@ ENV PROVIDER_NAME $PROVIDER_NAME
ARG PORT
ENV PORT $PORT
+ENV LOADER_PATH="gc/"
+
+COPY tmp-gc/gc-oqm-pubsub-*.jar gc/oqm-pubsub.jar
+COPY tmp-gc/gc-apd-acc-*.jar gc/apd-acc.jar
+
# Copy the jar to the production image from the builder stage.
COPY provider/indexer-${PROVIDER_NAME}/target/indexer-${PROVIDER_NAME}-*-spring-boot.jar indexer-${PROVIDER_NAME}.jar
@@ -20,4 +25,9 @@ USER 10001:10001
# Run the web service on container startup.
#--add-opens args used to open modules and allow illegal(reflection\private classes and fields) access from the code.
-CMD java --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED -Djava.security.egd=indexer:/dev/./urandom -Dserver.port=${PORT} -Dlog4j.formatMsgNoLookups=true -jar /app/indexer-${PROVIDER_NAME}.jar
+CMD java --add-opens java.base/java.lang=ALL-UNNAMED \
+ --add-opens java.base/java.lang.reflect=ALL-UNNAMED \
+ -Djava.security.egd=indexer:/dev/./urandom -Dserver.port=${PORT} \
+ -Dlog4j.formatMsgNoLookups=true \
+ -Dloader.main=org.opengroup.osdu.indexer.provider.gcp.IndexerGcpApplication \
+ -jar /app/indexer-${PROVIDER_NAME}.jar
diff --git a/provider/indexer-gc/docs/baremetal/README.md b/provider/indexer-gc/docs/baremetal/README.md
deleted file mode 100644
index 558013905ad189ef4455488ff39cd3a8c1ccc8a1..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/docs/baremetal/README.md
+++ /dev/null
@@ -1,304 +0,0 @@
-## Service Configuration for Anthos
-
-## Run args
-
-In order to run Indexer with Java 17 additional run args must be provided:
-
-```bash
---add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED
-```
-
-Full command:
-
-```bash
-java -jar indexer.jar --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED
-```
-
-## Environment variables:
-
-Define the following environment variables.
-
-Must have:
-
-| name | value | description | sensitive? | source |
-|----------------------------------------------|--------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------|--------|
-| `SPRING_PROFILES_ACTIVE` | ex `anthos` | Spring profile that activate default configuration for Google Cloud environment | false | - |
-| `OPENID_PROVIDER_CLIENT_ID` | `*****` | Client id that represents this service and serves to request tokens, example `workload-identity-legal` | yes | - |
-| `OPENID_PROVIDER_CLIENT_SECRET` | `*****` | This client secret that serves to request tokens | yes | - |
-| `OPENID_PROVIDER_URL` | `https://keycloack.com/auth/realms/master` | URL of OpenID Connect provider, it will be used as `<OpenID URL> + /.well-known/openid-configuration` to auto configure endpoint for token request | no | - |
-| `<ELASTICSEARCH_USER_ENV_VARIABLE_NAME>` | ex `user` | Elasticsearch user, name of that variable not defined at the service level, the name will be received through partition service. Each tenant can have it's own ENV name value, and it must be present in ENV of Indexer service, see [Partition properties set](#Properties-set-in-Partition-service) | yes | - |
-| `<ELASTICSEARCH_PASSWORD_ENV_VARIABLE_NAME>` | ex `password` | Elasticsearch password, name of that variable not defined at the service level, the name will be received through partition service. Each tenant can have it's own ENV name value, and it must be present in ENV of Indexer service, see [Partition properties set](#Properties-set-in-Partition-service) | false | - |
-
-Defined in default application property file but possible to override:
-
-| name | value | description | sensitive? | source |
-|------------------------------------|---------------------------------------------------------------------------|---------------------------------------------------------------------------|------------|-------------------------------------|
-| `LOG_PREFIX` | `service` | Logging prefix | no | - |
-| `LOG_LEVEL` | `****` | Logging level | no | - |
-| `SECURITY_HTTPS_CERTIFICATE_TRUST` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment |
-| `REDIS_SEARCH_HOST` | ex `127.0.0.1` | Redis host | no | |
-| `REDIS_SEARCH_PORT` | ex `6379` | Redis host port | no | |
-| `REDIS_SEARCH_PASSWORD` | ex `*****` | Redis host password | yes | |
-| `REDIS_SEARCH_WITH_SSL` | ex `true` or `false` | Redis host ssl config | no | |
-| `REDIS_SEARCH_EXPIRATION` | ex `30` | Redis cache expiration in seconds | no | |
-| `PARTITION_HOST` | ex `https://partition.com` | Partition host | no | output of infrastructure deployment |
-| `ENTITLEMENTS_HOST` | ex `https://entitlements.com` | Entitlements host | no | output of infrastructure deployment |
-| `STORAGE_HOST` | ex `https://storage.com` | Storage host | no | output of infrastructure deployment |
-| `SCHEMA_BASE_HOST` | ex `https://schema.com` | Schema service host | no | output of infrastructure deployment |
-
-These variables define service behavior, and are used to switch between `baremetal` or `gcp` environments, their overriding and usage in mixed mode was not tested.
-Usage of spring profiles is preferred.
-
-| name | value | description | sensitive? | source |
-|--------------------------|------------------------|---------------------------------------------------------------------------------------------------------------------------|------------|--------|
-| `PARTITION_AUTH_ENABLED` | ex `true` or `false` | Disable or enable auth token provisioning for requests to Partition service | no | - |
-| `OQMDRIVER` | `rabbitmq` or `pubsub` | Oqm driver mode that defines which message broker will be used | no | - |
-| `SERVICE_TOKEN_PROVIDER` | `GCP` or `OPENID` | Service account token provider, `GCP` means use Google service account `OPEIND` means use OpenId provider like `Keycloak` | no | - |
-| `RABBITMQ_RETRY_DELAY` | ex `20000` | Message retry interval after unsuccessful processing | no | - |
-| `RABBITMQ_RETRY_LIMIT` | ex `5` | Number of retries to send a message after unsuccessful processing | no | - |
-
-### Properties set in Partition service:
-
-Note that properties can be set in Partition as `sensitive` in that case in property `value` should be present not value itself, but ENV variable name.
-This variable should be present in environment of service that need that variable.
-
-Example:
-```json
- "elasticsearch.port": {
- "sensitive": false, <- value not sensitive
- "value": "9243" <- will be used as is.
- },
- "elasticsearch.password": {
- "sensitive": true, <- value is sensitive
- "value": "ELASTIC_SEARCH_PASSWORD_OSDU" <- service consumer should have env variable ELASTIC_SEARCH_PASSWORD_OSDU with elastic search password
- }
-```
-
-## Indexer account configuration
-Google cloud OSDU platform doesn't use a single Tenant account which provides access to all groups for each service,
-instead, separate accounts should be used. But the Indexer should have access to all data groups, no matter when they were created.
-To achieve that add an Indexer account to the partition configuration:
-```json
- "indexer.service.account": {
- "sensitive": false,
- "value": "indexer@service.local"
- }
-```
-Related issue: https://community.opengroup.org/osdu/platform/system/storage/-/issues/153
-
-## Elasticsearch configuration
-
-**prefix:** `elasticsearch`
-
-It can be overridden by:
-
-- through the Spring Boot property `elastic-search-properties-prefix`
-- environment variable `ELASTIC_SEARCH_PROPERTIES_PREFIX`
-
-**Propertyset:**
-
-| Property | Description |
-|------------------------|-------------|
-| elasticsearch.host | server URL |
-| elasticsearch.port | server port |
-| elasticsearch.user | username |
-| elasticsearch.password | password |
-
-<details><summary>Example of a definition for a single tenant</summary></details>
-
-```
-
-curl -L -X PATCH 'http://partition.com/api/partition/v1/partitions/opendes' -H 'data-partition-id: opendes' -H 'Authorization: Bearer ...' -H 'Content-Type: application/json' --data-raw '{
- "properties": {
- "elasticsearch.host": {
- "sensitive": false,
- "value": "elastic.us-central1.gc.cloud.es.io"
- },
- "elasticsearch.port": {
- "sensitive": false,
- "value": "9243"
- },
- "elasticsearch.user": {
- "sensitive": true,
- "value": "<ELASTICSEARCH_USER_ENV_VARIABLE_NAME>" <- (Not actual value, just name of env variable)
- },
- "elasticsearch.password": {
- "sensitive": true,
- "value": "<ELASTICSEARCH_PASSWORD_ENV_VARIABLE_NAME>" <- (Not actual value, just name of env variable)
- }
- }
-}'
-
-```
-
-### For OQM RabbitMQ
-
-**prefix:** `oqm.rabbitmq`
-It can be overridden by:
-
-- through the Spring Boot property `oqm.rabbitmq.partition-properties-prefix`
-- environment variable `OQM_RABBITMQ_PARTITION_PROPERTIES_PREFIX``
-
-**Propertyset** (for two types of connection: messaging and admin operations):
-
-| Property | Description |
-|-----------------------------|------------------------|
-| oqm.rabbitmq.amqp.host | messaging hostnameorIP |
-| oqm.rabbitmq.amqp.port | - port |
-| oqm.rabbitmq.amqp.path | - path |
-| oqm.rabbitmq.amqp.username | - username |
-| oqm.rabbitmq.amqp.password | - password |
-| oqm.rabbitmq.admin.schema | admin host schema |
-| oqm.rabbitmq.admin.host | - host name |
-| oqm.rabbitmq.admin.port | - port |
-| oqm.rabbitmq.admin.path | - path |
-| oqm.rabbitmq.admin.username | - username |
-| oqm.rabbitmq.admin.password | - password |
-
-<details><summary>Example of a single tenant definition</summary>
-
-```
-
-curl -L -X PATCH 'https://dev.osdu.club/api/partition/v1/partitions/opendes' -H 'data-partition-id: opendes' -H 'Authorization: Bearer ...' -H 'Content-Type: application/json' --data-raw '{
- "properties": {
- "oqm.rabbitmq.amqp.host": {
- "sensitive": false,
- "value": "localhost"
- },
- "oqm.rabbitmq.amqp.port": {
- "sensitive": false,
- "value": "5672"
- },
- "oqm.rabbitmq.amqp.path": {
- "sensitive": false,
- "value": ""
- },
- "oqm.rabbitmq.amqp.username": {
- "sensitive": false,
- "value": "guest"
- },
- "oqm.rabbitmq.amqp.password": {
- "sensitive": true,
- "value": "guest"
- },
-
- "oqm.rabbitmq.admin.schema": {
- "sensitive": false,
- "value": "http"
- },
- "oqm.rabbitmq.admin.host": {
- "sensitive": false,
- "value": "localhost"
- },
- "oqm.rabbitmq.admin.port": {
- "sensitive": false,
- "value": "9002"
- },
- "oqm.rabbitmq.admin.path": {
- "sensitive": false,
- "value": "/api"
- },
- "oqm.rabbitmq.admin.username": {
- "sensitive": false,
- "value": "guest"
- },
- "oqm.rabbitmq.admin.password": {
- "sensitive": true,
- "value": "guest"
- }
- }
-}'
-
-```
-
-</details>
-
-#### Exchanges and queues configuration
-
-
-
-RabbitMq should have exchanges and queues with names and configs:
-
-| EXCHANGE NAME | EXCHANGE CONFIG | Target queue name | Target queue config |
-|----------------------------------|-----------------------------------------------------------------------------|----------------------------|----------------------------------------------------------------------|
-| indexing-progress | `Type fanout` <br/>`durable: true` | (Consumer not implemented) | (Consumer not implemented) |
-| records-changed | `Type fanout` <br/>`durable: true` | indexer-records-changed | `x-delivery-limit: 5`<br/>`x-queue-type: quorum`<br/>`durable: true` |
-| indexer-records-changed-exchange | `Type x-delayed-message` <br/>`durable: true`<br/>`x-delayed-type: fanout` | indexer-records-changed | `x-delivery-limit: 5`<br/>`x-queue-type: quorum`<br/>`durable: true` |
-| reprocess | `Type fanout` <br/>`durable: true` | indexer-reprocess | `x-delivery-limit: 5`<br/>`x-queue-type: quorum`<br/>`durable: true` |
-| indexer-reprocess-exchange | `Type x-delayed-message` <br/>`durable: true`<br/>`x-delayed-type: fanout` | indexer-reprocess | `x-delivery-limit: 5`<br/>`x-queue-type: quorum`<br/>`durable: true` |
-| schema-changed | `Type fanout` <br/>`durable: true` | indexer-schema-changed | `x-delivery-limit: 5`<br/>`x-queue-type: quorum`<br/>`durable: true` |
-| indexer-schema-changed-exchange | `Type x-delayed-message` <br/>`durable: true`<br/>`x-delayed-type: fanout` | indexer-schema-changed | `x-delivery-limit: 5`<br/>`x-queue-type: quorum`<br/>`durable: true` |
-| reindex | `Type fanout` <br/>`durable: true` | indexer-reindex | `x-delivery-limit: 5`<br/>`x-queue-type: quorum`<br/>`durable: true` |
-| indexer-reindex-exchange | `Type x-delayed-message` <br/>`durable: true`<br/>`x-delayed-type: fanout` | indexer-reindex | `x-delivery-limit: 5`<br/>`x-queue-type: quorum`<br/>`durable: true` |
-
-## Keycloak configuration
-
-[Keycloak service accounts setup](https://www.keycloak.org/docs/latest/server_admin/#_service_accounts)
-
-Configure Clients. One Client per OSDU service. Set them “confidential”.
-
-
-
-Each Client has embedded Service Account (SA) option. Enable SAs for Clients, make “Authorization enabled”:
-
-
-
-Add `partition-and-entitlements` scope to `Default Client Scopes` and generate Keys.
-
-Give `client-id` and `client-secret` to services, which should be authorized within the platform.
-
-### Running E2E Tests
-
-You will need to have the following environment variables defined.
-
-| name | value | description | sensitive? | source |
-|--------------------------------------|-----------------------------------------------------------------------|---------------------------------------------------------------------------------------------------|-----------------------------------------|-------------------------------------|
-| `ELASTIC_PASSWORD` | `********` | Password for Elasticsearch | yes | output of infrastructure deployment |
-| `ELASTIC_USER_NAME` | `********` | User name for Elasticsearch | yes | output of infrastructure deployment |
-| `ELASTIC_HOST` | ex `elastic.domain.com` | Host Elasticsearch | yes | output of infrastructure deployment |
-| `ELASTIC_PORT` | ex `9243` | Port Elasticsearch | yes | output of infrastructure deployment |
-| `INDEXER_HOST` | ex `https://os-indexer-dot-opendes.appspot.com/api/indexer/v2/` | Indexer API endpoint | no | output of infrastructure deployment |
-| `GROUP_ID` | ex `opendes-gcp.projects.com` | OSDU R2 to run tests under | no | - |
-| `OTHER_RELEVANT_DATA_COUNTRIES` | ex `US` | valid legal tag with a other relevant data countries | no | - |
-| `LEGAL_TAG` | ex `opendes-demo-legaltag` | valid legal tag with a other relevant data countries from `DEFAULT_OTHER_RELEVANT_DATA_COUNTRIES` | no | - |
-| `DEFAULT_DATA_PARTITION_ID_TENANT1` | ex `opendes` | HTTP Header 'Data-Partition-ID' | no | - |
-| `DEFAULT_DATA_PARTITION_ID_TENANT2` | ex `opendes` | HTTP Header 'Data-Partition-ID' | no | - |
-| `SEARCH_HOST` | ex `http://localhost:8080/api/search/v2/` | Endpoint of search service | no | - |
-| `STORAGE_HOST` | ex `http://os-storage-dot-opendes.appspot.com/api/storage/v2/` | Storage API endpoint | no | output of infrastructure deployment |
-| `SECURITY_HTTPS_CERTIFICATE_TRUST` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment |
-| `TEST_OPENID_PROVIDER_CLIENT_ID` | `********` | Client Id for `$INTEGRATION_TESTER` | yes | -- |
-| `TEST_OPENID_PROVIDER_CLIENT_SECRET` | `********` | | Client secret for `$INTEGRATION_TESTER` | -- |
-| `TEST_OPENID_PROVIDER_URL` | `https://keycloak.com/auth/realms/osdu` | OpenID provider url | yes | -- |
-| `CUCUMBER_OPTIONS` | `--tags '~@indexer-extended'` OR `--tags '~@* and @indexer-extended'` | By default `--tags '~@indexer-extended'` to disable experimental feature testing | no | -- |
-
-**Entitlements configuration for integration accounts**
-
-| INTEGRATION_TESTER | NO_DATA_ACCESS_TESTER |
-|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------|
-| users<br/>users.datalake.ops<br/>service.storage.creator<br/>service.entitlements.user<br/>service.search.user<br/>service.search.admin<br/>data.test1<br/>data.integration.test<br/>users@{tenant1}@{groupId}.com | |
-
-Execute following command to build code and run all the integration tests:
-
-```bash
-# Note: this assumes that the environment variables for integration tests as outlined
-# above are already exported in your environment.
-$ (cd testing/indexer-test-baremetal/ && mvn clean test)
-```
-
-## License
-
-Copyright © Google LLC
-
-Copyright © EPAM Systems
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-[http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0)
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
\ No newline at end of file
diff --git a/provider/indexer-gc/docs/baremetal/pics/client.png b/provider/indexer-gc/docs/baremetal/pics/client.png
deleted file mode 100644
index 8a2014e672b65ddf644558e2c9e8cb8823f549dd..0000000000000000000000000000000000000000
Binary files a/provider/indexer-gc/docs/baremetal/pics/client.png and /dev/null differ
diff --git a/provider/indexer-gc/docs/baremetal/pics/indexer.png b/provider/indexer-gc/docs/baremetal/pics/indexer.png
deleted file mode 100644
index 711fc590e19dab1bbcb1837521ad58d96fff81da..0000000000000000000000000000000000000000
Binary files a/provider/indexer-gc/docs/baremetal/pics/indexer.png and /dev/null differ
diff --git a/provider/indexer-gc/docs/baremetal/pics/rabbit.PNG b/provider/indexer-gc/docs/baremetal/pics/rabbit.PNG
deleted file mode 100644
index e1c67734b91a536659fdf0b940a93208d749e3c8..0000000000000000000000000000000000000000
Binary files a/provider/indexer-gc/docs/baremetal/pics/rabbit.PNG and /dev/null differ
diff --git a/provider/indexer-gc/docs/baremetal/pics/sa.png b/provider/indexer-gc/docs/baremetal/pics/sa.png
deleted file mode 100644
index b9c8124047b5834a2cf79baecbf751dc106feeff..0000000000000000000000000000000000000000
Binary files a/provider/indexer-gc/docs/baremetal/pics/sa.png and /dev/null differ
diff --git a/provider/indexer-gc/docs/gc/README.md b/provider/indexer-gc/docs/gc/README.md
index 4a4847d025f890d73677a92f338e85338b86da6d..9264682d95e17383b2d5b35f164e144d471c0436 100644
--- a/provider/indexer-gc/docs/gc/README.md
+++ b/provider/indexer-gc/docs/gc/README.md
@@ -1,19 +1,5 @@
## Service Configuration for Google Cloud
-## Run args
-
-In order to run Indexer with Java 17 additional run args must be provided:
-
-```bash
---add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED
-```
-
-Full command:
-
-```bash
-java -jar indexer.jar --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED
-```
-
## Environment variables
Define the following environment variables.
@@ -22,7 +8,6 @@ Must have:
| name | value | description | sensitive? | source |
|----------------------------------------------|---------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------|---------------------------------------------------|
-| `SPRING_PROFILES_ACTIVE` | ex `gcp` | Spring profile that activate default configuration for Google Cloud environment | false | - |
| `<ELASTICSEARCH_USER_ENV_VARIABLE_NAME>` | ex `user` | Elasticsearch user, name of that variable not defined at the service level, the name will be received through partition service. Each tenant can have it's own ENV name value, and it must be present in ENV of Indexer service, see [Partition properties set](#properties-set-in-partition-service) | yes | - |
| `<ELASTICSEARCH_PASSWORD_ENV_VARIABLE_NAME>` | ex `password` | Elasticsearch password, name of that variable not defined at the service level, the name will be received through partition service. Each tenant can have it's own ENV name value, and it must be present in ENV of Indexer service, see [Partition properties set](#properties-set-in-partition-service) | false | - |
@@ -56,11 +41,11 @@ Usage of spring profiles is preferred.
| `OQMDRIVER` | `rabbitmq` or `pubsub` | Oqm driver mode that defines which message broker will be used | no | - |
| `SERVICE_TOKEN_PROVIDER` | `GCP` or `OPENID` | Service account token provider, `GCP` means use Google service account `OPEIND` means use OpenId provider like `Keycloak` | no | - |
-## Pubsub configuration
+## Pub/Sub configuration
-Pubsub should have topics and subscribers with names and configs:
+Pub/Sub should have topics and subscribers with names and configs:
| TOPIC NAME | Subscription name | Subscription config |
|----------------------------------|----------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
@@ -86,15 +71,15 @@ It is possible, but not necessary to adjust consumer throughput via Partition se
```json
"max.sub.parallel.streams": {
- "sensitive": false,
- "value": 2
+ "sensitive": false,
+ "value": 2
},
"max.sub.thread.per.stream": {
- "sensitive": false,
+ "sensitive": false,
"value": 5
},
"max.sub.max.outstanding.elements": {
- "sensitive": true,
+ "sensitive": true,
"value": 100
}
```
@@ -108,11 +93,11 @@ Example:
```json
"elasticsearch.port": {
- "sensitive": false, <- value not sensitive
+ "sensitive": false, <- value not sensitive
"value": "9243" <- will be used as is.
},
"elasticsearch.password": {
- "sensitive": true, <- value is sensitive
+ "sensitive": true, <- value is sensitive
"value": "ELASTIC_SEARCH_PASSWORD_OSDU" <- service consumer should have env variable ELASTIC_SEARCH_PASSWORD_OSDU with elastic search password
}
```
@@ -201,8 +186,7 @@ You will need to have the following environment variables defined.
| `LEGAL_TAG` | ex `opendes-demo-legaltag` | valid legal tag with a other relevant data countries from `DEFAULT_OTHER_RELEVANT_DATA_COUNTRIES` | no | - |
| `DEFAULT_DATA_PARTITION_ID_TENANT1` | ex `opendes` | HTTP Header 'Data-Partition-ID' | no | - |
| `DEFAULT_DATA_PARTITION_ID_TENANT2` | ex `opendes` | HTTP Header 'Data-Partition-ID' | no | - |
-| `SEARCH_INTEGRATION_TESTER` | `********` | Service account for API calls. Note: this user must have entitlements configured already | yes | <https://console.cloud.google.com/iam-admin/serviceaccounts> |
-| `SEARCH_HOST` | ex `http://localhost:8080/api/search/v2/` | Endpoint of search service | no | - |
+| `SEARCH_INTEGRATION_TESTER` | `ewogICJ0....` or `tmp/service-acc.json` | Service account for API calls. Note: this user must have entitlements configured already | yes | <https://console.cloud.google.com/iam-admin/serviceaccounts> |
| `STORAGE_HOST` | ex `http://os-storage-dot-opendes.appspot.com/api/storage/v2/` | Storage API endpoint | no | output of infrastructure deployment |
| `SECURITY_HTTPS_CERTIFICATE_TRUST` | ex `false` | Elastic client connection uses TrustSelfSignedStrategy(), if it is 'true' | false | output of infrastructure deployment |
| `CUCUMBER_OPTIONS` | `--tags '~@indexer-extended'` OR `--tags '~@* and @indexer-extended'` | By default `--tags '~@indexer-extended'` to disable experimental feature testing | no | -- |
diff --git a/provider/indexer-gc/pom.xml b/provider/indexer-gc/pom.xml
index 9e6b24c424546288aea85c83bad6b802c90fe695..c3f28136a286df6e760713acf14916630becd1d1 100644
--- a/provider/indexer-gc/pom.xml
+++ b/provider/indexer-gc/pom.xml
@@ -1,5 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
@@ -15,91 +16,11 @@
<description>Indexer Service Google Cloud</description>
<packaging>jar</packaging>
-
- <dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>com.google.cloud</groupId>
- <artifactId>libraries-bom</artifactId>
- <version>26.38.0</version>
- <type>pom</type>
- <scope>import</scope>
- </dependency>
- </dependencies>
- </dependencyManagement>
-
<dependencies>
- <dependency>
- <groupId>org.opengroup.osdu</groupId>
- <artifactId>core-lib-gc-spring6</artifactId>
- <version>0.26.0-rc5</version>
- </dependency>
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
- <artifactId>indexer-core</artifactId>
+ <artifactId>indexer-core-plus</artifactId>
<version>0.28.0-SNAPSHOT</version>
- <!-- excluded due to runtime conflict with latest core-lib-gc transient dependencies -->
- <exclusions>
- <exclusion>
- <groupId>com.google.api-client</groupId>
- <artifactId>google-api-client</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.http-client</groupId>
- <artifactId>google-http-client</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <!-- Test Dependencies -->
- <dependency>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-starter-test</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.springframework.security</groupId>
- <artifactId>spring-security-test</artifactId>
- <scope>test</scope>
- </dependency>
-
- <!-- Test Dependencies -->
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.12</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-core</artifactId>
- <version>3.12.0</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.projectlombok</groupId>
- <artifactId>lombok</artifactId>
- <version>1.18.26</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.codehaus.plexus</groupId>
- <artifactId>plexus-utils</artifactId>
- <version>4.0.1</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>com.auth0</groupId>
- <artifactId>java-jwt</artifactId>
- <version>3.8.1</version>
- </dependency>
-
- <!-- https://mvnrepository.com/artifact/com.github.stefanbirkner/system-rules -->
- <dependency>
- <groupId>com.github.stefanbirkner</groupId>
- <artifactId>system-rules</artifactId>
- <version>1.2.0</version>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>ch.qos.logback.contrib</groupId>
@@ -111,27 +32,6 @@
<artifactId>logback-jackson</artifactId>
<version>0.1.5</version>
</dependency>
- <dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-webmvc</artifactId>
- </dependency>
-
- <!-- Mappers -->
- <dependency>
- <groupId>org.opengroup.osdu</groupId>
- <artifactId>oqm</artifactId>
- <version>0.25.0</version>
- <exclusions>
- <exclusion>
- <artifactId>os-core-common</artifactId>
- <groupId>org.opengroup.osdu</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-starter-actuator</artifactId>
- </dependency>
</dependencies>
<build>
@@ -148,50 +48,31 @@
<configuration>
<classifier>spring-boot</classifier>
<mainClass>
- org.opengroup.osdu.indexer.provider.gcp.IndexerGcpApplication
+ org.springframework.boot.loader.launch.PropertiesLauncher
</mainClass>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
- <groupId>com.google.cloud.tools</groupId>
- <artifactId>appengine-maven-plugin</artifactId>
- <version>1.3.2</version>
- <configuration>
- <version>1</version>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <version>2.21.0</version>
- <configuration>
- <argLine>
- @{argLine} --add-opens java.base/java.util=ALL-UNNAMED
- --add-opens java.base/java.lang=ALL-UNNAMED
- </argLine>
- </configuration>
+ <groupId>org.jacoco</groupId>
+ <artifactId>jacoco-maven-plugin</artifactId>
+ <version>0.8.10</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>prepare-agent</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>report</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>report</goal>
+ </goals>
+ </execution>
+ </executions>
</plugin>
- <plugin>
- <groupId>org.jacoco</groupId>
- <artifactId>jacoco-maven-plugin</artifactId>
- <version>0.8.10</version>
- <executions>
- <execution>
- <goals>
- <goal>prepare-agent</goal>
- </goals>
- </execution>
- <execution>
- <id>report</id>
- <phase>prepare-package</phase>
- <goals>
- <goal>report</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
</plugins>
</build>
</project>
diff --git a/provider/indexer-gc/src/main/appengine/app.yaml b/provider/indexer-gc/src/main/appengine/app.yaml
deleted file mode 100644
index 363b4097e3a611b59391832910b92ca518800872..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/appengine/app.yaml
+++ /dev/null
@@ -1,42 +0,0 @@
-# Copyright 2017-2019, Schlumberger
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-service: os-indexer
-
-runtime: java
-env: flex
-
-runtime_config:
- jdk: openjdk8
-
-api_version: 1
-threadsafe: yes
-
-resources:
- cpu: 1
- memory_gb: 2.4
- disk_size_gb: 10
-
-handlers:
- - url: /.*
- script: this field is required, but ignored
-
-env_variables:
- GOOGLE_CLOUD_PROJECT: "GOOGLE_CLOUD_PROJECT_PLACE_HOLDER"
- AUTHORIZE_API: "AUTHORIZE_API_VAR"
- CRON_JOB_IP: "CRON_JOB_IP_VAR"
- LEGAL_HOSTNAME: "LEGAL_HOSTNAME_VAR"
- REGION: "REGION_VAR"
- SPRING_PROFILES_ACTIVE: 'ENVIRONMENT'
- SECURITY_HTTPS_CERTIFICATE_TRUST: 'SECURITY_HTTPS_CERTIFICATE_TRUST_VAR'
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/IndexerGcpApplication.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/IndexerGcpApplication.java
index 38dde12f2a8233e52aa8671898860f43ff8ab34d..99c3e8ea20311d8f6f8ba4c8c81bbc15e2a77c05 100644
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/IndexerGcpApplication.java
+++ b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/IndexerGcpApplication.java
@@ -1,6 +1,6 @@
/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
+ * Copyright 2020-2024 Google LLC
+ * Copyright 2020-2024 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -17,20 +17,30 @@
package org.opengroup.osdu.indexer.provider.gcp;
-import org.opengroup.osdu.indexer.provider.gcp.indexing.config.CustomContextConfiguration;
-import org.opengroup.osdu.indexer.provider.gcp.web.config.WebAppMainContextConfiguration;
+import org.opengroup.osdu.indexer.IndexerApplication;
+import org.opengroup.osdu.indexer.IndexerCorePlusApplication;
+import org.opengroup.osdu.indexer.indexing.config.CustomContextConfiguration;
+import org.opengroup.osdu.indexer.web.config.WebAppMainContextConfiguration;
import org.springframework.boot.SpringBootConfiguration;
import org.springframework.boot.WebApplicationType;
import org.springframework.boot.builder.SpringApplicationBuilder;
-import org.springframework.context.ConfigurableApplicationContext;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.FilterType;
import org.springframework.context.annotation.PropertySource;
+@ComponentScan(
+ excludeFilters = {
+ @ComponentScan.Filter(
+ type = FilterType.ASSIGNABLE_TYPE,
+ value = {IndexerCorePlusApplication.class, IndexerApplication.class})
+ })
@SpringBootConfiguration
@PropertySource("classpath:swagger.properties")
public class IndexerGcpApplication {
- public static void main(String[] args) {
- SpringApplicationBuilder springApplicationBuilder = new SpringApplicationBuilder(IndexerGcpApplication.class)
+ public static void main(String[] args) {
+ SpringApplicationBuilder springApplicationBuilder =
+ new SpringApplicationBuilder(IndexerGcpApplication.class)
.sources(IndexerGcpApplication.class)
.web(WebApplicationType.NONE)
.child(CustomContextConfiguration.class)
@@ -38,6 +48,6 @@ public class IndexerGcpApplication {
.child(WebAppMainContextConfiguration.class)
.web(WebApplicationType.SERVLET);
- springApplicationBuilder.run(args);
- }
+ springApplicationBuilder.run(args);
+ }
}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/cache/AttributesCache.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/cache/AttributesCache.java
deleted file mode 100644
index c635718be4cc32a7d0e3426fc73659e8705a962a..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/cache/AttributesCache.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.provider.gcp.common.cache;
-
-import java.util.Set;
-import jakarta.inject.Inject;
-import org.opengroup.osdu.core.common.cache.RedisCache;
-import org.opengroup.osdu.core.common.provider.interfaces.IAttributesCache;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.springframework.stereotype.Component;
-
-@Component
-public class AttributesCache implements IAttributesCache<String, Set>, AutoCloseable {
-
- private RedisCache<String, Set> cache;
-
- @Inject
- public AttributesCache(final IndexerConfigurationProperties properties) {
- cache = new RedisCache(properties.getRedisSearchHost(), Integer.parseInt(properties.getRedisSearchPort()),
- properties.getElasticCacheExpiration() * 60, String.class, Boolean.class);
- }
-
- @Override
- public void put(String key, Set value) {
- this.cache.put(key, value);
- }
-
- @Override
- public Set get(String key) {
- return this.cache.get(key);
- }
-
- @Override
- public void delete(String key) {
- this.cache.delete(key);
- }
-
- @Override
- public void clearAll() {
- this.cache.clearAll();
- }
-
- @Override
- public void close() {
- this.cache.close();
- }
-}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/cache/CacheConfig.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/cache/CacheConfig.java
deleted file mode 100644
index ebaddddcfb79d8ca12308ee67900cab62803c69c..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/cache/CacheConfig.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Copyright 2020-2023 Google LLC
- * Copyright 2020-2023 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.provider.gcp.common.cache;
-
-import lombok.RequiredArgsConstructor;
-import org.opengroup.osdu.core.common.cache.ICache;
-import org.opengroup.osdu.core.common.cache.RedisCache;
-import org.opengroup.osdu.core.common.cache.VmCache;
-import org.opengroup.osdu.core.common.model.search.ClusterSettings;
-import org.opengroup.osdu.core.common.partition.PartitionInfo;
-import org.opengroup.osdu.core.common.provider.interfaces.IElasticCredentialsCache;
-import org.opengroup.osdu.core.common.provider.interfaces.IIndexCache;
-import org.opengroup.osdu.core.gcp.cache.RedisCacheBuilder;
-import org.opengroup.osdu.indexer.provider.gcp.common.di.GcpConfigurationProperties;
-import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-
-@Configuration
-@RequiredArgsConstructor
-public class CacheConfig {
-
- private final RedisCacheBuilder<String, String> schemaCacheBuilder;
- private final RedisCacheBuilder<String, ClusterSettings> clusterSettingsCacheBuilder;
- private final RedisCacheBuilder<String, Boolean> redisCacheBuilder;
-
- @Bean
- public ISchemaCache fieldTypeMappingCache(RedisCache<String, String> schemaCache) {
- return new SchemaCache(schemaCache);
- }
-
- @Bean
- public RedisCache<String, String> schemaGCCache(GcpConfigurationProperties appProperties) {
- return schemaCacheBuilder.buildRedisCache(
- appProperties.getRedisSearchHost(),
- Integer.parseInt(appProperties.getRedisSearchPort()),
- appProperties.getRedisSearchPassword(),
- appProperties.getRedisSearchExpiration(),
- appProperties.getRedisSearchWithSsl(),
- String.class,
- String.class
- );
- }
-
- @Bean
- public IElasticCredentialsCache<String, ClusterSettings> elasticCredentialsCache(RedisCache<String, ClusterSettings> elasticCache) {
- return new ElasticCredentialsCache(elasticCache);
- }
-
- @Bean
- public RedisCache<String, ClusterSettings> elasticCache(GcpConfigurationProperties gcpAppServiceConfig) {
- return clusterSettingsCacheBuilder.buildRedisCache(
- gcpAppServiceConfig.getRedisSearchHost(),
- Integer.parseInt(gcpAppServiceConfig.getRedisSearchPort()),
- gcpAppServiceConfig.getRedisSearchPassword(),
- gcpAppServiceConfig.getRedisSearchExpiration(),
- gcpAppServiceConfig.getRedisSearchWithSsl(),
- String.class,
- ClusterSettings.class
- );
- }
-
- @Bean
- public IIndexCache cursorCache(RedisCache<String, Boolean> redisCache) {
- return new IndexCache(redisCache);
- }
-
- @Bean
- public RedisCache<String, Boolean> redisCache(GcpConfigurationProperties gcpAppServiceConfig) {
- return redisCacheBuilder.buildRedisCache(
- gcpAppServiceConfig.getRedisSearchHost(),
- Integer.parseInt(gcpAppServiceConfig.getRedisSearchPort()),
- gcpAppServiceConfig.getRedisSearchPassword(),
- gcpAppServiceConfig.getRedisSearchExpiration(),
- gcpAppServiceConfig.getRedisSearchWithSsl(),
- String.class,
- Boolean.class
- );
- }
-
- @Bean
- public ICache<String, PartitionInfo> partitionInfoCache() {
- return new VmCache<>(600, 2000);
- }
-}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/cache/ElasticCredentialsCache.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/cache/ElasticCredentialsCache.java
deleted file mode 100644
index ab5af737b666b656c27b62ac6d3ea836a0e7516b..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/cache/ElasticCredentialsCache.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.provider.gcp.common.cache;
-
-import com.lambdaworks.redis.RedisException;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.opengroup.osdu.core.common.cache.RedisCache;
-import org.opengroup.osdu.core.common.model.search.ClusterSettings;
-import org.opengroup.osdu.core.common.provider.interfaces.IElasticCredentialsCache;
-
-@Slf4j
-@RequiredArgsConstructor
-public class ElasticCredentialsCache implements IElasticCredentialsCache<String, ClusterSettings>, AutoCloseable {
-
- private final RedisCache<String, ClusterSettings> cache;
-
- @Override
- public void put(String key, ClusterSettings value) {
- this.cache.put(key, value);
- }
-
- @Override
- public ClusterSettings get(String key) {
- try {
- return this.cache.get(key);
- } catch (RedisException ex) {
- //In case the format of cache changes then clean the cache
- log.error("Unable to get value from Redis, trying to clean up by key.", ex);
- this.cache.delete(key);
- return null;
- }
- }
-
- @Override
- public void delete(String s) {
- this.cache.delete(s);
- }
-
- @Override
- public void clearAll() {
- this.cache.clearAll();
- }
-
- @Override
- public void close() {
- this.cache.close();
- }
-}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/cache/IndexCache.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/cache/IndexCache.java
deleted file mode 100644
index e47c951dd7c4d7aadf9c2983cb0212bed7c20a39..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/cache/IndexCache.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- Copyright 2020 Google LLC
- Copyright 2020 EPAM Systems, Inc
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.provider.gcp.common.cache;
-
-import lombok.RequiredArgsConstructor;
-import org.opengroup.osdu.core.common.cache.RedisCache;
-import org.opengroup.osdu.core.common.provider.interfaces.IIndexCache;
-
-@RequiredArgsConstructor
-public class IndexCache implements IIndexCache<String, Boolean>, AutoCloseable {
-
- private final RedisCache<String, Boolean> cache;
-
- @Override
- public void close() throws Exception {
- this.cache.close();
- }
-
- @Override
- public void put(String s, Boolean o) {
- this.cache.put(s, o);
- }
-
- @Override
- public Boolean get(String s) {
- return this.cache.get(s);
- }
-
- @Override
- public void delete(String s) {
- this.cache.delete(s);
- }
-
- @Override
- public void clearAll() {
- this.cache.clearAll();
- }
-}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/cache/SchemaCache.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/cache/SchemaCache.java
deleted file mode 100644
index 48d76fabf2aa7a35c8c5d81c87148f97193cb7d5..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/cache/SchemaCache.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- Copyright 2020 Google LLC
- Copyright 2020 EPAM Systems, Inc
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.provider.gcp.common.cache;
-
-import lombok.RequiredArgsConstructor;
-import org.opengroup.osdu.core.common.cache.RedisCache;
-import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache;
-
-@RequiredArgsConstructor
-public class SchemaCache implements ISchemaCache<String, String>, AutoCloseable {
-
- private final RedisCache<String, String> cache;
-
- @Override
- public void close() throws Exception {
- this.cache.close();
- }
-
- @Override
- public void put(String s, String o) {
- this.cache.put(s, o);
- }
-
- @Override
- public String get(String s) {
- return this.cache.get(s);
- }
-
- @Override
- public void delete(String s) {
- this.cache.delete(s);
- }
-
- @Override
- public void clearAll() {
- this.cache.clearAll();
- }
-}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/di/ElasticSearchConfig.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/di/ElasticSearchConfig.java
deleted file mode 100644
index 8e9e4dd84e116cddee55714d6825c9eb9e85715a..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/di/ElasticSearchConfig.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright 2020-2023 Google LLC
- * Copyright 2020-2023 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.provider.gcp.common.di;
-
-import org.opengroup.osdu.core.common.partition.IPartitionProvider;
-import org.opengroup.osdu.core.common.partition.IPropertyResolver;
-import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository;
-import org.opengroup.osdu.core.destination.elastic.ElasticSearchDestinationResolver;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-
-@Configuration
-public class ElasticSearchConfig {
-
- @Bean
- public IElasticRepository elasticRepository(GcpConfigurationProperties properties,
- IPartitionProvider partitionProvider, IPropertyResolver propertyResolver) {
- return new ElasticSearchDestinationResolver(properties.getElasticsearchPropertiesPrefix(), partitionProvider, propertyResolver);
- }
-}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/di/EntitlementsClientFactory.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/di/EntitlementsClientFactory.java
deleted file mode 100644
index 24f9cc19ff4f51d87bac4f31608eea9b0bb50ca7..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/di/EntitlementsClientFactory.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.provider.gcp.common.di;
-
-import org.opengroup.osdu.core.common.entitlements.EntitlementsAPIConfig;
-import org.opengroup.osdu.core.common.entitlements.EntitlementsFactory;
-import org.opengroup.osdu.core.common.entitlements.IEntitlementsFactory;
-import org.opengroup.osdu.core.common.http.json.HttpResponseBodyMapper;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.beans.factory.config.AbstractFactoryBean;
-import org.springframework.context.annotation.Lazy;
-import org.springframework.stereotype.Component;
-
-@Component
-@Lazy
-public class EntitlementsClientFactory extends AbstractFactoryBean<IEntitlementsFactory> {
-
- @Value("${AUTHORIZE_API}")
- private String authorizeApi;
-
- @Value("${AUTHORIZE_API_KEY:}")
- private String authorizeApiKey;
-
- @Autowired
- private HttpResponseBodyMapper mapper;
-
- @Override
- protected IEntitlementsFactory createInstance() throws Exception {
-
- return new EntitlementsFactory(EntitlementsAPIConfig
- .builder()
- .rootUrl(authorizeApi)
- .apiKey(authorizeApiKey)
- .build(),
- mapper);
- }
-
- @Override
- public Class<?> getObjectType() {
- return IEntitlementsFactory.class;
- }
-}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/di/GcpConfigurationProperties.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/di/GcpConfigurationProperties.java
deleted file mode 100644
index 2e651575e99ea4d4b7a6f1f7ec1c61f6cb7ed117..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/di/GcpConfigurationProperties.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.provider.gcp.common.di;
-
-import lombok.Getter;
-import lombok.Setter;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.springframework.boot.context.properties.ConfigurationProperties;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.Primary;
-
-@Primary
-@Configuration
-@ConfigurationProperties
-@Getter
-@Setter
-public class GcpConfigurationProperties extends IndexerConfigurationProperties {
-
- private String elasticsearchPropertiesPrefix = "elasticsearch";
-
- private String redisGroupPassword;
- private Integer redisGroupExpiration = 30;
- private Boolean redisGroupWithSsl = false;
-
- private String redisSearchPassword;
- private Integer redisSearchExpiration = 60 * 60;
- private Boolean redisSearchWithSsl = false;
-}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/publish/JobStatusJsonSerializer.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/publish/JobStatusJsonSerializer.java
deleted file mode 100644
index 01d88736ff10b20a61547db5931aa0b6868fad26..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/publish/JobStatusJsonSerializer.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.provider.gcp.common.publish;
-
-import com.google.gson.Gson;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonSerializationContext;
-import com.google.gson.JsonSerializer;
-import java.lang.reflect.Type;
-import org.opengroup.osdu.core.common.model.indexer.JobStatus;
-import org.springframework.stereotype.Component;
-
-@Component
-public class JobStatusJsonSerializer implements JsonSerializer<JobStatus> {
-
- private final Gson gson = new Gson();
-
- @Override
- public JsonElement serialize(JobStatus src, Type typeOfSrc, JsonSerializationContext context) {
- JsonObject jsonObject = new JsonObject();
- jsonObject.add("recordsStatus", this.gson.toJsonTree(src.getStatusesList()));
- jsonObject.add("debugInfo", this.gson.toJsonTree(src.getDebugInfos()));
- return jsonObject;
- }
-}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/publish/ReprocessingTaskPublisher.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/publish/ReprocessingTaskPublisher.java
deleted file mode 100644
index 87db1e6bb10aa5f081fc4ff8101c2da8484c5b1c..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/publish/ReprocessingTaskPublisher.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/*
- * Copyright 2020-2023 Google LLC
- * Copyright 2020-2023 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.provider.gcp.common.publish;
-
-import com.google.gson.Gson;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
-import org.opengroup.osdu.core.gcp.oqm.driver.OqmDriver;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmDestination;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmMessage;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmTopic;
-import org.opengroup.osdu.indexer.model.Constants;
-import org.opengroup.osdu.indexer.provider.gcp.indexing.config.MessagingConfigProperties;
-import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
-import org.springframework.context.annotation.Primary;
-import org.springframework.stereotype.Component;
-
-import jakarta.annotation.PostConstruct;
-import jakarta.validation.constraints.NotNull;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-@Slf4j
-@Primary
-@Component
-@RequiredArgsConstructor
-public class ReprocessingTaskPublisher extends IndexerQueueTaskBuilder {
-
- private final Gson gson = new Gson();
-
- private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
-
- private final OqmDriver driver;
-
- private final MessagingConfigProperties properties;
-
- private OqmTopic reprocessOqmTopic;
-
- private OqmTopic reindexTopic;
-
- @PostConstruct
- public void setUp() {
- reprocessOqmTopic = OqmTopic.builder().name(properties.getReprocessTopicName()).build();
- reindexTopic = OqmTopic.builder().name(properties.getReindexTopicName()).build();
- }
-
- public void createWorkerTask(String payload, DpsHeaders headers) {
- publishRecordsChangedTask(payload, headers);
- }
-
- public void createWorkerTask(String payload, Long countdownMillis, DpsHeaders headers) {
- DpsHeaders headersCopy = DpsHeaders.createFromMap(headers.getHeaders());
- scheduledExecutorService.schedule(
- () -> {
- try {
- publishRecordsChangedTask(payload, headersCopy);
- } catch (Exception e) {
- // If error or exception not caught, executor will die out silently.
- log.error("The exception was thrown during scheduled event publishing!", e);
- throw e;
- } catch (Throwable e) {
- log.error("The Error was thrown during scheduled event publishing!", e);
- throw e;
- }
- },
- countdownMillis,
- TimeUnit.MILLISECONDS
- );
- }
-
- public void createReIndexTask(String payload, DpsHeaders headers) {
- publishReindexTask(payload, headers);
- }
-
- public void createReIndexTask(String payload, Long countdownMillis, DpsHeaders headers) {
- DpsHeaders headersCopy = DpsHeaders.createFromMap(headers.getHeaders());
- scheduledExecutorService.schedule(
- () -> {
- try {
- publishReindexTask(payload, headersCopy);
- } catch (Exception e) {
- // If error or exception not caught, executor will die out silently.
- log.error("The exception was thrown during scheduled event publishing!", e);
- throw e;
- } catch (Throwable e) {
- log.error("The Error was thrown during scheduled event publishing!", e);
- throw e;
- }
- },
- countdownMillis,
- TimeUnit.MILLISECONDS
- );
- }
-
- private void publishRecordsChangedTask(String payload, DpsHeaders headers) {
- OqmDestination oqmDestination = OqmDestination.builder()
- .partitionId(headers.getPartitionId())
- .build();
-
- RecordChangedMessages recordChangedMessages = gson.fromJson(payload,
- RecordChangedMessages.class);
-
- Map<String, String> attributes = getAttributesFromHeaders(headers);
- // Append the ancestry kinds used to prevent circular chasing
- if(recordChangedMessages.getAttributes().containsKey(Constants.ANCESTRY_KINDS)) {
- attributes.put(Constants.ANCESTRY_KINDS, recordChangedMessages.getAttributes().get(Constants.ANCESTRY_KINDS));
- }
-
- OqmMessage oqmMessage = OqmMessage.builder()
- .id(headers.getCorrelationId())
- .data(recordChangedMessages.getData())
- .attributes(attributes)
- .build();
-
- log.info("Reindex task: {} ,has been published.", oqmMessage);
- driver.publish(oqmMessage, reindexTopic, oqmDestination);
- }
-
- private void publishReindexTask(String payload, DpsHeaders headers) {
- OqmDestination oqmDestination = OqmDestination.builder().partitionId(headers.getPartitionId())
- .build();
- Map<String, String> attributes = getAttributesFromHeaders(headers);
- OqmMessage oqmMessage = OqmMessage.builder().data(payload).attributes(attributes).build();
- log.info("Reprocessing task: {} ,has been published.", oqmMessage);
- driver.publish(oqmMessage, reprocessOqmTopic, oqmDestination);
- }
-
- @NotNull
- private Map<String, String> getAttributesFromHeaders(DpsHeaders headers) {
- Map<String, String> attributes = new HashMap<>();
- attributes.put(DpsHeaders.USER_EMAIL, headers.getUserEmail());
- attributes.put(DpsHeaders.ACCOUNT_ID, headers.getPartitionId());
- attributes.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
- headers.addCorrelationIdIfMissing();
- attributes.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
- return attributes;
- }
-}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/publish/StatusPublisherImpl.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/publish/StatusPublisherImpl.java
deleted file mode 100644
index 68c9b1cdaa7a11c44009c330da538282b8e17848..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/publish/StatusPublisherImpl.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.provider.gcp.common.publish;
-
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import com.google.gson.JsonSerializer;
-import java.util.HashMap;
-import java.util.Map;
-import jakarta.annotation.PostConstruct;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.indexer.JobStatus;
-import org.opengroup.osdu.core.gcp.oqm.driver.OqmDriver;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmDestination;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmMessage;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmTopic;
-import org.opengroup.osdu.indexer.provider.gcp.indexing.config.MessagingConfigProperties;
-import org.opengroup.osdu.indexer.provider.interfaces.IPublisher;
-import org.springframework.stereotype.Component;
-
-@Slf4j
-@Component
-@RequiredArgsConstructor
-public class StatusPublisherImpl implements IPublisher {
-
- private final OqmDriver driver;
- private final MessagingConfigProperties properties;
- private final JsonSerializer<JobStatus> statusJsonSerializer;
- private OqmTopic oqmTopic;
- private Gson gson;
-
- @PostConstruct
- public void setUp() {
- this.oqmTopic = OqmTopic.builder().name(properties.getStatusChangedTopicName()).build();
- this.gson = new GsonBuilder()
- .registerTypeHierarchyAdapter(JobStatus.class, statusJsonSerializer)
- .create();
- }
-
- @Override
- public void publishStatusChangedTagsToTopic(DpsHeaders headers, JobStatus indexerBatchStatus) {
- OqmDestination oqmDestination =
- OqmDestination.builder().partitionId(headers.getPartitionId()).build();
- String json = this.gson.toJson(indexerBatchStatus);
-
- Map<String, String> attributes = getAttributes(headers);
- OqmMessage oqmMessage = OqmMessage.builder().data(json).attributes(attributes).build();
- driver.publish(oqmMessage, oqmTopic, oqmDestination);
- }
-
- private Map<String, String> getAttributes(DpsHeaders headers) {
- Map<String, String> attributes = new HashMap<>();
- attributes.put(DpsHeaders.ACCOUNT_ID, headers.getPartitionIdWithFallbackToAccountId());
- attributes.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
- headers.addCorrelationIdIfMissing();
- attributes.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
- return attributes;
- }
-}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/util/ServiceAccountJwtGcpClientImpl.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/util/ServiceAccountJwtGcpClientImpl.java
deleted file mode 100644
index 698046499b105c845ce0c4d3d780be56afc833e1..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/util/ServiceAccountJwtGcpClientImpl.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.provider.gcp.common.util;
-
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.opengroup.osdu.core.auth.TokenProvider;
-import org.opengroup.osdu.core.common.util.IServiceAccountJwtClient;
-import org.springframework.stereotype.Component;
-
-@Slf4j
-@Component
-@RequiredArgsConstructor
-public class ServiceAccountJwtGcpClientImpl implements IServiceAccountJwtClient {
-
- private final TokenProvider tokenProvider;
-
- @Override
- public String getIdToken(String tenantName) {
- log.debug("Tenant name received for auth token is: {}", tenantName);
- return "Bearer " + tokenProvider.getIdToken();
- }
-}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/formatter/GoogleJsonFormatter.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/formatter/GoogleJsonFormatter.java
new file mode 100644
index 0000000000000000000000000000000000000000..4b5db20d66e384f950357bbc1b7fb341f45beb39
--- /dev/null
+++ b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/formatter/GoogleJsonFormatter.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2020-2024 Google LLC
+ * Copyright 2020-2024 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.opengroup.osdu.indexer.provider.gcp.formatter;
+
+import ch.qos.logback.contrib.jackson.JacksonJsonFormatter;
+import java.io.IOException;
+import java.util.Map;
+
+public class GoogleJsonFormatter extends JacksonJsonFormatter {
+
+ private static final String SEVERITY = "severity";
+ private static final String LEVEL = "level";
+
+ @Override
+ public String toJsonString(Map map) throws IOException {
+ map.put(SEVERITY, map.remove(LEVEL));
+ return super.toJsonString(map);
+ }
+}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/config/CustomContextConfiguration.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/config/CustomContextConfiguration.java
deleted file mode 100644
index 4f10b45c4e738d8342f72b0b54be8db925b32737..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/config/CustomContextConfiguration.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.provider.gcp.indexing.config;
-
-import java.util.Arrays;
-import jakarta.annotation.PostConstruct;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.opengroup.osdu.indexer.IndexerApplication;
-import org.opengroup.osdu.indexer.ServerletInitializer;
-import org.opengroup.osdu.indexer.provider.gcp.web.config.WebAppMainContextConfiguration;
-import org.opengroup.osdu.indexer.provider.gcp.web.security.GcpSecurityConfig;
-import org.opengroup.osdu.indexer.swagger.SwaggerConfiguration;
-import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
-import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
-import org.springframework.boot.autoconfigure.security.servlet.SecurityFilterAutoConfiguration;
-import org.springframework.boot.context.properties.EnableConfigurationProperties;
-import org.springframework.context.ApplicationContext;
-import org.springframework.context.annotation.ComponentScan;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.FilterType;
-import org.springframework.context.annotation.PropertySource;
-
-@Slf4j
-@Configuration
-@EnableConfigurationProperties
-@PropertySource("classpath:application.properties")
-@RequiredArgsConstructor
-@ComponentScan(
- value = {"org.opengroup.osdu"},
- excludeFilters = {
- @ComponentScan.Filter(
- type = FilterType.ASSIGNABLE_TYPE,
- value = {
- WebAppMainContextConfiguration.class,
- IndexerApplication.class,
- ServerletInitializer.class,
- SwaggerConfiguration.class,
- GcpSecurityConfig.class,
- SecurityAutoConfiguration.class,
- ManagementWebSecurityAutoConfiguration.class,
- SecurityFilterAutoConfiguration.class
- })
- })
-public class CustomContextConfiguration {
-
- private final ApplicationContext applicationContext;
-
- @PostConstruct
- public void setUp() {
- log.debug("Messaging context initialized with id: {}.", applicationContext.getId());
- log.debug("Messaging context status: {}.", applicationContext);
- String[] allBeansNames = applicationContext.getBeanDefinitionNames();
- log.debug("Messaging context beans definitions: {}.", Arrays.toString(allBeansNames));
- }
-}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/config/MessagingConfigProperties.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/config/MessagingConfigProperties.java
deleted file mode 100644
index 384768395976742f72174fae415e52e4bbd46302..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/config/MessagingConfigProperties.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Copyright 2020-2023 Google LLC
- * Copyright 2020-2023 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.provider.gcp.indexing.config;
-
-import lombok.Getter;
-import lombok.Setter;
-import org.springframework.boot.context.properties.ConfigurationProperties;
-import org.springframework.context.annotation.Configuration;
-
-@Setter
-@Getter
-@ConfigurationProperties
-@Configuration
-public class MessagingConfigProperties {
- @Deprecated
- private String defaultRelativeIndexerWorkerUrl;
- private String recordsChangedTopicName;
- private String schemaChangedTopicName;
- private String statusChangedTopicName;
- private String reprocessTopicName;
- private String reindexTopicName;
-}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/config/ScopeModifierPostProcessor.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/config/ScopeModifierPostProcessor.java
deleted file mode 100644
index f1b4ce0b80bbbd86d947b2f486fd97238b30a22a..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/config/ScopeModifierPostProcessor.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.provider.gcp.indexing.config;
-
-import java.util.Objects;
-import lombok.extern.slf4j.Slf4j;
-import org.opengroup.osdu.indexer.provider.gcp.indexing.thread.ThreadScope;
-import org.springframework.beans.BeansException;
-import org.springframework.beans.factory.config.BeanDefinition;
-import org.springframework.beans.factory.config.BeanFactoryPostProcessor;
-import org.springframework.beans.factory.config.ConfigurableListableBeanFactory;
-import org.springframework.stereotype.Component;
-
-@Slf4j
-@Component
-public class ScopeModifierPostProcessor implements BeanFactoryPostProcessor {
-
- public static final String SCOPE_THREAD = "scope_thread";
-
- @Override
- public void postProcessBeanFactory(ConfigurableListableBeanFactory factory) throws BeansException {
- factory.registerScope(SCOPE_THREAD, new ThreadScope());
-
- for (String beanName : factory.getBeanDefinitionNames()) {
- BeanDefinition beanDef = factory.getBeanDefinition(beanName);
- if (Objects.equals(beanDef.getScope(), "request")) {
- beanDef.setScope(SCOPE_THREAD);
- log.debug("Scope has been overridden for bean: {}", beanDef.getBeanClassName());
- }
- }
- }
-}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/initialization/OqmSubscriberManager.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/initialization/OqmSubscriberManager.java
deleted file mode 100644
index d4491b0ab1de0737f8b97cf6fb742de2797f9275..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/initialization/OqmSubscriberManager.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Copyright 2020-2023 Google LLC
- * Copyright 2020-2023 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.provider.gcp.indexing.initialization;
-
-import jakarta.annotation.Nullable;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.gcp.oqm.driver.OqmDriver;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmDestination;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmMessageReceiver;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmSubscriber;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmSubscriberThroughput;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmSubscription;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmSubscriptionQuery;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmTopic;
-import org.springframework.http.HttpStatus;
-import org.springframework.stereotype.Service;
-
-@Service
-@Slf4j
-@RequiredArgsConstructor
-public class OqmSubscriberManager {
-
- private final OqmDriver driver;
-
- public void registerSubscriber(String dataPartitionId, String topicName, String subscriptionName,
- OqmMessageReceiver messageReceiver, OqmSubscriberThroughput throughput) {
- OqmSubscription subscriptionForTenant = getSubscriptionForTenant(dataPartitionId, topicName, subscriptionName);
- log.info("OQM: registering Subscriber for subscription {}", subscriptionName);
-
- OqmDestination destination = getDestination(dataPartitionId);
- OqmSubscriber subscriber = OqmSubscriber.builder()
- .subscription(subscriptionForTenant)
- .messageReceiver(messageReceiver)
- .throughput(throughput)
- .build();
-
- driver.subscribe(subscriber, destination);
- log.info("OQM: provisioning subscription {}: Subscriber REGISTERED.", subscriptionName);
- }
-
- private OqmSubscription getSubscriptionForTenant(String dataPartitionId, String topicName, String subscriptionName) {
- log.info("OQM: provisioning tenant {}:", dataPartitionId);
- log.info("OQM: check for topic {} existence:", topicName);
- OqmTopic topic = driver.getTopic(topicName, getDestination(dataPartitionId)).orElse(null);
-
- if (topic == null) {
- log.error("OQM: check for topic: {}, tenant: {} existence: ABSENT.", topicName, dataPartitionId);
- throw new AppException(
- HttpStatus.INTERNAL_SERVER_ERROR.value(),
- "Required topic not exists.",
- String.format(
- "Required topic not exists. Create topic: %s for tenant: %s and restart service.",
- topicName, dataPartitionId
- )
- );
- }
-
- log.info("OQM: check for topic {} existence: PRESENT", topicName);
- OqmSubscription subscription = getSubscription(dataPartitionId, topic, subscriptionName);
-
- if (subscription == null) {
- log.error(
- "OQM: check for subscription {}, tenant: {} existence: ABSENT.",
- subscriptionName,
- dataPartitionId
- );
- throw new AppException(
- HttpStatus.INTERNAL_SERVER_ERROR.value(),
- "Required subscription not exists.",
- String.format(
- "Required subscription not exists. Create subscription: %s for tenant: %s and restart service.",
- subscriptionName,
- dataPartitionId
- )
- );
- }
- log.info("OQM: provisioning tenant {}: COMPLETED.", dataPartitionId);
- return subscription;
- }
-
- @Nullable
- private OqmSubscription getSubscription(String dataPartitionId, OqmTopic topic, String subscriptionName) {
- log.info("OQM: check for subscription {} existence:", subscriptionName);
- OqmSubscriptionQuery query = OqmSubscriptionQuery.builder()
- .namePrefix(subscriptionName)
- .subscriberable(true)
- .build();
- return driver
- .listSubscriptions(topic, query, getDestination(dataPartitionId)).stream()
- .findAny()
- .orElse(null);
- }
-
- private OqmDestination getDestination(String dataPartitionId) {
- return OqmDestination.builder()
- .partitionId(dataPartitionId)
- .build();
- }
-}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/initialization/TenantSubscriberConfiguration.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/initialization/TenantSubscriberConfiguration.java
deleted file mode 100644
index 60914db36a899d667e8f7d6a82fdbd9a3a86b295..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/initialization/TenantSubscriberConfiguration.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Copyright 2020-2023 Google LLC
- * Copyright 2020-2023 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.provider.gcp.indexing.initialization;
-
-import jakarta.annotation.PostConstruct;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.opengroup.osdu.core.auth.TokenProvider;
-import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
-import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmSubscriberThroughput;
-import org.opengroup.osdu.indexer.api.RecordIndexerApi;
-import org.opengroup.osdu.indexer.api.ReindexApi;
-import org.opengroup.osdu.indexer.provider.gcp.indexing.config.MessagingConfigProperties;
-import org.opengroup.osdu.indexer.provider.gcp.indexing.processing.RecordsChangedMessageReceiver;
-import org.opengroup.osdu.indexer.provider.gcp.indexing.processing.ReindexMessageReceiver;
-import org.opengroup.osdu.indexer.provider.gcp.indexing.processing.ReprocessorMessageReceiver;
-import org.opengroup.osdu.indexer.provider.gcp.indexing.processing.SchemaChangedMessageReceiver;
-import org.opengroup.osdu.indexer.provider.gcp.indexing.scope.ThreadDpsHeaders;
-import org.springframework.stereotype.Component;
-
-/**
- * Subscription configuration class.
- */
-@Slf4j
-@Component
-@RequiredArgsConstructor
-public class TenantSubscriberConfiguration {
-
- private static final String SUBSCRIPTION_PREFIX = "indexer-";
- private final MessagingConfigProperties properties;
- private final OqmSubscriberManager subscriberManager;
- private final ITenantFactory tenantInfoFactory;
- private final TokenProvider tokenProvider;
- private final ThreadDpsHeaders headers;
- private final RecordIndexerApi recordIndexerApi;
- private final ReindexApi reindexApi;
-
- /**
- * Tenant configurations provided by the Partition service will be used to configure subscribers. If tenants use the
- * same message broker(The same RabbitMQ instance, or the same GCP project Pub/Sub) then only one subscriber in this
- * broker will be used.
- */
- @PostConstruct
- void postConstruct() {
- log.info("OqmSubscriberManager provisioning STARTED");
- String recordsChangedTopicName = properties.getRecordsChangedTopicName();
- String schemaChangedTopicName = properties.getSchemaChangedTopicName();
- String reprocessTopicName = properties.getReprocessTopicName();
- String reindexTopicName = properties.getReindexTopicName();
-
- for (TenantInfo tenantInfo : tenantInfoFactory.listTenantInfo()) {
- String dataPartitionId = tenantInfo.getDataPartitionId();
- subscriberManager.registerSubscriber(
- dataPartitionId,
- recordsChangedTopicName,
- getSubscriptionName(recordsChangedTopicName),
- new RecordsChangedMessageReceiver(headers, tokenProvider, recordIndexerApi),
- OqmSubscriberThroughput.MAX
- );
- subscriberManager.registerSubscriber(
- dataPartitionId,
- schemaChangedTopicName,
- getSubscriptionName(schemaChangedTopicName),
- new SchemaChangedMessageReceiver(headers, tokenProvider, recordIndexerApi),
- OqmSubscriberThroughput.MIN
- );
- subscriberManager.registerSubscriber(
- dataPartitionId,
- reprocessTopicName,
- getSubscriptionName(reprocessTopicName),
- new ReprocessorMessageReceiver(headers, tokenProvider, reindexApi),
- OqmSubscriberThroughput.MIN
- );
- subscriberManager.registerSubscriber(
- dataPartitionId,
- reindexTopicName,
- getSubscriptionName(reindexTopicName),
- new ReindexMessageReceiver(headers, tokenProvider, recordIndexerApi),
- OqmSubscriberThroughput.MAX
- );
- }
- log.info("OqmSubscriberManager provisioning COMPLETED");
- }
-
- private String getSubscriptionName(String topicName) {
- return SUBSCRIPTION_PREFIX + topicName;
- }
-}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/package-info.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/package-info.java
deleted file mode 100644
index 4cc667aa15649f7e9022b31c829f95078f479270..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/package-info.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * This module is used to override request scope bean configuration enforced by the common code, to untie indexing from the web environment and allow async task
- * processing in a pulling manner. And configure non web context, which should not process user requests.
- * <p>
- * As a replacement for @RequestScope, ThreadScope is used, implementation based on SimpleThreadScope provided by Spring. Beans configuration provided by the
- * common code is overriden with help of BeanFactoryPostProcessor.
- * </p>
- */
-package org.opengroup.osdu.indexer.provider.gcp.indexing;
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/IndexerOqmMessageReceiver.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/IndexerOqmMessageReceiver.java
deleted file mode 100644
index 47f5d24015e41ab3cafe81580ad6687729a61cbc..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/IndexerOqmMessageReceiver.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Copyright 2020-2023 Google LLC
- * Copyright 2020-2023 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.provider.gcp.indexing.processing;
-
-import com.google.common.base.Strings;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.opengroup.osdu.core.auth.TokenProvider;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmAckReplier;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmMessage;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmMessageReceiver;
-import org.opengroup.osdu.indexer.provider.gcp.indexing.scope.ThreadDpsHeaders;
-import org.opengroup.osdu.indexer.provider.gcp.indexing.thread.ThreadScopeContextHolder;
-
-import jakarta.validation.constraints.NotNull;
-import java.util.Optional;
-import org.opengroup.osdu.indexer.schema.converter.exeption.SchemaProcessingException;
-
-@Slf4j
-@RequiredArgsConstructor
-public abstract class IndexerOqmMessageReceiver implements OqmMessageReceiver {
-
- protected final ThreadDpsHeaders dpsHeaders;
- private final TokenProvider tokenProvider;
-
- /**
- * Receives and processes an OQM message, emulating the behavior of the previous IndexQueue service and message bus.
- * Validates the input payload and handles exceptions appropriately for error processing precision.
- *
- * @param oqmMessage the OQM message to be processed
- * @param oqmAckReplier the ack replier for acknowledging message receipt
- */
- @Override
- public void receiveMessage(OqmMessage oqmMessage, OqmAckReplier oqmAckReplier) {
- log.info("OQM message: {} - {} - {}", oqmMessage.getId(), oqmMessage.getData(), oqmMessage.getAttributes());
- if (!validInput(oqmMessage)) {
- log.error("Not valid event payload, event will not be processed.");
- oqmAckReplier.ack();
- return;
- }
-
- try {
- DpsHeaders headers = getHeaders(oqmMessage);
- // Filling thread context required by the core services.
- dpsHeaders.setThreadContext(headers.getHeaders());
- sendMessage(oqmMessage);
- oqmAckReplier.ack();
- } catch (AppException appException) {
- if (isExceptionToSkip(appException)) {
- skipMessage(oqmMessage, dpsHeaders, oqmAckReplier, appException);
- } else {
- rescheduleMessage(oqmMessage, dpsHeaders, oqmAckReplier, getException(appException));
- }
- } catch (Exception exception) {
- rescheduleMessage(oqmMessage, dpsHeaders, oqmAckReplier, exception);
- } catch (Throwable e) {
- //Catching throwable is necessary, otherwise, errors like NoSuchMethodError will slip through silently.
- log.error(
- "HALT! Unrecoverable malfunction! Unexpected error was thrown during processing! Event ID: "
- + oqmMessage.getId() + ". Correlation ID: " + dpsHeaders.getCorrelationId(), e);
- throw e;
- } finally {
- // Cleaning thread context after processing is finished and the thread dies out.
- ThreadScopeContextHolder.currentThreadScopeAttributes().clear();
- }
- }
-
- /**
- * Determines whether an exception should be skipped during processing or not.
- * Exceptions within the specified range of status codes are considered skippable.
- * Additionally, SchemaProcessingException is marked as skippable, as retrying it is ineffective.
- *
- * @param exception the exception thrown during processing
- * @return true if the exception should be skipped, false otherwise
- */
- private static boolean isExceptionToSkip(AppException exception) {
- int statusCode = exception.getError().getCode();
- Exception originalException = getException(exception);
- return statusCode > 199 && statusCode < 300 || originalException instanceof SchemaProcessingException;
- }
-
- private static void skipMessage(OqmMessage oqmMessage, DpsHeaders dpsHeaders,
- OqmAckReplier oqmAckReplier, AppException appException) {
- log.info(
- "Event ID: " + oqmMessage.getId() + ". Correlation ID: " + dpsHeaders.getCorrelationId()
- + ", was not processed, and will NOT be rescheduled.", appException);
- oqmAckReplier.ack();
- }
-
- private static void rescheduleMessage(OqmMessage oqmMessage, DpsHeaders dpsHeaders,
- OqmAckReplier oqmAckReplier, Exception exception) {
- log.error(
- "Event id : " + oqmMessage.getId() + ". Correlation ID: " + dpsHeaders.getCorrelationId()
- + ", was not processed, and will BE rescheduled.", exception);
- oqmAckReplier.nack();
- }
-
- /**
- * It is possible to get both AppException that wraps original Exception
- * or the original Exception without any wrapper.
- */
- @NotNull
- private static Exception getException(AppException appException) {
- return Optional.ofNullable(appException.getOriginalException()).orElse(appException);
- }
-
- private boolean validInput(OqmMessage oqmMessage) {
- boolean isValid = true;
- if (Strings.isNullOrEmpty(oqmMessage.getData()) || oqmMessage.getData().equals("{}")) {
- log.error("Message body is empty, message id: {}, attributes: {}", oqmMessage.getId(), oqmMessage.getAttributes());
- isValid = false;
- }
- if (oqmMessage.getAttributes() == null || oqmMessage.getAttributes().size() == 0) {
- log.error("Attribute map not found, message id: {}, attributes: {}", oqmMessage.getId(), oqmMessage.getAttributes());
- isValid = false;
- }
- return isValid;
- }
-
- /**
- * Sends a message to the respective API endpoint. This method is abstract and should be implemented
- * according to the specific requirements of the endpoint.
- *
- * @param oqmMessage the message to be sent
- */
- protected abstract void sendMessage(OqmMessage oqmMessage) throws Exception;
-
- @NotNull
- private DpsHeaders getHeaders(OqmMessage oqmMessage) {
- DpsHeaders headers = DpsHeaders.createFromMap(oqmMessage.getAttributes());
- headers.getHeaders().put("authorization", "Bearer " + tokenProvider.getIdToken());
- return headers;
- }
-}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/RecordsChangedMessageReceiver.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/RecordsChangedMessageReceiver.java
deleted file mode 100644
index 9a784bf29f413544dadd2e6ed4bd9c504923e33b..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/RecordsChangedMessageReceiver.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.provider.gcp.indexing.processing;
-
-import java.time.LocalDateTime;
-import lombok.extern.slf4j.Slf4j;
-import org.opengroup.osdu.core.auth.TokenProvider;
-import org.opengroup.osdu.core.common.model.indexer.JobStatus;
-import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmMessage;
-import org.opengroup.osdu.indexer.api.RecordIndexerApi;
-import org.opengroup.osdu.indexer.provider.gcp.indexing.scope.ThreadDpsHeaders;
-import org.springframework.http.ResponseEntity;
-
-@Slf4j
-public class RecordsChangedMessageReceiver extends IndexerOqmMessageReceiver {
-
- private final RecordIndexerApi recordIndexerApi;
-
- public RecordsChangedMessageReceiver(ThreadDpsHeaders dpsHeaders, TokenProvider tokenProvider,
- RecordIndexerApi recordIndexerApi) {
- super(dpsHeaders, tokenProvider);
- this.recordIndexerApi = recordIndexerApi;
- }
-
- @Override
- protected void sendMessage(OqmMessage oqmMessage) throws Exception {
- RecordChangedMessages indexWorkerRequestBody = getIndexWorkerRequestBody(oqmMessage);
- log.debug("Job message body: {}", indexWorkerRequestBody);
- ResponseEntity<JobStatus> jobStatusResponse = recordIndexerApi.indexWorker(indexWorkerRequestBody);
- log.debug("Job status: {}", jobStatusResponse);
- }
-
- private RecordChangedMessages getIndexWorkerRequestBody(OqmMessage request) {
- RecordChangedMessages recordChangedMessages = new RecordChangedMessages();
- recordChangedMessages.setMessageId(dpsHeaders.getCorrelationId());
- recordChangedMessages.setData(request.getData());
- recordChangedMessages.setAttributes(request.getAttributes());
- recordChangedMessages.setPublishTime(LocalDateTime.now().toString());
- return recordChangedMessages;
- }
-}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/ReindexMessageReceiver.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/ReindexMessageReceiver.java
deleted file mode 100644
index e00c9c0e767fbe018618cfa9b59c7a988ce1a1cf..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/ReindexMessageReceiver.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright 2020-2023 Google LLC
- * Copyright 2020-2023 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.provider.gcp.indexing.processing;
-
-import java.time.LocalDateTime;
-import lombok.extern.slf4j.Slf4j;
-import org.opengroup.osdu.core.auth.TokenProvider;
-import org.opengroup.osdu.core.common.model.indexer.JobStatus;
-import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmMessage;
-import org.opengroup.osdu.indexer.api.RecordIndexerApi;
-import org.opengroup.osdu.indexer.provider.gcp.indexing.scope.ThreadDpsHeaders;
-import org.springframework.http.ResponseEntity;
-
-@Slf4j
-public class ReindexMessageReceiver extends IndexerOqmMessageReceiver {
-
- private final RecordIndexerApi recordIndexerApi;
-
- public ReindexMessageReceiver(ThreadDpsHeaders dpsHeaders, TokenProvider tokenProvider, RecordIndexerApi recordIndexerApi) {
- super(dpsHeaders, tokenProvider);
- this.recordIndexerApi = recordIndexerApi;
- }
-
- @Override
- protected void sendMessage(OqmMessage oqmMessage) throws Exception {
- RecordChangedMessages indexWorkerRequestBody = getIndexWorkerRequestBody(oqmMessage);
- log.debug("Reindex job message body: {}", indexWorkerRequestBody);
- ResponseEntity<JobStatus> jobStatusResponse = recordIndexerApi.indexWorker(indexWorkerRequestBody);
- log.debug("Job status: {}", jobStatusResponse);
- }
-
- private RecordChangedMessages getIndexWorkerRequestBody(OqmMessage request) {
- RecordChangedMessages recordChangedMessages = new RecordChangedMessages();
- recordChangedMessages.setMessageId(dpsHeaders.getCorrelationId());
- recordChangedMessages.setData(request.getData());
- recordChangedMessages.setAttributes(request.getAttributes());
- recordChangedMessages.setPublishTime(LocalDateTime.now().toString());
- return recordChangedMessages;
- }
-}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/ReprocessorMessageReceiver.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/ReprocessorMessageReceiver.java
deleted file mode 100644
index c5d71abf72d70f3765dda5c2999b00b0c7870969..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/ReprocessorMessageReceiver.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Copyright 2020-2023 Google LLC
- * Copyright 2020-2023 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.provider.gcp.indexing.processing;
-
-import com.google.gson.Gson;
-import lombok.extern.slf4j.Slf4j;
-import org.opengroup.osdu.core.auth.TokenProvider;
-import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmMessage;
-import org.opengroup.osdu.indexer.api.ReindexApi;
-import org.opengroup.osdu.indexer.provider.gcp.indexing.scope.ThreadDpsHeaders;
-import org.springframework.http.ResponseEntity;
-
-@Slf4j
-public class ReprocessorMessageReceiver extends IndexerOqmMessageReceiver {
-
- private final Gson gson = new Gson();
- private final ReindexApi reindexApi;
-
- public ReprocessorMessageReceiver(ThreadDpsHeaders dpsHeaders, TokenProvider tokenProvider, ReindexApi reindexApi) {
- super(dpsHeaders, tokenProvider);
- this.reindexApi = reindexApi;
- }
-
- @Override
- protected void sendMessage(OqmMessage oqmMessage) throws Exception {
- RecordReindexRequest reindexBody = getReindexBody(oqmMessage);
- log.debug("Reindex job message body: {}", reindexBody);
- ResponseEntity<?> reindexResponse = reindexApi.reindex(reindexBody, false);
- log.debug("Reindex job status: {}", reindexResponse);
- }
-
- private RecordReindexRequest getReindexBody(OqmMessage request) {
- return this.gson.fromJson(request.getData(), RecordReindexRequest.class);
- }
-}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/SchemaChangedMessageReceiver.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/SchemaChangedMessageReceiver.java
deleted file mode 100644
index cde2d8954c08e47f5b0e62f86ebbb732578044de..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/SchemaChangedMessageReceiver.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.provider.gcp.indexing.processing;
-
-import com.google.gson.Gson;
-import java.io.IOException;
-import java.time.LocalDateTime;
-import lombok.extern.slf4j.Slf4j;
-import org.opengroup.osdu.core.auth.TokenProvider;
-import org.opengroup.osdu.core.common.model.indexer.SchemaChangedMessages;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmMessage;
-import org.opengroup.osdu.indexer.api.RecordIndexerApi;
-import org.opengroup.osdu.indexer.provider.gcp.indexing.scope.ThreadDpsHeaders;
-import org.springframework.http.ResponseEntity;
-
-@Slf4j
-public class SchemaChangedMessageReceiver extends IndexerOqmMessageReceiver {
-
- private final RecordIndexerApi recordIndexerApi;
- private final Gson gson = new Gson();
-
- public SchemaChangedMessageReceiver(ThreadDpsHeaders dpsHeaders, TokenProvider tokenProvider,
- RecordIndexerApi recordIndexerApi) {
- super(dpsHeaders, tokenProvider);
- this.recordIndexerApi = recordIndexerApi;
- }
-
- @Override
- protected void sendMessage(OqmMessage oqmMessage) throws Exception {
- SchemaChangedMessages schemaChangedMessage = getSchemaWorkerRequestBody(oqmMessage);
- log.debug("Schema changed job message body: {}", schemaChangedMessage);
- ResponseEntity<?> schemaChangeResponse = recordIndexerApi.schemaWorker(schemaChangedMessage);
- log.debug("Schema changed job status: {}", schemaChangeResponse);
- }
-
- private SchemaChangedMessages getSchemaWorkerRequestBody(OqmMessage oqmMessage) {
- SchemaChangedMessages schemaChangedMessages = new SchemaChangedMessages();
- schemaChangedMessages.setMessageId(dpsHeaders.getCorrelationId());
- schemaChangedMessages.setData(oqmMessage.getData());
- schemaChangedMessages.setAttributes(oqmMessage.getAttributes());
- schemaChangedMessages.setPublishTime(LocalDateTime.now().toString());
- return schemaChangedMessages;
- }
-}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/scope/ThreadDpsHeaders.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/scope/ThreadDpsHeaders.java
deleted file mode 100644
index 3e9f007ceb7c5ab75f6c1dfcb40cc53ad8648c51..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/scope/ThreadDpsHeaders.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.provider.gcp.indexing.scope;
-
-import static org.springframework.context.annotation.ScopedProxyMode.TARGET_CLASS;
-
-import java.util.Map;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.opengroup.osdu.core.auth.TokenProvider;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.indexer.provider.gcp.indexing.config.ScopeModifierPostProcessor;
-import org.springframework.context.annotation.Primary;
-import org.springframework.context.annotation.Scope;
-import org.springframework.stereotype.Component;
-
-@Slf4j
-@Primary
-@Component
-@Scope(value = ScopeModifierPostProcessor.SCOPE_THREAD, proxyMode = TARGET_CLASS)
-@RequiredArgsConstructor
-public class ThreadDpsHeaders extends DpsHeaders {
-
- private final TokenProvider tokenProvider;
-
- public void setThreadContext(Map<String, String> headers) {
- this.put(DpsHeaders.AUTHORIZATION, "Bearer " + tokenProvider.getIdToken());
- this.addFromMap(headers);
- }
-}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/thread/ThreadScope.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/thread/ThreadScope.java
deleted file mode 100644
index c2e5fef7361b8477f2fc606b7025d9ccd21769d8..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/thread/ThreadScope.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.provider.gcp.indexing.thread;
-
-import java.util.Map;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.beans.factory.ObjectFactory;
-import org.springframework.beans.factory.config.Scope;
-
-@Slf4j
-public class ThreadScope implements Scope {
-
- public Object get(String name, ObjectFactory<?> factory) {
- log.trace("Get bean:{} with factory: {} current Thread: {}", name, factory,
- Thread.currentThread().getName());
- Object result = null;
- Map<String, Object> hBeans = ThreadScopeContextHolder.currentThreadScopeAttributes()
- .getBeanMap();
- if (!hBeans.containsKey(name)) {
- result = factory.getObject();
- log.trace(
- "No bean in context with name: {} factory provisioning result is: {} current Thread: {}",
- name, result, Thread.currentThread().getName());
- hBeans.put(name, result);
- } else {
- result = hBeans.get(name);
- }
-
- return result;
- }
-
- public Object remove(String name) {
- log.trace("Removing bean : {} current Thread: {}", name, Thread.currentThread().getName());
- Object result = null;
- Map<String, Object> hBeans = ThreadScopeContextHolder.currentThreadScopeAttributes()
- .getBeanMap();
- if (hBeans.containsKey(name)) {
- result = hBeans.get(name);
- hBeans.remove(name);
- }
-
- return result;
- }
-
- public void registerDestructionCallback(String name, Runnable callback) {
- ThreadScopeContextHolder.currentThreadScopeAttributes().registerRequestDestructionCallback(name, callback);
- }
-
- public Object resolveContextualObject(String key) {
- return null;
- }
-
- public String getConversationId() {
- return Thread.currentThread().getName();
- }
-}
-
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/thread/ThreadScopeAttributes.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/thread/ThreadScopeAttributes.java
deleted file mode 100644
index 7a9ac113c207b536d3b5df147dd3e22771b35583..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/thread/ThreadScopeAttributes.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.provider.gcp.indexing.thread;
-
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.lang.NonNull;
-
-@Slf4j
-@RequiredArgsConstructor
-public class ThreadScopeAttributes {
-
- protected final Map<String, Object> hBeans = new HashMap();
- protected final Map<String, Runnable> hRequestDestructionCallbacks = new LinkedHashMap();
-
- protected final Map<String, Object> getBeanMap() {
- return this.hBeans;
- }
-
- protected final void registerRequestDestructionCallback(
- @NonNull String name, @NonNull Runnable callback) {
- log.trace("Registering callback for: {} on runnable: {}", name, callback);
- this.hRequestDestructionCallbacks.put(name, callback);
- }
-
- public final void clear() {
- this.processDestructionCallbacks();
- this.hBeans.clear();
- }
-
- private void processDestructionCallbacks() {
- for (Map.Entry<String, Runnable> mapEntry : this.hRequestDestructionCallbacks.entrySet()) {
- Runnable callback = mapEntry.getValue();
- log.trace(
- "Performing destruction callback for: {} on thread: {}",
- mapEntry.getKey(),
- Thread.currentThread().getName());
- callback.run();
- }
- this.hRequestDestructionCallbacks.clear();
- }
-}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/thread/ThreadScopeContextHolder.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/thread/ThreadScopeContextHolder.java
deleted file mode 100644
index 24d7c2ef69103f9181a620ac7e88bcd1f464cb26..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/thread/ThreadScopeContextHolder.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.provider.gcp.indexing.thread;
-
-public class ThreadScopeContextHolder {
-
- private static final ThreadLocal<ThreadScopeAttributes> threadScopeAttributesHolder =
- new InheritableThreadLocal<ThreadScopeAttributes>() {
- @Override
- protected ThreadScopeAttributes initialValue() {
- return new ThreadScopeAttributes();
- }
- };
-
- private ThreadScopeContextHolder() {
- }
-
- public static ThreadScopeAttributes getThreadScopeAttributes() {
- return threadScopeAttributesHolder.get();
- }
-
- public static void setThreadScopeAttributes(ThreadScopeAttributes accessor) {
- threadScopeAttributesHolder.set(accessor);
- }
-
- public static ThreadScopeAttributes currentThreadScopeAttributes() throws IllegalStateException {
- ThreadScopeAttributes accessor = threadScopeAttributesHolder.get();
- if (accessor == null) {
- throw new IllegalStateException("No thread scoped attributes.");
- } else {
- return accessor;
- }
- }
-}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/web/config/WebAppMainContextConfiguration.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/web/config/WebAppMainContextConfiguration.java
deleted file mode 100644
index 58563b57ceeca8c608e81c9dd564234ed926cf17..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/web/config/WebAppMainContextConfiguration.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.provider.gcp.web.config;
-
-import java.util.Arrays;
-import jakarta.annotation.PostConstruct;
-import jakarta.servlet.http.HttpServletRequest;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.opengroup.osdu.core.common.http.DpsHeaderFactory;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.indexer.IndexerApplication;
-import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
-import org.springframework.context.ApplicationContext;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.ComponentScan;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.FilterType;
-import org.springframework.context.annotation.Primary;
-import org.springframework.context.annotation.PropertySource;
-import org.springframework.web.context.annotation.RequestScope;
-
-@Slf4j
-@Configuration
-@EnableAutoConfiguration
-@PropertySource("classpath:application.properties")
-@RequiredArgsConstructor
-@ComponentScan(
- value = {"org.opengroup.osdu"},
- excludeFilters = {
- @ComponentScan.Filter(
- type = FilterType.ASSIGNABLE_TYPE,
- value = {
- IndexerApplication.class
- }),
- @ComponentScan.Filter(
- type = FilterType.REGEX,
- pattern = {"org\\.opengroup\\.osdu\\.indexer\\.provider\\.gcp\\.indexing\\..*"}
- )
- }
-)
-public class WebAppMainContextConfiguration {
-
- private final ApplicationContext applicationContext;
-
- @PostConstruct
- public void setUp() {
- log.debug("Main web app context initialized with id: {}.", applicationContext.getId());
- log.debug("Main web app context status: {}.", applicationContext);
- String[] allBeansNames = applicationContext.getBeanDefinitionNames();
- log.debug("Main web app context beans definitions: {}.", Arrays.toString(allBeansNames));
- }
-
- @Primary
- @Bean
- @RequestScope
- public DpsHeaders dpsHeaders(HttpServletRequest request) {
- return new DpsHeaderFactory(request);
- }
-}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/web/di/TenantInfoService.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/web/di/TenantInfoService.java
deleted file mode 100644
index 9a432b6d51a5e878953c78110658106263df86e7..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/web/di/TenantInfoService.java
+++ /dev/null
@@ -1,32 +0,0 @@
-package org.opengroup.osdu.indexer.provider.gcp.web.di;
-
-import java.util.ArrayList;
-import java.util.List;
-import jakarta.inject.Inject;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
-import org.opengroup.osdu.core.common.multitenancy.ITenantInfoService;
-import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
-import org.springframework.stereotype.Component;
-import org.springframework.web.context.annotation.RequestScope;
-
-@RequestScope
-@Component
-public class TenantInfoService implements ITenantInfoService {
-
- @Inject
- private ITenantFactory tenantFactory;
-
- @Inject
- private DpsHeaders headers;
-
- @Override
- public TenantInfo getTenantInfo() {
- return tenantFactory.getTenantInfo(headers.getPartitionId());
- }
-
- @Override
- public List<TenantInfo> getAllTenantInfos() {
- return new ArrayList<>(tenantFactory.listTenantInfo());
- }
-}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/web/middleware/IndexFilter.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/web/middleware/IndexFilter.java
deleted file mode 100644
index 85456fbcbd8118bdb2f8abc8e4d6c937ee3b387e..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/web/middleware/IndexFilter.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- Copyright 2020 Google LLC
- Copyright 2020 EPAM Systems, Inc
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.provider.gcp.web.middleware;
-
-import com.google.common.base.Strings;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import jakarta.inject.Inject;
-import jakarta.servlet.Filter;
-import jakarta.servlet.FilterChain;
-import jakarta.servlet.FilterConfig;
-import jakarta.servlet.ServletException;
-import jakarta.servlet.ServletRequest;
-import jakarta.servlet.ServletResponse;
-import jakarta.servlet.http.HttpServletRequest;
-import jakarta.servlet.http.HttpServletResponse;
-import lombok.extern.java.Log;
-import org.apache.http.HttpStatus;
-import org.opengroup.osdu.core.common.http.ResponseHeaders;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.search.DeploymentEnvironment;
-import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.springframework.http.HttpMethod;
-import org.springframework.stereotype.Component;
-
-@Log
-@Component
-public class IndexFilter implements Filter {
-
- @Inject
- private DpsHeaders dpsHeaders;
-
- @Inject
- private IRequestInfo requestInfo;
-
- @Inject
- private IndexerConfigurationProperties properties;
-
- private static final String PATH_TASK_HANDLERS = "task-handlers";
- private static final String PATH_CRON_HANDLERS = "cron-handlers";
-
- @Override
- public void init(FilterConfig filterConfig) {
- }
-
- @Override
- public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain)
- throws IOException, ServletException {
- HttpServletRequest httpRequest = (HttpServletRequest) servletRequest;
- String uri = httpRequest.getRequestURI().toLowerCase();
-
- if (httpRequest.getMethod().equalsIgnoreCase(HttpMethod.POST.name()) && uri.contains(PATH_TASK_HANDLERS)) {
- if (properties.getDeploymentEnvironment() != DeploymentEnvironment.LOCAL) {
- checkWorkerApiAccess(requestInfo);
- }
- }
-
- if (httpRequest.getMethod().equalsIgnoreCase(HttpMethod.GET.name()) && uri.contains(PATH_CRON_HANDLERS)) {
- checkWorkerApiAccess(requestInfo);
- }
-
- filterChain.doFilter(servletRequest, servletResponse);
-
- HttpServletResponse httpResponse = (HttpServletResponse) servletResponse;
- Map<String, List<Object>> standardHeaders = ResponseHeaders.STANDARD_RESPONSE_HEADERS;
- for (Map.Entry<String, List<Object>> header : standardHeaders.entrySet()) {
- httpResponse.addHeader(header.getKey(), header.getValue().toString());
- }
- if (httpResponse.getHeader(DpsHeaders.CORRELATION_ID) == null) {
- httpResponse.addHeader(DpsHeaders.CORRELATION_ID, dpsHeaders.getCorrelationId());
- }
-
- }
-
- @Override
- public void destroy() {
- }
-
- private void checkWorkerApiAccess(IRequestInfo requestInfo) {
- if (requestInfo.isTaskQueueRequest()) {
- return;
- }
- throw AppException.createForbidden("invalid user agent, AppEngine Task Queue only");
- }
-
- private List<String> validateAccountId(DpsHeaders requestHeaders) {
- String accountHeader = requestHeaders.getPartitionIdWithFallbackToAccountId();
- String debuggingInfo = String.format("%s:%s", DpsHeaders.DATA_PARTITION_ID, accountHeader);
-
- if (Strings.isNullOrEmpty(accountHeader)) {
- throw new AppException(HttpStatus.SC_BAD_REQUEST, "Bad request", "invalid or empty data partition", debuggingInfo);
- }
-
- List<String> dataPartitions = Arrays.asList(accountHeader.trim().split("\\s*,\\s*"));
- if (dataPartitions.isEmpty() || dataPartitions.size() > 1) {
- throw new AppException(HttpStatus.SC_BAD_REQUEST, "Bad request", "invalid or empty data partition", debuggingInfo);
- }
- return dataPartitions;
- }
-
-}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/web/security/GcpSecurityConfig.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/web/security/GcpSecurityConfig.java
deleted file mode 100644
index 0ea195d2e92f0705b5f2238a07a9ba4cff5f7d1e..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/web/security/GcpSecurityConfig.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- Copyright 2020 Google LLC
- Copyright 2020 EPAM Systems, Inc
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.provider.gcp.web.security;
-
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.security.config.annotation.method.configuration.EnableMethodSecurity;
-import org.springframework.security.config.annotation.web.builders.HttpSecurity;
-import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
-import org.springframework.security.config.annotation.web.configuration.WebSecurityCustomizer;
-import org.springframework.security.config.annotation.web.configurers.AbstractHttpConfigurer;
-import org.springframework.security.config.http.SessionCreationPolicy;
-import org.springframework.security.web.SecurityFilterChain;
-import static org.springframework.security.config.Customizer.withDefaults;
-
-@Configuration
-@EnableWebSecurity
-@EnableMethodSecurity
-public class GcpSecurityConfig {
-
- @Bean
- public SecurityFilterChain securityFilterChain(HttpSecurity http) throws Exception {
- http
- .csrf(AbstractHttpConfigurer::disable)
- .sessionManagement(session -> session.sessionCreationPolicy(SessionCreationPolicy.STATELESS))
- .authorizeHttpRequests(authorize -> authorize.anyRequest().permitAll())
- .httpBasic(withDefaults());
- return http.build();
- }
-
- @Bean
- public WebSecurityCustomizer webSecurityCustomizer() {
- return (web) -> web.ignoring().requestMatchers("/api-docs", "/info", "/swagger");
- }
-
-}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/web/util/AppExceptionHandler.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/web/util/AppExceptionHandler.java
deleted file mode 100644
index a766de2693d24cbc2ac119796e198d2881692363..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/web/util/AppExceptionHandler.java
+++ /dev/null
@@ -1,58 +0,0 @@
-package org.opengroup.osdu.indexer.provider.gcp.web.util;
-
-import java.util.Objects;
-import lombok.extern.slf4j.Slf4j;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.springframework.http.HttpStatus;
-import org.springframework.http.ResponseEntity;
-import org.springframework.web.bind.annotation.ControllerAdvice;
-import org.springframework.web.bind.annotation.ExceptionHandler;
-
-@ControllerAdvice
-@Slf4j
-public class AppExceptionHandler {
-
- @ExceptionHandler(AppException.class)
- public ResponseEntity<Object> handleAppExceptions(AppException e) {
- return this.getErrorResponse(e);
- }
-
- private ResponseEntity<Object> getErrorResponse(AppException e) {
-
- String exceptionMsg = Objects.nonNull(e.getOriginalException())
- ? e.getOriginalException().getMessage()
- : e.getError().getMessage();
-
- Integer errorCode = e.getError().getCode();
-
- if (errorCode > 499) {
- log.error(exceptionMsg, e.getOriginalException());
- } else {
- log.warn(exceptionMsg, e.getOriginalException());
- }
-
- HttpStatus status = Objects.nonNull(HttpStatus.resolve(errorCode))
- ? HttpStatus.resolve(errorCode)
- : resolveNotSupportedStatus(errorCode);
-
- return new ResponseEntity<>(e.getError(), status);
- }
-
- //Currently not all codes provided from core can be resolved by HttpStatus
- //example org.opengroup.osdu.core.common.model.http.RequestStatus have not supported by HttpStatus codes
- private HttpStatus resolveNotSupportedStatus(int statusCode) {
- if (statusCode > 99 && statusCode < 200) {
- return HttpStatus.CONTINUE;
- }
- if (statusCode > 199 && statusCode < 300) {
- return HttpStatus.NO_CONTENT;
- }
- if (statusCode > 299 && statusCode < 400) {
- return HttpStatus.MULTIPLE_CHOICES;
- }
- if (statusCode > 399 && statusCode < 500) {
- return HttpStatus.BAD_REQUEST;
- }
- return HttpStatus.INTERNAL_SERVER_ERROR;
- }
-}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/web/util/RequestInfoImpl.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/web/util/RequestInfoImpl.java
deleted file mode 100644
index cc5f1c6d09a4fe6aa55a1337f98a20683f178cb0..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/web/util/RequestInfoImpl.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- Copyright 2020 Google LLC
- Copyright 2020 EPAM Systems, Inc
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.provider.gcp.web.util;
-
-import static org.opengroup.osdu.core.common.model.http.DpsHeaders.AUTHORIZATION;
-
-import com.google.common.base.Strings;
-import java.util.Map;
-import java.util.logging.Level;
-import jakarta.inject.Inject;
-import lombok.extern.java.Log;
-import org.apache.http.HttpStatus;
-import org.opengroup.osdu.core.auth.TokenProvider;
-import org.opengroup.osdu.core.common.Constants;
-import org.opengroup.osdu.core.common.model.entitlements.AuthorizationResponse;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.search.DeploymentEnvironment;
-import org.opengroup.osdu.core.common.model.search.SearchServiceRole;
-import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
-import org.opengroup.osdu.core.common.provider.interfaces.IAuthorizationService;
-import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
-import org.opengroup.osdu.core.gcp.model.CloudTaskHeaders;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.springframework.stereotype.Component;
-
-
-@Log
-@Component
-public class RequestInfoImpl implements IRequestInfo {
-
- @Inject
- private DpsHeaders dpsHeaders;
-
- @Inject
- private TokenProvider tokenProvider;
-
- @Inject
- private TenantInfo tenantInfo;
-
- @Inject
- private IndexerConfigurationProperties properties;
-
- @Inject
- private IAuthorizationService authorizationService;
-
- private static final String EXPECTED_CRON_HEADER_VALUE = "true";
-
- @Override
- public DpsHeaders getHeaders() {
-
- return this.dpsHeaders;
- }
-
- @Override
- public String getPartitionId() {
- return this.dpsHeaders.getPartitionId();
- }
-
- @Override
- public Map<String, String> getHeadersMap() {
- return this.dpsHeaders.getHeaders();
- }
-
- @Override
- public Map<String, String> getHeadersMapWithDwdAuthZ() {
- return getHeadersWithDwdAuthZ().getHeaders();
- }
-
- @Override
- public DpsHeaders getHeadersWithDwdAuthZ() {
- // Update DpsHeaders so that service account creds are passed down
- this.dpsHeaders.put(AUTHORIZATION, this.checkOrGetAuthorizationHeader());
- return this.dpsHeaders;
- }
-
- @Override
- public boolean isCronRequest() {
- String appEngineCronHeader = this.dpsHeaders.getHeaders().getOrDefault(CloudTaskHeaders.CLOUD_CRON_SERVICE, null);
- return EXPECTED_CRON_HEADER_VALUE.equalsIgnoreCase(appEngineCronHeader);
- }
-
- @Override
- public boolean isTaskQueueRequest() {
- if (this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.CLOUD_TASK_QUEUE_NAME)) {
- log.log(Level.INFO, "Request confirmed as cloud task, token validation in progress");
- return isCloudTaskRequest();
- }
- if (this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.APPENGINE_TASK_QUEUE_NAME)) {
- log.log(Level.INFO, "Request confirmed as AppEngine, headers validation in progress");
- return isAppEngineTaskRequest();
- }
- return false;
- }
-
- private boolean isCloudTaskRequest() {
- AuthorizationResponse authResponse = authorizationService.authorizeAny(dpsHeaders, SearchServiceRole.ADMIN);
- dpsHeaders.put(DpsHeaders.USER_EMAIL, authResponse.getUser());
- return true;
- }
-
- private boolean isAppEngineTaskRequest() {
- if (!this.dpsHeaders.getHeaders().containsKey(CloudTaskHeaders.APPENGINE_TASK_QUEUE_NAME)) {
- return false;
- }
- String queueId = this.dpsHeaders.getHeaders().get(CloudTaskHeaders.APPENGINE_TASK_QUEUE_NAME);
- return queueId.endsWith(Constants.INDEXER_QUEUE_IDENTIFIER);
- }
-
- public String checkOrGetAuthorizationHeader() {
- if (properties.getDeploymentEnvironment() == DeploymentEnvironment.LOCAL) {
- String authHeader = this.dpsHeaders.getAuthorization();
- if (Strings.isNullOrEmpty(authHeader)) {
- throw new AppException(HttpStatus.SC_UNAUTHORIZED, "Invalid authorization header", "Authorization token cannot be empty");
- }
- String user = this.dpsHeaders.getUserEmail();
- if (Strings.isNullOrEmpty(user)) {
- throw new AppException(HttpStatus.SC_UNAUTHORIZED, "Invalid user header", "User header cannot be empty");
- }
- return authHeader;
- } else {
- return "Bearer " + this.tokenProvider.getIdToken();
- }
- }
-}
diff --git a/provider/indexer-gc/src/main/resources/application-anthos.properties b/provider/indexer-gc/src/main/resources/application-anthos.properties
deleted file mode 100644
index 491a6ee820619c5ef157905ab94779d8580a14e3..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/resources/application-anthos.properties
+++ /dev/null
@@ -1,8 +0,0 @@
-oqmDriver=rabbitmq
-service.token.provider=OPENID
-partition-auth-enabled=false
-openid.provider-url=
-openid.provider-client-id=
-openid.provider-client-secret=
-rabbitmq-retry-delay=20000
-rabbitmq-retry-limit=5
diff --git a/provider/indexer-gc/src/main/resources/application-gcp.properties b/provider/indexer-gc/src/main/resources/application-gcp.properties
deleted file mode 100644
index 2e6be18d1a00c96e97bb3567da10699f5201428e..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/main/resources/application-gcp.properties
+++ /dev/null
@@ -1,4 +0,0 @@
-oqmDriver=pubsub
-service.token.provider=GCP
-partition-auth-enabled=true
-dead-lettering-required=true
diff --git a/provider/indexer-gc/src/main/resources/application.properties b/provider/indexer-gc/src/main/resources/application.properties
index 61dbc9c4d923b835586db826d20479749a428342..f25237671e243c50172c33a8378db9529866b1c6 100644
--- a/provider/indexer-gc/src/main/resources/application.properties
+++ b/provider/indexer-gc/src/main/resources/application.properties
@@ -10,6 +10,8 @@ logging.level.org.springframework=${LOG_LEVEL:DEBUG}
JAVA_OPTS=-Xms3072m -Xmx3072m
JAVA_GC_OPTS=-XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45
+googleAudiences=${GOOGLE_AUDIENCES:osdu}
+
#Default Cache Settings
schema-cache-expiration=60
index-cache-expiration=60
@@ -22,6 +24,8 @@ cron-empty-index-cleanup-threshold-days=7
# Indexer service config
propertyResolver.strategy=partition
+destination.resolver=partition
+
DEFAULT_DATA_COUNTRY=US
gae-service=indexer
security.https.certificate.trust=false
@@ -52,10 +56,6 @@ SCHEMA_BASE_HOST=http://schema
SCHEMA_PATH=/api/schema-service/v1/schema
SCHEMA_HOST=${SCHEMA_BASE_HOST}${SCHEMA_PATH}
-SEARCH_BASE_HOST=http://search
-SEARCH_PATH=/api/search/v2
-SEARCH_HOST=${SEARCH_BASE_HOST}${SEARCH_PATH}
-
# OQM config
records-changed-topic-name=records-changed
schema-changed-topic-name=schema-changed
@@ -64,15 +64,22 @@ reprocess-topic-name=reprocess
reindex-topic-name=reindex
# Feature flag settings
-featureFlag.strategy=appProperty
+featureFlag.strategy=dataPartition
featureFlag.xCollaboration.enabled=false
featureFlag.asIngestedCoordinates.enabled=true
featureFlag.keywordLower.enabled=true
featureFlag.bagOfWords.enabled=true
-featureFlag.xCollaboration.enabled=false
# Health checks
management.server.port=${MANAGEMENT_SERVER_PORT:8081}
management.endpoints.web.base-path=${MANAGEMENT_ENDPOINTS_WEB_BASE:/}
management.endpoints.web.exposure.include=health
management.health.probes.enabled=true
+
+elasticsearch-properties-prefix=elasticsearch.8
+
+# GC config
+oqmDriver=pubsub
+service.token.provider=GCP
+partition-auth-enabled=true
+dead-lettering-required=true
diff --git a/provider/indexer-gc/src/main/resources/logback.xml b/provider/indexer-gc/src/main/resources/logback.xml
index cdfa2e625e54da4cfb64929655942bfd3b739cbb..c27f98e52637c4c7b62ecb9bfacfe2aeb97518f9 100644
--- a/provider/indexer-gc/src/main/resources/logback.xml
+++ b/provider/indexer-gc/src/main/resources/logback.xml
@@ -1,37 +1,39 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
- <include resource="org/springframework/boot/logging/logback/defaults.xml"/>
- <logger name="org.opengroup.osdu" level="${LOG_LEVEL}"/>
- <springProfile name="local">
- <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
- <encoder>
- <pattern>%yellow([%thread]) %highlight(| %-5level |) %green(%d) %cyan(| %logger{15} |) %highlight(%msg) %n</pattern>
- <charset>utf8</charset>
- </encoder>
- </appender>
- <root level="info">
- <appender-ref ref="CONSOLE"/>
- </root>
- </springProfile>
+ <include resource="org/springframework/boot/logging/logback/defaults.xml"/>
+ <logger name="org.opengroup.osdu" level="${LOG_LEVEL}"/>
+ <springProfile name="local">
+ <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%yellow([%thread]) %highlight(| %-5level |) %green(%d) %cyan(| %logger{15} |) %highlight(%msg)
+ %n
+ </pattern>
+ <charset>utf8</charset>
+ </encoder>
+ </appender>
+ <root level="info">
+ <appender-ref ref="CONSOLE"/>
+ </root>
+ </springProfile>
- <springProfile name="!local">
- <appender name="stdout" class="ch.qos.logback.core.ConsoleAppender">
- <encoder class="ch.qos.logback.core.encoder.LayoutWrappingEncoder">
- <layout class="ch.qos.logback.contrib.json.classic.JsonLayout">
- <timestampFormat>yyyy-MM-dd HH:mm:ss.SSS</timestampFormat>
- <timestampFormatTimezoneId>Etc/UTC</timestampFormatTimezoneId>
- <appendLineSeparator>true</appendLineSeparator>
+ <springProfile name="!local">
+ <appender name="stdout" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder class="ch.qos.logback.core.encoder.LayoutWrappingEncoder">
+ <layout class="ch.qos.logback.contrib.json.classic.JsonLayout">
+ <timestampFormat>yyyy-MM-dd HH:mm:ss.SSS</timestampFormat>
+ <timestampFormatTimezoneId>Etc/UTC</timestampFormatTimezoneId>
+ <appendLineSeparator>true</appendLineSeparator>
- <jsonFormatter class="org.opengroup.osdu.core.gcp.logging.formatter.GoogleJsonFormatter">
- <prettyPrint>false</prettyPrint>
- </jsonFormatter>
- </layout>
- </encoder>
- </appender>
+ <jsonFormatter class="org.opengroup.osdu.indexer.provider.gcp.formatter.GoogleJsonFormatter">
+ <prettyPrint>false</prettyPrint>
+ </jsonFormatter>
+ </layout>
+ </encoder>
+ </appender>
- <root level="info">
- <appender-ref ref="stdout"/>
- </root>
- </springProfile>
+ <root level="info">
+ <appender-ref ref="stdout"/>
+ </root>
+ </springProfile>
</configuration>
diff --git a/provider/indexer-gc/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java b/provider/indexer-gc/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java
deleted file mode 100644
index 356f53c74100c2b5e24af1d172e47f978d7a12f1..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/test/java/org/opengroup/osdu/indexer/middleware/IndexFilterTest.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.middleware;
-
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.mockito.Mockito;
-import org.mockito.junit.MockitoJUnitRunner;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.indexer.provider.gcp.web.middleware.IndexFilter;
-
-import jakarta.servlet.FilterChain;
-import jakarta.servlet.ServletException;
-import jakarta.servlet.http.HttpServletRequest;
-import jakarta.servlet.http.HttpServletResponse;
-import java.io.IOException;
-import java.util.Collections;
-
-@RunWith(MockitoJUnitRunner.class)
-public class IndexFilterTest {
-
- @InjectMocks private IndexFilter indexFilter;
-
- @Mock private DpsHeaders dpsHeaders;
-
- @Test
- public void shouldSetCorrectResponseHeaders() throws IOException, ServletException {
- HttpServletRequest httpServletRequest = Mockito.mock(HttpServletRequest.class);
- HttpServletResponse httpServletResponse = Mockito.mock(HttpServletResponse.class);
- FilterChain filterChain = Mockito.mock(FilterChain.class);
- Mockito.when(httpServletRequest.getRequestURI()).thenReturn("https://test.com");
- Mockito.when(httpServletRequest.getMethod()).thenReturn("POST");
- Mockito.when(dpsHeaders.getCorrelationId()).thenReturn("correlation-id-value");
-
- indexFilter.doFilter(httpServletRequest, httpServletResponse, filterChain);
-
- Mockito.verify(httpServletResponse)
- .addHeader("Access-Control-Allow-Origin", Collections.singletonList("*").toString());
- Mockito.verify(httpServletResponse)
- .addHeader(
- "Access-Control-Allow-Headers",
- Collections.singletonList(
- "origin, content-type, accept, authorization, data-partition-id, correlation-id, appkey")
- .toString());
- Mockito.verify(httpServletResponse)
- .addHeader(
- "Access-Control-Allow-Methods",
- Collections.singletonList("GET, POST, PUT, DELETE, OPTIONS, HEAD, PATCH").toString());
- Mockito.verify(httpServletResponse)
- .addHeader(
- "Access-Control-Allow-Credentials", Collections.singletonList("true").toString());
- Mockito.verify(httpServletResponse)
- .addHeader("X-Frame-Options", Collections.singletonList("DENY").toString());
- Mockito.verify(httpServletResponse)
- .addHeader("X-XSS-Protection", Collections.singletonList("1; mode=block").toString());
- Mockito.verify(httpServletResponse)
- .addHeader("X-Content-Type-Options", Collections.singletonList("nosniff").toString());
- Mockito.verify(httpServletResponse)
- .addHeader(
- "Cache-Control",
- Collections.singletonList("no-cache, no-store, must-revalidate").toString());
- Mockito.verify(httpServletResponse)
- .addHeader(
- "Content-Security-Policy", Collections.singletonList("default-src 'self'").toString());
- Mockito.verify(httpServletResponse)
- .addHeader(
- "Strict-Transport-Security",
- Collections.singletonList("max-age=31536000; includeSubDomains").toString());
- Mockito.verify(httpServletResponse)
- .addHeader("Expires", Collections.singletonList("0").toString());
- Mockito.verify(httpServletResponse).addHeader("correlation-id", "correlation-id-value");
- Mockito.verify(filterChain).doFilter(httpServletRequest, httpServletResponse);
- }
-}
diff --git a/provider/indexer-gc/src/test/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/IndexerOqmMessageReceiverTest.java b/provider/indexer-gc/src/test/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/IndexerOqmMessageReceiverTest.java
deleted file mode 100644
index e26dfbcd6b2e9733802df867c912ddb1971fa851..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/test/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/IndexerOqmMessageReceiverTest.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Copyright 2020-2023 Google LLC
- * Copyright 2020-2023 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.provider.gcp.indexing.processing;
-
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.verify;
-
-import com.google.common.collect.ImmutableList;
-import com.google.gson.JsonParseException;
-import java.util.Arrays;
-import java.util.List;
-import org.apache.http.HttpStatus;
-import org.junit.Before;
-import org.junit.experimental.theories.DataPoints;
-import org.junit.experimental.theories.FromDataPoints;
-import org.junit.experimental.theories.Theories;
-import org.junit.experimental.theories.Theory;
-import org.junit.runner.RunWith;
-import org.mockito.Mockito;
-import org.opengroup.osdu.core.auth.TokenProvider;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.http.RequestStatus;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmAckReplier;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmMessage;
-import org.opengroup.osdu.indexer.provider.gcp.indexing.scope.ThreadDpsHeaders;
-import org.opengroup.osdu.indexer.schema.converter.exeption.SchemaProcessingException;
-
-@RunWith(Theories.class)
-public class IndexerOqmMessageReceiverTest {
-
- protected ThreadDpsHeaders dpsHeaders = Mockito.mock(ThreadDpsHeaders.class);
-
- protected TokenProvider tokenProvider = Mockito.mock(TokenProvider.class);
-
- protected OqmAckReplier ackReplier = Mockito.mock(OqmAckReplier.class);
-
- protected IndexerOqmMessageReceiver receiver;
-
- protected IndexerOqmMessageReceiver exceptionCheckReceiver;
-
- @Before
- public void setUp() {
- IndexerOqmMessageReceiver indexerOqmMessageReceiver = new IndexerOqmMessageReceiver(
- dpsHeaders, tokenProvider) {
- @Override
- protected void sendMessage(OqmMessage oqmMessage) throws Exception {
- //do nothing
- }
- };
- receiver = Mockito.spy(indexerOqmMessageReceiver);
- }
-
- @DataPoints("NOT_VALID_EVENTS")
- public static List<String> notValidEvents() {
- return ImmutableList.of(
- "/test-events/empty-data-event.json",
- "/test-events/empty-attributes-event.json"
- );
- }
-
- @DataPoints("RESCHEDULE_EXCEPTIONS")
- public static List<AppException> exceptionsThatShouldCauseEventRescheduling() {
- return Arrays.asList(
- new AppException(
- HttpStatus.SC_BAD_REQUEST,
- "not tested",
- "not tested",
- new JsonParseException("not tested")
- ),
- new AppException(
- HttpStatus.SC_INTERNAL_SERVER_ERROR,
- "not tested",
- "not tested",
- new Exception()
- ),
- new AppException(
- HttpStatus.SC_BAD_REQUEST,
- "not tested",
- "not tested")
- );
- }
-
- @DataPoints("SKIPPED_EXCEPTIONS")
- public static List<AppException> exceptionsThatShouldCauseEventSkip() {
- return Arrays.asList(
- new AppException(
- HttpStatus.SC_BAD_REQUEST,
- "not tested",
- "not tested",
- new SchemaProcessingException("not tested")
- ),
- new AppException(
- RequestStatus.INVALID_RECORD,
- "not tested",
- "not tested",
- new Exception()
- )
- );
- }
-
- @Theory
- public void shouldNotConsumeNotValidEvent(@FromDataPoints("NOT_VALID_EVENTS") String fileName)
- throws Exception {
- OqmMessage oqmMessage = ReadFromFileUtil.readEventFromFile(fileName);
- receiver.receiveMessage(oqmMessage, ackReplier);
- verify(ackReplier).ack();
- verify(receiver, never()).sendMessage(any());
- }
-
- @Theory
- public void shouldRescheduleForExceptions(@FromDataPoints("RESCHEDULE_EXCEPTIONS") AppException exception) {
- OqmMessage oqmMessage = ReadFromFileUtil.readEventFromFile("/test-events/storage-index-event.json");
- IndexerOqmMessageReceiver errorProneReceiver = new IndexerOqmMessageReceiver(dpsHeaders, tokenProvider) {
- @Override
- protected void sendMessage(OqmMessage oqmMessage) throws Exception {
- throw exception;
- }
- };
- errorProneReceiver.receiveMessage(oqmMessage, ackReplier);
- verify(ackReplier).nack();
- }
-
- @Theory
- public void shouldSkipForExceptions(@FromDataPoints("SKIPPED_EXCEPTIONS") AppException exception) {
- OqmMessage oqmMessage = ReadFromFileUtil.readEventFromFile("/test-events/storage-index-event.json");
- IndexerOqmMessageReceiver errorProneReceiver = new IndexerOqmMessageReceiver(dpsHeaders, tokenProvider) {
- @Override
- protected void sendMessage(OqmMessage oqmMessage) throws Exception {
- throw exception;
- }
- };
- errorProneReceiver.receiveMessage(oqmMessage, ackReplier);
- verify(ackReplier).ack();
- }
-
-}
diff --git a/provider/indexer-gc/src/test/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/ReadFromFileUtil.java b/provider/indexer-gc/src/test/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/ReadFromFileUtil.java
deleted file mode 100644
index 6a44900e75b9eea3e002cfe09d61c3aaf84a059d..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/test/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/ReadFromFileUtil.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright 2020-2023 Google LLC
- * Copyright 2020-2023 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.provider.gcp.indexing.processing;
-
-import com.google.gson.Gson;
-import com.google.gson.stream.JsonReader;
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmMessage;
-
-public class ReadFromFileUtil {
-
- private final static Gson gson = new Gson();
-
- public static OqmMessage readEventFromFile(String filename) {
- InputStream resourceAsStream = ReadFromFileUtil.class.getResourceAsStream(filename);
- BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(resourceAsStream));
- JsonReader reader = new JsonReader(bufferedReader);
- return gson.fromJson(reader, OqmMessage.class);
- }
-
-}
diff --git a/provider/indexer-gc/src/test/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/RecordsChangedMessageReceiverTest.java b/provider/indexer-gc/src/test/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/RecordsChangedMessageReceiverTest.java
deleted file mode 100644
index 72540d02f6fce90d93bb9997740d56d993c9f416..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/test/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/RecordsChangedMessageReceiverTest.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Copyright 2020-2023 Google LLC
- * Copyright 2020-2023 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.provider.gcp.indexing.processing;
-
-import static org.mockito.Mockito.verify;
-
-import com.google.common.collect.ImmutableList;
-import java.util.List;
-import org.junit.Before;
-import org.junit.experimental.theories.DataPoints;
-import org.junit.experimental.theories.FromDataPoints;
-import org.junit.experimental.theories.Theories;
-import org.junit.experimental.theories.Theory;
-import org.junit.runner.RunWith;
-import org.mockito.Mockito;
-import org.opengroup.osdu.core.auth.TokenProvider;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmAckReplier;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmMessage;
-import org.opengroup.osdu.indexer.api.RecordIndexerApi;
-import org.opengroup.osdu.indexer.provider.gcp.indexing.scope.ThreadDpsHeaders;
-
-@RunWith(Theories.class)
-public class RecordsChangedMessageReceiverTest {
-
- protected ThreadDpsHeaders dpsHeaders = Mockito.mock(ThreadDpsHeaders.class);
-
- protected TokenProvider tokenProvider = Mockito.mock(TokenProvider.class);
-
- protected OqmAckReplier ackReplier = Mockito.mock(OqmAckReplier.class);
-
- private RecordIndexerApi recordIndexerApi = Mockito.mock(RecordIndexerApi.class);
-
- private RecordsChangedMessageReceiver receiver;
-
- @Before
- public void setUp() {
- receiver = new RecordsChangedMessageReceiver(dpsHeaders, tokenProvider, recordIndexerApi);
- }
-
- @DataPoints("VALID_EVENTS")
- public static List<String> validEvents() {
- return ImmutableList.of(
- "/test-events/storage-index-event.json"
- );
- }
-
- @Theory
- public void shouldReceiveValidEvent(
- @FromDataPoints("VALID_EVENTS") String fileName) throws Exception {
- OqmMessage oqmMessage = ReadFromFileUtil.readEventFromFile(fileName);
- receiver.receiveMessage(oqmMessage, ackReplier);
- verify(ackReplier).ack();
- }
-}
diff --git a/provider/indexer-gc/src/test/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/ReprocessorMessageReceiverTest.java b/provider/indexer-gc/src/test/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/ReprocessorMessageReceiverTest.java
deleted file mode 100644
index ebddfb3c23686379ca347e2699a7c00063df2d09..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/test/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/ReprocessorMessageReceiverTest.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Copyright 2020-2023 Google LLC
- * Copyright 2020-2023 EPAM Systems, Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.opengroup.osdu.indexer.provider.gcp.indexing.processing;
-
-import static org.mockito.Mockito.verify;
-
-import com.google.common.collect.ImmutableList;
-import java.util.List;
-import org.junit.Before;
-import org.junit.experimental.theories.DataPoints;
-import org.junit.experimental.theories.FromDataPoints;
-import org.junit.experimental.theories.Theories;
-import org.junit.experimental.theories.Theory;
-import org.junit.runner.RunWith;
-import org.mockito.Mockito;
-import org.opengroup.osdu.core.auth.TokenProvider;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmAckReplier;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmMessage;
-import org.opengroup.osdu.indexer.api.ReindexApi;
-import org.opengroup.osdu.indexer.provider.gcp.indexing.scope.ThreadDpsHeaders;
-
-@RunWith(Theories.class)
-public class ReprocessorMessageReceiverTest {
-
- protected ThreadDpsHeaders dpsHeaders = Mockito.mock(ThreadDpsHeaders.class);
-
- protected TokenProvider tokenProvider = Mockito.mock(TokenProvider.class);
-
- protected OqmAckReplier ackReplier = Mockito.mock(OqmAckReplier.class);
-
- private ReindexApi reindexApi = Mockito.mock(ReindexApi.class);
-
- private ReprocessorMessageReceiver receiver;
-
- @Before
- public void setUp() {
- receiver = new ReprocessorMessageReceiver(dpsHeaders, tokenProvider, reindexApi);
- }
-
- @DataPoints("VALID_EVENTS")
- public static List<String> validEvents() {
- return ImmutableList.of(
- "/test-events/reindex-event.json"
- );
- }
-
- @Theory
- public void shouldReceiveValidEvent(
- @FromDataPoints("VALID_EVENTS") String fileName) throws Exception {
- OqmMessage oqmMessage = ReadFromFileUtil.readEventFromFile(fileName);
- receiver.receiveMessage(oqmMessage, ackReplier);
- verify(ackReplier).ack();
- }
-}
diff --git a/provider/indexer-gc/src/test/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceTest.java b/provider/indexer-gc/src/test/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceTest.java
deleted file mode 100644
index ffdfde118a63472e22e142523e0715dc1d6aa506..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/test/java/org/opengroup/osdu/indexer/service/ElasticSettingServiceTest.java
+++ /dev/null
@@ -1,114 +0,0 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.service;
-
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.opengroup.osdu.core.common.model.search.ClusterSettings;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
-import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.provider.interfaces.IElasticCredentialsCache;
-import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository;
-import org.opengroup.osdu.core.common.multitenancy.ITenantInfoService;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.springframework.test.context.junit4.SpringRunner;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.mockito.Mockito.when;
-
-@RunWith(SpringRunner.class)
-public class ElasticSettingServiceTest {
-
- @Mock
- private ITenantInfoService tenantInfoService;
- @Mock
- private IElasticRepository elasticRepository;
- @Mock
- private IElasticCredentialsCache elasticCredentialCache;
- @Mock
- private IndexerConfigurationProperties configurationProperties;
- @Mock
- private TenantInfo tenantInfo;
- @InjectMocks
- private ElasticSettingServiceImpl sut;
- @Mock
- private ClusterSettings clusterSettings;
- @Mock
- private DpsHeaders headersInfo;
-
- @Mock
- private JaxRsDpsLog log;
-
-
- public String GAE_SERVICE = "indexer";
-
- private final String host = "db5c51c1.us-central1.gcp.cloud.es.io";
- private final int port = 9243;
- private final String credentials = "name:password";
-
- String cacheKey = "";
-
-
- @Before
- public void setup() {
- when(tenantInfo.getName()).thenReturn("tenant1");
- when(this.headersInfo.getPartitionId()).thenReturn("tenant1");
- when(this.tenantInfoService.getTenantInfo()).thenReturn(tenantInfo);
- when(configurationProperties.getGaeService()).thenReturn("indexer");
- clusterSettings = ClusterSettings.builder().host(host).port(port).userNameAndPassword(credentials).build();
- cacheKey = String.format("%s-%s", GAE_SERVICE, tenantInfo.getName());
- }
-
- @Test
- public void should_getValid_clusterSettings_fromCache() {
-
- when(this.elasticCredentialCache.get(cacheKey)).thenReturn(clusterSettings);
-
- ClusterSettings response = this.sut.getElasticClusterInformation();
- assertNotNull(response);
- assertEquals(response.getHost(), host);
- assertEquals(response.getPort(), port);
- assertEquals(response.getUserNameAndPassword(), credentials);
- }
-
- @Test
- public void should_getValid_clusterSettings_fromCosmosDB() {
-
- when(this.elasticCredentialCache.get(cacheKey)).thenReturn(clusterSettings);
-
- when(this.elasticRepository.getElasticClusterSettings(tenantInfo)).thenReturn(clusterSettings);
-
- ClusterSettings response = this.sut.getElasticClusterInformation();
- assertNotNull(response);
- assertEquals(response.getHost(), host);
- assertEquals(response.getPort(), port);
- assertEquals(response.getUserNameAndPassword(), credentials);
- }
-
- @Test(expected = AppException.class)
- public void should_throwAppException_when_tenantClusterInfo_not_found() throws AppException {
-
- when(this.elasticRepository.getElasticClusterSettings(tenantInfo)).thenReturn(null);
-
- this.sut.getElasticClusterInformation();
-
- }
-}
diff --git a/provider/indexer-gc/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java b/provider/indexer-gc/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java
deleted file mode 100644
index 34102903c7c7038631551b05cbd724e3fa991b8f..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/test/java/org/opengroup/osdu/indexer/service/ReindexServiceTest.java
+++ /dev/null
@@ -1,154 +0,0 @@
-// Copyright 2017-2019, Schlumberger
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.opengroup.osdu.indexer.service;
-
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.mockito.MockedStatic;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
-import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
-import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
-import org.mockito.junit.MockitoJUnitRunner;
-
-import java.util.*;
-
-import static org.junit.Assert.fail;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.mockStatic;
-import static org.mockito.Mockito.when;
-import static org.mockito.MockitoAnnotations.initMocks;
-
-@RunWith(MockitoJUnitRunner.class)
-public class ReindexServiceTest {
-
- private static MockedStatic<UUID> mockedUUIDs;
-
- private final String cursor = "100";
-
- private final String correlationId = UUID.randomUUID().toString();
-
- @Mock
- private IndexerConfigurationProperties configurationProperties;
- @Mock
- private StorageService storageService;
- @Mock
- private IRequestInfo requestInfo;
- @Mock
- private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
- @Mock
- private JaxRsDpsLog log;
- @InjectMocks
- private ReindexServiceImpl sut;
-
- private RecordReindexRequest recordReindexRequest;
- private RecordQueryResponse recordQueryResponse;
-
- private Map<String, String> httpHeaders;
-
- @Before
- public void setup() {
- initMocks(this);
-
- mockedUUIDs = mockStatic(UUID.class);
-
- recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor(cursor).build();
- recordQueryResponse = new RecordQueryResponse();
-
- httpHeaders = new HashMap<>();
- httpHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
- httpHeaders.put(DpsHeaders.CORRELATION_ID, correlationId);
- DpsHeaders standardHeaders = DpsHeaders.createFromMap(httpHeaders);
- when(requestInfo.getHeadersWithDwdAuthZ()).thenReturn(standardHeaders);
- }
-
- @After
- public void close() {
- mockedUUIDs.close();
- }
-
- @Test
- public void should_returnNull_givenNullResponseResult_reIndexRecordsTest() {
- try {
- recordQueryResponse.setResults(null);
- when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
-
- String response = sut.reindexKind(recordReindexRequest, false, false);
-
- Assert.assertNull(response);
- } catch (Exception e) {
- fail("Should not throw this exception" + e.getMessage());
- }
- }
-
- @Test
- public void should_returnNull_givenEmptyResponseResult_reIndexRecordsTest() {
- try {
- recordQueryResponse.setResults(new ArrayList<>());
- when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
-
- String response = sut.reindexKind(recordReindexRequest, false, false);
-
- Assert.assertNull(response);
- } catch (Exception e) {
- fail("Should not throw this exception" + e.getMessage());
- }
- }
- @Ignore
- @Test
- public void should_returnRecordQueryRequestPayload_givenValidResponseResult_reIndexRecordsTest() {
- try {
- recordQueryResponse.setCursor(cursor);
- List<String> results = new ArrayList<>();
- results.add("test1");
- recordQueryResponse.setResults(results);
-
- when(configurationProperties.getStorageRecordsBatchSize()).thenReturn(1);
-
- when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
-
- String taskQueuePayload = sut.reindexKind(recordReindexRequest, false, false);
-
- Assert.assertEquals("{\"kind\":\"tenant:test:test:1.0.0\",\"cursor\":\"100\"}", taskQueuePayload);
- } catch (Exception e) {
- fail("Should not throw exception" + e.getMessage());
- }
- }
-
- @Test
- public void should_returnRecordChangedMessage_givenValidResponseResult_reIndexRecordsTest() {
- try {
- List<String> results = new ArrayList<>();
- results.add("test1");
- recordQueryResponse.setResults(results);
- when(storageService.getRecordsByKind(any())).thenReturn(recordQueryResponse);
-
- String taskQueuePayload = sut.reindexKind(recordReindexRequest, false, false);
-
- Assert.assertEquals(String.format("{\"data\":\"[{\\\"id\\\":\\\"test1\\\",\\\"kind\\\":\\\"tenant:test:test:1.0.0\\\",\\\"op\\\":\\\"create\\\"}]\",\"attributes\":{\"correlation-id\":\"%s\"}}", correlationId), taskQueuePayload);
- } catch (Exception e) {
- fail("Should not throw exception" + e.getMessage());
- }
- }
-}
diff --git a/provider/indexer-gc/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/provider/indexer-gc/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker
deleted file mode 100644
index ca6ee9cea8ec189a088d50559325d4e84ff8ad09..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker
+++ /dev/null
@@ -1 +0,0 @@
-mock-maker-inline
\ No newline at end of file
diff --git a/provider/indexer-gc/src/test/resources/test-events/empty-attributes-event.json b/provider/indexer-gc/src/test/resources/test-events/empty-attributes-event.json
deleted file mode 100644
index dedbede7da290e9f474b6541c3047cc9ec83a28a..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/test/resources/test-events/empty-attributes-event.json
+++ /dev/null
@@ -1,5 +0,0 @@
-{
- "id": "6523155266469533",
- "data": "{\"url\":\"/api/indexer/v2/_dps/task-handlers/reindex-worker\",\"message\":\"{\\\"kind\\\":\\\"osdu:wks:reference-data--AgreementType:1.0.0\\\",\\\"cursor\\\":\\\"12345\\\\u003d\\\\u003d\\\"}\",\"initialDelayMillis\":30000}",
- "attributes": {}
-}
diff --git a/provider/indexer-gc/src/test/resources/test-events/empty-data-event.json b/provider/indexer-gc/src/test/resources/test-events/empty-data-event.json
deleted file mode 100644
index e81f8e0413a527bbc627f4cbe5fc851fcd5a6eb7..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/test/resources/test-events/empty-data-event.json
+++ /dev/null
@@ -1,11 +0,0 @@
-{
- "id": "6524321148950995",
- "data": "",
- "attributes": {
- "correlation-id": "fbe4a2b4-b3be-48f6-bc2a-b7dc068bce62",
- "account-id": "osdu",
- "data-partition-id": "osdu",
- "user": "gcp-integration-tester@nice-etching-277309.iam.gserviceaccount.com",
- "googclient_deliveryattempt": "1"
- }
-}
diff --git a/provider/indexer-gc/src/test/resources/test-events/indexer-reprocess-event.json b/provider/indexer-gc/src/test/resources/test-events/indexer-reprocess-event.json
deleted file mode 100644
index 3ec3b31acdc096613cd564aac0b2238d638bfc00..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/test/resources/test-events/indexer-reprocess-event.json
+++ /dev/null
@@ -1,11 +0,0 @@
-{
- "id": "6524321148950995",
- "data": "{\"url\":\"/api/indexer/v2/_dps/task-handlers/index-worker\",\"message\":\"{\\\"data\\\":\\\"[{\\\\\\\"id\\\\\\\":\\\\\\\"osdu:reference-data--AgreementType:AgencyTrust\\\\\\\",\\\\\\\"kind\\\\\\\":\\\\\\\"osdu:wks:reference-data--AgreementType:1.0.0\\\\\\\",\\\\\\\"op\\\\\\\":\\\\\\\"create\\\\\\\"},{\\\\\\\"id\\\\\\\":\\\\\\\"osdu:reference-data--AgreementType:AmendingWorkingInterestClarification\\\\\\\",\\\\\\\"kind\\\\\\\":\\\\\\\"osdu:wks:reference-data--AgreementType:1.0.0\\\\\\\",\\\\\\\"op\\\\\\\":\\\\\\\"create\\\\\\\"},{\\\\\\\"id\\\\\\\":\\\\\\\"osdu:reference-data--AgreementType:AssetSaleAndPurchase\\\\\\\",\\\\\\\"kind\\\\\\\":\\\\\\\"osdu:wks:reference-data--AgreementType:1.0.0\\\\\\\",\\\\\\\"op\\\\\\\":\\\\\\\"create\\\\\\\"},{\\\\\\\"id\\\\\\\":\\\\\\\"osdu:reference-data--AgreementType:AssetSalePurchase\\\\\\\",\\\\\\\"kind\\\\\\\":\\\\\\\"osdu:wks:reference-data--AgreementType:1.0.0\\\\\\\",\\\\\\\"op\\\\\\\":\\\\\\\"create\\\\\\\"},{\\\\\\\"id\\\\\\\":\\\\\\\"osdu:reference-data--AgreementType:AssignmentNovation\\\\\\\",\\\\\\\"kind\\\\\\\":\\\\\\\"osdu:wks:reference-data--AgreementType:1.0.0\\\\\\\",\\\\\\\"op\\\\\\\":\\\\\\\"create\\\\\\\"},{\\\\\\\"id\\\\\\\":\\\\\\\"osdu:reference-data--AgreementType:AssignmentNovationAmending\\\\\\\",\\\\\\\"kind\\\\\\\":\\\\\\\"osdu:wks:reference-data--AgreementType:1.0.0\\\\\\\",\\\\\\\"op\\\\\\\":\\\\\\\"create\\\\\\\"},{\\\\\\\"id\\\\\\\":\\\\\\\"osdu:reference-data--AgreementType:CommonStreamOperating\\\\\\\",\\\\\\\"kind\\\\\\\":\\\\\\\"osdu:wks:reference-data--AgreementType:1.0.0\\\\\\\",\\\\\\\"op\\\\\\\":\\\\\\\"create\\\\\\\"},{\\\\\\\"id\\\\\\\":\\\\\\\"osdu:reference-data--AgreementType:Concession\\\\\\\",\\\\\\\"kind\\\\\\\":\\\\\\\"osdu:wks:reference-data--AgreementType:1.0.0\\\\\\\",\\\\\\\"op\\\\\\\":\\\\\\\"create\\\\\\\"},{\\\\\\\"id\\\\\\\":\\\\\\\"osdu:reference-data--AgreementType:ConfidentialDisclosure\\\\\\\",\\\\\\\"kind\\\\\\\":\\\\\\\"osdu:wks:reference-data--AgreementType:1.0.0\\\\\\\",\\\\\\\"op\\\\\\\":\\\\\\\"create\\\\\\\"},{\\\\\\\"id\\\\\\\":\\\\\\\"osdu:reference-data--AgreementType:Confidentiality\\\\\\\",\\\\\\\"kind\\\\\\\":\\\\\\\"osdu:wks:reference-data--AgreementType:1.0.0\\\\\\\",\\\\\\\"op\\\\\\\":\\\\\\\"create\\\\\\\"},{\\\\\\\"id\\\\\\\":\\\\\\\"osdu:reference-data--AgreementType:ConstructionOwnershipOperating\\\\\\\",\\\\\\\"kind\\\\\\\":\\\\\\\"osdu:wks:reference-data--AgreementType:1.0.0\\\\\\\",\\\\\\\"op\\\\\\\":\\\\\\\"create\\\\\\\"},{\\\\\\\"id\\\\\\\":\\\\\\\"osdu:reference-data--AgreementType:ConsultantBusinessDevelopment\\\\\\\",\\\\\\\"kind\\\\\\\":\\\\\\\"osdu:wks:reference-data--AgreementType:1.0.0\\\\\\\",\\\\\\\"op\\\\\\\":\\\\\\\"create\\\\\\\"},{\\\\\\\"id\\\\\\\":\\\\\\\"osdu:reference-data--AgreementType:ContractWellOperating\\\\\\\",\\\\\\\"kind\\\\\\\":\\\\\\\"osdu:wks:reference-data--AgreementType:1.0.0\\\\\\\",\\\\\\\"op\\\\\\\":\\\\\\\"create\\\\\\\"},{\\\\\\\"id\\\\\\\":\\\\\\\"osdu:reference-data--AgreementType:CrossConveyedPooling\\\\\\\",\\\\\\\"kind\\\\\\\":\\\\\\\"osdu:wks:reference-data--AgreementType:1.0.0\\\\\\\",\\\\\\\"op\\\\\\\":\\\\\\\"create\\\\\\\"},{\\\\\\\"id\\\\\\\":\\\\\\\"osdu:reference-data--AgreementType:CrossconveyedPooling\\\\\\\",\\\\\\\"kind\\\\\\\":\\\\\\\"osdu:wks:reference-data--AgreementType:1.0.0\\\\\\\",\\\\\\\"op\\\\\\\":\\\\\\\"create\\\\\\\"},{\\\\\\\"id\\\\\\\":\\\\\\\"osdu:reference-data--AgreementType:DataExchange\\\\\\\",\\\\\\\"kind\\\\\\\":\\\\\\\"osdu:wks:reference-data--AgreementType:1.0.0\\\\\\\",\\\\\\\"op\\\\\\\":\\\\\\\"create\\\\\\\"},{\\\\\\\"id\\\\\\\":\\\\\\\"osdu:reference-data--AgreementType:DataLicence\\\\\\\",\\\\\\\"kind\\\\\\\":\\\\\\\"osdu:wks:reference-data--AgreementType:1.0.0\\\\\\\",\\\\\\\"op\\\\\\\":\\\\\\\"create\\\\\\\"},{\\\\\\\"id\\\\\\\":\\\\\\\"osdu:reference-data--AgreementType:DataPurchase\\\\\\\",\\\\\\\"kind\\\\\\\":\\\\\\\"osdu:wks:reference-data--AgreementType:1.0.0\\\\\\\",\\\\\\\"op\\\\\\\":\\\\\\\"create\\\\\\\"},{\\\\\\\"id\\\\\\\":\\\\\\\"osdu:reference-data--AgreementType:DataRoom\\\\\\\",\\\\\\\"kind\\\\\\\":\\\\\\\"osdu:wks:reference-data--AgreementType:1.0.0\\\\\\\",\\\\\\\"op\\\\\\\":\\\\\\\"create\\\\\\\"},{\\\\\\\"id\\\\\\\":\\\\\\\"osdu:reference-data--AgreementType:DataSubscription\\\\\\\",\\\\\\\"kind\\\\\\\":\\\\\\\"osdu:wks:reference-data--AgreementType:1.0.0\\\\\\\",\\\\\\\"op\\\\\\\":\\\\\\\"create\\\\\\\"}]\\\",\\\"attributes\\\":{\\\"correlation-id\\\":\\\"fbe4a2b4-b3be-48f6-bc2a-b7dc068bce62\\\",\\\"data-partition-id\\\":\\\"osdu\\\"}}\",\"initialDelayMillis\":30000}",
- "attributes": {
- "correlation-id": "fbe4a2b4-b3be-48f6-bc2a-b7dc068bce62",
- "account-id": "osdu",
- "data-partition-id": "osdu",
- "user": "gcp-integration-tester@nice-etching-277309.iam.gserviceaccount.com",
- "googclient_deliveryattempt": "1"
- }
-}
diff --git a/provider/indexer-gc/src/test/resources/test-events/reindex-event.json b/provider/indexer-gc/src/test/resources/test-events/reindex-event.json
deleted file mode 100644
index 601e981be59935f99ad32a2bda99aab324fa5654..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/test/resources/test-events/reindex-event.json
+++ /dev/null
@@ -1,11 +0,0 @@
-{
- "id": "6523155266469533",
- "data": "{\"url\":\"/api/indexer/v2/_dps/task-handlers/reindex-worker\",\"message\":\"{\\\"kind\\\":\\\"osdu:wks:reference-data--AgreementType:1.0.0\\\",\\\"cursor\\\":\\\"12345\\\\u003d\\\\u003d\\\"}\",\"initialDelayMillis\":30000}",
- "attributes": {
- "data-partition-id": "osdu",
- "account-id": "osdu",
- "correlation-id": "46086b67-541a-449e-ae75-5e52659a775c",
- "user": "gcp-integration-tester@nice-etching-277309.iam.gserviceaccount.com",
- "googclient_deliveryattempt": "1"
- }
-}
diff --git a/provider/indexer-gc/src/test/resources/test-events/storage-index-event.json b/provider/indexer-gc/src/test/resources/test-events/storage-index-event.json
deleted file mode 100644
index c84a1e28fe3f5b0fba11c25b06e0e46ce1051baa..0000000000000000000000000000000000000000
--- a/provider/indexer-gc/src/test/resources/test-events/storage-index-event.json
+++ /dev/null
@@ -1,11 +0,0 @@
-{
- "id": "6405379623020546",
- "data": "[{\"id\":\"osdu:query:3b4ee153-48c4-4026-955a-59644ddaf3110\",\"kind\":\"osdu:ds:query:1.0.1670526387956\",\"op\":\"create\"}]",
- "attributes": {
- "account-id": "osdu",
- "data-partition-id": "osdu",
- "user": "gcp-integration-tester@nice-etching-277309.iam.gserviceaccount.com",
- "correlation-id": "c4e8b544-ad55-44e0-a6c8-90afb25aaad9",
- "googclient_deliveryattempt": "1"
- }
-}
diff --git a/provider/indexer-ibm/pom.xml b/provider/indexer-ibm/pom.xml
index 93f67b864acdc9d6584437ca24f5289cce5dce2d..675c38ee6c22ee4e5de950035619ce2698007db4 100644
--- a/provider/indexer-ibm/pom.xml
+++ b/provider/indexer-ibm/pom.xml
@@ -33,7 +33,7 @@
<properties>
<os-core-lib-ibm.version>0.26.0-rc8</os-core-lib-ibm.version>
<jackson.version>2.13.4</jackson.version>
- <os-core-common.version>0.27.0-rc3</os-core-common.version>
+ <os-core-common.version>3.0.0</os-core-common.version>
</properties>
<profiles>
@@ -81,7 +81,7 @@
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
- <artifactId>os-core-common-spring6</artifactId>
+ <artifactId>os-core-common</artifactId>
<version>${os-core-common.version}</version>
<exclusions>
<exclusion>
@@ -97,7 +97,7 @@
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec</artifactId>
- <version>4.1.86.Final</version>
+ <version>4.1.118.Final</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
@@ -105,9 +105,13 @@
<version>${os-core-lib-ibm.version}</version>
<exclusions>
<exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty-codec</artifactId>
+ <groupId>org.opengroup.osdu</groupId>
+ <artifactId>os-core-common-spring6</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -158,7 +162,18 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>net.minidev</groupId>
+ <artifactId>json-smart</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
+ <dependency>
+ <groupId>net.minidev</groupId>
+ <artifactId>json-smart</artifactId>
+ <version>2.5.2</version>
+ </dependency>
<dependency>
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-test</artifactId>
@@ -189,7 +204,19 @@
</dependency>
</dependencies>
-
+ <dependencyManagement>
+ <dependencies>
+ <!-- https://mvnrepository.com/artifact/io.netty/netty-bom -->
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-bom</artifactId>
+ <version>4.1.118.Final</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+
+ </dependencies>
+ </dependencyManagement>
<build>
<plugins>
diff --git a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/ReindexServiceTest.java b/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/ReindexServiceTest.java
deleted file mode 100644
index 0af01628e06425975aa4c0b35858ef3afcdecba5..0000000000000000000000000000000000000000
--- a/provider/indexer-ibm/src/test/java/org/opengroup/osdu/indexer/ibm/service/ReindexServiceTest.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/* Licensed Materials - Property of IBM */
-/* (c) Copyright IBM Corp. 2020. All Rights Reserved.*/
-package org.opengroup.osdu.indexer.ibm.service;
-
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.ArgumentMatchers;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.mockito.MockedStatic;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.indexer.RecordQueryResponse;
-import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
-import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
-import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.opengroup.osdu.indexer.service.ReindexServiceImpl;
-import org.opengroup.osdu.indexer.service.StorageService;
-import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
-import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
-import org.mockito.junit.MockitoJUnitRunner;
-
-import java.util.*;
-
-import static org.junit.Assert.fail;
-import static org.mockito.Mockito.mockStatic;
-import static org.mockito.Mockito.when;
-import static org.mockito.MockitoAnnotations.initMocks;
-
-@Ignore
-@RunWith(MockitoJUnitRunner.class)
-public class ReindexServiceTest {
-
- private static MockedStatic<UUID> mockedUUIDs;
-
- private final String cursor = "100";
-
- private final String correlationId = UUID.randomUUID().toString();
-
- @Mock
- private IndexerConfigurationProperties indexerConfigurationProperties;
-
- @Mock
- private StorageService storageService;
-
- @Mock
- private Map<String, String> httpHeaders;
- @Mock
- private IRequestInfo requestInfo;
- @Mock
- private IndexerQueueTaskBuilder indexerQueueTaskBuilder;
- @Mock
- private JaxRsDpsLog log;
- @InjectMocks
- private ReindexServiceImpl sut;
-
- private RecordReindexRequest recordReindexRequest;
- private RecordQueryResponse recordQueryResponse;
-
- @Before
- public void setup() {
- initMocks(this);
-
- mockedUUIDs = mockStatic(UUID.class);
-
- recordReindexRequest = RecordReindexRequest.builder().kind("tenant:test:test:1.0.0").cursor(cursor).build();
- recordQueryResponse = new RecordQueryResponse();
-
- httpHeaders = new HashMap<>();
- httpHeaders.put(DpsHeaders.AUTHORIZATION, "testAuth");
- httpHeaders.put(DpsHeaders.CORRELATION_ID, correlationId);
- DpsHeaders standardHeaders = DpsHeaders.createFromMap(httpHeaders);
- when(requestInfo.getHeaders()).thenReturn(standardHeaders);
- when(requestInfo.getHeadersMapWithDwdAuthZ()).thenReturn(httpHeaders);
- }
-
- @After
- public void close() {
- mockedUUIDs.close();
- }
-
- @Test
- public void should_returnNull_givenNullResponseResult_reIndexRecordsTest() {
- try {
- recordQueryResponse.setResults(null);
- when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
-
- String response = sut.reindexKind(recordReindexRequest, false, false);
-
- Assert.assertNull(response);
- } catch (Exception e) {
- fail("Should not throw this exception" + e.getMessage());
- }
- }
-
- @Test
- public void should_returnNull_givenEmptyResponseResult_reIndexRecordsTest() {
- try {
- recordQueryResponse.setResults(new ArrayList<>());
- when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
-
- String response = sut.reindexKind(recordReindexRequest, false, false);
-
- Assert.assertNull(response);
- } catch (Exception e) {
- fail("Should not throw this exception" + e.getMessage());
- }
- }
-
- @Test
- public void should_returnRecordQueryRequestPayload_givenValidResponseResult_reIndexRecordsTest() {
- try {
- recordQueryResponse.setCursor(cursor);
- List<String> results = new ArrayList<>();
- results.add("test1");
- recordQueryResponse.setResults(results);
-
- when(indexerConfigurationProperties.getStorageRecordsBatchSize()).thenReturn(1);
-
- when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
-
- String taskQueuePayload = sut.reindexKind(recordReindexRequest, false, false);
-
- Assert.assertEquals("{\"kind\":\"tenant:test:test:1.0.0\",\"cursor\":\"100\"}", taskQueuePayload);
- } catch (Exception e) {
- fail("Should not throw exception" + e.getMessage());
- }
- }
-
- @Test
- public void should_returnRecordChangedMessage_givenValidResponseResult_reIndexRecordsTest() {
- try {
- List<String> results = new ArrayList<>();
- results.add("test1");
- recordQueryResponse.setResults(results);
- when(storageService.getRecordsByKind(ArgumentMatchers.any())).thenReturn(recordQueryResponse);
-
- String taskQueuePayload = sut.reindexKind(recordReindexRequest, false, false);
-
- Assert.assertEquals(String.format("{\"data\":\"[{\\\"id\\\":\\\"test1\\\",\\\"kind\\\":\\\"tenant:test:test:1.0.0\\\",\\\"op\\\":\\\"create\\\"}]\",\"attributes\":{\"slb-correlation-id\":\"%s\"}}", correlationId), taskQueuePayload);
- } catch (Exception e) {
- fail("Should not throw exception" + e.getMessage());
- }
- }
-}
diff --git a/testing/indexer-test-aws/pom.xml b/testing/indexer-test-aws/pom.xml
index 66abf93456b9bc726d78d2274a8876a95e974a40..dbe41f12179e2d6625409964ab04e43f760dcc6e 100644
--- a/testing/indexer-test-aws/pom.xml
+++ b/testing/indexer-test-aws/pom.xml
@@ -58,7 +58,7 @@
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
<artifactId>os-core-lib-aws</artifactId>
- <version>0.26.0-rc2</version>
+ <version>0.29.0</version>
</dependency>
<!-- Testing -->
@@ -86,7 +86,7 @@
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
- <version>7.17.22</version>
+ <version>8.15.2</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
diff --git a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java
index 195ef84aef9f0f99fb06625765292d9b5f4ac0c7..946e3aaf1d6481430271dab993560baca1608433 100644
--- a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java
+++ b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java
@@ -38,11 +38,16 @@ public class Steps extends InfoSteps {
/******************Inputs being set**************/
- @When("^I send get request to version info endpoint")
+ @When("^I send get request to version info endpoint$")
public void i_send_get_request_to_version_info_endpoint() {
super.i_send_get_request_to_version_info_endpoint();
}
+ @When("^I send get request to version info endpoint with trailing slash$")
+ public void i_send_get_request_to_version_info_endpoint_with_trailing_slash() {
+ super.i_send_get_request_to_version_info_endpoint_with_trailing_slash();
+ }
+
/******************Assert final response**************/
@Then("^I should get version info in response$")
@@ -59,4 +64,4 @@ public class Steps extends InfoSteps {
protected String getApi() {
return Config.getIndexerBaseURL() + "info";
}
-}
\ No newline at end of file
+}
diff --git a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/ElasticUtilsAws.java b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/ElasticUtilsAws.java
index a3a7378d2477666cc4d2ae6760fb921e3e956c33..5f3a94f4abd532cf9baa1bcfc558cc93a76286d0 100644
--- a/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/ElasticUtilsAws.java
+++ b/testing/indexer-test-aws/src/test/java/org/opengroup/osdu/util/ElasticUtilsAws.java
@@ -70,8 +70,8 @@ public class ElasticUtilsAws extends ElasticUtils {
new BasicHeader("client.transport.nodes_sampler_interval", "30s"),
new BasicHeader("client.transport.ping_timeout", "30s"),
new BasicHeader("client.transport.sniff", "false"),
- new BasicHeader("request.headers.X-Found-Cluster", Config.getElasticHost()),
- new BasicHeader("cluster.name", Config.getElasticHost()),
+ new BasicHeader("request.headers.X-Found-Cluster", Config.getElastic8Host()),
+ new BasicHeader("cluster.name", Config.getElastic8Host()),
new BasicHeader("xpack.security.transport.ssl.enabled", Boolean.toString(true)),
new BasicHeader("Authorization", basicAuthenticationHeaderVal),
};
diff --git a/testing/indexer-test-aws/src/test/resources/cucumber.properties b/testing/indexer-test-aws/src/test/resources/cucumber.properties
index e30b479bad02b26a5117fc6d112f84e21420058b..c575929b88d37c9216d44d5b9f28f1c939ffd075 100644
--- a/testing/indexer-test-aws/src/test/resources/cucumber.properties
+++ b/testing/indexer-test-aws/src/test/resources/cucumber.properties
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# tag indexer-extended disabled by default
+# tag indexer-extended enabled by default
# tag as-ingested-coordinates disabled by default
# tag keyword-lower enabled by default
-cucumber.options=--tags '@default,@as-ingested-coordinates,@keyword-lower,@bag-of-words'
+cucumber.options=--tags '@default,@as-ingested-coordinates,@keyword-lower,@bag-of-words,@indexer-extended'
diff --git a/testing/indexer-test-azure/pom.xml b/testing/indexer-test-azure/pom.xml
index 09caabd5290845541239317fe5b23a3559742a39..780709b59803a923ec83699939a2b87b88c7ae7f 100644
--- a/testing/indexer-test-azure/pom.xml
+++ b/testing/indexer-test-azure/pom.xml
@@ -35,7 +35,7 @@
<maven.compiler.source>17</maven.compiler.source>
<cucumber.version>1.2.5</cucumber.version>
<junit.jupiter.version>5.6.0</junit.jupiter.version>
- <elasticsearch.version>7.17.22</elasticsearch.version>
+ <elasticsearch.version>8.15.2</elasticsearch.version>
<jackson.version>2.13.2</jackson.version>
<argLine>
--add-opens java.base/java.lang=ALL-UNNAMED
diff --git a/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java
index a7871ae9130150ddf1c07688489136ea58667cbb..bfacb48850262b6242e74d7e0783a65626cfa730 100644
--- a/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java
+++ b/testing/indexer-test-azure/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java
@@ -22,11 +22,16 @@ public class Steps extends InfoSteps {
/******************Inputs being set**************/
- @When("^I send get request to version info endpoint")
+ @When("^I send get request to version info endpoint$")
public void i_send_get_request_to_version_info_endpoint() {
super.i_send_get_request_to_version_info_endpoint();
}
+ @When("^I send get request to version info endpoint with trailing slash$")
+ public void i_send_get_request_to_version_info_endpoint_with_trailing_slash() {
+ super.i_send_get_request_to_version_info_endpoint_with_trailing_slash();
+ }
+
/******************Assert final response**************/
@Then("^I should get version info in response$")
@@ -43,4 +48,4 @@ public class Steps extends InfoSteps {
protected String getApi() {
return Config.getIndexerBaseURL() + "info";
}
-}
\ No newline at end of file
+}
diff --git a/testing/indexer-test-baremetal/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java b/testing/indexer-test-baremetal/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java
index 67ae1d3dbf5b81086ecd2c489f815d8bfcd6e424..c1d5ef080e47e293ffa004c02b659e484cbf0656 100644
--- a/testing/indexer-test-baremetal/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java
+++ b/testing/indexer-test-baremetal/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java
@@ -39,11 +39,16 @@ public class Steps extends InfoSteps {
/******************Inputs being set**************/
- @When("^I send get request to version info endpoint")
+ @When("^I send get request to version info endpoint$")
public void i_send_get_request_to_version_info_endpoint() {
super.i_send_get_request_to_version_info_endpoint();
}
+ @When("^I send get request to version info endpoint with trailing slash$")
+ public void i_send_get_request_to_version_info_endpoint_with_trailing_slash() {
+ super.i_send_get_request_to_version_info_endpoint_with_trailing_slash();
+ }
+
/******************Assert final response**************/
@Then("^I should get version info in response$")
@@ -60,4 +65,4 @@ public class Steps extends InfoSteps {
protected String getApi() {
return Config.getIndexerBaseURL() + "info";
}
-}
\ No newline at end of file
+}
diff --git a/testing/indexer-test-baremetal/src/test/java/org/opengroup/osdu/util/AnthosHTTPClient.java b/testing/indexer-test-baremetal/src/test/java/org/opengroup/osdu/util/AnthosHTTPClient.java
index 7bf1aeab02562e475333ebcb2623fbb589f70adf..90b4c606084c5ce5e13800c796c969f77af1179f 100644
--- a/testing/indexer-test-baremetal/src/test/java/org/opengroup/osdu/util/AnthosHTTPClient.java
+++ b/testing/indexer-test-baremetal/src/test/java/org/opengroup/osdu/util/AnthosHTTPClient.java
@@ -34,4 +34,4 @@ public class AnthosHTTPClient extends HTTPClient {
}
return token;
}
-}
\ No newline at end of file
+}
diff --git a/testing/indexer-test-core/pom.xml b/testing/indexer-test-core/pom.xml
index 92d2465710b0fe5c7bc6445a78c515a40a036aaf..4bda67d5c47ba8d92fcaac33b5c1d23250e6910f 100644
--- a/testing/indexer-test-core/pom.xml
+++ b/testing/indexer-test-core/pom.xml
@@ -108,12 +108,12 @@
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
- <version>7.17.22</version>
+ <version>8.15.2</version>
</dependency>
<dependency>
<groupId>co.elastic.clients</groupId>
<artifactId>elasticsearch-java</artifactId>
- <version>7.17.22</version>
+ <version>8.15.2</version>
</dependency>
<dependency>
<groupId>org.locationtech.jts.io</groupId>
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/InfoBase.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/InfoBase.java
index abcc33b527cb4625d0af7005865ae14641fc5d6a..9e83ec1c30e4bb6ea9c8d18905ab94b6aa1162d3 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/InfoBase.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/InfoBase.java
@@ -43,12 +43,17 @@ public class InfoBase extends TestsBase {
}
response =
- executeQuery(
- this.getApi(),
- Strings.EMPTY,
- headers,
- httpClient.getAccessToken(),
- InfoResponseMock.class);
+ executeQuery(
+ this.getApi(),
+ Strings.EMPTY,
+ headers,
+ httpClient.getAccessToken(),
+ InfoResponseMock.class);
+ }
+
+ public void i_send_get_request_to_version_info_endpoint_with_trailing_slash() {
+ response =
+ executeQuery(getApi()+"/", Strings.EMPTY, headers, httpClient.getAccessToken(), InfoResponseMock.class);
}
public void i_should_get_version_info_in_response() {
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/info/InfoSteps.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/info/InfoSteps.java
index 02924b4d055aa4a444ab79174ec2c0f978ae3818..2deee856b92d689b27c895bdc7f74feebef35493 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/info/InfoSteps.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/common/info/InfoSteps.java
@@ -13,4 +13,4 @@ public class InfoSteps extends InfoBase {
public InfoSteps(HTTPClient httpClient, ElasticUtils elasticUtils) {
super(httpClient, elasticUtils);
}
-}
\ No newline at end of file
+}
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/Config.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/Config.java
index 247213e690389136eab11202e3fd38cfb6144bbc..fc9e92bd5af81b2d62e996858797cae93cd4c8b2 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/Config.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/Config.java
@@ -29,6 +29,42 @@ public class Config {
return Integer.parseInt(getEnvironmentVariableOrDefaultValue("ELASTIC_PORT", String.valueOf(PORT)));
}
+ public static String getUserName() {
+ return getEnvironmentVariableOrDefaultValue("ELASTIC_USER_NAME", DEFAULT_ELASTIC_USER_NAME);
+ }
+
+ public static String getPassword() {
+ return getEnvironmentVariableOrDefaultValue("ELASTIC_PASSWORD", DEFAULT_ELASTIC_PASSWORD);
+ }
+
+ public static String getElasticHost() {
+ return getEnvironmentVariableOrDefaultValue("ELASTIC_HOST", DEFAULT_ELASTIC_HOST);
+ }
+
+ public static boolean isElasticSslEnabled() {
+ return Boolean.parseBoolean(getEnvironmentVariableOrDefaultValue("ELASTIC_SSL_ENABLED", DEFAULT_ELASTIC_SSL_ENABLED));
+ }
+
+ public static int getElastic8Port() {
+ return Integer.parseInt(getEnvironmentVariableOrDefaultValue("ELASTIC_8_PORT", String.valueOf(getPort())));
+ }
+
+ public static String getElastic8UserName() {
+ return getEnvironmentVariableOrDefaultValue("ELASTIC_8_USER_NAME", getUserName());
+ }
+
+ public static String getElastic8Password() {
+ return getEnvironmentVariableOrDefaultValue("ELASTIC_8_PASSWORD", getPassword());
+ }
+
+ public static String getElastic8Host() {
+ return getEnvironmentVariableOrDefaultValue("ELASTIC_8_HOST", getElasticHost());
+ }
+
+ public static boolean isElastic8SslEnabled() {
+ return Boolean.parseBoolean(getEnvironmentVariableOrDefaultValue("ELASTIC_8_SSL_ENABLED", String.valueOf(isElasticSslEnabled())));
+ }
+
public static String getOtherRelevantDataCountries() {
return getEnvironmentVariableOrDefaultValue("OTHER_RELEVANT_DATA_COUNTRIES", DEFAULT_OTHER_RELEVANT_DATA_COUNTRIES);
}
@@ -49,22 +85,6 @@ public class Config {
return getEnvironmentVariableOrDefaultValue("DEFAULT_DATA_PARTITION_ID_TENANT2", DEFAULT_DATA_PARTITION_ID_TENANT2);
}
- public static String getUserName() {
- return getEnvironmentVariableOrDefaultValue("ELASTIC_USER_NAME", DEFAULT_ELASTIC_USER_NAME);
- }
-
- public static String getPassword() {
- return getEnvironmentVariableOrDefaultValue("ELASTIC_PASSWORD", DEFAULT_ELASTIC_PASSWORD);
- }
-
- public static String getElasticHost() {
- return getEnvironmentVariableOrDefaultValue("ELASTIC_HOST", DEFAULT_ELASTIC_HOST);
- }
-
- public static boolean isElasticSslEnabled() {
- return Boolean.parseBoolean(getEnvironmentVariableOrDefaultValue("ELASTIC_SSL_ENABLED", DEFAULT_ELASTIC_SSL_ENABLED));
- }
-
public static String getIndexerBaseURL() {
return getEnvironmentVariableOrDefaultValue("INDEXER_HOST", DEFAULT_INDEXER_HOST);
}
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
index 49e2e2d5f869d2f3cf60e7eb1df243eb95a46e15..eeae9569c12bbdbe607350d0b03d67bf71b173e7 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/ElasticUtils.java
@@ -114,10 +114,10 @@ public class ElasticUtils {
private ElasticsearchClient elasticsearchClient;
public ElasticUtils() {
- this.username = Config.getUserName();
- this.password = Config.getPassword();
- this.host = Config.getElasticHost();
- this.sslEnabled = Config.isElasticSslEnabled();
+ this.username = Config.getElastic8UserName();
+ this.password = Config.getElastic8Password();
+ this.host = Config.getElastic8Host();
+ this.sslEnabled = Config.isElastic8SslEnabled();
}
public void createIndex(String index, String mapping) {
@@ -438,11 +438,13 @@ public class ElasticUtils {
String pointY, Double topPointY, Double bottomPointY) throws Exception {
ElasticsearchClient client = this.getOrCreateClient(username, password, host);
try {
- RangeQuery xRangeQuery = RangeQuery.of(
- builder -> builder.field(pointX).lte(JsonData.of(topPointX)).gte(JsonData.of(bottomPointX)));
+ RangeQuery xRangeQuery = RangeQuery.of(builder -> builder.untyped(
+ untypedBuilder -> untypedBuilder.field(pointX).lte(JsonData.of(topPointX))
+ .gte(JsonData.of(bottomPointX))));
- RangeQuery yRangeQuery = RangeQuery.of(
- builder -> builder.field(pointY).lte(JsonData.of(topPointY)).gte(JsonData.of(bottomPointY)));
+ RangeQuery yRangeQuery = RangeQuery.of(builder -> builder.untyped(
+ untypedBuilder -> untypedBuilder.field(pointY).lte(JsonData.of(topPointY))
+ .gte(JsonData.of(bottomPointY))));
BoolQuery boolQuery = BoolQuery.of(
builder -> builder.must(xRangeQuery._toQuery()).must(yRangeQuery._toQuery()));
@@ -611,7 +613,7 @@ public class ElasticUtils {
private ElasticsearchClient getOrCreateClient(String username, String password, String host) {
if (this.elasticsearchClient == null) {
ElasticsearchClient restHighLevelClient;
- int port = Config.getPort();
+ int port = Config.getElastic8Port();
try {
String rawString = String.format("%s:%s", username, password);
@@ -645,8 +647,8 @@ public class ElasticUtils {
new BasicHeader("client.transport.nodes_sampler_interval", "30s"),
new BasicHeader("client.transport.ping_timeout", "30s"),
new BasicHeader("client.transport.sniff", "false"),
- new BasicHeader("request.headers.X-Found-Cluster", Config.getElasticHost()),
- new BasicHeader("cluster.name", Config.getElasticHost()),
+ new BasicHeader("request.headers.X-Found-Cluster", Config.getElastic8Host()),
+ new BasicHeader("cluster.name", Config.getElastic8Host()),
new BasicHeader("xpack.security.transport.ssl.enabled", Boolean.toString(true)),
new BasicHeader("Authorization", String.format("Basic %s", Base64.getEncoder().encodeToString(usernameAndPassword.getBytes()))),
};
diff --git a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/SchemaServiceClient.java b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/SchemaServiceClient.java
index 5a1a90a0241a1b279c2b57ecfb5fe48c840a519e..04d74f8b7eb0ab84182d61b342e802e343114e86 100644
--- a/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/SchemaServiceClient.java
+++ b/testing/indexer-test-core/src/main/java/org/opengroup/osdu/util/SchemaServiceClient.java
@@ -65,7 +65,7 @@ public class SchemaServiceClient {
private String buildSchemaUri() {
return UriComponentsBuilder.fromHttpUrl(schemaBaseUrl)
- .path("/schema/")
+ .path("/schema")
.build().toUriString();
}
diff --git a/testing/indexer-test-core/src/main/resources/features/info/Info.feature b/testing/indexer-test-core/src/main/resources/features/info/Info.feature
index c4b99ba1150165f44d5bd5cdfb9ea020924cc385..92a5e03726dd2f968d6445b5bd7e10fa183b4c40 100644
--- a/testing/indexer-test-core/src/main/resources/features/info/Info.feature
+++ b/testing/indexer-test-core/src/main/resources/features/info/Info.feature
@@ -4,3 +4,8 @@ Feature: Fetch info about maven build and git repository.
Scenario: Verify version info endpoint content
When I send get request to version info endpoint
Then I should get version info in response
+
+ @default
+ Scenario: Verify version info endpoint content for request with trailing slash
+ When I send get request to version info endpoint with trailing slash
+ Then I should get version info in response
diff --git a/testing/indexer-test-core/src/main/resources/testData/osdu_wks_IndexPropertyPathConfiguration_v1.schema.json b/testing/indexer-test-core/src/main/resources/testData/osdu_wks_IndexPropertyPathConfiguration_v1.schema.json
index 7cf15888121f227c11303d934c9af6340f384798..1f1b3249c7d803c6d41b92bd95aacbe67fc17461 100644
--- a/testing/indexer-test-core/src/main/resources/testData/osdu_wks_IndexPropertyPathConfiguration_v1.schema.json
+++ b/testing/indexer-test-core/src/main/resources/testData/osdu_wks_IndexPropertyPathConfiguration_v1.schema.json
@@ -1,284 +1,780 @@
{
- "schemaInfo": {
- "schemaIdentity": {
- "authority": "osdu",
- "source": "wks",
- "entityType": "reference-data--IndexPropertyPathConfiguration",
- "schemaVersionMajor": 1,
- "schemaVersionMinor": 0,
- "schemaVersionPatch": 0,
- "id": "osdu:wks:reference-data--IndexPropertyPathConfiguration:1.0.0"
+ "schemaInfo": {
+ "schemaIdentity": {
+ "authority": "osdu",
+ "source": "wks",
+ "entityType": "reference-data--IndexPropertyPathConfiguration",
+ "schemaVersionMajor": 1,
+ "schemaVersionMinor": 0,
+ "schemaVersionPatch": 0,
+ "id": "osdu:wks:reference-data--IndexPropertyPathConfiguration:1.0.0"
+ },
+ "status": "PUBLISHED"
},
- "status": "DEVELOPMENT"
- },
- "schema": {
- "x-osdu-license": "Copyright 2023, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
- "$id": "https://schema.osdu.opengroup.org/json/reference-data/IndexPropertyPathConfiguration.1.0.0.json",
- "$schema": "http://json-schema.org/draft-07/schema#",
- "x-osdu-schema-source": "osdu:wks:reference-data--IndexPropertyPathConfiguration:1.0.0",
- "title": "IndexPropertyPathConfiguration",
- "description": "IndexPropertyPathConfiguration contains the de-normalization configuration settings for the Search index. The record id contains the kind so that there is a unique relationship between the kind and the kind's index extensions.",
- "type": "object",
- "properties": {
- "id": {
- "description": "Previously called ResourceID or SRN which identifies this OSDU resource object without version.",
- "title": "Entity ID",
- "type": "string",
- "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-IndexPropertyPathConfiguration:[\\w\\-\\.\\:\\%]+$",
- "example": "namespace:reference-data--IndexPropertyPathConfiguration:c9d84708-2b1b-5e0b-954e-9621132f7154"
- },
- "kind": {
- "description": "The schema identification for the OSDU resource object following the pattern {Namespace}:{Source}:{Type}:{VersionMajor}.{VersionMinor}.{VersionPatch}. The versioning scheme follows the semantic versioning, https://semver.org/.",
- "title": "Entity Kind",
- "type": "string",
- "pattern": "^[\\w\\-\\.]+:[\\w\\-\\.]+:[\\w\\-\\.]+:[0-9]+.[0-9]+.[0-9]+$",
- "example": "osdu:wks:reference-data--IndexPropertyPathConfiguration:1.0.0"
- },
- "version": {
- "description": "The version number of this OSDU resource; set by the framework.",
- "title": "Version Number",
- "type": "integer",
- "format": "int64",
- "example": 1562066009929332
- },
- "acl": {
- "description": "The access control tags associated with this entity.",
- "title": "Access Control List",
- "$ref": "osdu:wks:AbstractAccessControlList:1.0.0"
- },
- "legal": {
- "description": "The entity's legal tags and compliance status. The actual contents associated with the legal tags is managed by the Compliance Service.",
- "title": "Legal Tags",
- "$ref": "osdu:wks:AbstractLegalTags:1.0.0"
- },
- "tags": {
- "title": "Tag Dictionary",
- "description": "A generic dictionary of string keys mapping to string value. Only strings are permitted as keys and values.",
+ "schema": {
+ "x-osdu-governance-model": "OPEN",
+ "x-osdu-inheriting-from-kind": [{
+ "kind": "osdu:wks:AbstractReferenceType:1.0.0",
+ "name": "ReferenceType"
+ }
+ ],
+ "x-osdu-license": "Copyright 2023, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:reference-data--IndexPropertyPathConfiguration:1.0.0",
+ "description": "IndexPropertyPathConfiguration contains the de-normalization configuration settings for the Search index. The record id contains the kind so that there is a unique relationship between the kind and the kind's index extensions.",
+ "title": "IndexPropertyPathConfiguration",
"type": "object",
- "additionalProperties": {
- "type": "string"
+ "x-osdu-review-status": "Accepted",
+ "required": [
+ "kind",
+ "acl",
+ "legal"
+ ],
+ "x-osdu-governance-authorities": [
+ "OSDU"
+ ],
+ "x-osdu-virtual-properties": {
+ "data.VirtualProperties.DefaultName": {
+ "type": "string",
+ "priority": [{
+ "path": "data.Name"
+ }
+ ]
+ }
},
- "example": {
- "NameOfKey": "String value"
- }
- },
- "createTime": {
- "description": "Timestamp of the time at which initial version of this OSDU resource object was created. Set by the System. The value is a combined date-time string in ISO-8601 given in UTC.",
- "title": "Resource Object Creation DateTime",
- "type": "string",
- "format": "date-time",
- "example": "2020-12-16T11:46:20.163Z"
- },
- "createUser": {
- "title": "Resource Object Creation User Reference",
- "description": "The user reference, which created the first version of this resource object. Set by the System.",
- "type": "string",
- "example": "some-user@some-company-cloud.com"
- },
- "modifyTime": {
- "description": "Timestamp of the time at which this version of the OSDU resource object was created. Set by the System. The value is a combined date-time string in ISO-8601 given in UTC.",
- "title": "Resource Object Version Creation DateTime",
- "type": "string",
- "format": "date-time",
- "example": "2020-12-16T11:52:24.477Z"
- },
- "modifyUser": {
- "title": "Resource Object Version Creation User Reference",
- "description": "The user reference, which created this version of this resource object. Set by the System.",
- "type": "string",
- "example": "some-user@some-company-cloud.com"
- },
- "ancestry": {
- "description": "The links to data, which constitute the inputs, from which this record instance is derived.",
- "title": "Ancestry",
- "$ref": "osdu:wks:AbstractLegalParentList:1.0.0"
- },
- "meta": {
- "description": "The Frame of Reference meta data section linking the named properties to self-contained definitions.",
- "title": "Frame of Reference Meta Data",
- "type": "array",
- "items": {
- "$ref": "osdu:wks:AbstractMetaItem:1.0.0"
- }
- },
- "data": {
- "allOf": [
- {
- "$ref": "osdu:wks:AbstractCommonResources:1.0.0"
- },
- {
- "$ref": "osdu:wks:AbstractReferenceType:1.0.0"
- },
- {
- "type": "object",
- "properties": {
- "Configurations": {
- "type": "array",
- "title": "Configurations",
- "description": "The list of index property configurations for the specific kind.",
- "x-osdu-indexing": {
- "type": "nested"
- },
- "items": {
- "type": "object",
- "title": "Configuration",
- "description": "One single configuration to derive an Search index property value and assign it to the index 'column' with Name.",
- "properties": {
- "Name": {
- "type": "string",
- "title": "Name",
- "description": "The name of the indexed property, i.e., this is the property name used in Search.",
- "example": "CountryNames"
+ "additionalProperties": false,
+ "definitions": {
+ "osdu:wks:AbstractCommonResources:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractCommonResources:1.0.0",
+ "description": "Common resources to be injected at root 'data' level for every entity, which is persistable in Storage. The insertion is performed by the OsduSchemaComposer script.",
+ "x-osdu-review-status": "Accepted",
+ "title": "OSDU Common Resources",
+ "type": "object",
+ "properties": {
+ "ResourceHomeRegionID": {
+ "x-osdu-relationship": [{
+ "EntityType": "OSDURegion",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-OSDURegion:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The name of the home [cloud environment] region for this OSDU resource object.",
+ "type": "string",
+ "title": "Resource Home Region ID"
},
- "UseCase": {
- "type": "string",
- "title": "Use Case",
- "description": "The use case description this configuration satisfies.",
- "example": "As a user I want to find objects by a country name, with the understanding that an object may extend over country boundaries."
+ "ResourceHostRegionIDs": {
+ "description": "The name of the host [cloud environment] region(s) for this OSDU resource object.",
+ "type": "array",
+ "title": "Resource Host Region ID",
+ "items": {
+ "x-osdu-relationship": [{
+ "EntityType": "OSDURegion",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-OSDURegion:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "type": "string"
+ }
},
- "Policy": {
- "type": "string",
- "title": "Extraction Policy",
- "description": "Current supported policies are 'ExtractAllMatches' resulting in an array of values or 'ExtractFirstMatch' single value. The policy applies only to the Paths[].ValueExtraction.",
- "example": "ExtractAllMatches",
- "pattern": "^(ExtractFirstMatch|ExtractAllMatches)$"
+ "ResourceLifecycleStatus": {
+ "x-osdu-relationship": [{
+ "EntityType": "ResourceLifecycleStatus",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-ResourceLifecycleStatus:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Describes the current Resource Lifecycle status.",
+ "type": "string",
+ "title": "Resource Lifecycle Status"
+ },
+ "ResourceSecurityClassification": {
+ "x-osdu-relationship": [{
+ "EntityType": "ResourceSecurityClassification",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-ResourceSecurityClassification:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Classifies the security level of the resource.",
+ "type": "string",
+ "title": "Resource Security Classification"
+ },
+ "ResourceCurationStatus": {
+ "x-osdu-relationship": [{
+ "EntityType": "ResourceCurationStatus",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-ResourceCurationStatus:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Describes the current Curation status.",
+ "type": "string",
+ "title": "Resource Curation Status"
+ },
+ "ExistenceKind": {
+ "x-osdu-relationship": [{
+ "EntityType": "ExistenceKind",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-ExistenceKind:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "Where does this data resource sit in the cradle-to-grave span of its existence?",
+ "type": "string",
+ "title": "Existence Kind"
+ },
+ "TechnicalAssuranceID": {
+ "x-osdu-relationship": [{
+ "EntityType": "TechnicalAssuranceType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-TechnicalAssuranceType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "DEPRECATED: Describes a record's overall suitability for general business consumption based on data quality. Clarifications: Since Certified is the highest classification of suitable quality, any further change or versioning of a Certified record should be carefully considered and justified. If a Technical Assurance value is not populated then one can assume the data has not been evaluated or its quality is unknown (=Unevaluated). Technical Assurance values are not intended to be used for the identification of a single \"preferred\" or \"definitive\" record by comparison with other records.",
+ "type": "string",
+ "title": "Technical Assurance ID"
},
- "Paths": {
- "type": "array",
- "title": "Paths",
- "description": "The list of path definitions to derive the property value from.",
- "x-osdu-indexing": {
- "type": "nested"
- },
- "items": {
+ "Source": {
+ "description": "The entity that produced the record, or from which it is received; could be an organization, agency, system, internal team, or individual. For informational purposes only, the list of sources is not governed.",
+ "type": "string",
+ "title": "Data Source"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractCommonResources.1.0.0.json"
+ },
+ "osdu:wks:AbstractMetaItem:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "oneOf": [{
+ "title": "FrameOfReferenceUOM",
"type": "object",
- "title": "Path",
- "description": "A single path definition to derive a property value from.",
"properties": {
- "RelatedObjectsSpec": {
- "type": "object",
- "title": "Related Objects Specification",
- "description": "The specification to extract related objects, from which to derive the ValueExtraction. If this property is empty or absent, the ValueExtraction is done on the current object to be indexed.",
- "properties": {
- "RelatedObjectID": {
+ "persistableReference": {
+ "description": "The self-contained, persistable reference string uniquely identifying the Unit.",
+ "title": "UOM Persistable Reference",
"type": "string",
- "title": "Related Object ID",
- "description": "The path to the property containing the ID of the target record to chase. This property is only populated if the property is extracted from a related object, which must be chased. If the property is derived from 'within' the same record, which triggered the indexing, the RelatedObjectID is left absent.",
- "example": "GeoContexts[].GeoPoliticalEntityID"
- },
- "RelatedObjectKind": {
+ "example": "{\"abcd\":{\"a\":0.0,\"b\":1200.0,\"c\":3937.0,\"d\":0.0},\"symbol\":\"ft[US]\",\"baseMeasurement\":{\"ancestry\":\"L\",\"type\":\"UM\"},\"type\":\"UAD\"}"
+ },
+ "kind": {
+ "const": "Unit",
+ "description": "The kind of reference, 'Unit' for FrameOfReferenceUOM.",
+ "title": "UOM Reference Kind",
+ "type": "string"
+ },
+ "propertyNames": {
+ "description": "The list of property names, to which this meta data item provides Unit context to. A full path like \"StructureA.PropertyB\" is required to define a unique context; \"data\" is omitted since frame-of reference normalization only applies to the data block.",
+ "title": "UOM Property Names",
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "example": [
+ "HorizontalDeflection.EastWest",
+ "HorizontalDeflection.NorthSouth"
+ ]
+ },
+ "name": {
+ "description": "The unit symbol or name of the unit.",
+ "title": "UOM Unit Symbol",
"type": "string",
- "title": "Related Object Kind",
- "description": "The kind or schema id expected as the target object type. This property is only populated if the property is extracted from a related object, which must be chased. If the property is derived from 'within' the same record, which triggered the indexing, the RelatedObjectKind is left absent.",
- "example": "osdu:wks:master-data--GeoPoliticalEntity:1.",
- "pattern": "^[\\w\\-\\.]+:[\\w\\-\\.]+:[\\w\\-\\.]+:[0-9]+.$"
- },
- "RelationshipDirection": {
+ "example": "ft[US]"
+ },
+ "unitOfMeasureID": {
+ "x-osdu-relationship": [{
+ "EntityType": "UnitOfMeasure",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-UnitOfMeasure:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "SRN to unit of measure reference.",
"type": "string",
- "title": "Relationship Direction",
- "description": "The direction of the relationship definition seen from the object being indexed. 'ChildToParent' assumes an outgoing relationship with the target record defined in the object being indexed. 'ParentToChildren' assumes that the related objects have a relationship by RelatedObjectID to the id of the record being indexed.",
- "example": "ChildToParent",
- "pattern": "^(ChildToParent|ParentToChildren)$"
- },
- "RelatedConditionProperty": {
+ "example": "namespace:reference-data--UnitOfMeasure:ftUS:"
+ }
+ },
+ "required": [
+ "kind",
+ "persistableReference"
+ ]
+ }, {
+ "title": "FrameOfReferenceCRS",
+ "type": "object",
+ "properties": {
+ "coordinateReferenceSystemID": {
+ "x-osdu-relationship": [{
+ "EntityType": "CoordinateReferenceSystem",
+ "GroupType": "reference-data"
+ }
+ ],
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-CoordinateReferenceSystem:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "SRN to CRS reference.",
+ "type": "string",
+ "example": "namespace:reference-data--CoordinateReferenceSystem:Projected:EPSG::32615:"
+ },
+ "persistableReference": {
+ "description": "The self-contained, persistable reference string uniquely identifying the CRS.",
+ "title": "CRS Persistable Reference",
"type": "string",
- "title": "Related Condition Property",
- "description": "The property path of the target record data block, which needs subjected to the conditional matching. The data prefix is not required.",
- "example": "GeoContexts[].GeoTypeID"
- },
- "RelatedConditionMatches": {
+ "example": "{\"authCode\":{\"auth\":\"EPSG\",\"code\":\"32615\"},\"name\":\"WGS_1984_UTM_Zone_15N\",\"type\":\"LBC\",\"ver\":\"PE_10_9_1\",\"wkt\":\"PROJCS[\\\"WGS_1984_UTM_Zone_15N\\\",GEOGCS[\\\"GCS_WGS_1984\\\",DATUM[\\\"D_WGS_1984\\\",SPHEROID[\\\"WGS_1984\\\",6378137.0,298.257223563]],PRIMEM[\\\"Greenwich\\\",0.0],UNIT[\\\"Degree\\\",0.0174532925199433]],PROJECTION[\\\"Transverse_Mercator\\\"],PARAMETER[\\\"False_Easting\\\",500000.0],PARAMETER[\\\"False_Northing\\\",0.0],PARAMETER[\\\"Central_Meridian\\\",-93.0],PARAMETER[\\\"Scale_Factor\\\",0.9996],PARAMETER[\\\"Latitude_Of_Origin\\\",0.0],UNIT[\\\"Meter\\\",1.0],AUTHORITY[\\\"EPSG\\\",32615]]\"}"
+ },
+ "kind": {
+ "const": "CRS",
+ "description": "The kind of reference, constant 'CRS' for FrameOfReferenceCRS.",
+ "title": "CRS Reference Kind",
+ "type": "string"
+ },
+ "propertyNames": {
+ "description": "The list of property names, to which this meta data item provides CRS context to. A full path like \"StructureA.PropertyB\" is required to define a unique context; \"data\" is omitted since frame-of reference normalization only applies to the data block.",
+ "title": "CRS Property Names",
"type": "array",
- "title": "Related Condition Matches",
- "description": "The RelatedConditionProperty values, which need to match in order to be accepted as de-normalized value(s). If the Policy is ExtractFirstMatch, the list is prioritized and the first match is accepted as final value. Policy ExtractAllMatches collects all matching values as array.",
- "example": [
- "namespace:reference-data--GeoPoliticalEntityType:Country:"
- ],
"items": {
- "type": "string"
- }
- }
+ "type": "string"
+ },
+ "example": [
+ "KickOffPosition.X",
+ "KickOffPosition.Y"
+ ]
+ },
+ "name": {
+ "description": "The name of the CRS.",
+ "title": "CRS Name",
+ "type": "string",
+ "example": "WGS 84 / UTM zone 15N"
}
- },
- "ValueExtraction": {
- "type": "object",
- "title": "Value Extraction",
- "description": "The instructions from where to derive the value.",
- "properties": {
- "RelatedConditionProperty": {
+ },
+ "required": [
+ "kind",
+ "persistableReference"
+ ]
+ }, {
+ "title": "FrameOfReferenceDateTime",
+ "type": "object",
+ "properties": {
+ "persistableReference": {
+ "description": "The self-contained, persistable reference string uniquely identifying DateTime reference.",
+ "title": "DateTime Persistable Reference",
"type": "string",
- "title": "Related Condition Property",
- "description": "The property path of the target record data block, which needs to be subjected to the conditional matching. The data prefix is not required in the path."
- },
- "RelatedConditionMatches": {
+ "example": "{\"format\":\"yyyy-MM-ddTHH:mm:ssZ\",\"timeZone\":\"UTC\",\"type\":\"DTM\"}"
+ },
+ "kind": {
+ "const": "DateTime",
+ "description": "The kind of reference, constant 'DateTime', for FrameOfReferenceDateTime.",
+ "title": "DateTime Reference Kind",
+ "type": "string"
+ },
+ "propertyNames": {
+ "description": "The list of property names, to which this meta data item provides DateTime context to. A full path like \"StructureA.PropertyB\" is required to define a unique context; \"data\" is omitted since frame-of reference normalization only applies to the data block.",
+ "title": "DateTime Property Names",
"type": "array",
- "title": "Related Condition Matches",
- "description": "The RelatedConditionProperty values, which need to match in order to be accepted as de-normalized value(s). If the Policy is ExtractFirstMatch, the list is prioritized and the first match is accepted as final value. Policy ExtractAllMatches collects all matching values as array.",
"items": {
- "type": "string"
- }
- },
- "ValuePath": {
+ "type": "string"
+ },
+ "example": [
+ "Acquisition.StartTime",
+ "Acquisition.EndTime"
+ ]
+ },
+ "name": {
+ "description": "The name of the DateTime format and reference.",
+ "title": "DateTime Name",
+ "type": "string",
+ "example": "UTC"
+ }
+ },
+ "required": [
+ "kind",
+ "persistableReference"
+ ]
+ }, {
+ "title": "FrameOfReferenceAzimuthReference",
+ "type": "object",
+ "properties": {
+ "persistableReference": {
+ "description": "The self-contained, persistable reference string uniquely identifying AzimuthReference.",
+ "title": "AzimuthReference Persistable Reference",
"type": "string",
- "title": "Value Path",
- "description": "The path to the property from where to extract the de-normalized value. The data prefix is not required in the path.",
- "example": "GeoPoliticalEntityName"
- }
+ "example": "{\"code\":\"TrueNorth\",\"type\":\"AZR\"}"
+ },
+ "kind": {
+ "const": "AzimuthReference",
+ "description": "The kind of reference, constant 'AzimuthReference', for FrameOfReferenceAzimuthReference.",
+ "title": "AzimuthReference Reference Kind",
+ "type": "string"
+ },
+ "propertyNames": {
+ "description": "The list of property names, to which this meta data item provides AzimuthReference context to. A full path like \"StructureA.PropertyB\" is required to define a unique context; \"data\" is omitted since frame-of reference normalization only applies to the data block.",
+ "title": "AzimuthReference Property Names",
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "example": [
+ "Bearing"
+ ]
},
- "required": [
- "ValuePath"
- ]
- }
+ "name": {
+ "description": "The name of the CRS or the symbol/name of the unit.",
+ "title": "AzimuthReference Name",
+ "type": "string",
+ "example": "TrueNorth"
+ }
},
"required": [
- "ValueExtraction"
+ "kind",
+ "persistableReference"
]
- }
}
- }
+ ],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractMetaItem:1.0.0",
+ "description": "A meta data item, which allows the association of named properties or property values to a Unit/Measurement/CRS/Azimuth/Time context.",
+ "title": "Frame of Reference Meta Data Item",
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractMetaItem.1.0.0.json"
+ },
+ "osdu:wks:AbstractLegalTags:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractLegalTags:1.0.0",
+ "description": "Legal meta data like legal tags, relevant other countries, legal status. This structure is included by the SystemProperties \"legal\", which is part of all OSDU records. Not extensible.",
+ "additionalProperties": false,
+ "title": "Legal Meta Data",
+ "type": "object",
+ "properties": {
+ "legaltags": {
+ "description": "The list of legal tags, which resolve to legal properties (like country of origin, export classification code, etc.) and rules with the help of the Compliance Service.",
+ "title": "Legal Tags",
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "otherRelevantDataCountries": {
+ "description": "The list of other relevant data countries as an array of two-letter country codes, see https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2.",
+ "title": "Other Relevant Data Countries",
+ "type": "array",
+ "items": {
+ "pattern": "^[A-Z]{2}$",
+ "type": "string"
+ }
+ },
+ "status": {
+ "pattern": "^(compliant|uncompliant)$",
+ "description": "The legal status. Set by the system after evaluation against the compliance rules associated with the \"legaltags\" using the Compliance Service.",
+ "title": "Legal Status",
+ "type": "string"
+ }
+ },
+ "required": [
+ "legaltags",
+ "otherRelevantDataCountries"
+ ],
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractLegalTags.1.0.0.json"
+ },
+ "osdu:wks:AbstractReferenceType:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractReferenceType:1.0.0",
+ "description": "Generic reference object containing the universal properties of reference data, especially the ones commonly thought of as Types",
+ "x-osdu-review-status": "Accepted",
+ "title": "AbstractReferenceType",
+ "type": "object",
+ "properties": {
+ "AttributionAuthority": {
+ "description": "Name of the authority, or organisation, which governs the entity value and from which it is sourced.",
+ "type": "string"
+ },
+ "AttributionPublication": {
+ "description": "Name, URL, or other identifier of the publication, or repository, of the attribution source organisation from which the entity value is sourced.",
+ "type": "string"
+ },
+ "InactiveIndicator": {
+ "description": "By default reference values are considered as 'active'. An absent 'InactiveIndicator' property value means the reference value is in active use. When 'InactiveIndicator' is set true the reverence value is no longer in use and should no longer be offered as a choice.",
+ "type": "boolean",
+ "title": "Inactive Indicator"
+ },
+ "Description": {
+ "description": "The text which describes a NAME TYPE in detail.",
+ "type": "string"
+ },
+ "NameAlias": {
+ "x-osdu-indexing": {
+ "type": "nested"
+ },
+ "description": "Alternative names, including historical, by which this entity instance is/has been known.",
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/osdu:wks:AbstractAliasNames:1.0.0"
+ }
+ },
+ "AttributionRevision": {
+ "description": "The distinct instance of the attribution publication, by version number, sequence number, date of publication, etc., that was used for the entity value.",
+ "type": "string"
+ },
+ "ID": {
+ "description": "Native identifier from a Master Data Management System or other trusted source external to OSDU - stored here in order to allow for multi-system connection and synchronization. If used, the \"Source\" property should identify that source system.",
+ "type": "string",
+ "title": "External Identifier"
+ },
+ "Code": {
+ "description": "The abbreviation or mnemonic for a reference type if defined. Example: WELL and WLBR.",
+ "type": "string",
+ "x-osdu-natural-key": 0
+ },
+ "CommitDate": {
+ "format": "date-time",
+ "description": "For reference values published and governed by OSDU: The date and time the record was committed into the OSDU member GitLab reference-values repository. The sole purpose of this date is to optimise the OSDU milestone upgrades. It allows the upgrade code to figure out whether or not the record must be PUT into reference value storage.",
+ "type": "string",
+ "title": "Commit Date",
+ "example": "2021-02-08T21:36:40-05:00"
+ },
+ "Name": {
+ "description": "The name of the entity instance.",
+ "type": "string",
+ "x-osdu-natural-key": 1
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractReferenceType.1.0.0.json"
+ },
+ "osdu:wks:AbstractLegalParentList:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractLegalParentList:1.0.0",
+ "description": "A list of entity id:version references to record instances recorded in the data platform, from which the current record is derived and from which the legal tags must be derived. This structure is included by the SystemProperties \"ancestry\", which is part of all OSDU records. Not extensible.",
+ "additionalProperties": false,
+ "title": "Parent List",
+ "type": "object",
+ "properties": {
+ "parents": {
+ "description": "An array of none, one or many entity references of 'direct parents' in the data platform, which mark the current record as a derivative. In contrast to other relationships, the source record version is required. During record creation or update the ancestry.parents[] relationships are used to collect the legal tags from the sources and aggregate them in the legal.legaltags[] array. As a consequence, should e.g., one or more of the legal tags of the source data expire, the access to the derivatives is also terminated. For details, see ComplianceService tutorial, 'Creating derivative Records'.",
+ "title": "Parents",
+ "type": "array",
+ "items": {
+ "x-osdu-relationship": [],
+ "pattern": "^[\\w\\-\\.]+:[\\w\\-\\.]+:[\\w\\-\\.\\:\\%]+:[0-9]+$",
+ "type": "string"
+ },
+ "example": []
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractLegalParentList.1.0.0.json"
+ },
+ "osdu:wks:AbstractAliasNames:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractAliasNames:1.0.0",
+ "description": "A list of alternative names for an object. The preferred name is in a separate, scalar property. It may or may not be repeated in the alias list, though a best practice is to include it if the list is present, but to omit the list if there are no other names. Note that the abstract entity is an array so the $ref to it is a simple property reference.",
+ "x-osdu-review-status": "Accepted",
+ "title": "AbstractAliasNames",
+ "type": "object",
+ "properties": {
+ "AliasNameTypeID": {
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-AliasNameType:[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "A classification of alias names such as by role played or type of source, such as regulatory name, regulatory code, company code, international standard name, etc.",
+ "x-osdu-relationship": [{
+ "EntityType": "AliasNameType",
+ "GroupType": "reference-data"
+ }
+ ],
+ "type": "string"
+ },
+ "EffectiveDateTime": {
+ "format": "date-time",
+ "type": "string",
+ "description": "The date and time when an alias name becomes effective."
+ },
+ "AliasName": {
+ "type": "string",
+ "description": "Alternative Name value of defined name type for an object."
+ },
+ "TerminationDateTime": {
+ "format": "date-time",
+ "type": "string",
+ "description": "The data and time when an alias name is no longer in effect."
+ },
+ "DefinitionOrganisationID": {
+ "pattern": "^[\\w\\-\\.]+:(reference-data\\-\\-StandardsOrganisation|master-data\\-\\-Organisation):[\\w\\-\\.\\:\\%]+:[0-9]*$",
+ "description": "The StandardsOrganisation (reference-data) or Organisation (master-data) that provided the name (the source).",
+ "x-osdu-relationship": [{
+ "EntityType": "StandardsOrganisation",
+ "GroupType": "reference-data"
+ }, {
+ "EntityType": "Organisation",
+ "GroupType": "master-data"
+ }
+ ],
+ "type": "string"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractAliasNames.1.0.0.json"
+ },
+ "osdu:wks:AbstractAccessControlList:1.0.0": {
+ "x-osdu-inheriting-from-kind": [],
+ "x-osdu-license": "Copyright 2022, The Open Group \\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "x-osdu-schema-source": "osdu:wks:AbstractAccessControlList:1.0.0",
+ "description": "The access control tags associated with this entity. This structure is included by the SystemProperties \"acl\", which is part of all OSDU records. Not extensible.",
+ "additionalProperties": false,
+ "title": "Access Control List",
+ "type": "object",
+ "properties": {
+ "viewers": {
+ "description": "The list of viewers to which this data record is accessible/visible/discoverable formatted as an email (core.common.model.storage.validation.ValidationDoc.EMAIL_REGEX).",
+ "title": "List of Viewers",
+ "type": "array",
+ "items": {
+ "pattern": "^[a-zA-Z0-9_+&*-]+(?:\\.[a-zA-Z0-9_+&*-]+)*@(?:[a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,7}$",
+ "type": "string"
+ }
+ },
+ "owners": {
+ "description": "The list of owners of this data record formatted as an email (core.common.model.storage.validation.ValidationDoc.EMAIL_REGEX).",
+ "title": "List of Owners",
+ "type": "array",
+ "items": {
+ "pattern": "^[a-zA-Z0-9_+&*-]+(?:\\.[a-zA-Z0-9_+&*-]+)*@(?:[a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,7}$",
+ "type": "string"
+ }
+ }
+ },
+ "required": [
+ "owners",
+ "viewers"
+ ],
+ "$id": "https://schema.osdu.opengroup.org/json/abstract/AbstractAccessControlList.1.0.0.json"
+ }
+ },
+ "properties": {
+ "ancestry": {
+ "description": "The links to data, which constitute the inputs, from which this record instance is derived.",
+ "title": "Ancestry",
+ "$ref": "#/definitions/osdu:wks:AbstractLegalParentList:1.0.0"
+ },
+ "data": {
+ "allOf": [{
+ "$ref": "#/definitions/osdu:wks:AbstractCommonResources:1.0.0"
+ }, {
+ "$ref": "#/definitions/osdu:wks:AbstractReferenceType:1.0.0"
+ }, {
+ "type": "object",
+ "title": "IndividualProperties",
+ "properties": {
+ "Configurations": {
+ "x-osdu-indexing": {
+ "type": "nested"
+ },
+ "description": "The list of index property configurations for the specific kind.",
+ "type": "array",
+ "title": "Configurations",
+ "items": {
+ "description": "One single configuration to derive an Search index property value and assign it to the index 'column' with Name.",
+ "type": "object",
+ "title": "Configuration",
+ "properties": {
+ "Policy": {
+ "pattern": "^(ExtractFirstMatch|ExtractAllMatches)$",
+ "description": "Current supported policies are 'ExtractAllMatches' resulting in an array of values or 'ExtractFirstMatch' single value. The policy applies only to the Paths[].ValueExtraction.",
+ "type": "string",
+ "title": "Extraction Policy",
+ "example": "ExtractAllMatches"
+ },
+ "UseCase": {
+ "description": "The use case description this configuration satisfies.",
+ "type": "string",
+ "title": "Use Case",
+ "example": "As a user I want to find objects by a country name, with the understanding that an object may extend over country boundaries."
+ },
+ "Paths": {
+ "x-osdu-indexing": {
+ "type": "nested"
+ },
+ "description": "The list of path definitions to derive the property value from.",
+ "type": "array",
+ "title": "Paths",
+ "items": {
+ "description": "A single path definition to derive a property value from.",
+ "type": "object",
+ "title": "Path",
+ "properties": {
+ "RelatedObjectsSpec": {
+ "description": "The specification to extract related objects, from which to derive the ValueExtraction. If this property is empty or absent, the ValueExtraction is done on the current object to be indexed.",
+ "type": "object",
+ "title": "Related Objects Specification",
+ "properties": {
+ "RelatedObjectID": {
+ "description": "The path to the property containing the ID of the target record to chase. This property is only populated if the property is extracted from a related object, which must be chased. If the property is derived from 'within' the same record, which triggered the indexing, the RelatedObjectID is left absent.",
+ "type": "string",
+ "title": "Related Object ID",
+ "example": "GeoContexts[].GeoPoliticalEntityID"
+ },
+ "RelatedObjectKind": {
+ "pattern": "^[\\w\\-\\.]+:[\\w\\-\\.]+:[\\w\\-\\.]+:[0-9]+.$",
+ "description": "The kind or schema id expected as the target object type. This property is only populated if the property is extracted from a related object, which must be chased. If the property is derived from 'within' the same record, which triggered the indexing, the RelatedObjectKind is left absent.",
+ "type": "string",
+ "title": "Related Object Kind",
+ "example": "osdu:wks:master-data--GeoPoliticalEntity:1."
+ },
+ "RelationshipDirection": {
+ "pattern": "^(ChildToParent|ParentToChildren)$",
+ "description": "The direction of the relationship definition seen from the object being indexed. 'ChildToParent' assumes an outgoing relationship with the target record defined in the object being indexed. 'ParentToChildren' assumes that the related objects have a relationship by RelatedObjectID to the id of the record being indexed.",
+ "type": "string",
+ "title": "Relationship Direction",
+ "example": "ChildToParent"
+ },
+ "RelatedConditionMatches": {
+ "description": "The RelatedConditionProperty values, which need to match in order to be accepted as de-normalized value(s). If the Policy is ExtractFirstMatch, the list is prioritized and the first match is accepted as final value. Policy ExtractAllMatches collects all matching values as array.",
+ "type": "array",
+ "title": "Related Condition Matches",
+ "items": {
+ "type": "string"
+ },
+ "example": [
+ "namespace:reference-data--GeoPoliticalEntityType:Country:"
+ ]
+ },
+ "RelatedConditionProperty": {
+ "description": "The property path of the target record data block, which needs subjected to the conditional matching. The data prefix is not required.",
+ "type": "string",
+ "title": "Related Condition Property",
+ "example": "GeoContexts[].GeoTypeID"
+ }
+ }
+ },
+ "ValueExtraction": {
+ "description": "The instructions from where to derive the value.",
+ "type": "object",
+ "title": "Value Extraction",
+ "properties": {
+ "RelatedConditionMatches": {
+ "description": "The RelatedConditionProperty values, which need to match in order to be accepted as de-normalized value(s). If the Policy is ExtractFirstMatch, the list is prioritized and the first match is accepted as final value. Policy ExtractAllMatches collects all matching values as array.",
+ "type": "array",
+ "title": "Related Condition Matches",
+ "items": {
+ "type": "string"
+ }
+ },
+ "ValuePath": {
+ "description": "The path to the property from where to extract the de-normalized value. The data prefix is not required in the path.",
+ "type": "string",
+ "title": "Value Path",
+ "example": "GeoPoliticalEntityName"
+ },
+ "RelatedConditionProperty": {
+ "description": "The property path of the target record data block, which needs to be subjected to the conditional matching. The data prefix is not required in the path.",
+ "type": "string",
+ "title": "Related Condition Property"
+ }
+ },
+ "required": [
+ "ValuePath"
+ ]
+ }
+ },
+ "required": [
+ "ValueExtraction"
+ ]
+ }
+ },
+ "Name": {
+ "description": "The name of the indexed property, i.e., this is the property name used in Search.",
+ "type": "string",
+ "title": "Name",
+ "example": "CountryNames"
+ }
+ }
+ }
+ }
+ }
+ }, {
+ "type": "object",
+ "title": "ExtensionProperties",
+ "properties": {
+ "ExtensionProperties": {
+ "type": "object"
+ }
+ }
+ }
+ ]
+ },
+ "kind": {
+ "pattern": "^[\\w\\-\\.]+:[\\w\\-\\.]+:[\\w\\-\\.]+:[0-9]+.[0-9]+.[0-9]+$",
+ "description": "The schema identification for the OSDU resource object following the pattern {Namespace}:{Source}:{Type}:{VersionMajor}.{VersionMinor}.{VersionPatch}. The versioning scheme follows the semantic versioning, https://semver.org/.",
+ "title": "Entity Kind",
+ "type": "string",
+ "example": "osdu:wks:reference-data--IndexPropertyPathConfiguration:1.0.0"
+ },
+ "acl": {
+ "description": "The access control tags associated with this entity.",
+ "title": "Access Control List",
+ "$ref": "#/definitions/osdu:wks:AbstractAccessControlList:1.0.0"
+ },
+ "version": {
+ "format": "int64",
+ "description": "The version number of this OSDU resource; set by the framework.",
+ "title": "Version Number",
+ "type": "integer",
+ "example": 1562066009929332
+ },
+ "tags": {
+ "description": "A generic dictionary of string keys mapping to string value. Only strings are permitted as keys and values.",
+ "additionalProperties": {
+ "type": "string"
+ },
+ "title": "Tag Dictionary",
+ "type": "object",
+ "example": {
+ "NameOfKey": "String value"
}
- }
},
- "title": "IndividualProperties"
- },
- {
- "type": "object",
- "properties": {
- "ExtensionProperties": {
- "type": "object"
- }
+ "modifyUser": {
+ "description": "The user reference, which created this version of this resource object. Set by the System.",
+ "title": "Resource Object Version Creation User Reference",
+ "type": "string",
+ "example": "some-user@some-company-cloud.com"
},
- "title": "ExtensionProperties"
- }
- ]
- }
- },
- "required": [
- "kind",
- "acl",
- "legal"
- ],
- "additionalProperties": false,
- "x-osdu-review-status": "Accepted",
- "x-osdu-governance-model": "OPEN",
- "x-osdu-governance-authorities": [
- "Energistics",
- "OSDU"
- ],
- "x-osdu-virtual-properties": {
- "data.VirtualProperties.DefaultName": {
- "type": "string",
- "priority": [
- {
- "path": "data.Name"
- }
- ]
- }
- },
- "x-osdu-inheriting-from-kind": [
- {
- "name": "ReferenceType",
- "kind": "osdu:wks:AbstractReferenceType:1.0.0"
- }
- ]
- }
+ "modifyTime": {
+ "format": "date-time",
+ "description": "Timestamp of the time at which this version of the OSDU resource object was created. Set by the System. The value is a combined date-time string in ISO-8601 given in UTC.",
+ "title": "Resource Object Version Creation DateTime",
+ "type": "string",
+ "example": "2020-12-16T11:52:24.477Z"
+ },
+ "createTime": {
+ "format": "date-time",
+ "description": "Timestamp of the time at which initial version of this OSDU resource object was created. Set by the System. The value is a combined date-time string in ISO-8601 given in UTC.",
+ "title": "Resource Object Creation DateTime",
+ "type": "string",
+ "example": "2020-12-16T11:46:20.163Z"
+ },
+ "meta": {
+ "description": "The Frame of Reference meta data section linking the named properties to self-contained definitions.",
+ "title": "Frame of Reference Meta Data",
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/osdu:wks:AbstractMetaItem:1.0.0"
+ }
+ },
+ "legal": {
+ "description": "The entity's legal tags and compliance status. The actual contents associated with the legal tags is managed by the Compliance Service.",
+ "title": "Legal Tags",
+ "$ref": "#/definitions/osdu:wks:AbstractLegalTags:1.0.0"
+ },
+ "createUser": {
+ "description": "The user reference, which created the first version of this resource object. Set by the System.",
+ "title": "Resource Object Creation User Reference",
+ "type": "string",
+ "example": "some-user@some-company-cloud.com"
+ },
+ "id": {
+ "pattern": "^[\\w\\-\\.]+:reference-data\\-\\-IndexPropertyPathConfiguration:[\\w\\-\\.\\:\\%]+$",
+ "description": "Previously called ResourceID or SRN which identifies this OSDU resource object without version.",
+ "title": "Entity ID",
+ "type": "string",
+ "example": "namespace:reference-data--IndexPropertyPathConfiguration:c9d84708-2b1b-5e0b-954e-9621132f7154"
+ }
+ },
+ "$id": "https://schema.osdu.opengroup.org/json/reference-data/IndexPropertyPathConfiguration.1.0.0.json"
+ }
}
diff --git a/testing/indexer-test-gc/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java b/testing/indexer-test-gc/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java
index e9b432e6ca920063543e10e5fe2b1935e85c4a87..b4ef93d423a60341a94bb901eacdfc7af67a81dd 100644
--- a/testing/indexer-test-gc/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java
+++ b/testing/indexer-test-gc/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java
@@ -22,11 +22,16 @@ public class Steps extends InfoSteps {
/******************Inputs being set**************/
- @When("^I send get request to version info endpoint")
+ @When("^I send get request to version info endpoint$")
public void i_send_get_request_to_version_info_endpoint() {
super.i_send_get_request_to_version_info_endpoint();
}
+ @When("^I send get request to version info endpoint with trailing slash$")
+ public void i_send_get_request_to_version_info_endpoint_with_trailing_slash() {
+ super.i_send_get_request_to_version_info_endpoint_with_trailing_slash();
+ }
+
/******************Assert final response**************/
@Then("^I should get version info in response$")
@@ -43,4 +48,4 @@ public class Steps extends InfoSteps {
protected String getApi() {
return Config.getIndexerBaseURL() + "info";
}
-}
\ No newline at end of file
+}
diff --git a/testing/indexer-test-gc/src/test/java/org/opengroup/osdu/util/DecodedContentExtractor.java b/testing/indexer-test-gc/src/test/java/org/opengroup/osdu/util/DecodedContentExtractor.java
new file mode 100644
index 0000000000000000000000000000000000000000..b5809b522a600b1f2ca57ddb9108d4d9a8c5a86a
--- /dev/null
+++ b/testing/indexer-test-gc/src/test/java/org/opengroup/osdu/util/DecodedContentExtractor.java
@@ -0,0 +1,127 @@
+/*
+ * Copyright 2020-2024 Google LLC
+ * Copyright 2020-2024 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.util;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.InvalidPathException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Base64;
+import java.util.function.Predicate;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.java.Log;
+
+@Log
+@RequiredArgsConstructor
+public class DecodedContentExtractor {
+ private final String inputFilenameOrContent;
+ private final Predicate<String> contentAcceptanceTester;
+
+ private boolean validOutputContentFound;
+ private String outputContent;
+
+ public String getContent() {
+
+ validOutputContentFound = false;
+ outputContent = null;
+
+ log.info("Treat value as a content");
+ if (inputFilenameOrContent.trim().isEmpty()) {
+ log.info("provided value is empty. Output as is.");
+ return setValidOutputContent(inputFilenameOrContent);
+ }
+
+ if (!treatValueAsAContent(inputFilenameOrContent)) {
+ log.info("Value is not a valid content. Treat value as a filename");
+ if (!treatValueAsAFileName(inputFilenameOrContent)){
+ log.info("Value is not a filename with a valid content");
+ }
+
+ }
+
+ return getValidOutputContentIfFound();
+ }
+
+ private boolean treatValueAsAContent(String input) {
+ if (contentAcceptanceTester.test(input)) {
+ log.info("the value is a valid content. Output as is.");
+ setValidOutputContent(input);
+ return true;
+ }
+ String output;
+ try {
+ output = new String(Base64.getDecoder().decode(input));
+ log.info("the value is probably Base64 encoded. Just decoded");
+ if (contentAcceptanceTester.test(output)) {
+ log.info("the decoded value is a valid content. Output decoded value.");
+ setValidOutputContent(output);
+ } else {
+ log.info("the decoded value is not a valid content.");
+ }
+ } catch (IllegalArgumentException e) {
+ log.info("the value is not Base64 encoded. ");
+ }
+
+ return validOutputContentFound;
+ }
+
+ private boolean treatValueAsAFileName(String filename) {
+
+ if (treatFileContent(filename)) return true;
+
+ try {
+ filename = new String(Base64.getDecoder().decode(filename));
+ log.info("the filename is probably Base64 encoded. Just decoded");
+ if (treatFileContent(filename)) return true;
+ } catch (IllegalArgumentException e) {
+ log.info("the filename is not Base64 encoded. ");
+ }
+ return validOutputContentFound;
+ }
+
+ private boolean treatFileContent(String filename) {
+ try {
+ Path path = Paths.get(filename);
+ if (Files.exists(path)) {
+ log.info("the filename is of existing file. Read file.");
+ try {
+ String fileContent = new String(Files.readAllBytes(path));
+ if (treatValueAsAContent(fileContent)) {
+ return true;
+ }
+ } catch (IOException | SecurityException | OutOfMemoryError ex) {
+ log.info(() -> ("unable to read the file: " + ex.getClass().getSimpleName()));
+ }
+ }
+ } catch (InvalidPathException ex) {
+ log.info("the filename is not valid or the file doesn't exist.");
+ }
+ return false;
+ }
+
+ private String setValidOutputContent(String outputContent) {
+ this.outputContent = outputContent;
+ this.validOutputContentFound = true;
+ return getValidOutputContentIfFound();
+ }
+
+ public String getValidOutputContentIfFound() {
+ return validOutputContentFound ? outputContent : null;
+ }
+}
diff --git a/testing/indexer-test-gc/src/test/java/org/opengroup/osdu/util/JwtTokenUtil.java b/testing/indexer-test-gc/src/test/java/org/opengroup/osdu/util/JwtTokenUtil.java
index d42350bad53f95ed5c2f3b8f8cfba7159855ed33..064c39d8a24cf2fc2468144a05c4492f957cdcae 100644
--- a/testing/indexer-test-gc/src/test/java/org/opengroup/osdu/util/JwtTokenUtil.java
+++ b/testing/indexer-test-gc/src/test/java/org/opengroup/osdu/util/JwtTokenUtil.java
@@ -8,6 +8,13 @@ import com.google.api.client.json.webtoken.JsonWebToken;
import com.google.api.client.util.Clock;
import com.google.common.base.Strings;
import com.google.gson.Gson;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.GeneralSecurityException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Predicate;
import lombok.Data;
import org.apache.commons.io.Charsets;
import org.apache.commons.io.IOUtils;
@@ -19,20 +26,12 @@ import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.message.BasicNameValuePair;
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.security.GeneralSecurityException;
-import java.util.ArrayList;
-import java.util.Base64;
-import java.util.List;
-
class JwtTokenUtil {
+ private final static Predicate<String> contentAcceptanceTester = s -> s.trim().startsWith("{");
private static String accessToken;
static String getAccessToken() throws IOException {
-
if (Strings.isNullOrEmpty(accessToken)) {
accessToken = getServiceAccountAccessToken(getJwtForIntegrationTesterAccount());
}
@@ -61,14 +60,14 @@ class JwtTokenUtil {
}
private static String getJwtForIntegrationTesterAccount() throws IOException {
- String serviceAccountFile = Config.getKeyValue();
- return getJwt(serviceAccountFile);
+ String serviceAccountValue = new DecodedContentExtractor(Config.getKeyValue(), contentAcceptanceTester).getContent();
+ return getJwt(serviceAccountValue);
}
private static String getJwt(String serviceAccountFile) throws IOException {
long currentTime = Clock.SYSTEM.currentTimeMillis();
- InputStream stream = new ByteArrayInputStream(Base64.getDecoder().decode(serviceAccountFile));
+ InputStream stream = new ByteArrayInputStream(serviceAccountFile.getBytes());
GoogleCredential credential = GoogleCredential.fromStream(stream);
JsonWebSignature.Header header = new JsonWebSignature.Header();
diff --git a/testing/indexer-test-gc/src/test/resources/cucumber.properties b/testing/indexer-test-gc/src/test/resources/cucumber.properties
index 3ae231efb8a52d9736aaa08437425b1f3b980f3b..1b44376b5c42f0304a8246228ec448b5fee200de 100644
--- a/testing/indexer-test-gc/src/test/resources/cucumber.properties
+++ b/testing/indexer-test-gc/src/test/resources/cucumber.properties
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-# tag indexer-extended disabled by default
+# tag indexer-extended enabled by default
# tag as-ingested-coordinates disabled by default
# tag keyword-lower disabled by default
cucumber.options=--tags '@as-ingested-coordinates,@default,@bag-of-words,@keyword-lower,@indexer-extended'
diff --git a/testing/indexer-test-ibm/pom.xml b/testing/indexer-test-ibm/pom.xml
index b9d0bd2de3e216f3fe7f432ee30aa77f63f52383..a0e2de48f87af26dca26ccedc702161c2aa7b6f5 100644
--- a/testing/indexer-test-ibm/pom.xml
+++ b/testing/indexer-test-ibm/pom.xml
@@ -121,7 +121,7 @@
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
- <version>7.17.22</version>
+ <version>8.15.2</version>
</dependency>
<!--Logging-->
<dependency>
diff --git a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java
index 7524db63d9b6f35e2757972a5fa7ac245b2f7e96..9e30f41b948c77733444d6d54040a1930bc8efed 100644
--- a/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java
+++ b/testing/indexer-test-ibm/src/test/java/org/opengroup/osdu/step_definitions/info/Steps.java
@@ -22,11 +22,16 @@ public class Steps extends InfoSteps {
/******************Inputs being set**************/
- @When("^I send get request to version info endpoint")
+ @When("^I send get request to version info endpoint$")
public void i_send_get_request_to_version_info_endpoint() {
super.i_send_get_request_to_version_info_endpoint();
}
+ @When("^I send get request to version info endpoint with trailing slash$")
+ public void i_send_get_request_to_version_info_endpoint_with_trailing_slash() {
+ super.i_send_get_request_to_version_info_endpoint_with_trailing_slash();
+ }
+
/******************Assert final response**************/
@Then("^I should get version info in response$")
@@ -43,4 +48,4 @@ public class Steps extends InfoSteps {
protected String getApi() {
return Config.getIndexerBaseURL() + "info";
}
-}
\ No newline at end of file
+}