diff --git a/NOTICE~ b/NOTICE~
new file mode 100644
index 0000000000000000000000000000000000000000..f1bf5bc49258adf89571d007a7dadd1a2ef00bcc
--- /dev/null
+++ b/NOTICE~
@@ -0,0 +1,498 @@
+# 3rd-Party Software License Notice
+Generated by fossa-cli (https://github.com/fossas/fossa-cli).
+Formatted by fossa-with-cache (https://community.opengroup.org/divido/fossa-with-cache).
+This software includes the following software and licenses:
+
+========================================================================
+Android-Sdk
+========================================================================
+The following software have components provided under the terms of this license:
+
+- Android SDK (from https://www.android.com/)
+
+========================================================================
+Apache-1.1
+========================================================================
+The following software have components provided under the terms of this license:
+
+- Apache Commons CLI (from https://commons.apache.org/proper/commons-cli/, https://repo1.maven.org/maven2/commons-cli/commons-cli)
+- Default Plexus Container (from https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-container-default)
+- oro (from https://repo1.maven.org/maven2/oro/oro)
+
+========================================================================
+Apache-2.0
+========================================================================
+The following software have components provided under the terms of this license:
+
+- ASM based accessors helper used by json-smart (from https://urielch.github.io/)
+- AWS SDK For Java (from https://aws.amazon.com/sdkforjava)
+- AWS SDK for Java - BOM (from https://aws.amazon.com/sdkforjava)
+- AWS SDK for Java - Models (from https://aws.amazon.com/sdkforjava)
+- Apache Ant + JUnit (from http://ant.apache.org/, https://ant.apache.org/)
+- Apache Ant Core
+- Apache Ant Launcher (from http://ant.apache.org/)
+- Apache Commons BeanUtils (from http://commons.apache.org/proper/commons-beanutils/, https://repo1.maven.org/maven2/commons-beanutils/commons-beanutils)
+- Apache Commons CLI (from https://commons.apache.org/proper/commons-cli/, https://repo1.maven.org/maven2/commons-cli/commons-cli)
+- Apache Commons Codec (from http://commons.apache.org/proper/commons-codec/, https://commons.apache.org/proper/commons-codec/)
+- Apache Commons Collections (from http://commons.apache.org/proper/commons-collections/)
+- Apache Commons IO (from http://commons.apache.org/io/, https://commons.apache.org/proper/commons-io/, https://repo1.maven.org/maven2/commons-io/commons-io)
+- Apache Commons Lang (from http://commons.apache.org/proper/commons-lang/)
+- Apache Commons Logging (from http://commons.apache.org/logging/, http://commons.apache.org/proper/commons-logging/)
+- Apache Commons Validator (from http://commons.apache.org/proper/commons-validator/, http://jakarta.apache.org/commons/${pom.artifactId.substring(8)}/, https://repo1.maven.org/maven2/commons-validator/commons-validator)
+- Apache Geronimo JMS Spec 2.0 (from http://geronimo.apache.org/maven/${siteId}/${version})
+- Apache HttpAsyncClient (from http://hc.apache.org/httpcomponents-asyncclient)
+- Apache HttpClient (from http://hc.apache.org/httpcomponents-client)
+- Apache HttpCore (from http://hc.apache.org/httpcomponents-core-ga, http://hc.apache.org/httpcomponents-core-ga/)
+- Apache Maven Wagon :: API (from https://repo1.maven.org/maven2/org/apache/maven/wagon/wagon-provider-api)
+- Apache Maven Wagon :: Providers :: File Provider (from https://repo1.maven.org/maven2/org/apache/maven/wagon/wagon-file)
+- Apache Velocity (from http://velocity.apache.org/engine/releases/velocity-1.6.1/, http://velocity.apache.org/engine/releases/velocity-1.6.2/)
+- AssertJ fluent assertions (from https://repo1.maven.org/maven2/org/assertj/assertj-core)
+- AutoValue Annotations (from https://github.com/google/auto/tree/master/value, https://repo1.maven.org/maven2/com/google/auto/value/auto-value-annotations)
+- BSON (from http://bsonspec.org, https://bsonspec.org)
+- Bean Validation API (from http://beanvalidation.org)
+- Brave (from https://repo1.maven.org/maven2/io/zipkin/brave/brave)
+- Brave Instrumentation: Http Adapters (from https://repo1.maven.org/maven2/io/zipkin/brave/brave-instrumentation-http)
+- Byte Buddy (without dependencies) (from https://repo1.maven.org/maven2/net/bytebuddy/byte-buddy)
+- Byte Buddy Java agent (from https://repo1.maven.org/maven2/net/bytebuddy/byte-buddy-agent)
+- ClassMate (from http://github.com/cowtowncoder/java-classmate)
+- Collections (from https://repo1.maven.org/maven2/commons-collections/commons-collections)
+- Default Plexus Container (from https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-container-default)
+- Digester (from https://repo1.maven.org/maven2/commons-digester/commons-digester)
+- FindBugs-jsr305 (from http://findbugs.sourceforge.net/)
+- Gson (from http://code.google.com/p/google-gson/, https://repo1.maven.org/maven2/com/google/code/gson/gson)
+- Guava InternalFutureFailureAccess and InternalFutures (from https://repo1.maven.org/maven2/com/google/guava/failureaccess)
+- Guava: Google Core Libraries for Java (from http://code.google.com/p/guava-libraries, https://github.com/google/guava, https://repo1.maven.org/maven2/com/google/guava/guava)
+- IBM COS SDK For Java (from https://github.com/ibm/ibm-cos-sdk-java)
+- J2ObjC Annotations (from https://github.com/google/j2objc/)
+- JCIP Annotations under Apache License (from http://stephenc.github.com/jcip-annotations)
+- JDOM
+- JSON Small and Fast Parser (from https://repo1.maven.org/maven2/net/minidev/json-smart, https://urielch.github.io/)
+- JSON library from Android SDK (from http://developer.android.com/sdk)
+- JSON.simple (from http://code.google.com/p/json-simple/)
+- JSONassert (from http://github.com/skyscreamer/yoga, https://github.com/skyscreamer/JSONassert)
+- JSR107 API and SPI (from https://github.com/jsr107/jsr107spec)
+- Jackson (from http://jackson.codehaus.org)
+- Jackson dataformat: CBOR (from http://github.com/FasterXML/jackson-dataformats-binary)
+- Jackson dataformat: Smile (from http://github.com/FasterXML/jackson-dataformat-smile, http://github.com/FasterXML/jackson-dataformats-binary)
+- Jackson datatype: JSR310 (from http://wiki.fasterxml.com/JacksonModuleJSR310, https://repo1.maven.org/maven2/com/fasterxml/jackson/datatype/jackson-datatype-jsr310)
+- Jackson datatype: jdk8 (from https://repo1.maven.org/maven2/com/fasterxml/jackson/datatype/jackson-datatype-jdk8)
+- Jackson extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-jackson)
+- Jackson module: JAXB Annotations (from http://github.com/FasterXML/jackson-module-jaxb-annotations, http://wiki.fasterxml.com/JacksonJAXBAnnotations, https://github.com/FasterXML/jackson-modules-base)
+- Jackson-annotations (from http://github.com/FasterXML/jackson, http://wiki.fasterxml.com/JacksonHome)
+- Jackson-core (from http://wiki.fasterxml.com/JacksonHome, https://github.com/FasterXML/jackson-core)
+- Jackson-dataformat-YAML (from http://wiki.fasterxml.com/JacksonExtensionYAML, https://github.com/FasterXML/jackson, https://github.com/FasterXML/jackson-dataformats-text)
+- Jackson-module-parameter-names (from https://repo1.maven.org/maven2/com/fasterxml/jackson/module/jackson-module-parameter-names)
+- Jakarta Bean Validation API (from https://beanvalidation.org)
+- Java Native Access (from https://github.com/java-native-access/jna, https://github.com/twall/jna, https://repo1.maven.org/maven2/net/java/dev/jna/jna)
+- Java Native Access Platform (from https://github.com/java-native-access/jna)
+- Java UUID Generator (from http://wiki.fasterxml.com/JugHome)
+- Javassist (from http://www.javassist.org/)
+- JetBrains Java Annotations (from https://github.com/JetBrains/java-annotations)
+- Jetty :: Utilities (from http://jetty.mortbay.org, http://www.eclipse.org/jetty, https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-util, https://repo1.maven.org/maven2/org/mortbay/jetty/jetty-util)
+- Jetty Server (from http://jetty.mortbay.org, https://repo1.maven.org/maven2/org/mortbay/jetty/jetty)
+- Joda-Time (from http://joda-time.sourceforge.net, http://www.joda.org/joda-time/, https://www.joda.org/joda-time/)
+- Kotlin Stdlib (from https://kotlinlang.org/, https://repo1.maven.org/maven2/org/jetbrains/kotlin/kotlin-stdlib)
+- Kotlin Stdlib Common (from https://kotlinlang.org/)
+- Lang (from https://repo1.maven.org/maven2/commons-lang/commons-lang)
+- Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core)
+- Metrics Core (from https://repo1.maven.org/maven2/io/dropwizard/metrics/metrics-core)
+- Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java)
+- Mockito (from http://mockito.org, http://www.mockito.org, https://github.com/mockito/mockito)
+- MongoDB Java Driver (from http://mongodb.org/, http://www.mongodb.org)
+- Netty Reactive Streams Implementation (from https://repo1.maven.org/maven2/com/typesafe/netty/netty-reactive-streams)
+- Netty/Buffer (from https://repo1.maven.org/maven2/io/netty/netty-buffer)
+- Netty/Codec (from https://repo1.maven.org/maven2/io/netty/netty-codec)
+- Netty/Codec/HTTP (from https://repo1.maven.org/maven2/io/netty/netty-codec-http)
+- Netty/Common (from https://repo1.maven.org/maven2/io/netty/netty-common)
+- Netty/Handler (from https://repo1.maven.org/maven2/io/netty/netty-handler)
+- Netty/Resolver (from https://repo1.maven.org/maven2/io/netty/netty-resolver)
+- Netty/Transport (from https://repo1.maven.org/maven2/io/netty/netty-transport)
+- Netty/Transport/Native/Unix/Common (from https://repo1.maven.org/maven2/io/netty/netty-transport-native-unix-common)
+- Nimbus Content Type (from https://bitbucket.org/connect2id/nimbus-content-type)
+- Nimbus LangTag (from https://bitbucket.org/connect2id/nimbus-language-tags)
+- Non-Blocking Reactive Foundation for the JVM (from https://github.com/reactor/reactor, https://github.com/reactor/reactor-core)
+- OAuth 2.0 SDK with OpenID Connect extensions (from https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions)
+- Objenesis (from http://objenesis.org)
+- OkHttp (from https://repo1.maven.org/maven2/com/squareup/okhttp3/okhttp, https://square.github.io/okhttp/)
+- Okio (from https://github.com/square/okio/, https://repo1.maven.org/maven2/com/squareup/okio/okio)
+- OpenCensus (from https://github.com/census-instrumentation/opencensus-java)
+- Plexus Velocity Component (from https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-velocity)
+- PostgreSQL JDBC Driver
+- PowerMock (from http://www.powermock.org, https://repo1.maven.org/maven2/org/powermock/powermock-api-mockito)
+- Prometheus Java Simpleclient (from https://repo1.maven.org/maven2/io/prometheus/simpleclient)
+- Protocol Buffer extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-protobuf)
+- RabbitMQ Java Client (from http://www.rabbitmq.com, https://www.rabbitmq.com)
+- RxJava (from https://github.com/ReactiveX/RxJava)
+- Servlet Specification 2.5 API (from http://jetty.mortbay.org, https://repo1.maven.org/maven2/org/mortbay/jetty/servlet-api-2.5)
+- SnakeYAML (from http://code.google.com/p/snakeyaml/, http://www.snakeyaml.org)
+- Spring Boot AutoConfigure (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-autoconfigure, https://spring.io/projects/spring-boot)
+- Spring Plugin Core (from https://repo1.maven.org/maven2/org/springframework/plugin/spring-plugin-core)
+- Woodstox (from https://github.com/FasterXML/woodstox)
+- Xerces2-j (from https://repo1.maven.org/maven2/xerces/xercesImpl, https://xerces.apache.org/xerces2-j/)
+- Zipkin Reporter Brave (from https://repo1.maven.org/maven2/io/zipkin/reporter2/zipkin-reporter-brave)
+- Zipkin Reporter: Core (from https://repo1.maven.org/maven2/io/zipkin/reporter2/zipkin-reporter)
+- Zipkin v2 (from https://repo1.maven.org/maven2/io/zipkin/zipkin2/zipkin)
+- aalto-xml (from https://github.com/FasterXML/aalto-xml, https://repo1.maven.org/maven2/com/fasterxml/aalto-xml)
+- error-prone annotations (from https://repo1.maven.org/maven2/com/google/errorprone/error_prone_annotations)
+- grpc-google-cloud-pubsub-v1 (from https://github.com/googleapis/googleapis, https://repo1.maven.org/maven2/com/google/api/grpc/grpc-google-cloud-pubsub-v1)
+- io.grpc:grpc-api (from https://github.com/grpc/grpc-java)
+- io.grpc:grpc-context (from https://github.com/grpc/grpc-java)
+- ion-java (from https://github.com/amzn/ion-java/, https://github.com/amznlabs/ion-java/)
+- jackson-databind (from http://github.com/FasterXML/jackson, http://wiki.fasterxml.com/JacksonHome)
+- jakarta.inject (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/jakarta.inject)
+- javax.inject (from http://code.google.com/p/atinject/, https://repo1.maven.org/maven2/org/glassfish/hk2/external/javax.inject)
+- javax.ws.rs-api (from https://github.com/eclipse-ee4j/jaxrs-api)
+- json-path (from http://code.google.com/p/json-path/, https://github.com/jayway/JsonPath)
+- org.apiguardian:apiguardian-api (from https://github.com/apiguardian-team/apiguardian)
+- org.conscrypt:conscrypt-openjdk-uber (from https://conscrypt.org/)
+- org.opentest4j:opentest4j (from https://github.com/ota4j-team/opentest4j)
+- org.xmlunit:xmlunit-core (from http://www.xmlunit.org/, https://www.xmlunit.org/)
+- powermock-api-mockito2 (from https://repo1.maven.org/maven2/org/powermock/powermock-api-mockito2)
+- powermock-api-support (from https://repo1.maven.org/maven2/org/powermock/powermock-api-support)
+- powermock-module-junit4 (from http://www.powermock.org)
+- powermock-module-junit4-common (from https://repo1.maven.org/maven2/org/powermock/powermock-module-junit4-common)
+- powermock-reflect (from https://repo1.maven.org/maven2/org/powermock/powermock-reflect)
+- tomcat-embed-core (from http://tomcat.apache.org/)
+- xml-apis
+
+========================================================================
+BSD-2-Clause
+========================================================================
+The following software have components provided under the terms of this license:
+
+- ASM Analysis (from http://asm.ow2.io/)
+- ASM Commons (from http://asm.ow2.io/, https://repo1.maven.org/maven2/org/ow2/asm/asm-commons)
+- ASM Tree (from http://asm.ow2.io/, https://repo1.maven.org/maven2/org/ow2/asm/asm-tree)
+- ASM Util (from http://asm.ow2.io/)
+- Apache Commons CLI (from https://commons.apache.org/proper/commons-cli/, https://repo1.maven.org/maven2/commons-cli/commons-cli)
+- Hamcrest (from http://hamcrest.org/JavaHamcrest/)
+- Hamcrest Core (from http://hamcrest.org/, https://repo1.maven.org/maven2/org/hamcrest/hamcrest-core)
+- PostgreSQL JDBC Driver
+- Reflections (from http://code.google.com/p/reflections/, http://github.com/ronmamo/reflections)
+- Stax2 API (from http://github.com/FasterXML/stax2-api)
+- ThreeTen backport (from https://github.com/ThreeTen/threetenbp, https://www.threeten.org/threetenbp)
+
+========================================================================
+BSD-3-Clause
+========================================================================
+The following software have components provided under the terms of this license:
+
+- ASM Analysis (from http://asm.ow2.io/)
+- ASM Commons (from http://asm.ow2.io/, https://repo1.maven.org/maven2/org/ow2/asm/asm-commons)
+- ASM Core (from http://asm.ow2.io/, http://asm.ow2.org/)
+- ASM Tree (from http://asm.ow2.io/, https://repo1.maven.org/maven2/org/ow2/asm/asm-tree)
+- ASM Util (from http://asm.ow2.io/)
+- Apache Commons Codec (from http://commons.apache.org/proper/commons-codec/, https://commons.apache.org/proper/commons-codec/)
+- Hamcrest (from http://hamcrest.org/JavaHamcrest/)
+- Hamcrest Core (from http://hamcrest.org/, https://repo1.maven.org/maven2/org/hamcrest/hamcrest-core)
+- Jakarta Activation API (from https://github.com/eclipse-ee4j/jaf, https://repo1.maven.org/maven2/jakarta/activation/jakarta.activation-api)
+- Jakarta XML Binding API (from https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api, https://repo1.maven.org/maven2/org/jboss/spec/javax/xml/bind/jboss-jaxb-api_2.3_spec)
+- Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core)
+- Mockito (from http://mockito.org, http://www.mockito.org, https://github.com/mockito/mockito)
+- Netty/Codec/HTTP (from https://repo1.maven.org/maven2/io/netty/netty-codec-http)
+- PostgreSQL JDBC Driver
+- Protocol Buffer Java API (from http://code.google.com/p/protobuf, https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java)
+- Reflections (from http://code.google.com/p/reflections/, http://github.com/ronmamo/reflections)
+- SnakeYAML (from http://code.google.com/p/snakeyaml/, http://www.snakeyaml.org)
+- ThreeTen backport (from https://github.com/ThreeTen/threetenbp, https://www.threeten.org/threetenbp)
+- classworlds (from http://classworlds.codehaus.org/, https://repo1.maven.org/maven2/classworlds/classworlds)
+- jaxen (from http://jaxen.codehaus.org/, https://repo1.maven.org/maven2/jaxen/jaxen)
+
+========================================================================
+CC-BY-2.5
+========================================================================
+The following software have components provided under the terms of this license:
+
+- FindBugs-jsr305 (from http://findbugs.sourceforge.net/)
+- MongoDB Java Driver (from http://mongodb.org/, http://www.mongodb.org)
+
+========================================================================
+CC0-1.0
+========================================================================
+The following software have components provided under the terms of this license:
+
+- reactive-streams (from http://www.reactive-streams.org/)
+
+========================================================================
+CDDL-1.0
+========================================================================
+The following software have components provided under the terms of this license:
+
+- Jakarta Activation API (from https://github.com/eclipse-ee4j/jaf, https://repo1.maven.org/maven2/jakarta/activation/jakarta.activation-api)
+- Servlet Specification 2.5 API (from http://jetty.mortbay.org, https://repo1.maven.org/maven2/org/mortbay/jetty/servlet-api-2.5)
+
+========================================================================
+CDDL-1.1
+========================================================================
+The following software have components provided under the terms of this license:
+
+- HK2 Spring Bridge (from https://repo1.maven.org/maven2/org/glassfish/hk2/spring-bridge)
+- HK2 config types (from https://repo1.maven.org/maven2/org/glassfish/hk2/config-types)
+- HK2 configuration module (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-config)
+- Java Architecture for XML Binding (from http://jaxb.java.net/, https://repo1.maven.org/maven2/javax/xml/bind/jaxb-api)
+- JavaBeans Activation Framework
+- JavaBeans(TM) Activation Framework (from http://java.sun.com/javase/technologies/desktop/javabeans/jaf/index.jsp)
+- JavaMail API (from https://repo1.maven.org/maven2/com/sun/mail/javax.mail)
+- OSGi resource locator (from https://repo1.maven.org/maven2/org/glassfish/hk2/osgi-resource-locator)
+- Servlet Specification 2.5 API (from http://jetty.mortbay.org, https://repo1.maven.org/maven2/org/mortbay/jetty/servlet-api-2.5)
+- javax.annotation-api (from http://jcp.org/en/jsr/detail?id=250)
+- tomcat-embed-core (from http://tomcat.apache.org/)
+
+========================================================================
+CPL-1.0
+========================================================================
+The following software have components provided under the terms of this license:
+
+- JUnit (from http://junit.org)
+
+========================================================================
+EPL-1.0
+========================================================================
+The following software have components provided under the terms of this license:
+
+- HK2 Implementation Utilities (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-utils)
+- JUnit Jupiter (Aggregator) (from https://junit.org/junit5/)
+- JUnit Jupiter API (from http://junit.org/junit5/, https://junit.org/junit5/)
+- JUnit Jupiter Engine (from http://junit.org/junit5/, https://junit.org/junit5/)
+- JUnit Jupiter Params (from http://junit.org/junit5/, https://junit.org/junit5/)
+- JUnit Platform Commons (from http://junit.org/junit5/, https://junit.org/junit5/)
+- JUnit Platform Engine API (from http://junit.org/junit5/, https://junit.org/junit5/)
+- Jakarta Annotations API (from https://projects.eclipse.org/projects/ee4j.ca)
+- Logback Classic Module (from http://logback.qos.ch, https://repo1.maven.org/maven2/ch/qos/logback/logback-classic)
+- Logback Contrib :: JSON :: Classic (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-json-classic)
+- Logback Contrib :: JSON :: Core (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-json-core)
+- Logback Contrib :: Jackson (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-jackson)
+- Logback Core Module (from http://logback.qos.ch, https://repo1.maven.org/maven2/ch/qos/logback/logback-core)
+- OSGi resource locator (from https://repo1.maven.org/maven2/org/glassfish/hk2/osgi-resource-locator)
+- ServiceLocator Default Implementation (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-locator)
+- SnakeYAML (from http://code.google.com/p/snakeyaml/, http://www.snakeyaml.org)
+- aopalliance-repackaged (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/aopalliance-repackaged)
+- jakarta.inject (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/jakarta.inject)
+- javax.ws.rs-api (from https://github.com/eclipse-ee4j/jaxrs-api)
+
+========================================================================
+EPL-2.0
+========================================================================
+The following software have components provided under the terms of this license:
+
+- HK2 Implementation Utilities (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-utils)
+- JUnit Jupiter (Aggregator) (from https://junit.org/junit5/)
+- JUnit Jupiter API (from http://junit.org/junit5/, https://junit.org/junit5/)
+- JUnit Jupiter Engine (from http://junit.org/junit5/, https://junit.org/junit5/)
+- JUnit Jupiter Params (from http://junit.org/junit5/, https://junit.org/junit5/)
+- JUnit Platform Commons (from http://junit.org/junit5/, https://junit.org/junit5/)
+- JUnit Platform Engine API (from http://junit.org/junit5/, https://junit.org/junit5/)
+- Jakarta Annotations API (from https://projects.eclipse.org/projects/ee4j.ca)
+- OSGi resource locator (from https://repo1.maven.org/maven2/org/glassfish/hk2/osgi-resource-locator)
+- ServiceLocator Default Implementation (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-locator)
+- aopalliance-repackaged (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/aopalliance-repackaged)
+- jakarta.inject (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/jakarta.inject)
+- javax.ws.rs-api (from https://github.com/eclipse-ee4j/jaxrs-api)
+
+========================================================================
+GPL-2.0-only
+========================================================================
+The following software have components provided under the terms of this license:
+
+- Java Architecture for XML Binding (from http://jaxb.java.net/, https://repo1.maven.org/maven2/javax/xml/bind/jaxb-api)
+- JavaBeans Activation Framework
+- Lang (from https://repo1.maven.org/maven2/commons-lang/commons-lang)
+- OSGi resource locator (from https://repo1.maven.org/maven2/org/glassfish/hk2/osgi-resource-locator)
+- RabbitMQ Java Client (from http://www.rabbitmq.com, https://www.rabbitmq.com)
+- javax.annotation-api (from http://jcp.org/en/jsr/detail?id=250)
+- tomcat-embed-core (from http://tomcat.apache.org/)
+
+========================================================================
+GPL-2.0-or-later
+========================================================================
+The following software have components provided under the terms of this license:
+
+- SnakeYAML (from http://code.google.com/p/snakeyaml/, http://www.snakeyaml.org)
+
+========================================================================
+GPL-2.0-with-classpath-exception
+========================================================================
+The following software have components provided under the terms of this license:
+
+- Checker Qual (from https://checkerframework.org)
+- HK2 Implementation Utilities (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-utils)
+- Jakarta Annotations API (from https://projects.eclipse.org/projects/ee4j.ca)
+- Java Architecture for XML Binding (from http://jaxb.java.net/, https://repo1.maven.org/maven2/javax/xml/bind/jaxb-api)
+- JavaBeans Activation Framework
+- OSGi resource locator (from https://repo1.maven.org/maven2/org/glassfish/hk2/osgi-resource-locator)
+- RabbitMQ Java Client (from http://www.rabbitmq.com, https://www.rabbitmq.com)
+- ServiceLocator Default Implementation (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-locator)
+- aopalliance-repackaged (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/aopalliance-repackaged)
+- jakarta.inject (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/jakarta.inject)
+- javax.annotation-api (from http://jcp.org/en/jsr/detail?id=250)
+- javax.ws.rs-api (from https://github.com/eclipse-ee4j/jaxrs-api)
+- tomcat-embed-core (from http://tomcat.apache.org/)
+
+========================================================================
+GPL-3.0-only
+========================================================================
+The following software have components provided under the terms of this license:
+
+- Jakarta Annotations API (from https://projects.eclipse.org/projects/ee4j.ca)
+- OSGi resource locator (from https://repo1.maven.org/maven2/org/glassfish/hk2/osgi-resource-locator)
+- Project Lombok (from http://projectlombok.org, https://projectlombok.org)
+- javax.ws.rs-api (from https://github.com/eclipse-ee4j/jaxrs-api)
+
+========================================================================
+JSON
+========================================================================
+The following software have components provided under the terms of this license:
+
+- JSON in Java (from https://github.com/douglascrockford/JSON-java)
+
+========================================================================
+LGPL-2.1-only
+========================================================================
+The following software have components provided under the terms of this license:
+
+- Java Native Access (from https://github.com/java-native-access/jna, https://github.com/twall/jna, https://repo1.maven.org/maven2/net/java/dev/jna/jna)
+- Java Native Access Platform (from https://github.com/java-native-access/jna)
+- Javassist (from http://www.javassist.org/)
+- Lang (from https://repo1.maven.org/maven2/commons-lang/commons-lang)
+- Logback Classic Module (from http://logback.qos.ch, https://repo1.maven.org/maven2/ch/qos/logback/logback-classic)
+- Logback Contrib :: JSON :: Classic (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-json-classic)
+- Logback Contrib :: JSON :: Core (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-json-core)
+- Logback Contrib :: Jackson (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-jackson)
+- Logback Core Module (from http://logback.qos.ch, https://repo1.maven.org/maven2/ch/qos/logback/logback-core)
+
+========================================================================
+LGPL-2.1-or-later
+========================================================================
+The following software have components provided under the terms of this license:
+
+- Javassist (from http://www.javassist.org/)
+- SnakeYAML (from http://code.google.com/p/snakeyaml/, http://www.snakeyaml.org)
+
+========================================================================
+LGPL-3.0-only
+========================================================================
+- RabbitMQ Java Client (from http://www.rabbitmq.com, https://www.rabbitmq.com)
+
+========================================================================
+MIT
+========================================================================
+The following software have components provided under the terms of this license:
+
+- Animal Sniffer Annotations (from https://repo1.maven.org/maven2/org/codehaus/mojo/animal-sniffer-annotations)
+- Bouncy Castle Provider (from http://www.bouncycastle.org/java.html, https://www.bouncycastle.org/java.html)
+- Checker Qual (from https://checkerframework.org)
+- Default Plexus Container (from https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-container-default)
+- JOpt Simple (from http://jopt-simple.github.io/jopt-simple, http://pholser.github.io/jopt-simple)
+- JUL to SLF4J bridge (from http://www.slf4j.org)
+- Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core)
+- Microsoft Azure SDK for SQL API of Azure Cosmos DB Service (from https://github.com/Azure/azure-sdk-for-java)
+- Mockito (from http://mockito.org, http://www.mockito.org, https://github.com/mockito/mockito)
+- Netty/Codec/HTTP (from https://repo1.maven.org/maven2/io/netty/netty-codec-http)
+- Netty/Common (from https://repo1.maven.org/maven2/io/netty/netty-common)
+- Plexus Default Interactivity Handler (from https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-interactivity-api)
+- Project Lombok (from http://projectlombok.org, https://projectlombok.org)
+- SLF4J API Module (from http://www.slf4j.org)
+
+========================================================================
+MPL-1.1
+========================================================================
+The following software have components provided under the terms of this license:
+
+- Javassist (from http://www.javassist.org/)
+- RabbitMQ Java Client (from http://www.rabbitmq.com, https://www.rabbitmq.com)
+
+========================================================================
+MPL-2.0
+========================================================================
+The following software have components provided under the terms of this license:
+
+- Javassist (from http://www.javassist.org/)
+- OkHttp (from https://repo1.maven.org/maven2/com/squareup/okhttp3/okhttp, https://square.github.io/okhttp/)
+- RabbitMQ Java Client (from http://www.rabbitmq.com, https://www.rabbitmq.com)
+
+========================================================================
+PHP-3.01
+========================================================================
+The following software have components provided under the terms of this license:
+
+- Jakarta Activation API (from https://github.com/eclipse-ee4j/jaf, https://repo1.maven.org/maven2/jakarta/activation/jakarta.activation-api)
+- Jakarta XML Binding API (from https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api, https://repo1.maven.org/maven2/org/jboss/spec/javax/xml/bind/jboss-jaxb-api_2.3_spec)
+
+========================================================================
+SAX-PD
+========================================================================
+The following software have components provided under the terms of this license:
+
+- Apache Ant Core
+
+========================================================================
+SPL-1.0
+========================================================================
+The following software have components provided under the terms of this license:
+
+- Servlet Specification 2.5 API (from http://jetty.mortbay.org, https://repo1.maven.org/maven2/org/mortbay/jetty/servlet-api-2.5)
+
+========================================================================
+SunPro
+========================================================================
+The following software have components provided under the terms of this license:
+
+- Lucene Core (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-core)
+
+========================================================================
+W3C
+========================================================================
+The following software have components provided under the terms of this license:
+
+- Apache Ant Core
+- JTidy (from http://jtidy.sourceforge.net)
+- Xerces2-j (from https://repo1.maven.org/maven2/xerces/xercesImpl, https://xerces.apache.org/xerces2-j/)
+- jaxen (from http://jaxen.codehaus.org/, https://repo1.maven.org/maven2/jaxen/jaxen)
+- xml-apis
+
+========================================================================
+WTFPL
+========================================================================
+The following software have components provided under the terms of this license:
+
+- Reflections (from http://code.google.com/p/reflections/, http://github.com/ronmamo/reflections)
+
+========================================================================
+public-domain
+========================================================================
+The following software have components provided under the terms of this license:
+
+- Bouncy Castle PKIX, CMS, EAC, TSP, PKCS, OCSP, CMP, and CRMF APIs (from http://www.bouncycastle.org/java.html, https://www.bouncycastle.org/java.html)
+- Bouncy Castle Provider (from http://www.bouncycastle.org/java.html, https://www.bouncycastle.org/java.html)
+- Guava: Google Core Libraries for Java (from http://code.google.com/p/guava-libraries, https://github.com/google/guava, https://repo1.maven.org/maven2/com/google/guava/guava)
+- HdrHistogram (from http://hdrhistogram.github.io/HdrHistogram/)
+- JTidy (from http://jtidy.sourceforge.net)
+- LatencyUtils (from http://latencyutils.github.io/LatencyUtils/)
+- PostgreSQL JDBC Driver
+- Project Lombok (from http://projectlombok.org, https://projectlombok.org)
+- Prometheus Java Simpleclient (from https://repo1.maven.org/maven2/io/prometheus/simpleclient)
+- RabbitMQ Java Client (from http://www.rabbitmq.com, https://www.rabbitmq.com)
+- reactive-streams (from http://www.reactive-streams.org/)
+- xml-apis
+
+========================================================================
+unknown
+========================================================================
+The following software have components provided under the terms of this license:
+
+- Bouncy Castle PKIX, CMS, EAC, TSP, PKCS, OCSP, CMP, and CRMF APIs (from http://www.bouncycastle.org/java.html, https://www.bouncycastle.org/java.html)
+- Bouncy Castle Provider (from http://www.bouncycastle.org/java.html, https://www.bouncycastle.org/java.html)
+- Byte Buddy (without dependencies) (from https://repo1.maven.org/maven2/net/bytebuddy/byte-buddy)
+- Checker Qual (from https://checkerframework.org)
+- JSON in Java (from https://github.com/douglascrockford/JSON-java)
+- JTidy (from http://jtidy.sourceforge.net)
+- JUnit Jupiter (Aggregator) (from https://junit.org/junit5/)
+- JUnit Jupiter API (from http://junit.org/junit5/, https://junit.org/junit5/)
+- JUnit Jupiter Engine (from http://junit.org/junit5/, https://junit.org/junit5/)
+- JUnit Jupiter Params (from http://junit.org/junit5/, https://junit.org/junit5/)
+- JUnit Platform Commons (from http://junit.org/junit5/, https://junit.org/junit5/)
+- JUnit Platform Engine API (from http://junit.org/junit5/, https://junit.org/junit5/)
+- Jakarta Activation API (from https://github.com/eclipse-ee4j/jaf, https://repo1.maven.org/maven2/jakarta/activation/jakarta.activation-api)
+- Jakarta XML Binding API (from https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api, https://repo1.maven.org/maven2/org/jboss/spec/javax/xml/bind/jboss-jaxb-api_2.3_spec)
+- jaxen (from http://jaxen.codehaus.org/, https://repo1.maven.org/maven2/jaxen/jaxen)
+- xml-apis
\ No newline at end of file
diff --git a/docs/tutorial/IndexerService.md b/docs/tutorial/IndexerService.md
index 5208cc9b30936fdc9c2ebc75fab57e2f47f9fb82..c65489ba596b332bc6b85374cb954c3607b1778a 100644
--- a/docs/tutorial/IndexerService.md
+++ b/docs/tutorial/IndexerService.md
@@ -5,9 +5,11 @@
- [Indexer service](#indexer-service)
- [Introduction](#introduction)
- [Indexer API access](#indexer-api-access)
-- [Version info endpoint](#version-info-endpoint)
-- [Reindex](#reindex)
-- [Data Partition provision](#data-partition-provision)
+- [API Reference](#api-reference)
+ - [Version info endpoint](#version-info-endpoint)
+ - [Reindex](#reindex)
+ - [Data Partition provision](#data-partition-provision)
+ - [Schema change](#schema-change)
- [Schema Service adoption](#schema-service-adoption)
* [R3 Schema Support](#r3-schema-support)
- [Troubleshoot Indexing Issues](#troubleshoot-indexing-issues)
@@ -51,9 +53,17 @@ If the service is initiating the request, an ID should be generated. If the `cor
[Back to table of contents](#TOC)
-## Version info endpoint
+## API Reference
-For deployment available public `/info` endpoint, which provides build and git related information.
+### Version info endpoint
+
+Provides build and git related information.
+
+#### Request
+
+```http
+GET /api/indexer/v2/info HTTP/1.1
+```
#### Example response:
@@ -86,20 +96,16 @@ This endpoint takes information from files, generated by `spring-boot-maven-plug
[Back to table of contents](#TOC)
-## Reindex <a name="reindex"></a>
+### Reindex <a name="reindex"></a>
-Reindex API allows users to re-index a `kind` without re-ingesting the records via storage API. Reindexing a kind is an
-asynchronous operation and when a user calls this API, it will respond with HTTP 200 if it can launch the re-indexing or
-appropriate error code if it cannot. The current status of the indexing can be tracked by calling search API and making
-query with this particular kind. Please be advised, it may take few seconds to few hours to finish the re-indexing as
+Reindex API allows users to re-index a `kind` without re-ingesting the records via storage API. Reindexing a kind is an asynchronous operation and when a user calls this API, it will respond with HTTP 200 if it can launch the re-indexing or
+appropriate error code if it cannot. The current status of the indexing can be tracked by calling search API and making query with this particular kind. Please be advised, it may take few seconds to few hours to finish the re-indexing as
multiple factors contribute to latency, such as number of records in the kind, current load at the indexer service etc.
-__Note__: If a kind has been previously indexed with particular schema and if you wish to apply the schema changes
-during re-indexing, previous kind index has to be deleted via Index Delete API. In absence of this clean-up, reindex API
-will use the same schema and overwrite records with the same ids.
+#### Request
-```
-POST /api/indexer/v2/reindex
+```http
+POST /api/indexer/v2/reindex HTTP/1.1
{
"kind": "opendes:welldb:wellbore:1.0.0"
}
@@ -121,14 +127,30 @@ curl --request POST \
</details>
+#### Prerequisite
+
+Users must be a member of `users.datalake.admins` or `users.datalake.ops` group.
+
+#### Query parameters
+
+`force_clean` <br />
+  (optional, Boolean) If a kind has been previously indexed with a schema and if you wish to apply latest schema changes before re-indexing, than use this query parameter. It will drop the current Index schema, apply latest schema changes & re-index records. If `false`, reindex API
+will use the same schema and overwrite records with the same ids. Default value is `false`.
+
+#### Request body
+
+`kind` <br />
+  (required, String) Kind to be re-indexed.
+
+
[Back to table of contents](#TOC)
-## Data Partition provision <a name="data-partition-provision"></a>
+### Data Partition provision <a name="data-partition-provision"></a>
Configures Search backend for a data partition.
-```
-PUT /api/indexer/v2/partitions/provision
+```http
+PUT /api/indexer/v2/partitions/provision HTTP/1.1
```
<details><summary>**Curl**</summary>
@@ -143,10 +165,52 @@ curl --request PUT \
```
</details>
+#### Prerequisite
+
+Users must be a member of `users.datalake.ops` group.
+
> __NOTE__: API should be run at-least once at the data partition provisioning to configure required resources/settings.
[Back to table of contents](#TOC)
+### Schema change <a name="schema-change"></a>
+
+Schema change event listener endpoint.
+
+> __Note:__ This is internal API and shouldn't be exposed publicly.
+
+#### Request
+
+```http
+POST /api/indexer/v2/_dps/task-handlers/schema-worker HTTP/1.1
+{
+ "messageId": "676894654",
+ "publishTime": "2017-03-19T00:00:00",
+ "attributes": {
+ "data-partition-id": "opendes",
+ "correlation-id": "b5a281bd-f59d-4db2-9939-b2d85036fc7e"
+ },
+ "data": "[{\"kind\":\"slb:indexer:test-data--SchemaEventIntegration:1.0.0\",\"op\":\"create\"}]"
+}
+```
+
+#### Request body
+
+`messageId` <br />
+  (optional, String) Event message id.
+
+`publishTime` <br />
+  (optional, String) Event publish time.
+
+`attributes.data-partition-id` <br />
+  (required, String) Data partition id for which this message is targeted.
+
+`attributes.correlation-id` <br />
+  (optional, String) Correlation-id to enable tracing.
+
+`data` <br />
+  (required, String) Schema change event message json string. Only `create` and `update` events are supported.
+
## Schema Service adoption <a name="schema-service-adoption"></a>
Indexer service is in adaptation process to use schemas from the Schema service instead of Storage Service. The Indexer
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/RecordIndexerApi.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/RecordIndexerApi.java
index 5962141825b4a3c6bf33ddec01c270a560e27dc9..1f8df5cf5a6ae9b208d371ab24522075348150a5 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/RecordIndexerApi.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/RecordIndexerApi.java
@@ -22,13 +22,16 @@ import io.swagger.annotations.ApiOperation;
import lombok.extern.java.Log;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.http.AppException;
-import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
-import org.opengroup.osdu.indexer.SwaggerDoc;
import org.opengroup.osdu.core.common.model.indexer.JobStatus;
+import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
+import org.opengroup.osdu.core.common.model.indexer.SchemaChangedMessages;
+import org.opengroup.osdu.core.common.model.indexer.SchemaInfo;
+import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
+import org.opengroup.osdu.indexer.SwaggerDoc;
import org.opengroup.osdu.indexer.service.IndexerService;
import org.opengroup.osdu.indexer.service.ReindexService;
-import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
+import org.opengroup.osdu.indexer.service.SchemaService;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
@@ -37,6 +40,7 @@ import org.springframework.web.context.annotation.RequestScope;
import javax.inject.Inject;
import javax.validation.Valid;
import javax.validation.constraints.NotNull;
+import java.io.IOException;
import java.lang.reflect.Type;
import java.util.List;
@@ -50,6 +54,8 @@ public class RecordIndexerApi {
private IndexerService indexerService;
@Inject
private ReindexService reIndexService;
+ @Inject
+ private SchemaService schemaService;
// THIS IS AN INTERNAL USE API ONLY
// THAT MEANS WE DON'T DOCUMENT IT IN SWAGGER, ACCESS IS LIMITED TO CLOUD TASK QUEUE CALLS ONLY
@@ -96,4 +102,35 @@ public class RecordIndexerApi {
@Valid RecordReindexRequest recordReindexRequest) {
return new ResponseEntity<>(reIndexService.reindexRecords(recordReindexRequest, false), HttpStatus.OK);
}
+
+ // THIS IS AN INTERNAL USE API ONLY
+ // THAT MEANS WE DON'T DOCUMENT IT IN SWAGGER, ACCESS IS LIMITED TO CLOUD TASK QUEUE CALLS ONLY
+ @PostMapping("/schema-worker")
+ @ApiOperation(hidden = true, value = "", notes = "")
+ public ResponseEntity<?> schemaWorker(
+ @NotNull(message = SwaggerDoc.REQUEST_VALIDATION_NOT_NULL_BODY)
+ @Valid @RequestBody SchemaChangedMessages schemaChangedMessage) throws IOException {
+ if (schemaChangedMessage == null) {
+ log.warning("schema change messages is null");
+ }
+
+ if (schemaChangedMessage.missingAccountId()) {
+ throw new AppException(org.apache.http.HttpStatus.SC_BAD_REQUEST, "Invalid tenant",
+ String.format("Required header: '%s' not found", DpsHeaders.DATA_PARTITION_ID));
+ }
+
+ try {
+ Type listType = new TypeToken<List<SchemaInfo>>() {}.getType();
+ List<SchemaInfo> schemaInfos = new Gson().fromJson(schemaChangedMessage.getData(), listType);
+
+ if (schemaInfos.size() == 0) {
+ log.warning("none of schema-change message can be deserialized");
+ return new ResponseEntity(org.springframework.http.HttpStatus.OK);
+ }
+ this.schemaService.processSchemaMessages(schemaInfos);
+ return new ResponseEntity(HttpStatus.OK);
+ } catch (JsonParseException e) {
+ throw new AppException(org.apache.http.HttpStatus.SC_BAD_REQUEST, "Request payload parsing error", "Unable to parse request payload.", e);
+ }
+ }
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/config/SchemaEventsListenerConfiguration.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/config/SchemaEventsListenerConfiguration.java
new file mode 100644
index 0000000000000000000000000000000000000000..7f4bddefe2cbd10706e9ba82acb1f5815c1e45ba
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/config/SchemaEventsListenerConfiguration.java
@@ -0,0 +1,16 @@
+package org.opengroup.osdu.indexer.config;
+
+import lombok.Getter;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@Getter
+public class SchemaEventsListenerConfiguration {
+
+ @Value("${listner.schema.event.create:true}")
+ private boolean listenCreateEvent;
+
+ @Value("${listner.schema.event.update:true}")
+ private boolean listenUpdateEvent;
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditEvents.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditEvents.java
index 703d6747e2510d71ad0b27d1fe9843c697ea5ea8..fc07fc61ac12bbe429b0f8cb5e5b84117f8fc77a 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditEvents.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditEvents.java
@@ -27,7 +27,6 @@ public class AuditEvents {
private static final String INDEX_CREATE_RECORDS_SUCCESS = "Successfully created record in index";
private static final String INDEX_CREATE_RECORDS_FAILURE = "Failed creating record in index";
-
private static final String INDEX_UPDATE_RECORD_ACTION_ID = "IN002";
private static final String INDEX_UPDATE_RECORDS_SUCCESS = "Successfully updated record in index";
private static final String INDEX_UPDATE_RECORDS_FAILURE = "Failed updating record in index";
@@ -54,8 +53,8 @@ public class AuditEvents {
private static final String RUN_JOB_MESSAGE_SUCCESS = "Index clean-up status job run success";
private static final String INDEX_MAPPING_UPDATE_ACTION_ID = "IN0011";
- private static final String INDEX_MAPPING_UPDATE_SUCCESS = "Successfully updated index mapping";
- private static final String INDEX_MAPPING_UPDATE_FAILURE = "Failed updating index mapping";
+ private static final String INDEX_MAPPING_UPDATE_SUCCESS = "Successfully upserted index mapping";
+ private static final String INDEX_MAPPING_UPDATE_FAILURE = "Failed upserting index mapping";
private static final String CONFIGURE_PARTITION_ACTION_ID = "IN0012";
private static final String CONFIGURE_PARTITION_OPERATION = "Data partition cluster configuration update";
@@ -223,7 +222,7 @@ public class AuditEvents {
.build();
}
- public AuditPayload getIndexMappingUpdateEvent(List<String> resources, boolean isSuccess) {
+ public AuditPayload getIndexMappingUpsertEvent(List<String> resources, boolean isSuccess) {
if (isSuccess) {
return AuditPayload.builder()
.action(AuditAction.UPDATE)
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditLogger.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditLogger.java
index 1a37d76881a1bbc49bb2b4c97cc037f8a8af2a32..c85e6d7a2a4dda8fb78662cc165928efdc51c130 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditLogger.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/logging/AuditLogger.java
@@ -93,11 +93,12 @@ public class AuditLogger {
this.writeLog(this.getAuditEvents().getIndexCleanUpJobRunEvent(resources));
}
- public void indexMappingUpdateSuccess(List<String> resources) {
- this.writeLog(this.getAuditEvents().getIndexMappingUpdateEvent(resources,true));
+ public void indexMappingUpsertSuccess(List<String> resources) {
+ this.writeLog(this.getAuditEvents().getIndexMappingUpsertEvent(resources,true));
}
- public void indexMappingUpdateFail(List<String> resources) {
- this.writeLog(this.getAuditEvents().getIndexMappingUpdateEvent(resources,false));
+
+ public void indexMappingUpsertFail(List<String> resources) {
+ this.writeLog(this.getAuditEvents().getIndexMappingUpsertEvent(resources,false));
}
public void getConfigurePartition(List<String> resources) {
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/Kind.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/Kind.java
new file mode 100644
index 0000000000000000000000000000000000000000..483dd6b35911457760b85865b1fde1097a1a8912
--- /dev/null
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/model/Kind.java
@@ -0,0 +1,27 @@
+package org.opengroup.osdu.indexer.model;
+
+import lombok.Getter;
+
+@Getter
+public class Kind {
+
+ private String kind;
+ private String authority;
+ private String source;
+ private String type;
+ private String version;
+
+ public Kind(String kind) {
+ this.kind = kind;
+ String[] parts = this.kind.split(":");
+
+ if (parts.length != 4) {
+ throw new IllegalArgumentException("Invalid Kind, must be in format: authority:source:type:version");
+ }
+
+ this.authority = parts[0];
+ this.source = parts[1];
+ this.type = parts[2];
+ this.version = parts[3];
+ }
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IMappingService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IMappingService.java
index 689e7a654ae9fcbb6acc0fa659a51f804151ad03..f0009bd048fe18312c9cddefbb3b258fd591756f 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IMappingService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IMappingService.java
@@ -33,5 +33,5 @@ public interface IMappingService {
void updateIndexMappingForIndicesOfSameType(Set<String> indices, String fieldName) throws Exception;
- void syncIndexMappingIfRequired(RestHighLevelClient restClient, String index) throws Exception;
+ void syncIndexMappingIfRequired(RestHighLevelClient restClient, IndexSchema schema) throws Exception;
}
\ No newline at end of file
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaService.java
index 9691ada93cdfc8b90c6650b99e8782768afcaad2..e474feef6f284951667c80ff719407e07a2c0623 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaService.java
@@ -15,6 +15,8 @@
package org.opengroup.osdu.indexer.service;
import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.client.RestHighLevelClient;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
import org.opengroup.osdu.core.common.model.indexer.OperationType;
@@ -33,6 +35,8 @@ public interface IndexSchemaService {
void processSchemaMessages(Map<String, OperationType> schemaMsgs) throws IOException;
+ void processSchemaUpsertEvent(RestHighLevelClient restClient, String kind) throws IOException, ElasticsearchStatusException, URISyntaxException;
+
void syncIndexMappingWithStorageSchema(String kind) throws ElasticsearchException, IOException, AppException, URISyntaxException;
boolean isStorageSchemaSyncRequired(String kind, boolean forceClean) throws IOException;
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
index 7787fba463d36e76566575bc9e949bbb3ea92727..0f5bcd7b06c0695cd867c7fb4df1ec3a18c80ddc 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexSchemaServiceImpl.java
@@ -30,6 +30,7 @@ import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
import org.opengroup.osdu.core.common.model.storage.Schema;
import org.opengroup.osdu.core.common.model.storage.SchemaItem;
import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
+import org.opengroup.osdu.indexer.model.Kind;
import org.opengroup.osdu.indexer.provider.interfaces.ISchemaCache;
import org.opengroup.osdu.indexer.schema.converter.exeption.SchemaProcessingException;
import org.opengroup.osdu.indexer.util.ElasticClientHandler;
@@ -85,30 +86,7 @@ public class IndexSchemaServiceImpl implements IndexSchemaService {
boolean indexExist = this.indicesService.isIndexExist(restClient, index);
if (msg.getValue() == OperationType.create_schema) {
- // reset cache and get new schema
- this.invalidateCache(kind);
- IndexSchema schemaObj = this.getIndexerInputSchema(kind, true);
- if (schemaObj.isDataSchemaMissing()) {
- log.warning(String.format("schema not found for kind: %s", kind));
- return;
- }
-
- if (indexExist) {
- try {
- // merge the mapping
- this.mappingService.createMapping(restClient, schemaObj, index, true);
- } catch (AppException e) {
- // acknowledge for TaskQueue and not retry
- if (e.getError().getCode() == HttpStatus.SC_BAD_REQUEST) {
- throw new AppException(RequestStatus.SCHEMA_CONFLICT, e.getError().getReason(), "error creating or merging index mapping");
- }
- throw e;
- }
- } else {
- // create index with mapping
- Map<String, Object> mapping = this.mappingService.getIndexMappingFromRecordSchema(schemaObj);
- this.indicesService.createIndex(restClient, index, null, schemaObj.getType(), mapping);
- }
+ this.processSchemaUpsertEvent(restClient, kind);
} else if (msg.getValue() == OperationType.purge_schema) {
if (indexExist) {
// reset schema cache
@@ -120,6 +98,37 @@ public class IndexSchemaServiceImpl implements IndexSchemaService {
}
}
+ @Override
+ public void processSchemaUpsertEvent(RestHighLevelClient restClient, String kind) throws IOException, ElasticsearchStatusException, URISyntaxException {
+ String index = this.elasticIndexNameResolver.getIndexNameFromKind(kind);
+ boolean indexExist = this.indicesService.isIndexExist(restClient, index);
+
+ // reset cache and get new schema
+ this.invalidateCache(kind);
+ IndexSchema schemaObj = this.getIndexerInputSchema(kind, true);
+ if (schemaObj.isDataSchemaMissing()) {
+ log.warning(String.format("schema not found for kind: %s", kind));
+ return;
+ }
+
+ if (indexExist) {
+ try {
+ // merge the mapping
+ this.mappingService.createMapping(restClient, schemaObj, index, true);
+ } catch (AppException e) {
+ // acknowledge for TaskQueue and not retry
+ if (e.getError().getCode() == HttpStatus.SC_BAD_REQUEST) {
+ throw new AppException(RequestStatus.SCHEMA_CONFLICT, e.getError().getReason(), "error creating or merging index mapping");
+ }
+ throw e;
+ }
+ } else {
+ // create index with mapping
+ Map<String, Object> mapping = this.mappingService.getIndexMappingFromRecordSchema(schemaObj);
+ this.indicesService.createIndex(restClient, index, null, schemaObj.getType(), mapping);
+ }
+ }
+
@Override
public IndexSchema getIndexerInputSchema(String kind, List<String> errors) throws AppException, UnsupportedEncodingException, URISyntaxException {
try {
@@ -223,6 +232,8 @@ public class IndexSchemaServiceImpl implements IndexSchemaService {
}
}
+ Kind kind = new Kind(schemaObj.getKind());
+
// mandatory attributes
meta.put(RecordMetaAttribute.ID.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.ID));
meta.put(RecordMetaAttribute.NAMESPACE.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.NAMESPACE));
@@ -242,10 +253,7 @@ public class IndexSchemaServiceImpl implements IndexSchemaService {
meta.put(RecordMetaAttribute.MODIFY_USER.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.MODIFY_USER));
meta.put(RecordMetaAttribute.MODIFY_TIME.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.MODIFY_TIME));
- String kind = schemaObj.getKind();
- String type = kind.split(":")[2];
-
- return IndexSchema.builder().dataSchema(data).metaSchema(meta).kind(kind).type(type).build();
+ return IndexSchema.builder().dataSchema(data).metaSchema(meta).kind(schemaObj.getKind()).type(kind.getType()).build();
} catch (Exception e) {
throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Schema normalization error", "An error has occurred while normalizing the schema.", e);
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
index 6acd078e913b9733f3d95e335461f397a6966525..13738aefa36c8871481e6db4f9ad4949a45849b3 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceImpl.java
@@ -35,8 +35,11 @@ import org.opengroup.osdu.core.common.Constants;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
+import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
+import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
import org.opengroup.osdu.core.common.search.Preconditions;
import org.opengroup.osdu.indexer.cache.PartitionSafeIndexCache;
+import org.opengroup.osdu.indexer.model.Kind;
import org.opengroup.osdu.indexer.util.ElasticClientHandler;
import org.opengroup.osdu.indexer.util.TypeMapper;
import org.springframework.beans.factory.annotation.Autowired;
@@ -58,6 +61,8 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements IMa
private PartitionSafeIndexCache indexCache;
@Autowired
private IMappingService mappingService;
+ @Autowired
+ private ElasticIndexNameResolver elasticIndexNameResolver;
private static TimeValue REQUEST_TIMEOUT = TimeValue.timeValueMinutes(1);
@@ -111,12 +116,8 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements IMa
Map<String, Object> properties = new HashMap<>();
// meta attribute
- Map<String, Object> metaMapping = new HashMap<>();
- for (Map.Entry<String, Object> entry : schema.getMetaSchema().entrySet()) {
- metaMapping.put(entry.getKey(), TypeMapper.getMetaAttributeIndexerMapping(entry.getKey()));
- }
+ Map<String, Object> metaMapping = this.getMetaMapping(schema);
- // data-source attributes
// data-source attributes
Map<String, Object> dataMapping = this.getDataMapping(schema);
if (!dataMapping.isEmpty()) {
@@ -138,6 +139,22 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements IMa
return documentMapping;
}
+ private Map<String, Object> getMetaMapping(IndexSchema schema) {
+ Map<String, Object> metaMapping = new HashMap<>();
+ Kind kind = new Kind(schema.getKind());
+
+ for (Map.Entry<String, Object> entry : schema.getMetaSchema().entrySet()) {
+ if (entry.getKey() == RecordMetaAttribute.AUTHORITY.getValue()) {
+ metaMapping.put(entry.getKey(), TypeMapper.getMetaAttributeIndexerMapping(entry.getKey(), kind.getAuthority()));
+ } else if (entry.getKey() == RecordMetaAttribute.SOURCE.getValue()) {
+ metaMapping.put(entry.getKey(), TypeMapper.getMetaAttributeIndexerMapping(entry.getKey(), kind.getSource()));
+ } else {
+ metaMapping.put(entry.getKey(), TypeMapper.getMetaAttributeIndexerMapping(entry.getKey(), null));
+ }
+ }
+ return metaMapping;
+ }
+
private Map<String, Object> getDataMapping(IndexSchema schema) {
Map<String, Object> dataMapping = new HashMap<>();
if (schema.getDataSchema() == null || schema.getDataSchema().isEmpty()) return dataMapping;
@@ -158,7 +175,8 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements IMa
}
@Override
- public void syncIndexMappingIfRequired(RestHighLevelClient restClient, String index) throws Exception {
+ public void syncIndexMappingIfRequired(RestHighLevelClient restClient, IndexSchema schema) throws Exception {
+ String index = this.elasticIndexNameResolver.getIndexNameFromKind(schema.getKind());
final String cacheKey = String.format("metaAttributeMappingSynced-%s", index);
try {
@@ -191,8 +209,15 @@ public class IndexerMappingServiceImpl extends MappingServiceImpl implements IMa
}
Map<String, Object> properties = new HashMap<>();
+ Kind kind = new Kind(schema.getKind());
for (String attribute : missing) {
- properties.put(attribute, TypeMapper.getMetaAttributeIndexerMapping(attribute));
+ if (attribute == RecordMetaAttribute.AUTHORITY.getValue()) {
+ properties.put(attribute, TypeMapper.getMetaAttributeIndexerMapping(attribute, kind.getAuthority()));
+ } else if (attribute == RecordMetaAttribute.SOURCE.getValue()) {
+ properties.put(attribute, TypeMapper.getMetaAttributeIndexerMapping(attribute, kind.getSource()));
+ } else {
+ properties.put(attribute, TypeMapper.getMetaAttributeIndexerMapping(attribute, null));
+ }
}
Map<String, Object> documentMapping = new HashMap<>();
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
index 24544bc2fb9e84fd2ed1b4691771f54fd7cfe457..6bec8964f4095c5bf8b6f118c1da2cda440ac6cd 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
@@ -115,6 +115,10 @@ public class IndexerServiceImpl implements IndexerService {
this.jobStatus.initialize(recordInfos);
try {
+ auditLogger.indexStarted(recordInfos.stream()
+ .map(RecordInfo::getKind)
+ .collect(Collectors.toList()));
+
// get upsert records
Map<String, Map<String, OperationType>> upsertRecordMap = RecordInfo.getUpsertRecordIds(recordInfos);
if (upsertRecordMap != null && !upsertRecordMap.isEmpty()) {
@@ -129,11 +133,7 @@ public class IndexerServiceImpl implements IndexerService {
retryRecordIds.addAll(deleteFailureRecordIds);
}
- auditLogger.indexStarted(recordInfos.stream()
- .map(RecordInfo::getKind)
- .collect(Collectors.toList()));
-
- // process schema change messages
+ // process legacy storage schema change messages
Map<String, OperationType> schemaMsgs = RecordInfo.getSchemaMsgs(recordInfos);
if (schemaMsgs != null && !schemaMsgs.isEmpty()) {
this.schemaService.processSchemaMessages(schemaMsgs);
@@ -367,7 +367,7 @@ public class IndexerServiceImpl implements IndexerService {
// check if index exist and sync meta attribute schema if required
if (this.indicesService.isIndexReady(restClient, index)) {
- this.mappingService.syncIndexMappingIfRequired(restClient, index);
+ this.mappingService.syncIndexMappingIfRequired(restClient, schema);
continue;
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SchemaProviderImpl.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SchemaProviderImpl.java
index 164cf5fa4fac298e761ea2195246c591067c76b8..c82d538a234ae20cbc41f166ec36e08749891da0 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SchemaProviderImpl.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SchemaProviderImpl.java
@@ -15,29 +15,41 @@
package org.opengroup.osdu.indexer.service;
import com.google.api.client.http.HttpMethods;
+import com.google.gson.Gson;
import org.apache.http.HttpStatus;
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.client.RestHighLevelClient;
import org.opengroup.osdu.core.common.http.FetchServiceHttpRequest;
import org.opengroup.osdu.core.common.http.IUrlFetchService;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.HttpResponse;
+import org.opengroup.osdu.core.common.model.indexer.SchemaInfo;
+import org.opengroup.osdu.core.common.model.indexer.SchemaOperationType;
import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
-import org.opengroup.osdu.indexer.schema.converter.exeption.SchemaProcessingException;
+import org.opengroup.osdu.indexer.config.SchemaEventsListenerConfiguration;
+import org.opengroup.osdu.indexer.logging.AuditLogger;
import org.opengroup.osdu.indexer.schema.converter.interfaces.SchemaToStorageFormat;
+import org.opengroup.osdu.indexer.util.ElasticClientHandler;
import org.springframework.stereotype.Component;
import javax.inject.Inject;
+import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URISyntaxException;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
-import java.util.Objects;
+import java.util.*;
+
+import static java.util.Collections.singletonList;
/**
* Provides implementation of the client service that retrieves schemas from the Schema Service
*/
@Component
public class SchemaProviderImpl implements SchemaService {
+
@Inject
private JaxRsDpsLog log;
@@ -56,12 +68,58 @@ public class SchemaProviderImpl implements SchemaService {
@Inject
private StorageService storageService;
+ @Inject
+ private IndexSchemaService indexSchemaService;
+
+ @Inject
+ private ElasticClientHandler elasticClientHandler;
+
+ @Inject
+ private AuditLogger auditLogger;
+
+ @Inject
+ private SchemaEventsListenerConfiguration schemaEventsListenerConfiguration;
+
+
@Override
public String getSchema(String kind) throws URISyntaxException, UnsupportedEncodingException {
String schemaServiceSchema = getFromSchemaService(kind);
return Objects.nonNull(schemaServiceSchema) ? schemaServiceSchema : getFromStorageService(kind);
}
+ @Override
+ public void processSchemaMessages(List<SchemaInfo> schemaInfos) throws IOException {
+ Map<String, SchemaOperationType> messages = new HashMap<>();
+
+ if (schemaEventsListenerConfiguration.isListenCreateEvent()) {
+ Map<String, SchemaOperationType> createSchemaMessages = SchemaInfo.getCreateSchemaEvents(schemaInfos);
+ if (createSchemaMessages != null && !createSchemaMessages.isEmpty()) {
+ messages.putAll(createSchemaMessages);
+ }
+ }
+
+ if (schemaEventsListenerConfiguration.isListenUpdateEvent()) {
+ Map<String, SchemaOperationType> updateSchemaMessages = SchemaInfo.getUpdateSchemaEvents(schemaInfos);
+ if (updateSchemaMessages != null && !updateSchemaMessages.isEmpty()) {
+ messages.putAll(updateSchemaMessages);
+ }
+ }
+
+ if (messages.isEmpty()) return;
+
+ try (RestHighLevelClient restClient = this.elasticClientHandler.createRestClient()) {
+ messages.entrySet().forEach(msg -> {
+ try {
+ this.indexSchemaService.processSchemaUpsertEvent(restClient, msg.getKey());
+ this.auditLogger.indexMappingUpsertSuccess(singletonList(msg.getKey()));
+ } catch (IOException | ElasticsearchStatusException | URISyntaxException e) {
+ this.auditLogger.indexMappingUpsertFail(singletonList(msg.getKey()));
+ throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "unable to process schema upsert event", e.getMessage());
+ }
+ });
+ }
+ }
+
protected String getFromSchemaService(String kind) throws UnsupportedEncodingException, URISyntaxException {
HttpResponse response = getSchemaServiceResponse(kind);
@@ -90,10 +148,10 @@ public class SchemaProviderImpl implements SchemaService {
String url = String.format("%s/%s", configurationProperties.getSchemaHost(), URLEncoder.encode(kind, StandardCharsets.UTF_8.toString()));
FetchServiceHttpRequest request = FetchServiceHttpRequest.builder()
.httpMethod(HttpMethods.GET)
- .headers(this.requestInfo.getHeadersMap())
+ .headers(this.requestInfo.getHeadersMapWithDwdAuthZ())
.url(url)
.build();
-
+
return this.urlFetchService.sendRequest(request);
}
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SchemaService.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SchemaService.java
index 174630771365a1aa6761298dc0f0c0d7582c960b..5e87ca8ddf3f86cc7a8571ab8736720602240af8 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SchemaService.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/service/SchemaService.java
@@ -14,11 +14,15 @@
package org.opengroup.osdu.indexer.service;
+import org.opengroup.osdu.core.common.model.indexer.SchemaInfo;
+
+import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URISyntaxException;
+import java.util.List;
/**
- * Interface to retriew schemas from the Schema Service
+ * Interface to consume schemas from the Schema Service
*/
public interface SchemaService {
/**
@@ -29,4 +33,11 @@ public interface SchemaService {
* @throws UnsupportedEncodingException
*/
String getSchema(String kind) throws URISyntaxException, UnsupportedEncodingException;
+
+ /**
+ *
+ * @param schemaInfos schema change event
+ * @throws IOException
+ */
+ void processSchemaMessages(List<SchemaInfo> schemaInfos) throws IOException;
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java
index bd48202110fae6ebb501e6b9de7a2e7ce61e1b63..7ef011331ea576303495db1c5cb4ceb86ea09ad1 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/util/TypeMapper.java
@@ -51,8 +51,8 @@ public class TypeMapper {
metaAttributeIndexerType.put(RecordMetaAttribute.LEGAL.getValue(), getLegalIndexerMapping());
metaAttributeIndexerType.put(RecordMetaAttribute.ANCESTRY.getValue(), getAncestryIndexerMapping());
metaAttributeIndexerType.put(RecordMetaAttribute.INDEX_STATUS.getValue(), getIndexStatusMapping());
- metaAttributeIndexerType.put(RecordMetaAttribute.AUTHORITY.getValue(), ElasticType.CONSTANT_KEYWORD.getValue());
- metaAttributeIndexerType.put(RecordMetaAttribute.SOURCE.getValue(), ElasticType.CONSTANT_KEYWORD.getValue());
+ metaAttributeIndexerType.put(RecordMetaAttribute.AUTHORITY.getValue(), getConstantKeywordMap());
+ metaAttributeIndexerType.put(RecordMetaAttribute.SOURCE.getValue(), getConstantKeywordMap());
metaAttributeIndexerType.put(RecordMetaAttribute.CREATE_USER.getValue(), ElasticType.KEYWORD.getValue());
metaAttributeIndexerType.put(RecordMetaAttribute.MODIFY_USER.getValue(), ElasticType.KEYWORD.getValue());
metaAttributeIndexerType.put(RecordMetaAttribute.CREATE_TIME.getValue(), ElasticType.DATE.getValue());
@@ -91,14 +91,22 @@ public class TypeMapper {
return metaAttributeIndexerType.getOrDefault(attribute.getValue(), null);
}
+ private static Object getConstantIndexerType(String key, String value) {
+ Map<String, Object> constantAttribute = (Map<String, Object>) metaAttributeIndexerType.get(key);
+ constantAttribute.put("value", value);
+ return constantAttribute;
+ }
+
public static List<String> getMetaAttributesKeys() {
return new ArrayList<>(metaAttributeIndexerType.keySet());
}
- public static Object getMetaAttributeIndexerMapping(String key) {
+ public static Object getMetaAttributeIndexerMapping(String key, String value) {
if (key.equals(RecordMetaAttribute.ACL.getValue())
|| key.equals(RecordMetaAttribute.LEGAL.getValue()) || key.equals(RecordMetaAttribute.ANCESTRY.getValue()) || key.equals(RecordMetaAttribute.INDEX_STATUS.getValue())) {
return metaAttributeIndexerType.get(key);
+ } else if (key.equals(RecordMetaAttribute.AUTHORITY.getValue()) || key.equals(RecordMetaAttribute.SOURCE.getValue())) {
+ return getConstantIndexerType(key, value);
}
return Records.Type.builder().type(metaAttributeIndexerType.get(key).toString()).build();
}
@@ -211,4 +219,10 @@ public class TypeMapper {
fieldIndexTypeMap.put("keyword", keywordMap);
return fieldIndexTypeMap;
}
+
+ private static Map<String, Object> getConstantKeywordMap() {
+ Map<String, Object> map = new HashMap<>();
+ map.put("type", "constant_keyword");
+ return map;
+ }
}
\ No newline at end of file
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/RecordIndexerApiTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/RecordIndexerApiTest.java
index 1a3752139d2bbf9d642f85dbeb9f0f636aa8ba8a..a54d2161f71a06c37c21456ff1af00f3ef6f948b 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/RecordIndexerApiTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/api/RecordIndexerApiTest.java
@@ -15,24 +15,26 @@
package org.opengroup.osdu.indexer.api;
import com.google.gson.Gson;
-import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
-import org.opengroup.osdu.core.common.model.http.DpsHeaders;
-import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.http.HeadersUtil;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.model.indexer.SchemaChangedMessages;
+import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
import org.opengroup.osdu.indexer.service.IndexerService;
+import org.opengroup.osdu.indexer.service.SchemaService;
import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
-import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
-import org.opengroup.osdu.core.common.http.HeadersUtil;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import static org.mockito.MockitoAnnotations.initMocks;
@@ -40,7 +42,8 @@ import static org.mockito.MockitoAnnotations.initMocks;
@PrepareForTest({HeadersUtil.class, IndexerQueueTaskBuilder.class, DpsHeaders.class})
public class RecordIndexerApiTest {
- private final String messageValid = "{\"data\":\"[{\\\"id\\\":\\\"opendes:welldb:wellbore-d9033ae1-fb15-496c-9ba0-880fd1d2b2cf\\\",\\\"kind\\\":\\\"tenant1:welldb:wellbore:1.0.0\\\",\\\"op\\\":\\\"create\\\"}]\",\"attributes\":{\"account-id\":\"opendes\",\"correlation-id\":\"b5a281bd-f59d-4db2-9939-b2d85036fc7e\"},\"messageId\":\"75328163778221\",\"publishTime\":\"2018-05-08T21:48:56.131Z\"}";
+ private final String recordMessageValid = "{\"data\":\"[{\\\"id\\\":\\\"opendes:welldb:wellbore-d9033ae1-fb15-496c-9ba0-880fd1d2b2cf\\\",\\\"kind\\\":\\\"tenant1:welldb:wellbore:1.0.0\\\",\\\"op\\\":\\\"create\\\"}]\",\"attributes\":{\"account-id\":\"opendes\",\"correlation-id\":\"b5a281bd-f59d-4db2-9939-b2d85036fc7e\"},\"messageId\":\"75328163778221\",\"publishTime\":\"2018-05-08T21:48:56.131Z\"}";
+ private final String schemaMessageValid = "{\"data\":\"[{\\\"kind\\\":\\\"tenant1:welldb:wellbore:1.0.0\\\",\\\"op\\\":\\\"create\\\"}]\",\"attributes\":{\"account-id\":\"opendes\",\"correlation-id\":\"b5a281bd-f59d-4db2-9939-b2d85036fc7e\"},\"messageId\":\"75328163778221\",\"publishTime\":\"2018-05-08T21:48:56.131Z\"}";
private final String messageEmpty = "{}";
private final String messageWithEmptyData = "{\"data\":\"[]\",\"attributes\":{\"account-id\":\"opendes\",\"correlation-id\":\"b5a281bd-f59d-4db2-9939-b2d85036fc7e\"},\"messageId\":\"75328163778221\",\"publishTime\":\"2018-05-08T21:48:56.131Z\"}";
private final String messageWithIncorrectJsonFormat = "{\"data\":\"[{}}]\",\"attributes\":{\"account-id\":\"opendes\",\"correlation-id\":\"b5a281bd-f59d-4db2-9939-b2d85036fc7e\"},\"messageId\":\"75328163778221\",\"publishTime\":\"2018-05-08T21:48:56.131Z\"}";
@@ -54,6 +57,8 @@ public class RecordIndexerApiTest {
private JaxRsDpsLog log;
@Mock
private IndexerService indexService;
+ @Mock
+ private SchemaService schemaService;
@Mock
private DpsHeaders dpsHeaders;
@@ -68,7 +73,7 @@ public class RecordIndexerApiTest {
@Test
public void should_return200_given_validMessage_indexWorkerTest() throws Exception {
- should_return200_indexerWorkerTest(messageValid);
+ should_return200_indexerWorkerTest(recordMessageValid);
}
@Test
@@ -86,9 +91,24 @@ public class RecordIndexerApiTest {
should_return400_indexerWorkerTest(messageWithIncorrectJsonFormat, "Unable to parse request payload.");
}
+ @Test
+ public void should_return200_given_validMessage_schemaWorkerTest() throws Exception {
+ should_return200_schemaWorkerTest(schemaMessageValid);
+ }
+
+ @Test
+ public void should_return200_given_emptyData_schemaWorkerTest() throws Exception {
+ should_return200_schemaWorkerTest(messageWithEmptyData);
+ }
+
+ @Test
+ public void should_return400_given_incorrectJsonFormatMessage_SchemaWorkerTest() {
+ should_return400_schemaWorkerTest(messageWithIncorrectJsonFormat, "Unable to parse request payload.");
+ }
+
private void should_return200_indexerWorkerTest(String message) throws Exception {
ResponseEntity response = this.sut.indexWorker(createRecordChangedMessage(message));
- Assert.assertEquals(HttpStatus.OK.value(), response.getStatusCodeValue());
+ assertEquals(HttpStatus.OK.value(), response.getStatusCodeValue());
}
private void should_return400_indexerWorkerTest(String message, String errorMessage) {
@@ -96,8 +116,8 @@ public class RecordIndexerApiTest {
this.sut.indexWorker(createRecordChangedMessage(message));
fail("Should throw exception");
} catch (AppException e) {
- Assert.assertEquals(HttpStatus.BAD_REQUEST.value(), e.getError().getCode());
- Assert.assertEquals(errorMessage, e.getError().getMessage());
+ assertEquals(HttpStatus.BAD_REQUEST.value(), e.getError().getCode());
+ assertEquals(errorMessage, e.getError().getMessage());
} catch (Exception e) {
fail("Should not throw this exception" + e.getMessage());
}
@@ -106,4 +126,25 @@ public class RecordIndexerApiTest {
private RecordChangedMessages createRecordChangedMessage(String message) {
return (new Gson()).fromJson(message, RecordChangedMessages.class);
}
+
+ private void should_return200_schemaWorkerTest(String message) throws Exception {
+ ResponseEntity response = this.sut.schemaWorker(createSchemaChangedMessage(message));
+ assertEquals(HttpStatus.OK.value(), response.getStatusCodeValue());
+ }
+
+ private void should_return400_schemaWorkerTest(String message, String errorMessage) {
+ try {
+ this.sut.schemaWorker(createSchemaChangedMessage(message));
+ fail("Should throw exception");
+ } catch (AppException e) {
+ assertEquals(HttpStatus.BAD_REQUEST.value(), e.getError().getCode());
+ assertEquals(errorMessage, e.getError().getMessage());
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ private SchemaChangedMessages createSchemaChangedMessage(String message) {
+ return (new Gson()).fromJson(message, SchemaChangedMessages.class);
+ }
}
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/logging/AuditEventsTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/logging/AuditEventsTest.java
index 685482e2833c2bfb02509b27c8c29b07b73691a8..27d1e4b8e7c8ce8de1076fb8a464505c1efedd48 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/logging/AuditEventsTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/logging/AuditEventsTest.java
@@ -205,24 +205,25 @@ public class AuditEventsTest {
@SuppressWarnings({"unchecked", "rawtypes"})
public void should_getIndexMappingUpdateEventSuccess() {
AuditEvents auditEvent = new AuditEvents("testUser");
- Map<String, String> payload = (Map) auditEvent.getIndexMappingUpdateEvent(Lists.newArrayList("anything"),true)
+ Map<String, String> payload = (Map) auditEvent.getIndexMappingUpsertEvent(Lists.newArrayList("anything"),true)
.get("auditLog");
assertEquals(Lists.newArrayList("anything"), payload.get("resources"));
assertEquals(AuditStatus.SUCCESS, payload.get("status"));
- assertEquals("Successfully updated index mapping", payload.get("message"));
+ assertEquals("Successfully upserted index mapping", payload.get("message"));
assertEquals(AuditAction.UPDATE, payload.get("action"));
assertEquals("IN0011", payload.get("actionId"));
assertEquals("testUser", payload.get("user"));
}
+
@Test
@SuppressWarnings({"unchecked", "rawtypes"})
public void should_getIndexMappingUpdateEventFail() {
AuditEvents auditEvent = new AuditEvents("testUser");
- Map<String, String> payload = (Map) auditEvent.getIndexMappingUpdateEvent(Lists.newArrayList("anything"),false)
+ Map<String, String> payload = (Map) auditEvent.getIndexMappingUpsertEvent(Lists.newArrayList("anything"),false)
.get("auditLog");
assertEquals(Lists.newArrayList("anything"), payload.get("resources"));
assertEquals(AuditStatus.FAILURE, payload.get("status"));
- assertEquals("Failed updating index mapping", payload.get("message"));
+ assertEquals("Failed upserting index mapping", payload.get("message"));
assertEquals(AuditAction.UPDATE, payload.get("action"));
assertEquals("IN0011", payload.get("actionId"));
assertEquals("testUser", payload.get("user"));
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/logging/AuditLoggerTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/logging/AuditLoggerTest.java
index 63906cfdc636439514809aded529e17524b14a23..63318bde1b809886d57a7a5b42ae4a069d43baf5 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/logging/AuditLoggerTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/logging/AuditLoggerTest.java
@@ -189,7 +189,7 @@ public class AuditLoggerTest {
@Test
@SuppressWarnings("rawtypes")
public void should_createAuditLogEvent_when_indexMappingUpdateFail() {
- this.sut.indexMappingUpdateFail(Lists.newArrayList("anything"));
+ this.sut.indexMappingUpsertFail(Lists.newArrayList("anything"));
ArgumentCaptor<AuditPayload> payloadCaptor = ArgumentCaptor.forClass(AuditPayload.class);
verify(this.logger).audit(payloadCaptor.capture());
AuditPayload payload = payloadCaptor.getValue();
@@ -200,7 +200,7 @@ public class AuditLoggerTest {
@Test
@SuppressWarnings("rawtypes")
public void should_createAuditLogEvent_when_indexMappingUpdateSuccess() {
- this.sut.indexMappingUpdateSuccess(Lists.newArrayList("anything"));
+ this.sut.indexMappingUpsertSuccess(Lists.newArrayList("anything"));
ArgumentCaptor<AuditPayload> payloadCaptor = ArgumentCaptor.forClass(AuditPayload.class);
verify(this.logger).audit(payloadCaptor.capture());
AuditPayload payload = payloadCaptor.getValue();
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java
index c3ff4bf968ffbf21d41dd516f340bd92cb3e6a41..ec1f61e4bf77cb496265cfc3142ebcbe77c4e71b 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/IndexerMappingServiceTest.java
@@ -27,6 +27,7 @@ import org.mockito.Mock;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
+import org.opengroup.osdu.core.common.search.ElasticIndexNameResolver;
import org.opengroup.osdu.indexer.cache.PartitionSafeIndexCache;
import org.opengroup.osdu.indexer.util.ElasticClientHandler;
import org.opengroup.osdu.indexer.util.TypeMapper;
@@ -56,8 +57,8 @@ public class IndexerMappingServiceTest {
private final String kind = "tenant:test:test:1.0.0";
private final String index = "tenant-test-test-1.0.0";
private final String type = "test";
- private final String validMapping = "{\"dynamic\":false,\"properties\":{\"data\":{\"properties\":{\"Msg\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"null_value\":\"null\",\"ignore_above\":256,\"type\":\"keyword\"}}},\"Location\":{\"type\":\"geo_point\"}}},\"id\":{\"type\":\"keyword\"},\"acl\":{\"properties\":{\"viewers\":{\"type\":\"keyword\"},\"owners\":{\"type\":\"keyword\"}}}}}";
- private final String emptyDataValidMapping = "{\"dynamic\":false,\"properties\":{\"id\":{\"type\":\"keyword\"},\"acl\":{\"properties\":{\"viewers\":{\"type\":\"keyword\"},\"owners\":{\"type\":\"keyword\"}}}}}";
+ private final String validMapping = "{\"dynamic\":false,\"properties\":{\"data\":{\"properties\":{\"Msg\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"null_value\":\"null\",\"ignore_above\":256,\"type\":\"keyword\"}}},\"Location\":{\"type\":\"geo_point\"}}},\"authority\":{\"type\":\"constant_keyword\",\"value\":\"tenant\"},\"id\":{\"type\":\"keyword\"},\"acl\":{\"properties\":{\"viewers\":{\"type\":\"keyword\"},\"owners\":{\"type\":\"keyword\"}}}}}";
+ private final String emptyDataValidMapping = "{\"dynamic\":false,\"properties\":{\"id\":{\"type\":\"keyword\"},\"acl\":{\"properties\":{\"viewers\":{\"type\":\"keyword\"},\"owners\":{\"type\":\"keyword\"}}},\"authority\":{\"type\":\"constant_keyword\",\"value\":\"tenant\"}}}";
@Mock
private RestClient restClient;
@@ -73,6 +74,8 @@ public class IndexerMappingServiceTest {
private PartitionSafeIndexCache indexCache;
@Mock
private IMappingService mappingService;
+ @Mock
+ private ElasticIndexNameResolver elasticIndexNameResolver;
@InjectMocks
private IndexerMappingServiceImpl sut;
@@ -89,6 +92,7 @@ public class IndexerMappingServiceTest {
this.indicesClient = PowerMockito.mock(IndicesClient.class);
this.restHighLevelClient = PowerMockito.mock(RestHighLevelClient.class);
+ when(this.elasticIndexNameResolver.getIndexNameFromKind(kind)).thenReturn(index);
when(this.restHighLevelClient.getLowLevelClient()).thenReturn(restClient);
when(this.restClient.performRequest(any())).thenReturn(response);
when(this.response.getStatusLine()).thenReturn(statusLine);
@@ -99,6 +103,7 @@ public class IndexerMappingServiceTest {
Map<String, Object> metaMapping = new HashMap<>();
metaMapping.put(RecordMetaAttribute.ID.getValue(), "keyword");
metaMapping.put(RecordMetaAttribute.ACL.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.ACL));
+ metaMapping.put(RecordMetaAttribute.AUTHORITY.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.AUTHORITY));
return metaMapping;
}
@@ -177,7 +182,7 @@ public class IndexerMappingServiceTest {
final String cacheKey = String.format("metaAttributeMappingSynced-%s", index);
when(this.indexCache.get(cacheKey)).thenReturn(true);
- this.sut.syncIndexMappingIfRequired(restHighLevelClient, index);
+ this.sut.syncIndexMappingIfRequired(restHighLevelClient, indexSchema);
verifyNoMoreInteractions(this.mappingService);
}
@@ -188,7 +193,7 @@ public class IndexerMappingServiceTest {
final String mapping = "{\"dynamic\":\"false\",\"properties\":{\"acl\":{\"properties\":{\"owners\":{\"type\":\"keyword\"},\"viewers\":{\"type\":\"keyword\"}}},\"ancestry\":{\"properties\":{\"parents\":{\"type\":\"keyword\"}}},\"authority\":{\"type\":\"constant_keyword\",\"value\":\"opendes\"},\"createTime\":{\"type\":\"date\"},\"createUser\":{\"type\":\"keyword\"},\"data\":{\"properties\":{\"message\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"null_value\":\"null\",\"ignore_above\":256}}}}},\"id\":{\"type\":\"keyword\"},\"index\":{\"properties\":{\"lastUpdateTime\":{\"type\":\"date\"},\"statusCode\":{\"type\":\"integer\"},\"trace\":{\"type\":\"text\"}}},\"kind\":{\"type\":\"keyword\"},\"legal\":{\"properties\":{\"legaltags\":{\"type\":\"keyword\"},\"otherRelevantDataCountries\":{\"type\":\"keyword\"},\"status\":{\"type\":\"keyword\"}}},\"modifyTime\":{\"type\":\"date\"},\"modifyUser\":{\"type\":\"keyword\"},\"namespace\":{\"type\":\"keyword\"},\"source\":{\"type\":\"constant_keyword\",\"value\":\"test\"},\"tags\":{\"type\":\"flattened\"},\"type\":{\"type\":\"keyword\"},\"version\":{\"type\":\"long\"},\"x-acl\":{\"type\":\"keyword\"}}}";
when(this.mappingService.getIndexMapping(restHighLevelClient, index)).thenReturn(mapping);
- this.sut.syncIndexMappingIfRequired(restHighLevelClient, index);
+ this.sut.syncIndexMappingIfRequired(restHighLevelClient, indexSchema);
verify(this.indexCache, times(1)).get(cacheKey);
verify(this.indexCache, times(1)).put(cacheKey, true);
@@ -204,7 +209,7 @@ public class IndexerMappingServiceTest {
doReturn(this.indicesClient).when(this.restHighLevelClient).indices();
doReturn(mappingResponse).when(this.indicesClient).putMapping(any(PutMappingRequest.class), any(RequestOptions.class));
- this.sut.syncIndexMappingIfRequired(restHighLevelClient, index);
+ this.sut.syncIndexMappingIfRequired(restHighLevelClient, indexSchema);
verify(this.indexCache, times(1)).get(cacheKey);
verify(this.indexCache, times(1)).put(cacheKey, true);
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/SchemaProviderImplTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/SchemaProviderImplTest.java
index 9e6da1a355f2057eec76feaaa87ad47451a7ae52..3c40aa24ca8ffc38ed84c51275d67975092364ac 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/SchemaProviderImplTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/service/SchemaProviderImplTest.java
@@ -17,29 +17,41 @@ package org.opengroup.osdu.indexer.service;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.http.HttpStatus;
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.rest.RestStatus;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.*;
import org.opengroup.osdu.core.common.http.IUrlFetchService;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
+import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.HttpResponse;
+import org.opengroup.osdu.core.common.model.indexer.SchemaInfo;
import org.opengroup.osdu.core.common.provider.interfaces.IRequestInfo;
import org.opengroup.osdu.indexer.config.IndexerConfigurationProperties;
+import org.opengroup.osdu.indexer.config.SchemaEventsListenerConfiguration;
+import org.opengroup.osdu.indexer.logging.AuditLogger;
import org.opengroup.osdu.indexer.schema.converter.SchemaToStorageFormatImpl;
-import org.opengroup.osdu.indexer.service.StorageService;
+import org.opengroup.osdu.indexer.util.ElasticClientHandler;
import org.powermock.api.mockito.PowerMockito;
import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder;
import org.springframework.test.context.junit4.SpringRunner;
+import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URISyntaxException;
import java.util.Map;
+import static java.util.Collections.singletonList;
import static org.junit.Assert.*;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.*;
import static org.powermock.api.mockito.PowerMockito.when;
+import static org.springframework.http.HttpStatus.BAD_REQUEST;
+import static org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR;
@RunWith(SpringRunner.class)
public class SchemaProviderImplTest {
@@ -61,10 +73,25 @@ public class SchemaProviderImplTest {
@Mock
private IndexerConfigurationProperties configurationProperties;
-
+ @Mock
+ private ElasticClientHandler elasticClientHandler;
+ @Mock
+ private AuditLogger auditLogger;
+ @Mock
+ private IndexSchemaService indexSchemaService;
+ @Mock
+ private SchemaEventsListenerConfiguration schemaEventsListenerConfiguration;
@InjectMocks
private SchemaProviderImpl sut;
+ private RestHighLevelClient restClient;
+
+ @Before
+ public void setup() {
+ when(this.schemaEventsListenerConfiguration.isListenCreateEvent()).thenReturn(true);
+ when(this.schemaEventsListenerConfiguration.isListenUpdateEvent()).thenReturn(true);
+ }
+
@Test
public void test_empty_schema() throws UnsupportedEncodingException, URISyntaxException {
org.opengroup.osdu.core.common.model.http.HttpResponse httpResponse =
@@ -112,7 +139,8 @@ public class SchemaProviderImplTest {
String recordSchemaResponse = this.sut.getSchema(kind);
Map<String, Object> result = objectMapper.readValue(recordSchemaResponse,
- new TypeReference<Map<String,Object>>(){});
+ new TypeReference<Map<String, Object>>() {
+ });
assertEquals("Schema must have two root items", 2, result.size());
assertEquals("Wrong kind", "tenant:test:test:1.0.0", result.get("kind"));
assertEquals("Wrong schema attributes", "[{path=WellID, kind=link}]", result.get("schema").toString());
@@ -174,4 +202,61 @@ public class SchemaProviderImplTest {
verify(schemaService, times(0)).getFromStorageService(any());
}
+ @Test
+ public void should_process_validSchemaCreateEvent() throws IOException, URISyntaxException {
+ SchemaInfo event1 = new SchemaInfo("slb:indexer:test-data--SchemaEventIntegration:1.0.0", "create");
+ this.restClient = PowerMockito.mock(RestHighLevelClient.class);
+ when(elasticClientHandler.createRestClient()).thenReturn(restClient);
+
+ this.sut.processSchemaMessages(singletonList(event1));
+
+ verify(this.indexSchemaService, times(1)).processSchemaUpsertEvent(this.restClient, event1.getKind());
+ verify(this.auditLogger, times(1)).indexMappingUpsertSuccess(singletonList(event1.getKind()));
+ }
+
+ @Test
+ public void should_process_validSchemaUpdateEvent() throws IOException, URISyntaxException {
+ SchemaInfo event1 = new SchemaInfo("slb:indexer:test-data--SchemaEventIntegration:1.0.0", "update");
+ this.restClient = PowerMockito.mock(RestHighLevelClient.class);
+ when(elasticClientHandler.createRestClient()).thenReturn(restClient);
+
+ this.sut.processSchemaMessages(singletonList(event1));
+
+ verify(this.indexSchemaService, times(1)).processSchemaUpsertEvent(this.restClient, event1.getKind());
+ verify(this.auditLogger, times(1)).indexMappingUpsertSuccess(singletonList(event1.getKind()));
+ }
+
+ @Test
+ public void should_throwError_given_unsupportedEvent() {
+ SchemaInfo event1 = new SchemaInfo("slb:indexer:test-data--SchemaEventIntegration:1.0.0", "delete");
+
+ try {
+ this.sut.processSchemaMessages(singletonList(event1));
+ fail("Should throw exception");
+ } catch (AppException e) {
+ assertEquals(BAD_REQUEST.value(), e.getError().getCode());
+ assertEquals("Error parsing schema events in request payload.", e.getError().getMessage());
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void should_throwError_given_schemaUpsertFails() throws IOException, URISyntaxException {
+ SchemaInfo event1 = new SchemaInfo("slb:indexer:test-data--SchemaEventIntegration:1.0.0", "update");
+ this.restClient = PowerMockito.mock(RestHighLevelClient.class);
+ when(elasticClientHandler.createRestClient()).thenReturn(restClient);
+ doThrow(new ElasticsearchStatusException("unknown error", RestStatus.INTERNAL_SERVER_ERROR)).when(this.indexSchemaService).processSchemaUpsertEvent(any(RestHighLevelClient.class), anyString());
+
+ try {
+ this.sut.processSchemaMessages(singletonList(event1));
+ fail("Should throw exception");
+ } catch (AppException e) {
+ assertEquals(INTERNAL_SERVER_ERROR.value(), e.getError().getCode());
+ assertEquals("unknown error", e.getError().getMessage());
+ verify(this.auditLogger, times(1)).indexMappingUpsertFail(singletonList(event1.getKind()));
+ } catch (Exception e) {
+ fail("Should not throw this exception" + e.getMessage());
+ }
+ }
}
\ No newline at end of file
diff --git a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/TypeMapperTest.java b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/TypeMapperTest.java
index 6dbc2206dd4a482c3d93731ded079c0ececf8451..2768f5c93f7ac496517e62af1194de288fda815e 100644
--- a/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/TypeMapperTest.java
+++ b/indexer-core/src/test/java/org/opengroup/osdu/indexer/util/TypeMapperTest.java
@@ -2,12 +2,15 @@ package org.opengroup.osdu.indexer.util;
import org.junit.Test;
import org.opengroup.osdu.core.common.model.indexer.ElasticType;
+import org.opengroup.osdu.core.common.model.indexer.Records;
import org.opengroup.osdu.core.common.model.indexer.StorageType;
import org.opengroup.osdu.core.common.model.search.RecordMetaAttribute;
import java.util.List;
+import java.util.Map;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
public class TypeMapperTest {
@@ -34,8 +37,6 @@ public class TypeMapperTest {
assertEquals(ElasticType.LONG.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.VERSION));
assertEquals(ElasticType.KEYWORD.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.X_ACL));
- assertEquals(ElasticType.CONSTANT_KEYWORD.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.AUTHORITY));
- assertEquals(ElasticType.CONSTANT_KEYWORD.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.SOURCE));
assertEquals(ElasticType.KEYWORD.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.CREATE_USER));
assertEquals(ElasticType.KEYWORD.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.MODIFY_USER));
assertEquals(ElasticType.DATE.getValue(), TypeMapper.getIndexerType(RecordMetaAttribute.CREATE_TIME));
@@ -46,9 +47,21 @@ public class TypeMapperTest {
public void validate_meta_attributes() {
List<String> keys = TypeMapper.getMetaAttributesKeys();
- String[] meta = new String[] {"id", "kind", "authority", "source", "namespace", "type", "version", "acl", "tags", "legal", "ancestry", "createUser", "modifyUser", "createTime", "modifyTime", "index"};
- for(String attributeKey : meta) {
+ String[] meta = new String[]{"id", "kind", "authority", "source", "namespace", "type", "version", "acl", "tags", "legal", "ancestry", "createUser", "modifyUser", "createTime", "modifyTime", "index"};
+ for (String attributeKey : meta) {
assertTrue(keys.contains(attributeKey));
}
}
+
+ @Test
+ public void validate_constantAttribute_indexerMapping() {
+ Object value = TypeMapper.getMetaAttributeIndexerMapping(RecordMetaAttribute.ID.getValue(), "");
+ Records.Type recordsType = (Records.Type) value;
+ assertEquals("keyword", recordsType.getType());
+
+ value = TypeMapper.getMetaAttributeIndexerMapping(RecordMetaAttribute.AUTHORITY.getValue(), "opendes");
+ Map<String, Object> mapping = (Map<String, Object>) value;
+ assertEquals("constant_keyword", mapping.get("type"));
+ assertEquals("opendes", mapping.get("value"));
+ }
}
diff --git a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
index 81da502c69dc5dc20160417bd1ff1085f823b139..505180648b2233d5062d3a48c895efe898e51add 100644
--- a/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
+++ b/provider/indexer-reference/src/main/java/org/opengroup/osdu/indexer/service/IndexerServiceImpl.java
@@ -425,7 +425,7 @@ public class IndexerServiceImpl implements IndexerService {
// check if index exist and sync meta attribute schema if required
if (this.indicesService.isIndexExist(restClient, index)) {
- this.mappingService.syncIndexMappingIfRequired(restClient, index);
+ this.mappingService.syncIndexMappingIfRequired(restClient, schema);
continue;
}