diff --git a/NOTICE b/NOTICE
index 1886307c88c0fc0e719778dc7a7c0b47f92e21d4..ba7891fad64d315704b373871343c6a9acbd3352 100644
--- a/NOTICE
+++ b/NOTICE
@@ -363,7 +363,6 @@ The following software have components provided under the terms of this license:
- Asynchronous Http Client (from https://repo1.maven.org/maven2/org/asynchttpclient/async-http-client)
- Asynchronous Http Client Netty Utils (from https://repo1.maven.org/maven2/org/asynchttpclient/async-http-client-netty-utils)
- AutoValue Annotations (from https://github.com/google/auto/tree/master/value, https://repo1.maven.org/maven2/com/google/auto/value/auto-value-annotations)
-- AutoValue Processor (from https://github.com/google/auto/tree/master/value)
- BSON (from http://bsonspec.org, https://bsonspec.org)
- BSON Record Codec (from <https://www.mongodb.com/>, https://www.mongodb.com/)
- Bean Validation API (from http://beanvalidation.org)
@@ -373,8 +372,7 @@ The following software have components provided under the terms of this license:
- Byte Buddy (without dependencies) (from https://repo1.maven.org/maven2/net/bytebuddy/byte-buddy)
- Byte Buddy Java agent (from https://repo1.maven.org/maven2/net/bytebuddy/byte-buddy-agent)
- ClassMate (from http://github.com/cowtowncoder/java-classmate)
-- Cloud Key Management Service (KMS) API (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-cloudkms)
-- Cloud Storage JSON API v1-rev20230301-2.0.0 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-storage)
+- Cloud Key Management Service (KMS) API v1-rev20230407-2.0.0 (from https://repo1.maven.org/maven2/com/google/apis/google-api-services-cloudkms)
- CloudWatch Metrics for AWS Java SDK (from https://aws.amazon.com/sdkforjava)
- Cobertura (from http://cobertura.sourceforge.net)
- Cobertura Limited Runtime (from http://cobertura.sourceforge.net)
@@ -394,18 +392,12 @@ The following software have components provided under the terms of this license:
- Doxia Sitetools :: Site Renderer (from http://maven.apache.org/doxia/doxia-sitetools/doxia-site-renderer/, https://repo1.maven.org/maven2/org/apache/maven/doxia/doxia-site-renderer)
- Elastic JNA Distribution (from https://github.com/java-native-access/jna)
- FindBugs-jsr305 (from http://findbugs.sourceforge.net/)
-- GAX (Google Api eXtensions) for Java (HTTP JSON) (from <https://repo1.maven.org/maven2/com/google/api/gax-httpjson>, https://repo1.maven.org/maven2/com/google/api/gax-httpjson)
- GSON extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-gson)
- Google APIs Client Library for Java (from https://repo1.maven.org/maven2/com/google/api-client/google-api-client)
-- Google App Engine extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-appengine)
- Google Cloud Core (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-core, https://github.com/googleapis/java-core, https://repo1.maven.org/maven2/com/google/cloud/google-cloud-core)
-- Google Cloud Core HTTP (from https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-core-http, https://github.com/googleapis/google-cloud-java, https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-core-http, https://github.com/googleapis/java-core)
-- Google Cloud Core gRPC (from https://github.com/googleapis/google-cloud-java, https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-core-grpc, https://github.com/googleapis/java-core)
-- Google Cloud Datastore (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-datastore, https://github.com/googleapis/java-datastore)
-- Google Cloud IAM Service Account Credentials (from https://github.com/googleapis/google-cloud-java, https://github.com/googleapis/java-iamcredentials)
+- Google Cloud Core gRPC (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-core-grpc, https://github.com/googleapis/java-core, https://repo1.maven.org/maven2/com/google/cloud/google-cloud-core-grpc)
- Google Cloud Logging (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-logging, https://github.com/googleapis/java-logging)
- Google Cloud Pub/Sub (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-pubsub, https://github.com/googleapis/java-pubsub)
-- Google Cloud Storage (from https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-storage, https://github.com/googleapis/java-storage)
- Google HTTP Client Library for Java (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client)
- Google OAuth Client Library for Java (from https://repo1.maven.org/maven2/com/google/oauth-client/google-oauth-client)
- Gson (from http://code.google.com/p/google-gson/, https://repo1.maven.org/maven2/com/google/code/gson/gson)
@@ -416,7 +408,6 @@ The following software have components provided under the terms of this license:
- HdrHistogram (from http://hdrhistogram.github.io/HdrHistogram/)
- Hibernate Validator (from https://repo1.maven.org/maven2/org/hibernate/hibernate-validator, https://repo1.maven.org/maven2/org/hibernate/validator/hibernate-validator)
- High Performance Primitive Collections (from https://github.com/carrotsearch/hppc)
-- HikariCP (from https://github.com/brettwooldridge/HikariCP)
- Hop (from https://github.com/rabbitmq/hop, https://www.rabbitmq.com)
- IBM COS Java SDK for Amazon S3 (from https://github.com/ibm/ibm-cos-sdk-java)
- IBM COS Java SDK for COS KMS (from https://github.com/ibm/ibm-cos-sdk-java)
@@ -436,6 +427,7 @@ The following software have components provided under the terms of this license:
- JSON.simple (from http://code.google.com/p/json-simple/)
- JSONassert (from http://github.com/skyscreamer/yoga, https://github.com/skyscreamer/JSONassert)
- JSR107 API and SPI (from https://github.com/jsr107/jsr107spec)
+- Jackson 2 extensions to the Google APIs Client Library for Java (from https://repo1.maven.org/maven2/com/google/api-client/google-api-client-jackson2)
- Jackson 2 extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-jackson2)
- Jackson dataformat: CBOR (from http://github.com/FasterXML/jackson-dataformats-binary)
- Jackson dataformat: Smile (from http://github.com/FasterXML/jackson-dataformat-smile, http://github.com/FasterXML/jackson-dataformats-binary)
@@ -510,7 +502,6 @@ The following software have components provided under the terms of this license:
- Mojo's Maven plugin for Cobertura (from http://mojo.codehaus.org/cobertura-maven-plugin/)
- MongoDB Driver (from https://www.mongodb.com/)
- MongoDB Java Driver (from http://mongodb.org/, http://www.mongodb.org, https://www.mongodb.com/)
-- NanoHttpd-Core (from https://repo1.maven.org/maven2/org/nanohttpd/nanohttpd)
- Netty Reactive Streams Implementation (from https://repo1.maven.org/maven2/com/typesafe/netty/netty-reactive-streams)
- Netty/Buffer (from https://repo1.maven.org/maven2/io/netty/netty-buffer)
- Netty/Codec (from https://repo1.maven.org/maven2/io/netty/netty-codec)
@@ -525,7 +516,6 @@ The following software have components provided under the terms of this license:
- Netty/Resolver/DNS (from https://repo1.maven.org/maven2/io/netty/netty-resolver-dns)
- Netty/Resolver/DNS/Classes/MacOS (from https://repo1.maven.org/maven2/io/netty/netty-resolver-dns-classes-macos)
- Netty/TomcatNative [BoringSSL - Static] (from https://github.com/netty/netty-tcnative/netty-tcnative-boringssl-static/)
-- Netty/TomcatNative [OpenSSL - Classes] (from https://repo1.maven.org/maven2/io/netty/netty-tcnative-classes)
- Netty/Transport (from https://repo1.maven.org/maven2/io/netty/netty-transport)
- Netty/Transport/Classes/Epoll (from https://repo1.maven.org/maven2/io/netty/netty-transport-classes-epoll)
- Netty/Transport/Classes/KQueue (from https://repo1.maven.org/maven2/io/netty/netty-transport-classes-kqueue)
@@ -541,19 +531,17 @@ The following software have components provided under the terms of this license:
- OkHttp Logging Interceptor (from https://github.com/square/okhttp, https://repo1.maven.org/maven2/com/squareup/okhttp3/logging-interceptor, https://square.github.io/okhttp/)
- OkHttp URLConnection (from https://repo1.maven.org/maven2/com/squareup/okhttp3/okhttp-urlconnection, https://square.github.io/okhttp/)
- Okio (from https://github.com/square/okio/, https://repo1.maven.org/maven2/com/squareup/okio/okio)
-- OpenCensus (from https://github.com/census-instrumentation/opencensus-java, https://github.com/census-instrumentation/opencensus-proto)
+- OpenCensus (from https://github.com/census-instrumentation/opencensus-java)
- PWDB :: Database (from https://repo1.maven.org/maven2/org/linguafranca/pwdb/database)
- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils, https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-utils)
- Plexus I18N Component (from https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-i18n)
- Plexus Velocity Component (from https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-velocity)
-- PostgreSQL JDBC Driver
- PowerMock (from http://www.powermock.org, https://repo1.maven.org/maven2/org/powermock/powermock-api-mockito)
- Prometheus Java Simpleclient (from <https://repo1.maven.org/maven2/io/prometheus/simpleclient>, https://repo1.maven.org/maven2/io/prometheus/simpleclient)
- Prometheus Java Simpleclient Common (from <https://repo1.maven.org/maven2/io/prometheus/simpleclient_common>, https://repo1.maven.org/maven2/io/prometheus/simpleclient_common)
- Prometheus Java Span Context Supplier - Common (from <https://repo1.maven.org/maven2/io/prometheus/simpleclient_tracer_common>, https://repo1.maven.org/maven2/io/prometheus/simpleclient_tracer_common)
- Prometheus Java Span Context Supplier - OpenTelemetry (from <https://repo1.maven.org/maven2/io/prometheus/simpleclient_tracer_otel>, https://repo1.maven.org/maven2/io/prometheus/simpleclient_tracer_otel)
- Prometheus Java Span Context Supplier - OpenTelemetry Agent (from <https://repo1.maven.org/maven2/io/prometheus/simpleclient_tracer_otel_agent>, https://repo1.maven.org/maven2/io/prometheus/simpleclient_tracer_otel_agent)
-- Protocol Buffer extensions to the Google HTTP Client Library for Java. (from https://repo1.maven.org/maven2/com/google/http-client/google-http-client-protobuf)
- Proton-J (from https://repo1.maven.org/maven2/org/apache/qpid/proton-j)
- QpidJMS Client (from https://repo1.maven.org/maven2/org/apache/qpid/qpid-jms-client)
- RabbitMQ Java Client (from http://www.rabbitmq.com, https://www.rabbitmq.com)
@@ -561,8 +549,8 @@ The following software have components provided under the terms of this license:
- Redisson (from http://redisson.org)
- Retrofit (from https://github.com/square/retrofit, https://repo1.maven.org/maven2/com/squareup/retrofit2/retrofit)
- RxJava (from https://github.com/ReactiveX/RxJava)
+- Servlet API (from https://repo1.maven.org/maven2/org/mortbay/jetty/servlet-api)
- Servlet Specification 2.5 API (from http://jetty.mortbay.org, https://repo1.maven.org/maven2/org/mortbay/jetty/servlet-api-2.5)
-- Simple XML (safe) (from https://github.com/dweiss/simplexml)
- SnakeYAML (from http://code.google.com/p/snakeyaml/, http://www.snakeyaml.org, https://bitbucket.org/snakeyaml/snakeyaml)
- Spatial4J (from https://projects.eclipse.org/projects/locationtech.spatial4j)
- Spring AOP (from http://www.springframework.org, https://github.com/spring-projects/spring-framework, https://repo1.maven.org/maven2/org/springframework/spring-aop)
@@ -575,7 +563,6 @@ The following software have components provided under the terms of this license:
- Spring Boot AutoConfigure (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-autoconfigure, https://spring.io/projects/spring-boot)
- Spring Boot Configuration Processor (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-tools/spring-boot-configuration-processor, https://spring.io/projects/spring-boot)
- Spring Boot Dependencies (from http://projects.spring.io/spring-boot/, https://spring.io/projects/spring-boot)
-- Spring Boot JDBC Starter (from http://projects.spring.io/spring-boot/, https://spring.io/projects/spring-boot)
- Spring Boot Jersey Starter (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-jersey, https://spring.io/projects/spring-boot)
- Spring Boot Json Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-json, https://spring.io/projects/spring-boot)
- Spring Boot Log4j 2 Starter (from http://projects.spring.io/spring-boot/, https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-log4j2, https://spring.io/projects/spring-boot)
@@ -596,7 +583,6 @@ The following software have components provided under the terms of this license:
- Spring Data Core (from https://spring.io/projects/spring-data)
- Spring Data MongoDB - Core (from https://repo1.maven.org/maven2/org/springframework/data/spring-data-mongodb)
- Spring Expression Language (SpEL) (from https://github.com/SpringSource/spring-framework, https://github.com/spring-projects/spring-framework, https://repo1.maven.org/maven2/org/springframework/spring-expression)
-- Spring JDBC (from https://github.com/SpringSource/spring-framework, https://github.com/spring-projects/spring-framework, https://repo1.maven.org/maven2/org/springframework/spring-jdbc)
- Spring JMS (from http://www.springframework.org, https://github.com/SpringSource/spring-framework, https://github.com/spring-projects/spring-framework, https://repo1.maven.org/maven2/org/springframework/spring-jms)
- Spring Messaging (from https://github.com/spring-projects/spring-framework)
- Spring Plugin - Metadata Extension (from https://repo1.maven.org/maven2/org/springframework/plugin/spring-plugin-metadata)
@@ -621,7 +607,6 @@ The following software have components provided under the terms of this license:
- aalto-xml (from https://github.com/FasterXML/aalto-xml, https://repo1.maven.org/maven2/com/fasterxml/aalto-xml)
- aggs-matrix-stats (from https://github.com/elastic/elasticsearch, https://github.com/elastic/elasticsearch.git)
- compiler (from http://github.com/spullara/mustache.java)
-- datastore-v1-proto-client (from https://repo1.maven.org/maven2/com/google/cloud/datastore/datastore-v1-proto-client)
- documentdb-bulkexecutor (from http://azure.microsoft.com/en-us/services/documentdb/)
- elasticsearch (from https://github.com/elastic/elasticsearch, https://github.com/elastic/elasticsearch.git, https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch)
- elasticsearch-cli (from https://github.com/elastic/elasticsearch, https://github.com/elastic/elasticsearch.git)
@@ -630,20 +615,16 @@ The following software have components provided under the terms of this license:
- elasticsearch-secure-sm (from https://github.com/elastic/elasticsearch, https://github.com/elastic/elasticsearch.git)
- elasticsearch-x-content (from https://github.com/elastic/elasticsearch, https://github.com/elastic/elasticsearch.git)
- error-prone annotations (from https://repo1.maven.org/maven2/com/google/errorprone/error_prone_annotations)
-- grpc-google-cloud-datastore-admin-v1 (from https://github.com/googleapis/java-datastore/grpc-google-cloud-datastore-admin-v1)
- io.grpc:grpc-alts (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-api (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-auth (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-context (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-core (from https://github.com/grpc/grpc-java)
-- io.grpc:grpc-googleapis (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-grpclb (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-netty-shaded (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-protobuf (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-protobuf-lite (from https://github.com/grpc/grpc-java)
-- io.grpc:grpc-services (from https://github.com/grpc/grpc-java)
- io.grpc:grpc-stub (from https://github.com/grpc/grpc-java)
-- io.grpc:grpc-xds (from https://github.com/grpc/grpc-java)
- ion-java (from https://github.com/amzn/ion-java/, https://github.com/amznlabs/ion-java/)
- jackson-databind (from http://github.com/FasterXML/jackson, http://wiki.fasterxml.com/JacksonHome, https://github.com/FasterXML/jackson)
- jakarta.inject (from https://repo1.maven.org/maven2/org/glassfish/hk2/external/jakarta.inject)
@@ -663,16 +644,12 @@ The following software have components provided under the terms of this license:
- micrometer-core (from https://github.com/micrometer-metrics/micrometer)
- micrometer-registry-azure-monitor (from https://github.com/micrometer-metrics/micrometer)
- micrometer-registry-prometheus (from <https://github.com/micrometer-metrics/micrometer>, https://github.com/micrometer-metrics/micrometer)
-- minio (from https://github.com/minio/minio-java)
- org.apiguardian:apiguardian-api (from https://github.com/apiguardian-team/apiguardian)
- org.conscrypt:conscrypt-openjdk-uber (from https://conscrypt.org/)
- org.opentest4j:opentest4j (from https://github.com/ota4j-team/opentest4j)
- org.xmlunit:xmlunit-core (from http://www.xmlunit.org/, https://www.xmlunit.org/)
- parent-join (from https://github.com/elastic/elasticsearch, https://github.com/elastic/elasticsearch.git)
- perfmark:perfmark-api (from https://github.com/perfmark/perfmark)
-- proto-google-cloud-datastore-admin-v1 (from https://github.com/googleapis/java-datastore/proto-google-cloud-datastore-admin-v1)
-- proto-google-cloud-datastore-v1 (from https://github.com/googleapis/googleapis, https://github.com/googleapis/java-datastore/proto-google-cloud-datastore-v1)
-- proto-google-cloud-iamcredentials-v1 (from https://github.com/googleapis/google-cloud-java, https://github.com/googleapis/java-iamcredentials/proto-google-cloud-iamcredentials-v1, https://repo1.maven.org/maven2/com/google/api/grpc/proto-google-cloud-iamcredentials-v1)
- proto-google-cloud-logging-v2 (from https://github.com/googleapis/java-logging/proto-google-cloud-logging-v2, https://repo1.maven.org/maven2/com/google/api/grpc/proto-google-cloud-logging-v2)
- proto-google-cloud-pubsub-v1 (from https://github.com/googleapis/googleapis, https://github.com/googleapis/java-pubsub/proto-google-cloud-pubsub-v1)
- proto-google-common-protos (from https://github.com/googleapis/api-client-staging, https://github.com/googleapis/gapic-generator-java, https://github.com/googleapis/googleapis, https://github.com/googleapis/java-iam/proto-google-common-protos)
@@ -726,7 +703,6 @@ The following software have components provided under the terms of this license:
- Lucene Sandbox (from https://lucene.apache.org/, https://repo1.maven.org/maven2/org/apache/lucene/lucene-sandbox)
- Lucene Spatial 3D (from https://lucene.apache.org/, https://repo1.maven.org/maven2/org/apache/lucene/lucene-spatial3d)
- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils, https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-utils)
-- PostgreSQL JDBC Driver
- Stax2 API (from http://github.com/FasterXML/stax2-api)
- jaxen (from http://jaxen.codehaus.org/, https://repo1.maven.org/maven2/jaxen/jaxen)
- jersey-core-server (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-server)
@@ -749,7 +725,6 @@ The following software have components provided under the terms of this license:
- AspectJ Weaver (from http://www.aspectj.org, https://www.eclipse.org/aspectj/)
- Class Model for Hk2 (from https://repo1.maven.org/maven2/org/glassfish/hk2/class-model)
- GAX (Google Api eXtensions) for Java (Core) (from https://github.com/googleapis, https://github.com/googleapis/gax-java, https://repo1.maven.org/maven2/com/google/api/gax)
-- GAX (Google Api eXtensions) for Java (HTTP JSON) (from <https://repo1.maven.org/maven2/com/google/api/gax-httpjson>, https://repo1.maven.org/maven2/com/google/api/gax-httpjson)
- GAX (Google Api eXtensions) for Java (gRPC) (from <https://repo1.maven.org/maven2/com/google/api/gax-grpc>, https://repo1.maven.org/maven2/com/google/api/gax-grpc)
- Google APIs Client Library for Java (from https://repo1.maven.org/maven2/com/google/api-client/google-api-client)
- Google Auth Library for Java - Credentials (from https://repo1.maven.org/maven2/com/google/auth/google-auth-library-credentials)
@@ -779,13 +754,10 @@ The following software have components provided under the terms of this license:
- Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
- Mockito (from http://mockito.org, http://www.mockito.org, https://github.com/mockito/mockito)
-- NanoHttpd-Core (from https://repo1.maven.org/maven2/org/nanohttpd/nanohttpd)
- Netty/Codec/HTTP (from https://repo1.maven.org/maven2/io/netty/netty-codec-http)
- Plexus Common Utilities (from http://plexus.codehaus.org/plexus-utils, https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-utils)
-- PostgreSQL JDBC Driver
- Protocol Buffer Java API (from http://code.google.com/p/protobuf, https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java)
- Protocol Buffers [Util] (from https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java-util)
-- RE2/J (from http://github.com/google/re2j)
- Redisson (from http://redisson.org)
- ServiceLocator Default Implementation (from https://repo1.maven.org/maven2/org/glassfish/hk2/hk2-locator)
- Spring Core (from http://www.springframework.org, https://github.com/spring-projects/spring-framework, https://repo1.maven.org/maven2/org/springframework/spring-core)
@@ -921,6 +893,7 @@ The following software have components provided under the terms of this license:
- Jakarta Annotations API (from https://projects.eclipse.org/projects/ee4j.ca)
- Jakarta Bean Validation API (from https://beanvalidation.org)
- Jakarta XML Binding API (from https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api, https://repo1.maven.org/maven2/org/jboss/spec/javax/xml/bind/jboss-jaxb-api_2.3_spec)
+- Jetty :: Utilities (from http://jetty.mortbay.org, http://www.eclipse.org/jetty, https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-util, https://repo1.maven.org/maven2/org/mortbay/jetty/jetty-util)
- Logback Contrib :: JSON :: Classic (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-json-classic)
- Logback Contrib :: JSON :: Core (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-json-core)
- Logback Contrib :: Jackson (from https://repo1.maven.org/maven2/ch/qos/logback/contrib/logback-jackson)
@@ -1339,7 +1312,6 @@ The following software have components provided under the terms of this license:
- JSON in Java (from https://github.com/douglascrockford/JSON-java)
- LatencyUtils (from http://latencyutils.github.io/LatencyUtils/)
- Microsoft Azure client library for Blob Storage (from https://github.com/Azure/azure-sdk-for-java)
-- PostgreSQL JDBC Driver
- jersey-core-common (from https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-common)
========================================================================
diff --git a/README.md b/README.md
index 0b2fc26ecb867f1ebb0c569a5f4f4c948b3a7471..51aa2465b3294c3a40c34d25ccf93ae614f92228 100644
--- a/README.md
+++ b/README.md
@@ -14,3 +14,10 @@ All documentation for the Google Cloud implementation of `os-indexer` lives [her
## AWS Implementation
All documentation for the AWS implementation of `os-indexer` lives [here](./provider/indexer-aws/README.md)
+
+## Open API 3.0 - Swagger
+- Swagger UI : https://host/context-path/swagger (will redirect to https://host/context-path/swagger-ui/index.html)
+- api-docs (JSON) : https://host/context-path/api-docs
+- api-docs (YAML) : https://host/context-path/api-docs.yaml
+
+All the Swagger and OpenAPI related common properties are managed here [swagger.properties](./indexer-core/src/main/resources/swagger.properties)
diff --git a/devops/gc/deploy/templates/authorization-policy.yaml b/devops/gc/deploy/templates/authorization-policy.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..f694999500c9a2e3318e61a0f633f8d13d56c11a
--- /dev/null
+++ b/devops/gc/deploy/templates/authorization-policy.yaml
@@ -0,0 +1,35 @@
+# FIXME: remove it when migrate to istio 1.6 and use directResponse instead
+apiVersion: security.istio.io/v1beta1
+kind: AuthorizationPolicy
+metadata:
+ name: {{ printf "%s-allow-policy" .Values.conf.appName | quote }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ selector:
+ matchLabels:
+ app: {{ .Values.conf.appName | quote }}
+ action: ALLOW
+ rules:
+ - to:
+ - operation:
+ paths:
+ - /api/indexer/v2/*
+---
+apiVersion: security.istio.io/v1beta1
+kind: AuthorizationPolicy
+metadata:
+ name: {{ printf "%s-deny-policy" .Values.conf.appName | quote }}
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ selector:
+ matchLabels:
+ app: {{ .Values.conf.appName | quote }}
+ action: DENY
+ rules:
+ - from:
+ - source:
+ notNamespaces: [ {{ .Release.Namespace | quote }} ]
+ to:
+ - operation:
+ paths:
+ - /api/indexer/v2/_dps/*
diff --git a/devops/gc/deploy/templates/virtual-service.yaml b/devops/gc/deploy/templates/virtual-service.yaml
index 8903b299b9577d298288734003857dcfe5877709..0f882c05077190c1a18e59f6338eadada4354e42 100644
--- a/devops/gc/deploy/templates/virtual-service.yaml
+++ b/devops/gc/deploy/templates/virtual-service.yaml
@@ -15,6 +15,14 @@ spec:
gateways:
- service-gateway
http:
+ # FIXME: use it when migrate to istio 1.6
+ # - match:
+ # - uri:
+ # prefix: "/api/indexer/v2/_dps"
+ # directResponse:
+ # status: 403
+ # body:
+ # string: "Forbidden"
- match:
- uri:
prefix: "/api/indexer/v2"
diff --git a/indexer-core/pom.xml b/indexer-core/pom.xml
index 3fd6fa05d20e9098b1dc904022513d8cb93ab994..b888f32f7dc8a12d46e2ff07887954dbb1da7af5 100644
--- a/indexer-core/pom.xml
+++ b/indexer-core/pom.xml
@@ -19,7 +19,7 @@
<gson.version>2.9.1</gson.version>
<netty.version>4.1.70.Final</netty.version>
<spring-webmvc.version>5.3.22</spring-webmvc.version>
- <os-core-common.version>0.19.0</os-core-common.version>
+ <os-core-common.version>0.21.0-rc4</os-core-common.version>
</properties>
<dependencyManagement>
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/IndexerApplication.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/IndexerApplication.java
index 416bf89dd51c5e78838ea5f6fcc86166c429d4a5..56be9f76d16aba9019957c6073c6bec470ae6420 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/IndexerApplication.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/IndexerApplication.java
@@ -7,10 +7,12 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.PropertySource;
@Configuration
@ComponentScan({"org.opengroup.osdu.core.common","org.opengroup.osdu.indexer", "org.opengroup.osdu.is"})
@SpringBootApplication(exclude = {ElasticSearchRestHealthContributorAutoConfiguration.class, SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
+@PropertySource("classpath:swagger.properties")
public class IndexerApplication {
public static void main( String[] args )
{
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/CleanupIndiciesApi.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/CleanupIndiciesApi.java
index a1f427d8e0396531d964437cadcf48654afe2c5b..9160b175a36154e5e8b32b7ed58fbce931e8892f 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/CleanupIndiciesApi.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/CleanupIndiciesApi.java
@@ -17,16 +17,16 @@ package org.opengroup.osdu.indexer.api;
import com.google.common.reflect.TypeToken;
import com.google.gson.Gson;
import com.google.gson.JsonParseException;
-import java.lang.reflect.Type;
-import java.util.List;
-import java.util.stream.Collectors;
-import javax.inject.Inject;
-import javax.validation.Valid;
-import javax.validation.constraints.NotBlank;
-import javax.validation.constraints.NotNull;
-
import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.Parameter;
+import io.swagger.v3.oas.annotations.media.Content;
+import io.swagger.v3.oas.annotations.media.Schema;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.responses.ApiResponses;
+import io.swagger.v3.oas.annotations.security.SecurityRequirement;
+import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.extern.java.Log;
+import org.opengroup.osdu.core.common.model.http.AppError;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
@@ -49,11 +49,21 @@ import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.context.annotation.RequestScope;
+
+import javax.inject.Inject;
+import javax.validation.Valid;
+import javax.validation.constraints.NotBlank;
+import javax.validation.constraints.NotNull;
+import java.lang.reflect.Type;
+import java.util.List;
+import java.util.stream.Collectors;
+
import static java.util.Collections.singletonList;
@Log
@RestController
@RequestScope
+@Tag(name = "cleanup-indicies-api", description = "Cleanup Indicies API")
public class CleanupIndiciesApi {
@Autowired
@@ -108,9 +118,22 @@ public class CleanupIndiciesApi {
}
}
+ @Operation(summary = "${cleanupIndiciesApi.deleteIndex.summary}", description = "${cleanupIndiciesApi.deleteIndex.description}",
+ security = {@SecurityRequirement(name = "Authorization")}, tags = { "cleanup-indicies-api" })
+ @ApiResponses(value = {
+ @ApiResponse(responseCode = "200", description = "OK"),
+ @ApiResponse(responseCode = "400", description = "Bad Request", content = {@Content(schema = @Schema(implementation = AppError.class))}),
+ @ApiResponse(responseCode = "401", description = "Unauthorized", content = {@Content(schema = @Schema(implementation = AppError.class))}),
+ @ApiResponse(responseCode = "403", description = "User not authorized to perform the action", content = {@Content(schema = @Schema(implementation = AppError.class))}),
+ @ApiResponse(responseCode = "404", description = "Not Found", content = {@Content(schema = @Schema(implementation = AppError.class))}),
+ @ApiResponse(responseCode = "500", description = "Internal Server Error", content = {@Content(schema = @Schema(implementation = AppError.class))}),
+ @ApiResponse(responseCode = "502", description = "Bad Gateway", content = {@Content(schema = @Schema(implementation = AppError.class))}),
+ @ApiResponse(responseCode = "503", description = "Service Unavailable", content = {@Content(schema = @Schema(implementation = AppError.class))})
+ })
@DeleteMapping(value = "/index", produces = MediaType.APPLICATION_JSON_VALUE)
@PreAuthorize("@authorizationFilter.hasPermission('" + ENTITLEMENT_GROUP + "')")
- public ResponseEntity deleteIndex(@RequestParam("kind") @NotBlank @ValidKind String kind) {
+ public ResponseEntity deleteIndex(@Parameter(description = "Kind", example = "tenant1:public:well:1.0.2")
+ @RequestParam("kind") @NotBlank @ValidKind String kind) {
String index = elasticIndexNameResolver.getIndexNameFromKind(kind);
try {
boolean responseStatus = indicesService.deleteIndex(index);
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/HealthCheckApi.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/HealthCheckApi.java
index d19ac9d042d9727b901d0148b34ea4a0dc3289e4..5bb8e0ed8d7709c0a145ee71dd1870653f73b284 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/HealthCheckApi.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/HealthCheckApi.java
@@ -14,6 +14,12 @@
package org.opengroup.osdu.indexer.api;
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.media.Content;
+import io.swagger.v3.oas.annotations.media.Schema;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.responses.ApiResponses;
+import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.extern.java.Log;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
@@ -28,18 +34,29 @@ import javax.annotation.security.PermitAll;
@RestController
@RequestMapping("/")
@RequestScope
+@Tag(name = "health-check-api", description = "Health Check API")
public class HealthCheckApi {
+ @Operation(summary = "${healthCheckApi.livenessCheck.summary}",
+ description = "${healthCheckApi.livenessCheck.description}", tags = { "health-check-api" })
+ @ApiResponses(value = {
+ @ApiResponse(responseCode = "200", description = "OK", content = { @Content(schema = @Schema(implementation = String.class)) })
+ })
@PermitAll
@GetMapping("/liveness_check")
public ResponseEntity<String> livenessCheck(){
return new ResponseEntity<String>("Indexer service is alive", HttpStatus.OK);
}
+ @Operation(summary = "${healthCheckApi.readinessCheck.summary}",
+ description = "${healthCheckApi.readinessCheck.description}", tags = { "health-check-api" })
+ @ApiResponses(value = {
+ @ApiResponse(responseCode = "200", description = "OK", content = { @Content(schema = @Schema(implementation = String.class)) })
+ })
@PermitAll
@GetMapping("/readiness_check")
public ResponseEntity<String> readinessCheck() {
return new ResponseEntity<String>("Indexer service is ready", HttpStatus.OK);
}
-}
\ No newline at end of file
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/InfoApi.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/InfoApi.java
index c9e7b6b15b62775df9cca6ecbdb26cbf63f8a58a..dc33b57dd92114ec84842e31038a4a1d1251f264 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/InfoApi.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/InfoApi.java
@@ -18,6 +18,13 @@
package org.opengroup.osdu.indexer.api;
import java.io.IOException;
+
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.media.Content;
+import io.swagger.v3.oas.annotations.media.Schema;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.responses.ApiResponses;
+import io.swagger.v3.oas.annotations.tags.Tag;
import org.opengroup.osdu.core.common.info.VersionInfoBuilder;
import org.opengroup.osdu.core.common.model.info.VersionInfo;
import org.springframework.beans.factory.annotation.Autowired;
@@ -28,13 +35,18 @@ import org.springframework.web.bind.annotation.RestController;
@RestController
@RequestMapping
+@Tag(name = "info", description = "Version info endpoint")
public class InfoApi {
@Autowired
private VersionInfoBuilder versionInfoBuilder;
+ @Operation(summary = "${infoApi.info.summary}", description = "${infoApi.info.description}", tags = { "info" })
+ @ApiResponses(value = {
+ @ApiResponse(responseCode = "200", description = "Version info.", content = { @Content(schema = @Schema(implementation = VersionInfo.class)) })
+ })
@GetMapping(value = "/info", produces = MediaType.APPLICATION_JSON_VALUE)
public VersionInfo info() throws IOException {
return versionInfoBuilder.buildVersionInfo();
}
-}
\ No newline at end of file
+}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/PartitionSetupApi.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/PartitionSetupApi.java
index 26435bfa27a8f90ff6d66d4721088b916a216f4c..56566ed015536ad821fe7328215f8e3f7ed98b24 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/PartitionSetupApi.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/PartitionSetupApi.java
@@ -14,6 +14,14 @@
package org.opengroup.osdu.indexer.api;
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.media.Content;
+import io.swagger.v3.oas.annotations.media.Schema;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.responses.ApiResponses;
+import io.swagger.v3.oas.annotations.security.SecurityRequirement;
+import io.swagger.v3.oas.annotations.tags.Tag;
+import org.opengroup.osdu.core.common.model.http.AppError;
import org.opengroup.osdu.indexer.logging.AuditLogger;
import org.opengroup.osdu.indexer.service.IClusterConfigurationService;
import org.springframework.beans.factory.annotation.Autowired;
@@ -33,6 +41,7 @@ import static org.opengroup.osdu.core.common.model.http.DpsHeaders.DATA_PARTITIO
@RestController
@RequestMapping("/partitions")
@RequestScope
+@Tag(name = "partition-setup-api", description = "Partition Setup API")
public class PartitionSetupApi {
private static final String OPS = "users.datalake.ops";
@@ -42,6 +51,18 @@ public class PartitionSetupApi {
@Autowired
private AuditLogger auditLogger;
+ @Operation(summary = "${partitionSetupApi.provisionPartition.summary}", description = "${partitionSetupApi.provisionPartition.description}",
+ security = {@SecurityRequirement(name = "Authorization")}, tags = { "partition-setup-api" })
+ @ApiResponses(value = {
+ @ApiResponse(responseCode = "200", description = "OK"),
+ @ApiResponse(responseCode = "400", description = "Bad Request", content = {@Content(schema = @Schema(implementation = AppError.class))}),
+ @ApiResponse(responseCode = "401", description = "Unauthorized", content = {@Content(schema = @Schema(implementation = AppError.class))}),
+ @ApiResponse(responseCode = "403", description = "User not authorized to perform the action", content = {@Content(schema = @Schema(implementation = AppError.class))}),
+ @ApiResponse(responseCode = "404", description = "Not Found", content = {@Content(schema = @Schema(implementation = AppError.class))}),
+ @ApiResponse(responseCode = "500", description = "Internal Server Error", content = {@Content(schema = @Schema(implementation = AppError.class))}),
+ @ApiResponse(responseCode = "502", description = "Bad Gateway", content = {@Content(schema = @Schema(implementation = AppError.class))}),
+ @ApiResponse(responseCode = "503", description = "Service Unavailable", content = {@Content(schema = @Schema(implementation = AppError.class))})
+ })
@PreAuthorize("@authorizationFilter.hasPermission('" + OPS + "')")
@PutMapping(path = "/provision", consumes = "application/json")
public ResponseEntity<?> provisionPartition(@RequestHeader(DATA_PARTITION_ID) String dataPartitionId) throws IOException {
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java
index a4f38e31d144498d1b4bdb010df91d8a3cf22f77..8d719c247bb1a6aa8fcc8f24877703a31aa743b8 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/api/ReindexApi.java
@@ -14,21 +14,33 @@
package org.opengroup.osdu.indexer.api;
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.Parameter;
+import io.swagger.v3.oas.annotations.media.Content;
+import io.swagger.v3.oas.annotations.media.Schema;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.responses.ApiResponses;
+import io.swagger.v3.oas.annotations.security.SecurityRequirement;
+import io.swagger.v3.oas.annotations.tags.Tag;
+import org.opengroup.osdu.core.common.model.http.AppError;
+import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
import org.opengroup.osdu.core.common.model.search.SearchServiceRole;
import org.opengroup.osdu.indexer.logging.AuditLogger;
-import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
import org.opengroup.osdu.indexer.service.IndexSchemaService;
import org.opengroup.osdu.indexer.service.ReindexService;
import org.springframework.http.ResponseEntity;
import org.springframework.security.access.prepost.PreAuthorize;
-
-import org.springframework.web.bind.annotation.*;
+import org.springframework.web.bind.annotation.PatchMapping;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.context.annotation.RequestScope;
import javax.inject.Inject;
import javax.validation.Valid;
import javax.validation.constraints.NotNull;
-
import java.io.IOException;
import static java.util.Collections.singletonList;
@@ -36,6 +48,7 @@ import static java.util.Collections.singletonList;
@RestController
@RequestMapping("/reindex")
@RequestScope
+@Tag(name = "reindex-api", description = "Reindex API")
public class ReindexApi {
@Inject
@@ -45,19 +58,44 @@ public class ReindexApi {
@Inject
private AuditLogger auditLogger;
+ @Operation(summary = "${reindexApi.reindex.summary}", description = "${reindexApi.reindex.description}",
+ security = {@SecurityRequirement(name = "Authorization")}, tags = { "reindex-api" })
+ @ApiResponses(value = {
+ @ApiResponse(responseCode = "200", description = "OK"),
+ @ApiResponse(responseCode = "400", description = "Bad Request", content = {@Content(schema = @Schema(implementation = AppError.class))}),
+ @ApiResponse(responseCode = "401", description = "Unauthorized", content = {@Content(schema = @Schema(implementation = AppError.class))}),
+ @ApiResponse(responseCode = "403", description = "User not authorized to perform the action", content = {@Content(schema = @Schema(implementation = AppError.class))}),
+ @ApiResponse(responseCode = "404", description = "Not Found", content = {@Content(schema = @Schema(implementation = AppError.class))}),
+ @ApiResponse(responseCode = "500", description = "Internal Server Error", content = {@Content(schema = @Schema(implementation = AppError.class))}),
+ @ApiResponse(responseCode = "502", description = "Bad Gateway", content = {@Content(schema = @Schema(implementation = AppError.class))}),
+ @ApiResponse(responseCode = "503", description = "Service Unavailable", content = {@Content(schema = @Schema(implementation = AppError.class))})
+ })
@PreAuthorize("@authorizationFilter.hasPermission('" + SearchServiceRole.ADMIN + "')")
@PostMapping
- public ResponseEntity<?> reindex(
- @NotNull @Valid @RequestBody RecordReindexRequest recordReindexRequest,
+ public ResponseEntity<?> reindex(@NotNull @Valid @RequestBody RecordReindexRequest recordReindexRequest,
+ @Parameter(description = "Force Clean")
@RequestParam(value = "force_clean", defaultValue = "false") boolean forceClean) throws IOException {
this.reIndexService.reindexRecords(recordReindexRequest, this.indexSchemaService.isStorageSchemaSyncRequired(recordReindexRequest.getKind(), forceClean));
this.auditLogger.getReindex(singletonList(recordReindexRequest.getKind()));
return new ResponseEntity<>(org.springframework.http.HttpStatus.OK);
}
+ @Operation(summary = "${reindexApi.fullReindex.summary}", description = "${reindexApi.fullReindex.description}",
+ security = {@SecurityRequirement(name = "Authorization")}, tags = { "reindex-api" })
+ @ApiResponses(value = {
+ @ApiResponse(responseCode = "200", description = "OK"),
+ @ApiResponse(responseCode = "400", description = "Bad Request", content = {@Content(schema = @Schema(implementation = AppError.class))}),
+ @ApiResponse(responseCode = "401", description = "Unauthorized", content = {@Content(schema = @Schema(implementation = AppError.class))}),
+ @ApiResponse(responseCode = "403", description = "User not authorized to perform the action", content = {@Content(schema = @Schema(implementation = AppError.class))}),
+ @ApiResponse(responseCode = "404", description = "Not Found", content = {@Content(schema = @Schema(implementation = AppError.class))}),
+ @ApiResponse(responseCode = "500", description = "Internal Server Error", content = {@Content(schema = @Schema(implementation = AppError.class))}),
+ @ApiResponse(responseCode = "502", description = "Bad Gateway", content = {@Content(schema = @Schema(implementation = AppError.class))}),
+ @ApiResponse(responseCode = "503", description = "Service Unavailable", content = {@Content(schema = @Schema(implementation = AppError.class))})
+ })
@PreAuthorize("@authorizationFilter.hasPermission('" + SearchServiceRole.ADMIN + "')")
@PatchMapping
- public ResponseEntity<String> fullReindex(@RequestParam(value = "force_clean", defaultValue = "false") boolean forceClean) throws IOException {
+ public ResponseEntity<String> fullReindex( @Parameter(description = "Force Clean")
+ @RequestParam(value = "force_clean", defaultValue = "false") boolean forceClean) throws IOException {
this.reIndexService.fullReindex(forceClean);
return new ResponseEntity<>(org.springframework.http.HttpStatus.OK);
}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/swagger/HomeController.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/swagger/HomeController.java
deleted file mode 100644
index b3ab482f149f66ca89fa993ad858d4c364ce508e..0000000000000000000000000000000000000000
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/swagger/HomeController.java
+++ /dev/null
@@ -1,12 +0,0 @@
-package org.opengroup.osdu.indexer.swagger;
-
-import org.springframework.stereotype.Controller;
-import org.springframework.web.bind.annotation.RequestMapping;
-
-@Controller
-public class HomeController {
- @RequestMapping(value = "/swagger")
- public String swagger() {
- return "redirect:swagger-ui.html";
- }
-}
diff --git a/indexer-core/src/main/java/org/opengroup/osdu/indexer/swagger/SwaggerConfiguration.java b/indexer-core/src/main/java/org/opengroup/osdu/indexer/swagger/SwaggerConfiguration.java
index f40606837a5b52a489ba3de99cdcf8ba07f348bd..957ccd927789ccd3a2168d9275a1c9305e22beb1 100644
--- a/indexer-core/src/main/java/org/opengroup/osdu/indexer/swagger/SwaggerConfiguration.java
+++ b/indexer-core/src/main/java/org/opengroup/osdu/indexer/swagger/SwaggerConfiguration.java
@@ -1,53 +1,57 @@
package org.opengroup.osdu.indexer.swagger;
-import io.swagger.v3.oas.models.servers.Server;
-import org.springframework.context.annotation.Configuration;
-import io.swagger.v3.oas.models.Components;
-import io.swagger.v3.oas.models.OpenAPI;
-import io.swagger.v3.oas.models.info.Info;
+import io.swagger.v3.oas.annotations.OpenAPIDefinition;
+import io.swagger.v3.oas.annotations.enums.SecuritySchemeType;
+import io.swagger.v3.oas.annotations.info.Contact;
+import io.swagger.v3.oas.annotations.info.Info;
+import io.swagger.v3.oas.annotations.info.License;
+import io.swagger.v3.oas.annotations.security.SecurityRequirement;
+import io.swagger.v3.oas.annotations.security.SecurityScheme;
+import io.swagger.v3.oas.annotations.servers.Server;
+import io.swagger.v3.oas.annotations.tags.Tag;
+import io.swagger.v3.oas.models.media.StringSchema;
import io.swagger.v3.oas.models.parameters.Parameter;
-import io.swagger.v3.oas.models.security.SecurityRequirement;
-import io.swagger.v3.oas.models.security.SecurityScheme;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.springdoc.core.customizers.OperationCustomizer;
import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
-import javax.servlet.ServletContext;
-import java.util.Collections;
-
+@OpenAPIDefinition(
+ info = @Info(
+ title = "${api.title}",
+ description = "${api.description}",
+ version = "${api.version}",
+ contact = @Contact(name = "${api.contact.name}", email = "${api.contact.email}"),
+ license = @License(name = "${api.license.name}", url = "${api.license.url}")),
+ servers = @Server(url = "${api.server.url}"),
+ security = @SecurityRequirement(name = "Authorization"),
+ tags = {
+ @Tag(name = "cleanup-indicies-api", description = "Cleanup Indicies API"),
+ @Tag(name = "partition-setup-api", description = "Partition Setup API"),
+ @Tag(name = "reindex-api", description = "Reindex API"),
+ @Tag(name = "health-check-api", description = "Health Check API"),
+ @Tag(name = "info", description = "Version info endpoint")
+ }
+)
+@SecurityScheme(name = "Authorization", scheme = "bearer", bearerFormat = "Authorization", type = SecuritySchemeType.HTTP)
@Configuration
public class SwaggerConfiguration {
@Bean
- public OpenAPI openApi(ServletContext servletContext) {
- Server server = new Server().url(servletContext.getContextPath());
- return new OpenAPI()
- .servers(Collections.singletonList(server))
- .info(new Info()
- .description("Indexer service that provides a set of APIs " +
- "to index storage records against Elasticsearch")
- .title("Indexer Service")
- .version("1.0"))
- .components(new Components()
- .addSecuritySchemes("Authorization",
- new SecurityScheme()
- .type(SecurityScheme.Type.HTTP)
- .scheme("bearer")
- .bearerFormat("Authorization")
- .in(SecurityScheme.In.HEADER)
- .name("Authorization")))
- .addSecurityItem(
- new SecurityRequirement()
- .addList("Authorization"));
+ public OperationCustomizer operationCustomizer() {
+ return (operation, handlerMethod) -> {
+ Parameter dataPartitionId = new Parameter()
+ .name(DpsHeaders.DATA_PARTITION_ID)
+ .description("Data Partition Id")
+ .in("header")
+ .required(true)
+ .schema(new StringSchema());
+ // PartitionSetupApi - provisionPartition endpoint has 'data-partition-id' header defined at method level.
+ if(!operation.getOperationId().equals("provisionPartition"))
+ return operation.addParametersItem(dataPartitionId);
+ else
+ return operation;
+ };
}
- @Bean
- public OperationCustomizer customize() {
- return (operation, handlerMethod) -> operation.addParametersItem(
- new Parameter()
- .in("header")
- .required(true)
- .description("Tenant Id")
- .name(DpsHeaders.DATA_PARTITION_ID));
- }
}
diff --git a/indexer-core/src/main/resources/swagger.properties b/indexer-core/src/main/resources/swagger.properties
new file mode 100644
index 0000000000000000000000000000000000000000..12506ebeb2e9aa1af8fdb396baaa543879cbcba0
--- /dev/null
+++ b/indexer-core/src/main/resources/swagger.properties
@@ -0,0 +1,45 @@
+#Manage common SpringDoc and OpenAPI related properties
+
+#SpringDoc related properties
+springdoc.swagger-ui.tagsSorter=alpha
+springdoc.swagger-ui.operationsSorter=alpha
+springdoc.swagger-ui.doc-expansion=none
+springdoc.swagger-ui.path=/swagger
+springdoc.swagger-ui.displayOperationId=true
+springdoc.api-docs.path=/api-docs
+
+#OpenAPI 3.0 - Indexer Service properties
+api.title=Indexer Service
+api.description=Indexer service creates an index, which is a methodical arrangement of records designed to enable users to locate information quickly.
+api.version=2.0
+api.contact.name=OSDU Data Platform Team
+api.contact.email=dps@OSDU.org
+api.license.name=Apache 2.0
+api.license.url=https://www.apache.org/licenses/LICENSE-2.0.html
+api.server.url=${server.servlet.contextPath}
+
+
+#CleanupIndiciesApi related properties
+cleanupIndiciesApi.deleteIndex.summary=Delete Index for the given kind
+cleanupIndiciesApi.deleteIndex.description=Delete Index for the given kind. Required roles: `users.datalake.ops`
+
+#PartitionSetupApi related properties
+partitionSetupApi.provisionPartition.summary=Provision partition
+partitionSetupApi.provisionPartition.description=Provision partition. Required roles: `users.datalake.ops`
+
+#Reindex API related properties
+reindexApi.reindex.summary=Re-index given 'kind'
+reindexApi.reindex.description=This API allows users to re-index a 'kind' without re-ingesting the records via storage API. \
+Required roles: `service.search.admin`
+reindexApi.fullReindex.summary=Full Re-index by data partition
+reindexApi.fullReindex.description=This API allows users to re-index an entire partition without re-ingesting the records via storage API.\
+Required roles: `service.search.admin`
+
+#Info & Health API related properties
+infoApi.info.summary=Version info
+infoApi.info.description=For deployment available public `/info` endpoint, which provides build and git related information.
+healthCheckApi.livenessCheck.summary=Liveness Check endpoint
+healthCheckApi.livenessCheck.description=For deployment available public `/liveness_check` endpoint.
+healthCheckApi.readinessCheck.summary=Readiness Check endpoint
+healthCheckApi.readinessCheck.description=For deployment available public `/readiness_check` endpoint.
+
diff --git a/pom.xml b/pom.xml
index 317711bed092e226c1f04270335d0dbf2d71c396..41919837141fd99c52bad08bf487f436c68e4721 100644
--- a/pom.xml
+++ b/pom.xml
@@ -13,7 +13,7 @@
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
<spring-cloud.version>Greenwich.SR2</spring-cloud.version>
- <os-core-common.version>0.19.0</os-core-common.version>
+ <os-core-common.version>0.21.0-rc4</os-core-common.version>
<snakeyaml.version>2.0</snakeyaml.version>
<hibernate-validator.version>6.1.5.Final</hibernate-validator.version>
<jackson-databind.version>2.14.1</jackson-databind.version>
@@ -33,7 +33,7 @@
<!-- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>-->
<!-- <failOnMissingWebXml>false</failOnMissingWebXml>-->
<!-- <spring-cloud.version>Greenwich.SR2</spring-cloud.version>-->
- <openapi.version>1.6.9</openapi.version>
+ <openapi.version>1.6.14</openapi.version>
</properties>
<licenses>
diff --git a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/IndexerAwsApplication.java b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/IndexerAwsApplication.java
index 04de6340810e6d84e8c551a70d9b7e353b08b265..fb73bef6d03fb9c606b555d9187d9b1373e4399c 100644
--- a/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/IndexerAwsApplication.java
+++ b/provider/indexer-aws/src/main/java/org/opengroup/osdu/indexer/aws/IndexerAwsApplication.java
@@ -21,6 +21,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.FilterType;
+import org.springframework.context.annotation.PropertySource;
@SpringBootApplication(exclude = { SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class })
@ComponentScan(
@@ -30,6 +31,7 @@ import org.springframework.context.annotation.FilterType;
type=FilterType.ASSIGNABLE_TYPE,
value=IndexerQueueTaskBuilder.class)
})
+@PropertySource("classpath:swagger.properties")
public class IndexerAwsApplication {
public static void main(String[] args) {
diff --git a/provider/indexer-azure/README.md b/provider/indexer-azure/README.md
index 9b8842fbbbac2f3fc449c23ef8967c28e3ad5569..cb13417cb40902bdfa6cea76f058604ad25bae21 100644
--- a/provider/indexer-azure/README.md
+++ b/provider/indexer-azure/README.md
@@ -118,7 +118,7 @@ $ mvn spring-boot:run
### Test the application
-After the service has started it should be accessible via a web browser by visiting [http://localhost:8080/swagger-ui/index.html](http://localhost:8080/swagger-ui/index.html). If the request does not fail, you can then run the integration tests.
+After the service has started it should be accessible via a web browser by visiting [http://localhost:8080/api/indexer/v2/swagger](http://localhost:8080/api/indexer/v2/swagger). If the request does not fail, you can then run the integration tests.
> **Note**: the integration tests for `os-indexer-azure` work by validating that records submitted to `os-storage-azure` can eventually be queried by `os-search-azure`. This only works if the messages emitted by `os-storage-azure` can be consumed by `os-indexer-queue-azure`, which will submit the indexing request to `os-indexer-azure`.
>
@@ -140,6 +140,13 @@ $ (cd testing/indexer-test-core/ && mvn clean install)
$ (cd testing/indexer-test-azure/ && mvn clean test)
```
+## Open API 3.0 - Swagger
+- Swagger UI: http://localhost:8080/api/indexer/v2/swagger (will redirect to http://localhost:8080/api/indexer/v2/swagger-ui/index.html)
+- api-docs (JSON) : http://localhost:8080/api/indexer/v2/api-docs
+- api-docs (YAML) : http://localhost:8080/api/indexer/v2/api-docs.yaml
+
+All the Swagger and OpenAPI related common properties are managed here [swagger.properties](../../indexer-core/src/main/resources/swagger.properties)
+
## Debugging
Jet Brains - the authors of Intellij IDEA, have written an [excellent guide](https://www.jetbrains.com/help/idea/debugging-your-first-java-application.html) on how to debug java programs.
diff --git a/provider/indexer-azure/pom.xml b/provider/indexer-azure/pom.xml
index ec47402e269a35dd1599831de2fcc3e79e7a605a..9533622b4fc072ad626e2559dd4844c0121b4a8c 100644
--- a/provider/indexer-azure/pom.xml
+++ b/provider/indexer-azure/pom.xml
@@ -38,7 +38,7 @@
<azure.appservice.appname />
<azure.appservice.subscription />
<log4j.version>2.17.1</log4j.version>
- <nimbus-jose-jwt.version>8.2</nimbus-jose-jwt.version>
+ <nimbus-jose-jwt.version>8.20.2</nimbus-jose-jwt.version>
<indexer-core.version>0.21.0-SNAPSHOT</indexer-core.version>
<spring-security-jwt.version>1.1.1.RELEASE</spring-security-jwt.version>
<osdu.corelibazure.version>0.20.0-rc5</osdu.corelibazure.version>
@@ -219,6 +219,11 @@
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ <version>6.1.23</version>
+ </dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/IndexerAzureApplication.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/IndexerAzureApplication.java
index f0ed79107e68a6d71bb98296734c237de5a62c4a..d09c7c85a851b8697f80745faa9362824f1e67a4 100644
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/IndexerAzureApplication.java
+++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/IndexerAzureApplication.java
@@ -23,6 +23,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.FilterType;
+import org.springframework.context.annotation.PropertySource;
@SpringBootApplication(exclude = { SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class })
@@ -34,6 +35,7 @@ import org.springframework.context.annotation.FilterType;
@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value=ElasticSettingServiceImpl.class),
}
)
+@PropertySource("classpath:swagger.properties")
public class IndexerAzureApplication {
public static void main(String[] args) {
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/security/AADSecurityConfig.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/security/AADSecurityConfig.java
index a1261867617574a2ae0ea39f9648998e1db880e8..62427d5ea6e94f7d054219196aa9904e14905325 100644
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/security/AADSecurityConfig.java
+++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/security/AADSecurityConfig.java
@@ -32,24 +32,26 @@ public class AADSecurityConfig extends WebSecurityConfigurerAdapter {
@Inject
private AADAppRoleStatelessAuthenticationFilter appRoleAuthFilter;
+ private static final String[] AUTH_ALLOWLIST = {"/", "/index.html",
+ "/index-worker", "/_dps/task-handlers", "/_dps/task-handlers/**",
+ "/reindex",
+ "/actuator/*",
+ "/v2/api-docs.yaml",
+ "/v2/api-docs/swagger-config",
+ "/v2/api-docs/**",
+ "/info",
+ "/v2/swagger",
+ "/v2/swagger-ui/**"
+ };
+
@Override
protected void configure(HttpSecurity http) throws Exception {
http.csrf().disable().sessionManagement().sessionCreationPolicy(SessionCreationPolicy.NEVER)
.and()
.authorizeRequests()
- .antMatchers("/", "/index.html",
- "/index-worker", "/_dps/task-handlers", "/_dps/task-handlers/**",
- "/reindex",
- "/v2/api-docs",
- "/v3/api-docs",
- "/swagger-resources/**",
- "/configuration/security",
- "/swagger",
- "/info",
- "/swagger-ui/index.html",
- "/swagger-ui/**",
- "/webjars/**").permitAll()
+ .antMatchers(AUTH_ALLOWLIST).permitAll()
.anyRequest().authenticated()
.and().addFilterBefore(appRoleAuthFilter, UsernamePasswordAuthenticationFilter.class);
}
+
}
diff --git a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/security/WhoamiController.java b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/security/WhoamiController.java
index 6f83843d54672a3d8f5bb47152e02d82431d3779..70828d0e9ad608987a19db399763566901a9422c 100644
--- a/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/security/WhoamiController.java
+++ b/provider/indexer-azure/src/main/java/org/opengroup/osdu/indexer/azure/security/WhoamiController.java
@@ -14,12 +14,14 @@
package org.opengroup.osdu.indexer.azure.security;
+import io.swagger.v3.oas.annotations.Hidden;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.ResponseBody;
+@Hidden
@Controller
public class WhoamiController {
@RequestMapping(value = {"/", "/whoami"})
diff --git a/provider/indexer-gc/pom.xml b/provider/indexer-gc/pom.xml
index 94b23a7ca4e7ae5331c0a15875c42b43e02489bd..634032280d30488dc829882408020011869d670d 100644
--- a/provider/indexer-gc/pom.xml
+++ b/provider/indexer-gc/pom.xml
@@ -19,24 +19,26 @@
<dependencyManagement>
<dependencies>
<dependency>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-dependencies</artifactId>
- <version>2.7.10</version>
- <type>pom</type>
- <scope>import</scope>
+ <groupId>com.fasterxml.jackson</groupId>
+ <artifactId>jackson-bom</artifactId>
+ <version>2.14.2</version>
+ <type>pom</type>
+ <scope>import</scope>
</dependency>
<dependency>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-dependencies</artifactId>
- <version>2.7.10</version>
- </dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-dependencies</artifactId>
+ <version>2.7.10</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<dependency>
<groupId>org.opengroup.osdu</groupId>
- <artifactId>core-lib-gcp</artifactId>
- <version>0.20.0-rc2</version>
+ <artifactId>core-lib-gc</artifactId>
+ <version>0.21.0-rc4</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu.indexer</groupId>
@@ -67,6 +69,11 @@
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ <version>6.1.23</version>
+ </dependency>
<!-- Test Dependencies -->
<dependency>
@@ -148,6 +155,12 @@
<artifactId>spring-webmvc</artifactId>
<version>5.3.22</version>
</dependency>
+
+ <dependency>
+ <groupId>org.opengroup.osdu</groupId>
+ <artifactId>oqm</artifactId>
+ <version>0.21.0-rc3</version>
+ </dependency>
</dependencies>
<build>
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/IndexerGcpApplication.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/IndexerGcpApplication.java
index 418959415ebd132ccc18f9c7089169a91f09be3e..f9252594aef24cd48d66d5509cd7d1672bfba863 100644
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/IndexerGcpApplication.java
+++ b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/IndexerGcpApplication.java
@@ -23,8 +23,10 @@ import org.springframework.boot.SpringBootConfiguration;
import org.springframework.boot.WebApplicationType;
import org.springframework.boot.builder.SpringApplicationBuilder;
import org.springframework.context.ConfigurableApplicationContext;
+import org.springframework.context.annotation.PropertySource;
@SpringBootConfiguration
+@PropertySource("classpath:swagger.properties")
public class IndexerGcpApplication {
public static void main(String[] args) {
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/cache/CacheConfig.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/cache/CacheConfig.java
index 05757b74681b2c9d8771558200b11355f92619a0..2371910d15764b64bc7ba0aa60e6e28c662055be 100644
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/cache/CacheConfig.java
+++ b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/cache/CacheConfig.java
@@ -1,6 +1,6 @@
/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
+ * Copyright 2020-2023 Google LLC
+ * Copyright 2020-2023 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -18,8 +18,11 @@
package org.opengroup.osdu.indexer.provider.gcp.common.cache;
import lombok.RequiredArgsConstructor;
+import org.opengroup.osdu.core.common.cache.ICache;
import org.opengroup.osdu.core.common.cache.RedisCache;
+import org.opengroup.osdu.core.common.cache.VmCache;
import org.opengroup.osdu.core.common.model.search.ClusterSettings;
+import org.opengroup.osdu.core.common.partition.PartitionInfo;
import org.opengroup.osdu.core.common.provider.interfaces.IElasticCredentialsCache;
import org.opengroup.osdu.core.common.provider.interfaces.IIndexCache;
import org.opengroup.osdu.core.gcp.cache.RedisCacheBuilder;
@@ -89,4 +92,9 @@ public class CacheConfig {
Boolean.class
);
}
+
+ @Bean
+ public ICache<String, PartitionInfo> partitionInfoCache() {
+ return new VmCache<>(600, 2000);
+ }
}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/di/ElasticSearchConfig.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/di/ElasticSearchConfig.java
index 9ef3bc8585f45297d42a54f4a51863e893ec3133..8e9e4dd84e116cddee55714d6825c9eb9e85715a 100644
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/di/ElasticSearchConfig.java
+++ b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/di/ElasticSearchConfig.java
@@ -1,6 +1,6 @@
/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
+ * Copyright 2020-2023 Google LLC
+ * Copyright 2020-2023 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -18,9 +18,9 @@
package org.opengroup.osdu.indexer.provider.gcp.common.di;
import org.opengroup.osdu.core.common.partition.IPartitionProvider;
+import org.opengroup.osdu.core.common.partition.IPropertyResolver;
import org.opengroup.osdu.core.common.provider.interfaces.IElasticRepository;
import org.opengroup.osdu.core.destination.elastic.ElasticSearchDestinationResolver;
-import org.opengroup.osdu.core.destination.util.IPartitionPropertyResolver;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@@ -29,7 +29,7 @@ public class ElasticSearchConfig {
@Bean
public IElasticRepository elasticRepository(GcpConfigurationProperties properties,
- IPartitionProvider partitionProvider, IPartitionPropertyResolver propertyResolver) {
+ IPartitionProvider partitionProvider, IPropertyResolver propertyResolver) {
return new ElasticSearchDestinationResolver(properties.getElasticsearchPropertiesPrefix(), partitionProvider, propertyResolver);
}
}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/publish/ReprocessingTaskPublisher.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/publish/ReprocessingTaskPublisher.java
index 4caa625e53d6687f4362c6892182dae7f2676407..720628c8a042376ddf4d2ac549a428918674c95c 100644
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/publish/ReprocessingTaskPublisher.java
+++ b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/common/publish/ReprocessingTaskPublisher.java
@@ -1,6 +1,6 @@
/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
+ * Copyright 2020-2023 Google LLC
+ * Copyright 2020-2023 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -17,19 +17,11 @@
package org.opengroup.osdu.indexer.provider.gcp.common.publish;
-import static org.opengroup.osdu.core.common.Constants.REINDEX_RELATIVE_URL;
-import static org.opengroup.osdu.core.common.Constants.WORKER_RELATIVE_URL;
-
import com.google.gson.Gson;
-import java.util.HashMap;
-import java.util.Map;
-import javax.annotation.PostConstruct;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
-import org.jetbrains.annotations.NotNull;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
-import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
import org.opengroup.osdu.core.gcp.oqm.driver.OqmDriver;
import org.opengroup.osdu.core.gcp.oqm.model.OqmDestination;
import org.opengroup.osdu.core.gcp.oqm.model.OqmMessage;
@@ -39,6 +31,14 @@ import org.opengroup.osdu.indexer.util.IndexerQueueTaskBuilder;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
+import javax.annotation.PostConstruct;
+import javax.validation.constraints.NotNull;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
@Slf4j
@Primary
@Component
@@ -47,9 +47,9 @@ public class ReprocessingTaskPublisher extends IndexerQueueTaskBuilder {
private final Gson gson = new Gson();
- private final OqmDriver driver;
+ private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
- private final TenantInfo tenantInfo;
+ private final OqmDriver driver;
private final IndexerMessagingConfigProperties properties;
@@ -64,23 +64,54 @@ public class ReprocessingTaskPublisher extends IndexerQueueTaskBuilder {
}
public void createWorkerTask(String payload, DpsHeaders headers) {
- publishRecordsChangedTask(WORKER_RELATIVE_URL, payload, 0l, headers);
+ publishRecordsChangedTask(payload, headers);
}
public void createWorkerTask(String payload, Long countdownMillis, DpsHeaders headers) {
- publishRecordsChangedTask(WORKER_RELATIVE_URL, payload, countdownMillis, headers);
+ DpsHeaders headersCopy = DpsHeaders.createFromMap(headers.getHeaders());
+ scheduledExecutorService.schedule(
+ () -> {
+ try {
+ publishRecordsChangedTask(payload, headersCopy);
+ } catch (Exception e) {
+ // If error or exception not caught, executor will die out silently.
+ log.error("The exception was thrown during scheduled event publishing!", e);
+ throw e;
+ } catch (Throwable e) {
+ log.error("The Error was thrown during scheduled event publishing!", e);
+ throw e;
+ }
+ },
+ countdownMillis,
+ TimeUnit.MILLISECONDS
+ );
}
public void createReIndexTask(String payload, DpsHeaders headers) {
- publishReindexTask(REINDEX_RELATIVE_URL, payload, 0l, headers);
+ publishReindexTask(payload, headers);
}
public void createReIndexTask(String payload, Long countdownMillis, DpsHeaders headers) {
- publishReindexTask(REINDEX_RELATIVE_URL, payload, countdownMillis, headers);
+ DpsHeaders headersCopy = DpsHeaders.createFromMap(headers.getHeaders());
+ scheduledExecutorService.schedule(
+ () -> {
+ try {
+ publishReindexTask(payload, headersCopy);
+ } catch (Exception e) {
+ // If error or exception not caught, executor will die out silently.
+ log.error("The exception was thrown during scheduled event publishing!", e);
+ throw e;
+ } catch (Throwable e) {
+ log.error("The Error was thrown during scheduled event publishing!", e);
+ throw e;
+ }
+ },
+ countdownMillis,
+ TimeUnit.MILLISECONDS
+ );
}
- private void publishReindexTask(String url, String payload, Long countdownMillis,
- DpsHeaders headers) {
+ private void publishReindexTask(String payload, DpsHeaders headers) {
OqmDestination oqmDestination = OqmDestination.builder().partitionId(headers.getPartitionId())
.build();
Map<String, String> attributes = getAttributesFromHeaders(headers);
@@ -89,8 +120,7 @@ public class ReprocessingTaskPublisher extends IndexerQueueTaskBuilder {
driver.publish(oqmMessage, reprocessOqmTopic, oqmDestination);
}
- private void publishRecordsChangedTask(String url, String payload, Long countdownMillis,
- DpsHeaders headers) {
+ private void publishRecordsChangedTask(String payload, DpsHeaders headers) {
OqmDestination oqmDestination = OqmDestination.builder()
.partitionId(headers.getPartitionId())
.build();
@@ -112,7 +142,7 @@ public class ReprocessingTaskPublisher extends IndexerQueueTaskBuilder {
private Map<String, String> getAttributesFromHeaders(DpsHeaders headers) {
Map<String, String> attributes = new HashMap<>();
attributes.put(DpsHeaders.USER_EMAIL, headers.getUserEmail());
- attributes.put(DpsHeaders.ACCOUNT_ID, this.tenantInfo.getName());
+ attributes.put(DpsHeaders.ACCOUNT_ID, headers.getPartitionId());
attributes.put(DpsHeaders.DATA_PARTITION_ID, headers.getPartitionIdWithFallbackToAccountId());
headers.addCorrelationIdIfMissing();
attributes.put(DpsHeaders.CORRELATION_ID, headers.getCorrelationId());
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/IndexerOqmMessageReceiver.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/IndexerOqmMessageReceiver.java
index 3c73f93c1d893dea4be35ee1191eafee23546946..c14b4df4e59f4ce235d8eca7803de135fcea98aa 100644
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/IndexerOqmMessageReceiver.java
+++ b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/IndexerOqmMessageReceiver.java
@@ -18,11 +18,8 @@
package org.opengroup.osdu.indexer.provider.gcp.indexing.processing;
import com.google.common.base.Strings;
-
-import java.util.Optional;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
-import org.jetbrains.annotations.NotNull;
import org.opengroup.osdu.core.auth.TokenProvider;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
@@ -33,6 +30,9 @@ import org.opengroup.osdu.core.gcp.oqm.model.OqmMessageReceiver;
import org.opengroup.osdu.indexer.provider.gcp.indexing.scope.ThreadDpsHeaders;
import org.opengroup.osdu.indexer.provider.gcp.indexing.thread.ThreadScopeContextHolder;
+import javax.validation.constraints.NotNull;
+import java.util.Optional;
+
@Slf4j
@RequiredArgsConstructor
public abstract class IndexerOqmMessageReceiver implements OqmMessageReceiver {
@@ -58,25 +58,37 @@ public abstract class IndexerOqmMessageReceiver implements OqmMessageReceiver {
} catch (AppException appException) {
int statusCode = appException.getError().getCode();
if (statusCode > 199 && statusCode < 300 && statusCode != RequestStatus.INVALID_RECORD) {
- skipMessage(oqmMessage, oqmAckReplier, appException);
+ skipMessage(oqmMessage, dpsHeaders, oqmAckReplier, appException);
} else {
- rescheduleMessage(oqmMessage, oqmAckReplier, getException(appException));
+ rescheduleMessage(oqmMessage, dpsHeaders, oqmAckReplier, getException(appException));
}
} catch (Exception exception) {
- rescheduleMessage(oqmMessage, oqmAckReplier, exception);
+ rescheduleMessage(oqmMessage, dpsHeaders, oqmAckReplier, exception);
+ } catch (Throwable e) {
+ //Catching throwable is necessary, otherwise, errors like NoSuchMethodError will slip through silently.
+ log.error(
+ "HALT! Unrecoverable malfunction! Unexpected error was thrown during processing! Event ID: "
+ + oqmMessage.getId() + ". Correlation ID: " + dpsHeaders.getCorrelationId(), e);
+ throw e;
} finally {
// Cleaning thread context after processing is finished and the thread dies out.
ThreadScopeContextHolder.currentThreadScopeAttributes().clear();
}
}
- private static void skipMessage(OqmMessage oqmMessage, OqmAckReplier oqmAckReplier, AppException appException) {
- log.info("Event id : " + oqmMessage.getId() + ", was not processed, and will NOT be rescheduled.", appException);
+ private static void skipMessage(OqmMessage oqmMessage, DpsHeaders dpsHeaders,
+ OqmAckReplier oqmAckReplier, AppException appException) {
+ log.info(
+ "Event ID: " + oqmMessage.getId() + ". Correlation ID: " + dpsHeaders.getCorrelationId()
+ + ", was not processed, and will NOT be rescheduled.", appException);
oqmAckReplier.ack();
}
- private static void rescheduleMessage(OqmMessage oqmMessage, OqmAckReplier oqmAckReplier, Exception exception) {
- log.error("Event id : " + oqmMessage.getId() + ", was not processed, and will BE rescheduled.", exception);
+ private static void rescheduleMessage(OqmMessage oqmMessage, DpsHeaders dpsHeaders,
+ OqmAckReplier oqmAckReplier, Exception exception) {
+ log.error(
+ "Event id : " + oqmMessage.getId() + ". Correlation ID: " + dpsHeaders.getCorrelationId()
+ + ", was not processed, and will BE rescheduled.", exception);
oqmAckReplier.nack();
}
diff --git a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/OqmSubscriberManager.java b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/OqmSubscriberManager.java
index b7f6bfd7d5687e1f663db57b5f64d6a13bc7cd10..acc4ecd7a9fa8fec98cf531077e993f44a95f919 100644
--- a/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/OqmSubscriberManager.java
+++ b/provider/indexer-gc/src/main/java/org/opengroup/osdu/indexer/provider/gcp/indexing/processing/OqmSubscriberManager.java
@@ -1,6 +1,6 @@
/*
- * Copyright 2020-2022 Google LLC
- * Copyright 2020-2022 EPAM Systems, Inc
+ * Copyright 2020-2023 Google LLC
+ * Copyright 2020-2023 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -19,18 +19,13 @@ package org.opengroup.osdu.indexer.provider.gcp.indexing.processing;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
-import org.jetbrains.annotations.Nullable;
import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
import org.opengroup.osdu.core.gcp.oqm.driver.OqmDriver;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmDestination;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmMessageReceiver;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmSubscriber;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmSubscriberThroughput;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmSubscription;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmSubscriptionQuery;
-import org.opengroup.osdu.core.gcp.oqm.model.OqmTopic;
+import org.opengroup.osdu.core.gcp.oqm.model.*;
import org.springframework.stereotype.Service;
+import javax.annotation.Nullable;
+
@Service
@Slf4j
@RequiredArgsConstructor
diff --git a/provider/indexer-gc/src/main/resources/application.properties b/provider/indexer-gc/src/main/resources/application.properties
index a5e5077aa785f9ee0c02f8e25b31d35395ed8ba2..1a2613ffe5df6a87e20724b7fcfb6dd1f027c239 100644
--- a/provider/indexer-gc/src/main/resources/application.properties
+++ b/provider/indexer-gc/src/main/resources/application.properties
@@ -24,7 +24,7 @@ cron-empty-index-cleanup-threshold-days=7
DEFAULT_DATA_COUNTRY=US
gae-service=indexer
security.https.certificate.trust=false
-storage-records-by-kind-batch-size=20
+storage-records-by-kind-batch-size=1000
storage-records-batch-size=20
REDIS_SEARCH_PORT=6379
@@ -54,3 +54,5 @@ records-changed-topic-name=records-changed
schema-changed-topic-name=schema-changed
reprocess-topic-name=reprocess
status-changed-topic-name=indexing-progress
+
+propertyResolver.strategy=partition
\ No newline at end of file
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/IndexerIBMApplication.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/IndexerIBMApplication.java
index 3e8637487818bc8d46c1ad5547513ff5a309ff87..c92579519b64e659604161554c5e1ce7846008ef 100644
--- a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/IndexerIBMApplication.java
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/IndexerIBMApplication.java
@@ -10,6 +10,7 @@ import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.FilterType;
+import org.springframework.context.annotation.PropertySource;
@SpringBootApplication
@@ -21,6 +22,7 @@ import org.springframework.context.annotation.FilterType;
@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value=ElasticSettingServiceImpl.class),
}
)
+@PropertySource("classpath:swagger.properties")
public class IndexerIBMApplication {
public static void main(String[] args) throws Exception {
diff --git a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/security/WhoamiController.java b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/security/WhoamiController.java
index abf80357b50f2ffa5f42fa8dba7631622673e265..b0d399ff56660a897ce89c6520e98406c692366c 100644
--- a/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/security/WhoamiController.java
+++ b/provider/indexer-ibm/src/main/java/org/opengroup/osdu/indexer/ibm/security/WhoamiController.java
@@ -3,12 +3,14 @@
package org.opengroup.osdu.indexer.ibm.security;
+import io.swagger.v3.oas.annotations.Hidden;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.ResponseBody;
+@Hidden
@Controller
public class WhoamiController {
@RequestMapping(value = { "/", "/whoami" })